Remove legacy raw pipeline contracts

This commit is contained in:
2026-07-17 08:13:08 +00:00
parent 814fcdc6ba
commit adfe3825ee
68 changed files with 823 additions and 7919 deletions

View File

@@ -39,9 +39,9 @@ without exposing Scriptorium types through stage contracts.
7. injecting that one shared client into complete pipeline preparation before 7. injecting that one shared client into complete pipeline preparation before
the source file is read or the runner is invoked. the source file is read or the runner is invoked.
The D&D scene chunker retains this injected client and uses it for every scene The D&D scene chunker and spell extractor retain this injected client and use
completion. Later legacy LLM-backed operations still receive the same shared it for every structured completion. Operation requests do not carry an LLM
client through their operation requests. client.
The CLI separately gathers explicit profile IDs from resolved LLM-capable stage The CLI separately gathers explicit profile IDs from resolved LLM-capable stage
and validator bindings. It prepares a small internal check prompt for each ID so and validator bindings. It prepares a small internal check prompt for each ID so

View File

@@ -112,8 +112,8 @@ schema, strict option decoder, injected shared LLM client, and prompt/schema
manifest metadata. The separate `internal/modules/dnd/codec/spells` package manifest metadata. The separate `internal/modules/dnd/codec/spells` package
owns the durable schema and stable JSON representation for artifact kind owns the durable schema and stable JSON representation for artifact kind
`dnd/spell-list`. The runner keeps the result typed through validators and later `dnd/spell-list`. The runner keeps the result typed through validators and later
stages. Explicit migration-only codec adapters preserve the existing raw stages, using the codec only for checkpoint, debug, and output boundaries.
checkpoint, debug, and output envelopes. Shared D&D helpers keep prompt input Shared D&D helpers keep prompt input
names and source-unit reference conversion consistent with the scene chunker. names and source-unit reference conversion consistent with the scene chunker.
The durable payload and manifest metadata shapes are defined in the The durable payload and manifest metadata shapes are defined in the
@@ -123,16 +123,14 @@ The durable payload and manifest metadata shapes are defined in the
### `internal/modules/generic/merge/appendorder` ### `internal/modules/generic/merge/appendorder`
The typed merger passes values to an injected combine function in framework The merger passes typed values to an injected combine function in framework
source-chunk order. The D&D registrar specializes it with a spell-list append source-chunk order. The D&D registrar specializes it with a spell-list append
function. Its temporary raw implementation retains the prior JSON merge function.
behavior for the current runner.
### `internal/modules/generic/normalize/noop` ### `internal/modules/generic/normalize/noop`
The typed normalizer returns the merged domain value unchanged and is reusable The normalizer returns the merged domain value unchanged and is reusable for
for any registered artifact type. Its temporary raw implementation defensively any registered artifact type.
clones the accepted payload for the current runner.
## Output Encoder ## Output Encoder

View File

@@ -47,8 +47,8 @@ validator set before the runner receives source bytes.
| `internal/framework/checkpoint` | Workspace-backed checkpoint loading, recording, and payload serialization. | | `internal/framework/checkpoint` | Workspace-backed checkpoint loading, recording, and payload serialization. |
| `internal/framework/debug` | Workspace-backed framework and LLM debug recording. | | `internal/framework/debug` | Workspace-backed framework and LLM debug recording. |
Framework contracts provide both the production raw stage interfaces and typed Framework contracts provide typed artifact, provenance-wrapper, chunk-validator,
artifact, provenance-wrapper, chunk-validator, serialized-validator, and serialized-validator, and
typed-validator interfaces. The runner owns handoff provenance, validation typed-validator interfaces. The runner owns handoff provenance, validation
sequencing, rejection handling, checkpoint and debug boundaries, and final sequencing, rejection handling, checkpoint and debug boundaries, and final
manifest assembly. manifest assembly.
@@ -60,15 +60,13 @@ type equality across the lane, and records schema identity in the resolved lane
and pipeline digest. Registry entries carry separate option-validation and and pipeline digest. Registry entries carry separate option-validation and
run-local construction closures. Preparation injects shared dependencies and run-local construction closures. Preparation injects shared dependencies and
constructs input, chunk, validators, ordered lanes, and output before source constructs input, chunk, validators, ordered lanes, and output before source
parsing. Production input, chunk, and output modules use strict construction-time parsing. Production modules use strict construction-time option decoding, and
option decoding, and the LLM-backed scene chunker retains the injected shared LLM-backed modules retain the injected shared client. The D&D family registers
client. The D&D family registers the canonical `dnd/spell-list` codec, typed the canonical `dnd/spell-list` codec, typed spell extractor and validators, and
spell extractor and validators, and kind-specific generic merge and normalize kind-specific generic merge and normalize strategies; generic JSON validators
strategies; generic JSON validators use the serialized-validation contract. The use the serialized-validation contract. The runner executes lanes through
runner executes the production D&D lane through private exact-type-checked private exact-type-checked closures and serializes artifacts only through their
closures and uses migration-only codec adapters for the existing raw codec at checkpoint, debug, and output boundaries.
checkpoint, debug, and output envelopes. Legacy raw lanes retain their separate
executor while they migrate.
## Production Extensions ## Production Extensions

View File

@@ -75,20 +75,18 @@ mismatches are rejected deterministically.
Production composition registers the D&D spell-list codec and typed extractor, Production composition registers the D&D spell-list codec and typed extractor,
matching typed merge, normalize, and semantic-validator variants, and matching typed merge, normalize, and semantic-validator variants, and
serialized JSON validators. The production D&D lane has no parallel raw stage serialized JSON validators. Every artifact lane resolves through the typed
registration. A standalone raw registration cannot satisfy a typed lane. registries and a matching codec.
A `ModuleSpec` declares its stage plus required and provided capabilities. A `ModuleSpec` declares its stage plus required and provided capabilities.
Chunk, extract, merge, and normalize specs may also declare reference slots. Chunk, extract, merge, and normalize specs may also declare reference slots.
Registry implementations defensively copy spec metadata, reject duplicate keys, Registry implementations defensively copy spec metadata, reject duplicate keys,
and verify that a constructed implementation reports the registered key. and verify that a constructed implementation reports the registered key.
Builder registrations accept `ModuleDependencies` and cloned raw options through Builder registrations accept `ModuleDependencies` and cloned configuration
one `BuildRequest`. Production input, chunk, and output builders decode those options through one `BuildRequest`. Builders decode those options and retain
options and retain typed values or injected dependencies in the constructed typed values or injected dependencies in the constructed implementation.
implementation. A typed extractor registration may explicitly supply a raw Extractors declare their artifact kind, and merger, normalizer, and validator
adapter builder for a still-raw downstream lane; the adapter is selected as one resolution selects the matching typed variant.
unit and does not expose the typed value to raw consumers. Remaining production
raw-stage registrations are adapted from their zero-argument constructors.
A `ValidatorSpec` declares a validator key and execution class. Resolution uses A `ValidatorSpec` declares a validator key and execution class. Resolution uses
the execution class to reject incompatible profile bindings before execution. the execution class to reject incompatible profile bindings before execution.
@@ -116,10 +114,9 @@ an LLM client; an LLM-backed chunker receives the shared client during
preparation. Their operation requests retain run-specific source, reference, preparation. Their operation requests retain run-specific source, reference,
profile, session, and metadata context as applicable. profile, session, and metadata context as applicable.
Prepared typed lanes retain exact-type-checked erased operation closures. The Prepared lanes retain exact-type-checked erased operation closures. The runner
runner uses those closures to keep each value typed through extraction, uses those closures to keep each value typed through extraction, validation,
validation, merge, and normalization. Legacy raw lanes continue through their merge, and normalization.
existing executor while they migrate independently.
Source validation requires every unit to carry a canonical self-reference to Source validation requires every unit to carry a canonical self-reference to
its containing document and its own unit ID. Explicit clone, checkpoint, and its containing document and its own unit ID. Explicit clone, checkpoint, and
@@ -225,7 +222,7 @@ and recorder implementation are inventoried in
The runner owns manifest assembly and handoff summaries but not the durable JSON The runner owns manifest assembly and handoff summaries but not the durable JSON
schema. It records resolved module and lane provenance, validator chains, schema. It records resolved module and lane provenance, validator chains,
source/reference identities, selected LLM profiles, normalized and rejected source/reference identities, selected LLM profiles, normalized and rejected
summaries, status, and timing. Raw payload bytes remain outside the manifest. summaries, status, and timing. Serialized artifact content remains outside the manifest.
Module metadata providers may add non-secret singleton or lane-scoped metadata. Module metadata providers may add non-secret singleton or lane-scoped metadata.
Execution errors include stage, module, lane, or validator context. Once a Execution errors include stage, module, lane, or validator context. Once a

View File

@@ -170,12 +170,12 @@ func TestProductionCatalogIncludesProductionModulesValidatorsAndDefaults(t *test
{ {
name: "appendorder merger", name: "appendorder merger",
got: func() (pipeline.ModuleSpec, bool) { return catalog.Mergers.Spec(appendorder.Key) }, got: func() (pipeline.ModuleSpec, bool) { return catalog.Mergers.Spec(appendorder.Key) },
want: appendorder.ModuleSpec(), want: appendorder.TypedModuleSpec(spells.ModuleSpec().ArtifactKind),
}, },
{ {
name: "noop normalizer", name: "noop normalizer",
got: func() (pipeline.ModuleSpec, bool) { return catalog.Normalizers.Spec(noop.Key) }, got: func() (pipeline.ModuleSpec, bool) { return catalog.Normalizers.Spec(noop.Key) },
want: noop.ModuleSpec(), want: noop.TypedModuleSpec(spells.ModuleSpec().ArtifactKind),
}, },
{ {
name: "json output", name: "json output",
@@ -3556,6 +3556,10 @@ func fakeExecutionRegistries(t *testing.T) pipeline.Registries {
mergers := pipeline.NewMergerRegistry() mergers := pipeline.NewMergerRegistry()
normalizers := pipeline.NewNormalizerRegistry() normalizers := pipeline.NewNormalizerRegistry()
outputs := pipeline.NewOutputEncoderRegistry() outputs := pipeline.NewOutputEncoderRegistry()
codecs := pipeline.NewArtifactCodecRegistry()
if err := pipeline.RegisterArtifactCodec(codecs, fakeRunCodec{}); err != nil {
t.Fatal(err)
}
if err := inputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "fake/input", Stage: pipeline.StageInput, Provides: []string{"source"}}, func() (contracts.InputAdapter, error) { if err := inputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "fake/input", Stage: pipeline.StageInput, Provides: []string{"source"}}, func() (contracts.InputAdapter, error) {
return fakeRunInputAdapter{}, nil return fakeRunInputAdapter{}, nil
@@ -3567,25 +3571,25 @@ func fakeExecutionRegistries(t *testing.T) pipeline.Registries {
}); err != nil { }); err != nil {
t.Fatalf("register fake chunker: %v", err) t.Fatalf("register fake chunker: %v", err)
} }
if err := extractors.RegisterLegacyRawWithSpec(pipeline.ModuleSpec{ if err := pipeline.RegisterExtractor(extractors, pipeline.ModuleSpec{
Key: "fake/extract", Key: "fake/extract",
Stage: pipeline.StageExtract, Stage: pipeline.StageExtract,
Requires: []string{"chunks"}, Requires: []string{"chunks"},
Provides: []string{"artifact"}, Provides: []string{"artifact"},
ReferenceSlots: []contracts.ReferenceSlot{ ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster"}, {Name: "roster"},
}, }, ArtifactKind: fakeRunArtifactKind,
}, func() (contracts.LegacyRawExtractor, error) { }, func() (contracts.Extractor[fakeRunArtifact], error) {
return fakeRunExtractor{}, nil return fakeRunExtractor{}, nil
}); err != nil { }); err != nil {
t.Fatalf("register fake extractor: %v", err) t.Fatalf("register fake extractor: %v", err)
} }
if err := mergers.RegisterLegacyRawWithSpec(pipeline.ModuleSpec{Key: "appendorder", Stage: pipeline.StageMerge, Requires: []string{"artifact"}, Provides: []string{"merged"}}, func() (contracts.LegacyRawMerger, error) { if err := pipeline.RegisterMerger(mergers, pipeline.ModuleSpec{Key: "appendorder", Stage: pipeline.StageMerge, Requires: []string{"artifact"}, Provides: []string{"merged"}, ArtifactKind: fakeRunArtifactKind}, func() (contracts.Merger[fakeRunArtifact], error) {
return fakeRunMerger{}, nil return fakeRunMerger{}, nil
}); err != nil { }); err != nil {
t.Fatalf("register fake merger: %v", err) t.Fatalf("register fake merger: %v", err)
} }
if err := normalizers.RegisterLegacyRawWithSpec(pipeline.ModuleSpec{Key: "noop", Stage: pipeline.StageNormalize, Requires: []string{"merged"}, Provides: []string{"normalized"}}, func() (contracts.LegacyRawNormalizer, error) { if err := pipeline.RegisterNormalizer(normalizers, pipeline.ModuleSpec{Key: "noop", Stage: pipeline.StageNormalize, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: fakeRunArtifactKind}, func() (contracts.Normalizer[fakeRunArtifact], error) {
return fakeRunNormalizer{}, nil return fakeRunNormalizer{}, nil
}); err != nil { }); err != nil {
t.Fatalf("register fake normalizer: %v", err) t.Fatalf("register fake normalizer: %v", err)
@@ -3597,7 +3601,7 @@ func fakeExecutionRegistries(t *testing.T) pipeline.Registries {
return pipeline.Registries{ return pipeline.Registries{
Inputs: inputs, Inputs: inputs,
Chunkers: chunkers, Chunkers: chunkers,
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), ArtifactCodecs: codecs,
Extractors: extractors, Extractors: extractors,
Mergers: mergers, Mergers: mergers,
Normalizers: normalizers, Normalizers: normalizers,
@@ -3643,6 +3647,25 @@ func (fakeRunChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (co
type fakeRunExtractor struct{} type fakeRunExtractor struct{}
const fakeRunArtifactKind contracts.ArtifactKind = "test/fake"
type fakeRunArtifact struct {
Value bool `json:"value"`
}
type fakeRunCodec struct{}
func (fakeRunCodec) Kind() contracts.ArtifactKind { return fakeRunArtifactKind }
func (fakeRunCodec) Schema() contracts.ArtifactSchema {
return contracts.ArtifactSchema{ID: "fake.artifact", Name: "fake_artifact", Version: "v1", JSONSchema: []byte(`{"type":"object"}`)}
}
func (fakeRunCodec) MediaType() string { return "application/json" }
func (fakeRunCodec) Encode(v fakeRunArtifact) ([]byte, error) { return json.Marshal(v) }
func (fakeRunCodec) Decode(b []byte) (fakeRunArtifact, error) {
var v fakeRunArtifact
err := json.Unmarshal(b, &v)
return v, err
}
func (fakeRunExtractor) Key() string { func (fakeRunExtractor) Key() string {
return "fake/extract" return "fake/extract"
} }
@@ -3651,16 +3674,8 @@ func (fakeRunExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return []contracts.ReferenceSlot{{Name: "roster"}} return []contracts.ReferenceSlot{{Name: "roster"}}
} }
func (fakeRunExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) { func (fakeRunExtractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[fakeRunArtifact], error) {
return contracts.ExtractionResult{ return contracts.TypedExtractionResult[fakeRunArtifact]{Value: fakeRunArtifact{Value: true}}, nil
Output: contracts.ExtractOutput{
Schema: contracts.ResponseSchema{ID: "fake.artifact", Name: "fake_artifact", Version: "v1"},
Payload: contracts.RawPayload{
Content: []byte(`{"value":true}`),
MediaType: "application/json",
},
},
}, nil
} }
type fakeRunMerger struct{} type fakeRunMerger struct{}
@@ -3669,21 +3684,11 @@ func (fakeRunMerger) Key() string {
return "appendorder" return "appendorder"
} }
func (fakeRunMerger) Merge(ctx context.Context, req contracts.MergeRequest) (contracts.MergeResult, error) { func (fakeRunMerger) Merge(ctx context.Context, req contracts.TypedMergeRequest[fakeRunArtifact]) (contracts.TypedMergeResult[fakeRunArtifact], error) {
output := contracts.MergeOutput{
LaneID: req.LaneID,
SourceID: req.Source.ID,
Schema: contracts.ResponseSchema{ID: "fake.artifact", Name: "fake_artifact", Version: "v1"},
Payload: contracts.RawPayload{
Content: []byte(`{"merged":true}`),
MediaType: "application/json",
},
}
if len(req.ExtractOutputs) > 0 { if len(req.ExtractOutputs) > 0 {
output.Schema = req.ExtractOutputs[0].Schema return contracts.TypedMergeResult[fakeRunArtifact]{Value: req.ExtractOutputs[0].Value}, nil
output.Payload = req.ExtractOutputs[0].Payload
} }
return contracts.MergeResult{Output: output}, nil return contracts.TypedMergeResult[fakeRunArtifact]{}, nil
} }
type fakeRunNormalizer struct{} type fakeRunNormalizer struct{}
@@ -3696,15 +3701,8 @@ func (fakeRunNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
return nil return nil
} }
func (fakeRunNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) { func (fakeRunNormalizer) Normalize(ctx context.Context, req contracts.TypedNormalizeRequest[fakeRunArtifact]) (contracts.TypedNormalizeResult[fakeRunArtifact], error) {
return contracts.NormalizeResult{ return contracts.TypedNormalizeResult[fakeRunArtifact]{Value: req.MergeOutput.Value}, nil
Output: contracts.NormalizeOutput{
LaneID: req.LaneID,
SourceID: req.MergeOutput.SourceID,
Schema: req.MergeOutput.Schema,
Payload: req.MergeOutput.Payload,
},
}, nil
} }
func onlyChildDir(t *testing.T, root string) string { func onlyChildDir(t *testing.T, root string) string {
@@ -3966,9 +3964,18 @@ func fakeCatalog(t *testing.T, overrides ...pipeline.ModuleSpec) pipeline.Module
for _, override := range overrides { for _, override := range overrides {
specs[override.Key] = override specs[override.Key] = override
} }
for _, key := range []string{"fake/extract", "appendorder", "noop"} {
spec := specs[key]
spec.ArtifactKind = fakeRunArtifactKind
specs[key] = spec
}
mustRegisterInput(t, inputs, specs["fake/input"]) mustRegisterInput(t, inputs, specs["fake/input"])
mustRegisterChunker(t, chunkers, specs["generic"]) mustRegisterChunker(t, chunkers, specs["generic"])
codecs := pipeline.NewArtifactCodecRegistry()
if err := pipeline.RegisterArtifactCodec(codecs, fakeRunCodec{}); err != nil {
t.Fatal(err)
}
mustRegisterExtractor(t, extractors, specs["fake/extract"]) mustRegisterExtractor(t, extractors, specs["fake/extract"])
mustRegisterMerger(t, mergers, specs["appendorder"]) mustRegisterMerger(t, mergers, specs["appendorder"])
mustRegisterNormalizer(t, normalizers, specs["noop"]) mustRegisterNormalizer(t, normalizers, specs["noop"])
@@ -3977,7 +3984,7 @@ func fakeCatalog(t *testing.T, overrides ...pipeline.ModuleSpec) pipeline.Module
return pipeline.ModuleCatalog{ return pipeline.ModuleCatalog{
Inputs: inputs, Inputs: inputs,
Chunkers: chunkers, Chunkers: chunkers,
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), ArtifactCodecs: codecs,
Extractors: extractors, Extractors: extractors,
Mergers: mergers, Mergers: mergers,
Normalizers: normalizers, Normalizers: normalizers,
@@ -4003,21 +4010,21 @@ func mustRegisterChunker(t *testing.T, registry *pipeline.ChunkerRegistry, spec
func mustRegisterExtractor(t *testing.T, registry *pipeline.ExtractorRegistry, spec pipeline.ModuleSpec) { func mustRegisterExtractor(t *testing.T, registry *pipeline.ExtractorRegistry, spec pipeline.ModuleSpec) {
t.Helper() t.Helper()
if err := registry.RegisterLegacyRawWithSpec(spec, func() (contracts.LegacyRawExtractor, error) { return fakeRunExtractor{}, nil }); err != nil { if err := pipeline.RegisterExtractor(registry, spec, func() (contracts.Extractor[fakeRunArtifact], error) { return fakeRunExtractor{}, nil }); err != nil {
t.Fatalf("register extractor: %v", err) t.Fatalf("register extractor: %v", err)
} }
} }
func mustRegisterMerger(t *testing.T, registry *pipeline.MergerRegistry, spec pipeline.ModuleSpec) { func mustRegisterMerger(t *testing.T, registry *pipeline.MergerRegistry, spec pipeline.ModuleSpec) {
t.Helper() t.Helper()
if err := registry.RegisterLegacyRawWithSpec(spec, func() (contracts.LegacyRawMerger, error) { return fakeRunMerger{}, nil }); err != nil { if err := pipeline.RegisterMerger(registry, spec, func() (contracts.Merger[fakeRunArtifact], error) { return fakeRunMerger{}, nil }); err != nil {
t.Fatalf("register merger: %v", err) t.Fatalf("register merger: %v", err)
} }
} }
func mustRegisterNormalizer(t *testing.T, registry *pipeline.NormalizerRegistry, spec pipeline.ModuleSpec) { func mustRegisterNormalizer(t *testing.T, registry *pipeline.NormalizerRegistry, spec pipeline.ModuleSpec) {
t.Helper() t.Helper()
if err := registry.RegisterLegacyRawWithSpec(spec, func() (contracts.LegacyRawNormalizer, error) { return fakeRunNormalizer{}, nil }); err != nil { if err := pipeline.RegisterNormalizer(registry, spec, func() (contracts.Normalizer[fakeRunArtifact], error) { return fakeRunNormalizer{}, nil }); err != nil {
t.Fatalf("register normalizer: %v", err) t.Fatalf("register normalizer: %v", err)
} }
} }
@@ -4031,11 +4038,16 @@ func mustRegisterOutput(t *testing.T, registry *pipeline.OutputEncoderRegistry,
func mustRegisterValidator(t *testing.T, registry *pipeline.ValidatorRegistry, spec pipeline.ValidatorSpec) { func mustRegisterValidator(t *testing.T, registry *pipeline.ValidatorRegistry, spec pipeline.ValidatorSpec) {
t.Helper() t.Helper()
if err := registry.RegisterLegacyRawWithSpec(spec, func() (contracts.LegacyRawValidator, error) { if err := pipeline.RegisterChunkValidator(registry, spec, func() (contracts.ChunkValidator, error) {
return fakeConfigValidator{name: spec.Key, executionClass: spec.ExecutionClass}, nil return fakeConfigValidator{name: spec.Key, executionClass: spec.ExecutionClass}, nil
}); err != nil { }); err != nil {
t.Fatalf("register validator: %v", err) t.Fatalf("register validator: %v", err)
} }
if err := pipeline.RegisterTypedValidator[fakeRunArtifact](registry, fakeRunArtifactKind, spec, func() (contracts.TypedValidator[fakeRunArtifact], error) {
return fakeConfigTypedValidator{fakeConfigValidator{name: spec.Key, executionClass: spec.ExecutionClass}}, nil
}); err != nil {
t.Fatalf("register typed validator: %v", err)
}
} }
type fakeConfigValidator struct { type fakeConfigValidator struct {
@@ -4051,7 +4063,13 @@ func (validator fakeConfigValidator) ExecutionClass() contracts.ExecutionClass {
return validator.executionClass return validator.executionClass
} }
func (validator fakeConfigValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) { func (validator fakeConfigValidator) Validate(ctx context.Context, req contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: true}, nil
}
type fakeConfigTypedValidator struct{ fakeConfigValidator }
func (validator fakeConfigTypedValidator) Validate(ctx context.Context, req contracts.TypedValidationRequest[fakeRunArtifact]) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: true}, nil return contracts.ValidationResult{Approved: true}, nil
} }

View File

@@ -250,7 +250,7 @@ pipelines:
} }
lane := profile.Artifacts["events"] lane := profile.Artifacts["events"]
if !reflect.DeepEqual(lane.References, map[string]string{"lore": "./lore.md", "roster": "./legacy-roster.yml"}) { if !reflect.DeepEqual(lane.References, map[string]string{"lore": "./lore.md", "roster": "./legacy-roster.yml"}) {
t.Fatalf("legacy lane references = %#v, want trimmed map", lane.References) t.Fatalf("lane references = %#v, want trimmed map", lane.References)
} }
wantExtract := map[string]string{ wantExtract := map[string]string{
"glossary": "./glossary.md", "glossary": "./glossary.md",

View File

@@ -1,6 +1,7 @@
package config package config
import ( import (
"fmt"
"strings" "strings"
"testing" "testing"
@@ -497,6 +498,11 @@ func fakeCatalog(t *testing.T, overrides ...pipeline.ModuleSpec) pipeline.Module
for _, override := range overrides { for _, override := range overrides {
specs[override.Key] = override specs[override.Key] = override
} }
for _, key := range []string{"fake/extract", "appendorder", "noop"} {
spec := specs[key]
spec.ArtifactKind = fakeArtifactKind
specs[key] = spec
}
inputs := pipeline.NewInputAdapterRegistry() inputs := pipeline.NewInputAdapterRegistry()
chunkers := pipeline.NewChunkerRegistry() chunkers := pipeline.NewChunkerRegistry()
@@ -515,10 +521,15 @@ func fakeCatalog(t *testing.T, overrides ...pipeline.ModuleSpec) pipeline.Module
mustRegisterValidator(t, validators, specs["fake/llm-validator"]) mustRegisterValidator(t, validators, specs["fake/llm-validator"])
mustRegisterOutput(t, outputs, specs["json"]) mustRegisterOutput(t, outputs, specs["json"])
codecs := pipeline.NewArtifactCodecRegistry()
if err := pipeline.RegisterArtifactCodec(codecs, fakeArtifactCodec{}); err != nil {
t.Fatalf("register artifact codec: %v", err)
}
return pipeline.ModuleCatalog{ return pipeline.ModuleCatalog{
Inputs: inputs, Inputs: inputs,
Chunkers: chunkers, Chunkers: chunkers,
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), ArtifactCodecs: codecs,
Extractors: extractors, Extractors: extractors,
Mergers: mergers, Mergers: mergers,
Normalizers: normalizers, Normalizers: normalizers,
@@ -544,21 +555,32 @@ func mustRegisterChunker(t *testing.T, registry *pipeline.ChunkerRegistry, spec
func mustRegisterExtractor(t *testing.T, registry *pipeline.ExtractorRegistry, spec pipeline.ModuleSpec) { func mustRegisterExtractor(t *testing.T, registry *pipeline.ExtractorRegistry, spec pipeline.ModuleSpec) {
t.Helper() t.Helper()
if err := registry.RegisterLegacyRawWithSpec(spec, func() (contracts.LegacyRawExtractor, error) { return nil, nil }); err != nil { validateOptions := func(options map[string]any) error {
if err := pipeline.RejectUnknownOptions(options, "temperature"); err != nil {
return err
}
if value, ok := options["temperature"]; ok {
if _, ok := value.(float64); !ok {
return fmt.Errorf("temperature must be a number")
}
}
return nil
}
if err := pipeline.RegisterExtractorBuilder[fakeArtifact](registry, spec, validateOptions, func(pipeline.BuildRequest) (contracts.Extractor[fakeArtifact], error) { return nil, nil }); err != nil {
t.Fatalf("register extractor: %v", err) t.Fatalf("register extractor: %v", err)
} }
} }
func mustRegisterMerger(t *testing.T, registry *pipeline.MergerRegistry, spec pipeline.ModuleSpec) { func mustRegisterMerger(t *testing.T, registry *pipeline.MergerRegistry, spec pipeline.ModuleSpec) {
t.Helper() t.Helper()
if err := registry.RegisterLegacyRawWithSpec(spec, func() (contracts.LegacyRawMerger, error) { return nil, nil }); err != nil { if err := pipeline.RegisterMerger[fakeArtifact](registry, spec, func() (contracts.Merger[fakeArtifact], error) { return nil, nil }); err != nil {
t.Fatalf("register merger: %v", err) t.Fatalf("register merger: %v", err)
} }
} }
func mustRegisterNormalizer(t *testing.T, registry *pipeline.NormalizerRegistry, spec pipeline.ModuleSpec) { func mustRegisterNormalizer(t *testing.T, registry *pipeline.NormalizerRegistry, spec pipeline.ModuleSpec) {
t.Helper() t.Helper()
if err := registry.RegisterLegacyRawWithSpec(spec, func() (contracts.LegacyRawNormalizer, error) { return nil, nil }); err != nil { if err := pipeline.RegisterNormalizer[fakeArtifact](registry, spec, func() (contracts.Normalizer[fakeArtifact], error) { return nil, nil }); err != nil {
t.Fatalf("register normalizer: %v", err) t.Fatalf("register normalizer: %v", err)
} }
} }
@@ -570,11 +592,32 @@ func mustRegisterValidator(t *testing.T, registry *pipeline.ValidatorRegistry, s
executionClass = contracts.ExecutionClassLLMBacked executionClass = contracts.ExecutionClassLLMBacked
} }
validatorSpec := pipeline.ValidatorSpec{Key: spec.Key, ExecutionClass: executionClass} validatorSpec := pipeline.ValidatorSpec{Key: spec.Key, ExecutionClass: executionClass}
if err := registry.RegisterLegacyRawWithSpec(validatorSpec, func() (contracts.LegacyRawValidator, error) { return nil, nil }); err != nil { if err := pipeline.RegisterTypedValidator[fakeArtifact](registry, fakeArtifactKind, validatorSpec, func() (contracts.TypedValidator[fakeArtifact], error) { return nil, nil }); err != nil {
t.Fatalf("register validator: %v", err) t.Fatalf("register validator: %v", err)
} }
} }
const fakeArtifactKind contracts.ArtifactKind = "test/artifact"
type fakeArtifact string
type fakeArtifactCodec struct{}
func (fakeArtifactCodec) Kind() contracts.ArtifactKind { return fakeArtifactKind }
func (fakeArtifactCodec) Schema() contracts.ArtifactSchema {
return contracts.ArtifactSchema{ID: "urn:notarius:test:artifact", Name: "Test artifact", Version: "1", JSONSchema: []byte(`{"type":"string"}`)}
}
func (fakeArtifactCodec) MediaType() string { return "application/json" }
func (fakeArtifactCodec) Encode(value fakeArtifact) ([]byte, error) {
return []byte(fmt.Sprintf("%q", value)), nil
}
func (fakeArtifactCodec) Decode(content []byte) (fakeArtifact, error) {
if len(content) < 2 {
return "", fmt.Errorf("invalid test artifact")
}
return fakeArtifact(content[1 : len(content)-1]), nil
}
func mustRegisterOutput(t *testing.T, registry *pipeline.OutputEncoderRegistry, spec pipeline.ModuleSpec) { func mustRegisterOutput(t *testing.T, registry *pipeline.OutputEncoderRegistry, spec pipeline.ModuleSpec) {
t.Helper() t.Helper()
if err := registry.RegisterWithSpec(spec, func() (contracts.OutputEncoder, error) { return nil, nil }); err != nil { if err := registry.RegisterWithSpec(spec, func() (contracts.OutputEncoder, error) { return nil, nil }); err != nil {

View File

@@ -88,147 +88,77 @@ func (l *WorkspaceLoader) Chunk(moduleKey string, sourceDigest string) (pipeline
return pipeline.ChunkCheckpoint{Chunks: chunks, Warnings: cloneWarnings(payload.Warnings)}, reusedDecision() return pipeline.ChunkCheckpoint{Chunks: chunks, Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
} }
func (l *WorkspaceLoader) Extract(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.ExtractCheckpoint, pipeline.CheckpointDecision) { func (l *WorkspaceLoader) Extract(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.ExtractCheckpoint, pipeline.CheckpointDecision) {
var manifest coreworkspace.ExtractLaneManifest
if decision := l.readJSON(laneManifestPath("extract", laneID), &manifest); !decision.Reused {
return pipeline.ExtractCheckpoint{}, decision
}
if decision := l.validateLaneManifest(manifest.StageManifest, coreworkspace.StageExtract, laneID, moduleKey, dependencies, coreworkspace.StatusSucceeded, coreworkspace.StatusSucceededWithRejections); !decision.Reused {
return pipeline.ExtractCheckpoint{}, decision
}
var payload extractOutputsEnvelope
if decision := l.readJSON(lanePayloadPath("extract", laneID, "outputs.json"), &payload); !decision.Reused {
return pipeline.ExtractCheckpoint{}, decision
}
outputs, err := extractOutputsFromEnvelope(payload.Outputs)
if err != nil {
return pipeline.ExtractCheckpoint{}, invalidDecision("extract checkpoint payload is invalid: %v", err)
}
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), rawOutputDigests(extractPayloads(outputs))) {
return pipeline.ExtractCheckpoint{}, invalidDecision("extract checkpoint output digests do not match payload")
}
return pipeline.ExtractCheckpoint{
Outputs: outputs,
Rejected: cloneRejectedOutputs(payload.Rejected),
Warnings: cloneWarnings(payload.Warnings),
}, reusedDecision()
}
func (l *WorkspaceLoader) ArtifactExtract(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.ArtifactExtractCheckpoint, pipeline.CheckpointDecision) {
var manifest coreworkspace.ExtractLaneManifest var manifest coreworkspace.ExtractLaneManifest
if d := l.readJSON(laneManifestPath("extract", laneID), &manifest); !d.Reused { if d := l.readJSON(laneManifestPath("extract", laneID), &manifest); !d.Reused {
return pipeline.ArtifactExtractCheckpoint{}, d return pipeline.ExtractCheckpoint{}, d
} }
if d := l.validateLaneManifest(manifest.StageManifest, coreworkspace.StageExtract, laneID, moduleKey, dependencies, coreworkspace.StatusSucceeded, coreworkspace.StatusSucceededWithRejections); !d.Reused { if d := l.validateLaneManifest(manifest.StageManifest, coreworkspace.StageExtract, laneID, moduleKey, dependencies, coreworkspace.StatusSucceeded, coreworkspace.StatusSucceededWithRejections); !d.Reused {
return pipeline.ArtifactExtractCheckpoint{}, d return pipeline.ExtractCheckpoint{}, d
} }
var payload artifactExtractEnvelope var payload artifactExtractEnvelope
if d := l.readJSON(lanePayloadPath("extract", laneID, "outputs.json"), &payload); !d.Reused { if d := l.readJSON(lanePayloadPath("extract", laneID, "outputs.json"), &payload); !d.Reused {
return pipeline.ArtifactExtractCheckpoint{}, d return pipeline.ExtractCheckpoint{}, d
} }
outputs, err := artifactCheckpointOutputs(payload.Outputs) outputs, err := artifactCheckpointOutputs(payload.Outputs)
if err != nil { if err != nil {
return pipeline.ArtifactExtractCheckpoint{}, invalidDecision("extract artifact checkpoint payload is invalid: %v", err) return pipeline.ExtractCheckpoint{}, invalidDecision("extract artifact checkpoint payload is invalid: %v", err)
} }
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), artifactOutputDigests(outputs)) { if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), artifactOutputDigests(outputs)) {
return pipeline.ArtifactExtractCheckpoint{}, invalidDecision("extract artifact checkpoint output digests do not match payload") return pipeline.ExtractCheckpoint{}, invalidDecision("extract artifact checkpoint output digests do not match payload")
} }
return pipeline.ArtifactExtractCheckpoint{Outputs: outputs, Rejected: cloneRejectedOutputs(payload.Rejected), Warnings: cloneWarnings(payload.Warnings)}, reusedDecision() return pipeline.ExtractCheckpoint{Outputs: outputs, Rejected: cloneRejectedOutputs(payload.Rejected), Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
} }
func (l *WorkspaceLoader) Merge(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.MergeCheckpoint, pipeline.CheckpointDecision) { func (l *WorkspaceLoader) Merge(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.MergeCheckpoint, pipeline.CheckpointDecision) {
var manifest coreworkspace.MergeLaneManifest
if decision := l.readJSON(laneManifestPath("merge", laneID), &manifest); !decision.Reused {
return pipeline.MergeCheckpoint{}, decision
}
if decision := l.validateLaneManifest(manifest.StageManifest, coreworkspace.StageMerge, laneID, moduleKey, dependencies, coreworkspace.StatusSucceeded); !decision.Reused {
return pipeline.MergeCheckpoint{}, decision
}
var payload mergeOutputEnvelope
if decision := l.readJSON(lanePayloadPath("merge", laneID, "output.json"), &payload); !decision.Reused {
return pipeline.MergeCheckpoint{}, decision
}
output, err := mergeOutputFromEnvelope(payload.Output)
if err != nil {
return pipeline.MergeCheckpoint{}, invalidDecision("merge checkpoint payload is invalid: %v", err)
}
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), rawOutputDigests([]contracts.RawPayload{output.Payload})) {
return pipeline.MergeCheckpoint{}, invalidDecision("merge checkpoint output digest does not match payload")
}
return pipeline.MergeCheckpoint{Output: output, Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
}
func (l *WorkspaceLoader) ArtifactMerge(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.ArtifactMergeCheckpoint, pipeline.CheckpointDecision) {
var manifest coreworkspace.MergeLaneManifest var manifest coreworkspace.MergeLaneManifest
if d := l.readJSON(laneManifestPath("merge", laneID), &manifest); !d.Reused { if d := l.readJSON(laneManifestPath("merge", laneID), &manifest); !d.Reused {
return pipeline.ArtifactMergeCheckpoint{}, d return pipeline.MergeCheckpoint{}, d
} }
if d := l.validateLaneManifest(manifest.StageManifest, coreworkspace.StageMerge, laneID, moduleKey, dependencies, coreworkspace.StatusSucceeded); !d.Reused { if d := l.validateLaneManifest(manifest.StageManifest, coreworkspace.StageMerge, laneID, moduleKey, dependencies, coreworkspace.StatusSucceeded); !d.Reused {
return pipeline.ArtifactMergeCheckpoint{}, d return pipeline.MergeCheckpoint{}, d
} }
var payload artifactSingleEnvelope var payload artifactSingleEnvelope
if d := l.readJSON(lanePayloadPath("merge", laneID, "output.json"), &payload); !d.Reused { if d := l.readJSON(lanePayloadPath("merge", laneID, "output.json"), &payload); !d.Reused {
return pipeline.ArtifactMergeCheckpoint{}, d return pipeline.MergeCheckpoint{}, d
} }
values, err := artifactCheckpointOutputs([]artifactCheckpointEnvelope{payload.Output}) values, err := artifactCheckpointOutputs([]artifactCheckpointEnvelope{payload.Output})
if err != nil { if err != nil {
return pipeline.ArtifactMergeCheckpoint{}, invalidDecision("merge artifact checkpoint payload is invalid: %v", err) return pipeline.MergeCheckpoint{}, invalidDecision("merge artifact checkpoint payload is invalid: %v", err)
} }
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), artifactOutputDigests(values)) { if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), artifactOutputDigests(values)) {
return pipeline.ArtifactMergeCheckpoint{}, invalidDecision("merge artifact checkpoint output digest does not match payload") return pipeline.MergeCheckpoint{}, invalidDecision("merge artifact checkpoint output digest does not match payload")
} }
return pipeline.ArtifactMergeCheckpoint{Output: values[0], Warnings: cloneWarnings(payload.Warnings)}, reusedDecision() return pipeline.MergeCheckpoint{Output: values[0], Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
} }
func (l *WorkspaceLoader) Normalize(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.NormalizeCheckpoint, pipeline.CheckpointDecision) { func (l *WorkspaceLoader) Normalize(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.NormalizeCheckpoint, pipeline.CheckpointDecision) {
var manifest coreworkspace.NormalizeLaneManifest
if decision := l.readJSON(laneManifestPath("normalize", laneID), &manifest); !decision.Reused {
return pipeline.NormalizeCheckpoint{}, decision
}
if decision := l.validateLaneManifest(manifest.StageManifest, coreworkspace.StageNormalize, laneID, moduleKey, dependencies, coreworkspace.StatusSucceeded); !decision.Reused {
return pipeline.NormalizeCheckpoint{}, decision
}
var payload normalizeOutputEnvelope
if decision := l.readJSON(lanePayloadPath("normalize", laneID, "output.json"), &payload); !decision.Reused {
return pipeline.NormalizeCheckpoint{}, decision
}
output, err := normalizeOutputFromEnvelope(payload.Output)
if err != nil {
return pipeline.NormalizeCheckpoint{}, invalidDecision("normalize checkpoint payload is invalid: %v", err)
}
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), rawOutputDigests([]contracts.RawPayload{output.Payload})) {
return pipeline.NormalizeCheckpoint{}, invalidDecision("normalize checkpoint output digest does not match payload")
}
return pipeline.NormalizeCheckpoint{Output: output, Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
}
func (l *WorkspaceLoader) ArtifactNormalize(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.ArtifactNormalizeCheckpoint, pipeline.CheckpointDecision) {
var manifest coreworkspace.NormalizeLaneManifest var manifest coreworkspace.NormalizeLaneManifest
if d := l.readJSON(laneManifestPath("normalize", laneID), &manifest); !d.Reused { if d := l.readJSON(laneManifestPath("normalize", laneID), &manifest); !d.Reused {
return pipeline.ArtifactNormalizeCheckpoint{}, d return pipeline.NormalizeCheckpoint{}, d
} }
if d := l.validateLaneManifest(manifest.StageManifest, coreworkspace.StageNormalize, laneID, moduleKey, dependencies, coreworkspace.StatusSucceeded); !d.Reused { if d := l.validateLaneManifest(manifest.StageManifest, coreworkspace.StageNormalize, laneID, moduleKey, dependencies, coreworkspace.StatusSucceeded); !d.Reused {
return pipeline.ArtifactNormalizeCheckpoint{}, d return pipeline.NormalizeCheckpoint{}, d
} }
var payload artifactSingleEnvelope var payload artifactSingleEnvelope
if d := l.readJSON(lanePayloadPath("normalize", laneID, "output.json"), &payload); !d.Reused { if d := l.readJSON(lanePayloadPath("normalize", laneID, "output.json"), &payload); !d.Reused {
return pipeline.ArtifactNormalizeCheckpoint{}, d return pipeline.NormalizeCheckpoint{}, d
} }
values, err := artifactCheckpointOutputs([]artifactCheckpointEnvelope{payload.Output}) values, err := artifactCheckpointOutputs([]artifactCheckpointEnvelope{payload.Output})
if err != nil { if err != nil {
return pipeline.ArtifactNormalizeCheckpoint{}, invalidDecision("normalize artifact checkpoint payload is invalid: %v", err) return pipeline.NormalizeCheckpoint{}, invalidDecision("normalize artifact checkpoint payload is invalid: %v", err)
} }
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), artifactOutputDigests(values)) { if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), artifactOutputDigests(values)) {
return pipeline.ArtifactNormalizeCheckpoint{}, invalidDecision("normalize artifact checkpoint output digest does not match payload") return pipeline.NormalizeCheckpoint{}, invalidDecision("normalize artifact checkpoint output digest does not match payload")
} }
return pipeline.ArtifactNormalizeCheckpoint{Output: values[0], Warnings: cloneWarnings(payload.Warnings)}, reusedDecision() return pipeline.NormalizeCheckpoint{Output: values[0], Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
} }
func artifactCheckpointOutputs(values []artifactCheckpointEnvelope) ([]pipeline.ArtifactCheckpointOutput, error) { func artifactCheckpointOutputs(values []artifactCheckpointEnvelope) ([]pipeline.CheckpointArtifact, error) {
if len(values) == 0 { if len(values) == 0 {
return nil, nil return nil, nil
} }
out := make([]pipeline.ArtifactCheckpointOutput, 0, len(values)) out := make([]pipeline.CheckpointArtifact, 0, len(values))
for _, v := range values { for _, v := range values {
content, err := contentFromEnvelope(v.Content) content, err := contentFromEnvelope(v.Content)
if err != nil { if err != nil {
@@ -237,7 +167,7 @@ func artifactCheckpointOutputs(values []artifactCheckpointEnvelope) ([]pipeline.
if strings.TrimSpace(string(v.Kind)) == "" || strings.TrimSpace(v.Schema.ID) == "" || strings.TrimSpace(v.Schema.Version) == "" || strings.TrimSpace(v.SchemaDigest) == "" { if strings.TrimSpace(string(v.Kind)) == "" || strings.TrimSpace(v.Schema.ID) == "" || strings.TrimSpace(v.Schema.Version) == "" || strings.TrimSpace(v.SchemaDigest) == "" {
return nil, fmt.Errorf("artifact codec identity is incomplete") return nil, fmt.Errorf("artifact codec identity is incomplete")
} }
out = append(out, pipeline.ArtifactCheckpointOutput{LaneID: v.LaneID, ModuleKey: v.ModuleKey, SourceID: v.SourceID, ChunkID: v.ChunkID, ChunkIndex: v.ChunkIndex, ChunkRef: v.ChunkRef, SchemaDigest: v.SchemaDigest, Artifact: contracts.SerializedArtifact{Kind: v.Kind, Schema: v.Schema, MediaType: v.Content.MediaType, Content: content, Metadata: cloneMetadata(v.Content.Metadata)}}) out = append(out, pipeline.CheckpointArtifact{LaneID: v.LaneID, ModuleKey: v.ModuleKey, SourceID: v.SourceID, ChunkID: v.ChunkID, ChunkIndex: v.ChunkIndex, ChunkRef: v.ChunkRef, SchemaDigest: v.SchemaDigest, Artifact: contracts.SerializedArtifact{Kind: v.Kind, Schema: v.Schema, MediaType: v.Content.MediaType, Content: content, Metadata: cloneMetadata(v.Content.Metadata)}})
} }
return out, nil return out, nil
} }
@@ -326,70 +256,6 @@ func sourceChunksFromEnvelope(values []chunkEnvelope) ([]source.Chunk, error) {
return out, nil return out, nil
} }
func extractOutputsFromEnvelope(values []extractOutputEnvelope) ([]contracts.ExtractOutput, error) {
if len(values) == 0 {
return nil, nil
}
out := make([]contracts.ExtractOutput, 0, len(values))
for _, value := range values {
payload, err := rawPayloadFromEnvelope(value.Payload)
if err != nil {
return nil, err
}
out = append(out, contracts.ExtractOutput{
LaneID: value.LaneID,
ExtractorKey: value.ExtractorKey,
SourceID: value.SourceID,
ChunkID: value.ChunkID,
ChunkIndex: value.ChunkIndex,
Schema: value.Schema,
Payload: payload,
})
}
return out, nil
}
func mergeOutputFromEnvelope(value mergeOutputPayload) (contracts.MergeOutput, error) {
payload, err := rawPayloadFromEnvelope(value.Payload)
if err != nil {
return contracts.MergeOutput{}, err
}
return contracts.MergeOutput{
LaneID: value.LaneID,
MergerKey: value.MergerKey,
SourceID: value.SourceID,
Schema: value.Schema,
Payload: payload,
}, nil
}
func normalizeOutputFromEnvelope(value normalizeOutputPayload) (contracts.NormalizeOutput, error) {
payload, err := rawPayloadFromEnvelope(value.Payload)
if err != nil {
return contracts.NormalizeOutput{}, err
}
return contracts.NormalizeOutput{
LaneID: value.LaneID,
NormalizerKey: value.NormalizerKey,
SourceID: value.SourceID,
Schema: value.Schema,
Payload: payload,
}, nil
}
func rawPayloadFromEnvelope(value binaryEnvelope) (contracts.RawPayload, error) {
content, err := contentFromEnvelope(value)
if err != nil {
return contracts.RawPayload{}, err
}
return contracts.RawPayload{
Content: content,
MediaType: value.MediaType,
Metadata: cloneMetadata(value.Metadata),
Warnings: cloneWarnings(value.Warnings),
}, nil
}
func contentFromEnvelope(value binaryEnvelope) ([]byte, error) { func contentFromEnvelope(value binaryEnvelope) ([]byte, error) {
content, err := base64.StdEncoding.DecodeString(value.ContentBase64) content, err := base64.StdEncoding.DecodeString(value.ContentBase64)
if err != nil { if err != nil {

View File

@@ -119,28 +119,7 @@ func (r *WorkspaceRecorder) ExtractRunning(laneID string, moduleKey string, depe
return r.writeManifest(laneManifestPath("extract", laneID), coreworkspace.ExtractLaneManifest{StageManifest: manifest}) return r.writeManifest(laneManifestPath("extract", laneID), coreworkspace.ExtractLaneManifest{StageManifest: manifest})
} }
func (r *WorkspaceRecorder) ExtractSucceeded(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, outputs []contracts.ExtractOutput, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error { func (r *WorkspaceRecorder) ExtractSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, outputs []pipeline.CheckpointArtifact, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error {
payload := extractOutputsEnvelope{
Outputs: extractOutputEnvelopes(outputs),
Rejected: cloneRejectedOutputs(rejected),
Warnings: cloneWarnings(warnings),
}
if err := r.writePayload(lanePayloadPath("extract", laneID, "outputs.json"), payload); err != nil {
return err
}
manifest := r.laneManifest(coreworkspace.StageExtract, statusForRejected(rejected), laneID, moduleKey, dependencies)
manifest.OutputDigests = workspaceFingerprints(rawOutputDigests(extractPayloads(outputs)))
manifest.ValidationStatus = validationStatusString(warnings, rejected)
manifest.Rejections = rejectionSummaries(rejected)
manifest.CompletedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("extract", laneID), coreworkspace.ExtractLaneManifest{
StageManifest: manifest,
ChunkCount: len(outputs) + len(rejected),
OutputCount: len(outputs),
})
}
func (r *WorkspaceRecorder) ArtifactExtractSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, outputs []pipeline.ArtifactCheckpointOutput, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error {
payload := artifactExtractEnvelope{Outputs: artifactCheckpointEnvelopes(outputs), Rejected: cloneRejectedOutputs(rejected), Warnings: cloneWarnings(warnings)} payload := artifactExtractEnvelope{Outputs: artifactCheckpointEnvelopes(outputs), Rejected: cloneRejectedOutputs(rejected), Warnings: cloneWarnings(warnings)}
if err := r.writePayload(lanePayloadPath("extract", laneID, "outputs.json"), payload); err != nil { if err := r.writePayload(lanePayloadPath("extract", laneID, "outputs.json"), payload); err != nil {
return err return err
@@ -166,27 +145,12 @@ func (r *WorkspaceRecorder) MergeRunning(laneID string, moduleKey string, depend
return r.writeManifest(laneManifestPath("merge", laneID), coreworkspace.MergeLaneManifest{StageManifest: manifest}) return r.writeManifest(laneManifestPath("merge", laneID), coreworkspace.MergeLaneManifest{StageManifest: manifest})
} }
func (r *WorkspaceRecorder) MergeSucceeded(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output contracts.MergeOutput, warnings []contracts.Warning) error { func (r *WorkspaceRecorder) MergeSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.CheckpointArtifact, warnings []contracts.Warning) error {
payload := mergeOutputEnvelope{Output: mergeOutputEnvelopeFromOutput(output), Warnings: cloneWarnings(warnings)}
if err := r.writePayload(lanePayloadPath("merge", laneID, "output.json"), payload); err != nil {
return err
}
manifest := r.laneManifest(coreworkspace.StageMerge, coreworkspace.StatusSucceeded, laneID, moduleKey, dependencies)
manifest.OutputDigests = workspaceFingerprints(rawOutputDigests([]contracts.RawPayload{output.Payload}))
manifest.ValidationStatus = validationStatusString(warnings, nil)
manifest.CompletedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("merge", laneID), coreworkspace.MergeLaneManifest{
StageManifest: manifest,
InputCount: len(dependencies),
})
}
func (r *WorkspaceRecorder) ArtifactMergeSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.ArtifactCheckpointOutput, warnings []contracts.Warning) error {
if err := r.writePayload(lanePayloadPath("merge", laneID, "output.json"), artifactSingleEnvelope{Output: artifactCheckpointEnvelopeFromOutput(output), Warnings: cloneWarnings(warnings)}); err != nil { if err := r.writePayload(lanePayloadPath("merge", laneID, "output.json"), artifactSingleEnvelope{Output: artifactCheckpointEnvelopeFromOutput(output), Warnings: cloneWarnings(warnings)}); err != nil {
return err return err
} }
manifest := r.laneManifest(coreworkspace.StageMerge, coreworkspace.StatusSucceeded, laneID, moduleKey, dependencies) manifest := r.laneManifest(coreworkspace.StageMerge, coreworkspace.StatusSucceeded, laneID, moduleKey, dependencies)
manifest.OutputDigests = workspaceFingerprints(artifactOutputDigests([]pipeline.ArtifactCheckpointOutput{output})) manifest.OutputDigests = workspaceFingerprints(artifactOutputDigests([]pipeline.CheckpointArtifact{output}))
manifest.ValidationStatus = validationStatusString(warnings, nil) manifest.ValidationStatus = validationStatusString(warnings, nil)
manifest.CompletedAt = timePtr(r.timestamp()) manifest.CompletedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("merge", laneID), coreworkspace.MergeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)}) return r.writeManifest(laneManifestPath("merge", laneID), coreworkspace.MergeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
@@ -213,24 +177,12 @@ func (r *WorkspaceRecorder) NormalizeRunning(laneID string, moduleKey string, de
return r.writeManifest(laneManifestPath("normalize", laneID), coreworkspace.NormalizeLaneManifest{StageManifest: manifest}) return r.writeManifest(laneManifestPath("normalize", laneID), coreworkspace.NormalizeLaneManifest{StageManifest: manifest})
} }
func (r *WorkspaceRecorder) NormalizeSucceeded(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output contracts.NormalizeOutput, warnings []contracts.Warning) error { func (r *WorkspaceRecorder) NormalizeSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.CheckpointArtifact, warnings []contracts.Warning) error {
payload := normalizeOutputEnvelope{Output: normalizeOutputEnvelopeFromOutput(output), Warnings: cloneWarnings(warnings)}
if err := r.writePayload(lanePayloadPath("normalize", laneID, "output.json"), payload); err != nil {
return err
}
manifest := r.laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusSucceeded, laneID, moduleKey, dependencies)
manifest.OutputDigests = workspaceFingerprints(rawOutputDigests([]contracts.RawPayload{output.Payload}))
manifest.ValidationStatus = validationStatusString(warnings, nil)
manifest.CompletedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("normalize", laneID), coreworkspace.NormalizeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
}
func (r *WorkspaceRecorder) ArtifactNormalizeSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.ArtifactCheckpointOutput, warnings []contracts.Warning) error {
if err := r.writePayload(lanePayloadPath("normalize", laneID, "output.json"), artifactSingleEnvelope{Output: artifactCheckpointEnvelopeFromOutput(output), Warnings: cloneWarnings(warnings)}); err != nil { if err := r.writePayload(lanePayloadPath("normalize", laneID, "output.json"), artifactSingleEnvelope{Output: artifactCheckpointEnvelopeFromOutput(output), Warnings: cloneWarnings(warnings)}); err != nil {
return err return err
} }
manifest := r.laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusSucceeded, laneID, moduleKey, dependencies) manifest := r.laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusSucceeded, laneID, moduleKey, dependencies)
manifest.OutputDigests = workspaceFingerprints(artifactOutputDigests([]pipeline.ArtifactCheckpointOutput{output})) manifest.OutputDigests = workspaceFingerprints(artifactOutputDigests([]pipeline.CheckpointArtifact{output}))
manifest.ValidationStatus = validationStatusString(warnings, nil) manifest.ValidationStatus = validationStatusString(warnings, nil)
manifest.CompletedAt = timePtr(r.timestamp()) manifest.CompletedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("normalize", laneID), coreworkspace.NormalizeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)}) return r.writeManifest(laneManifestPath("normalize", laneID), coreworkspace.NormalizeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
@@ -308,48 +260,6 @@ type chunkEnvelope struct {
Metadata map[string]any `json:"metadata,omitempty"` Metadata map[string]any `json:"metadata,omitempty"`
} }
type extractOutputsEnvelope struct {
Outputs []extractOutputEnvelope `json:"outputs"`
Rejected []contracts.RejectedOutput `json:"rejected,omitempty"`
Warnings []contracts.Warning `json:"warnings,omitempty"`
}
type extractOutputEnvelope struct {
LaneID string `json:"lane_id"`
ExtractorKey string `json:"extractor_key"`
SourceID string `json:"source_id"`
ChunkID string `json:"chunk_id"`
ChunkIndex int `json:"chunk_index"`
Schema contracts.ResponseSchema `json:"schema,omitempty"`
Payload binaryEnvelope `json:"payload"`
}
type mergeOutputEnvelope struct {
Output mergeOutputPayload `json:"output"`
Warnings []contracts.Warning `json:"warnings,omitempty"`
}
type mergeOutputPayload struct {
LaneID string `json:"lane_id"`
MergerKey string `json:"merger_key"`
SourceID string `json:"source_id,omitempty"`
Schema contracts.ResponseSchema `json:"schema,omitempty"`
Payload binaryEnvelope `json:"payload"`
}
type normalizeOutputEnvelope struct {
Output normalizeOutputPayload `json:"output"`
Warnings []contracts.Warning `json:"warnings,omitempty"`
}
type normalizeOutputPayload struct {
LaneID string `json:"lane_id"`
NormalizerKey string `json:"normalizer_key"`
SourceID string `json:"source_id,omitempty"`
Schema contracts.ResponseSchema `json:"schema,omitempty"`
Payload binaryEnvelope `json:"payload"`
}
type binaryEnvelope struct { type binaryEnvelope struct {
ContentBase64 string `json:"content_base64,omitempty"` ContentBase64 string `json:"content_base64,omitempty"`
ContentDigest string `json:"content_digest,omitempty"` ContentDigest string `json:"content_digest,omitempty"`
@@ -380,12 +290,12 @@ type artifactSingleEnvelope struct {
Warnings []contracts.Warning `json:"warnings,omitempty"` Warnings []contracts.Warning `json:"warnings,omitempty"`
} }
func artifactCheckpointEnvelopeFromOutput(output pipeline.ArtifactCheckpointOutput) artifactCheckpointEnvelope { func artifactCheckpointEnvelopeFromOutput(output pipeline.CheckpointArtifact) artifactCheckpointEnvelope {
schema := contracts.CloneArtifactSchema(output.Artifact.Schema) schema := contracts.CloneArtifactSchema(output.Artifact.Schema)
schema.JSONSchema = nil schema.JSONSchema = nil
return artifactCheckpointEnvelope{LaneID: output.LaneID, ModuleKey: output.ModuleKey, SourceID: output.SourceID, ChunkID: output.ChunkID, ChunkIndex: output.ChunkIndex, ChunkRef: output.ChunkRef, Kind: output.Artifact.Kind, Schema: schema, SchemaDigest: output.SchemaDigest, Content: binaryEnvelopeFromContent(output.Artifact.Content, output.Artifact.MediaType, output.Artifact.Metadata, nil)} return artifactCheckpointEnvelope{LaneID: output.LaneID, ModuleKey: output.ModuleKey, SourceID: output.SourceID, ChunkID: output.ChunkID, ChunkIndex: output.ChunkIndex, ChunkRef: output.ChunkRef, Kind: output.Artifact.Kind, Schema: schema, SchemaDigest: output.SchemaDigest, Content: binaryEnvelopeFromContent(output.Artifact.Content, output.Artifact.MediaType, output.Artifact.Metadata, nil)}
} }
func artifactCheckpointEnvelopes(outputs []pipeline.ArtifactCheckpointOutput) []artifactCheckpointEnvelope { func artifactCheckpointEnvelopes(outputs []pipeline.CheckpointArtifact) []artifactCheckpointEnvelope {
if len(outputs) == 0 { if len(outputs) == 0 {
return nil return nil
} }
@@ -395,7 +305,7 @@ func artifactCheckpointEnvelopes(outputs []pipeline.ArtifactCheckpointOutput) []
} }
return out return out
} }
func artifactOutputDigests(outputs []pipeline.ArtifactCheckpointOutput) []pipeline.CheckpointFingerprint { func artifactOutputDigests(outputs []pipeline.CheckpointArtifact) []pipeline.CheckpointFingerprint {
values := make([]pipeline.CheckpointFingerprint, 0, len(outputs)) values := make([]pipeline.CheckpointFingerprint, 0, len(outputs))
for i, v := range outputs { for i, v := range outputs {
values = append(values, pipeline.CheckpointFingerprint{Name: fmt.Sprintf("artifact[%d]", i), Value: contentDigest(v.Artifact.Content)}) values = append(values, pipeline.CheckpointFingerprint{Name: fmt.Sprintf("artifact[%d]", i), Value: contentDigest(v.Artifact.Content)})
@@ -422,54 +332,6 @@ func chunkEnvelopes(chunks []source.Chunk) []chunkEnvelope {
return out return out
} }
func extractOutputEnvelopes(outputs []contracts.ExtractOutput) []extractOutputEnvelope {
if len(outputs) == 0 {
return nil
}
out := make([]extractOutputEnvelope, 0, len(outputs))
for _, output := range outputs {
out = append(out, extractOutputEnvelope{
LaneID: output.LaneID,
ExtractorKey: output.ExtractorKey,
SourceID: output.SourceID,
ChunkID: output.ChunkID,
ChunkIndex: output.ChunkIndex,
Schema: schemaEnvelope(output.Schema),
Payload: binaryEnvelopeFromPayload(output.Payload),
})
}
return out
}
func mergeOutputEnvelopeFromOutput(output contracts.MergeOutput) mergeOutputPayload {
return mergeOutputPayload{
LaneID: output.LaneID,
MergerKey: output.MergerKey,
SourceID: output.SourceID,
Schema: schemaEnvelope(output.Schema),
Payload: binaryEnvelopeFromPayload(output.Payload),
}
}
func normalizeOutputEnvelopeFromOutput(output contracts.NormalizeOutput) normalizeOutputPayload {
return normalizeOutputPayload{
LaneID: output.LaneID,
NormalizerKey: output.NormalizerKey,
SourceID: output.SourceID,
Schema: schemaEnvelope(output.Schema),
Payload: binaryEnvelopeFromPayload(output.Payload),
}
}
func schemaEnvelope(schema contracts.ResponseSchema) contracts.ResponseSchema {
schema.JSONSchema = nil
return schema
}
func binaryEnvelopeFromPayload(payload contracts.RawPayload) binaryEnvelope {
return binaryEnvelopeFromContent(payload.Content, payload.MediaType, payload.Metadata, payload.Warnings)
}
func binaryEnvelopeFromContent(content []byte, mediaType string, metadata map[string]any, warnings []contracts.Warning) binaryEnvelope { func binaryEnvelopeFromContent(content []byte, mediaType string, metadata map[string]any, warnings []contracts.Warning) binaryEnvelope {
return binaryEnvelope{ return binaryEnvelope{
ContentBase64: base64.StdEncoding.EncodeToString(content), ContentBase64: base64.StdEncoding.EncodeToString(content),
@@ -528,28 +390,6 @@ func cloneMetadata(metadata map[string]any) map[string]any {
return out return out
} }
func rawOutputDigests(payloads []contracts.RawPayload) []pipeline.CheckpointFingerprint {
values := make([]pipeline.CheckpointFingerprint, 0, len(payloads))
for i, payload := range payloads {
values = append(values, pipeline.CheckpointFingerprint{
Name: fmt.Sprintf("payload[%d]", i),
Value: contentDigest(payload.Content),
})
}
return normalizeFingerprints(values)
}
func extractPayloads(outputs []contracts.ExtractOutput) []contracts.RawPayload {
if len(outputs) == 0 {
return nil
}
payloads := make([]contracts.RawPayload, 0, len(outputs))
for _, output := range outputs {
payloads = append(payloads, output.Payload)
}
return payloads
}
func chunkOutputDigests(chunks []source.Chunk) ([]pipeline.CheckpointFingerprint, error) { func chunkOutputDigests(chunks []source.Chunk) ([]pipeline.CheckpointFingerprint, error) {
values := make([]pipeline.CheckpointFingerprint, 0, len(chunks)) values := make([]pipeline.CheckpointFingerprint, 0, len(chunks))
for _, chunk := range chunks { for _, chunk := range chunks {

View File

@@ -79,117 +79,19 @@ func TestWorkspaceRecorderWritesSuccessfulCheckpointFiles(t *testing.T) {
} }
} }
func TestWorkspaceLoaderReusesSuccessfulCheckpointFiles(t *testing.T) {
root := t.TempDir()
recorder := newTestRecorder(t, root)
loader := &WorkspaceLoader{root: root}
doc := &source.SourceDocument{
ID: "source-1",
Kind: "document",
Format: "text/plain",
Digest: "sha256:source",
Units: []source.SourceUnit{{ID: 1, Kind: "line", Text: "hello", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}}},
}
chunks := []source.Chunk{
{
ID: "chunk-1",
SourceID: "source-1",
Index: 0,
Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1},
Content: []byte("chunk content"),
MediaType: "text/plain",
Units: doc.Units,
},
}
extractOutput := contracts.ExtractOutput{
LaneID: "spells",
ExtractorKey: "dnd/spells",
SourceID: doc.ID,
ChunkID: "chunk-1",
Payload: contracts.RawPayload{
Content: []byte(`{"spell":"cure wounds"}`),
MediaType: "application/json",
},
}
mergeOutput := contracts.MergeOutput{
LaneID: "spells",
MergerKey: "appendorder",
SourceID: doc.ID,
Payload: contracts.RawPayload{
Content: []byte(`{"merged":true}`),
MediaType: "application/json",
},
}
normalizeOutput := contracts.NormalizeOutput{
LaneID: "spells",
NormalizerKey: "noop",
SourceID: doc.ID,
Payload: contracts.RawPayload{
Content: []byte(`{"normalized":true}`),
MediaType: "application/json",
},
}
if err := recorder.SourceSucceeded("seriatim", doc); err != nil {
t.Fatalf("SourceSucceeded: %v", err)
}
if err := recorder.ChunkSucceeded("generic", doc.Digest, chunks, nil); err != nil {
t.Fatalf("ChunkSucceeded: %v", err)
}
extractDeps := []pipeline.CheckpointFingerprint{{Name: "chunks", Value: "sha256:chunks"}}
if err := recorder.ExtractSucceeded("spells", "dnd/spells", extractDeps, []contracts.ExtractOutput{extractOutput}, nil, nil); err != nil {
t.Fatalf("ExtractSucceeded: %v", err)
}
mergeDeps := rawOutputDigests([]contracts.RawPayload{extractOutput.Payload})
if err := recorder.MergeSucceeded("spells", "appendorder", mergeDeps, mergeOutput, nil); err != nil {
t.Fatalf("MergeSucceeded: %v", err)
}
normalizeDeps := rawOutputDigests([]contracts.RawPayload{mergeOutput.Payload})
if err := recorder.NormalizeSucceeded("spells", "noop", normalizeDeps, normalizeOutput, nil); err != nil {
t.Fatalf("NormalizeSucceeded: %v", err)
}
sourceCheckpoint, decision := loader.Source("seriatim")
if !decision.Reused || sourceCheckpoint.Document.ID != "source-1" {
t.Fatalf("source decision = %#v checkpoint=%#v, want reused", decision, sourceCheckpoint)
}
if got, want := sourceCheckpoint.Document.Units[0].Ref, doc.Units[0].Ref; got != want {
t.Fatalf("checkpoint source unit ref = %#v, want %#v", got, want)
}
chunkCheckpoint, decision := loader.Chunk("generic", doc.Digest)
if !decision.Reused || len(chunkCheckpoint.Chunks) != 1 || string(chunkCheckpoint.Chunks[0].Content) != "chunk content" {
t.Fatalf("chunk decision = %#v checkpoint=%#v, want reused", decision, chunkCheckpoint)
}
if got, want := chunkCheckpoint.Chunks[0].Ref, chunks[0].Ref; got != want {
t.Fatalf("checkpoint chunk ref = %#v, want %#v", got, want)
}
extractCheckpoint, decision := loader.Extract("spells", "dnd/spells", extractDeps)
if !decision.Reused || len(extractCheckpoint.Outputs) != 1 || string(extractCheckpoint.Outputs[0].Payload.Content) != `{"spell":"cure wounds"}` {
t.Fatalf("extract decision = %#v checkpoint=%#v, want reused", decision, extractCheckpoint)
}
mergeCheckpoint, decision := loader.Merge("spells", "appendorder", mergeDeps)
if !decision.Reused || string(mergeCheckpoint.Output.Payload.Content) != `{"merged":true}` {
t.Fatalf("merge decision = %#v checkpoint=%#v, want reused", decision, mergeCheckpoint)
}
normalizeCheckpoint, decision := loader.Normalize("spells", "noop", normalizeDeps)
if !decision.Reused || string(normalizeCheckpoint.Output.Payload.Content) != `{"normalized":true}` {
t.Fatalf("normalize decision = %#v checkpoint=%#v, want reused", decision, normalizeCheckpoint)
}
}
func TestWorkspaceArtifactCheckpointsRoundTripCodecIdentityAndBytes(t *testing.T) { func TestWorkspaceArtifactCheckpointsRoundTripCodecIdentityAndBytes(t *testing.T) {
root := t.TempDir() root := t.TempDir()
recorder := newTestRecorder(t, root) recorder := newTestRecorder(t, root)
loader := &WorkspaceLoader{root: root} loader := &WorkspaceLoader{root: root}
schema := contracts.ArtifactSchema{ID: "dnd.spell_response", Name: "spell response", Version: "v1", JSONSchema: []byte(`{"type":"object"}`)} schema := contracts.ArtifactSchema{ID: "dnd.spell_response", Name: "spell response", Version: "v1", JSONSchema: []byte(`{"type":"object"}`)}
artifact := contracts.SerializedArtifact{Kind: "dnd.spells", Schema: schema, MediaType: "application/json", Content: []byte(`{"spell_casts":[]}`), Metadata: map[string]any{"spell_cast_count": float64(0)}} artifact := contracts.SerializedArtifact{Kind: "dnd.spells", Schema: schema, MediaType: "application/json", Content: []byte(`{"spell_casts":[]}`), Metadata: map[string]any{"spell_cast_count": float64(0)}}
stored := pipeline.ArtifactCheckpointOutput{LaneID: "spells", ModuleKey: "dnd/spells", SourceID: "source-1", ChunkID: "chunk-1", ChunkIndex: 2, ChunkRef: source.SourceRef{SourceID: "source-1", StartUnitID: 4, EndUnitID: 8}, Artifact: artifact, SchemaDigest: contracts.DigestArtifactSchema(schema)} stored := pipeline.CheckpointArtifact{LaneID: "spells", ModuleKey: "dnd/spells", SourceID: "source-1", ChunkID: "chunk-1", ChunkIndex: 2, ChunkRef: source.SourceRef{SourceID: "source-1", StartUnitID: 4, EndUnitID: 8}, Artifact: artifact, SchemaDigest: contracts.DigestArtifactSchema(schema)}
extractDeps := []pipeline.CheckpointFingerprint{{Name: "chunks", Value: "sha256:chunks"}} extractDeps := []pipeline.CheckpointFingerprint{{Name: "chunks", Value: "sha256:chunks"}}
if err := recorder.ArtifactExtractSucceeded("spells", "dnd/spells", extractDeps, []pipeline.ArtifactCheckpointOutput{stored}, nil, nil); err != nil { if err := recorder.ExtractSucceeded("spells", "dnd/spells", extractDeps, []pipeline.CheckpointArtifact{stored}, nil, nil); err != nil {
t.Fatalf("ArtifactExtractSucceeded: %v", err) t.Fatalf("ExtractSucceeded: %v", err)
} }
extracted, decision := loader.ArtifactExtract("spells", "dnd/spells", extractDeps) extracted, decision := loader.Extract("spells", "dnd/spells", extractDeps)
if !decision.Reused || len(extracted.Outputs) != 1 { if !decision.Reused || len(extracted.Outputs) != 1 {
t.Fatalf("extract decision=%#v checkpoint=%#v, want reused", decision, extracted) t.Fatalf("extract decision=%#v checkpoint=%#v, want reused", decision, extracted)
} }
@@ -198,20 +100,20 @@ func TestWorkspaceArtifactCheckpointsRoundTripCodecIdentityAndBytes(t *testing.T
t.Fatalf("artifact checkpoint = %#v, want codec identity, bytes, and provenance", got) t.Fatalf("artifact checkpoint = %#v, want codec identity, bytes, and provenance", got)
} }
mergeDeps := artifactOutputDigests([]pipeline.ArtifactCheckpointOutput{stored}) mergeDeps := artifactOutputDigests([]pipeline.CheckpointArtifact{stored})
if err := recorder.ArtifactMergeSucceeded("spells", "merge", mergeDeps, stored, nil); err != nil { if err := recorder.MergeSucceeded("spells", "merge", mergeDeps, stored, nil); err != nil {
t.Fatalf("ArtifactMergeSucceeded: %v", err) t.Fatalf("MergeSucceeded: %v", err)
} }
merged, decision := loader.ArtifactMerge("spells", "merge", mergeDeps) merged, decision := loader.Merge("spells", "merge", mergeDeps)
if !decision.Reused || string(merged.Output.Artifact.Content) != string(artifact.Content) { if !decision.Reused || string(merged.Output.Artifact.Content) != string(artifact.Content) {
t.Fatalf("merge decision=%#v checkpoint=%#v, want reused", decision, merged) t.Fatalf("merge decision=%#v checkpoint=%#v, want reused", decision, merged)
} }
normalizeDeps := artifactOutputDigests([]pipeline.ArtifactCheckpointOutput{stored}) normalizeDeps := artifactOutputDigests([]pipeline.CheckpointArtifact{stored})
if err := recorder.ArtifactNormalizeSucceeded("spells", "normalize", normalizeDeps, stored, nil); err != nil { if err := recorder.NormalizeSucceeded("spells", "normalize", normalizeDeps, stored, nil); err != nil {
t.Fatalf("ArtifactNormalizeSucceeded: %v", err) t.Fatalf("NormalizeSucceeded: %v", err)
} }
normalized, decision := loader.ArtifactNormalize("spells", "normalize", normalizeDeps) normalized, decision := loader.Normalize("spells", "normalize", normalizeDeps)
if !decision.Reused || normalized.Output.SchemaDigest != stored.SchemaDigest { if !decision.Reused || normalized.Output.SchemaDigest != stored.SchemaDigest {
t.Fatalf("normalize decision=%#v checkpoint=%#v, want reused", decision, normalized) t.Fatalf("normalize decision=%#v checkpoint=%#v, want reused", decision, normalized)
} }
@@ -361,23 +263,20 @@ func TestWorkspaceRecorderRecordsFailedStages(t *testing.T) {
func TestWorkspaceRecorderRecordsWarningOnlyValidation(t *testing.T) { func TestWorkspaceRecorderRecordsWarningOnlyValidation(t *testing.T) {
root := t.TempDir() root := t.TempDir()
recorder := newTestRecorder(t, root) recorder := newTestRecorder(t, root)
output := contracts.NormalizeOutput{ schema := contracts.ArtifactSchema{ID: "test.artifact", Name: "test_artifact", Version: "v1", JSONSchema: []byte(`{"type":"object"}`)}
LaneID: "spells", output := pipeline.CheckpointArtifact{
NormalizerKey: "noop", LaneID: "events", ModuleKey: "noop", SourceID: "source-1",
SourceID: "source-1", Artifact: contracts.SerializedArtifact{Kind: "test/artifact", Schema: schema, MediaType: "application/json", Content: []byte(`{"ok":true}`)},
Payload: contracts.RawPayload{ SchemaDigest: contracts.DigestArtifactSchema(schema),
Content: []byte(`{"ok":true}`),
MediaType: "application/json",
},
} }
warnings := []contracts.Warning{{ReasonCode: "note", Message: "warning"}} warnings := []contracts.Warning{{ReasonCode: "note", Message: "warning"}}
if err := recorder.NormalizeSucceeded("spells", "noop", nil, output, warnings); err != nil { if err := recorder.NormalizeSucceeded("events", "noop", nil, output, warnings); err != nil {
t.Fatalf("NormalizeSucceeded: %v", err) t.Fatalf("NormalizeSucceeded: %v", err)
} }
var manifest coreworkspace.NormalizeLaneManifest var manifest coreworkspace.NormalizeLaneManifest
readJSON(t, filepath.Join(root, "normalize", "spells", "manifest.json"), &manifest) readJSON(t, filepath.Join(root, "normalize", "events", "manifest.json"), &manifest)
if manifest.Status != coreworkspace.StatusSucceeded || manifest.ValidationStatus != "approved_with_warnings" { if manifest.Status != coreworkspace.StatusSucceeded || manifest.ValidationStatus != "approved_with_warnings" {
t.Fatalf("normalize manifest status = %q validation=%q", manifest.Status, manifest.ValidationStatus) t.Fatalf("normalize manifest status = %q validation=%q", manifest.Status, manifest.ValidationStatus)
} }

View File

@@ -1,289 +0,0 @@
package contracts_test
import (
"context"
"encoding/json"
"errors"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
var _ contracts.InputAdapter = compositionAdapter{}
var _ contracts.Chunker = compositionChunker{}
var _ contracts.LegacyRawExtractor = compositionExtractor{}
var _ contracts.LegacyRawMerger = compositionMerger{}
var _ contracts.LegacyRawNormalizer = compositionNormalizer{}
var _ contracts.LegacyRawValidator = compositionValidator{}
var _ contracts.OutputEncoder = compositionOutputEncoder{}
func TestContractsComposeAcrossPackages(t *testing.T) {
ctx := context.Background()
adapter := compositionAdapter{}
chunker := compositionChunker{}
extractor := compositionExtractor{}
merger := compositionMerger{}
normalizer := compositionNormalizer{}
encoder := compositionOutputEncoder{}
doc, err := adapter.Parse(ctx, contracts.ParseRequest{SourceID: "source-1"})
if err != nil {
t.Fatalf("Parse() error = %v, want nil", err)
}
if err := source.ValidateDocument(doc); err != nil {
t.Fatalf("ValidateDocument() error = %v, want nil", err)
}
chunking, err := chunker.Chunk(ctx, contracts.ChunkRequest{
Source: doc,
Metadata: map[string]any{"request": "test"},
})
if err != nil {
t.Fatalf("Chunk() error = %v, want nil", err)
}
if len(chunking.Chunks) != 1 {
t.Fatalf("len(Chunks) = %d, want 1", len(chunking.Chunks))
}
extraction, err := extractor.Extract(ctx, contracts.ExtractionRequest{
Source: doc,
Chunk: &chunking.Chunks[0],
AmbientContext: map[string]any{"synopsis": "example synopsis"},
})
if err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
if extraction.Output.Payload.MediaType != "application/json" {
t.Fatalf("extract media type = %q, want application/json", extraction.Output.Payload.MediaType)
}
merge, err := merger.Merge(ctx, contracts.MergeRequest{
Source: doc,
LaneID: "generic-lane",
ExtractOutputs: []contracts.ExtractOutput{extraction.Output},
})
if err != nil {
t.Fatalf("Merge() error = %v, want nil", err)
}
if string(merge.Output.Payload.Content) != `{"value":"example"}` {
t.Fatalf("merge output = %s, want extract payload", merge.Output.Payload.Content)
}
normalize, err := normalizer.Normalize(ctx, contracts.NormalizeRequest{
Source: doc,
LaneID: "generic-lane",
MergeOutput: merge.Output,
})
if err != nil {
t.Fatalf("Normalize() error = %v, want nil", err)
}
if string(normalize.Output.Payload.Content) != `{"value":"example"}` {
t.Fatalf("normalize output = %s, want merge payload", normalize.Output.Payload.Content)
}
output, err := encoder.Encode(ctx, contracts.OutputRequest{
Manifest: artifacts.RunManifest{RunID: "run-1"},
NormalizeOutputs: []contracts.SerializedOutput{{LaneID: normalize.Output.LaneID, NormalizerKey: normalize.Output.NormalizerKey, SourceID: normalize.Output.SourceID, Artifact: contracts.SerializedArtifact{Schema: contracts.ArtifactSchema{ID: normalize.Output.Schema.ID, Name: normalize.Output.Schema.Name, Version: normalize.Output.Schema.Version}, MediaType: normalize.Output.Payload.MediaType, Content: append([]byte(nil), normalize.Output.Payload.Content...)}}},
})
if err != nil {
t.Fatalf("Encode() error = %v, want nil", err)
}
if len(output.Files) != 1 {
t.Fatalf("len(Files) = %d, want 1", len(output.Files))
}
if output.Files[0].ContentType != "application/json" {
t.Fatalf("ContentType = %q, want application/json", output.Files[0].ContentType)
}
if len(output.Files[0].Bytes) == 0 {
t.Fatal("len(Bytes) = 0, want encoded bytes")
}
}
type compositionAdapter struct{}
func (adapter compositionAdapter) Key() string {
return "generic-input"
}
func (adapter compositionAdapter) Parse(ctx context.Context, req contracts.ParseRequest) (*source.SourceDocument, error) {
return &source.SourceDocument{
ID: req.SourceID,
Kind: "document",
Format: "text/plain",
Digest: "sha256:abc123",
Units: []source.SourceUnit{
{ID: 1, Kind: "unit", Text: "First source unit.", Ref: source.SourceRef{SourceID: req.SourceID, StartUnitID: 1, EndUnitID: 1}},
{ID: 2, Kind: "unit", Text: "Second source unit.", Ref: source.SourceRef{SourceID: req.SourceID, StartUnitID: 2, EndUnitID: 2}},
},
}, nil
}
type compositionChunker struct{}
func (chunker compositionChunker) Key() string {
return "generic-chunker"
}
func (chunker compositionChunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (chunker compositionChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
if req.Source == nil {
return contracts.ChunkResult{}, errors.New("source document is required")
}
return contracts.ChunkResult{
Chunks: []source.Chunk{
{
ID: req.Source.ID + ":chunk:0",
SourceID: req.Source.ID,
Index: 0,
Ref: source.SourceRef{
SourceID: req.Source.ID,
StartUnitID: req.Source.Units[0].ID,
EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID,
},
Content: []byte(`{"units":[{"id":1,"kind":"unit","text":"First source unit."},{"id":2,"kind":"unit","text":"Second source unit."}]}`),
MediaType: "application/json",
Units: append([]source.SourceUnit(nil), req.Source.Units...),
Metadata: map[string]any{"strategy": "whole-document"},
},
},
}, nil
}
type compositionExtractor struct{}
func (extractor compositionExtractor) Key() string {
return "generic-extractor"
}
func (extractor compositionExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (extractor compositionExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
if req.Source == nil {
return contracts.ExtractionResult{}, errors.New("source document is required")
}
if req.AmbientContext["synopsis"] == "" {
return contracts.ExtractionResult{}, errors.New("ambient synopsis is required")
}
return contracts.ExtractionResult{
Output: contracts.ExtractOutput{
Schema: contracts.ResponseSchema{ID: "schema-id", Name: "schema-name", Version: "v1"},
Payload: contracts.RawPayload{
Content: []byte(`{"value":"example"}`),
MediaType: "application/json",
},
},
}, nil
}
type compositionMerger struct{}
func (merger compositionMerger) Key() string {
return "generic-merger"
}
func (merger compositionMerger) Merge(ctx context.Context, req contracts.MergeRequest) (contracts.MergeResult, error) {
output := req.ExtractOutputs[0]
return contracts.MergeResult{Output: contracts.MergeOutput{
LaneID: req.LaneID,
MergerKey: merger.Key(),
SourceID: output.SourceID,
Schema: output.Schema,
Payload: cloneCompositionPayload(output.Payload),
}}, nil
}
type compositionNormalizer struct{}
func (normalizer compositionNormalizer) Key() string {
return "generic-normalizer"
}
func (normalizer compositionNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (normalizer compositionNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
return contracts.NormalizeResult{Output: contracts.NormalizeOutput{
LaneID: req.LaneID,
NormalizerKey: normalizer.Key(),
SourceID: req.MergeOutput.SourceID,
Schema: req.MergeOutput.Schema,
Payload: cloneCompositionPayload(req.MergeOutput.Payload),
}}, nil
}
func cloneCompositionPayload(payload contracts.RawPayload) contracts.RawPayload {
return contracts.RawPayload{
Content: append([]byte(nil), payload.Content...),
MediaType: payload.MediaType,
Metadata: cloneCompositionMetadata(payload.Metadata),
Warnings: append([]contracts.Warning(nil), payload.Warnings...),
}
}
func cloneCompositionMetadata(metadata map[string]any) map[string]any {
if len(metadata) == 0 {
return nil
}
out := make(map[string]any, len(metadata))
for key, value := range metadata {
out[key] = value
}
return out
}
type compositionValidator struct{}
func (validator compositionValidator) Name() string {
return "generic-validator"
}
func (validator compositionValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (validator compositionValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
return contracts.ValidationResult{
Approved: true,
ReasonCode: "accepted",
Message: "output accepted",
}, nil
}
type compositionOutputEncoder struct{}
func (encoder compositionOutputEncoder) Key() string {
return "generic-output"
}
func (encoder compositionOutputEncoder) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
payload := struct {
RunID string `json:"run_id"`
OutputCount int `json:"output_count"`
}{
RunID: req.Manifest.RunID,
OutputCount: len(req.NormalizeOutputs),
}
encoded, err := json.Marshal(payload)
if err != nil {
return contracts.OutputResult{}, err
}
return contracts.OutputResult{
Files: []contracts.OutputFile{
{
Name: "artifacts/generic.json",
ContentType: "application/json",
Bytes: encoded,
},
},
}, nil
}

View File

@@ -207,37 +207,6 @@ type ReferenceSet struct {
Slots map[string]ResolvedReferenceSlot `json:"slots,omitempty"` Slots map[string]ResolvedReferenceSlot `json:"slots,omitempty"`
} }
type ExtractionRequest struct {
Source *source.SourceDocument `json:"-"`
Chunk *source.Chunk `json:"chunk,omitempty"`
AmbientContext map[string]any `json:"ambient_context,omitempty"`
SourceInput LLMInputMaterial `json:"source_input,omitempty"`
SessionID string `json:"session_id,omitempty"`
References ReferenceSet `json:"references,omitempty"`
LLMClient StructuredLLMClient `json:"-"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type ExtractionResult struct {
Output ExtractOutput `json:"output"`
Warnings []Warning `json:"warnings,omitempty"`
}
type LegacyRawExtractor interface {
Key() string
ReferenceSlots() []ReferenceSlot
Extract(ctx context.Context, req ExtractionRequest) (ExtractionResult, error)
}
type RawPayload struct {
Content []byte `json:"-"`
MediaType string `json:"media_type"`
Metadata map[string]any `json:"metadata,omitempty"`
Warnings []Warning `json:"warnings,omitempty"`
}
type ExecutionClass string type ExecutionClass string
const ( const (
@@ -245,29 +214,6 @@ const (
ExecutionClassLLMBacked ExecutionClass = "llm_backed" ExecutionClassLLMBacked ExecutionClass = "llm_backed"
) )
type ValidationRequest struct {
Stage string `json:"stage"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key"`
Source *source.SourceDocument `json:"-"`
SourceID string `json:"source_id,omitempty"`
SourceInput LLMInputMaterial `json:"source_input,omitempty"`
SessionID string `json:"session_id,omitempty"`
References ReferenceSet `json:"references,omitempty"`
LLMClient StructuredLLMClient `json:"-"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
Schema ResponseSchema `json:"schema,omitempty"`
Payload RawPayload `json:"payload"`
ChunkID string `json:"chunk_id,omitempty"`
ChunkIndex int `json:"chunk_index,omitempty"`
Chunk *source.Chunk `json:"chunk,omitempty"`
Chunks []source.Chunk `json:"chunks,omitempty"`
ExtractOutputs []ExtractOutput `json:"extract_outputs,omitempty"`
MergeOutput MergeOutput `json:"merge_output,omitempty"`
}
type ValidationResult struct { type ValidationResult struct {
Approved bool `json:"approved"` Approved bool `json:"approved"`
ReasonCode string `json:"reason_code,omitempty"` ReasonCode string `json:"reason_code,omitempty"`
@@ -276,92 +222,6 @@ type ValidationResult struct {
Warnings []Warning `json:"warnings,omitempty"` Warnings []Warning `json:"warnings,omitempty"`
} }
type LegacyRawValidator interface {
Name() string
ExecutionClass() ExecutionClass
Validate(ctx context.Context, req ValidationRequest) (ValidationResult, error)
}
type ResponseSchema struct {
ID string `json:"id,omitempty"`
Name string `json:"name,omitempty"`
Version string `json:"version,omitempty"`
JSONSchema []byte `json:"-"`
}
type ExtractOutput struct {
LaneID string `json:"lane_id"`
ExtractorKey string `json:"extractor_key"`
SourceID string `json:"source_id"`
ChunkID string `json:"chunk_id"`
ChunkIndex int `json:"chunk_index"`
Schema ResponseSchema `json:"schema,omitempty"`
Payload RawPayload `json:"payload"`
}
type MergeRequest struct {
Source *source.SourceDocument `json:"-"`
LaneID string `json:"lane_id"`
ExtractOutputs []ExtractOutput `json:"extract_outputs"`
SourceInput LLMInputMaterial `json:"source_input,omitempty"`
SessionID string `json:"session_id,omitempty"`
References ReferenceSet `json:"references,omitempty"`
LLMClient StructuredLLMClient `json:"-"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type MergeResult struct {
Output MergeOutput `json:"output"`
Warnings []Warning `json:"warnings,omitempty"`
}
type MergeOutput struct {
LaneID string `json:"lane_id"`
MergerKey string `json:"merger_key"`
SourceID string `json:"source_id,omitempty"`
Schema ResponseSchema `json:"schema,omitempty"`
Payload RawPayload `json:"payload"`
}
type LegacyRawMerger interface {
Key() string
Merge(ctx context.Context, req MergeRequest) (MergeResult, error)
}
type NormalizeRequest struct {
Source *source.SourceDocument `json:"-"`
LaneID string `json:"lane_id"`
MergeOutput MergeOutput `json:"merge_output"`
SourceInput LLMInputMaterial `json:"source_input,omitempty"`
SessionID string `json:"session_id,omitempty"`
References ReferenceSet `json:"references,omitempty"`
LLMClient StructuredLLMClient `json:"-"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type NormalizeResult struct {
Output NormalizeOutput `json:"output"`
Warnings []Warning `json:"warnings,omitempty"`
}
type NormalizeOutput struct {
LaneID string `json:"lane_id"`
NormalizerKey string `json:"normalizer_key"`
SourceID string `json:"source_id,omitempty"`
Schema ResponseSchema `json:"schema,omitempty"`
Payload RawPayload `json:"payload"`
}
type LegacyRawNormalizer interface {
Key() string
ReferenceSlots() []ReferenceSlot
Normalize(ctx context.Context, req NormalizeRequest) (NormalizeResult, error)
}
type Warning struct { type Warning struct {
Scope string `json:"scope,omitempty"` Scope string `json:"scope,omitempty"`
ReasonCode string `json:"reason_code"` ReasonCode string `json:"reason_code"`

View File

@@ -12,14 +12,13 @@ import (
var _ InputAdapter = fakeAdapter{} var _ InputAdapter = fakeAdapter{}
var _ Chunker = fakeChunker{} var _ Chunker = fakeChunker{}
var _ LegacyRawExtractor = fakeExtractor{} var _ Extractor[fakeArtifact] = fakeExtractor{}
var _ LegacyRawMerger = fakeMerger{} var _ Merger[fakeArtifact] = fakeMerger{}
var _ LegacyRawNormalizer = fakeNormalizer{} var _ Normalizer[fakeArtifact] = fakeNormalizer{}
var _ LegacyRawValidator = fakeValidator{}
var _ StructuredLLMClient = fakeLLMClient{} var _ StructuredLLMClient = fakeLLMClient{}
var _ OutputEncoder = fakeOutputEncoder{} var _ OutputEncoder = fakeOutputEncoder{}
func TestFakeExtractorReturnsRawOutput(t *testing.T) { func TestFakeExtractorReturnsTypedOutput(t *testing.T) {
extractor := fakeExtractor{ extractor := fakeExtractor{
key: "generic-extractor", key: "generic-extractor",
} }
@@ -33,7 +32,7 @@ func TestFakeExtractorReturnsRawOutput(t *testing.T) {
}, },
} }
result, err := extractor.Extract(context.Background(), ExtractionRequest{Source: doc}) result, err := extractor.Extract(context.Background(), TypedExtractionRequest{Source: doc})
if err != nil { if err != nil {
t.Fatalf("Extract() error = %v, want nil", err) t.Fatalf("Extract() error = %v, want nil", err)
} }
@@ -41,14 +40,8 @@ func TestFakeExtractorReturnsRawOutput(t *testing.T) {
if extractor.Key() != "generic-extractor" { if extractor.Key() != "generic-extractor" {
t.Fatalf("Key() = %q, want generic-extractor", extractor.Key()) t.Fatalf("Key() = %q, want generic-extractor", extractor.Key())
} }
if result.Output.ExtractorKey != "" { if result.Value.Value != "example" {
t.Fatalf("ExtractorKey = %q, want runner-owned empty value", result.Output.ExtractorKey) t.Fatalf("Value = %q, want example", result.Value.Value)
}
if result.Output.Schema.Version != "v1" {
t.Fatalf("Schema.Version = %q, want v1", result.Output.Schema.Version)
}
if result.Output.Payload.MediaType != "application/json" || string(result.Output.Payload.Content) != `{"value":"example"}` {
t.Fatalf("payload = %q %s, want JSON raw output", result.Output.Payload.MediaType, result.Output.Payload.Content)
} }
} }
@@ -139,7 +132,7 @@ func TestFakeExtractorReceivesChunkAndAmbientContext(t *testing.T) {
Units: []source.SourceUnit{doc.Units[1]}, Units: []source.SourceUnit{doc.Units[1]},
} }
result, err := extractor.Extract(context.Background(), ExtractionRequest{ result, err := extractor.Extract(context.Background(), TypedExtractionRequest{
Source: doc, Source: doc,
Chunk: &chunk, Chunk: &chunk,
AmbientContext: map[string]any{"mode": "chunked"}, AmbientContext: map[string]any{"mode": "chunked"},
@@ -147,11 +140,8 @@ func TestFakeExtractorReceivesChunkAndAmbientContext(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("Extract() error = %v, want nil", err) t.Fatalf("Extract() error = %v, want nil", err)
} }
if result.Output.ChunkID != "" || result.Output.ChunkIndex != 0 { if result.Value.Value != "chunked" {
t.Fatalf("chunk provenance = %q/%d, want runner-owned zero values", result.Output.ChunkID, result.Output.ChunkIndex) t.Fatalf("Value = %q, want chunked", result.Value.Value)
}
if string(result.Output.Payload.Content) != `{"value":"chunked"}` {
t.Fatalf("Payload.Content = %s, want chunked payload", result.Output.Payload.Content)
} }
} }
@@ -320,8 +310,8 @@ func TestLLMInputSetCloneCopiesContent(t *testing.T) {
} }
} }
func TestResponseSchemaJSONOmitRawSchemaContent(t *testing.T) { func TestArtifactSchemaJSONOmitsSchemaContent(t *testing.T) {
schema := ResponseSchema{ schema := ArtifactSchema{
ID: "schema-id", ID: "schema-id",
Name: "schema-name", Name: "schema-name",
Version: "v1", Version: "v1",
@@ -347,26 +337,14 @@ func TestResponseSchemaJSONOmitRawSchemaContent(t *testing.T) {
} }
func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) { func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
extractOutput := ExtractOutput{ extractOutput := ExtractArtifact[fakeArtifact]{LaneID: "generic-lane", ExtractorKey: "generic-extractor", SourceID: "source-1", ChunkID: "source-1:chunk:0", ChunkIndex: 0, Value: fakeArtifact{Value: "example"}}
LaneID: "generic-lane",
ExtractorKey: "generic-extractor",
SourceID: "source-1",
ChunkID: "source-1:chunk:0",
ChunkIndex: 0,
Schema: ResponseSchema{ID: "schema-id", Name: "schema-name", Version: "v1"},
Payload: RawPayload{
Content: []byte(`{"value":"example"}`),
MediaType: "application/json",
Metadata: map[string]any{"confidence": 0.75},
},
}
merger := fakeMerger{key: "generic-merger"} merger := fakeMerger{key: "generic-merger"}
normalizer := fakeNormalizer{key: "generic-normalizer"} normalizer := fakeNormalizer{key: "generic-normalizer"}
encoder := fakeOutputEncoder{key: "generic-output"} encoder := fakeOutputEncoder{key: "generic-output"}
merged, err := merger.Merge(context.Background(), MergeRequest{ merged, err := merger.Merge(context.Background(), TypedMergeRequest[fakeArtifact]{
LaneID: "generic-lane", LaneID: "generic-lane",
ExtractOutputs: []ExtractOutput{extractOutput}, ExtractOutputs: []ExtractArtifact[fakeArtifact]{extractOutput},
}) })
if err != nil { if err != nil {
t.Fatalf("Merge() error = %v, want nil", err) t.Fatalf("Merge() error = %v, want nil", err)
@@ -374,13 +352,13 @@ func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
if merger.Key() != "generic-merger" { if merger.Key() != "generic-merger" {
t.Fatalf("Merger.Key() = %q, want generic-merger", merger.Key()) t.Fatalf("Merger.Key() = %q, want generic-merger", merger.Key())
} }
if string(merged.Output.Payload.Content) != `{"value":"example"}` { if merged.Value.Value != "example" {
t.Fatalf("merged content = %s, want raw extract content", merged.Output.Payload.Content) t.Fatalf("merged value = %q, want example", merged.Value.Value)
} }
normalized, err := normalizer.Normalize(context.Background(), NormalizeRequest{ normalized, err := normalizer.Normalize(context.Background(), TypedNormalizeRequest[fakeArtifact]{
LaneID: "generic-lane", LaneID: "generic-lane",
MergeOutput: merged.Output, MergeOutput: MergeArtifact[fakeArtifact]{LaneID: "generic-lane", MergerKey: merger.Key(), SourceID: "source-1", Value: merged.Value},
}) })
if err != nil { if err != nil {
t.Fatalf("Normalize() error = %v, want nil", err) t.Fatalf("Normalize() error = %v, want nil", err)
@@ -388,13 +366,13 @@ func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
if normalizer.Key() != "generic-normalizer" { if normalizer.Key() != "generic-normalizer" {
t.Fatalf("Normalizer.Key() = %q, want generic-normalizer", normalizer.Key()) t.Fatalf("Normalizer.Key() = %q, want generic-normalizer", normalizer.Key())
} }
if string(normalized.Output.Payload.Content) != `{"value":"example"}` { if normalized.Value.Value != "example" {
t.Fatalf("normalized content = %s, want raw merge content", normalized.Output.Payload.Content) t.Fatalf("normalized value = %q, want example", normalized.Value.Value)
} }
encoded, err := encoder.Encode(context.Background(), OutputRequest{ encoded, err := encoder.Encode(context.Background(), OutputRequest{
Manifest: artifacts.RunManifest{RunID: "run-1"}, Manifest: artifacts.RunManifest{RunID: "run-1"},
NormalizeOutputs: []SerializedOutput{serializedTestOutput(normalized.Output)}, NormalizeOutputs: []SerializedOutput{{LaneID: "generic-lane", NormalizerKey: normalizer.Key(), SourceID: "source-1", Artifact: SerializedArtifact{Kind: "test/artifact", Schema: ArtifactSchema{ID: "schema-id", Name: "schema-name", Version: "v1"}, MediaType: "application/json", Content: []byte(`{"value":"example"}`)}}},
}) })
if err != nil { if err != nil {
t.Fatalf("Encode() error = %v, want nil", err) t.Fatalf("Encode() error = %v, want nil", err)
@@ -413,10 +391,6 @@ func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
} }
} }
func serializedTestOutput(output NormalizeOutput) SerializedOutput {
return SerializedOutput{LaneID: output.LaneID, NormalizerKey: output.NormalizerKey, SourceID: output.SourceID, Artifact: SerializedArtifact{Schema: ArtifactSchema{ID: output.Schema.ID, Name: output.Schema.Name, Version: output.Schema.Version, JSONSchema: append([]byte(nil), output.Schema.JSONSchema...)}, MediaType: output.Payload.MediaType, Content: append([]byte(nil), output.Payload.Content...), Metadata: cloneArtifactMetadata(output.Payload.Metadata)}}
}
func TestOutputFileJSONShapeOmitsBytes(t *testing.T) { func TestOutputFileJSONShapeOmitsBytes(t *testing.T) {
file := OutputFile{ file := OutputFile{
Name: "artifacts/events.json", Name: "artifacts/events.json",
@@ -514,6 +488,8 @@ type fakeExtractor struct {
key string key string
} }
type fakeArtifact struct{ Value string }
func (extractor fakeExtractor) Key() string { func (extractor fakeExtractor) Key() string {
return extractor.key return extractor.key
} }
@@ -522,21 +498,12 @@ func (extractor fakeExtractor) ReferenceSlots() []ReferenceSlot {
return nil return nil
} }
func (extractor fakeExtractor) Extract(ctx context.Context, req ExtractionRequest) (ExtractionResult, error) { func (extractor fakeExtractor) Extract(ctx context.Context, req TypedExtractionRequest) (TypedExtractionResult[fakeArtifact], error) {
payload := json.RawMessage(`{"value":"example"}`) value := "example"
if req.AmbientContext["mode"] == "chunked" { if req.AmbientContext["mode"] == "chunked" {
payload = json.RawMessage(`{"value":"chunked"}`) value = "chunked"
} }
return TypedExtractionResult[fakeArtifact]{Value: fakeArtifact{Value: value}}, nil
return ExtractionResult{
Output: ExtractOutput{
Schema: ResponseSchema{ID: "schema-id", Name: "schema-name", Version: "v1"},
Payload: RawPayload{
Content: append([]byte(nil), payload...),
MediaType: "application/json",
},
},
}, nil
} }
type fakeMerger struct { type fakeMerger struct {
@@ -547,15 +514,8 @@ func (merger fakeMerger) Key() string {
return merger.key return merger.key
} }
func (merger fakeMerger) Merge(ctx context.Context, req MergeRequest) (MergeResult, error) { func (merger fakeMerger) Merge(ctx context.Context, req TypedMergeRequest[fakeArtifact]) (TypedMergeResult[fakeArtifact], error) {
output := req.ExtractOutputs[0] return TypedMergeResult[fakeArtifact]{Value: req.ExtractOutputs[0].Value}, nil
return MergeResult{Output: MergeOutput{
LaneID: req.LaneID,
MergerKey: merger.key,
SourceID: output.SourceID,
Schema: output.Schema,
Payload: cloneTestRawPayload(output.Payload),
}}, nil
} }
type fakeNormalizer struct { type fakeNormalizer struct {
@@ -570,54 +530,8 @@ func (normalizer fakeNormalizer) ReferenceSlots() []ReferenceSlot {
return nil return nil
} }
func (normalizer fakeNormalizer) Normalize(ctx context.Context, req NormalizeRequest) (NormalizeResult, error) { func (normalizer fakeNormalizer) Normalize(ctx context.Context, req TypedNormalizeRequest[fakeArtifact]) (TypedNormalizeResult[fakeArtifact], error) {
return NormalizeResult{Output: NormalizeOutput{ return TypedNormalizeResult[fakeArtifact]{Value: req.MergeOutput.Value}, nil
LaneID: req.LaneID,
NormalizerKey: normalizer.key,
SourceID: req.MergeOutput.SourceID,
Schema: req.MergeOutput.Schema,
Payload: cloneTestRawPayload(req.MergeOutput.Payload),
}}, nil
}
func cloneTestRawPayload(payload RawPayload) RawPayload {
return RawPayload{
Content: append([]byte(nil), payload.Content...),
MediaType: payload.MediaType,
Metadata: cloneTestMetadata(payload.Metadata),
Warnings: append([]Warning(nil), payload.Warnings...),
}
}
func cloneTestMetadata(metadata map[string]any) map[string]any {
if len(metadata) == 0 {
return nil
}
out := make(map[string]any, len(metadata))
for key, value := range metadata {
out[key] = value
}
return out
}
type fakeValidator struct {
name string
}
func (validator fakeValidator) Name() string {
return validator.name
}
func (validator fakeValidator) ExecutionClass() ExecutionClass {
return ExecutionClassDeterministic
}
func (validator fakeValidator) Validate(ctx context.Context, req ValidationRequest) (ValidationResult, error) {
return ValidationResult{
Approved: true,
ReasonCode: "accepted",
Message: "output accepted",
}, nil
} }
type fakeLLMClient struct{} type fakeLLMClient struct{}

View File

@@ -25,14 +25,14 @@ type CheckpointRecorder interface {
ChunkRejected(moduleKey string, sourceDigest string, rejected contracts.RejectedOutput) error ChunkRejected(moduleKey string, sourceDigest string, rejected contracts.RejectedOutput) error
ChunkFailed(moduleKey string, sourceDigest string, err error) error ChunkFailed(moduleKey string, sourceDigest string, err error) error
ExtractRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error ExtractRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
ExtractSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, outputs []contracts.ExtractOutput, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error ExtractSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error
ExtractFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error ExtractFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
MergeRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error MergeRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
MergeSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, output contracts.MergeOutput, warnings []contracts.Warning) error MergeSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, output CheckpointArtifact, warnings []contracts.Warning) error
MergeRejected(laneID string, moduleKey string, dependencies []CheckpointFingerprint, rejected contracts.RejectedOutput) error MergeRejected(laneID string, moduleKey string, dependencies []CheckpointFingerprint, rejected contracts.RejectedOutput) error
MergeFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error MergeFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
NormalizeRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error NormalizeRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
NormalizeSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, output contracts.NormalizeOutput, warnings []contracts.Warning) error NormalizeSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, output CheckpointArtifact, warnings []contracts.Warning) error
NormalizeRejected(laneID string, moduleKey string, dependencies []CheckpointFingerprint, rejected contracts.RejectedOutput) error NormalizeRejected(laneID string, moduleKey string, dependencies []CheckpointFingerprint, rejected contracts.RejectedOutput) error
NormalizeFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error NormalizeFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
} }
@@ -59,25 +59,9 @@ type ChunkCheckpoint struct {
Warnings []contracts.Warning Warnings []contracts.Warning
} }
type ExtractCheckpoint struct { // CheckpointArtifact is the durable, domain-neutral value stored at a lane
Outputs []contracts.ExtractOutput // checkpoint boundary.
Rejected []contracts.RejectedOutput type CheckpointArtifact struct {
Warnings []contracts.Warning
}
type MergeCheckpoint struct {
Output contracts.MergeOutput
Warnings []contracts.Warning
}
type NormalizeCheckpoint struct {
Output contracts.NormalizeOutput
Warnings []contracts.Warning
}
// ArtifactCheckpointOutput is the durable, domain-neutral value stored at a
// typed lane checkpoint boundary.
type ArtifactCheckpointOutput struct {
LaneID string LaneID string
ModuleKey string ModuleKey string
SourceID string SourceID string
@@ -88,33 +72,21 @@ type ArtifactCheckpointOutput struct {
SchemaDigest string SchemaDigest string
} }
type ArtifactExtractCheckpoint struct { type ExtractCheckpoint struct {
Outputs []ArtifactCheckpointOutput Outputs []CheckpointArtifact
Rejected []contracts.RejectedOutput Rejected []contracts.RejectedOutput
Warnings []contracts.Warning Warnings []contracts.Warning
} }
type ArtifactMergeCheckpoint struct { type MergeCheckpoint struct {
Output ArtifactCheckpointOutput Output CheckpointArtifact
Warnings []contracts.Warning Warnings []contracts.Warning
} }
type ArtifactNormalizeCheckpoint struct { type NormalizeCheckpoint struct {
Output ArtifactCheckpointOutput Output CheckpointArtifact
Warnings []contracts.Warning Warnings []contracts.Warning
} }
type ArtifactCheckpointRecorder interface {
ArtifactExtractSucceeded(string, string, []CheckpointFingerprint, []ArtifactCheckpointOutput, []contracts.RejectedOutput, []contracts.Warning) error
ArtifactMergeSucceeded(string, string, []CheckpointFingerprint, ArtifactCheckpointOutput, []contracts.Warning) error
ArtifactNormalizeSucceeded(string, string, []CheckpointFingerprint, ArtifactCheckpointOutput, []contracts.Warning) error
}
type ArtifactCheckpointLoader interface {
ArtifactExtract(string, string, []CheckpointFingerprint) (ArtifactExtractCheckpoint, CheckpointDecision)
ArtifactMerge(string, string, []CheckpointFingerprint) (ArtifactMergeCheckpoint, CheckpointDecision)
ArtifactNormalize(string, string, []CheckpointFingerprint) (ArtifactNormalizeCheckpoint, CheckpointDecision)
}
type CheckpointLoader interface { type CheckpointLoader interface {
Enabled() bool Enabled() bool
Source(moduleKey string) (SourceCheckpoint, CheckpointDecision) Source(moduleKey string) (SourceCheckpoint, CheckpointDecision)
@@ -144,14 +116,14 @@ func (noopCheckpointRecorder) ChunkFailed(string, string, error) error { return
func (noopCheckpointRecorder) ExtractRunning(string, string, []CheckpointFingerprint) error { func (noopCheckpointRecorder) ExtractRunning(string, string, []CheckpointFingerprint) error {
return nil return nil
} }
func (noopCheckpointRecorder) ExtractSucceeded(string, string, []CheckpointFingerprint, []contracts.ExtractOutput, []contracts.RejectedOutput, []contracts.Warning) error { func (noopCheckpointRecorder) ExtractSucceeded(string, string, []CheckpointFingerprint, []CheckpointArtifact, []contracts.RejectedOutput, []contracts.Warning) error {
return nil return nil
} }
func (noopCheckpointRecorder) ExtractFailed(string, string, []CheckpointFingerprint, error) error { func (noopCheckpointRecorder) ExtractFailed(string, string, []CheckpointFingerprint, error) error {
return nil return nil
} }
func (noopCheckpointRecorder) MergeRunning(string, string, []CheckpointFingerprint) error { return nil } func (noopCheckpointRecorder) MergeRunning(string, string, []CheckpointFingerprint) error { return nil }
func (noopCheckpointRecorder) MergeSucceeded(string, string, []CheckpointFingerprint, contracts.MergeOutput, []contracts.Warning) error { func (noopCheckpointRecorder) MergeSucceeded(string, string, []CheckpointFingerprint, CheckpointArtifact, []contracts.Warning) error {
return nil return nil
} }
func (noopCheckpointRecorder) MergeRejected(string, string, []CheckpointFingerprint, contracts.RejectedOutput) error { func (noopCheckpointRecorder) MergeRejected(string, string, []CheckpointFingerprint, contracts.RejectedOutput) error {
@@ -163,7 +135,7 @@ func (noopCheckpointRecorder) MergeFailed(string, string, []CheckpointFingerprin
func (noopCheckpointRecorder) NormalizeRunning(string, string, []CheckpointFingerprint) error { func (noopCheckpointRecorder) NormalizeRunning(string, string, []CheckpointFingerprint) error {
return nil return nil
} }
func (noopCheckpointRecorder) NormalizeSucceeded(string, string, []CheckpointFingerprint, contracts.NormalizeOutput, []contracts.Warning) error { func (noopCheckpointRecorder) NormalizeSucceeded(string, string, []CheckpointFingerprint, CheckpointArtifact, []contracts.Warning) error {
return nil return nil
} }
func (noopCheckpointRecorder) NormalizeRejected(string, string, []CheckpointFingerprint, contracts.RejectedOutput) error { func (noopCheckpointRecorder) NormalizeRejected(string, string, []CheckpointFingerprint, contracts.RejectedOutput) error {
@@ -190,28 +162,6 @@ func (noopCheckpointLoader) Normalize(string, string, []CheckpointFingerprint) (
return NormalizeCheckpoint{}, CheckpointDecision{Reason: "checkpoint loading disabled"} return NormalizeCheckpoint{}, CheckpointDecision{Reason: "checkpoint loading disabled"}
} }
func rawOutputDigests(payloads []contracts.RawPayload) []CheckpointFingerprint {
values := make([]CheckpointFingerprint, 0, len(payloads))
for i, payload := range payloads {
values = append(values, CheckpointFingerprint{
Name: fmt.Sprintf("payload[%d]", i),
Value: checkpointContentDigest(payload.Content),
})
}
return normalizeCheckpointFingerprints(values)
}
func extractPayloads(outputs []contracts.ExtractOutput) []contracts.RawPayload {
if len(outputs) == 0 {
return nil
}
payloads := make([]contracts.RawPayload, 0, len(outputs))
for _, output := range outputs {
payloads = append(payloads, output.Payload)
}
return payloads
}
func digestFingerprints(name string, digest string) []CheckpointFingerprint { func digestFingerprints(name string, digest string) []CheckpointFingerprint {
digest = strings.TrimSpace(digest) digest = strings.TrimSpace(digest)
if digest == "" { if digest == "" {

View File

@@ -32,7 +32,7 @@ func (r *ChunkerRegistry) RegisterWithSpec(spec ModuleSpec, constructor ChunkerC
if constructor == nil { if constructor == nil {
return fmt.Errorf("chunker constructor for %q must not be nil", strings.TrimSpace(spec.Key)) return fmt.Errorf("chunker constructor for %q must not be nil", strings.TrimSpace(spec.Key))
} }
return r.RegisterBuilderWithSpec(spec, allowLegacyOptions, func(BuildRequest) (contracts.Chunker, error) { return r.RegisterBuilderWithSpec(spec, rejectUnconfiguredOptions, func(BuildRequest) (contracts.Chunker, error) {
return constructor() return constructor()
}) })
} }

View File

@@ -338,34 +338,6 @@ func (chunker registryChunker) Chunk(ctx context.Context, req contracts.ChunkReq
return contracts.ChunkResult{}, nil return contracts.ChunkResult{}, nil
} }
type registryMerger struct {
key string
}
func (merger registryMerger) Key() string {
return merger.key
}
func (merger registryMerger) Merge(ctx context.Context, req contracts.MergeRequest) (contracts.MergeResult, error) {
return contracts.MergeResult{}, nil
}
type registryNormalizer struct {
key string
}
func (normalizer registryNormalizer) Key() string {
return normalizer.key
}
func (normalizer registryNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (normalizer registryNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
return contracts.NormalizeResult{}, nil
}
type registryOutputEncoder struct { type registryOutputEncoder struct {
key string key string
} }
@@ -377,19 +349,3 @@ func (encoder registryOutputEncoder) Key() string {
func (encoder registryOutputEncoder) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) { func (encoder registryOutputEncoder) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
return contracts.OutputResult{}, nil return contracts.OutputResult{}, nil
} }
type registryValidator struct {
name string
}
func (validator registryValidator) Name() string {
return validator.name
}
func (validator registryValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (validator registryValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: true}, nil
}

View File

@@ -24,7 +24,9 @@ type BuildRequest struct {
// OptionValidator validates one module binding without constructing it. // OptionValidator validates one module binding without constructing it.
type OptionValidator func(map[string]any) error type OptionValidator func(map[string]any) error
func allowLegacyOptions(map[string]any) error { return nil } func rejectUnconfiguredOptions(options map[string]any) error {
return RejectUnknownOptions(options)
}
func validateRegisteredOptions(validator OptionValidator, options map[string]any) error { func validateRegisteredOptions(validator OptionValidator, options map[string]any) error {
if validator == nil { if validator == nil {

View File

@@ -82,10 +82,6 @@ type debugBinaryEnvelope struct {
Warnings []contracts.Warning `json:"warnings,omitempty"` Warnings []contracts.Warning `json:"warnings,omitempty"`
} }
type debugRawPayload struct {
Content debugBinaryEnvelope `json:"content"`
}
type debugSourceInput struct { type debugSourceInput struct {
SourceID string `json:"source_id,omitempty"` SourceID string `json:"source_id,omitempty"`
Path string `json:"path,omitempty"` Path string `json:"path,omitempty"`
@@ -113,32 +109,6 @@ type debugSourceChunk struct {
Metadata map[string]any `json:"metadata,omitempty"` Metadata map[string]any `json:"metadata,omitempty"`
} }
type debugExtractOutput struct {
LaneID string `json:"lane_id"`
ExtractorKey string `json:"extractor_key"`
SourceID string `json:"source_id"`
ChunkID string `json:"chunk_id"`
ChunkIndex int `json:"chunk_index"`
Schema contracts.ResponseSchema `json:"schema,omitempty"`
Payload debugBinaryEnvelope `json:"payload"`
}
type debugMergeOutput struct {
LaneID string `json:"lane_id"`
MergerKey string `json:"merger_key"`
SourceID string `json:"source_id,omitempty"`
Schema contracts.ResponseSchema `json:"schema,omitempty"`
Payload debugBinaryEnvelope `json:"payload"`
}
type debugNormalizeOutput struct {
LaneID string `json:"lane_id"`
NormalizerKey string `json:"normalizer_key"`
SourceID string `json:"source_id,omitempty"`
Schema contracts.ResponseSchema `json:"schema,omitempty"`
Payload debugBinaryEnvelope `json:"payload"`
}
type debugSerializedOutput struct { type debugSerializedOutput struct {
LaneID string `json:"lane_id"` LaneID string `json:"lane_id"`
NormalizerKey string `json:"normalizer_key"` NormalizerKey string `json:"normalizer_key"`
@@ -202,25 +172,6 @@ type debugLLMCallReference struct {
Error bool `json:"error,omitempty"` Error bool `json:"error,omitempty"`
} }
type debugValidationRequest struct {
Stage string `json:"stage"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key"`
SourceID string `json:"source_id,omitempty"`
SessionID string `json:"session_id,omitempty"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
Schema contracts.ResponseSchema `json:"schema,omitempty"`
Payload *debugBinaryEnvelope `json:"payload,omitempty"`
ChunkID string `json:"chunk_id,omitempty"`
ChunkIndex int `json:"chunk_index,omitempty"`
Chunk *debugSourceChunk `json:"chunk,omitempty"`
Chunks []debugSourceChunk `json:"chunks,omitempty"`
ExtractOutputs []debugExtractOutput `json:"extract_outputs,omitempty"`
MergeOutput *debugMergeOutput `json:"merge_output,omitempty"`
}
type debugValidationCall struct { type debugValidationCall struct {
ValidatorName string `json:"validator_name"` ValidatorName string `json:"validator_name"`
Request any `json:"request"` Request any `json:"request"`
@@ -448,10 +399,6 @@ func debugContentEnvelope(content []byte, mediaType string, metadata map[string]
} }
} }
func debugPayloadEnvelope(payload contracts.RawPayload) debugBinaryEnvelope {
return debugContentEnvelope(payload.Content, payload.MediaType, payload.Metadata, payload.Warnings)
}
func debugSourceDocumentEnvelope(doc *source.SourceDocument) *debugSourceDocument { func debugSourceDocumentEnvelope(doc *source.SourceDocument) *debugSourceDocument {
if doc == nil { if doc == nil {
return nil return nil
@@ -489,63 +436,6 @@ func debugSourceChunkEnvelopes(chunks []source.Chunk) []debugSourceChunk {
return out return out
} }
func debugExtractOutputEnvelope(output contracts.ExtractOutput) debugExtractOutput {
output.Schema.JSONSchema = nil
return debugExtractOutput{
LaneID: output.LaneID,
ExtractorKey: output.ExtractorKey,
SourceID: output.SourceID,
ChunkID: output.ChunkID,
ChunkIndex: output.ChunkIndex,
Schema: output.Schema,
Payload: debugPayloadEnvelope(output.Payload),
}
}
func debugExtractOutputEnvelopes(outputs []contracts.ExtractOutput) []debugExtractOutput {
if len(outputs) == 0 {
return nil
}
out := make([]debugExtractOutput, 0, len(outputs))
for _, output := range outputs {
out = append(out, debugExtractOutputEnvelope(output))
}
return out
}
func debugMergeOutputEnvelope(output contracts.MergeOutput) debugMergeOutput {
output.Schema.JSONSchema = nil
return debugMergeOutput{
LaneID: output.LaneID,
MergerKey: output.MergerKey,
SourceID: output.SourceID,
Schema: output.Schema,
Payload: debugPayloadEnvelope(output.Payload),
}
}
func debugNormalizeOutputEnvelope(output contracts.NormalizeOutput) debugNormalizeOutput {
output.Schema.JSONSchema = nil
return debugNormalizeOutput{
LaneID: output.LaneID,
NormalizerKey: output.NormalizerKey,
SourceID: output.SourceID,
Schema: output.Schema,
Payload: debugPayloadEnvelope(output.Payload),
}
}
func debugNormalizeOutputEnvelopes(outputs []contracts.NormalizeOutput) []debugNormalizeOutput {
if len(outputs) == 0 {
return nil
}
out := make([]debugNormalizeOutput, 0, len(outputs))
for _, output := range outputs {
out = append(out, debugNormalizeOutputEnvelope(output))
}
return out
}
func debugSerializedOutputEnvelope(output contracts.SerializedOutput) debugSerializedOutput { func debugSerializedOutputEnvelope(output contracts.SerializedOutput) debugSerializedOutput {
schema := contracts.CloneArtifactSchema(output.Artifact.Schema) schema := contracts.CloneArtifactSchema(output.Artifact.Schema)
digest := contracts.DigestArtifactSchema(schema) digest := contracts.DigestArtifactSchema(schema)
@@ -714,36 +604,6 @@ func debugResponseModel(response contracts.StructuredCompletionResponse) string
return response.Debug.Response.ModelName return response.Debug.Response.ModelName
} }
func debugValidationRequestEnvelope(req contracts.ValidationRequest) debugValidationRequest {
req.Schema.JSONSchema = nil
out := debugValidationRequest{
Stage: req.Stage,
LaneID: req.LaneID,
ModuleKey: req.ModuleKey,
SourceID: req.SourceID,
SessionID: req.SessionID,
LLMProfile: req.LLMProfile,
Options: redactSensitiveMap(req.Options),
Metadata: redactSensitiveMap(req.Metadata),
Schema: req.Schema,
ChunkID: req.ChunkID,
ChunkIndex: req.ChunkIndex,
}
payload := debugPayloadEnvelope(req.Payload)
out.Payload = &payload
if req.Chunk != nil {
chunk := debugSourceChunkEnvelope(*req.Chunk)
out.Chunk = &chunk
}
out.Chunks = debugSourceChunkEnvelopes(req.Chunks)
out.ExtractOutputs = debugExtractOutputEnvelopes(req.ExtractOutputs)
if len(req.MergeOutput.Payload.Content) > 0 || req.MergeOutput.LaneID != "" {
merge := debugMergeOutputEnvelope(req.MergeOutput)
out.MergeOutput = &merge
}
return out
}
func debugValidationResultEnvelope(result contracts.ValidationResult) contracts.ValidationResult { func debugValidationResultEnvelope(result contracts.ValidationResult) contracts.ValidationResult {
result.Message = string(redactSecretBytes([]byte(result.Message))) result.Message = string(redactSecretBytes([]byte(result.Message)))
result.DiagnosticArtifactPath = string(redactSecretBytes([]byte(result.DiagnosticArtifactPath))) result.DiagnosticArtifactPath = string(redactSecretBytes([]byte(result.DiagnosticArtifactPath)))

View File

@@ -2,6 +2,7 @@ package pipeline_test
import ( import (
"context" "context"
"encoding/json"
"testing" "testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/config" "gitea.maximumdirect.net/eric/notarius/internal/core/config"
@@ -71,30 +72,40 @@ func defaultModuleCatalog(t *testing.T) pipeline.ModuleCatalog {
if err := units.Register(chunkers); err != nil { if err := units.Register(chunkers); err != nil {
t.Fatalf("register generic chunker: %v", err) t.Fatalf("register generic chunker: %v", err)
} }
if err := extractors.RegisterLegacyRawWithSpec(pipeline.ModuleSpec{ if err := pipeline.RegisterExtractor[defaultArtifact](extractors, pipeline.ModuleSpec{
Key: "extract", Key: "extract",
Stage: pipeline.StageExtract, Stage: pipeline.StageExtract,
Requires: []string{"chunks"}, ArtifactKind: defaultArtifactKind,
Provides: []string{"records"}, Requires: []string{"chunks"},
}, func() (contracts.LegacyRawExtractor, error) { Provides: []string{"records"},
}, func() (contracts.Extractor[defaultArtifact], error) {
return defaultExtractor{}, nil return defaultExtractor{}, nil
}); err != nil { }); err != nil {
t.Fatalf("register extractor: %v", err) t.Fatalf("register extractor: %v", err)
} }
if err := appendorder.Register(mergers); err != nil { if err := appendorder.RegisterTyped(mergers, defaultArtifactKind, func(values []defaultArtifact) (defaultArtifact, error) {
if len(values) == 0 {
return defaultArtifact{}, nil
}
return values[0], nil
}); err != nil {
t.Fatalf("register appendorder merger: %v", err) t.Fatalf("register appendorder merger: %v", err)
} }
if err := noop.Register(normalizers); err != nil { if err := noop.RegisterTyped[defaultArtifact](normalizers, defaultArtifactKind); err != nil {
t.Fatalf("register noop normalizer: %v", err) t.Fatalf("register noop normalizer: %v", err)
} }
if err := jsonoutput.Register(outputs); err != nil { if err := jsonoutput.Register(outputs); err != nil {
t.Fatalf("register json output: %v", err) t.Fatalf("register json output: %v", err)
} }
codecs := pipeline.NewArtifactCodecRegistry()
if err := pipeline.RegisterArtifactCodec(codecs, defaultArtifactCodec{}); err != nil {
t.Fatalf("register artifact codec: %v", err)
}
return pipeline.ModuleCatalog{ return pipeline.ModuleCatalog{
Inputs: inputs, Inputs: inputs,
Chunkers: chunkers, Chunkers: chunkers,
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), ArtifactCodecs: codecs,
Extractors: extractors, Extractors: extractors,
Mergers: mergers, Mergers: mergers,
Normalizers: normalizers, Normalizers: normalizers,
@@ -117,6 +128,25 @@ func (defaultExtractor) Key() string { return "extract" }
func (defaultExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil } func (defaultExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (defaultExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) { func (defaultExtractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[defaultArtifact], error) {
return contracts.ExtractionResult{}, nil return contracts.TypedExtractionResult[defaultArtifact]{}, nil
}
const defaultArtifactKind contracts.ArtifactKind = "test/default"
type defaultArtifact struct {
Value string `json:"value"`
}
type defaultArtifactCodec struct{}
func (defaultArtifactCodec) Kind() contracts.ArtifactKind { return defaultArtifactKind }
func (defaultArtifactCodec) Schema() contracts.ArtifactSchema {
return contracts.ArtifactSchema{ID: "urn:notarius:test:default", Name: "default", Version: "1", JSONSchema: []byte(`{"type":"object"}`)}
}
func (defaultArtifactCodec) MediaType() string { return "application/json" }
func (defaultArtifactCodec) Encode(value defaultArtifact) ([]byte, error) { return json.Marshal(value) }
func (defaultArtifactCodec) Decode(content []byte) (defaultArtifact, error) {
var value defaultArtifact
err := json.Unmarshal(content, &value)
return value, err
} }

View File

@@ -9,14 +9,9 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
) )
type LegacyRawExtractorConstructor func() (contracts.LegacyRawExtractor, error)
type LegacyRawExtractorBuilder func(BuildRequest) (contracts.LegacyRawExtractor, error)
type ExtractorRegistry struct { type ExtractorRegistry struct {
legacyBuilders map[string]LegacyRawExtractorBuilder typedEntries map[string]typedExtractorEntry
legacyValidators map[string]OptionValidator specs map[string]ModuleSpec
typedEntries map[string]typedExtractorEntry
specs map[string]ModuleSpec
} }
type typedExtractorEntry struct { type typedExtractorEntry struct {
@@ -25,201 +20,79 @@ type typedExtractorEntry struct {
validateOptions OptionValidator validateOptions OptionValidator
builder func(BuildRequest) (any, error) builder func(BuildRequest) (any, error)
extract typedExtractOperation extract typedExtractOperation
rawBuilder LegacyRawExtractorBuilder
} }
func NewExtractorRegistry() *ExtractorRegistry { func NewExtractorRegistry() *ExtractorRegistry {
return &ExtractorRegistry{ return &ExtractorRegistry{typedEntries: map[string]typedExtractorEntry{}, specs: map[string]ModuleSpec{}}
legacyBuilders: make(map[string]LegacyRawExtractorBuilder),
legacyValidators: make(map[string]OptionValidator),
typedEntries: make(map[string]typedExtractorEntry),
specs: make(map[string]ModuleSpec),
}
}
func (r *ExtractorRegistry) RegisterLegacyRaw(key string, constructor LegacyRawExtractorConstructor) error {
return r.RegisterLegacyRawWithSpec(defaultModuleSpec(key, StageExtract), constructor)
}
func (r *ExtractorRegistry) RegisterLegacyRawWithSpec(spec ModuleSpec, constructor LegacyRawExtractorConstructor) error {
if constructor == nil {
return fmt.Errorf("extractor constructor for %q must not be nil", strings.TrimSpace(spec.Key))
}
return r.RegisterLegacyRawBuilderWithSpec(spec, allowLegacyOptions, func(BuildRequest) (contracts.LegacyRawExtractor, error) {
return constructor()
})
}
func (r *ExtractorRegistry) RegisterLegacyRawBuilderWithSpec(spec ModuleSpec, validateOptions OptionValidator, builder LegacyRawExtractorBuilder) error {
if r == nil {
return fmt.Errorf("extractor registry must not be nil")
}
normalizedSpec := normalizeModuleSpec(spec)
if err := validateModuleSpec("extractor", StageExtract, normalizedSpec); err != nil {
return err
}
if normalizedSpec.ArtifactKind != "" {
return fmt.Errorf("legacy raw extractor %q must not declare an artifact kind", normalizedSpec.Key)
}
if validateOptions == nil {
return fmt.Errorf("extractor option validator for %q must not be nil", normalizedSpec.Key)
}
if builder == nil {
return fmt.Errorf("extractor builder for %q must not be nil", normalizedSpec.Key)
}
if _, ok := r.specs[normalizedSpec.Key]; ok {
return fmt.Errorf("extractor %q is already registered", normalizedSpec.Key)
}
if r.legacyBuilders == nil {
r.legacyBuilders = make(map[string]LegacyRawExtractorBuilder)
}
if r.legacyValidators == nil {
r.legacyValidators = make(map[string]OptionValidator)
}
if r.specs == nil {
r.specs = make(map[string]ModuleSpec)
}
r.legacyBuilders[normalizedSpec.Key] = builder
r.legacyValidators[normalizedSpec.Key] = validateOptions
r.specs[normalizedSpec.Key] = cloneModuleSpec(normalizedSpec)
return nil
} }
func RegisterExtractor[T any](registry *ExtractorRegistry, spec ModuleSpec, constructor func() (contracts.Extractor[T], error)) error { func RegisterExtractor[T any](registry *ExtractorRegistry, spec ModuleSpec, constructor func() (contracts.Extractor[T], error)) error {
if constructor == nil { if constructor == nil {
return fmt.Errorf("extractor constructor for %q must not be nil", strings.TrimSpace(spec.Key)) return fmt.Errorf("extractor constructor for %q must not be nil", strings.TrimSpace(spec.Key))
} }
return RegisterExtractorBuilder(registry, spec, allowLegacyOptions, func(BuildRequest) (contracts.Extractor[T], error) { return RegisterExtractorBuilder(registry, spec, rejectUnconfiguredOptions, func(BuildRequest) (contracts.Extractor[T], error) { return constructor() })
return constructor()
})
} }
func RegisterExtractorBuilder[T any](registry *ExtractorRegistry, spec ModuleSpec, validateOptions OptionValidator, builder func(BuildRequest) (contracts.Extractor[T], error)) error { func RegisterExtractorBuilder[T any](registry *ExtractorRegistry, spec ModuleSpec, validateOptions OptionValidator, builder func(BuildRequest) (contracts.Extractor[T], error)) error {
return registerExtractorBuilder(registry, spec, validateOptions, builder, nil)
}
// RegisterExtractorBuilderWithRawAdapter registers a typed extractor while a
// raw downstream remains in use. Resolution selects the adapter until the
// registration is replaced with the typed-only builder.
func RegisterExtractorBuilderWithRawAdapter[T any](registry *ExtractorRegistry, spec ModuleSpec, validateOptions OptionValidator, builder func(BuildRequest) (contracts.Extractor[T], error), rawBuilder LegacyRawExtractorBuilder) error {
if rawBuilder == nil {
return fmt.Errorf("extractor raw adapter builder for %q must not be nil", strings.TrimSpace(spec.Key))
}
return registerExtractorBuilder(registry, spec, validateOptions, builder, rawBuilder)
}
func registerExtractorBuilder[T any](registry *ExtractorRegistry, spec ModuleSpec, validateOptions OptionValidator, builder func(BuildRequest) (contracts.Extractor[T], error), rawBuilder LegacyRawExtractorBuilder) error {
if registry == nil { if registry == nil {
return fmt.Errorf("extractor registry must not be nil") return fmt.Errorf("extractor registry must not be nil")
} }
normalizedSpec := normalizeModuleSpec(spec) normalized := normalizeModuleSpec(spec)
if err := validateModuleSpec("extractor", StageExtract, normalizedSpec); err != nil { if err := validateModuleSpec("extractor", StageExtract, normalized); err != nil {
return err return err
} }
if normalizedSpec.ArtifactKind == "" { if normalized.ArtifactKind == "" {
return fmt.Errorf("typed extractor %q artifact kind must not be empty", normalizedSpec.Key) return fmt.Errorf("typed extractor %q artifact kind must not be empty", normalized.Key)
} }
if validateOptions == nil { if validateOptions == nil {
return fmt.Errorf("extractor option validator for %q must not be nil", normalizedSpec.Key) return fmt.Errorf("extractor option validator for %q must not be nil", normalized.Key)
} }
if builder == nil { if builder == nil {
return fmt.Errorf("extractor builder for %q must not be nil", normalizedSpec.Key) return fmt.Errorf("extractor builder for %q must not be nil", normalized.Key)
} }
if _, ok := registry.specs[normalizedSpec.Key]; ok { if _, ok := registry.specs[normalized.Key]; ok {
return fmt.Errorf("extractor %q is already registered", normalizedSpec.Key) return fmt.Errorf("extractor %q is already registered", normalized.Key)
}
entry := typedExtractorEntry{
spec: cloneModuleSpec(normalizedSpec),
valueType: reflect.TypeFor[T](),
validateOptions: validateOptions,
builder: func(request BuildRequest) (any, error) {
return builder(cloneBuildRequest(request))
},
extract: func(ctx context.Context, implementation any, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
extractor, ok := implementation.(contracts.Extractor[T])
if !ok {
return erasedTypedResult{}, fmt.Errorf("extractor %q has incompatible implementation %T", normalizedSpec.Key, implementation)
}
result, err := extractor.Extract(ctx, request)
if err != nil {
return erasedTypedResult{}, err
}
return erasedTypedResult{Value: result.Value, Warnings: result.Warnings}, nil
},
rawBuilder: rawBuilder,
} }
entry := typedExtractorEntry{spec: cloneModuleSpec(normalized), valueType: reflect.TypeFor[T](), validateOptions: validateOptions, builder: func(request BuildRequest) (any, error) { return builder(cloneBuildRequest(request)) }, extract: func(ctx context.Context, implementation any, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
extractor, ok := implementation.(contracts.Extractor[T])
if !ok {
return erasedTypedResult{}, fmt.Errorf("extractor %q has incompatible implementation %T", normalized.Key, implementation)
}
result, err := extractor.Extract(ctx, request)
if err != nil {
return erasedTypedResult{}, err
}
return erasedTypedResult{Value: result.Value, Warnings: result.Warnings}, nil
}}
if registry.typedEntries == nil { if registry.typedEntries == nil {
registry.typedEntries = make(map[string]typedExtractorEntry) registry.typedEntries = map[string]typedExtractorEntry{}
} }
if registry.specs == nil { if registry.specs == nil {
registry.specs = make(map[string]ModuleSpec) registry.specs = map[string]ModuleSpec{}
} }
registry.typedEntries[normalizedSpec.Key] = entry registry.typedEntries[normalized.Key] = entry
registry.specs[normalizedSpec.Key] = cloneModuleSpec(normalizedSpec) registry.specs[normalized.Key] = cloneModuleSpec(normalized)
return nil return nil
} }
func (r *ExtractorRegistry) BuildLegacyRaw(key string) (contracts.LegacyRawExtractor, error) {
return r.BuildLegacyRawWithRequest(key, BuildRequest{})
}
func (r *ExtractorRegistry) BuildLegacyRawWithRequest(key string, request BuildRequest) (contracts.LegacyRawExtractor, error) {
if r == nil {
return nil, fmt.Errorf("extractor registry must not be nil")
}
normalizedKey := strings.TrimSpace(key)
if normalizedKey == "" {
return nil, fmt.Errorf("extractor key must not be empty")
}
builder, ok := r.legacyBuilders[normalizedKey]
if !ok {
if entry, typedOK := r.typedEntries[normalizedKey]; typedOK && entry.rawBuilder != nil {
builder = entry.rawBuilder
ok = true
}
}
if !ok {
return nil, fmt.Errorf("legacy raw extractor %q is not registered", normalizedKey)
}
extractor, err := builder(cloneBuildRequest(request))
if err != nil {
return nil, fmt.Errorf("build extractor %q: %w", normalizedKey, err)
}
if extractor == nil {
return nil, fmt.Errorf("extractor %q constructor returned nil", normalizedKey)
}
if extractor.Key() != normalizedKey {
return nil, fmt.Errorf("extractor %q returned key %q", normalizedKey, extractor.Key())
}
return extractor, nil
}
func (r *ExtractorRegistry) validateOptions(key string, options map[string]any) error { func (r *ExtractorRegistry) validateOptions(key string, options map[string]any) error {
if r == nil { if r == nil {
return fmt.Errorf("extractor registry must not be nil") return fmt.Errorf("extractor registry must not be nil")
} }
normalizedKey := strings.TrimSpace(key) normalized := strings.TrimSpace(key)
if entry, ok := r.typedEntries[normalizedKey]; ok { entry, ok := r.typedEntries[normalized]
return validateRegisteredOptions(entry.validateOptions, options)
}
validator, ok := r.legacyValidators[normalizedKey]
if !ok { if !ok {
return fmt.Errorf("extractor %q is not registered", normalizedKey) return fmt.Errorf("extractor %q is not registered", normalized)
} }
return validateRegisteredOptions(validator, options) return validateRegisteredOptions(entry.validateOptions, options)
} }
func (r *ExtractorRegistry) Spec(key string) (ModuleSpec, bool) { func (r *ExtractorRegistry) Spec(key string) (ModuleSpec, bool) {
if r == nil { if r == nil {
return ModuleSpec{}, false return ModuleSpec{}, false
} }
spec, ok := r.specs[strings.TrimSpace(key)] spec, ok := r.specs[strings.TrimSpace(key)]
if !ok { return cloneModuleSpec(spec), ok
return ModuleSpec{}, false
}
return cloneModuleSpec(spec), true
} }
func (r *ExtractorRegistry) typedEntry(key string) (typedExtractorEntry, bool) { func (r *ExtractorRegistry) typedEntry(key string) (typedExtractorEntry, bool) {
if r == nil { if r == nil {
return typedExtractorEntry{}, false return typedExtractorEntry{}, false
@@ -227,12 +100,6 @@ func (r *ExtractorRegistry) typedEntry(key string) (typedExtractorEntry, bool) {
entry, ok := r.typedEntries[strings.TrimSpace(key)] entry, ok := r.typedEntries[strings.TrimSpace(key)]
return entry, ok return entry, ok
} }
func (r *ExtractorRegistry) usesRawAdapter(key string) bool {
entry, ok := r.typedEntry(key)
return ok && entry.rawBuilder != nil
}
func (r *ExtractorRegistry) RegisteredKeys() []string { func (r *ExtractorRegistry) RegisteredKeys() []string {
if r == nil { if r == nil {
return nil return nil

View File

@@ -1,414 +0,0 @@
package pipeline
import (
"context"
"errors"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestExtractorRegistryRegisterAndBuild(t *testing.T) {
registry := NewExtractorRegistry()
if err := registry.RegisterLegacyRaw("generic-extractor", fakeExtractorConstructor("generic-extractor")); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
extractor, err := registry.BuildLegacyRaw("generic-extractor")
if err != nil {
t.Fatalf("Build() error = %v, want nil", err)
}
if extractor.Key() != "generic-extractor" {
t.Fatalf("extractor.Key() = %q, want generic-extractor", extractor.Key())
}
}
func TestExtractorRegistryRegisterAndBuildTrimKeys(t *testing.T) {
registry := NewExtractorRegistry()
if err := registry.RegisterLegacyRaw(" generic-extractor ", fakeExtractorConstructor("generic-extractor")); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
extractor, err := registry.BuildLegacyRaw("\tgeneric-extractor\n")
if err != nil {
t.Fatalf("Build() error = %v, want nil", err)
}
if extractor.Key() != "generic-extractor" {
t.Fatalf("extractor.Key() = %q, want generic-extractor", extractor.Key())
}
}
func TestExtractorRegistryRegisterWithSpecStoresMetadata(t *testing.T) {
registry := NewExtractorRegistry()
spec := ModuleSpec{
Key: " generic-extractor ",
Stage: StageExtract,
Provides: []string{" generic-artifact ", "source-citations", "generic-artifact", ""},
Requires: []string{" source-document ", "source-document", ""},
ReferenceSlots: []contracts.ReferenceSlot{
{
Name: " glossary ",
Description: " Supporting terms ",
AcceptedMediaTypes: []string{" text/plain ", "text/markdown", "text/plain", ""},
MaxBytes: 1024,
},
{
Name: " roster ",
Description: " Characters ",
Required: true,
Multiple: true,
},
},
}
if err := registry.RegisterLegacyRawWithSpec(spec, fakeExtractorConstructor("generic-extractor")); err != nil {
t.Fatalf("RegisterWithSpec() error = %v, want nil", err)
}
got, ok := registry.Spec("\tgeneric-extractor\n")
if !ok {
t.Fatal("Spec() ok = false, want true")
}
want := ModuleSpec{
Key: "generic-extractor",
Stage: StageExtract,
Provides: []string{"generic-artifact", "source-citations"},
Requires: []string{"source-document"},
ReferenceSlots: []contracts.ReferenceSlot{
{
Name: "glossary",
Description: "Supporting terms",
AcceptedMediaTypes: []string{"text/markdown", "text/plain"},
MaxBytes: 1024,
},
{
Name: "roster",
Description: "Characters",
Required: true,
Multiple: true,
},
},
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("Spec() = %#v, want %#v", got, want)
}
got.Provides[0] = "changed"
got.ReferenceSlots[0].Name = "changed"
got.ReferenceSlots[0].AcceptedMediaTypes[0] = "changed"
again, ok := registry.Spec("generic-extractor")
if !ok {
t.Fatal("Spec() after caller mutation ok = false, want true")
}
if !reflect.DeepEqual(again, want) {
t.Fatalf("Spec() after caller mutation = %#v, want %#v", again, want)
}
}
func TestExtractorRegistryRegisterStoresDefaultSpec(t *testing.T) {
registry := NewExtractorRegistry()
if err := registry.RegisterLegacyRaw(" generic-extractor ", fakeExtractorConstructor("generic-extractor")); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
got, ok := registry.Spec("generic-extractor")
if !ok {
t.Fatal("Spec() ok = false, want true")
}
want := ModuleSpec{Key: "generic-extractor", Stage: StageExtract}
if !reflect.DeepEqual(got, want) {
t.Fatalf("Spec() = %#v, want %#v", got, want)
}
}
func TestExtractorRegistryRegisterWithSpecRejectsWrongStage(t *testing.T) {
registry := NewExtractorRegistry()
err := registry.RegisterLegacyRawWithSpec(ModuleSpec{Key: "generic-extractor", Stage: StageInput}, fakeExtractorConstructor("generic-extractor"))
if err == nil {
t.Fatal("RegisterWithSpec() error = nil, want error")
}
if !strings.Contains(err.Error(), "stage") {
t.Fatalf("RegisterWithSpec() error = %q, want stage error", err.Error())
}
}
func TestExtractorRegistryRejectsInvalidReferenceSlots(t *testing.T) {
tests := []struct {
name string
slots []contracts.ReferenceSlot
want string
}{
{
name: "empty name",
slots: []contracts.ReferenceSlot{{Name: " "}},
want: "name",
},
{
name: "duplicate name after trim",
slots: []contracts.ReferenceSlot{
{Name: "roster"},
{Name: " roster "},
},
want: "duplicated",
},
{
name: "negative max bytes",
slots: []contracts.ReferenceSlot{{Name: "roster", MaxBytes: -1}},
want: "max_bytes",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
registry := NewExtractorRegistry()
err := registry.RegisterLegacyRawWithSpec(ModuleSpec{
Key: "generic-extractor",
Stage: StageExtract,
ReferenceSlots: test.slots,
}, fakeExtractorConstructor("generic-extractor"))
if err == nil {
t.Fatal("RegisterWithSpec() error = nil, want error")
}
if !strings.Contains(err.Error(), test.want) {
t.Fatalf("RegisterWithSpec() error = %q, want %q", err.Error(), test.want)
}
})
}
}
func TestExtractorRegistrySpecRejectsUnknownKey(t *testing.T) {
registry := NewExtractorRegistry()
if _, ok := registry.Spec("missing-extractor"); ok {
t.Fatal("Spec() ok = true, want false")
}
}
func TestExtractorRegistryRegisterRejectsEmptyKey(t *testing.T) {
registry := NewExtractorRegistry()
err := registry.RegisterLegacyRaw(" \t", fakeExtractorConstructor("generic-extractor"))
if err == nil {
t.Fatal("Register() error = nil, want error")
}
if !strings.Contains(err.Error(), "key must not be empty") {
t.Fatalf("Register() error = %q, want empty key error", err.Error())
}
}
func TestExtractorRegistryRegisterRejectsDuplicateKey(t *testing.T) {
registry := NewExtractorRegistry()
if err := registry.RegisterLegacyRaw("generic-extractor", fakeExtractorConstructor("generic-extractor")); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
err := registry.RegisterLegacyRaw(" generic-extractor ", fakeExtractorConstructor("generic-extractor"))
if err == nil {
t.Fatal("Register() error = nil, want error")
}
if !strings.Contains(err.Error(), "already registered") {
t.Fatalf("Register() error = %q, want duplicate key error", err.Error())
}
}
func TestExtractorRegistryRegisterRejectsNilConstructor(t *testing.T) {
registry := NewExtractorRegistry()
err := registry.RegisterLegacyRaw("generic-extractor", nil)
if err == nil {
t.Fatal("Register() error = nil, want error")
}
if !strings.Contains(err.Error(), "constructor") {
t.Fatalf("Register() error = %q, want constructor error", err.Error())
}
}
func TestExtractorRegistryBuildRejectsUnknownKey(t *testing.T) {
registry := NewExtractorRegistry()
_, err := registry.BuildLegacyRaw("missing-extractor")
if err == nil {
t.Fatal("Build() error = nil, want error")
}
if !strings.Contains(err.Error(), "not registered") {
t.Fatalf("Build() error = %q, want unknown key error", err.Error())
}
}
func TestExtractorRegistryBuildWrapsConstructorError(t *testing.T) {
registry := NewExtractorRegistry()
constructorErr := errors.New("constructor failed")
if err := registry.RegisterLegacyRaw("generic-extractor", func() (contracts.LegacyRawExtractor, error) {
return nil, constructorErr
}); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
_, err := registry.BuildLegacyRaw("generic-extractor")
if err == nil {
t.Fatal("Build() error = nil, want error")
}
if !errors.Is(err, constructorErr) {
t.Fatalf("Build() error = %v, want wrapped constructor error", err)
}
if !strings.Contains(err.Error(), "generic-extractor") {
t.Fatalf("Build() error = %q, want key context", err.Error())
}
}
func TestExtractorRegistryBuildRejectsNilExtractor(t *testing.T) {
registry := NewExtractorRegistry()
if err := registry.RegisterLegacyRaw("generic-extractor", func() (contracts.LegacyRawExtractor, error) {
return nil, nil
}); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
_, err := registry.BuildLegacyRaw("generic-extractor")
if err == nil {
t.Fatal("Build() error = nil, want error")
}
if !strings.Contains(err.Error(), "returned nil") {
t.Fatalf("Build() error = %q, want nil extractor error", err.Error())
}
}
func TestExtractorRegistryBuildRejectsExtractorKeyMismatch(t *testing.T) {
registry := NewExtractorRegistry()
if err := registry.RegisterLegacyRaw("generic-extractor", fakeExtractorConstructor("other-extractor")); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
_, err := registry.BuildLegacyRaw("generic-extractor")
if err == nil {
t.Fatal("Build() error = nil, want error")
}
if !strings.Contains(err.Error(), "returned key") {
t.Fatalf("Build() error = %q, want key mismatch error", err.Error())
}
}
func TestExtractorRegistryRegisteredKeysReturnsSortedCopy(t *testing.T) {
registry := NewExtractorRegistry()
for _, key := range []string{"zeta", "alpha", "middle"} {
if err := registry.RegisterLegacyRaw(key, fakeExtractorConstructor(key)); err != nil {
t.Fatalf("Register(%q) error = %v, want nil", key, err)
}
}
keys := registry.RegisteredKeys()
want := []string{"alpha", "middle", "zeta"}
if !reflect.DeepEqual(keys, want) {
t.Fatalf("RegisteredKeys() = %#v, want %#v", keys, want)
}
keys[0] = "changed"
if got := registry.RegisteredKeys(); !reflect.DeepEqual(got, want) {
t.Fatalf("RegisteredKeys() after caller mutation = %#v, want %#v", got, want)
}
}
func TestExtractorRegistryNilRegistryBehavior(t *testing.T) {
var registry *ExtractorRegistry
if err := registry.RegisterLegacyRaw("generic-extractor", fakeExtractorConstructor("generic-extractor")); err == nil {
t.Fatal("Register() error = nil, want error")
}
if _, err := registry.BuildLegacyRaw("generic-extractor"); err == nil {
t.Fatal("Build() error = nil, want error")
}
if _, ok := registry.Spec("generic-extractor"); ok {
t.Fatal("Spec() ok = true, want false")
}
if keys := registry.RegisteredKeys(); keys != nil {
t.Fatalf("RegisteredKeys() = %#v, want nil", keys)
}
}
func TestExtractorRegistryBuildRejectsEmptyKey(t *testing.T) {
registry := NewExtractorRegistry()
_, err := registry.BuildLegacyRaw(" \n")
if err == nil {
t.Fatal("Build() error = nil, want error")
}
if !strings.Contains(err.Error(), "key must not be empty") {
t.Fatalf("Build() error = %q, want empty key error", err.Error())
}
}
func TestExtractorRegistryTypedRegistrationCanProvideRawAdapter(t *testing.T) {
registry := NewExtractorRegistry()
spec := ModuleSpec{Key: "typed-extractor", Stage: StageExtract, ArtifactKind: "test/value"}
if err := RegisterExtractorBuilderWithRawAdapter(registry, spec, func(map[string]any) error { return nil },
func(BuildRequest) (contracts.Extractor[registryTypedValue], error) {
return registryTypedExtractor{key: spec.Key}, nil
},
func(BuildRequest) (contracts.LegacyRawExtractor, error) {
return registryFakeExtractor{key: spec.Key}, nil
},
); err != nil {
t.Fatalf("RegisterExtractorBuilderWithRawAdapter() error = %v", err)
}
if !registry.usesRawAdapter(spec.Key) {
t.Fatal("usesRawAdapter() = false, want true")
}
if _, ok := registry.typedEntry(spec.Key); !ok {
t.Fatal("typedEntry() ok = false, want true")
}
adapter, err := registry.BuildLegacyRaw(spec.Key)
if err != nil || adapter.Key() != spec.Key {
t.Fatalf("BuildLegacyRaw() = %#v, %v", adapter, err)
}
if err := RegisterExtractorBuilderWithRawAdapter[registryTypedValue](NewExtractorRegistry(), spec, func(map[string]any) error { return nil }, nil, nil); err == nil || !strings.Contains(err.Error(), "raw adapter") {
t.Fatalf("nil raw builder error = %v, want raw adapter context", err)
}
}
type registryFakeExtractor struct {
key string
}
type registryTypedValue struct{ Value string }
type registryTypedExtractor struct{ key string }
func (extractor registryTypedExtractor) Key() string { return extractor.key }
func (registryTypedExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (registryTypedExtractor) Extract(context.Context, contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[registryTypedValue], error) {
return contracts.TypedExtractionResult[registryTypedValue]{}, nil
}
func fakeExtractorConstructor(key string) LegacyRawExtractorConstructor {
return func() (contracts.LegacyRawExtractor, error) {
return registryFakeExtractor{key: key}, nil
}
}
func (extractor registryFakeExtractor) Key() string {
return extractor.key
}
func (extractor registryFakeExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (extractor registryFakeExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
return contracts.ExtractionResult{}, nil
}

View File

@@ -32,7 +32,7 @@ func (r *InputAdapterRegistry) RegisterWithSpec(spec ModuleSpec, constructor Inp
if constructor == nil { if constructor == nil {
return fmt.Errorf("input adapter constructor for %q must not be nil", strings.TrimSpace(spec.Key)) return fmt.Errorf("input adapter constructor for %q must not be nil", strings.TrimSpace(spec.Key))
} }
return r.RegisterBuilderWithSpec(spec, allowLegacyOptions, func(BuildRequest) (contracts.InputAdapter, error) { return r.RegisterBuilderWithSpec(spec, rejectUnconfiguredOptions, func(BuildRequest) (contracts.InputAdapter, error) {
return constructor() return constructor()
}) })
} }

View File

@@ -9,19 +9,13 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
) )
type LegacyRawMergerConstructor func() (contracts.LegacyRawMerger, error)
type LegacyRawMergerBuilder func(BuildRequest) (contracts.LegacyRawMerger, error)
type artifactVariantKey struct { type artifactVariantKey struct {
module string module string
kind contracts.ArtifactKind kind contracts.ArtifactKind
} }
type MergerRegistry struct { type MergerRegistry struct {
legacyBuilders map[string]LegacyRawMergerBuilder typedEntries map[artifactVariantKey]typedMergerEntry
legacyValidators map[string]OptionValidator
legacySpecs map[string]ModuleSpec
typedEntries map[artifactVariantKey]typedMergerEntry
} }
type typedMergerEntry struct { type typedMergerEntry struct {
@@ -34,66 +28,15 @@ type typedMergerEntry struct {
func NewMergerRegistry() *MergerRegistry { func NewMergerRegistry() *MergerRegistry {
return &MergerRegistry{ return &MergerRegistry{
legacyBuilders: make(map[string]LegacyRawMergerBuilder), typedEntries: make(map[artifactVariantKey]typedMergerEntry),
legacyValidators: make(map[string]OptionValidator),
legacySpecs: make(map[string]ModuleSpec),
typedEntries: make(map[artifactVariantKey]typedMergerEntry),
} }
} }
func (r *MergerRegistry) RegisterLegacyRaw(key string, constructor LegacyRawMergerConstructor) error {
return r.RegisterLegacyRawWithSpec(defaultModuleSpec(key, StageMerge), constructor)
}
func (r *MergerRegistry) RegisterLegacyRawWithSpec(spec ModuleSpec, constructor LegacyRawMergerConstructor) error {
if constructor == nil {
return fmt.Errorf("merger constructor for %q must not be nil", strings.TrimSpace(spec.Key))
}
return r.RegisterLegacyRawBuilderWithSpec(spec, allowLegacyOptions, func(BuildRequest) (contracts.LegacyRawMerger, error) {
return constructor()
})
}
func (r *MergerRegistry) RegisterLegacyRawBuilderWithSpec(spec ModuleSpec, validateOptions OptionValidator, builder LegacyRawMergerBuilder) error {
if r == nil {
return fmt.Errorf("merger registry must not be nil")
}
normalizedSpec := normalizeModuleSpec(spec)
if err := validateModuleSpec("merger", StageMerge, normalizedSpec); err != nil {
return err
}
if normalizedSpec.ArtifactKind != "" {
return fmt.Errorf("legacy raw merger %q must not declare an artifact kind", normalizedSpec.Key)
}
if validateOptions == nil {
return fmt.Errorf("merger option validator for %q must not be nil", normalizedSpec.Key)
}
if builder == nil {
return fmt.Errorf("merger builder for %q must not be nil", normalizedSpec.Key)
}
if _, ok := r.legacyBuilders[normalizedSpec.Key]; ok {
return fmt.Errorf("legacy raw merger %q is already registered", normalizedSpec.Key)
}
if r.legacyBuilders == nil {
r.legacyBuilders = make(map[string]LegacyRawMergerBuilder)
}
if r.legacyValidators == nil {
r.legacyValidators = make(map[string]OptionValidator)
}
if r.legacySpecs == nil {
r.legacySpecs = make(map[string]ModuleSpec)
}
r.legacyBuilders[normalizedSpec.Key] = builder
r.legacyValidators[normalizedSpec.Key] = validateOptions
r.legacySpecs[normalizedSpec.Key] = cloneModuleSpec(normalizedSpec)
return nil
}
func RegisterMerger[T any](registry *MergerRegistry, spec ModuleSpec, constructor func() (contracts.Merger[T], error)) error { func RegisterMerger[T any](registry *MergerRegistry, spec ModuleSpec, constructor func() (contracts.Merger[T], error)) error {
if constructor == nil { if constructor == nil {
return fmt.Errorf("merger constructor for %q must not be nil", strings.TrimSpace(spec.Key)) return fmt.Errorf("merger constructor for %q must not be nil", strings.TrimSpace(spec.Key))
} }
return RegisterMergerBuilder(registry, spec, allowLegacyOptions, func(BuildRequest) (contracts.Merger[T], error) { return RegisterMergerBuilder(registry, spec, rejectUnconfiguredOptions, func(BuildRequest) (contracts.Merger[T], error) {
return constructor() return constructor()
}) })
} }
@@ -152,63 +95,29 @@ func RegisterMergerBuilder[T any](registry *MergerRegistry, spec ModuleSpec, val
return nil return nil
} }
func (r *MergerRegistry) BuildLegacyRaw(key string) (contracts.LegacyRawMerger, error) {
return r.BuildLegacyRawWithRequest(key, BuildRequest{})
}
func (r *MergerRegistry) BuildLegacyRawWithRequest(key string, request BuildRequest) (contracts.LegacyRawMerger, error) {
if r == nil {
return nil, fmt.Errorf("merger registry must not be nil")
}
normalizedKey := strings.TrimSpace(key)
if normalizedKey == "" {
return nil, fmt.Errorf("merger key must not be empty")
}
builder, ok := r.legacyBuilders[normalizedKey]
if !ok {
return nil, fmt.Errorf("legacy raw merger %q is not registered", normalizedKey)
}
merger, err := builder(cloneBuildRequest(request))
if err != nil {
return nil, fmt.Errorf("build merger %q: %w", normalizedKey, err)
}
if merger == nil {
return nil, fmt.Errorf("merger %q constructor returned nil", normalizedKey)
}
if merger.Key() != normalizedKey {
return nil, fmt.Errorf("merger %q returned key %q", normalizedKey, merger.Key())
}
return merger, nil
}
func (r *MergerRegistry) validateOptions(key string, kind contracts.ArtifactKind, options map[string]any) error { func (r *MergerRegistry) validateOptions(key string, kind contracts.ArtifactKind, options map[string]any) error {
if r == nil { if r == nil {
return fmt.Errorf("merger registry must not be nil") return fmt.Errorf("merger registry must not be nil")
} }
normalizedKey := strings.TrimSpace(key) normalizedKey := strings.TrimSpace(key)
if kind != "" { entry, ok := r.typedEntry(normalizedKey, kind)
entry, ok := r.typedEntry(normalizedKey, kind)
if !ok {
return fmt.Errorf("merger %q variant for artifact kind %q is not registered", normalizedKey, kind)
}
return validateRegisteredOptions(entry.validateOptions, options)
}
validator, ok := r.legacyValidators[normalizedKey]
if !ok { if !ok {
return fmt.Errorf("legacy raw merger %q is not registered", normalizedKey) return fmt.Errorf("merger %q variant for artifact kind %q is not registered", normalizedKey, kind)
} }
return validateRegisteredOptions(validator, options) return validateRegisteredOptions(entry.validateOptions, options)
} }
func (r *MergerRegistry) Spec(key string) (ModuleSpec, bool) { func (r *MergerRegistry) Spec(key string) (ModuleSpec, bool) {
if r == nil { if r == nil {
return ModuleSpec{}, false return ModuleSpec{}, false
} }
spec, ok := r.legacySpecs[strings.TrimSpace(key)] module := strings.TrimSpace(key)
if !ok { for variant, entry := range r.typedEntries {
return ModuleSpec{}, false if variant.module == module {
return cloneModuleSpec(entry.spec), true
}
} }
return cloneModuleSpec(spec), true return ModuleSpec{}, false
} }
func (r *MergerRegistry) typedEntry(key string, kind contracts.ArtifactKind) (typedMergerEntry, bool) { func (r *MergerRegistry) typedEntry(key string, kind contracts.ArtifactKind) (typedMergerEntry, bool) {
@@ -238,10 +147,7 @@ func (r *MergerRegistry) RegisteredKeys() []string {
if r == nil { if r == nil {
return nil return nil
} }
keys := make(map[string]struct{}, len(r.legacySpecs)+len(r.typedEntries)) keys := make(map[string]struct{}, len(r.typedEntries))
for key := range r.legacySpecs {
keys[key] = struct{}{}
}
for key := range r.typedEntries { for key := range r.typedEntries {
keys[key.module] = struct{}{} keys[key.module] = struct{}{}
} }

View File

@@ -1,58 +0,0 @@
package pipeline
import (
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestMergerRegistryBehavior(t *testing.T) {
runRegistryBehaviorTests(t, registryBehaviorCase[contracts.LegacyRawMerger]{
name: "MergerRegistry",
key: "generic-merger",
stage: StageMerge,
wrongStage: StageExtract,
newRegistry: func() any {
return NewMergerRegistry()
},
register: func(registry any, key string, constructor func() (contracts.LegacyRawMerger, error)) error {
return registry.(*MergerRegistry).RegisterLegacyRaw(key, constructor)
},
registerWithSpec: func(registry any, spec ModuleSpec, constructor func() (contracts.LegacyRawMerger, error)) error {
return registry.(*MergerRegistry).RegisterLegacyRawWithSpec(spec, constructor)
},
build: func(registry any, key string) (contracts.LegacyRawMerger, error) {
return registry.(*MergerRegistry).BuildLegacyRaw(key)
},
spec: func(registry any, key string) (ModuleSpec, bool) {
return registry.(*MergerRegistry).Spec(key)
},
registeredKeys: func(registry any) []string {
return registry.(*MergerRegistry).RegisteredKeys()
},
nilRegister: func(key string, constructor func() (contracts.LegacyRawMerger, error)) error {
var registry *MergerRegistry
return registry.RegisterLegacyRaw(key, constructor)
},
nilBuild: func(key string) (contracts.LegacyRawMerger, error) {
var registry *MergerRegistry
return registry.BuildLegacyRaw(key)
},
nilSpec: func(key string) (ModuleSpec, bool) {
var registry *MergerRegistry
return registry.Spec(key)
},
nilRegisteredKey: func() []string {
var registry *MergerRegistry
return registry.RegisteredKeys()
},
constructor: func(key string) func() (contracts.LegacyRawMerger, error) {
return func() (contracts.LegacyRawMerger, error) {
return registryMerger{key: key}, nil
}
},
moduleKey: func(module contracts.LegacyRawMerger) string {
return module.Key()
},
})
}

View File

@@ -9,14 +9,8 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
) )
type LegacyRawNormalizerConstructor func() (contracts.LegacyRawNormalizer, error)
type LegacyRawNormalizerBuilder func(BuildRequest) (contracts.LegacyRawNormalizer, error)
type NormalizerRegistry struct { type NormalizerRegistry struct {
legacyBuilders map[string]LegacyRawNormalizerBuilder typedEntries map[artifactVariantKey]typedNormalizerEntry
legacyValidators map[string]OptionValidator
legacySpecs map[string]ModuleSpec
typedEntries map[artifactVariantKey]typedNormalizerEntry
} }
type typedNormalizerEntry struct { type typedNormalizerEntry struct {
@@ -29,66 +23,15 @@ type typedNormalizerEntry struct {
func NewNormalizerRegistry() *NormalizerRegistry { func NewNormalizerRegistry() *NormalizerRegistry {
return &NormalizerRegistry{ return &NormalizerRegistry{
legacyBuilders: make(map[string]LegacyRawNormalizerBuilder), typedEntries: make(map[artifactVariantKey]typedNormalizerEntry),
legacyValidators: make(map[string]OptionValidator),
legacySpecs: make(map[string]ModuleSpec),
typedEntries: make(map[artifactVariantKey]typedNormalizerEntry),
} }
} }
func (r *NormalizerRegistry) RegisterLegacyRaw(key string, constructor LegacyRawNormalizerConstructor) error {
return r.RegisterLegacyRawWithSpec(defaultModuleSpec(key, StageNormalize), constructor)
}
func (r *NormalizerRegistry) RegisterLegacyRawWithSpec(spec ModuleSpec, constructor LegacyRawNormalizerConstructor) error {
if constructor == nil {
return fmt.Errorf("normalizer constructor for %q must not be nil", strings.TrimSpace(spec.Key))
}
return r.RegisterLegacyRawBuilderWithSpec(spec, allowLegacyOptions, func(BuildRequest) (contracts.LegacyRawNormalizer, error) {
return constructor()
})
}
func (r *NormalizerRegistry) RegisterLegacyRawBuilderWithSpec(spec ModuleSpec, validateOptions OptionValidator, builder LegacyRawNormalizerBuilder) error {
if r == nil {
return fmt.Errorf("normalizer registry must not be nil")
}
normalizedSpec := normalizeModuleSpec(spec)
if err := validateModuleSpec("normalizer", StageNormalize, normalizedSpec); err != nil {
return err
}
if normalizedSpec.ArtifactKind != "" {
return fmt.Errorf("legacy raw normalizer %q must not declare an artifact kind", normalizedSpec.Key)
}
if validateOptions == nil {
return fmt.Errorf("normalizer option validator for %q must not be nil", normalizedSpec.Key)
}
if builder == nil {
return fmt.Errorf("normalizer builder for %q must not be nil", normalizedSpec.Key)
}
if _, ok := r.legacyBuilders[normalizedSpec.Key]; ok {
return fmt.Errorf("legacy raw normalizer %q is already registered", normalizedSpec.Key)
}
if r.legacyBuilders == nil {
r.legacyBuilders = make(map[string]LegacyRawNormalizerBuilder)
}
if r.legacyValidators == nil {
r.legacyValidators = make(map[string]OptionValidator)
}
if r.legacySpecs == nil {
r.legacySpecs = make(map[string]ModuleSpec)
}
r.legacyBuilders[normalizedSpec.Key] = builder
r.legacyValidators[normalizedSpec.Key] = validateOptions
r.legacySpecs[normalizedSpec.Key] = cloneModuleSpec(normalizedSpec)
return nil
}
func RegisterNormalizer[T any](registry *NormalizerRegistry, spec ModuleSpec, constructor func() (contracts.Normalizer[T], error)) error { func RegisterNormalizer[T any](registry *NormalizerRegistry, spec ModuleSpec, constructor func() (contracts.Normalizer[T], error)) error {
if constructor == nil { if constructor == nil {
return fmt.Errorf("normalizer constructor for %q must not be nil", strings.TrimSpace(spec.Key)) return fmt.Errorf("normalizer constructor for %q must not be nil", strings.TrimSpace(spec.Key))
} }
return RegisterNormalizerBuilder(registry, spec, allowLegacyOptions, func(BuildRequest) (contracts.Normalizer[T], error) { return RegisterNormalizerBuilder(registry, spec, rejectUnconfiguredOptions, func(BuildRequest) (contracts.Normalizer[T], error) {
return constructor() return constructor()
}) })
} }
@@ -143,63 +86,29 @@ func RegisterNormalizerBuilder[T any](registry *NormalizerRegistry, spec ModuleS
return nil return nil
} }
func (r *NormalizerRegistry) BuildLegacyRaw(key string) (contracts.LegacyRawNormalizer, error) {
return r.BuildLegacyRawWithRequest(key, BuildRequest{})
}
func (r *NormalizerRegistry) BuildLegacyRawWithRequest(key string, request BuildRequest) (contracts.LegacyRawNormalizer, error) {
if r == nil {
return nil, fmt.Errorf("normalizer registry must not be nil")
}
normalizedKey := strings.TrimSpace(key)
if normalizedKey == "" {
return nil, fmt.Errorf("normalizer key must not be empty")
}
builder, ok := r.legacyBuilders[normalizedKey]
if !ok {
return nil, fmt.Errorf("legacy raw normalizer %q is not registered", normalizedKey)
}
normalizer, err := builder(cloneBuildRequest(request))
if err != nil {
return nil, fmt.Errorf("build normalizer %q: %w", normalizedKey, err)
}
if normalizer == nil {
return nil, fmt.Errorf("normalizer %q constructor returned nil", normalizedKey)
}
if normalizer.Key() != normalizedKey {
return nil, fmt.Errorf("normalizer %q returned key %q", normalizedKey, normalizer.Key())
}
return normalizer, nil
}
func (r *NormalizerRegistry) validateOptions(key string, kind contracts.ArtifactKind, options map[string]any) error { func (r *NormalizerRegistry) validateOptions(key string, kind contracts.ArtifactKind, options map[string]any) error {
if r == nil { if r == nil {
return fmt.Errorf("normalizer registry must not be nil") return fmt.Errorf("normalizer registry must not be nil")
} }
normalizedKey := strings.TrimSpace(key) normalizedKey := strings.TrimSpace(key)
if kind != "" { entry, ok := r.typedEntry(normalizedKey, kind)
entry, ok := r.typedEntry(normalizedKey, kind)
if !ok {
return fmt.Errorf("normalizer %q variant for artifact kind %q is not registered", normalizedKey, kind)
}
return validateRegisteredOptions(entry.validateOptions, options)
}
validator, ok := r.legacyValidators[normalizedKey]
if !ok { if !ok {
return fmt.Errorf("legacy raw normalizer %q is not registered", normalizedKey) return fmt.Errorf("normalizer %q variant for artifact kind %q is not registered", normalizedKey, kind)
} }
return validateRegisteredOptions(validator, options) return validateRegisteredOptions(entry.validateOptions, options)
} }
func (r *NormalizerRegistry) Spec(key string) (ModuleSpec, bool) { func (r *NormalizerRegistry) Spec(key string) (ModuleSpec, bool) {
if r == nil { if r == nil {
return ModuleSpec{}, false return ModuleSpec{}, false
} }
spec, ok := r.legacySpecs[strings.TrimSpace(key)] module := strings.TrimSpace(key)
if !ok { for variant, entry := range r.typedEntries {
return ModuleSpec{}, false if variant.module == module {
return cloneModuleSpec(entry.spec), true
}
} }
return cloneModuleSpec(spec), true return ModuleSpec{}, false
} }
func (r *NormalizerRegistry) typedEntry(key string, kind contracts.ArtifactKind) (typedNormalizerEntry, bool) { func (r *NormalizerRegistry) typedEntry(key string, kind contracts.ArtifactKind) (typedNormalizerEntry, bool) {
@@ -229,10 +138,7 @@ func (r *NormalizerRegistry) RegisteredKeys() []string {
if r == nil { if r == nil {
return nil return nil
} }
keys := make(map[string]struct{}, len(r.legacySpecs)+len(r.typedEntries)) keys := make(map[string]struct{}, len(r.typedEntries))
for key := range r.legacySpecs {
keys[key] = struct{}{}
}
for key := range r.typedEntries { for key := range r.typedEntries {
keys[key.module] = struct{}{} keys[key.module] = struct{}{}
} }

View File

@@ -1,58 +0,0 @@
package pipeline
import (
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestNormalizerRegistryBehavior(t *testing.T) {
runRegistryBehaviorTests(t, registryBehaviorCase[contracts.LegacyRawNormalizer]{
name: "NormalizerRegistry",
key: "generic-normalizer",
stage: StageNormalize,
wrongStage: StageExtract,
newRegistry: func() any {
return NewNormalizerRegistry()
},
register: func(registry any, key string, constructor func() (contracts.LegacyRawNormalizer, error)) error {
return registry.(*NormalizerRegistry).RegisterLegacyRaw(key, constructor)
},
registerWithSpec: func(registry any, spec ModuleSpec, constructor func() (contracts.LegacyRawNormalizer, error)) error {
return registry.(*NormalizerRegistry).RegisterLegacyRawWithSpec(spec, constructor)
},
build: func(registry any, key string) (contracts.LegacyRawNormalizer, error) {
return registry.(*NormalizerRegistry).BuildLegacyRaw(key)
},
spec: func(registry any, key string) (ModuleSpec, bool) {
return registry.(*NormalizerRegistry).Spec(key)
},
registeredKeys: func(registry any) []string {
return registry.(*NormalizerRegistry).RegisteredKeys()
},
nilRegister: func(key string, constructor func() (contracts.LegacyRawNormalizer, error)) error {
var registry *NormalizerRegistry
return registry.RegisterLegacyRaw(key, constructor)
},
nilBuild: func(key string) (contracts.LegacyRawNormalizer, error) {
var registry *NormalizerRegistry
return registry.BuildLegacyRaw(key)
},
nilSpec: func(key string) (ModuleSpec, bool) {
var registry *NormalizerRegistry
return registry.Spec(key)
},
nilRegisteredKey: func() []string {
var registry *NormalizerRegistry
return registry.RegisteredKeys()
},
constructor: func(key string) func() (contracts.LegacyRawNormalizer, error) {
return func() (contracts.LegacyRawNormalizer, error) {
return registryNormalizer{key: key}, nil
}
},
moduleKey: func(module contracts.LegacyRawNormalizer) string {
return module.Key()
},
})
}

View File

@@ -32,7 +32,7 @@ func (r *OutputEncoderRegistry) RegisterWithSpec(spec ModuleSpec, constructor Ou
if constructor == nil { if constructor == nil {
return fmt.Errorf("output encoder constructor for %q must not be nil", strings.TrimSpace(spec.Key)) return fmt.Errorf("output encoder constructor for %q must not be nil", strings.TrimSpace(spec.Key))
} }
return r.RegisterBuilderWithSpec(spec, allowLegacyOptions, func(BuildRequest) (contracts.OutputEncoder, error) { return r.RegisterBuilderWithSpec(spec, rejectUnconfiguredOptions, func(BuildRequest) (contracts.OutputEncoder, error) {
return constructor() return constructor()
}) })
} }

View File

@@ -1,11 +1,13 @@
package pipeline package pipeline
import ( import (
"context"
"errors" "errors"
"reflect" "reflect"
"strings" "strings"
"testing" "testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
) )
@@ -143,7 +145,7 @@ func constructionProfile() PipelineProfile {
} }
} }
func constructionRegistries(t *testing.T, built *[]string, failure *constructionFailure) (Registries, *runnerInputAdapter) { func constructionRegistries(t *testing.T, built *[]string, failure *constructionFailure) (Registries, *constructionInput) {
t.Helper() t.Helper()
if built == nil { if built == nil {
built = &[]string{} built = &[]string{}
@@ -153,48 +155,64 @@ func constructionRegistries(t *testing.T, built *[]string, failure *construction
} }
record := func(name string) { *built = append(*built, name) } record := func(name string) { *built = append(*built, name) }
strict := func(options map[string]any) error { return RejectUnknownOptions(options, "known") } strict := func(options map[string]any) error { return RejectUnknownOptions(options, "known") }
modules := defaultRunnerModules() input := &constructionInput{key: "input"}
registries := Registries{ registries := Registries{
Inputs: NewInputAdapterRegistry(), Chunkers: NewChunkerRegistry(), ArtifactCodecs: NewArtifactCodecRegistry(), Inputs: NewInputAdapterRegistry(), Chunkers: NewChunkerRegistry(), ArtifactCodecs: NewArtifactCodecRegistry(),
Extractors: NewExtractorRegistry(), Mergers: NewMergerRegistry(), Normalizers: NewNormalizerRegistry(), Extractors: NewExtractorRegistry(), Mergers: NewMergerRegistry(), Normalizers: NewNormalizerRegistry(),
Validators: NewValidatorRegistry(), ValidatorChains: NewValidatorChainRegistry(), Outputs: NewOutputEncoderRegistry(), Validators: NewValidatorRegistry(), ValidatorChains: NewValidatorChainRegistry(), Outputs: NewOutputEncoderRegistry(),
} }
if err := RegisterArtifactCodec(registries.ArtifactCodecs, notesCodec()); err != nil {
t.Fatal(err)
}
if err := registries.Inputs.RegisterBuilderWithSpec(defaultModuleSpec("input", StageInput), strict, func(BuildRequest) (contracts.InputAdapter, error) { if err := registries.Inputs.RegisterBuilderWithSpec(defaultModuleSpec("input", StageInput), strict, func(BuildRequest) (contracts.InputAdapter, error) {
record("input") record("input")
return modules.input, nil return input, nil
}); err != nil { }); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if err := registries.Chunkers.RegisterBuilderWithSpec(defaultModuleSpec("chunk", StageChunk), strict, func(BuildRequest) (contracts.Chunker, error) { if err := registries.Chunkers.RegisterBuilderWithSpec(defaultModuleSpec("chunk", StageChunk), strict, func(BuildRequest) (contracts.Chunker, error) {
record("chunk") record("chunk")
return modules.chunker, nil return &typedTestChunker{key: "chunk"}, nil
}); err != nil { }); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if err := registries.Extractors.RegisterLegacyRawBuilderWithSpec(defaultModuleSpec("extract", StageExtract), strict, func(request BuildRequest) (contracts.LegacyRawExtractor, error) { extractSpec := defaultModuleSpec("extract", StageExtract)
extractSpec.ArtifactKind = "test/notes"
if err := RegisterExtractorBuilder(registries.Extractors, extractSpec, strict, func(request BuildRequest) (contracts.Extractor[codecNotes], error) {
record("extract") record("extract")
if failure.requireExtractorLLM && request.Dependencies.LLM == nil { if failure.requireExtractorLLM && request.Dependencies.LLM == nil {
return nil, errors.New("structured LLM client is required") return nil, errors.New("structured LLM client is required")
} }
return &runnerExtractor{key: "extract"}, nil return typedTestExtractor[codecNotes]{key: "extract"}, nil
}); err != nil { }); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if err := registries.Mergers.RegisterLegacyRawBuilderWithSpec(defaultModuleSpec("merge", StageMerge), strict, func(BuildRequest) (contracts.LegacyRawMerger, error) { mergeSpec := defaultModuleSpec("merge", StageMerge)
mergeSpec.ArtifactKind = "test/notes"
if err := RegisterMergerBuilder(registries.Mergers, mergeSpec, strict, func(BuildRequest) (contracts.Merger[codecNotes], error) {
record("merge") record("merge")
return modules.mergers["merge"], nil return typedTestMerger[codecNotes]{key: "merge"}, nil
}); err != nil { }); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if err := registries.Normalizers.RegisterLegacyRawBuilderWithSpec(defaultModuleSpec("normalize", StageNormalize), strict, func(BuildRequest) (contracts.LegacyRawNormalizer, error) { normalizeSpec := defaultModuleSpec("normalize", StageNormalize)
normalizeSpec.ArtifactKind = "test/notes"
if err := RegisterNormalizerBuilder(registries.Normalizers, normalizeSpec, strict, func(BuildRequest) (contracts.Normalizer[codecNotes], error) {
record("normalize") record("normalize")
return modules.normalizers["normalize"], nil return typedTestNormalizer[codecNotes]{key: "normalize"}, nil
}); err != nil { }); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if err := registries.Validators.RegisterLegacyRawBuilderWithSpec(ValidatorSpec{Key: "configured", ExecutionClass: contracts.ExecutionClassDeterministic}, strict, func(BuildRequest) (contracts.LegacyRawValidator, error) { validatorSpec := ValidatorSpec{Key: "configured", ExecutionClass: contracts.ExecutionClassDeterministic}
if err := RegisterChunkValidatorBuilder(registries.Validators, validatorSpec, strict, func(BuildRequest) (contracts.ChunkValidator, error) {
record("validator") record("validator")
return modules.validators["configured"], nil return typedTestChunkValidator{key: "configured"}, nil
}); err != nil {
t.Fatal(err)
}
if err := RegisterTypedValidatorBuilder(registries.Validators, "test/notes", validatorSpec, strict, func(BuildRequest) (contracts.TypedValidator[codecNotes], error) {
record("validator")
return typedTestValidator[codecNotes]{key: "configured"}, nil
}); err != nil { }); err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -203,9 +221,20 @@ func constructionRegistries(t *testing.T, built *[]string, failure *construction
if failure.output != nil { if failure.output != nil {
return nil, failure.output return nil, failure.output
} }
return modules.output, nil return &typedTestOutput{key: "output"}, nil
}); err != nil { }); err != nil {
t.Fatal(err) t.Fatal(err)
} }
return registries, modules.input return registries, input
}
type constructionInput struct {
key string
requests []contracts.ParseRequest
}
func (input *constructionInput) Key() string { return input.key }
func (input *constructionInput) Parse(_ context.Context, request contracts.ParseRequest) (*source.SourceDocument, error) {
input.requests = append(input.requests, request)
return typedTestDocument(), nil
} }

View File

@@ -32,19 +32,12 @@ type PreparedArtifactLane struct {
type preparedLaneExecutor struct { type preparedLaneExecutor struct {
resolved ResolvedArtifactLane resolved ResolvedArtifactLane
legacy *preparedLegacyLane
typed *preparedTypedLane typed *preparedTypedLane
extractValidators preparedValidatorChain extractValidators preparedValidatorChain
mergeValidators preparedValidatorChain mergeValidators preparedValidatorChain
normalizeValidators preparedValidatorChain normalizeValidators preparedValidatorChain
} }
type preparedLegacyLane struct {
extractor contracts.LegacyRawExtractor
merger contracts.LegacyRawMerger
normalizer contracts.LegacyRawNormalizer
}
type preparedTypedLane struct { type preparedTypedLane struct {
extractor any extractor any
merger any merger any
@@ -62,7 +55,6 @@ type preparedValidatorChain struct {
type preparedValidator struct { type preparedValidator struct {
resolved ResolvedValidator resolved ResolvedValidator
legacy contracts.LegacyRawValidator
typed any typed any
typedValidate typedValidateOperation typedValidate typedValidateOperation
chunk contracts.ChunkValidator chunk contracts.ChunkValidator
@@ -130,75 +122,50 @@ func prepareLane(pipeline ResolvedPipeline, lane ResolvedArtifactLane, registrie
request := func(binding ModuleBinding) BuildRequest { request := func(binding ModuleBinding) BuildRequest {
return BuildRequest{Dependencies: deps, Options: cloneOptions(binding.Options)} return BuildRequest{Dependencies: deps, Options: cloneOptions(binding.Options)}
} }
if lane.ArtifactKind == "" { extractEntry, ok := registries.Extractors.typedEntry(lane.Extract.Module)
extractor, err := registries.Extractors.BuildLegacyRawWithRequest(lane.Extract.Module, request(lane.Extract)) if !ok {
if err != nil { return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageExtract, lane.Extract.Module, "", fmt.Errorf("typed construction entry is not registered"))
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageExtract, lane.Extract.Module, "", err)
}
executor.legacy = &preparedLegacyLane{extractor: extractor}
} else {
entry, ok := registries.Extractors.typedEntry(lane.Extract.Module)
if !ok {
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageExtract, lane.Extract.Module, "", fmt.Errorf("typed construction entry is not registered"))
}
module, err := buildErasedModule(entry.builder, request(lane.Extract), lane.Extract.Module, "extractor")
if err != nil {
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageExtract, lane.Extract.Module, "", err)
}
codec, _, codecErr := registries.ArtifactCodecs.entry(lane.ArtifactKind)
if codecErr != nil {
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageExtract, lane.Extract.Module, "", codecErr)
}
executor.typed = &preparedTypedLane{extractor: module, extract: entry.extract, codec: codec}
} }
module, err := buildErasedModule(extractEntry.builder, request(lane.Extract), lane.Extract.Module, "extractor")
if err != nil {
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageExtract, lane.Extract.Module, "", err)
}
codec, _, codecErr := registries.ArtifactCodecs.entry(lane.ArtifactKind)
if codecErr != nil {
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageExtract, lane.Extract.Module, "", codecErr)
}
executor.typed = &preparedTypedLane{extractor: module, extract: extractEntry.extract, codec: codec}
var err error
executor.extractValidators, err = prepareValidatorChain(pipeline, registries, deps, StageExtract, lane.ID, lane.Extract.Module) executor.extractValidators, err = prepareValidatorChain(pipeline, registries, deps, StageExtract, lane.ID, lane.Extract.Module)
if err != nil { if err != nil {
return preparedLaneExecutor{}, err return preparedLaneExecutor{}, err
} }
if lane.ArtifactKind == "" { mergeEntry, ok := registries.Mergers.typedEntry(lane.Merge.Module, lane.ArtifactKind)
module, err := registries.Mergers.BuildLegacyRawWithRequest(lane.Merge.Module, request(lane.Merge)) if !ok {
if err != nil { return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageMerge, lane.Merge.Module, "", fmt.Errorf("typed construction entry is not registered"))
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageMerge, lane.Merge.Module, "", err)
}
executor.legacy.merger = module
} else {
entry, ok := registries.Mergers.typedEntry(lane.Merge.Module, lane.ArtifactKind)
if !ok {
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageMerge, lane.Merge.Module, "", fmt.Errorf("typed construction entry is not registered"))
}
module, err := buildErasedModule(entry.builder, request(lane.Merge), lane.Merge.Module, "merger")
if err != nil {
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageMerge, lane.Merge.Module, "", err)
}
executor.typed.merger = module
executor.typed.merge = entry.merge
} }
module, err = buildErasedModule(mergeEntry.builder, request(lane.Merge), lane.Merge.Module, "merger")
if err != nil {
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageMerge, lane.Merge.Module, "", err)
}
executor.typed.merger = module
executor.typed.merge = mergeEntry.merge
executor.mergeValidators, err = prepareValidatorChain(pipeline, registries, deps, StageMerge, lane.ID, lane.Merge.Module) executor.mergeValidators, err = prepareValidatorChain(pipeline, registries, deps, StageMerge, lane.ID, lane.Merge.Module)
if err != nil { if err != nil {
return preparedLaneExecutor{}, err return preparedLaneExecutor{}, err
} }
if lane.ArtifactKind == "" { normalizeEntry, ok := registries.Normalizers.typedEntry(lane.Normalize.Module, lane.ArtifactKind)
module, err := registries.Normalizers.BuildLegacyRawWithRequest(lane.Normalize.Module, request(lane.Normalize)) if !ok {
if err != nil { return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageNormalize, lane.Normalize.Module, "", fmt.Errorf("typed construction entry is not registered"))
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageNormalize, lane.Normalize.Module, "", err)
}
executor.legacy.normalizer = module
} else {
entry, ok := registries.Normalizers.typedEntry(lane.Normalize.Module, lane.ArtifactKind)
if !ok {
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageNormalize, lane.Normalize.Module, "", fmt.Errorf("typed construction entry is not registered"))
}
module, err := buildErasedModule(entry.builder, request(lane.Normalize), lane.Normalize.Module, "normalizer")
if err != nil {
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageNormalize, lane.Normalize.Module, "", err)
}
executor.typed.normalizer = module
executor.typed.normalize = entry.normalize
} }
module, err = buildErasedModule(normalizeEntry.builder, request(lane.Normalize), lane.Normalize.Module, "normalizer")
if err != nil {
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageNormalize, lane.Normalize.Module, "", err)
}
executor.typed.normalizer = module
executor.typed.normalize = normalizeEntry.normalize
executor.normalizeValidators, err = prepareValidatorChain(pipeline, registries, deps, StageNormalize, lane.ID, lane.Normalize.Module) executor.normalizeValidators, err = prepareValidatorChain(pipeline, registries, deps, StageNormalize, lane.ID, lane.Normalize.Module)
if err != nil { if err != nil {
return preparedLaneExecutor{}, err return preparedLaneExecutor{}, err
@@ -249,8 +216,7 @@ func buildPreparedValidator(registry *ValidatorRegistry, resolved ResolvedValida
prepared.serialized, err = entry.builder(cloneBuildRequest(request)) prepared.serialized, err = entry.builder(cloneBuildRequest(request))
implementation = prepared.serialized implementation = prepared.serialized
default: default:
prepared.legacy, err = registry.BuildLegacyRawWithRequest(key, request) return preparedValidator{}, fmt.Errorf("validator construction target %q is not supported", resolved.Target)
implementation = prepared.legacy
} }
if err != nil { if err != nil {
return preparedValidator{}, err return preparedValidator{}, err

View File

@@ -384,7 +384,7 @@ func configuredValidatorsError(pipelineID string, laneID string) error {
func resolveArtifactIdentity(pipelineID, laneID string, lane *ResolvedArtifactLane, extractSpec ModuleSpec, catalog ModuleCatalog) (reflect.Type, error) { func resolveArtifactIdentity(pipelineID, laneID string, lane *ResolvedArtifactLane, extractSpec ModuleSpec, catalog ModuleCatalog) (reflect.Type, error) {
if extractSpec.ArtifactKind == "" { if extractSpec.ArtifactKind == "" {
return nil, nil return nil, fmt.Errorf("pipeline %q lane %q extract module %q does not declare an artifact kind", pipelineID, laneID, lane.Extract.Module)
} }
if catalog.Extractors == nil { if catalog.Extractors == nil {
return nil, fmt.Errorf("pipeline %q lane %q extractor registry must not be nil", pipelineID, laneID) return nil, fmt.Errorf("pipeline %q lane %q extractor registry must not be nil", pipelineID, laneID)
@@ -393,9 +393,6 @@ func resolveArtifactIdentity(pipelineID, laneID string, lane *ResolvedArtifactLa
if !ok { if !ok {
return nil, fmt.Errorf("pipeline %q lane %q extract module %q declares artifact kind %q without a typed registration", pipelineID, laneID, lane.Extract.Module, extractSpec.ArtifactKind) return nil, fmt.Errorf("pipeline %q lane %q extract module %q declares artifact kind %q without a typed registration", pipelineID, laneID, lane.Extract.Module, extractSpec.ArtifactKind)
} }
if catalog.Extractors.usesRawAdapter(lane.Extract.Module) {
return nil, nil
}
if catalog.ArtifactCodecs == nil { if catalog.ArtifactCodecs == nil {
return nil, fmt.Errorf("pipeline %q lane %q artifact codec registry must not be nil for kind %q", pipelineID, laneID, extractSpec.ArtifactKind) return nil, fmt.Errorf("pipeline %q lane %q artifact codec registry must not be nil for kind %q", pipelineID, laneID, extractSpec.ArtifactKind)
} }
@@ -470,7 +467,7 @@ func validatorSpecForTarget(registry *ValidatorRegistry, stage ModuleStage, key
if spec, ok := registry.Spec(key); ok { if spec, ok := registry.Spec(key); ok {
return spec, "", nil return spec, "", nil
} }
return ValidatorSpec{}, "", fmt.Errorf("references unknown validator %q on legacy raw path", key) return ValidatorSpec{}, "", fmt.Errorf("references unknown validator %q without an artifact kind", key)
} }
if entry, ok := registry.typedEntry(key, kind); ok { if entry, ok := registry.typedEntry(key, kind); ok {
if entry.valueType != expectedType { if entry.valueType != expectedType {
@@ -646,9 +643,6 @@ func validatePipelineReferenceDefaults(
merge := resolveBinding(laneProfile.Merge, DefaultMergeModule) merge := resolveBinding(laneProfile.Merge, DefaultMergeModule)
var artifactType reflect.Type var artifactType reflect.Type
artifactKind := extractSpec.ArtifactKind artifactKind := extractSpec.ArtifactKind
if catalog.Extractors != nil && catalog.Extractors.usesRawAdapter(extract.Module) {
artifactKind = ""
}
if artifactKind != "" && catalog.Extractors != nil { if artifactKind != "" && catalog.Extractors != nil {
if entry, ok := catalog.Extractors.typedEntry(extract.Module); ok { if entry, ok := catalog.Extractors.typedEntry(extract.Module); ok {
artifactType = entry.valueType artifactType = entry.valueType

View File

@@ -766,20 +766,22 @@ func TestResolvePipelineUsesReferenceSlotsFromSpecWithoutConstructingExtractor(t
profile := baselineProfile() profile := baselineProfile()
profile.References = map[string]string{"roster": "./roster.yml"} profile.References = map[string]string{"roster": "./roster.yml"}
catalog := emptyProfileCatalog() catalog := emptyProfileCatalog()
mustRegisterArtifactCodec(t, catalog.ArtifactCodecs, notesCodec())
for _, spec := range defaultProfileSpecs() { for _, spec := range defaultProfileSpecs() {
if spec.Key != "event-extractor" { if spec.Key != "event-extractor" {
registerProfileSpecs(t, catalog, spec) registerProfileSpecs(t, catalog, spec)
} }
} }
if err := catalog.Extractors.RegisterLegacyRawWithSpec(ModuleSpec{ if err := RegisterExtractor[codecNotes](catalog.Extractors, ModuleSpec{
Key: "event-extractor", Key: "event-extractor",
Stage: StageExtract, Stage: StageExtract,
Requires: []string{"chunk"}, ArtifactKind: "test/notes",
Provides: []string{"candidate"}, Requires: []string{"chunk"},
Provides: []string{"candidate"},
ReferenceSlots: []contracts.ReferenceSlot{ ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster", Required: true}, {Name: "roster", Required: true},
}, },
}, func() (contracts.LegacyRawExtractor, error) { }, func() (contracts.Extractor[codecNotes], error) {
return nil, errors.New("constructor should not run") return nil, errors.New("constructor should not run")
}); err != nil { }); err != nil {
t.Fatalf("RegisterWithSpec() error = %v, want nil", err) t.Fatalf("RegisterWithSpec() error = %v, want nil", err)
@@ -1177,6 +1179,7 @@ func newProfileCatalog(t *testing.T) ModuleCatalog {
t.Helper() t.Helper()
catalog := emptyProfileCatalog() catalog := emptyProfileCatalog()
mustRegisterArtifactCodec(t, catalog.ArtifactCodecs, notesCodec())
registerProfileSpecs(t, catalog, defaultProfileSpecs()...) registerProfileSpecs(t, catalog, defaultProfileSpecs()...)
return catalog return catalog
} }
@@ -1192,6 +1195,9 @@ func newProfileCatalogWithOverrides(t *testing.T, overrides ...ModuleSpec) Modul
specs := defaultProfileSpecs() specs := defaultProfileSpecs()
for _, override := range overrides { for _, override := range overrides {
if override.ArtifactKind == "" && (override.Stage == StageExtract || override.Stage == StageMerge || override.Stage == StageNormalize) {
override.ArtifactKind = "test/notes"
}
replaced := false replaced := false
for index, spec := range specs { for index, spec := range specs {
if spec.Stage == override.Stage && spec.Key == override.Key { if spec.Stage == override.Stage && spec.Key == override.Key {
@@ -1206,6 +1212,7 @@ func newProfileCatalogWithOverrides(t *testing.T, overrides ...ModuleSpec) Modul
} }
catalog := emptyProfileCatalog() catalog := emptyProfileCatalog()
mustRegisterArtifactCodec(t, catalog.ArtifactCodecs, notesCodec())
registerProfileSpecs(t, catalog, specs...) registerProfileSpecs(t, catalog, specs...)
return catalog return catalog
} }
@@ -1228,10 +1235,10 @@ func defaultProfileSpecs() []ModuleSpec {
return []ModuleSpec{ return []ModuleSpec{
ModuleSpec{Key: "text", Stage: StageInput, Provides: []string{"source"}}, ModuleSpec{Key: "text", Stage: StageInput, Provides: []string{"source"}},
ModuleSpec{Key: "generic", Stage: StageChunk, Requires: []string{"source"}, Provides: []string{"chunk"}}, ModuleSpec{Key: "generic", Stage: StageChunk, Requires: []string{"source"}, Provides: []string{"chunk"}},
ModuleSpec{Key: "event-extractor", Stage: StageExtract, Requires: []string{"chunk"}, Provides: []string{"candidate"}}, ModuleSpec{Key: "event-extractor", Stage: StageExtract, ArtifactKind: "test/notes", Requires: []string{"chunk"}, Provides: []string{"candidate"}},
ModuleSpec{Key: "note-extractor", Stage: StageExtract, Requires: []string{"chunk"}, Provides: []string{"candidate"}}, ModuleSpec{Key: "note-extractor", Stage: StageExtract, ArtifactKind: "test/notes", Requires: []string{"chunk"}, Provides: []string{"candidate"}},
ModuleSpec{Key: "appendorder", Stage: StageMerge, Requires: []string{"candidate"}, Provides: []string{"merged"}}, ModuleSpec{Key: "appendorder", Stage: StageMerge, ArtifactKind: "test/notes", Requires: []string{"candidate"}, Provides: []string{"merged"}},
ModuleSpec{Key: "noop", Stage: StageNormalize, Requires: []string{"merged"}, Provides: []string{"normalized"}}, ModuleSpec{Key: "noop", Stage: StageNormalize, ArtifactKind: "test/notes", Requires: []string{"merged"}, Provides: []string{"normalized"}},
ModuleSpec{Key: "grounded", Stage: StageValidate, Requires: []string{"normalized"}, Provides: []string{"validated"}}, ModuleSpec{Key: "grounded", Stage: StageValidate, Requires: []string{"normalized"}, Provides: []string{"validated"}},
ModuleSpec{Key: "json", Stage: StageOutput, Requires: []string{"normalized"}, Provides: []string{"encoded"}}, ModuleSpec{Key: "json", Stage: StageOutput, Requires: []string{"normalized"}, Provides: []string{"encoded"}},
} }
@@ -1241,30 +1248,40 @@ func registerProfileSpecs(t *testing.T, catalog ModuleCatalog, specs ...ModuleSp
t.Helper() t.Helper()
for _, spec := range specs { for _, spec := range specs {
if spec.ArtifactKind == "" && (spec.Stage == StageExtract || spec.Stage == StageMerge || spec.Stage == StageNormalize) {
spec.ArtifactKind = "test/notes"
}
switch spec.Stage { switch spec.Stage {
case StageInput: case StageInput:
if err := catalog.Inputs.RegisterWithSpec(spec, profileInputConstructor(spec.Key)); err != nil { if err := catalog.Inputs.RegisterWithSpec(spec, profileInputConstructor(spec.Key)); err != nil {
t.Fatalf("register input spec %#v: %v", spec, err) t.Fatalf("register input spec %#v: %v", spec, err)
} }
case StageChunk: case StageChunk:
if err := catalog.Chunkers.RegisterWithSpec(spec, profileChunkerConstructor(spec.Key)); err != nil { validateOptions := func(options map[string]any) error { return RejectUnknownOptions(options, "a", "b", "size") }
if err := catalog.Chunkers.RegisterBuilderWithSpec(spec, validateOptions, func(BuildRequest) (contracts.Chunker, error) { return &typedTestChunker{key: spec.Key}, nil }); err != nil {
t.Fatalf("register chunk spec %#v: %v", spec, err) t.Fatalf("register chunk spec %#v: %v", spec, err)
} }
case StageExtract: case StageExtract:
if err := catalog.Extractors.RegisterLegacyRawWithSpec(spec, profileExtractorConstructor(spec.Key)); err != nil { if err := RegisterExtractor(catalog.Extractors, spec, func() (contracts.Extractor[codecNotes], error) {
return typedTestExtractor[codecNotes]{key: spec.Key}, nil
}); err != nil {
t.Fatalf("register extractor spec %#v: %v", spec, err) t.Fatalf("register extractor spec %#v: %v", spec, err)
} }
case StageMerge: case StageMerge:
if err := catalog.Mergers.RegisterLegacyRawWithSpec(spec, profileMergerConstructor(spec.Key)); err != nil { if err := RegisterMerger(catalog.Mergers, spec, func() (contracts.Merger[codecNotes], error) { return typedTestMerger[codecNotes]{key: spec.Key}, nil }); err != nil {
t.Fatalf("register merger spec %#v: %v", spec, err) t.Fatalf("register merger spec %#v: %v", spec, err)
} }
case StageNormalize: case StageNormalize:
if err := catalog.Normalizers.RegisterLegacyRawWithSpec(spec, profileNormalizerConstructor(spec.Key)); err != nil { if err := RegisterNormalizer(catalog.Normalizers, spec, func() (contracts.Normalizer[codecNotes], error) {
return typedTestNormalizer[codecNotes]{key: spec.Key}, nil
}); err != nil {
t.Fatalf("register normalizer spec %#v: %v", spec, err) t.Fatalf("register normalizer spec %#v: %v", spec, err)
} }
case StageValidate: case StageValidate:
validatorSpec := ValidatorSpec{Key: spec.Key, ExecutionClass: contracts.ExecutionClassDeterministic} validatorSpec := ValidatorSpec{Key: spec.Key, ExecutionClass: contracts.ExecutionClassDeterministic}
if err := catalog.Validators.RegisterLegacyRawWithSpec(validatorSpec, profileValidatorConstructor(spec.Key)); err != nil { if err := RegisterTypedValidator(catalog.Validators, "test/notes", validatorSpec, func() (contracts.TypedValidator[codecNotes], error) {
return typedTestValidator[codecNotes]{key: spec.Key}, nil
}); err != nil {
t.Fatalf("register validator spec %#v: %v", spec, err) t.Fatalf("register validator spec %#v: %v", spec, err)
} }
case StageOutput: case StageOutput:
@@ -1279,7 +1296,9 @@ func registerProfileSpecs(t *testing.T, catalog ModuleCatalog, specs ...ModuleSp
func registerProfileValidatorSpec(t *testing.T, catalog ModuleCatalog, spec ValidatorSpec) { func registerProfileValidatorSpec(t *testing.T, catalog ModuleCatalog, spec ValidatorSpec) {
t.Helper() t.Helper()
if err := catalog.Validators.RegisterLegacyRawWithSpec(spec, profileValidatorConstructor(spec.Key)); err != nil { if err := RegisterTypedValidator(catalog.Validators, "test/notes", spec, func() (contracts.TypedValidator[codecNotes], error) {
return typedTestValidator[codecNotes]{key: spec.Key}, nil
}); err != nil {
t.Fatalf("register validator spec %#v: %v", spec, err) t.Fatalf("register validator spec %#v: %v", spec, err)
} }
} }
@@ -1302,36 +1321,6 @@ func (adapter profileInputAdapter) Parse(ctx context.Context, req contracts.Pars
return &source.SourceDocument{}, nil return &source.SourceDocument{}, nil
} }
func profileChunkerConstructor(key string) ChunkerConstructor {
return func() (contracts.Chunker, error) {
return registryChunker{key: key}, nil
}
}
func profileExtractorConstructor(key string) LegacyRawExtractorConstructor {
return func() (contracts.LegacyRawExtractor, error) {
return registryFakeExtractor{key: key}, nil
}
}
func profileMergerConstructor(key string) LegacyRawMergerConstructor {
return func() (contracts.LegacyRawMerger, error) {
return registryMerger{key: key}, nil
}
}
func profileNormalizerConstructor(key string) LegacyRawNormalizerConstructor {
return func() (contracts.LegacyRawNormalizer, error) {
return registryNormalizer{key: key}, nil
}
}
func profileValidatorConstructor(key string) LegacyRawValidatorConstructor {
return func() (contracts.LegacyRawValidator, error) {
return registryValidator{name: key}, nil
}
}
func profileOutputConstructor(key string) OutputEncoderConstructor { func profileOutputConstructor(key string) OutputEncoderConstructor {
return func() (contracts.OutputEncoder, error) { return func() (contracts.OutputEncoder, error) {
return registryOutputEncoder{key: key}, nil return registryOutputEncoder{key: key}, nil

View File

@@ -0,0 +1,14 @@
package pipeline
import "gitea.maximumdirect.net/eric/notarius/internal/core/source"
func validSourceDocument() *source.SourceDocument {
return &source.SourceDocument{
ID: "source-1", Kind: "document", Format: "text/plain", Digest: "sha256:source",
Units: []source.SourceUnit{
{ID: 1, Kind: "unit", Text: "Source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}},
{ID: 2, Kind: "unit", Text: "Second source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 2, EndUnitID: 2}},
{ID: 3, Kind: "unit", Text: "Third source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 3, EndUnitID: 3}},
},
}
}

View File

@@ -1,265 +0,0 @@
package pipeline
import (
"context"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestRunnerUsesRegistries(t *testing.T) {
var built []string
var executed []string
registries := integrationRegistries(t, &built, &executed)
output, err := newPreparedRunner(t, registries).Run(context.Background(), RunInput{
pipeline: integrationPipeline(),
SourceID: "source-1",
RawInput: []byte("source text"),
})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
wantBuilt := []string{"input", "chunk", "extract-first", "merge", "normalize", "extract-second", "merge", "normalize", "output"}
if !reflect.DeepEqual(built, wantBuilt) {
t.Fatalf("built = %#v, want %#v", built, wantBuilt)
}
if !reflect.DeepEqual(executed, []string{"extract-first:chunk-0", "extract-second:chunk-0"}) {
t.Fatalf("executed = %#v, want extractor chunk execution", executed)
}
if got := normalizeOutputKeys(output.NormalizeOutputs); !reflect.DeepEqual(got, []string{"normalize", "normalize"}) {
t.Fatalf("normalize output keys = %#v, want one output from each lane", got)
}
if len(output.Rejected) != 0 {
t.Fatalf("len(Rejected) = %d, want none", len(output.Rejected))
}
}
func integrationRegistries(t *testing.T, built, executed *[]string) Registries {
t.Helper()
registries := Registries{
Inputs: NewInputAdapterRegistry(),
Chunkers: NewChunkerRegistry(),
ArtifactCodecs: NewArtifactCodecRegistry(),
Extractors: NewExtractorRegistry(),
Mergers: NewMergerRegistry(),
Normalizers: NewNormalizerRegistry(),
Outputs: NewOutputEncoderRegistry(),
}
if err := registries.Inputs.Register("input", func() (contracts.InputAdapter, error) {
*built = append(*built, "input")
return integrationInput{}, nil
}); err != nil {
t.Fatalf("register input: %v", err)
}
if err := registries.Chunkers.Register("chunk", func() (contracts.Chunker, error) {
*built = append(*built, "chunk")
return integrationChunker{}, nil
}); err != nil {
t.Fatalf("register chunker: %v", err)
}
registerIntegrationExtractor(t, registries.Extractors, "extract-first", built, executed)
registerIntegrationExtractor(t, registries.Extractors, "extract-second", built, executed)
if err := registries.Mergers.RegisterLegacyRaw("merge", func() (contracts.LegacyRawMerger, error) {
*built = append(*built, "merge")
return integrationMerger{}, nil
}); err != nil {
t.Fatalf("register merger: %v", err)
}
if err := registries.Normalizers.RegisterLegacyRaw("normalize", func() (contracts.LegacyRawNormalizer, error) {
*built = append(*built, "normalize")
return integrationNormalizer{}, nil
}); err != nil {
t.Fatalf("register normalizer: %v", err)
}
if err := registries.Outputs.Register("output", func() (contracts.OutputEncoder, error) {
*built = append(*built, "output")
return integrationOutput{}, nil
}); err != nil {
t.Fatalf("register output: %v", err)
}
return registries
}
func registerIntegrationExtractor(t *testing.T, registry *ExtractorRegistry, key string, built, executed *[]string) {
t.Helper()
if err := registry.RegisterLegacyRaw(key, func() (contracts.LegacyRawExtractor, error) {
*built = append(*built, key)
return integrationExtractor{key: key, executed: executed}, nil
}); err != nil {
t.Fatalf("Register(%q) error = %v, want nil", key, err)
}
}
type integrationInput struct{}
func (input integrationInput) Key() string {
return "input"
}
func (input integrationInput) Parse(ctx context.Context, req contracts.ParseRequest) (*source.SourceDocument, error) {
return integrationSourceDocument(), nil
}
type integrationChunker struct{}
func (chunker integrationChunker) Key() string {
return "chunk"
}
func (chunker integrationChunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (chunker integrationChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
return contracts.ChunkResult{
Chunks: []source.Chunk{
{
ID: "chunk-0",
SourceID: req.Source.ID,
Index: 0,
Ref: source.SourceRef{
SourceID: req.Source.ID,
StartUnitID: req.Source.Units[0].ID,
EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID,
},
Content: []byte(`{"units":[1]}`),
MediaType: "application/json",
Units: req.Source.Units,
},
},
}, nil
}
type integrationExtractor struct {
key string
executed *[]string
}
func (extractor integrationExtractor) Key() string {
return extractor.key
}
func (extractor integrationExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (extractor integrationExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
*extractor.executed = append(*extractor.executed, extractor.key+":"+req.Chunk.ID)
return contracts.ExtractionResult{
Output: contracts.ExtractOutput{
Schema: contracts.ResponseSchema{ID: "integration", Name: "integration", Version: "v1"},
Payload: contracts.RawPayload{
Content: []byte(`{"value":true}`),
MediaType: "application/json",
},
},
}, nil
}
type integrationNormalizer struct{}
type integrationMerger struct{}
func (merger integrationMerger) Key() string {
return "merge"
}
func (merger integrationMerger) Merge(ctx context.Context, req contracts.MergeRequest) (contracts.MergeResult, error) {
output := contracts.MergeOutput{
LaneID: req.LaneID,
Schema: contracts.ResponseSchema{ID: "integration", Name: "integration", Version: "v1"},
Payload: contracts.RawPayload{
Content: []byte(`{"merged":true}`),
MediaType: "application/json",
},
}
if len(req.ExtractOutputs) > 0 {
output.SourceID = req.ExtractOutputs[0].SourceID
output.Schema = req.ExtractOutputs[0].Schema
output.Payload = req.ExtractOutputs[0].Payload
}
return contracts.MergeResult{Output: output}, nil
}
func (normalizer integrationNormalizer) Key() string {
return "normalize"
}
func (normalizer integrationNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (normalizer integrationNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
return contracts.NormalizeResult{
Output: contracts.NormalizeOutput{
LaneID: req.LaneID,
SourceID: req.MergeOutput.SourceID,
Schema: req.MergeOutput.Schema,
Payload: req.MergeOutput.Payload,
},
}, nil
}
type integrationOutput struct{}
func (output integrationOutput) Key() string {
return "output"
}
func (output integrationOutput) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
return contracts.OutputResult{
Files: []contracts.OutputFile{
{Name: "output.json", ContentType: "application/json", Bytes: []byte(`{}`)},
},
}, nil
}
func integrationPipeline() ResolvedPipeline {
return ResolvedPipeline{
ID: "pipeline-1",
Digest: "sha256:pipeline",
Input: Binding("input"),
Chunk: Binding("chunk"),
ArtifactLanes: []ResolvedArtifactLane{
{
ID: "first",
Extract: Binding("extract-first"),
Merge: Binding("merge"),
Normalize: Binding("normalize"),
},
{
ID: "second",
Extract: Binding("extract-second"),
Merge: Binding("merge"),
Normalize: Binding("normalize"),
},
},
Output: Binding("output"),
}
}
func integrationSourceDocument() *source.SourceDocument {
return &source.SourceDocument{
ID: "source-1",
Kind: "document",
Format: "text/plain",
Digest: "sha256:abc123",
Units: []source.SourceUnit{
{ID: 1, Kind: "unit", Text: "Source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}},
},
}
}
func normalizeOutputKeys(outputs []contracts.SerializedOutput) []string {
keys := make([]string, 0, len(outputs))
for _, output := range outputs {
keys = append(keys, output.NormalizerKey)
}
return keys
}

View File

@@ -245,7 +245,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
}, llmScope)) }, llmScope))
return false, nil, err return false, nil, err
} }
validationWarnings, rejection, err := r.validateChunks(attemptCtx, doc, chunker.Key(), chunks, sourceInput, sessionID, input.pipeline.ChunkReferences.ReferenceSet, input.llmClient, input.Metadata, input.Prepared.chunkValidators, attempt, input.Debug) validationWarnings, rejection, err := r.validateChunks(attemptCtx, doc, chunker.Key(), chunks, sourceInput, sessionID, input.pipeline.ChunkReferences.ReferenceSet, input.Metadata, input.Prepared.chunkValidators, attempt, input.Debug)
if err != nil || rejection != nil { if err != nil || rejection != nil {
_ = writeDebugTimed(debugRecorder, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{ _ = writeDebugTimed(debugRecorder, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(StageChunk), Stage: string(StageChunk),
@@ -323,7 +323,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
} else { } else {
output.Manifest.ValidationStatus = "approved" output.Manifest.ValidationStatus = "approved"
} }
populateRawOutputManifest(&output) populateOutputManifest(&output)
output.Manifest.CompletedAt = timePtr(time.Now().UTC()) output.Manifest.CompletedAt = timePtr(time.Now().UTC())
encoder := input.Prepared.output encoder := input.Prepared.output
@@ -377,534 +377,16 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
} }
func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, checkpointLoader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, prepared preparedLaneExecutor, output *RunOutput) error { func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, checkpointLoader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, prepared preparedLaneExecutor, output *RunOutput) error {
if prepared.typed != nil { return r.runTypedLane(ctx, input, checkpoints, checkpointLoader, doc, sourceInput, sessionID, chunks, prepared, output)
return r.runTypedLane(ctx, input, checkpoints, checkpointLoader, doc, sourceInput, sessionID, chunks, prepared, output)
}
return r.runLegacyLane(ctx, input, checkpoints, checkpointLoader, doc, sourceInput, sessionID, chunks, prepared, output)
}
func (r *Runner) runLegacyLane(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, checkpointLoader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, prepared preparedLaneExecutor, output *RunOutput) error {
lane := prepared.resolved
if prepared.legacy == nil {
return fmt.Errorf("resolved pipeline lane %q uses typed artifact kind %q, which the legacy raw runner cannot execute", lane.ID, lane.ArtifactKind)
}
extractor := prepared.legacy.extractor
merger := prepared.legacy.merger
normalizer := prepared.legacy.normalizer
setLaneManifestMetadata(output, lane.ID, extractor, merger, normalizer)
extractOutputs := make([]contracts.ExtractOutput, 0, len(chunks))
extractWarnings := []contracts.Warning{}
extractRejectedStart := len(output.Rejected)
chunksDigest, err := joinedChunkDigest(chunks)
if err != nil {
return fmt.Errorf("digest chunks for lane %q: %w", lane.ID, err)
}
extractDependencies := digestFingerprints("chunks", chunksDigest)
extractCheckpoint, extractDecision := checkpointLoader.Extract(lane.ID, extractor.Key(), extractDependencies)
recordCheckpointEvent(output, checkpointLoader, string(StageExtract), lane.ID, extractor.Key(), extractDecision)
extractStarted := time.Now().UTC()
if err := writeDebugTimed(input.Debug, path.Join("extract", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{
Stage: string(StageExtract),
LaneID: lane.ID,
ModuleKey: extractor.Key(),
StartedAt: extractStarted,
Payload: map[string]any{
"reused": extractDecision.Reused,
"decision": extractDecision,
"source": debugSourceDocumentEnvelope(doc),
"chunks": debugSourceChunkEnvelopes(chunks),
"options": redactSensitiveMap(lane.Extract.Options),
"metadata": redactSensitiveMap(input.Metadata),
},
}); err != nil {
return fmt.Errorf("write extract debug artifact for lane %q: %w", lane.ID, err)
}
if extractDecision.Reused {
extractOutputs = cloneExtractOutputs(extractCheckpoint.Outputs)
extractWarnings = cloneWarnings(extractCheckpoint.Warnings)
output.Rejected = append(output.Rejected, cloneRejectedOutputs(extractCheckpoint.Rejected)...)
output.Warnings = append(output.Warnings, extractWarnings...)
} else {
if err := checkpoints.ExtractRunning(lane.ID, extractor.Key(), extractDependencies); err != nil {
return fmt.Errorf("write extract checkpoint for lane %q: %w", lane.ID, err)
}
for index := range chunks {
chunk := chunks[index]
var acceptedOutput contracts.ExtractOutput
var acceptedWarnings []contracts.Warning
accepted, rejection, err := runWithRetry(ctx, lane.Extract.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
attemptStarted := time.Now().UTC()
attemptPath := path.Join("extract", debugPathComponent(lane.ID), fmt.Sprintf("chunk-%06d", chunk.Index+1), fmt.Sprintf("attempt-%02d", attempt))
attemptCtx, llmScope := withDebugLLMScope(ctx, attemptPath)
result, err := extractor.Extract(attemptCtx, contracts.ExtractionRequest{
Source: doc,
Chunk: &chunk,
SourceInput: chunkInputMaterial(sourceInput, chunk),
SessionID: sessionID,
References: CloneReferenceSet(lane.ExtractReferences.ReferenceSet),
LLMClient: input.llmClient,
LLMProfile: lane.Extract.LLMProfile,
Options: cloneOptions(lane.Extract.Options),
Metadata: input.Metadata,
})
if err != nil {
_ = writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(StageExtract),
LaneID: lane.ID,
ModuleKey: extractor.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Error: err.Error(),
}, llmScope))
return false, nil, fmt.Errorf("extract lane %q chunk %q with extractor %q: %w", lane.ID, chunk.ID, extractor.Key(), err)
}
extractOutput := result.Output
extractOutput.LaneID = lane.ID
extractOutput.ExtractorKey = extractor.Key()
extractOutput.SourceID = doc.ID
extractOutput.ChunkID = chunk.ID
extractOutput.ChunkIndex = chunk.Index
extractOutput.Payload.Warnings = append(extractOutput.Payload.Warnings, result.Warnings...)
validationWarnings, rejection, err := r.validateRaw(attemptCtx, rawValidationTarget{
stage: StageExtract,
laneID: lane.ID,
moduleKey: extractor.Key(),
source: doc,
sourceID: doc.ID,
chunkID: chunk.ID,
chunkIndex: chunk.Index,
chunk: &chunk,
sourceInput: chunkInputMaterial(sourceInput, chunk),
sessionID: sessionID,
references: lane.ExtractReferences.ReferenceSet,
llmClient: input.llmClient,
schema: extractOutput.Schema,
payload: extractOutput.Payload,
metadata: input.Metadata,
prepared: prepared.extractValidators,
attempt: attempt,
debug: input.Debug,
})
if err != nil || rejection != nil {
_ = writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(StageExtract),
LaneID: lane.ID,
ModuleKey: extractor.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Payload: map[string]any{
"output": debugExtractOutputEnvelope(extractOutput),
"warnings": append(cloneWarnings(result.Warnings), validationWarnings...),
"rejection": debugRejectedOutputPtr(rejection),
},
}, llmScope))
return false, rejection, err
}
acceptedOutput = cloneExtractOutput(extractOutput)
acceptedWarnings = append(cloneWarnings(result.Warnings), validationWarnings...)
if err := writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(StageExtract),
LaneID: lane.ID,
ModuleKey: extractor.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Payload: map[string]any{
"output": debugExtractOutputEnvelope(extractOutput),
"warnings": acceptedWarnings,
},
}, llmScope)); err != nil {
return false, nil, err
}
return true, nil, nil
})
if err != nil {
_ = checkpoints.ExtractFailed(lane.ID, extractor.Key(), extractDependencies, err)
return err
}
if !accepted {
output.Rejected = append(output.Rejected, *rejection)
continue
}
output.Warnings = append(output.Warnings, acceptedWarnings...)
extractWarnings = append(extractWarnings, acceptedWarnings...)
extractOutputs = append(extractOutputs, acceptedOutput)
}
extractRejected := cloneRejectedOutputs(output.Rejected[extractRejectedStart:])
if err := checkpoints.ExtractSucceeded(lane.ID, extractor.Key(), extractDependencies, extractOutputs, extractRejected, extractWarnings); err != nil {
return fmt.Errorf("write extract checkpoint for lane %q: %w", lane.ID, err)
}
}
if err := writeDebugTimed(input.Debug, path.Join("extract", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{
Stage: string(StageExtract),
LaneID: lane.ID,
ModuleKey: extractor.Key(),
StartedAt: extractStarted,
Payload: map[string]any{
"reused": extractDecision.Reused,
"outputs": debugExtractOutputEnvelopes(extractOutputs),
"rejected": debugRejectedOutputEnvelopes(output.Rejected[extractRejectedStart:]),
"warnings": extractWarnings,
},
}); err != nil {
return fmt.Errorf("write extract debug artifact for lane %q: %w", lane.ID, err)
}
if len(extractOutputs) == 0 {
return nil
}
var acceptedMerge contracts.MergeOutput
var mergeWarnings []contracts.Warning
mergeDependencies := rawOutputDigests(extractPayloads(extractOutputs))
mergeCheckpoint, mergeDecision := checkpointLoader.Merge(lane.ID, merger.Key(), mergeDependencies)
recordCheckpointEvent(output, checkpointLoader, string(StageMerge), lane.ID, merger.Key(), mergeDecision)
mergeStarted := time.Now().UTC()
if err := writeDebugTimed(input.Debug, path.Join("merge", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{
Stage: string(StageMerge),
LaneID: lane.ID,
ModuleKey: merger.Key(),
StartedAt: mergeStarted,
Payload: map[string]any{
"reused": mergeDecision.Reused,
"decision": mergeDecision,
"source": debugSourceDocumentEnvelope(doc),
"extract_outputs": debugExtractOutputEnvelopes(extractOutputs),
"options": redactSensitiveMap(lane.Merge.Options),
"metadata": redactSensitiveMap(input.Metadata),
},
}); err != nil {
return fmt.Errorf("write merge debug artifact for lane %q: %w", lane.ID, err)
}
if mergeDecision.Reused {
acceptedMerge = cloneMergeOutput(mergeCheckpoint.Output)
mergeWarnings = cloneWarnings(mergeCheckpoint.Warnings)
output.Warnings = append(output.Warnings, mergeWarnings...)
} else {
if err := checkpoints.MergeRunning(lane.ID, merger.Key(), mergeDependencies); err != nil {
return fmt.Errorf("write merge checkpoint for lane %q: %w", lane.ID, err)
}
mergeAccepted, mergeRejection, err := runWithRetry(ctx, lane.Merge.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
attemptStarted := time.Now().UTC()
attemptPath := path.Join("merge", debugPathComponent(lane.ID), fmt.Sprintf("attempt-%02d", attempt))
attemptCtx, llmScope := withDebugLLMScope(ctx, attemptPath)
mergeResult, err := merger.Merge(attemptCtx, contracts.MergeRequest{
Source: doc,
LaneID: lane.ID,
ExtractOutputs: cloneExtractOutputs(extractOutputs),
SourceInput: sourceInput.Clone(),
SessionID: sessionID,
References: CloneReferenceSet(lane.MergeReferences.ReferenceSet),
LLMClient: input.llmClient,
LLMProfile: lane.Merge.LLMProfile,
Options: cloneOptions(lane.Merge.Options),
Metadata: input.Metadata,
})
if err != nil {
_ = writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(StageMerge),
LaneID: lane.ID,
ModuleKey: merger.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Error: err.Error(),
}, llmScope))
return false, nil, fmt.Errorf("merge lane %q with merger %q: %w", lane.ID, merger.Key(), err)
}
mergeOutput := mergeResult.Output
mergeOutput.LaneID = lane.ID
mergeOutput.MergerKey = merger.Key()
mergeOutput.SourceID = doc.ID
mergeOutput.Payload.Warnings = append(mergeOutput.Payload.Warnings, mergeResult.Warnings...)
validationWarnings, rejection, err := r.validateRaw(attemptCtx, rawValidationTarget{
stage: StageMerge,
laneID: lane.ID,
moduleKey: merger.Key(),
source: doc,
sourceID: doc.ID,
sourceInput: sourceInput.Clone(),
sessionID: sessionID,
references: lane.MergeReferences.ReferenceSet,
llmClient: input.llmClient,
schema: mergeOutput.Schema,
payload: mergeOutput.Payload,
extractOutputs: extractOutputs,
metadata: input.Metadata,
prepared: prepared.mergeValidators,
attempt: attempt,
debug: input.Debug,
})
if err != nil || rejection != nil {
_ = writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(StageMerge),
LaneID: lane.ID,
ModuleKey: merger.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Payload: map[string]any{
"output": debugMergeOutputEnvelope(mergeOutput),
"warnings": append(cloneWarnings(mergeResult.Warnings), validationWarnings...),
"rejection": debugRejectedOutputPtr(rejection),
},
}, llmScope))
return false, rejection, err
}
acceptedMerge = cloneMergeOutput(mergeOutput)
mergeWarnings = append(cloneWarnings(mergeResult.Warnings), validationWarnings...)
if err := writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(StageMerge),
LaneID: lane.ID,
ModuleKey: merger.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Payload: map[string]any{
"output": debugMergeOutputEnvelope(mergeOutput),
"warnings": mergeWarnings,
},
}, llmScope)); err != nil {
return false, nil, err
}
return true, nil, nil
})
if err != nil {
_ = checkpoints.MergeFailed(lane.ID, merger.Key(), mergeDependencies, err)
return err
}
if !mergeAccepted {
output.Rejected = append(output.Rejected, *mergeRejection)
if err := checkpoints.MergeRejected(lane.ID, merger.Key(), mergeDependencies, *mergeRejection); err != nil {
return fmt.Errorf("write merge checkpoint for lane %q: %w", lane.ID, err)
}
if err := writeDebugTimed(input.Debug, path.Join("merge", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{
Stage: string(StageMerge),
LaneID: lane.ID,
ModuleKey: merger.Key(),
StartedAt: mergeStarted,
Payload: map[string]any{
"accepted": false,
"rejection": debugRejectedOutputEnvelope(*mergeRejection),
"warnings": mergeWarnings,
},
}); err != nil {
return fmt.Errorf("write merge debug artifact for lane %q: %w", lane.ID, err)
}
return nil
}
output.Warnings = append(output.Warnings, mergeWarnings...)
if err := checkpoints.MergeSucceeded(lane.ID, merger.Key(), mergeDependencies, acceptedMerge, mergeWarnings); err != nil {
return fmt.Errorf("write merge checkpoint for lane %q: %w", lane.ID, err)
}
}
if err := writeDebugTimed(input.Debug, path.Join("merge", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{
Stage: string(StageMerge),
LaneID: lane.ID,
ModuleKey: merger.Key(),
StartedAt: mergeStarted,
Payload: map[string]any{
"reused": mergeDecision.Reused,
"accepted": true,
"output": debugMergeOutputEnvelope(acceptedMerge),
"warnings": mergeWarnings,
},
}); err != nil {
return fmt.Errorf("write merge debug artifact for lane %q: %w", lane.ID, err)
}
var acceptedNormalize contracts.NormalizeOutput
var normalizeWarnings []contracts.Warning
normalizeDependencies := rawOutputDigests([]contracts.RawPayload{acceptedMerge.Payload})
normalizeCheckpoint, normalizeDecision := checkpointLoader.Normalize(lane.ID, normalizer.Key(), normalizeDependencies)
recordCheckpointEvent(output, checkpointLoader, string(StageNormalize), lane.ID, normalizer.Key(), normalizeDecision)
normalizeStarted := time.Now().UTC()
if err := writeDebugTimed(input.Debug, path.Join("normalize", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{
Stage: string(StageNormalize),
LaneID: lane.ID,
ModuleKey: normalizer.Key(),
StartedAt: normalizeStarted,
Payload: map[string]any{
"reused": normalizeDecision.Reused,
"decision": normalizeDecision,
"source": debugSourceDocumentEnvelope(doc),
"merge_output": debugMergeOutputEnvelope(acceptedMerge),
"options": redactSensitiveMap(lane.Normalize.Options),
"metadata": redactSensitiveMap(input.Metadata),
},
}); err != nil {
return fmt.Errorf("write normalize debug artifact for lane %q: %w", lane.ID, err)
}
if normalizeDecision.Reused {
acceptedNormalize = cloneNormalizeOutput(normalizeCheckpoint.Output)
normalizeWarnings = cloneWarnings(normalizeCheckpoint.Warnings)
output.Warnings = append(output.Warnings, normalizeWarnings...)
} else {
if err := checkpoints.NormalizeRunning(lane.ID, normalizer.Key(), normalizeDependencies); err != nil {
return fmt.Errorf("write normalize checkpoint for lane %q: %w", lane.ID, err)
}
normalizeAccepted, normalizeRejection, err := runWithRetry(ctx, lane.Normalize.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
attemptStarted := time.Now().UTC()
attemptPath := path.Join("normalize", debugPathComponent(lane.ID), fmt.Sprintf("attempt-%02d", attempt))
attemptCtx, llmScope := withDebugLLMScope(ctx, attemptPath)
normalizeResult, err := normalizer.Normalize(attemptCtx, contracts.NormalizeRequest{
Source: doc,
LaneID: lane.ID,
MergeOutput: cloneMergeOutput(acceptedMerge),
SourceInput: sourceInput.Clone(),
SessionID: sessionID,
References: CloneReferenceSet(lane.NormalizeReferences.ReferenceSet),
LLMClient: input.llmClient,
LLMProfile: lane.Normalize.LLMProfile,
Options: cloneOptions(lane.Normalize.Options),
Metadata: input.Metadata,
})
if err != nil {
_ = writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(StageNormalize),
LaneID: lane.ID,
ModuleKey: normalizer.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Error: err.Error(),
}, llmScope))
return false, nil, fmt.Errorf("normalize lane %q with normalizer %q: %w", lane.ID, normalizer.Key(), err)
}
normalizeOutput := normalizeResult.Output
normalizeOutput.LaneID = lane.ID
normalizeOutput.NormalizerKey = normalizer.Key()
normalizeOutput.SourceID = doc.ID
normalizeOutput.Payload.Warnings = append(normalizeOutput.Payload.Warnings, normalizeResult.Warnings...)
validationWarnings, rejection, err := r.validateRaw(attemptCtx, rawValidationTarget{
stage: StageNormalize,
laneID: lane.ID,
moduleKey: normalizer.Key(),
source: doc,
sourceID: doc.ID,
sourceInput: sourceInput.Clone(),
sessionID: sessionID,
references: lane.NormalizeReferences.ReferenceSet,
llmClient: input.llmClient,
schema: normalizeOutput.Schema,
payload: normalizeOutput.Payload,
mergeOutput: acceptedMerge,
metadata: input.Metadata,
prepared: prepared.normalizeValidators,
attempt: attempt,
debug: input.Debug,
})
if err != nil || rejection != nil {
_ = writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(StageNormalize),
LaneID: lane.ID,
ModuleKey: normalizer.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Payload: map[string]any{
"output": debugNormalizeOutputEnvelope(normalizeOutput),
"warnings": append(cloneWarnings(normalizeResult.Warnings), validationWarnings...),
"rejection": debugRejectedOutputPtr(rejection),
},
}, llmScope))
return false, rejection, err
}
acceptedNormalize = cloneNormalizeOutput(normalizeOutput)
normalizeWarnings = append(cloneWarnings(normalizeResult.Warnings), validationWarnings...)
if err := writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(StageNormalize),
LaneID: lane.ID,
ModuleKey: normalizer.Key(),
Attempt: attempt,
StartedAt: attemptStarted,
Payload: map[string]any{
"output": debugNormalizeOutputEnvelope(normalizeOutput),
"warnings": normalizeWarnings,
},
}, llmScope)); err != nil {
return false, nil, err
}
return true, nil, nil
})
if err != nil {
_ = checkpoints.NormalizeFailed(lane.ID, normalizer.Key(), normalizeDependencies, err)
return err
}
if !normalizeAccepted {
output.Rejected = append(output.Rejected, *normalizeRejection)
if err := checkpoints.NormalizeRejected(lane.ID, normalizer.Key(), normalizeDependencies, *normalizeRejection); err != nil {
return fmt.Errorf("write normalize checkpoint for lane %q: %w", lane.ID, err)
}
if err := writeDebugTimed(input.Debug, path.Join("normalize", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{
Stage: string(StageNormalize),
LaneID: lane.ID,
ModuleKey: normalizer.Key(),
StartedAt: normalizeStarted,
Payload: map[string]any{
"accepted": false,
"rejection": debugRejectedOutputEnvelope(*normalizeRejection),
"warnings": normalizeWarnings,
},
}); err != nil {
return fmt.Errorf("write normalize debug artifact for lane %q: %w", lane.ID, err)
}
return nil
}
output.Warnings = append(output.Warnings, normalizeWarnings...)
if err := checkpoints.NormalizeSucceeded(lane.ID, normalizer.Key(), normalizeDependencies, acceptedNormalize, normalizeWarnings); err != nil {
return fmt.Errorf("write normalize checkpoint for lane %q: %w", lane.ID, err)
}
}
if err := writeDebugTimed(input.Debug, path.Join("normalize", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{
Stage: string(StageNormalize),
LaneID: lane.ID,
ModuleKey: normalizer.Key(),
StartedAt: normalizeStarted,
Payload: map[string]any{
"reused": normalizeDecision.Reused,
"accepted": true,
"output": debugNormalizeOutputEnvelope(acceptedNormalize),
"warnings": normalizeWarnings,
},
}); err != nil {
return fmt.Errorf("write normalize debug artifact for lane %q: %w", lane.ID, err)
}
output.NormalizeOutputs = append(output.NormalizeOutputs, serializedOutputFromLegacy(acceptedNormalize))
return nil
}
type rawValidationTarget struct {
stage ModuleStage
laneID string
moduleKey string
source *source.SourceDocument
sourceID string
sourceInput contracts.LLMInputMaterial
sessionID string
references contracts.ReferenceSet
llmClient contracts.StructuredLLMClient
chunkID string
chunkIndex int
chunk *source.Chunk
chunks []source.Chunk
schema contracts.ResponseSchema
payload contracts.RawPayload
extractOutputs []contracts.ExtractOutput
mergeOutput contracts.MergeOutput
metadata map[string]any
prepared preparedValidatorChain
attempt int
debug DebugRecorder
} }
func runWithRetry(ctx context.Context, retries int, run func(attempt int) (bool, *contracts.RejectedOutput, error)) (bool, *contracts.RejectedOutput, error) { func runWithRetry(ctx context.Context, retries int, run func(attempt int) (bool, *contracts.RejectedOutput, error)) (bool, *contracts.RejectedOutput, error) {
attempts := 1 attempts := retries + 1
if retries > 0 { var last *contracts.RejectedOutput
attempts += retries
}
var lastRejection *contracts.RejectedOutput
for attempt := 1; attempt <= attempts; attempt++ { for attempt := 1; attempt <= attempts; attempt++ {
if err := ctx.Err(); err != nil { if err := ctx.Err(); err != nil {
return false, nil, err return false, nil, err
} }
accepted, rejection, err := run(attempt) accepted, rejection, err := run(attempt)
if err != nil { if err != nil {
if ctxErr := ctx.Err(); ctxErr != nil { if ctxErr := ctx.Err(); ctxErr != nil {
@@ -920,55 +402,22 @@ func runWithRetry(ctx context.Context, retries int, run func(attempt int) (bool,
} }
if rejection != nil { if rejection != nil {
rejection.AttemptCount = attempt rejection.AttemptCount = attempt
lastRejection = rejection last = rejection
} }
if ctxErr := ctx.Err(); ctxErr != nil { if ctxErr := ctx.Err(); ctxErr != nil {
return false, nil, ctxErr return false, nil, ctxErr
} }
if attempt == attempts { if attempt == attempts {
if lastRejection == nil { if last == nil {
lastRejection = &contracts.RejectedOutput{ last = &contracts.RejectedOutput{ReasonCode: "output_rejected", Message: "output rejected", AttemptCount: attempt}
ReasonCode: "raw_output_rejected",
Message: "raw output rejected",
AttemptCount: attempt,
}
} }
return false, lastRejection, nil return false, last, nil
} }
} }
return false, last, nil
return false, lastRejection, nil
} }
func (r *Runner) validateChunksRaw(ctx context.Context, doc *source.SourceDocument, moduleKey string, chunks []source.Chunk, sourceInput contracts.LLMInputMaterial, sessionID string, references contracts.ReferenceSet, llmClient contracts.StructuredLLMClient, metadata map[string]any, prepared preparedValidatorChain, attempt int, debug DebugRecorder) ([]contracts.Warning, *contracts.RejectedOutput, error) { func (r *Runner) validateChunks(ctx context.Context, doc *source.SourceDocument, moduleKey string, chunks []source.Chunk, sourceInput contracts.LLMInputMaterial, sessionID string, references contracts.ReferenceSet, metadata map[string]any, prepared preparedValidatorChain, attempt int, debug DebugRecorder) ([]contracts.Warning, *contracts.RejectedOutput, error) {
return r.validateRaw(ctx, rawValidationTarget{
stage: StageChunk,
moduleKey: moduleKey,
source: doc,
sourceID: doc.ID,
sourceInput: sourceInput.Clone(),
sessionID: sessionID,
references: references,
llmClient: llmClient,
chunks: chunks,
metadata: metadata,
prepared: prepared,
attempt: attempt,
debug: debug,
})
}
func (r *Runner) validateChunks(ctx context.Context, doc *source.SourceDocument, moduleKey string, chunks []source.Chunk, sourceInput contracts.LLMInputMaterial, sessionID string, references contracts.ReferenceSet, llmClient contracts.StructuredLLMClient, metadata map[string]any, prepared preparedValidatorChain, attempt int, debug DebugRecorder) ([]contracts.Warning, *contracts.RejectedOutput, error) {
allLegacy := true
for _, item := range prepared.validators {
if item.resolved.Target != ValidatorTargetLegacyRaw && item.resolved.Target != "" {
allLegacy = false
break
}
}
if allLegacy {
return r.validateChunksRaw(ctx, doc, moduleKey, chunks, sourceInput, sessionID, references, llmClient, metadata, prepared, attempt, debug)
}
content, err := json.Marshal(chunks) content, err := json.Marshal(chunks)
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("encode canonical chunks for validation: %w", err) return nil, nil, fmt.Errorf("encode canonical chunks for validation: %w", err)
@@ -989,8 +438,9 @@ func (r *Runner) validateChunks(ctx context.Context, doc *source.SourceDocument,
default: default:
return nil, nil, fmt.Errorf("validator %q is incompatible with chunk validation", binding.Module) return nil, nil, fmt.Errorf("validator %q is incompatible with chunk validation", binding.Module)
} }
debugRequest := contracts.ValidationRequest{Stage: string(StageChunk), ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, Metadata: cloneMetadata(metadata), Chunks: cloneSourceChunks(chunks), Schema: contracts.ResponseSchema{ID: schema.ID, Name: schema.Name, Version: schema.Version, JSONSchema: append([]byte(nil), schema.JSONSchema...)}, Payload: contracts.RawPayload{Content: content, MediaType: "application/json"}} debugContent := debugContentEnvelope(content, "application/json", nil, nil)
debugCall := debugValidationCall{ValidatorName: binding.Module, Request: debugValidationRequestEnvelope(debugRequest), Result: debugValidationResultEnvelope(result)} debugContent.ContentDigest = debugContentDigest(content)
debugCall := debugValidationCall{ValidatorName: binding.Module, Request: map[string]any{"stage": string(StageChunk), "module_key": moduleKey, "source_id": doc.ID, "schema": schema, "schema_digest": contracts.DigestArtifactSchema(schema), "content": debugContent, "metadata": redactSensitiveMap(metadata)}, Result: debugValidationResultEnvelope(result)}
if err != nil { if err != nil {
debugCall.Error = err.Error() debugCall.Error = err.Error()
} }
@@ -1003,11 +453,11 @@ func (r *Runner) validateChunks(ctx context.Context, doc *source.SourceDocument,
if !result.Approved { if !result.Approved {
reason := result.ReasonCode reason := result.ReasonCode
if reason == "" { if reason == "" {
reason = "raw_output_rejected" reason = "output_rejected"
} }
message := result.Message message := result.Message
if message == "" { if message == "" {
message = "raw output rejected" message = "output rejected"
} }
return nil, &contracts.RejectedOutput{Stage: string(StageChunk), ModuleKey: moduleKey, ValidatorName: binding.Module, ReasonCode: reason, Message: message, AttemptCount: attempt, DiagnosticArtifactPath: result.DiagnosticArtifactPath}, nil return nil, &contracts.RejectedOutput{Stage: string(StageChunk), ModuleKey: moduleKey, ValidatorName: binding.Module, ReasonCode: reason, Message: message, AttemptCount: attempt, DiagnosticArtifactPath: result.DiagnosticArtifactPath}, nil
} }
@@ -1016,97 +466,6 @@ func (r *Runner) validateChunks(ctx context.Context, doc *source.SourceDocument,
return warnings, nil, nil return warnings, nil, nil
} }
func (r *Runner) validateRaw(ctx context.Context, target rawValidationTarget) ([]contracts.Warning, *contracts.RejectedOutput, error) {
if len(target.prepared.validators) == 0 {
return nil, nil, nil
}
var warnings []contracts.Warning
for index, preparedValidator := range target.prepared.validators {
validator := preparedValidator.legacy
validatorBinding := preparedValidator.resolved
if validator == nil {
return nil, nil, fmt.Errorf("validator %q is not available on the legacy raw path", validatorBinding.Binding.Module)
}
request := target.validationRequest(validatorBinding.Binding)
started := time.Now().UTC()
attemptPath := path.Join("validate", debugPathComponent(string(target.stage)), debugPathComponent(target.laneID), debugPathComponent(target.moduleKey), fmt.Sprintf("%02d-%s-attempt-%02d", index+1, debugPathComponent(validator.Name()), target.attempt))
validatorCtx, llmScope := withDebugLLMScope(ctx, attemptPath)
result, err := validator.Validate(validatorCtx, request)
debugPayload := debugValidationCall{
ValidatorName: validator.Name(),
Request: debugValidationRequestEnvelope(request),
Result: debugValidationResultEnvelope(result),
}
if err != nil {
debugPayload.Error = err.Error()
}
if debugErr := writeDebugTimed(target.debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
Stage: string(target.stage),
LaneID: target.laneID,
ModuleKey: target.moduleKey,
Attempt: target.attempt,
StartedAt: started,
Payload: debugPayload,
Error: debugPayload.Error,
}, llmScope)); debugErr != nil {
return nil, nil, fmt.Errorf("write validation debug artifact: %w", debugErr)
}
if err != nil {
return nil, nil, fmt.Errorf("validate raw %s output with validator %q: %w", target.stage, validator.Name(), err)
}
if !result.Approved {
reasonCode := strings.TrimSpace(result.ReasonCode)
if reasonCode == "" {
reasonCode = "raw_output_rejected"
}
message := strings.TrimSpace(result.Message)
if message == "" {
message = "raw output rejected"
}
return nil, &contracts.RejectedOutput{
Stage: string(target.stage),
LaneID: target.laneID,
ModuleKey: target.moduleKey,
ChunkID: target.chunkID,
ChunkIndex: target.chunkIndex,
ValidatorName: validator.Name(),
ReasonCode: reasonCode,
Message: message,
AttemptCount: target.attempt,
DiagnosticArtifactPath: result.DiagnosticArtifactPath,
}, nil
}
warnings = append(warnings, result.Warnings...)
}
return warnings, nil, nil
}
func (target rawValidationTarget) validationRequest(binding ModuleBinding) contracts.ValidationRequest {
return contracts.ValidationRequest{
Stage: string(target.stage),
LaneID: target.laneID,
ModuleKey: target.moduleKey,
Source: target.source,
SourceID: target.sourceID,
SourceInput: target.sourceInput.Clone(),
SessionID: target.sessionID,
References: CloneReferenceSet(target.references),
LLMClient: target.llmClient,
LLMProfile: binding.LLMProfile,
Options: cloneOptions(binding.Options),
Metadata: cloneMetadata(target.metadata),
Schema: cloneResponseSchema(target.schema),
Payload: cloneRawPayload(target.payload),
ChunkID: target.chunkID,
ChunkIndex: target.chunkIndex,
Chunk: cloneSourceChunkPtr(target.chunk),
Chunks: cloneSourceChunks(target.chunks),
ExtractOutputs: cloneExtractOutputs(target.extractOutputs),
MergeOutput: cloneMergeOutput(target.mergeOutput),
}
}
func resolvedValidatorChain(stage ModuleStage, laneID string, moduleKey string, chains []ResolvedValidatorChain) ResolvedValidatorChain { func resolvedValidatorChain(stage ModuleStage, laneID string, moduleKey string, chains []ResolvedValidatorChain) ResolvedValidatorChain {
for _, chain := range chains { for _, chain := range chains {
if chain.Stage != stage { if chain.Stage != stage {
@@ -1250,7 +609,7 @@ func validatorChainManifests(chains []ResolvedValidatorChain) []artifacts.Valida
func failOutput(output RunOutput) RunOutput { func failOutput(output RunOutput) RunOutput {
if output.Manifest.PipelineID != "" { if output.Manifest.PipelineID != "" {
populateRawOutputManifest(&output) populateOutputManifest(&output)
output.Manifest.ValidationStatus = "failed" output.Manifest.ValidationStatus = "failed"
output.Manifest.CompletedAt = timePtr(time.Now().UTC()) output.Manifest.CompletedAt = timePtr(time.Now().UTC())
} }
@@ -1274,7 +633,7 @@ func recordCheckpointEvent(output *RunOutput, loader CheckpointLoader, stage str
}) })
} }
func populateRawOutputManifest(output *RunOutput) { func populateOutputManifest(output *RunOutput) {
if output == nil { if output == nil {
return return
} }
@@ -1325,59 +684,18 @@ func rejectedOutputManifests(rejected []contracts.RejectedOutput) []artifacts.Re
return manifests return manifests
} }
func setLaneManifestMetadata(output *RunOutput, laneID string, modules ...any) {
if output == nil {
return
}
for i := range output.Manifest.ArtifactLanes {
if output.Manifest.ArtifactLanes[i].ID != laneID {
continue
}
metadata := make(map[string]any)
for _, module := range modules {
moduleMetadata, ok := moduleManifestMetadata(module)
if !ok {
continue
}
key := manifestMetadataKey(module)
if key == "" {
continue
}
metadata[key] = moduleMetadata
}
if len(metadata) > 0 {
output.Manifest.ArtifactLanes[i].Metadata = metadata
}
return
}
}
func attachModuleManifestMetadata(output *RunOutput, moduleKey string, module any) { func attachModuleManifestMetadata(output *RunOutput, moduleKey string, module any) {
if output == nil { if output == nil {
return return
} }
moduleMetadata, ok := moduleManifestMetadata(module) metadata, ok := moduleManifestMetadata(module)
if !ok { if !ok {
return return
} }
if output.Manifest.ModuleMetadata == nil { if output.Manifest.ModuleMetadata == nil {
output.Manifest.ModuleMetadata = make(map[string]map[string]any) output.Manifest.ModuleMetadata = make(map[string]map[string]any)
} }
output.Manifest.ModuleMetadata[moduleKey] = moduleMetadata output.Manifest.ModuleMetadata[moduleKey] = metadata
}
func manifestMetadataKey(module any) string {
switch module.(type) {
case contracts.LegacyRawExtractor:
return "extractor"
case contracts.LegacyRawMerger:
return "merger"
case contracts.LegacyRawNormalizer:
return "normalizer"
default:
return ""
}
} }
func moduleManifestMetadata(module any) (map[string]any, bool) { func moduleManifestMetadata(module any) (map[string]any, bool) {
@@ -1557,20 +875,6 @@ func cloneWarnings(warnings []contracts.Warning) []contracts.Warning {
return append([]contracts.Warning(nil), warnings...) return append([]contracts.Warning(nil), warnings...)
} }
func cloneRawPayload(payload contracts.RawPayload) contracts.RawPayload {
return contracts.RawPayload{
Content: append([]byte(nil), payload.Content...),
MediaType: payload.MediaType,
Metadata: cloneMetadata(payload.Metadata),
Warnings: cloneWarnings(payload.Warnings),
}
}
func cloneResponseSchema(schema contracts.ResponseSchema) contracts.ResponseSchema {
schema.JSONSchema = append([]byte(nil), schema.JSONSchema...)
return schema
}
func cloneSourceChunkPtr(chunk *source.Chunk) *source.Chunk { func cloneSourceChunkPtr(chunk *source.Chunk) *source.Chunk {
if chunk == nil { if chunk == nil {
return nil return nil
@@ -1608,35 +912,6 @@ func cloneSourceUnits(units []source.SourceUnit) []source.SourceUnit {
return out return out
} }
func cloneExtractOutput(output contracts.ExtractOutput) contracts.ExtractOutput {
output.Schema = cloneResponseSchema(output.Schema)
output.Payload = cloneRawPayload(output.Payload)
return output
}
func cloneExtractOutputs(outputs []contracts.ExtractOutput) []contracts.ExtractOutput {
if len(outputs) == 0 {
return nil
}
out := make([]contracts.ExtractOutput, 0, len(outputs))
for _, output := range outputs {
out = append(out, cloneExtractOutput(output))
}
return out
}
func cloneMergeOutput(output contracts.MergeOutput) contracts.MergeOutput {
output.Schema = cloneResponseSchema(output.Schema)
output.Payload = cloneRawPayload(output.Payload)
return output
}
func cloneNormalizeOutput(output contracts.NormalizeOutput) contracts.NormalizeOutput {
output.Schema = cloneResponseSchema(output.Schema)
output.Payload = cloneRawPayload(output.Payload)
return output
}
func cloneSerializedOutputs(outputs []contracts.SerializedOutput) []contracts.SerializedOutput { func cloneSerializedOutputs(outputs []contracts.SerializedOutput) []contracts.SerializedOutput {
if len(outputs) == 0 { if len(outputs) == 0 {
return nil return nil
@@ -1648,16 +923,6 @@ func cloneSerializedOutputs(outputs []contracts.SerializedOutput) []contracts.Se
return out return out
} }
func serializedOutputFromLegacy(output contracts.NormalizeOutput) contracts.SerializedOutput {
return contracts.SerializedOutput{
LaneID: output.LaneID, NormalizerKey: output.NormalizerKey, SourceID: output.SourceID,
Artifact: contracts.SerializedArtifact{
Schema: contracts.ArtifactSchema{ID: output.Schema.ID, Name: output.Schema.Name, Version: output.Schema.Version, JSONSchema: append([]byte(nil), output.Schema.JSONSchema...)},
MediaType: output.Payload.MediaType, Content: append([]byte(nil), output.Payload.Content...), Metadata: cloneMetadata(output.Payload.Metadata),
},
}
}
func cloneRejectedOutputs(rejected []contracts.RejectedOutput) []contracts.RejectedOutput { func cloneRejectedOutputs(rejected []contracts.RejectedOutput) []contracts.RejectedOutput {
if len(rejected) == 0 { if len(rejected) == 0 {
return nil return nil

File diff suppressed because it is too large Load Diff

View File

@@ -13,54 +13,30 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
) )
func loadArtifactExtract(loader CheckpointLoader, laneID, moduleKey string, deps []CheckpointFingerprint) (ArtifactExtractCheckpoint, CheckpointDecision) { func loadExtract(loader CheckpointLoader, laneID, moduleKey string, deps []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
typed, ok := loader.(ArtifactCheckpointLoader) return loader.Extract(laneID, moduleKey, deps)
if !ok {
return ArtifactExtractCheckpoint{}, CheckpointDecision{Reason: "artifact checkpoint loading is unavailable"}
}
return typed.ArtifactExtract(laneID, moduleKey, deps)
} }
func loadArtifactMerge(loader CheckpointLoader, laneID, moduleKey string, deps []CheckpointFingerprint) (ArtifactMergeCheckpoint, CheckpointDecision) { func loadMerge(loader CheckpointLoader, laneID, moduleKey string, deps []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) {
typed, ok := loader.(ArtifactCheckpointLoader) return loader.Merge(laneID, moduleKey, deps)
if !ok {
return ArtifactMergeCheckpoint{}, CheckpointDecision{Reason: "artifact checkpoint loading is unavailable"}
}
return typed.ArtifactMerge(laneID, moduleKey, deps)
} }
func loadArtifactNormalize(loader CheckpointLoader, laneID, moduleKey string, deps []CheckpointFingerprint) (ArtifactNormalizeCheckpoint, CheckpointDecision) { func loadNormalize(loader CheckpointLoader, laneID, moduleKey string, deps []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) {
typed, ok := loader.(ArtifactCheckpointLoader) return loader.Normalize(laneID, moduleKey, deps)
if !ok {
return ArtifactNormalizeCheckpoint{}, CheckpointDecision{Reason: "artifact checkpoint loading is unavailable"}
}
return typed.ArtifactNormalize(laneID, moduleKey, deps)
} }
func recordArtifactExtract(recorder CheckpointRecorder, laneID, moduleKey string, deps []CheckpointFingerprint, outputs []ArtifactCheckpointOutput, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error { func recordExtract(recorder CheckpointRecorder, laneID, moduleKey string, deps []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error {
typed, ok := recorder.(ArtifactCheckpointRecorder) return recorder.ExtractSucceeded(laneID, moduleKey, deps, outputs, rejected, warnings)
if !ok {
return nil
}
return typed.ArtifactExtractSucceeded(laneID, moduleKey, deps, outputs, rejected, warnings)
} }
func recordArtifactMerge(recorder CheckpointRecorder, laneID, moduleKey string, deps []CheckpointFingerprint, output ArtifactCheckpointOutput, warnings []contracts.Warning) error { func recordMerge(recorder CheckpointRecorder, laneID, moduleKey string, deps []CheckpointFingerprint, output CheckpointArtifact, warnings []contracts.Warning) error {
typed, ok := recorder.(ArtifactCheckpointRecorder) return recorder.MergeSucceeded(laneID, moduleKey, deps, output, warnings)
if !ok {
return nil
}
return typed.ArtifactMergeSucceeded(laneID, moduleKey, deps, output, warnings)
} }
func recordArtifactNormalize(recorder CheckpointRecorder, laneID, moduleKey string, deps []CheckpointFingerprint, output ArtifactCheckpointOutput, warnings []contracts.Warning) error { func recordNormalize(recorder CheckpointRecorder, laneID, moduleKey string, deps []CheckpointFingerprint, output CheckpointArtifact, warnings []contracts.Warning) error {
typed, ok := recorder.(ArtifactCheckpointRecorder) return recorder.NormalizeSucceeded(laneID, moduleKey, deps, output, warnings)
if !ok {
return nil
}
return typed.ArtifactNormalizeSucceeded(laneID, moduleKey, deps, output, warnings)
} }
func cloneArtifactCheckpointOutput(output ArtifactCheckpointOutput) ArtifactCheckpointOutput { func cloneCheckpointArtifact(output CheckpointArtifact) CheckpointArtifact {
output.Artifact = contracts.CloneSerializedArtifact(output.Artifact) output.Artifact = contracts.CloneSerializedArtifact(output.Artifact)
return output return output
} }
func hydrateCheckpointArtifact(codec artifactCodecEntry, output ArtifactCheckpointOutput, value any) ArtifactCheckpointOutput { func hydrateCheckpointArtifact(codec artifactCodecEntry, output CheckpointArtifact, value any) CheckpointArtifact {
output.Artifact.Schema = contracts.CloneArtifactSchema(codec.spec.Schema) output.Artifact.Schema = contracts.CloneArtifactSchema(codec.spec.Schema)
if codec.metadata != nil { if codec.metadata != nil {
output.Artifact.Metadata = cloneMetadata(codec.metadata(value)) output.Artifact.Metadata = cloneMetadata(codec.metadata(value))
@@ -69,7 +45,7 @@ func hydrateCheckpointArtifact(codec artifactCodecEntry, output ArtifactCheckpoi
} }
return output return output
} }
func artifactCheckpointDigests(outputs []ArtifactCheckpointOutput) []CheckpointFingerprint { func artifactCheckpointDigests(outputs []CheckpointArtifact) []CheckpointFingerprint {
values := make([]CheckpointFingerprint, 0, len(outputs)) values := make([]CheckpointFingerprint, 0, len(outputs))
for i, output := range outputs { for i, output := range outputs {
sum := sha256.Sum256(output.Artifact.Content) sum := sha256.Sum256(output.Artifact.Content)
@@ -77,7 +53,7 @@ func artifactCheckpointDigests(outputs []ArtifactCheckpointOutput) []CheckpointF
} }
return normalizeCheckpointFingerprints(values) return normalizeCheckpointFingerprints(values)
} }
func debugArtifactCheckpointOutput(output ArtifactCheckpointOutput) map[string]any { func debugCheckpointArtifact(output CheckpointArtifact) map[string]any {
artifact := output.Artifact artifact := output.Artifact
schema := contracts.CloneArtifactSchema(artifact.Schema) schema := contracts.CloneArtifactSchema(artifact.Schema)
digest := output.SchemaDigest digest := output.SchemaDigest
@@ -89,13 +65,13 @@ func debugArtifactCheckpointOutput(output ArtifactCheckpointOutput) map[string]a
content.ContentDigest = debugContentDigest(artifact.Content) content.ContentDigest = debugContentDigest(artifact.Content)
return map[string]any{"lane_id": output.LaneID, "module_key": output.ModuleKey, "source_id": output.SourceID, "chunk_id": output.ChunkID, "chunk_index": output.ChunkIndex, "chunk_ref": output.ChunkRef, "artifact_kind": artifact.Kind, "schema": schema, "schema_digest": digest, "content": content} return map[string]any{"lane_id": output.LaneID, "module_key": output.ModuleKey, "source_id": output.SourceID, "chunk_id": output.ChunkID, "chunk_index": output.ChunkIndex, "chunk_ref": output.ChunkRef, "artifact_kind": artifact.Kind, "schema": schema, "schema_digest": digest, "content": content}
} }
func debugArtifactCheckpointOutputs(outputs []ArtifactCheckpointOutput) []map[string]any { func debugCheckpointArtifacts(outputs []CheckpointArtifact) []map[string]any {
if len(outputs) == 0 { if len(outputs) == 0 {
return nil return nil
} }
out := make([]map[string]any, 0, len(outputs)) out := make([]map[string]any, 0, len(outputs))
for _, output := range outputs { for _, output := range outputs {
out = append(out, debugArtifactCheckpointOutput(output)) out = append(out, debugCheckpointArtifact(output))
} }
return out return out
} }
@@ -117,7 +93,7 @@ func serializeArtifact(codec artifactCodecEntry, value any, candidate bool) (con
return contracts.SerializedArtifact{Kind: codec.spec.Kind, Schema: contracts.CloneArtifactSchema(schema), MediaType: codec.spec.MediaType, Content: append([]byte(nil), content...), Metadata: cloneMetadata(metadata)}, nil return contracts.SerializedArtifact{Kind: codec.spec.Kind, Schema: contracts.CloneArtifactSchema(schema), MediaType: codec.spec.MediaType, Content: append([]byte(nil), content...), Metadata: cloneMetadata(metadata)}, nil
} }
func decodeCheckpointArtifact(codec artifactCodecEntry, artifact ArtifactCheckpointOutput) (any, error) { func decodeCheckpointArtifact(codec artifactCodecEntry, artifact CheckpointArtifact) (any, error) {
expectedDigest := contracts.DigestArtifactSchema(codec.spec.Schema) expectedDigest := contracts.DigestArtifactSchema(codec.spec.Schema)
if artifact.Artifact.Kind != codec.spec.Kind { if artifact.Artifact.Kind != codec.spec.Kind {
return nil, fmt.Errorf("artifact kind %q does not match codec %q", artifact.Artifact.Kind, codec.spec.Kind) return nil, fmt.Errorf("artifact kind %q does not match codec %q", artifact.Artifact.Kind, codec.spec.Kind)
@@ -134,12 +110,12 @@ func decodeCheckpointArtifact(codec artifactCodecEntry, artifact ArtifactCheckpo
return codec.decode(append([]byte(nil), artifact.Artifact.Content...)) return codec.decode(append([]byte(nil), artifact.Artifact.Content...))
} }
func checkpointArtifact(codec artifactCodecEntry, laneID, moduleKey, sourceID string, value any) (ArtifactCheckpointOutput, error) { func checkpointArtifact(codec artifactCodecEntry, laneID, moduleKey, sourceID string, value any) (CheckpointArtifact, error) {
serialized, err := serializeArtifact(codec, value, false) serialized, err := serializeArtifact(codec, value, false)
if err != nil { if err != nil {
return ArtifactCheckpointOutput{}, err return CheckpointArtifact{}, err
} }
return ArtifactCheckpointOutput{LaneID: laneID, ModuleKey: moduleKey, SourceID: sourceID, Artifact: serialized, SchemaDigest: contracts.DigestArtifactSchema(serialized.Schema)}, nil return CheckpointArtifact{LaneID: laneID, ModuleKey: moduleKey, SourceID: sourceID, Artifact: serialized, SchemaDigest: contracts.DigestArtifactSchema(serialized.Schema)}, nil
} }
func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, prepared preparedLaneExecutor, output *RunOutput) error { func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, prepared preparedLaneExecutor, output *RunOutput) error {
@@ -150,7 +126,7 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
setTypedLaneManifestMetadata(output, lane.ID, typed.extractor, typed.merger, typed.normalizer) setTypedLaneManifestMetadata(output, lane.ID, typed.extractor, typed.merger, typed.normalizer)
values := make([]erasedExtractArtifact, 0, len(chunks)) values := make([]erasedExtractArtifact, 0, len(chunks))
serializedExtracts := make([]ArtifactCheckpointOutput, 0, len(chunks)) serializedExtracts := make([]CheckpointArtifact, 0, len(chunks))
extractWarnings := []contracts.Warning{} extractWarnings := []contracts.Warning{}
rejectedStart := len(output.Rejected) rejectedStart := len(output.Rejected)
chunksDigest, err := joinedChunkDigest(chunks) chunksDigest, err := joinedChunkDigest(chunks)
@@ -158,7 +134,7 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
return fmt.Errorf("digest chunks for lane %q: %w", lane.ID, err) return fmt.Errorf("digest chunks for lane %q: %w", lane.ID, err)
} }
extractDeps := digestFingerprints("chunks", chunksDigest) extractDeps := digestFingerprints("chunks", chunksDigest)
cp, decision := loadArtifactExtract(loader, lane.ID, lane.Extract.Module, extractDeps) cp, decision := loadExtract(loader, lane.ID, lane.Extract.Module, extractDeps)
if decision.Reused { if decision.Reused {
for _, stored := range cp.Outputs { for _, stored := range cp.Outputs {
if _, decodeErr := decodeCheckpointArtifact(typed.codec, stored); decodeErr != nil { if _, decodeErr := decodeCheckpointArtifact(typed.codec, stored); decodeErr != nil {
@@ -183,7 +159,7 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
artifact.ChunkRef = chunks[stored.ChunkIndex].Ref artifact.ChunkRef = chunks[stored.ChunkIndex].Ref
} }
values = append(values, artifact) values = append(values, artifact)
serializedExtracts = append(serializedExtracts, cloneArtifactCheckpointOutput(stored)) serializedExtracts = append(serializedExtracts, cloneCheckpointArtifact(stored))
} }
extractWarnings = cloneWarnings(cp.Warnings) extractWarnings = cloneWarnings(cp.Warnings)
output.Warnings = append(output.Warnings, extractWarnings...) output.Warnings = append(output.Warnings, extractWarnings...)
@@ -195,7 +171,7 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
for i := range chunks { for i := range chunks {
chunk := chunks[i] chunk := chunks[i]
var accepted erasedExtractArtifact var accepted erasedExtractArtifact
var serializedAccepted ArtifactCheckpointOutput var serializedAccepted CheckpointArtifact
var acceptedWarnings []contracts.Warning var acceptedWarnings []contracts.Warning
ok, rejection, runErr := runWithRetry(ctx, lane.Extract.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) { ok, rejection, runErr := runWithRetry(ctx, lane.Extract.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
started := time.Now().UTC() started := time.Now().UTC()
@@ -218,7 +194,7 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
stored.ChunkID, stored.ChunkIndex, stored.ChunkRef = artifact.ChunkID, artifact.ChunkIndex, artifact.ChunkRef stored.ChunkID, stored.ChunkIndex, stored.ChunkRef = artifact.ChunkID, artifact.ChunkIndex, artifact.ChunkRef
accepted, serializedAccepted = artifact, stored accepted, serializedAccepted = artifact, stored
acceptedWarnings = append(cloneWarnings(result.Warnings), warnings...) acceptedWarnings = append(cloneWarnings(result.Warnings), warnings...)
if debugErr := writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{Stage: string(StageExtract), LaneID: lane.ID, ModuleKey: lane.Extract.Module, Attempt: attempt, StartedAt: started, Payload: map[string]any{"output": debugArtifactCheckpointOutput(stored), "warnings": debugWarningEnvelopes(acceptedWarnings)}}, llmScope)); debugErr != nil { if debugErr := writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{Stage: string(StageExtract), LaneID: lane.ID, ModuleKey: lane.Extract.Module, Attempt: attempt, StartedAt: started, Payload: map[string]any{"output": debugCheckpointArtifact(stored), "warnings": debugWarningEnvelopes(acceptedWarnings)}}, llmScope)); debugErr != nil {
return false, nil, debugErr return false, nil, debugErr
} }
return true, nil, nil return true, nil, nil
@@ -236,13 +212,13 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
extractWarnings = append(extractWarnings, acceptedWarnings...) extractWarnings = append(extractWarnings, acceptedWarnings...)
output.Warnings = append(output.Warnings, acceptedWarnings...) output.Warnings = append(output.Warnings, acceptedWarnings...)
} }
if err := recordArtifactExtract(checkpoints, lane.ID, lane.Extract.Module, extractDeps, serializedExtracts, cloneRejectedOutputs(output.Rejected[rejectedStart:]), extractWarnings); err != nil { if err := recordExtract(checkpoints, lane.ID, lane.Extract.Module, extractDeps, serializedExtracts, cloneRejectedOutputs(output.Rejected[rejectedStart:]), extractWarnings); err != nil {
return fmt.Errorf("write extract checkpoint for lane %q: %w", lane.ID, err) return fmt.Errorf("write extract checkpoint for lane %q: %w", lane.ID, err)
} }
} }
sort.SliceStable(values, func(i, j int) bool { return values[i].ChunkIndex < values[j].ChunkIndex }) sort.SliceStable(values, func(i, j int) bool { return values[i].ChunkIndex < values[j].ChunkIndex })
sort.SliceStable(serializedExtracts, func(i, j int) bool { return serializedExtracts[i].ChunkIndex < serializedExtracts[j].ChunkIndex }) sort.SliceStable(serializedExtracts, func(i, j int) bool { return serializedExtracts[i].ChunkIndex < serializedExtracts[j].ChunkIndex })
if err := writeDebugTimed(input.Debug, path.Join("extract", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageExtract), LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": decision.Reused, "outputs": debugArtifactCheckpointOutputs(serializedExtracts), "rejected": debugRejectedOutputEnvelopes(output.Rejected[rejectedStart:]), "warnings": debugWarningEnvelopes(extractWarnings)}}); err != nil { if err := writeDebugTimed(input.Debug, path.Join("extract", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageExtract), LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": decision.Reused, "outputs": debugCheckpointArtifacts(serializedExtracts), "rejected": debugRejectedOutputEnvelopes(output.Rejected[rejectedStart:]), "warnings": debugWarningEnvelopes(extractWarnings)}}); err != nil {
return err return err
} }
if len(values) == 0 { if len(values) == 0 {
@@ -254,18 +230,18 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
mergeInputs[i] = contracts.ExtractArtifact[any]{LaneID: value.LaneID, ExtractorKey: value.ExtractorKey, SourceID: value.SourceID, ChunkID: value.ChunkID, ChunkIndex: value.ChunkIndex, ChunkRef: value.ChunkRef, Value: value.Value} mergeInputs[i] = contracts.ExtractArtifact[any]{LaneID: value.LaneID, ExtractorKey: value.ExtractorKey, SourceID: value.SourceID, ChunkID: value.ChunkID, ChunkIndex: value.ChunkIndex, ChunkRef: value.ChunkRef, Value: value.Value}
} }
mergeDeps := artifactCheckpointDigests(serializedExtracts) mergeDeps := artifactCheckpointDigests(serializedExtracts)
mergeCP, mergeDecision := loadArtifactMerge(loader, lane.ID, lane.Merge.Module, mergeDeps) mergeCP, mergeDecision := loadMerge(loader, lane.ID, lane.Merge.Module, mergeDeps)
if mergeDecision.Reused { if mergeDecision.Reused {
if _, decodeErr := decodeCheckpointArtifact(typed.codec, mergeCP.Output); decodeErr != nil { if _, decodeErr := decodeCheckpointArtifact(typed.codec, mergeCP.Output); decodeErr != nil {
mergeDecision = CheckpointDecision{Reason: "merge artifact checkpoint codec is incompatible: " + decodeErr.Error()} mergeDecision = CheckpointDecision{Reason: "merge artifact checkpoint codec is incompatible: " + decodeErr.Error()}
} }
} }
recordCheckpointEvent(output, loader, string(StageMerge), lane.ID, lane.Merge.Module, mergeDecision) recordCheckpointEvent(output, loader, string(StageMerge), lane.ID, lane.Merge.Module, mergeDecision)
if err := writeDebugTimed(input.Debug, path.Join("merge", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageMerge), LaneID: lane.ID, ModuleKey: lane.Merge.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": mergeDecision.Reused, "decision": mergeDecision, "source": debugSourceDocumentEnvelope(doc), "extract_outputs": debugArtifactCheckpointOutputs(serializedExtracts), "options": redactSensitiveMap(lane.Merge.Options), "metadata": redactSensitiveMap(input.Metadata)}}); err != nil { if err := writeDebugTimed(input.Debug, path.Join("merge", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageMerge), LaneID: lane.ID, ModuleKey: lane.Merge.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": mergeDecision.Reused, "decision": mergeDecision, "source": debugSourceDocumentEnvelope(doc), "extract_outputs": debugCheckpointArtifacts(serializedExtracts), "options": redactSensitiveMap(lane.Merge.Options), "metadata": redactSensitiveMap(input.Metadata)}}); err != nil {
return err return err
} }
var merged erasedMergeArtifact var merged erasedMergeArtifact
var serializedMerge ArtifactCheckpointOutput var serializedMerge CheckpointArtifact
var mergeWarnings []contracts.Warning var mergeWarnings []contracts.Warning
if mergeDecision.Reused { if mergeDecision.Reused {
value, decodeErr := decodeCheckpointArtifact(typed.codec, mergeCP.Output) value, decodeErr := decodeCheckpointArtifact(typed.codec, mergeCP.Output)
@@ -273,7 +249,7 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
return fmt.Errorf("decode merge checkpoint for lane %q: %w", lane.ID, decodeErr) return fmt.Errorf("decode merge checkpoint for lane %q: %w", lane.ID, decodeErr)
} }
merged = erasedMergeArtifact{LaneID: lane.ID, MergerKey: lane.Merge.Module, SourceID: doc.ID, Value: value} merged = erasedMergeArtifact{LaneID: lane.ID, MergerKey: lane.Merge.Module, SourceID: doc.ID, Value: value}
serializedMerge = hydrateCheckpointArtifact(typed.codec, cloneArtifactCheckpointOutput(mergeCP.Output), value) serializedMerge = hydrateCheckpointArtifact(typed.codec, cloneCheckpointArtifact(mergeCP.Output), value)
mergeWarnings = cloneWarnings(mergeCP.Warnings) mergeWarnings = cloneWarnings(mergeCP.Warnings)
output.Warnings = append(output.Warnings, mergeWarnings...) output.Warnings = append(output.Warnings, mergeWarnings...)
} else { } else {
@@ -310,33 +286,33 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
return nil return nil
} }
output.Warnings = append(output.Warnings, mergeWarnings...) output.Warnings = append(output.Warnings, mergeWarnings...)
if err := recordArtifactMerge(checkpoints, lane.ID, lane.Merge.Module, mergeDeps, serializedMerge, mergeWarnings); err != nil { if err := recordMerge(checkpoints, lane.ID, lane.Merge.Module, mergeDeps, serializedMerge, mergeWarnings); err != nil {
return err return err
} }
} }
if err := writeDebugTimed(input.Debug, path.Join("merge", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageMerge), LaneID: lane.ID, ModuleKey: lane.Merge.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": mergeDecision.Reused, "accepted": true, "output": debugArtifactCheckpointOutput(serializedMerge), "warnings": debugWarningEnvelopes(mergeWarnings)}}); err != nil { if err := writeDebugTimed(input.Debug, path.Join("merge", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageMerge), LaneID: lane.ID, ModuleKey: lane.Merge.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": mergeDecision.Reused, "accepted": true, "output": debugCheckpointArtifact(serializedMerge), "warnings": debugWarningEnvelopes(mergeWarnings)}}); err != nil {
return err return err
} }
normalizeDeps := artifactCheckpointDigests([]ArtifactCheckpointOutput{serializedMerge}) normalizeDeps := artifactCheckpointDigests([]CheckpointArtifact{serializedMerge})
normalizeCP, normalizeDecision := loadArtifactNormalize(loader, lane.ID, lane.Normalize.Module, normalizeDeps) normalizeCP, normalizeDecision := loadNormalize(loader, lane.ID, lane.Normalize.Module, normalizeDeps)
if normalizeDecision.Reused { if normalizeDecision.Reused {
if _, decodeErr := decodeCheckpointArtifact(typed.codec, normalizeCP.Output); decodeErr != nil { if _, decodeErr := decodeCheckpointArtifact(typed.codec, normalizeCP.Output); decodeErr != nil {
normalizeDecision = CheckpointDecision{Reason: "normalize artifact checkpoint codec is incompatible: " + decodeErr.Error()} normalizeDecision = CheckpointDecision{Reason: "normalize artifact checkpoint codec is incompatible: " + decodeErr.Error()}
} }
} }
recordCheckpointEvent(output, loader, string(StageNormalize), lane.ID, lane.Normalize.Module, normalizeDecision) recordCheckpointEvent(output, loader, string(StageNormalize), lane.ID, lane.Normalize.Module, normalizeDecision)
if err := writeDebugTimed(input.Debug, path.Join("normalize", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageNormalize), LaneID: lane.ID, ModuleKey: lane.Normalize.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": normalizeDecision.Reused, "decision": normalizeDecision, "source": debugSourceDocumentEnvelope(doc), "merge_output": debugArtifactCheckpointOutput(serializedMerge), "options": redactSensitiveMap(lane.Normalize.Options), "metadata": redactSensitiveMap(input.Metadata)}}); err != nil { if err := writeDebugTimed(input.Debug, path.Join("normalize", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageNormalize), LaneID: lane.ID, ModuleKey: lane.Normalize.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": normalizeDecision.Reused, "decision": normalizeDecision, "source": debugSourceDocumentEnvelope(doc), "merge_output": debugCheckpointArtifact(serializedMerge), "options": redactSensitiveMap(lane.Normalize.Options), "metadata": redactSensitiveMap(input.Metadata)}}); err != nil {
return err return err
} }
var serializedNormalize ArtifactCheckpointOutput var serializedNormalize CheckpointArtifact
var normalizeWarnings []contracts.Warning var normalizeWarnings []contracts.Warning
if normalizeDecision.Reused { if normalizeDecision.Reused {
value, decodeErr := decodeCheckpointArtifact(typed.codec, normalizeCP.Output) value, decodeErr := decodeCheckpointArtifact(typed.codec, normalizeCP.Output)
if decodeErr != nil { if decodeErr != nil {
return fmt.Errorf("decode normalize checkpoint for lane %q: %w", lane.ID, decodeErr) return fmt.Errorf("decode normalize checkpoint for lane %q: %w", lane.ID, decodeErr)
} }
serializedNormalize, normalizeWarnings = hydrateCheckpointArtifact(typed.codec, cloneArtifactCheckpointOutput(normalizeCP.Output), value), cloneWarnings(normalizeCP.Warnings) serializedNormalize, normalizeWarnings = hydrateCheckpointArtifact(typed.codec, cloneCheckpointArtifact(normalizeCP.Output), value), cloneWarnings(normalizeCP.Warnings)
output.Warnings = append(output.Warnings, normalizeWarnings...) output.Warnings = append(output.Warnings, normalizeWarnings...)
} else { } else {
if err := checkpoints.NormalizeRunning(lane.ID, lane.Normalize.Module, normalizeDeps); err != nil { if err := checkpoints.NormalizeRunning(lane.ID, lane.Normalize.Module, normalizeDeps); err != nil {
@@ -371,11 +347,11 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
return nil return nil
} }
output.Warnings = append(output.Warnings, normalizeWarnings...) output.Warnings = append(output.Warnings, normalizeWarnings...)
if err := recordArtifactNormalize(checkpoints, lane.ID, lane.Normalize.Module, normalizeDeps, serializedNormalize, normalizeWarnings); err != nil { if err := recordNormalize(checkpoints, lane.ID, lane.Normalize.Module, normalizeDeps, serializedNormalize, normalizeWarnings); err != nil {
return err return err
} }
} }
if err := writeDebugTimed(input.Debug, path.Join("normalize", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageNormalize), LaneID: lane.ID, ModuleKey: lane.Normalize.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": normalizeDecision.Reused, "accepted": true, "output": debugArtifactCheckpointOutput(serializedNormalize), "warnings": debugWarningEnvelopes(normalizeWarnings)}}); err != nil { if err := writeDebugTimed(input.Debug, path.Join("normalize", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageNormalize), LaneID: lane.ID, ModuleKey: lane.Normalize.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": normalizeDecision.Reused, "accepted": true, "output": debugCheckpointArtifact(serializedNormalize), "warnings": debugWarningEnvelopes(normalizeWarnings)}}); err != nil {
return err return err
} }
output.NormalizeOutputs = append(output.NormalizeOutputs, contracts.SerializedOutput{LaneID: lane.ID, NormalizerKey: lane.Normalize.Module, SourceID: doc.ID, Artifact: contracts.CloneSerializedArtifact(serializedNormalize.Artifact)}) output.NormalizeOutputs = append(output.NormalizeOutputs, contracts.SerializedOutput{LaneID: lane.ID, NormalizerKey: lane.Normalize.Module, SourceID: doc.ID, Artifact: contracts.CloneSerializedArtifact(serializedNormalize.Artifact)})
@@ -430,7 +406,7 @@ func (r *Runner) validateTypedArtifact(ctx context.Context, codec artifactCodecE
return nil, nil, fmt.Errorf("validator %q is incompatible with typed artifact validation", binding.Module) return nil, nil, fmt.Errorf("validator %q is incompatible with typed artifact validation", binding.Module)
} }
artifact, _ := serializeArtifact(codec, target.value, true) artifact, _ := serializeArtifact(codec, target.value, true)
debugCall := debugValidationCall{ValidatorName: binding.Module, Request: map[string]any{"stage": string(target.stage), "lane_id": target.laneID, "module_key": target.moduleKey, "source_id": target.sourceID, "artifact": debugArtifactCheckpointOutput(ArtifactCheckpointOutput{Artifact: artifact, SchemaDigest: contracts.DigestArtifactSchema(artifact.Schema)}), "metadata": redactSensitiveMap(target.metadata)}, Result: debugValidationResultEnvelope(result)} debugCall := debugValidationCall{ValidatorName: binding.Module, Request: map[string]any{"stage": string(target.stage), "lane_id": target.laneID, "module_key": target.moduleKey, "source_id": target.sourceID, "artifact": debugCheckpointArtifact(CheckpointArtifact{Artifact: artifact, SchemaDigest: contracts.DigestArtifactSchema(artifact.Schema)}), "metadata": redactSensitiveMap(target.metadata)}, Result: debugValidationResultEnvelope(result)}
if err != nil { if err != nil {
debugCall.Error = err.Error() debugCall.Error = err.Error()
} }

View File

@@ -20,22 +20,22 @@ func TestDecodeCheckpointArtifactRejectsIncompatibleCodecIdentityAndBytes(t *tes
if err != nil { if err != nil {
t.Fatalf("serializeArtifact: %v", err) t.Fatalf("serializeArtifact: %v", err)
} }
base := ArtifactCheckpointOutput{Artifact: artifact, SchemaDigest: contracts.DigestArtifactSchema(artifact.Schema)} base := CheckpointArtifact{Artifact: artifact, SchemaDigest: contracts.DigestArtifactSchema(artifact.Schema)}
tests := []struct { tests := []struct {
name string name string
mutate func(*ArtifactCheckpointOutput) mutate func(*CheckpointArtifact)
want string want string
}{ }{
{name: "missing kind", mutate: func(v *ArtifactCheckpointOutput) { v.Artifact.Kind = "" }, want: "artifact kind"}, {name: "missing kind", mutate: func(v *CheckpointArtifact) { v.Artifact.Kind = "" }, want: "artifact kind"},
{name: "schema version", mutate: func(v *ArtifactCheckpointOutput) { v.Artifact.Schema.Version = "v999" }, want: "does not match codec schema"}, {name: "schema version", mutate: func(v *CheckpointArtifact) { v.Artifact.Schema.Version = "v999" }, want: "does not match codec schema"},
{name: "schema digest", mutate: func(v *ArtifactCheckpointOutput) { v.SchemaDigest = "sha256:different" }, want: "schema digest"}, {name: "schema digest", mutate: func(v *CheckpointArtifact) { v.SchemaDigest = "sha256:different" }, want: "schema digest"},
{name: "media type", mutate: func(v *ArtifactCheckpointOutput) { v.Artifact.MediaType = "text/plain" }, want: "media type"}, {name: "media type", mutate: func(v *CheckpointArtifact) { v.Artifact.MediaType = "text/plain" }, want: "media type"},
{name: "decode failure", mutate: func(v *ArtifactCheckpointOutput) { v.Artifact.Content = []byte(`{"items":[`) }, want: "unexpected EOF"}, {name: "decode failure", mutate: func(v *CheckpointArtifact) { v.Artifact.Content = []byte(`{"items":[`) }, want: "unexpected EOF"},
} }
for _, test := range tests { for _, test := range tests {
t.Run(test.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
stored := cloneArtifactCheckpointOutput(base) stored := cloneCheckpointArtifact(base)
test.mutate(&stored) test.mutate(&stored)
if _, err := decodeCheckpointArtifact(codec, stored); err == nil || !strings.Contains(err.Error(), test.want) { if _, err := decodeCheckpointArtifact(codec, stored); err == nil || !strings.Contains(err.Error(), test.want) {
t.Fatalf("decode error = %v, want %q", err, test.want) t.Fatalf("decode error = %v, want %q", err, test.want)

View File

@@ -55,6 +55,40 @@ func (typedTestChunkValidator) Validate(context.Context, contracts.ChunkValidati
type typedTestSerializedValidator struct{ key string } type typedTestSerializedValidator struct{ key string }
type typedTestInput struct {
key string
doc *source.SourceDocument
}
func (v *typedTestInput) Key() string { return v.key }
func (v *typedTestInput) Parse(context.Context, contracts.ParseRequest) (*source.SourceDocument, error) {
return v.doc, nil
}
type typedTestChunker struct {
key string
chunks []source.Chunk
}
func (v *typedTestChunker) Key() string { return v.key }
func (v *typedTestChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (v *typedTestChunker) Chunk(context.Context, contracts.ChunkRequest) (contracts.ChunkResult, error) {
return contracts.ChunkResult{Chunks: v.chunks}, nil
}
type typedTestOutput struct{ key string }
func (v *typedTestOutput) Key() string { return v.key }
func (v *typedTestOutput) Encode(context.Context, contracts.OutputRequest) (contracts.OutputResult, error) {
return contracts.OutputResult{}, nil
}
func typedTestDocument() *source.SourceDocument {
ref := source.SourceRef{SourceID: "source", StartUnitID: 1, EndUnitID: 1}
doc := &source.SourceDocument{ID: "source", Kind: "document", Format: "text/plain", Units: []source.SourceUnit{{ID: 1, Kind: "line", Text: "text", Ref: ref}}}
doc.Digest, _ = source.DigestDocument(doc)
return doc
}
func (v typedTestSerializedValidator) Name() string { return v.key } func (v typedTestSerializedValidator) Name() string { return v.key }
func (typedTestSerializedValidator) ExecutionClass() contracts.ExecutionClass { func (typedTestSerializedValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic return contracts.ExecutionClassDeterministic
@@ -177,20 +211,6 @@ func TestResolveTypedLaneRejectsIncompatibleComposition(t *testing.T) {
} }
} }
func TestLegacyRawRegistrationCannotSatisfyTypedLane(t *testing.T) {
options := completeTypedCatalogOptions()
options.registerScoreMerger = false
catalog := typedResolutionCatalog(t, options)
if err := catalog.Mergers.RegisterLegacyRaw("typed/merge", func() (contracts.LegacyRawMerger, error) { return nil, nil }); err != nil {
t.Fatalf("RegisterLegacyRaw() error = %v, want nil", err)
}
_, err := ResolvePipeline(typedResolutionProfile(), ResolveOptions{}, catalog)
if err == nil || !strings.Contains(err.Error(), `no typed variant for artifact kind "test/score"`) {
t.Fatalf("ResolvePipeline() error = %v, want typed variant error", err)
}
}
func TestTypedVariantRegistrationRejectsDuplicates(t *testing.T) { func TestTypedVariantRegistrationRejectsDuplicates(t *testing.T) {
registry := NewMergerRegistry() registry := NewMergerRegistry()
spec := ModuleSpec{Key: "typed/merge", Stage: StageMerge, ArtifactKind: "test/notes"} spec := ModuleSpec{Key: "typed/merge", Stage: StageMerge, ArtifactKind: "test/notes"}
@@ -205,6 +225,48 @@ func TestTypedVariantRegistrationRejectsDuplicates(t *testing.T) {
} }
} }
func TestConstructorRegistrationsRejectUnconfiguredOptions(t *testing.T) {
extractors := NewExtractorRegistry()
if err := RegisterExtractor(extractors, ModuleSpec{Key: "typed/extract", Stage: StageExtract, ArtifactKind: "test/notes"}, func() (contracts.Extractor[codecNotes], error) {
return typedTestExtractor[codecNotes]{key: "typed/extract"}, nil
}); err != nil {
t.Fatalf("RegisterExtractor() error = %v", err)
}
if err := extractors.validateOptions("typed/extract", map[string]any{"unexpected": true}); err == nil || !strings.Contains(err.Error(), "unknown option") {
t.Fatalf("extractor option validation error = %v, want unknown option", err)
}
mergers := NewMergerRegistry()
if err := RegisterMerger(mergers, ModuleSpec{Key: "typed/merge", Stage: StageMerge, ArtifactKind: "test/notes"}, func() (contracts.Merger[codecNotes], error) {
return typedTestMerger[codecNotes]{key: "typed/merge"}, nil
}); err != nil {
t.Fatalf("RegisterMerger() error = %v", err)
}
if err := mergers.validateOptions("typed/merge", "test/notes", map[string]any{"unexpected": true}); err == nil || !strings.Contains(err.Error(), "unknown option") {
t.Fatalf("merger option validation error = %v, want unknown option", err)
}
normalizers := NewNormalizerRegistry()
if err := RegisterNormalizer(normalizers, ModuleSpec{Key: "typed/normalize", Stage: StageNormalize, ArtifactKind: "test/notes"}, func() (contracts.Normalizer[codecNotes], error) {
return typedTestNormalizer[codecNotes]{key: "typed/normalize"}, nil
}); err != nil {
t.Fatalf("RegisterNormalizer() error = %v", err)
}
if err := normalizers.validateOptions("typed/normalize", "test/notes", map[string]any{"unexpected": true}); err == nil || !strings.Contains(err.Error(), "unknown option") {
t.Fatalf("normalizer option validation error = %v, want unknown option", err)
}
validators := NewValidatorRegistry()
if err := RegisterTypedValidator(validators, "test/notes", ValidatorSpec{Key: "typed/check", ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.TypedValidator[codecNotes], error) {
return typedTestValidator[codecNotes]{key: "typed/check"}, nil
}); err != nil {
t.Fatalf("RegisterTypedValidator() error = %v", err)
}
if err := validators.validateOptions(ResolvedValidator{Binding: ModuleBinding{Module: "typed/check", Options: map[string]any{"unexpected": true}}, Target: ValidatorTargetTyped, ArtifactKind: "test/notes"}); err == nil || !strings.Contains(err.Error(), "unknown option") {
t.Fatalf("validator option validation error = %v, want unknown option", err)
}
}
func TestResolvedPipelineDigestIncludesArtifactSchemaIdentity(t *testing.T) { func TestResolvedPipelineDigestIncludesArtifactSchemaIdentity(t *testing.T) {
baseOptions := completeTypedCatalogOptions() baseOptions := completeTypedCatalogOptions()
base, err := ResolvePipeline(typedResolutionProfile(), ResolveOptions{}, typedResolutionCatalog(t, baseOptions)) base, err := ResolvePipeline(typedResolutionProfile(), ResolveOptions{}, typedResolutionCatalog(t, baseOptions))
@@ -302,18 +364,18 @@ func typedResolutionCatalog(t *testing.T, options typedCatalogOptions) ModuleCat
func mustRegisterTypedTestBase(t *testing.T, catalog ModuleCatalog) { func mustRegisterTypedTestBase(t *testing.T, catalog ModuleCatalog) {
t.Helper() t.Helper()
if err := catalog.Inputs.RegisterWithSpec(ModuleSpec{Key: "typed/input", Stage: StageInput}, func() (contracts.InputAdapter, error) { if err := catalog.Inputs.RegisterWithSpec(ModuleSpec{Key: "typed/input", Stage: StageInput}, func() (contracts.InputAdapter, error) {
return &runnerInputAdapter{key: "typed/input", doc: validSourceDocument()}, nil return &typedTestInput{key: "typed/input", doc: typedTestDocument()}, nil
}); err != nil { }); err != nil {
t.Fatalf("register input: %v", err) t.Fatalf("register input: %v", err)
} }
if err := catalog.Chunkers.RegisterWithSpec(ModuleSpec{Key: "typed/chunk", Stage: StageChunk}, func() (contracts.Chunker, error) { if err := catalog.Chunkers.RegisterWithSpec(ModuleSpec{Key: "typed/chunk", Stage: StageChunk}, func() (contracts.Chunker, error) {
doc := validSourceDocument() doc := typedTestDocument()
return &runnerChunker{key: "typed/chunk", chunks: []source.Chunk{{ID: "chunk-1", SourceID: doc.ID, Index: 0, Ref: doc.Units[0].Ref, Content: []byte(`{"chunk":1}`), MediaType: "application/json", Units: []source.SourceUnit{doc.Units[0]}}}}, nil return &typedTestChunker{key: "typed/chunk", chunks: []source.Chunk{{ID: "chunk-1", SourceID: doc.ID, Index: 0, Ref: doc.Units[0].Ref, Content: []byte(`{"chunk":1}`), MediaType: "application/json", Units: []source.SourceUnit{doc.Units[0]}}}}, nil
}); err != nil { }); err != nil {
t.Fatalf("register chunker: %v", err) t.Fatalf("register chunker: %v", err)
} }
if err := catalog.Outputs.RegisterWithSpec(ModuleSpec{Key: "typed/output", Stage: StageOutput}, func() (contracts.OutputEncoder, error) { if err := catalog.Outputs.RegisterWithSpec(ModuleSpec{Key: "typed/output", Stage: StageOutput}, func() (contracts.OutputEncoder, error) {
return &runnerOutputEncoder{key: "typed/output"}, nil return &typedTestOutput{key: "typed/output"}, nil
}); err != nil { }); err != nil {
t.Fatalf("register output: %v", err) t.Fatalf("register output: %v", err)
} }

View File

@@ -10,9 +10,6 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
) )
type LegacyRawValidatorConstructor func() (contracts.LegacyRawValidator, error)
type LegacyRawValidatorBuilder func(BuildRequest) (contracts.LegacyRawValidator, error)
type ValidatorSpec struct { type ValidatorSpec struct {
Key string `json:"key"` Key string `json:"key"`
ExecutionClass contracts.ExecutionClass `json:"execution_class"` ExecutionClass contracts.ExecutionClass `json:"execution_class"`
@@ -27,16 +24,12 @@ type SerializedValidatorSpec struct {
type ValidatorTarget string type ValidatorTarget string
const ( const (
ValidatorTargetLegacyRaw ValidatorTarget = "legacy_raw"
ValidatorTargetChunk ValidatorTarget = "chunk" ValidatorTargetChunk ValidatorTarget = "chunk"
ValidatorTargetSerialized ValidatorTarget = "serialized" ValidatorTargetSerialized ValidatorTarget = "serialized"
ValidatorTargetTyped ValidatorTarget = "typed" ValidatorTargetTyped ValidatorTarget = "typed"
) )
type ValidatorRegistry struct { type ValidatorRegistry struct {
legacyBuilders map[string]LegacyRawValidatorBuilder
legacyValidators map[string]OptionValidator
legacySpecs map[string]ValidatorSpec
typedEntries map[artifactVariantKey]typedValidatorEntry typedEntries map[artifactVariantKey]typedValidatorEntry
chunkEntries map[string]chunkValidatorEntry chunkEntries map[string]chunkValidatorEntry
serializedEntries map[string]serializedValidatorEntry serializedEntries map[string]serializedValidatorEntry
@@ -65,65 +58,17 @@ type serializedValidatorEntry struct {
func NewValidatorRegistry() *ValidatorRegistry { func NewValidatorRegistry() *ValidatorRegistry {
return &ValidatorRegistry{ return &ValidatorRegistry{
legacyBuilders: make(map[string]LegacyRawValidatorBuilder),
legacyValidators: make(map[string]OptionValidator),
legacySpecs: make(map[string]ValidatorSpec),
typedEntries: make(map[artifactVariantKey]typedValidatorEntry), typedEntries: make(map[artifactVariantKey]typedValidatorEntry),
chunkEntries: make(map[string]chunkValidatorEntry), chunkEntries: make(map[string]chunkValidatorEntry),
serializedEntries: make(map[string]serializedValidatorEntry), serializedEntries: make(map[string]serializedValidatorEntry),
} }
} }
func (r *ValidatorRegistry) RegisterLegacyRaw(key string, constructor LegacyRawValidatorConstructor) error {
return r.RegisterLegacyRawWithSpec(ValidatorSpec{Key: key, ExecutionClass: contracts.ExecutionClassDeterministic}, constructor)
}
func (r *ValidatorRegistry) RegisterLegacyRawWithSpec(spec ValidatorSpec, constructor LegacyRawValidatorConstructor) error {
if constructor == nil {
return fmt.Errorf("validator constructor for %q must not be nil", strings.TrimSpace(spec.Key))
}
return r.RegisterLegacyRawBuilderWithSpec(spec, allowLegacyOptions, func(BuildRequest) (contracts.LegacyRawValidator, error) {
return constructor()
})
}
func (r *ValidatorRegistry) RegisterLegacyRawBuilderWithSpec(spec ValidatorSpec, validateOptions OptionValidator, builder LegacyRawValidatorBuilder) error {
if r == nil {
return fmt.Errorf("validator registry must not be nil")
}
normalizedSpec, err := normalizeValidatorSpec(spec)
if err != nil {
return err
}
if validateOptions == nil {
return fmt.Errorf("validator option validator for %q must not be nil", normalizedSpec.Key)
}
if builder == nil {
return fmt.Errorf("validator builder for %q must not be nil", normalizedSpec.Key)
}
if _, ok := r.legacyBuilders[normalizedSpec.Key]; ok {
return fmt.Errorf("legacy raw validator %q is already registered", normalizedSpec.Key)
}
if r.legacyBuilders == nil {
r.legacyBuilders = make(map[string]LegacyRawValidatorBuilder)
}
if r.legacyValidators == nil {
r.legacyValidators = make(map[string]OptionValidator)
}
if r.legacySpecs == nil {
r.legacySpecs = make(map[string]ValidatorSpec)
}
r.legacyBuilders[normalizedSpec.Key] = builder
r.legacyValidators[normalizedSpec.Key] = validateOptions
r.legacySpecs[normalizedSpec.Key] = normalizedSpec
return nil
}
func RegisterTypedValidator[T any](registry *ValidatorRegistry, kind contracts.ArtifactKind, spec ValidatorSpec, constructor func() (contracts.TypedValidator[T], error)) error { func RegisterTypedValidator[T any](registry *ValidatorRegistry, kind contracts.ArtifactKind, spec ValidatorSpec, constructor func() (contracts.TypedValidator[T], error)) error {
if constructor == nil { if constructor == nil {
return fmt.Errorf("validator constructor for %q must not be nil", strings.TrimSpace(spec.Key)) return fmt.Errorf("validator constructor for %q must not be nil", strings.TrimSpace(spec.Key))
} }
return RegisterTypedValidatorBuilder(registry, kind, spec, allowLegacyOptions, func(BuildRequest) (contracts.TypedValidator[T], error) { return RegisterTypedValidatorBuilder(registry, kind, spec, rejectUnconfiguredOptions, func(BuildRequest) (contracts.TypedValidator[T], error) {
return constructor() return constructor()
}) })
} }
@@ -180,7 +125,7 @@ func RegisterChunkValidator(registry *ValidatorRegistry, spec ValidatorSpec, con
if constructor == nil { if constructor == nil {
return fmt.Errorf("validator constructor for %q must not be nil", strings.TrimSpace(spec.Key)) return fmt.Errorf("validator constructor for %q must not be nil", strings.TrimSpace(spec.Key))
} }
return RegisterChunkValidatorBuilder(registry, spec, allowLegacyOptions, func(BuildRequest) (contracts.ChunkValidator, error) { return RegisterChunkValidatorBuilder(registry, spec, rejectUnconfiguredOptions, func(BuildRequest) (contracts.ChunkValidator, error) {
return constructor() return constructor()
}) })
} }
@@ -213,7 +158,7 @@ func RegisterSerializedValidator(registry *ValidatorRegistry, spec SerializedVal
if constructor == nil { if constructor == nil {
return fmt.Errorf("validator constructor for %q must not be nil", strings.TrimSpace(spec.Key)) return fmt.Errorf("validator constructor for %q must not be nil", strings.TrimSpace(spec.Key))
} }
return RegisterSerializedValidatorBuilder(registry, spec, allowLegacyOptions, func(BuildRequest) (contracts.SerializedValidator, error) { return RegisterSerializedValidatorBuilder(registry, spec, rejectUnconfiguredOptions, func(BuildRequest) (contracts.SerializedValidator, error) {
return constructor() return constructor()
}) })
} }
@@ -246,39 +191,6 @@ func RegisterSerializedValidatorBuilder(registry *ValidatorRegistry, spec Serial
return nil return nil
} }
func (r *ValidatorRegistry) BuildLegacyRaw(key string) (contracts.LegacyRawValidator, error) {
return r.BuildLegacyRawWithRequest(key, BuildRequest{})
}
func (r *ValidatorRegistry) BuildLegacyRawWithRequest(key string, request BuildRequest) (contracts.LegacyRawValidator, error) {
if r == nil {
return nil, fmt.Errorf("validator registry must not be nil")
}
normalizedKey := strings.TrimSpace(key)
if normalizedKey == "" {
return nil, fmt.Errorf("validator key must not be empty")
}
builder, ok := r.legacyBuilders[normalizedKey]
if !ok {
return nil, fmt.Errorf("legacy raw validator %q is not registered", normalizedKey)
}
validator, err := builder(cloneBuildRequest(request))
if err != nil {
return nil, fmt.Errorf("build validator %q: %w", normalizedKey, err)
}
if validator == nil {
return nil, fmt.Errorf("validator %q constructor returned nil", normalizedKey)
}
if validator.Name() != normalizedKey {
return nil, fmt.Errorf("validator %q returned name %q", normalizedKey, validator.Name())
}
spec := r.legacySpecs[normalizedKey]
if validator.ExecutionClass() != spec.ExecutionClass {
return nil, fmt.Errorf("validator %q returned execution class %q, want %q", normalizedKey, validator.ExecutionClass(), spec.ExecutionClass)
}
return validator, nil
}
func (r *ValidatorRegistry) validateOptions(resolved ResolvedValidator) error { func (r *ValidatorRegistry) validateOptions(resolved ResolvedValidator) error {
if r == nil { if r == nil {
return fmt.Errorf("validator registry must not be nil") return fmt.Errorf("validator registry must not be nil")
@@ -301,8 +213,6 @@ func (r *ValidatorRegistry) validateOptions(resolved ResolvedValidator) error {
if ok { if ok {
validator = entry.validateOptions validator = entry.validateOptions
} }
default:
validator = r.legacyValidators[key]
} }
if validator == nil { if validator == nil {
return fmt.Errorf("validator %q construction entry is not registered", key) return fmt.Errorf("validator %q construction entry is not registered", key)
@@ -314,10 +224,6 @@ func (r *ValidatorRegistry) Spec(key string) (ValidatorSpec, bool) {
if r == nil { if r == nil {
return ValidatorSpec{}, false return ValidatorSpec{}, false
} }
spec, ok := r.legacySpecs[strings.TrimSpace(key)]
if ok {
return spec, true
}
normalized := strings.TrimSpace(key) normalized := strings.TrimSpace(key)
if entry, found := r.chunkEntries[normalized]; found { if entry, found := r.chunkEntries[normalized]; found {
return entry.spec, true return entry.spec, true
@@ -329,7 +235,7 @@ func (r *ValidatorRegistry) Spec(key string) (ValidatorSpec, bool) {
entry, found := r.typedEntry(normalized, kinds[0]) entry, found := r.typedEntry(normalized, kinds[0])
return entry.spec, found return entry.spec, found
} }
return spec, ok return ValidatorSpec{}, false
} }
func (r *ValidatorRegistry) typedEntry(key string, kind contracts.ArtifactKind) (typedValidatorEntry, bool) { func (r *ValidatorRegistry) typedEntry(key string, kind contracts.ArtifactKind) (typedValidatorEntry, bool) {
@@ -372,13 +278,15 @@ func (r *ValidatorRegistry) registeredTypedKinds(key string) []contracts.Artifac
} }
func (r *ValidatorRegistry) RegisteredSpecs() []ValidatorSpec { func (r *ValidatorRegistry) RegisteredSpecs() []ValidatorSpec {
if r == nil || len(r.legacySpecs) == 0 { if r == nil {
return nil return nil
} }
keys := sortedRegistryKeys(r.legacySpecs) keys := r.RegisteredKeys()
specs := make([]ValidatorSpec, 0, len(keys)) specs := make([]ValidatorSpec, 0, len(keys))
for _, key := range keys { for _, key := range keys {
specs = append(specs, r.legacySpecs[key]) if spec, ok := r.Spec(key); ok {
specs = append(specs, spec)
}
} }
return specs return specs
} }
@@ -388,9 +296,6 @@ func (r *ValidatorRegistry) RegisteredKeys() []string {
return nil return nil
} }
keys := make(map[string]struct{}) keys := make(map[string]struct{})
for key := range r.legacySpecs {
keys[key] = struct{}{}
}
for key := range r.typedEntries { for key := range r.typedEntries {
keys[key.module] = struct{}{} keys[key.module] = struct{}{}
} }

View File

@@ -1,117 +0,0 @@
package pipeline
import (
"context"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestValidatorRegistryBehavior(t *testing.T) {
registry := NewValidatorRegistry()
if err := registry.RegisterLegacyRaw(" generic-validator ", validatorConstructor("generic-validator", contracts.ExecutionClassDeterministic)); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
validator, err := registry.BuildLegacyRaw("generic-validator")
if err != nil {
t.Fatalf("Build() error = %v, want nil", err)
}
if validator.Name() != "generic-validator" {
t.Fatalf("validator name = %q, want generic-validator", validator.Name())
}
spec, ok := registry.Spec(" generic-validator ")
if !ok {
t.Fatal("Spec() ok = false, want true")
}
want := ValidatorSpec{Key: "generic-validator", ExecutionClass: contracts.ExecutionClassDeterministic}
if !reflect.DeepEqual(spec, want) {
t.Fatalf("Spec() = %#v, want %#v", spec, want)
}
}
func TestValidatorRegistryRegistersSpecs(t *testing.T) {
registry := NewValidatorRegistry()
spec := ValidatorSpec{Key: " llm-validator ", ExecutionClass: contracts.ExecutionClassLLMBacked}
if err := registry.RegisterLegacyRawWithSpec(spec, validatorConstructor("llm-validator", contracts.ExecutionClassLLMBacked)); err != nil {
t.Fatalf("RegisterWithSpec() error = %v, want nil", err)
}
got, ok := registry.Spec("llm-validator")
if !ok {
t.Fatal("Spec() ok = false, want true")
}
want := ValidatorSpec{Key: "llm-validator", ExecutionClass: contracts.ExecutionClassLLMBacked}
if !reflect.DeepEqual(got, want) {
t.Fatalf("Spec() = %#v, want %#v", got, want)
}
}
func TestValidatorRegistryRegisteredSpecsAreSorted(t *testing.T) {
registry := NewValidatorRegistry()
for _, key := range []string{"zeta", "alpha"} {
if err := registry.RegisterLegacyRaw(key, validatorConstructor(key, contracts.ExecutionClassDeterministic)); err != nil {
t.Fatalf("Register(%q) error = %v", key, err)
}
}
specs := registry.RegisteredSpecs()
if len(specs) != 2 || specs[0].Key != "alpha" || specs[1].Key != "zeta" {
t.Fatalf("RegisteredSpecs() = %#v, want sorted specs", specs)
}
}
func TestValidatorRegistryRejectsUnsupportedExecutionClass(t *testing.T) {
registry := NewValidatorRegistry()
err := registry.RegisterLegacyRawWithSpec(
ValidatorSpec{Key: "invalid-validator", ExecutionClass: contracts.ExecutionClass("unsupported")},
validatorConstructor("invalid-validator", contracts.ExecutionClass("unsupported")),
)
if err == nil {
t.Fatal("RegisterWithSpec() error = nil, want unsupported execution class error")
}
}
func TestValidatorRegistryRejectsConstructorExecutionClassMismatch(t *testing.T) {
registry := NewValidatorRegistry()
if err := registry.RegisterLegacyRawWithSpec(
ValidatorSpec{Key: "validator", ExecutionClass: contracts.ExecutionClassDeterministic},
validatorConstructor("validator", contracts.ExecutionClassLLMBacked),
); err != nil {
t.Fatalf("RegisterWithSpec() error = %v, want nil", err)
}
_, err := registry.BuildLegacyRaw("validator")
if err == nil {
t.Fatal("Build() error = nil, want execution class mismatch")
}
if !strings.Contains(err.Error(), "execution class") {
t.Fatalf("Build() error = %q, want execution class context", err.Error())
}
}
type testValidator struct {
name string
executionClass contracts.ExecutionClass
}
func validatorConstructor(name string, executionClass contracts.ExecutionClass) LegacyRawValidatorConstructor {
return func() (contracts.LegacyRawValidator, error) {
return testValidator{name: name, executionClass: executionClass}, nil
}
}
func (validator testValidator) Name() string {
return validator.name
}
func (validator testValidator) ExecutionClass() contracts.ExecutionClass {
return validator.executionClass
}
func (validator testValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: true}, nil
}

View File

@@ -1,467 +0,0 @@
package pipeline
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"os"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestWalkingSkeletonFixture(t *testing.T) {
inputBytes := readTestFixture(t, "testdata/walking_skeleton_input.json")
expectedBytes := readTestFixture(t, "testdata/walking_skeleton_output.json")
llmClient := &walkingSkeletonLLMClient{}
resolved, err := ResolvePipeline(walkingSkeletonProfile(), ResolveOptions{}, walkingSkeletonCatalog(t))
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
output, err := newPreparedRunner(t, walkingSkeletonRegistries(t)).Run(context.Background(), RunInput{
pipeline: resolved,
SourceID: "fixture-source",
Path: "walking_skeleton_input.json",
RawInput: inputBytes,
llmClient: llmClient,
})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.OutputFiles) != 1 {
t.Fatalf("len(OutputFiles) = %d, want 1", len(output.OutputFiles))
}
if output.OutputFiles[0].ContentType != "application/json" {
t.Fatalf("ContentType = %q, want application/json", output.OutputFiles[0].ContentType)
}
assertStructuralJSONEqual(t, output.OutputFiles[0].Bytes, expectedBytes)
if llmClient.calls != 3 {
t.Fatalf("LLM calls = %d, want extractor calls plus normalizer call", llmClient.calls)
}
}
func TestWalkingSkeletonResolutionRejectsMissingCapability(t *testing.T) {
catalog := walkingSkeletonCatalog(t)
catalog.Extractors = NewExtractorRegistry()
if err := catalog.Extractors.RegisterLegacyRawWithSpec(ModuleSpec{
Key: "fake/extract",
Stage: StageExtract,
Requires: []string{"missing"},
Provides: []string{"fake_artifacts"},
}, func() (contracts.LegacyRawExtractor, error) {
return walkingSkeletonExtractor{}, nil
}); err != nil {
t.Fatalf("RegisterWithSpec() error = %v, want nil", err)
}
_, err := ResolvePipeline(walkingSkeletonProfile(), ResolveOptions{}, catalog)
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want error")
}
if !strings.Contains(err.Error(), "missing") {
t.Fatalf("ResolvePipeline() error = %q, want missing capability", err.Error())
}
}
func TestWalkingSkeletonResolutionRejectsUnknownOnlyLane(t *testing.T) {
_, err := ResolvePipeline(walkingSkeletonProfile(), ResolveOptions{Only: []string{"missing"}}, walkingSkeletonCatalog(t))
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want error")
}
if !strings.Contains(err.Error(), "missing") || !strings.Contains(err.Error(), "not declared") {
t.Fatalf("ResolvePipeline() error = %q, want unknown lane error", err.Error())
}
}
func walkingSkeletonProfile() PipelineProfile {
return PipelineProfile{
ID: "walking-skeleton",
Input: Binding("fake/input"),
Chunk: Binding("fake/chunk"),
Output: Binding("json"),
Artifacts: map[string]ArtifactLaneProfile{
"events": {Extract: Binding("fake/extract")},
},
}
}
func walkingSkeletonCatalog(t *testing.T) ModuleCatalog {
t.Helper()
catalog := ModuleCatalog{
Inputs: NewInputAdapterRegistry(),
Chunkers: NewChunkerRegistry(),
ArtifactCodecs: NewArtifactCodecRegistry(),
Extractors: NewExtractorRegistry(),
Mergers: NewMergerRegistry(),
Normalizers: NewNormalizerRegistry(),
ValidatorChains: NewValidatorChainRegistry(),
Outputs: NewOutputEncoderRegistry(),
}
if err := catalog.Inputs.RegisterWithSpec(ModuleSpec{
Key: "fake/input",
Stage: StageInput,
Provides: []string{"plain_text"},
}, func() (contracts.InputAdapter, error) {
return walkingSkeletonInput{}, nil
}); err != nil {
t.Fatalf("register fake input: %v", err)
}
if err := catalog.Chunkers.RegisterWithSpec(ModuleSpec{
Key: "fake/chunk",
Stage: StageChunk,
Requires: []string{"plain_text"},
Provides: []string{"chunks"},
}, func() (contracts.Chunker, error) {
return walkingSkeletonChunker{}, nil
}); err != nil {
t.Fatalf("register fake chunker: %v", err)
}
if err := catalog.Extractors.RegisterLegacyRawWithSpec(ModuleSpec{
Key: "fake/extract",
Stage: StageExtract,
Requires: []string{"chunks"},
Provides: []string{"fake_artifacts"},
}, func() (contracts.LegacyRawExtractor, error) {
return walkingSkeletonExtractor{}, nil
}); err != nil {
t.Fatalf("register fake extractor: %v", err)
}
if err := catalog.Mergers.RegisterLegacyRawWithSpec(ModuleSpec{
Key: DefaultMergeModule,
Stage: StageMerge,
Requires: []string{"fake_artifacts"},
}, func() (contracts.LegacyRawMerger, error) {
return walkingSkeletonMerger{}, nil
}); err != nil {
t.Fatalf("register append-order merger: %v", err)
}
if err := catalog.Normalizers.RegisterLegacyRawWithSpec(ModuleSpec{
Key: DefaultNormalizeModule,
Stage: StageNormalize,
}, func() (contracts.LegacyRawNormalizer, error) {
return walkingSkeletonNormalizer{}, nil
}); err != nil {
t.Fatalf("register no-op normalizer: %v", err)
}
if err := catalog.Outputs.RegisterWithSpec(ModuleSpec{
Key: "json",
Stage: StageOutput,
}, func() (contracts.OutputEncoder, error) {
return walkingSkeletonOutput{}, nil
}); err != nil {
t.Fatalf("register fake output: %v", err)
}
return catalog
}
func walkingSkeletonRegistries(t *testing.T) Registries {
t.Helper()
catalog := walkingSkeletonCatalog(t)
return Registries{
Inputs: catalog.Inputs,
Chunkers: catalog.Chunkers,
ArtifactCodecs: catalog.ArtifactCodecs,
Extractors: catalog.Extractors,
Mergers: catalog.Mergers,
Normalizers: catalog.Normalizers,
Outputs: catalog.Outputs,
}
}
type walkingSkeletonInput struct{}
func (input walkingSkeletonInput) Key() string {
return "fake/input"
}
func (input walkingSkeletonInput) Parse(ctx context.Context, req contracts.ParseRequest) (*source.SourceDocument, error) {
var fixture struct {
ID string `json:"id"`
Units []struct {
ID int `json:"id"`
Text string `json:"text"`
} `json:"units"`
}
if err := json.Unmarshal(req.Raw, &fixture); err != nil {
return nil, err
}
units := make([]source.SourceUnit, 0, len(fixture.Units))
for _, unit := range fixture.Units {
units = append(units, source.SourceUnit{
ID: unit.ID,
Kind: "unit",
Text: unit.Text,
Ref: source.SourceRef{
SourceID: fixture.ID,
StartUnitID: unit.ID,
EndUnitID: unit.ID,
},
})
}
return &source.SourceDocument{
ID: fixture.ID,
Kind: "fixture",
Format: "application/json",
Digest: rawDigest(req.Raw),
Units: units,
}, nil
}
type walkingSkeletonChunker struct{}
func (chunker walkingSkeletonChunker) Key() string {
return "fake/chunk"
}
func (chunker walkingSkeletonChunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (chunker walkingSkeletonChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
if len(req.Source.Units) < 3 {
return contracts.ChunkResult{}, fmt.Errorf("fixture source must contain at least three units")
}
return contracts.ChunkResult{
Chunks: []source.Chunk{
{
ID: req.Source.ID + ":chunk:0",
SourceID: req.Source.ID,
Index: 0,
Ref: source.SourceRef{SourceID: req.Source.ID, StartUnitID: req.Source.Units[0].ID, EndUnitID: req.Source.Units[1].ID},
Content: []byte(`{"units":[1,2]}`),
MediaType: "application/json",
Units: append([]source.SourceUnit(nil), req.Source.Units[:2]...),
},
{
ID: req.Source.ID + ":chunk:1",
SourceID: req.Source.ID,
Index: 1,
Ref: source.SourceRef{SourceID: req.Source.ID, StartUnitID: req.Source.Units[2].ID, EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID},
Content: []byte(`{"units":[3]}`),
MediaType: "application/json",
Units: append([]source.SourceUnit(nil), req.Source.Units[2:]...),
},
},
}, nil
}
type walkingSkeletonExtractor struct{}
func (extractor walkingSkeletonExtractor) Key() string {
return "fake/extract"
}
func (extractor walkingSkeletonExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (extractor walkingSkeletonExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
var response struct {
Call int `json:"call"`
}
if _, err := req.LLMClient.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
StageName: "fake/extract",
PromptID: "fake.event",
PromptVersion: "v1",
}, &response); err != nil {
return contracts.ExtractionResult{}, err
}
payload, err := json.Marshal(map[string]any{
"chunk_id": req.Chunk.ID,
"llm_call": response.Call,
"text": chunkText(req.Chunk.Units),
})
if err != nil {
return contracts.ExtractionResult{}, err
}
return contracts.ExtractionResult{
Output: contracts.ExtractOutput{
Schema: contracts.ResponseSchema{ID: "fake_event", Name: "fake_event", Version: "v1"},
Payload: contracts.RawPayload{
Content: payload,
MediaType: "application/json",
},
},
}, nil
}
type walkingSkeletonLLMClient struct {
calls int
}
func (client *walkingSkeletonLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
client.calls++
if response, ok := out.(*struct {
Call int `json:"call"`
}); ok {
response.Call = client.calls
}
content, err := json.Marshal(map[string]any{"call": client.calls})
if err != nil {
return contracts.StructuredCompletionResponse{}, err
}
return contracts.StructuredCompletionResponse{
Content: content,
}, nil
}
type walkingSkeletonMerger struct{}
func (merger walkingSkeletonMerger) Key() string {
return DefaultMergeModule
}
func (merger walkingSkeletonMerger) Merge(ctx context.Context, req contracts.MergeRequest) (contracts.MergeResult, error) {
outputs := make([]json.RawMessage, 0, len(req.ExtractOutputs))
for _, output := range req.ExtractOutputs {
outputs = append(outputs, json.RawMessage(output.Payload.Content))
}
content, err := json.Marshal(map[string]any{"outputs": outputs})
if err != nil {
return contracts.MergeResult{}, err
}
return contracts.MergeResult{
Output: contracts.MergeOutput{
LaneID: req.LaneID,
SourceID: req.Source.ID,
Schema: contracts.ResponseSchema{ID: "fake_event", Name: "fake_event", Version: "v1"},
Payload: contracts.RawPayload{
Content: content,
MediaType: "application/json",
},
},
}, nil
}
type walkingSkeletonNormalizer struct{}
func (normalizer walkingSkeletonNormalizer) Key() string {
return DefaultNormalizeModule
}
func (normalizer walkingSkeletonNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (normalizer walkingSkeletonNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
var response struct {
Call int `json:"call"`
}
if _, err := req.LLMClient.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
StageName: "fake/normalize",
PromptID: "fake.normalize",
PromptVersion: "v1",
}, &response); err != nil {
return contracts.NormalizeResult{}, err
}
return contracts.NormalizeResult{
Output: contracts.NormalizeOutput{
LaneID: req.LaneID,
SourceID: req.MergeOutput.SourceID,
Schema: req.MergeOutput.Schema,
Payload: req.MergeOutput.Payload,
},
}, nil
}
type walkingSkeletonOutput struct{}
func (output walkingSkeletonOutput) Key() string {
return "json"
}
func (output walkingSkeletonOutput) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
type rawOutput struct {
LaneID string `json:"lane_id"`
NormalizerKey string `json:"normalizer_key"`
SourceID string `json:"source_id"`
Schema contracts.ResponseSchema `json:"schema"`
MediaType string `json:"media_type"`
Content json.RawMessage `json:"content"`
}
rawOutputs := make([]rawOutput, 0, len(req.NormalizeOutputs))
for _, output := range req.NormalizeOutputs {
rawOutputs = append(rawOutputs, rawOutput{
LaneID: output.LaneID,
NormalizerKey: output.NormalizerKey,
SourceID: output.SourceID,
Schema: contracts.ResponseSchema{ID: output.Artifact.Schema.ID, Name: output.Artifact.Schema.Name, Version: output.Artifact.Schema.Version},
MediaType: output.Artifact.MediaType,
Content: json.RawMessage(output.Artifact.Content),
})
}
encoded, err := json.Marshal(struct {
Manifest artifacts.RunManifest `json:"manifest"`
NormalizeOutputs []rawOutput `json:"normalize_outputs"`
}{
Manifest: artifacts.RunManifest{
PipelineID: req.Manifest.PipelineID,
PipelineDigest: req.Manifest.PipelineDigest,
ArtifactLanes: req.Manifest.ArtifactLanes,
ValidationStatus: req.Manifest.ValidationStatus,
},
NormalizeOutputs: rawOutputs,
})
if err != nil {
return contracts.OutputResult{}, err
}
return contracts.OutputResult{
Files: []contracts.OutputFile{
{Name: "output.json", ContentType: "application/json", Bytes: encoded},
},
}, nil
}
func readTestFixture(t *testing.T, path string) []byte {
t.Helper()
bytes, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read fixture %q: %v", path, err)
}
return bytes
}
func assertStructuralJSONEqual(t *testing.T, gotBytes, wantBytes []byte) {
t.Helper()
var got any
if err := json.Unmarshal(gotBytes, &got); err != nil {
t.Fatalf("unmarshal actual JSON: %v\n%s", err, gotBytes)
}
var want any
if err := json.Unmarshal(wantBytes, &want); err != nil {
t.Fatalf("unmarshal expected JSON: %v\n%s", err, wantBytes)
}
if !reflect.DeepEqual(got, want) {
gotFormatted, _ := json.MarshalIndent(got, "", " ")
wantFormatted, _ := json.MarshalIndent(want, "", " ")
t.Fatalf("actual JSON:\n%s\nwant:\n%s", gotFormatted, wantFormatted)
}
}
func chunkText(units []source.SourceUnit) string {
parts := make([]string, 0, len(units))
for _, unit := range units {
parts = append(parts, unit.Text)
}
return strings.Join(parts, " ")
}
func rawDigest(raw []byte) string {
sum := sha256.Sum256(raw)
return "sha256:" + hex.EncodeToString(sum[:])
}

View File

@@ -52,7 +52,7 @@ func (c *Codec) Encode(value dnd.SpellList) ([]byte, error) {
} }
// EncodeCandidate provides the same stable representation before typed // EncodeCandidate provides the same stable representation before typed
// validators have approved a value on the temporary raw downstream path. // validators have approved a value.
func (c *Codec) EncodeCandidate(value dnd.SpellList) ([]byte, error) { func (c *Codec) EncodeCandidate(value dnd.SpellList) ([]byte, error) {
content, err := json.Marshal(value) content, err := json.Marshal(value)
if err != nil { if err != nil {
@@ -73,7 +73,7 @@ func (c *Codec) Decode(content []byte) (dnd.SpellList, error) {
} }
// DecodeCandidate reads the durable representation before semantic validators // DecodeCandidate reads the durable representation before semantic validators
// have approved it on the temporary raw runner path. // have approved it.
func (c *Codec) DecodeCandidate(content []byte) (dnd.SpellList, error) { func (c *Codec) DecodeCandidate(content []byte) (dnd.SpellList, error) {
decoder := json.NewDecoder(bytes.NewReader(content)) decoder := json.NewDecoder(bytes.NewReader(content))
decoder.DisallowUnknownFields() decoder.DisallowUnknownFields()

View File

@@ -39,18 +39,6 @@ type Extractor struct {
llm contracts.StructuredLLMClient llm contracts.StructuredLLMClient
} }
type rawAdapter struct {
extractor *Extractor
codec RawAdapterCodec
}
type RawAdapterCodec interface {
contracts.ArtifactCodec[dnd.SpellList]
EncodeCandidate(dnd.SpellList) ([]byte, error)
}
var _ contracts.LegacyRawExtractor = (*rawAdapter)(nil)
func New(llmClient contracts.StructuredLLMClient, _ Options) (*Extractor, error) { func New(llmClient contracts.StructuredLLMClient, _ Options) (*Extractor, error) {
if llmClient == nil { if llmClient == nil {
return nil, extractorErrorf("LLM client must not be nil") return nil, extractorErrorf("LLM client must not be nil")
@@ -169,66 +157,6 @@ func Register(registry *pipeline.ExtractorRegistry) error {
}) })
} }
// RegisterWithRawAdapter keeps existing raw downstream implementations usable
// while the extractor itself produces the canonical typed artifact.
func RegisterWithRawAdapter(registry *pipeline.ExtractorRegistry, codec RawAdapterCodec) error {
if codec == nil {
return extractorErrorf("artifact codec must not be nil")
}
build := func(request pipeline.BuildRequest) (*Extractor, error) {
options, err := DecodeOptions(request.Options)
if err != nil {
return nil, err
}
return New(request.Dependencies.LLM, options)
}
return pipeline.RegisterExtractorBuilderWithRawAdapter(registry, ModuleSpec(), validateOptions,
func(request pipeline.BuildRequest) (contracts.Extractor[dnd.SpellList], error) {
return build(request)
},
func(request pipeline.BuildRequest) (contracts.LegacyRawExtractor, error) {
extractor, err := build(request)
if err != nil {
return nil, err
}
return &rawAdapter{extractor: extractor, codec: codec}, nil
},
)
}
func (adapter *rawAdapter) Key() string { return Key }
func (adapter *rawAdapter) ReferenceSlots() []contracts.ReferenceSlot {
return adapter.extractor.ReferenceSlots()
}
func (adapter *rawAdapter) ManifestMetadata() map[string]any {
return adapter.extractor.ManifestMetadata()
}
func (adapter *rawAdapter) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
result, err := adapter.extractor.Extract(ctx, contracts.TypedExtractionRequest{
Source: req.Source, Chunk: req.Chunk, AmbientContext: req.AmbientContext,
SourceInput: req.SourceInput, SessionID: req.SessionID, References: req.References,
LLMProfile: req.LLMProfile, Metadata: req.Metadata,
})
if err != nil {
return contracts.ExtractionResult{}, err
}
content, err := adapter.codec.EncodeCandidate(result.Value)
if err != nil {
return contracts.ExtractionResult{}, extractorErrorf("encode canonical output: %w", err)
}
schema := adapter.codec.Schema()
return contracts.ExtractionResult{
Output: contracts.ExtractOutput{
Schema: contracts.ResponseSchema{ID: schema.ID, Name: schema.Name, Version: schema.Version, JSONSchema: append([]byte(nil), schema.JSONSchema...)},
Payload: contracts.RawPayload{Content: content, MediaType: adapter.codec.MediaType(), Metadata: map[string]any{"spell_cast_count": len(result.Value.SpellCasts)}},
},
Warnings: result.Warnings,
}, nil
}
func validateOptions(options map[string]any) error { func validateOptions(options map[string]any) error {
_, err := DecodeOptions(options) _, err := DecodeOptions(options)
return err return err

View File

@@ -76,9 +76,6 @@ func TestRegisterMakesExtractorBuildable(t *testing.T) {
t.Fatalf("Register() error = %v, want nil", err) t.Fatalf("Register() error = %v, want nil", err)
} }
if _, err := registry.BuildLegacyRaw(Key); err == nil || !strings.Contains(err.Error(), "legacy raw") {
t.Fatalf("BuildLegacyRaw() error = %v, want typed registration error", err)
}
if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil || !strings.Contains(err.Error(), "unknown option") { if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil || !strings.Contains(err.Error(), "unknown option") {
t.Fatalf("DecodeOptions() error = %v, want unknown option error", err) t.Fatalf("DecodeOptions() error = %v, want unknown option error", err)
} }

View File

@@ -15,10 +15,6 @@ const ReasonCode = "invalid_spell_shape"
type Options struct{} type Options struct{}
type Validator struct{} type Validator struct{}
type legacyValidator struct{ codec decoder }
type decoder interface {
DecodeCandidate([]byte) (dnd.SpellList, error)
}
var _ contracts.TypedValidator[dnd.SpellList] = (*Validator)(nil) var _ contracts.TypedValidator[dnd.SpellList] = (*Validator)(nil)
@@ -58,18 +54,6 @@ func Validate(value dnd.SpellList) error {
return nil return nil
} }
func (v *legacyValidator) Name() string { return Key }
func (v *legacyValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v *legacyValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
value, err := v.codec.DecodeCandidate(req.Payload.Content)
if err != nil {
return rejection(err.Error()), nil
}
return New(Options{}).Validate(ctx, contracts.TypedValidationRequest[dnd.SpellList]{Value: value})
}
func Spec() pipeline.ValidatorSpec { func Spec() pipeline.ValidatorSpec {
return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic} return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic}
} }
@@ -82,17 +66,6 @@ func Register(registry *pipeline.ValidatorRegistry) error {
return New(options), nil return New(options), nil
}) })
} }
func RegisterLegacy(registry *pipeline.ValidatorRegistry, codec decoder) error {
if codec == nil {
return fmt.Errorf("spell shape validator codec must not be nil")
}
return registry.RegisterLegacyRawBuilderWithSpec(Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.LegacyRawValidator, error) {
if _, err := DecodeOptions(request.Options); err != nil {
return nil, err
}
return &legacyValidator{codec: codec}, nil
})
}
func DecodeOptions(options map[string]any) (Options, error) { func DecodeOptions(options map[string]any) (Options, error) {
if err := pipeline.RejectUnknownOptions(options); err != nil { if err := pipeline.RejectUnknownOptions(options); err != nil {
return Options{}, err return Options{}, err

View File

@@ -16,10 +16,6 @@ const ReasonCode = "invalid_source_refs"
type Options struct{} type Options struct{}
type Validator struct{} type Validator struct{}
type legacyValidator struct{ codec decoder }
type decoder interface {
DecodeCandidate([]byte) (dnd.SpellList, error)
}
var _ contracts.TypedValidator[dnd.SpellList] = (*Validator)(nil) var _ contracts.TypedValidator[dnd.SpellList] = (*Validator)(nil)
@@ -41,20 +37,6 @@ func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationReq
} }
return contracts.ValidationResult{Approved: true}, nil return contracts.ValidationResult{Approved: true}, nil
} }
func (v *legacyValidator) Name() string { return Key }
func (v *legacyValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v *legacyValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
value, err := v.codec.DecodeCandidate(req.Payload.Content)
if err != nil {
return rejection(err.Error()), nil
}
if err := spellshape.Validate(value); err != nil {
return rejection(err.Error()), nil
}
return New(Options{}).Validate(ctx, contracts.TypedValidationRequest[dnd.SpellList]{Source: req.Source, Value: value})
}
func Spec() pipeline.ValidatorSpec { func Spec() pipeline.ValidatorSpec {
return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic} return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic}
} }
@@ -67,17 +49,6 @@ func Register(registry *pipeline.ValidatorRegistry) error {
return New(options), nil return New(options), nil
}) })
} }
func RegisterLegacy(registry *pipeline.ValidatorRegistry, codec decoder) error {
if codec == nil {
return fmt.Errorf("spell source references validator codec must not be nil")
}
return registry.RegisterLegacyRawBuilderWithSpec(Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.LegacyRawValidator, error) {
if _, err := DecodeOptions(request.Options); err != nil {
return nil, err
}
return &legacyValidator{codec: codec}, nil
})
}
func DecodeOptions(options map[string]any) (Options, error) { func DecodeOptions(options map[string]any) (Options, error) {
if err := pipeline.RejectUnknownOptions(options); err != nil { if err := pipeline.RejectUnknownOptions(options); err != nil {
return Options{}, err return Options{}, err

View File

@@ -17,10 +17,6 @@ const WarningReasonCode = "spell_not_near_source"
type Options struct{} type Options struct{}
type Validator struct{} type Validator struct{}
type legacyValidator struct{ codec decoder }
type decoder interface {
DecodeCandidate([]byte) (dnd.SpellList, error)
}
var _ contracts.TypedValidator[dnd.SpellList] = (*Validator)(nil) var _ contracts.TypedValidator[dnd.SpellList] = (*Validator)(nil)
@@ -41,20 +37,6 @@ func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationReq
} }
return contracts.ValidationResult{Approved: true, Warnings: warnings}, nil return contracts.ValidationResult{Approved: true, Warnings: warnings}, nil
} }
func (v *legacyValidator) Name() string { return Key }
func (v *legacyValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v *legacyValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
value, err := v.codec.DecodeCandidate(req.Payload.Content)
if err != nil {
return contracts.ValidationResult{Approved: true}, nil
}
if err := spellshape.Validate(value); err != nil {
return contracts.ValidationResult{Approved: true}, nil
}
return New(Options{}).Validate(ctx, contracts.TypedValidationRequest[dnd.SpellList]{Source: req.Source, Value: value})
}
func spellAppearsInCitedText(doc *source.SourceDocument, spell dnd.SpellCast) bool { func spellAppearsInCitedText(doc *source.SourceDocument, spell dnd.SpellCast) bool {
name := strings.ToLower(strings.TrimSpace(spell.Spell)) name := strings.ToLower(strings.TrimSpace(spell.Spell))
if name == "" { if name == "" {
@@ -100,17 +82,6 @@ func Register(registry *pipeline.ValidatorRegistry) error {
return New(options), nil return New(options), nil
}) })
} }
func RegisterLegacy(registry *pipeline.ValidatorRegistry, codec decoder) error {
if codec == nil {
return fmt.Errorf("spell source relatedness validator codec must not be nil")
}
return registry.RegisterLegacyRawBuilderWithSpec(Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.LegacyRawValidator, error) {
if _, err := DecodeOptions(request.Options); err != nil {
return nil, err
}
return &legacyValidator{codec: codec}, nil
})
}
func DecodeOptions(options map[string]any) (Options, error) { func DecodeOptions(options map[string]any) (Options, error) {
if err := pipeline.RejectUnknownOptions(options); err != nil { if err := pipeline.RejectUnknownOptions(options); err != nil {
return Options{}, err return Options{}, err

View File

@@ -1,234 +1,15 @@
package appendorder package appendorder
import ( import (
"context"
"encoding/json"
"fmt" "fmt"
"mime"
"sort"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
) )
const Key = "appendorder" const Key = "appendorder"
var _ contracts.LegacyRawMerger = (*Merger)(nil)
type Merger struct{}
func New() *Merger {
return &Merger{}
}
func (m *Merger) Key() string {
return Key
}
func (m *Merger) Merge(ctx context.Context, req contracts.MergeRequest) (contracts.MergeResult, error) {
if m == nil {
return contracts.MergeResult{}, mergerErrorf("merger must not be nil")
}
if ctx == nil {
return contracts.MergeResult{}, mergerErrorf("context must not be nil")
}
if err := ctx.Err(); err != nil {
return contracts.MergeResult{}, mergerErrorf("context error before merge: %w", err)
}
outputs, err := orderedOutputs(req.ExtractOutputs)
if err != nil {
return contracts.MergeResult{}, err
}
if len(outputs) == 1 {
payload := cloneRawPayload(outputs[0].Payload)
return contracts.MergeResult{
Output: contracts.MergeOutput{
LaneID: req.LaneID,
MergerKey: Key,
SourceID: outputs[0].SourceID,
Schema: outputs[0].Schema,
Payload: payload,
},
}, nil
}
content, err := mergedContent(outputs)
if err != nil {
return contracts.MergeResult{}, err
}
return contracts.MergeResult{
Output: contracts.MergeOutput{
LaneID: req.LaneID,
MergerKey: Key,
SourceID: sourceID(outputs),
Schema: commonSchema(outputs),
Payload: contracts.RawPayload{
Content: content,
MediaType: "application/json",
},
},
}, nil
}
func ModuleSpec() pipeline.ModuleSpec { func ModuleSpec() pipeline.ModuleSpec {
return pipeline.ModuleSpec{ return pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageMerge, Provides: []string{"merged"}}
Key: Key,
Stage: pipeline.StageMerge,
Provides: []string{"merged"},
}
}
func Register(registry *pipeline.MergerRegistry) error {
return registry.RegisterLegacyRawWithSpec(ModuleSpec(), func() (contracts.LegacyRawMerger, error) {
return New(), nil
})
}
func orderedOutputs(outputs []contracts.ExtractOutput) ([]contracts.ExtractOutput, error) {
ordered := make([]contracts.ExtractOutput, 0, len(outputs))
for _, output := range outputs {
if !isJSONMediaType(output.Payload.MediaType) {
return nil, mergerErrorf("extract output for chunk %q has unsupported media type %q", output.ChunkID, output.Payload.MediaType)
}
if !json.Valid(output.Payload.Content) {
return nil, mergerErrorf("extract output for chunk %q contains invalid JSON", output.ChunkID)
}
ordered = append(ordered, cloneExtractOutput(output))
}
sort.SliceStable(ordered, func(i, j int) bool {
return ordered[i].ChunkIndex < ordered[j].ChunkIndex
})
return ordered, nil
}
func mergedContent(outputs []contracts.ExtractOutput) ([]byte, error) {
values := make([]any, 0, len(outputs))
objects := make([]map[string]any, 0, len(outputs))
for _, output := range outputs {
var value any
if err := json.Unmarshal(output.Payload.Content, &value); err != nil {
return nil, mergerErrorf("decode extract output for chunk %q: %w", output.ChunkID, err)
}
values = append(values, value)
object, ok := value.(map[string]any)
if !ok {
continue
}
objects = append(objects, object)
}
if len(objects) == len(outputs) {
if field, ok := commonArrayField(objects); ok {
merged := make([]any, 0)
for _, object := range objects {
items := object[field].([]any)
merged = append(merged, items...)
}
return marshalMerged(map[string]any{field: merged})
}
}
return marshalMerged(values)
}
func commonArrayField(objects []map[string]any) (string, bool) {
if len(objects) == 0 {
return "", false
}
candidates := map[string]struct{}{}
for key, value := range objects[0] {
if _, ok := value.([]any); ok {
candidates[key] = struct{}{}
}
}
for _, object := range objects[1:] {
for key := range candidates {
if _, ok := object[key].([]any); !ok {
delete(candidates, key)
}
}
}
if len(candidates) != 1 {
return "", false
}
for key := range candidates {
return key, true
}
return "", false
}
func marshalMerged(value any) ([]byte, error) {
content, err := json.Marshal(value)
if err != nil {
return nil, mergerErrorf("encode merged output: %w", err)
}
return content, nil
}
func isJSONMediaType(mediaType string) bool {
base, _, err := mime.ParseMediaType(strings.TrimSpace(mediaType))
if err != nil {
base = strings.TrimSpace(mediaType)
}
return strings.EqualFold(base, "application/json")
}
func sourceID(outputs []contracts.ExtractOutput) string {
for _, output := range outputs {
if output.SourceID != "" {
return output.SourceID
}
}
return ""
}
func commonSchema(outputs []contracts.ExtractOutput) contracts.ResponseSchema {
if len(outputs) == 0 {
return contracts.ResponseSchema{}
}
schema := outputs[0].Schema
for _, output := range outputs[1:] {
if !sameResponseSchema(output.Schema, schema) {
return contracts.ResponseSchema{}
}
}
return schema
}
func sameResponseSchema(left contracts.ResponseSchema, right contracts.ResponseSchema) bool {
return left.ID == right.ID && left.Name == right.Name && left.Version == right.Version && string(left.JSONSchema) == string(right.JSONSchema)
}
func cloneExtractOutput(output contracts.ExtractOutput) contracts.ExtractOutput {
output.Schema = cloneResponseSchema(output.Schema)
output.Payload = cloneRawPayload(output.Payload)
return output
}
func cloneResponseSchema(schema contracts.ResponseSchema) contracts.ResponseSchema {
schema.JSONSchema = append([]byte(nil), schema.JSONSchema...)
return schema
}
func cloneRawPayload(payload contracts.RawPayload) contracts.RawPayload {
return contracts.RawPayload{
Content: append([]byte(nil), payload.Content...),
MediaType: payload.MediaType,
Metadata: cloneMetadata(payload.Metadata),
Warnings: append([]contracts.Warning(nil), payload.Warnings...),
}
}
func cloneMetadata(metadata map[string]any) map[string]any {
if len(metadata) == 0 {
return nil
}
out := make(map[string]any, len(metadata))
for key, value := range metadata {
out[key] = value
}
return out
} }
func mergerErrorf(format string, args ...any) error { func mergerErrorf(format string, args ...any) error {

View File

@@ -1,215 +0,0 @@
package appendorder
import (
"context"
"encoding/json"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func TestModuleSpecAndRegister(t *testing.T) {
want := pipeline.ModuleSpec{
Key: Key,
Stage: pipeline.StageMerge,
Provides: []string{"merged"},
}
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
}
registry := pipeline.NewMergerRegistry()
if err := Register(registry); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
spec, ok := registry.Spec(Key)
if !ok {
t.Fatalf("Spec(%q) ok = false, want true", Key)
}
if !reflect.DeepEqual(spec, want) {
t.Fatalf("registered spec = %#v, want %#v", spec, want)
}
}
func TestMergePassesThroughSingleExtractOutput(t *testing.T) {
input := extractOutput("chunk-0", 0, `{"name":"original"}`)
result, err := New().Merge(context.Background(), contracts.MergeRequest{
LaneID: "events",
ExtractOutputs: []contracts.ExtractOutput{input},
})
if err != nil {
t.Fatalf("Merge() error = %v, want nil", err)
}
if result.Output.LaneID != "events" || result.Output.MergerKey != Key {
t.Fatalf("output provenance = %#v, want lane and merger", result.Output)
}
if string(result.Output.Payload.Content) != `{"name":"original"}` {
t.Fatalf("content = %s, want original content", result.Output.Payload.Content)
}
if result.Output.Payload.Metadata["name"] != "chunk-0" {
t.Fatalf("metadata = %#v, want original metadata", result.Output.Payload.Metadata)
}
}
func TestMergeDefensivelyCopiesRawPayload(t *testing.T) {
input := extractOutput("chunk-0", 0, `{"name":"original"}`)
result, err := New().Merge(context.Background(), contracts.MergeRequest{
LaneID: "events",
ExtractOutputs: []contracts.ExtractOutput{input},
})
if err != nil {
t.Fatalf("Merge() error = %v, want nil", err)
}
input.Payload.Content[0] = '['
input.Payload.Metadata["name"] = "changed"
if string(result.Output.Payload.Content) != `{"name":"original"}` {
t.Fatalf("content changed after input mutation: %s", result.Output.Payload.Content)
}
if result.Output.Payload.Metadata["name"] != "chunk-0" {
t.Fatalf("metadata changed after input mutation: %#v", result.Output.Payload.Metadata)
}
}
func TestMergeConcatenatesCommonTopLevelArrayFieldInChunkOrder(t *testing.T) {
result, err := New().Merge(context.Background(), contracts.MergeRequest{
LaneID: "events",
ExtractOutputs: []contracts.ExtractOutput{
extractOutput("chunk-1", 1, `{"events":[{"name":"second"}]}`),
extractOutput("chunk-0", 0, `{"events":[{"name":"first"}]}`),
},
})
if err != nil {
t.Fatalf("Merge() error = %v, want nil", err)
}
if result.Output.Payload.MediaType != "application/json" {
t.Fatalf("MediaType = %q, want application/json", result.Output.Payload.MediaType)
}
var decoded struct {
Events []struct {
Name string `json:"name"`
} `json:"events"`
}
if err := json.Unmarshal(result.Output.Payload.Content, &decoded); err != nil {
t.Fatalf("Unmarshal() error = %v, want nil", err)
}
if len(decoded.Events) != 2 || decoded.Events[0].Name != "first" || decoded.Events[1].Name != "second" {
t.Fatalf("events = %#v, want concatenated chunk order", decoded.Events)
}
if result.Output.Schema.ID != "schema-id" {
t.Fatalf("schema = %#v, want common extract schema", result.Output.Schema)
}
}
func TestMergeFallsBackToOrderedJSONValueArrayWhenShapesDiffer(t *testing.T) {
result, err := New().Merge(context.Background(), contracts.MergeRequest{
LaneID: "events",
ExtractOutputs: []contracts.ExtractOutput{
extractOutput("chunk-1", 1, `{"notes":["second"]}`),
extractOutput("chunk-0", 0, `{"events":[{"name":"first"}]}`),
},
})
if err != nil {
t.Fatalf("Merge() error = %v, want nil", err)
}
var decoded []map[string]any
if err := json.Unmarshal(result.Output.Payload.Content, &decoded); err != nil {
t.Fatalf("Unmarshal() error = %v, want nil", err)
}
if len(decoded) != 2 {
t.Fatalf("len(decoded) = %d, want 2", len(decoded))
}
if _, ok := decoded[0]["events"]; !ok {
t.Fatalf("decoded[0] = %#v, want first chunk value", decoded[0])
}
if _, ok := decoded[1]["notes"]; !ok {
t.Fatalf("decoded[1] = %#v, want second chunk value", decoded[1])
}
}
func TestMergeRejectsInvalidJSONAndNonJSONMediaTypes(t *testing.T) {
tests := []struct {
name string
output contracts.ExtractOutput
want string
}{
{
name: "invalid JSON",
output: extractOutput("chunk-0", 0, `{"events":[`),
want: "invalid JSON",
},
{
name: "non JSON media type",
output: func() contracts.ExtractOutput {
output := extractOutput("chunk-0", 0, `{"events":[]}`)
output.Payload.MediaType = "text/plain"
return output
}(),
want: "unsupported media type",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
_, err := New().Merge(context.Background(), contracts.MergeRequest{
LaneID: "events",
ExtractOutputs: []contracts.ExtractOutput{test.output},
})
if err == nil {
t.Fatal("Merge() error = nil, want error")
}
if !strings.Contains(err.Error(), test.want) {
t.Fatalf("Merge() error = %q, want %q", err.Error(), test.want)
}
})
}
}
func TestTypedMergeUsesRequestOrderForReusableValueType(t *testing.T) {
type notes struct{ Values []string }
merger, err := NewTyped(func(values []notes) (notes, error) {
var combined notes
for _, value := range values {
combined.Values = append(combined.Values, value.Values...)
}
return combined, nil
})
if err != nil {
t.Fatalf("NewTyped() error = %v", err)
}
result, err := merger.Merge(context.Background(), contracts.TypedMergeRequest[notes]{ExtractOutputs: []contracts.ExtractArtifact[notes]{
{ChunkIndex: 4, Value: notes{Values: []string{"first"}}},
{ChunkIndex: 1, Value: notes{Values: []string{"second"}}},
}})
if err != nil {
t.Fatalf("Merge() error = %v", err)
}
if got := result.Value.Values; !reflect.DeepEqual(got, []string{"first", "second"}) {
t.Fatalf("Values = %#v", got)
}
}
func extractOutput(chunkID string, chunkIndex int, content string) contracts.ExtractOutput {
return contracts.ExtractOutput{
LaneID: "events",
ExtractorKey: "extract",
SourceID: "source-1",
ChunkID: chunkID,
ChunkIndex: chunkIndex,
Schema: contracts.ResponseSchema{ID: "schema-id", Name: "schema-name", Version: "v1"},
Payload: contracts.RawPayload{
Content: []byte(content),
MediaType: "application/json",
Metadata: map[string]any{"name": chunkID},
},
}
}

View File

@@ -0,0 +1,25 @@
package appendorder
import (
"context"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestTypedMergerCombinesValuesInFrameworkOrder(t *testing.T) {
merger, err := NewTyped(func(values []string) (string, error) {
if !reflect.DeepEqual(values, []string{"first", "second"}) {
t.Fatalf("values=%#v", values)
}
return values[0] + values[1], nil
})
if err != nil {
t.Fatal(err)
}
result, err := merger.Merge(context.Background(), contracts.TypedMergeRequest[string]{ExtractOutputs: []contracts.ExtractArtifact[string]{{ChunkIndex: 0, Value: "first"}, {ChunkIndex: 1, Value: "second"}}})
if err != nil || result.Value != "firstsecond" {
t.Fatalf("result=%#v err=%v", result, err)
}
}

View File

@@ -1,85 +1,15 @@
package noop package noop
import ( import (
"context"
"fmt" "fmt"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
) )
const Key = "noop" const Key = "noop"
var _ contracts.LegacyRawNormalizer = (*Normalizer)(nil)
type Normalizer struct{}
func New() *Normalizer {
return &Normalizer{}
}
func (n *Normalizer) Key() string {
return Key
}
func (n *Normalizer) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (n *Normalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
if n == nil {
return contracts.NormalizeResult{}, normalizerErrorf("normalizer must not be nil")
}
if ctx == nil {
return contracts.NormalizeResult{}, normalizerErrorf("context must not be nil")
}
if err := ctx.Err(); err != nil {
return contracts.NormalizeResult{}, normalizerErrorf("context error before normalize: %w", err)
}
return contracts.NormalizeResult{
Output: contracts.NormalizeOutput{
LaneID: req.LaneID,
NormalizerKey: Key,
SourceID: req.MergeOutput.SourceID,
Schema: req.MergeOutput.Schema,
Payload: cloneRawPayload(req.MergeOutput.Payload),
},
}, nil
}
func ModuleSpec() pipeline.ModuleSpec { func ModuleSpec() pipeline.ModuleSpec {
return pipeline.ModuleSpec{ return pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, Requires: []string{"merged"}, Provides: []string{"normalized"}}
Key: Key,
Stage: pipeline.StageNormalize,
Requires: []string{"merged"},
Provides: []string{"normalized"},
}
}
func Register(registry *pipeline.NormalizerRegistry) error {
return registry.RegisterLegacyRawWithSpec(ModuleSpec(), func() (contracts.LegacyRawNormalizer, error) {
return New(), nil
})
}
func cloneRawPayload(payload contracts.RawPayload) contracts.RawPayload {
return contracts.RawPayload{
Content: append([]byte(nil), payload.Content...),
MediaType: payload.MediaType,
Metadata: cloneMetadata(payload.Metadata),
Warnings: append([]contracts.Warning(nil), payload.Warnings...),
}
}
func cloneMetadata(metadata map[string]any) map[string]any {
if len(metadata) == 0 {
return nil
}
out := make(map[string]any, len(metadata))
for key, value := range metadata {
out[key] = value
}
return out
} }
func normalizerErrorf(format string, args ...any) error { func normalizerErrorf(format string, args ...any) error {

View File

@@ -1,112 +0,0 @@
package noop
import (
"context"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func TestModuleSpecAndRegister(t *testing.T) {
want := pipeline.ModuleSpec{
Key: Key,
Stage: pipeline.StageNormalize,
Requires: []string{"merged"},
Provides: []string{"normalized"},
}
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
}
registry := pipeline.NewNormalizerRegistry()
if err := Register(registry); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
spec, ok := registry.Spec(Key)
if !ok {
t.Fatalf("Spec(%q) ok = false, want true", Key)
}
if !reflect.DeepEqual(spec, want) {
t.Fatalf("registered spec = %#v, want %#v", spec, want)
}
normalizer, err := registry.BuildLegacyRaw(Key)
if err != nil {
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
}
if slots := normalizer.ReferenceSlots(); len(slots) != 0 {
t.Fatalf("ReferenceSlots() = %#v, want none", slots)
}
}
func TestNormalizePassesThroughMergeOutput(t *testing.T) {
input := mergeOutput(`{"name":"original"}`)
result, err := New().Normalize(context.Background(), contracts.NormalizeRequest{
LaneID: "events",
MergeOutput: input,
})
if err != nil {
t.Fatalf("Normalize() error = %v, want nil", err)
}
if result.Output.LaneID != "events" || result.Output.NormalizerKey != Key {
t.Fatalf("output provenance = %#v, want lane and normalizer", result.Output)
}
if string(result.Output.Payload.Content) != `{"name":"original"}` {
t.Fatalf("content = %s, want original content", result.Output.Payload.Content)
}
if result.Output.Payload.Metadata["name"] != "original" {
t.Fatalf("metadata = %#v, want original metadata", result.Output.Payload.Metadata)
}
}
func TestNormalizeDefensivelyCopiesRawPayload(t *testing.T) {
input := mergeOutput(`{"name":"original"}`)
result, err := New().Normalize(context.Background(), contracts.NormalizeRequest{
LaneID: "events",
MergeOutput: input,
})
if err != nil {
t.Fatalf("Normalize() error = %v, want nil", err)
}
input.Payload.Content[0] = '['
input.Payload.Metadata["name"] = "changed"
if string(result.Output.Payload.Content) != `{"name":"original"}` {
t.Fatalf("content changed after input mutation: %s", result.Output.Payload.Content)
}
if result.Output.Payload.Metadata["name"] != "original" {
t.Fatalf("metadata changed after input mutation: %#v", result.Output.Payload.Metadata)
}
}
func TestTypedNormalizePassesThroughReusableValueType(t *testing.T) {
type score struct{ Value int }
result, err := NewTyped[score]().Normalize(context.Background(), contracts.TypedNormalizeRequest[score]{
MergeOutput: contracts.MergeArtifact[score]{Value: score{Value: 7}},
})
if err != nil {
t.Fatalf("Normalize() error = %v", err)
}
if result.Value.Value != 7 {
t.Fatalf("Value = %d, want 7", result.Value.Value)
}
}
func mergeOutput(content string) contracts.MergeOutput {
return contracts.MergeOutput{
LaneID: "events",
MergerKey: "merge",
SourceID: "source-1",
Schema: contracts.ResponseSchema{ID: "schema-id", Name: "schema-name", Version: "v1"},
Payload: contracts.RawPayload{
Content: []byte(content),
MediaType: "application/json",
Metadata: map[string]any{"name": "original"},
},
}
}

View File

@@ -0,0 +1,16 @@
package noop
import (
"context"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestTypedNormalizerPreservesValue(t *testing.T) {
normalizer := NewTyped[string]()
result, err := normalizer.Normalize(context.Background(), contracts.TypedNormalizeRequest[string]{MergeOutput: contracts.MergeArtifact[string]{Value: "value"}})
if err != nil || result.Value != "value" {
t.Fatalf("result=%#v err=%v", result, err)
}
}

View File

@@ -253,15 +253,6 @@ func cloneNormalizeOutputs(outputs []contracts.SerializedOutput) []contracts.Ser
return out return out
} }
func cloneRawPayload(payload contracts.RawPayload) contracts.RawPayload {
return contracts.RawPayload{
Content: append([]byte(nil), payload.Content...),
MediaType: payload.MediaType,
Metadata: cloneMetadata(payload.Metadata),
Warnings: append([]contracts.Warning(nil), payload.Warnings...),
}
}
func cloneRejected(rejected []contracts.RejectedOutput) []contracts.RejectedOutput { func cloneRejected(rejected []contracts.RejectedOutput) []contracts.RejectedOutput {
if len(rejected) == 0 { if len(rejected) == 0 {
return []contracts.RejectedOutput{} return []contracts.RejectedOutput{}

View File

@@ -7,8 +7,6 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm" "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/chunk/units" "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/chunk/units"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
jsonoutput "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/output/json" jsonoutput "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/output/json"
alwaysaccept "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/validate/always_accept" alwaysaccept "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/validate/always_accept"
alwaysreject "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/validate/always_reject" alwaysreject "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/validate/always_reject"
@@ -27,8 +25,6 @@ func Register(registries pipeline.Registries, assets *llm.AssetRegistry) error {
register func() error register func() error
}{ }{
{name: "generic chunker", register: func() error { return units.Register(registries.Chunkers) }}, {name: "generic chunker", register: func() error { return units.Register(registries.Chunkers) }},
{name: "appendorder merger", register: func() error { return appendorder.Register(registries.Mergers) }},
{name: "noop normalizer", register: func() error { return noop.Register(registries.Normalizers) }},
{name: "always accept validator", register: func() error { return alwaysaccept.Register(registries.Validators) }}, {name: "always accept validator", register: func() error { return alwaysaccept.Register(registries.Validators) }},
{name: "always reject validator", register: func() error { return alwaysreject.Register(registries.Validators) }}, {name: "always reject validator", register: func() error { return alwaysreject.Register(registries.Validators) }},
{name: "valid json validator", register: func() error { return validjson.Register(registries.Validators) }}, {name: "valid json validator", register: func() error { return validjson.Register(registries.Validators) }},

View File

@@ -14,8 +14,8 @@ func TestRegisterAddsGenericFamily(t *testing.T) {
t.Fatalf("Register() error = %v, want nil", err) t.Fatalf("Register() error = %v, want nil", err)
} }
assertKeys(t, "chunkers", registries.Chunkers.RegisteredKeys(), []string{"generic"}) assertKeys(t, "chunkers", registries.Chunkers.RegisteredKeys(), []string{"generic"})
assertKeys(t, "mergers", registries.Mergers.RegisteredKeys(), []string{"appendorder"}) assertKeys(t, "mergers", registries.Mergers.RegisteredKeys(), nil)
assertKeys(t, "normalizers", registries.Normalizers.RegisteredKeys(), []string{"noop"}) assertKeys(t, "normalizers", registries.Normalizers.RegisteredKeys(), nil)
assertKeys(t, "validators", registries.Validators.RegisteredKeys(), []string{ assertKeys(t, "validators", registries.Validators.RegisteredKeys(), []string{
"generic/always_accept", "generic/always_accept",
"generic/always_reject", "generic/always_reject",

View File

@@ -12,7 +12,6 @@ const Key = "generic/always_accept"
type Options struct{} type Options struct{}
type ChunkValidator struct{} type ChunkValidator struct{}
type TypedValidator[T any] struct{} type TypedValidator[T any] struct{}
type legacyValidator struct{}
var _ contracts.ChunkValidator = (*ChunkValidator)(nil) var _ contracts.ChunkValidator = (*ChunkValidator)(nil)
@@ -35,33 +34,17 @@ func (v *TypedValidator[T]) Validate(context.Context, contracts.TypedValidationR
return contracts.ValidationResult{Approved: true}, nil return contracts.ValidationResult{Approved: true}, nil
} }
func (v *legacyValidator) Name() string { return Key }
func (v *legacyValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v *legacyValidator) Validate(context.Context, contracts.ValidationRequest) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: true}, nil
}
func Spec() pipeline.ValidatorSpec { func Spec() pipeline.ValidatorSpec {
return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic} return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic}
} }
func Register(registry *pipeline.ValidatorRegistry) error { func Register(registry *pipeline.ValidatorRegistry) error {
if err := pipeline.RegisterChunkValidatorBuilder(registry, Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.ChunkValidator, error) { return pipeline.RegisterChunkValidatorBuilder(registry, Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.ChunkValidator, error) {
options, err := DecodeOptions(request.Options) options, err := DecodeOptions(request.Options)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return NewChunk(options), nil return NewChunk(options), nil
}); err != nil {
return err
}
return registry.RegisterLegacyRawBuilderWithSpec(Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.LegacyRawValidator, error) {
if _, err := DecodeOptions(request.Options); err != nil {
return nil, err
}
return &legacyValidator{}, nil
}) })
} }

View File

@@ -30,11 +30,7 @@ func TestSpecAndRegister(t *testing.T) {
if err := Register(registry); err != nil { if err := Register(registry); err != nil {
t.Fatalf("Register() error = %v, want nil", err) t.Fatalf("Register() error = %v, want nil", err)
} }
validator, err := registry.BuildLegacyRaw(Key) if registered, ok := registry.Spec(Key); !ok || registered.Key != Key {
if err != nil { t.Fatalf("Spec(%q) = %#v, %v", Key, registered, ok)
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
}
if validator.Name() != Key {
t.Fatalf("Name() = %q, want %q", validator.Name(), Key)
} }
} }

View File

@@ -13,7 +13,6 @@ const ReasonCode = "always_reject"
type Options struct{} type Options struct{}
type ChunkValidator struct{} type ChunkValidator struct{}
type TypedValidator[T any] struct{} type TypedValidator[T any] struct{}
type legacyValidator struct{}
func NewChunk(Options) *ChunkValidator { return &ChunkValidator{} } func NewChunk(Options) *ChunkValidator { return &ChunkValidator{} }
func NewTyped[T any](Options) *TypedValidator[T] { return &TypedValidator[T]{} } func NewTyped[T any](Options) *TypedValidator[T] { return &TypedValidator[T]{} }
@@ -35,32 +34,16 @@ func (v *TypedValidator[T]) ExecutionClass() contracts.ExecutionClass {
func (v *TypedValidator[T]) Validate(context.Context, contracts.TypedValidationRequest[T]) (contracts.ValidationResult, error) { func (v *TypedValidator[T]) Validate(context.Context, contracts.TypedValidationRequest[T]) (contracts.ValidationResult, error) {
return rejection(), nil return rejection(), nil
} }
func (v *legacyValidator) Name() string { return Key }
func (v *legacyValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v *legacyValidator) Validate(context.Context, contracts.ValidationRequest) (contracts.ValidationResult, error) {
return rejection(), nil
}
func Spec() pipeline.ValidatorSpec { func Spec() pipeline.ValidatorSpec {
return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic} return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic}
} }
func Register(registry *pipeline.ValidatorRegistry) error { func Register(registry *pipeline.ValidatorRegistry) error {
if err := pipeline.RegisterChunkValidatorBuilder(registry, Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.ChunkValidator, error) { return pipeline.RegisterChunkValidatorBuilder(registry, Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.ChunkValidator, error) {
options, err := DecodeOptions(request.Options) options, err := DecodeOptions(request.Options)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return NewChunk(options), nil return NewChunk(options), nil
}); err != nil {
return err
}
return registry.RegisterLegacyRawBuilderWithSpec(Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.LegacyRawValidator, error) {
if _, err := DecodeOptions(request.Options); err != nil {
return nil, err
}
return &legacyValidator{}, nil
}) })
} }
func RegisterTyped[T any](registry *pipeline.ValidatorRegistry, kind contracts.ArtifactKind) error { func RegisterTyped[T any](registry *pipeline.ValidatorRegistry, kind contracts.ArtifactKind) error {

View File

@@ -33,11 +33,7 @@ func TestSpecAndRegister(t *testing.T) {
if err := Register(registry); err != nil { if err := Register(registry); err != nil {
t.Fatalf("Register() error = %v, want nil", err) t.Fatalf("Register() error = %v, want nil", err)
} }
validator, err := registry.BuildLegacyRaw(Key) if registered, ok := registry.Spec(Key); !ok || registered.Key != Key {
if err != nil { t.Fatalf("Spec(%q) = %#v, %v", Key, registered, ok)
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
}
if validator.Name() != Key {
t.Fatalf("Name() = %q, want %q", validator.Name(), Key)
} }
} }

View File

@@ -15,10 +15,7 @@ type Options struct{}
type Validator struct{} type Validator struct{}
type legacyValidator struct{}
var _ contracts.SerializedValidator = (*Validator)(nil) var _ contracts.SerializedValidator = (*Validator)(nil)
var _ contracts.LegacyRawValidator = (*legacyValidator)(nil)
func New(Options) *Validator { return &Validator{} } func New(Options) *Validator { return &Validator{} }
@@ -32,16 +29,6 @@ func (v *Validator) Validate(_ context.Context, req contracts.SerializedValidati
return validate(req.Content), nil return validate(req.Content), nil
} }
func (v *legacyValidator) Name() string { return Key }
func (v *legacyValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v *legacyValidator) Validate(_ context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
return validate(req.Payload.Content), nil
}
func validate(content []byte) contracts.ValidationResult { func validate(content []byte) contracts.ValidationResult {
if !json.Valid(content) { if !json.Valid(content) {
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCodeInvalidJSON, Message: "payload is not valid JSON"} return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCodeInvalidJSON, Message: "payload is not valid JSON"}
@@ -54,7 +41,7 @@ func Spec() pipeline.ValidatorSpec {
} }
func Register(registry *pipeline.ValidatorRegistry) error { func Register(registry *pipeline.ValidatorRegistry) error {
if err := pipeline.RegisterSerializedValidatorBuilder(registry, pipeline.SerializedValidatorSpec{ return pipeline.RegisterSerializedValidatorBuilder(registry, pipeline.SerializedValidatorSpec{
ValidatorSpec: Spec(), SupportsChunks: true, SupportsArtifacts: true, ValidatorSpec: Spec(), SupportsChunks: true, SupportsArtifacts: true,
}, validateOptions, func(request pipeline.BuildRequest) (contracts.SerializedValidator, error) { }, validateOptions, func(request pipeline.BuildRequest) (contracts.SerializedValidator, error) {
options, err := DecodeOptions(request.Options) options, err := DecodeOptions(request.Options)
@@ -62,14 +49,6 @@ func Register(registry *pipeline.ValidatorRegistry) error {
return nil, err return nil, err
} }
return New(options), nil return New(options), nil
}); err != nil {
return err
}
return registry.RegisterLegacyRawBuilderWithSpec(Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.LegacyRawValidator, error) {
if _, err := DecodeOptions(request.Options); err != nil {
return nil, err
}
return &legacyValidator{}, nil
}) })
} }

View File

@@ -47,12 +47,8 @@ func TestSpecAndRegister(t *testing.T) {
if err := Register(registry); err != nil { if err := Register(registry); err != nil {
t.Fatalf("Register() error = %v, want nil", err) t.Fatalf("Register() error = %v, want nil", err)
} }
validator, err := registry.BuildLegacyRaw(Key) if registered, ok := registry.Spec(Key); !ok || registered.Key != Key {
if err != nil { t.Fatalf("Spec(%q) = %#v, %v", Key, registered, ok)
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
}
if validator.Name() != Key {
t.Fatalf("Name() = %q, want %q", validator.Name(), Key)
} }
} }

View File

@@ -17,10 +17,8 @@ const ReasonCodeSchemaInvalid = "json_schema_invalid"
type Options struct{} type Options struct{}
type Validator struct{} type Validator struct{}
type legacyValidator struct{}
var _ contracts.SerializedValidator = (*Validator)(nil) var _ contracts.SerializedValidator = (*Validator)(nil)
var _ contracts.LegacyRawValidator = (*legacyValidator)(nil)
func New(Options) *Validator { return &Validator{} } func New(Options) *Validator { return &Validator{} }
func (v *Validator) Name() string { return Key } func (v *Validator) Name() string { return Key }
@@ -32,14 +30,6 @@ func (v *Validator) Validate(_ context.Context, req contracts.SerializedValidati
return validate(req.Content, req.Schema.JSONSchema) return validate(req.Content, req.Schema.JSONSchema)
} }
func (v *legacyValidator) Name() string { return Key }
func (v *legacyValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v *legacyValidator) Validate(_ context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
return validate(req.Payload.Content, req.Schema.JSONSchema)
}
func validate(content, schemaContent []byte) (contracts.ValidationResult, error) { func validate(content, schemaContent []byte) (contracts.ValidationResult, error) {
if len(schemaContent) == 0 { if len(schemaContent) == 0 {
return contracts.ValidationResult{}, fmt.Errorf("response schema content is not available") return contracts.ValidationResult{}, fmt.Errorf("response schema content is not available")
@@ -71,7 +61,7 @@ func Spec() pipeline.ValidatorSpec {
} }
func Register(registry *pipeline.ValidatorRegistry) error { func Register(registry *pipeline.ValidatorRegistry) error {
if err := pipeline.RegisterSerializedValidatorBuilder(registry, pipeline.SerializedValidatorSpec{ return pipeline.RegisterSerializedValidatorBuilder(registry, pipeline.SerializedValidatorSpec{
ValidatorSpec: Spec(), SupportsChunks: true, SupportsArtifacts: true, ValidatorSpec: Spec(), SupportsChunks: true, SupportsArtifacts: true,
}, validateOptions, func(request pipeline.BuildRequest) (contracts.SerializedValidator, error) { }, validateOptions, func(request pipeline.BuildRequest) (contracts.SerializedValidator, error) {
options, err := DecodeOptions(request.Options) options, err := DecodeOptions(request.Options)
@@ -79,14 +69,6 @@ func Register(registry *pipeline.ValidatorRegistry) error {
return nil, err return nil, err
} }
return New(options), nil return New(options), nil
}); err != nil {
return err
}
return registry.RegisterLegacyRawBuilderWithSpec(Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.LegacyRawValidator, error) {
if _, err := DecodeOptions(request.Options); err != nil {
return nil, err
}
return &legacyValidator{}, nil
}) })
} }

View File

@@ -74,12 +74,8 @@ func TestSpecAndRegister(t *testing.T) {
if err := Register(registry); err != nil { if err := Register(registry); err != nil {
t.Fatalf("Register() error = %v, want nil", err) t.Fatalf("Register() error = %v, want nil", err)
} }
validator, err := registry.BuildLegacyRaw(Key) if registered, ok := registry.Spec(Key); !ok || registered.Key != Key {
if err != nil { t.Fatalf("Spec(%q) = %#v, %v", Key, registered, ok)
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
}
if validator.Name() != Key {
t.Fatalf("Name() = %q, want %q", validator.Name(), Key)
} }
} }

View File

@@ -10,6 +10,7 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
spellcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/spells" spellcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/spells"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder" "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
@@ -178,37 +179,39 @@ func dndSpellsTestCatalog(t *testing.T, specs dndSpellsCatalogSpecs) pipeline.Mo
t.Fatalf("register chunker: %v", err) t.Fatalf("register chunker: %v", err)
} }
codec := spellcodec.New()
if err := pipeline.RegisterArtifactCodec(codecs, codec); err != nil {
t.Fatalf("register dnd spells codec: %v", err)
}
if specs.extractor.Key == "" { if specs.extractor.Key == "" {
codec := spellcodec.New() if err := spells.Register(extractors); err != nil {
if err := pipeline.RegisterArtifactCodec(codecs, codec); err != nil {
t.Fatalf("register dnd spells codec: %v", err)
}
if err := spells.RegisterWithRawAdapter(extractors, codec); err != nil {
t.Fatalf("register dnd spells extractor: %v", err) t.Fatalf("register dnd spells extractor: %v", err)
} }
} else { } else {
specs.extractor.ArtifactKind = "" specs.extractor.ArtifactKind = dnd.SpellListKind
if err := extractors.RegisterLegacyRawWithSpec(specs.extractor, func() (contracts.LegacyRawExtractor, error) { if err := pipeline.RegisterExtractor[dnd.SpellList](extractors, specs.extractor, func() (contracts.Extractor[dnd.SpellList], error) {
return configLegacyExtractor{key: specs.extractor.Key}, nil return configExtractor{key: specs.extractor.Key}, nil
}); err != nil { }); err != nil {
t.Fatalf("register dnd spells extractor override: %v", err) t.Fatalf("register dnd spells extractor override: %v", err)
} }
} }
if err := mergers.RegisterLegacyRawWithSpec(pipeline.ModuleSpec{ if err := pipeline.RegisterMerger[dnd.SpellList](mergers, pipeline.ModuleSpec{
Key: pipeline.DefaultMergeModule, Key: pipeline.DefaultMergeModule,
Stage: pipeline.StageMerge, Stage: pipeline.StageMerge,
Requires: []string{"dnd.spell_casts"}, ArtifactKind: dnd.SpellListKind,
}, func() (contracts.LegacyRawMerger, error) { Requires: []string{"dnd.spell_casts"},
return appendorder.New(), nil }, func() (contracts.Merger[dnd.SpellList], error) {
return appendorder.NewTyped(appendSpellLists)
}); err != nil { }); err != nil {
t.Fatalf("register merger: %v", err) t.Fatalf("register merger: %v", err)
} }
if err := normalizers.RegisterLegacyRawWithSpec(pipeline.ModuleSpec{ if err := pipeline.RegisterNormalizer[dnd.SpellList](normalizers, pipeline.ModuleSpec{
Key: pipeline.DefaultNormalizeModule, Key: pipeline.DefaultNormalizeModule,
Stage: pipeline.StageNormalize, Stage: pipeline.StageNormalize,
}, func() (contracts.LegacyRawNormalizer, error) { ArtifactKind: dnd.SpellListKind,
return noop.New(), nil }, func() (contracts.Normalizer[dnd.SpellList], error) {
return noop.NewTyped[dnd.SpellList](), nil
}); err != nil { }); err != nil {
t.Fatalf("register normalizer: %v", err) t.Fatalf("register normalizer: %v", err)
} }
@@ -233,12 +236,20 @@ func dndSpellsTestCatalog(t *testing.T, specs dndSpellsCatalogSpecs) pipeline.Mo
} }
} }
type configLegacyExtractor struct{ key string } type configExtractor struct{ key string }
func (extractor configLegacyExtractor) Key() string { return extractor.key } func (extractor configExtractor) Key() string { return extractor.key }
func (configLegacyExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil } func (configExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (configLegacyExtractor) Extract(context.Context, contracts.ExtractionRequest) (contracts.ExtractionResult, error) { func (configExtractor) Extract(context.Context, contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.SpellList], error) {
return contracts.ExtractionResult{}, nil return contracts.TypedExtractionResult[dnd.SpellList]{}, nil
}
func appendSpellLists(values []dnd.SpellList) (dnd.SpellList, error) {
combined := dnd.SpellList{SpellCasts: []dnd.SpellCast{}}
for _, value := range values {
combined.SpellCasts = append(combined.SpellCasts, value.SpellCasts...)
}
return combined, nil
} }
func dndSpellsChunkerSpec() pipeline.ModuleSpec { func dndSpellsChunkerSpec() pipeline.ModuleSpec {

View File

@@ -60,11 +60,11 @@ func TestRunnerProcessesSeriatimInputWithDNDSpellsExtractor(t *testing.T) {
if len(output.NormalizeOutputs) != 1 { if len(output.NormalizeOutputs) != 1 {
t.Fatalf("len(NormalizeOutputs) = %d, want 1", len(output.NormalizeOutputs)) t.Fatalf("len(NormalizeOutputs) = %d, want 1", len(output.NormalizeOutputs))
} }
rawOutput := output.NormalizeOutputs[0] serializedOutput := output.NormalizeOutputs[0]
if rawOutput.LaneID != "spells" || rawOutput.Artifact.Schema.ID != spells.ResponseSchemaID || rawOutput.Artifact.Schema.Version != spells.SchemaVersion { if serializedOutput.LaneID != "spells" || serializedOutput.Artifact.Schema.ID != spells.ResponseSchemaID || serializedOutput.Artifact.Schema.Version != spells.SchemaVersion {
t.Fatalf("raw output envelope = %#v, want dnd spells schema on spells lane", rawOutput) t.Fatalf("serialized output envelope = %#v, want dnd spells schema on spells lane", serializedOutput)
} }
response := decodeRunnerSpellResponse(t, rawOutput.Artifact.Content) response := decodeRunnerSpellResponse(t, serializedOutput.Artifact.Content)
if len(response.SpellCasts) != 2 { if len(response.SpellCasts) != 2 {
t.Fatalf("len(spell_casts) = %d, want 2", len(response.SpellCasts)) t.Fatalf("len(spell_casts) = %d, want 2", len(response.SpellCasts))
} }
@@ -204,7 +204,7 @@ func TestRunnerDoesNotExtractSpellMentionedOnlyInPartyReference(t *testing.T) {
} }
} }
func TestRunnerCarriesDNDSpellCastWithInvalidSourceRefAsRawOutput(t *testing.T) { func TestRunnerCarriesDNDSpellCastWithInvalidSourceRefToSerializedOutput(t *testing.T) {
raw := readDNDSpellsFixture(t) raw := readDNDSpellsFixture(t)
resolved := resolveDNDSpellsPipeline(t) resolved := resolveDNDSpellsPipeline(t)
llmClient := &fakeSpellsLLMClient{ llmClient := &fakeSpellsLLMClient{
@@ -235,7 +235,7 @@ func TestRunnerCarriesDNDSpellCastWithInvalidSourceRefAsRawOutput(t *testing.T)
t.Fatalf("len(spell_casts) = %d, want 1", len(response.SpellCasts)) t.Fatalf("len(spell_casts) = %d, want 1", len(response.SpellCasts))
} }
if response.SpellCasts[0].SourceRefs[0].SourceID != "spell-session" { if response.SpellCasts[0].SourceRefs[0].SourceID != "spell-session" {
t.Fatalf("SourceID = %q, want raw invalid source ref preserved", response.SpellCasts[0].SourceRefs[0].SourceID) t.Fatalf("SourceID = %q, want invalid source ref preserved", response.SpellCasts[0].SourceRefs[0].SourceID)
} }
if len(output.Rejected) != 0 { if len(output.Rejected) != 0 {
t.Fatalf("len(Rejected) = %d, want 0", len(output.Rejected)) t.Fatalf("len(Rejected) = %d, want 0", len(output.Rejected))
@@ -282,7 +282,7 @@ func dndSpellsReferenceSet(party string, glossary string) contracts.ReferenceSet
return contracts.ReferenceSet{Slots: slots} return contracts.ReferenceSet{Slots: slots}
} }
func TestRunnerCarriesMalformedDNDSpellsExtractorOutput(t *testing.T) { func TestRunnerRejectsMalformedDNDSpellsArtifactAtSerializationBoundary(t *testing.T) {
raw := readDNDSpellsFixture(t) raw := readDNDSpellsFixture(t)
resolved := resolveDNDSpellsPipeline(t) resolved := resolveDNDSpellsPipeline(t)
llmClient := &fakeSpellsLLMClient{response: extractionResponse{}} llmClient := &fakeSpellsLLMClient{response: extractionResponse{}}
@@ -290,17 +290,11 @@ func TestRunnerCarriesMalformedDNDSpellsExtractorOutput(t *testing.T) {
output, err := runPreparedPipeline(t, dndSpellsRunnerRegistries(t), resolved.ResolvedPipeline, llmClient, pipeline.RunInput{ output, err := runPreparedPipeline(t, dndSpellsRunnerRegistries(t), resolved.ResolvedPipeline, llmClient, pipeline.RunInput{
RawInput: raw, RawInput: raw,
}) })
if err != nil { if err == nil || !strings.Contains(err.Error(), "spell_casts must be present") {
t.Fatalf("Run() error = %v, want nil", err) t.Fatalf("Run() error = %v, want invalid spell-list serialization error", err)
} }
if len(output.NormalizeOutputs) != 1 { if len(output.NormalizeOutputs) != 0 {
t.Fatalf("len(NormalizeOutputs) = %d, want raw output", len(output.NormalizeOutputs)) t.Fatalf("len(NormalizeOutputs) = %d, want no serialized malformed artifact", len(output.NormalizeOutputs))
}
if string(output.NormalizeOutputs[0].Artifact.Content) != `{"spell_casts":null}` {
t.Fatalf("content = %s, want canonical structured output", output.NormalizeOutputs[0].Artifact.Content)
}
if output.Manifest.ValidationStatus != "approved" {
t.Fatalf("ValidationStatus = %q, want approved", output.Manifest.ValidationStatus)
} }
} }

View File

@@ -2,12 +2,14 @@ package transcript
import ( import (
"context" "context"
"encoding/json"
"os" "os"
"reflect" "reflect"
"strings" "strings"
"testing" "testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/config" "gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder" "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
@@ -131,29 +133,36 @@ func seriatimTestCatalog(t *testing.T, inputSpec pipeline.ModuleSpec) pipeline.M
Provides: []string{"chunks"}, Provides: []string{"chunks"},
}) })
mustRegisterExtractor(t, extractors, pipeline.ModuleSpec{ mustRegisterExtractor(t, extractors, pipeline.ModuleSpec{
Key: "fake/extract", Key: "fake/extract",
Stage: pipeline.StageExtract, Stage: pipeline.StageExtract,
Requires: []string{"chunks", "transcript.speaker", "transcript.timestamps"}, ArtifactKind: seriatimArtifactKind,
Provides: []string{"fake.artifacts"}, Requires: []string{"chunks", "transcript.speaker", "transcript.timestamps"},
Provides: []string{"fake.artifacts"},
}) })
mustRegisterMerger(t, mergers, pipeline.ModuleSpec{ mustRegisterMerger(t, mergers, pipeline.ModuleSpec{
Key: pipeline.DefaultMergeModule, Key: pipeline.DefaultMergeModule,
Stage: pipeline.StageMerge, Stage: pipeline.StageMerge,
Requires: []string{"fake.artifacts"}, ArtifactKind: seriatimArtifactKind,
Requires: []string{"fake.artifacts"},
}) })
mustRegisterNormalizer(t, normalizers, pipeline.ModuleSpec{ mustRegisterNormalizer(t, normalizers, pipeline.ModuleSpec{
Key: pipeline.DefaultNormalizeModule, Key: pipeline.DefaultNormalizeModule,
Stage: pipeline.StageNormalize, Stage: pipeline.StageNormalize,
ArtifactKind: seriatimArtifactKind,
}) })
mustRegisterOutput(t, outputs, pipeline.ModuleSpec{ mustRegisterOutput(t, outputs, pipeline.ModuleSpec{
Key: pipeline.DefaultOutputModule, Key: pipeline.DefaultOutputModule,
Stage: pipeline.StageOutput, Stage: pipeline.StageOutput,
}) })
codecs := pipeline.NewArtifactCodecRegistry()
if err := pipeline.RegisterArtifactCodec(codecs, seriatimArtifactCodec{}); err != nil {
t.Fatalf("register artifact codec: %v", err)
}
return pipeline.ModuleCatalog{ return pipeline.ModuleCatalog{
Inputs: inputs, Inputs: inputs,
Chunkers: chunkers, Chunkers: chunkers,
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), ArtifactCodecs: codecs,
Extractors: extractors, Extractors: extractors,
Mergers: mergers, Mergers: mergers,
Normalizers: normalizers, Normalizers: normalizers,
@@ -173,7 +182,7 @@ func mustRegisterChunker(t *testing.T, registry *pipeline.ChunkerRegistry, spec
func mustRegisterExtractor(t *testing.T, registry *pipeline.ExtractorRegistry, spec pipeline.ModuleSpec) { func mustRegisterExtractor(t *testing.T, registry *pipeline.ExtractorRegistry, spec pipeline.ModuleSpec) {
t.Helper() t.Helper()
if err := registry.RegisterLegacyRawWithSpec(spec, func() (contracts.LegacyRawExtractor, error) { if err := pipeline.RegisterExtractor[seriatimArtifact](registry, spec, func() (contracts.Extractor[seriatimArtifact], error) {
return fakeExtractor{}, nil return fakeExtractor{}, nil
}); err != nil { }); err != nil {
t.Fatalf("register extractor: %v", err) t.Fatalf("register extractor: %v", err)
@@ -182,8 +191,13 @@ func mustRegisterExtractor(t *testing.T, registry *pipeline.ExtractorRegistry, s
func mustRegisterMerger(t *testing.T, registry *pipeline.MergerRegistry, spec pipeline.ModuleSpec) { func mustRegisterMerger(t *testing.T, registry *pipeline.MergerRegistry, spec pipeline.ModuleSpec) {
t.Helper() t.Helper()
if err := registry.RegisterLegacyRawWithSpec(spec, func() (contracts.LegacyRawMerger, error) { if err := pipeline.RegisterMerger[seriatimArtifact](registry, spec, func() (contracts.Merger[seriatimArtifact], error) {
return appendorder.New(), nil return appendorder.NewTyped(func(values []seriatimArtifact) (seriatimArtifact, error) {
if len(values) == 0 {
return seriatimArtifact{}, nil
}
return values[0], nil
})
}); err != nil { }); err != nil {
t.Fatalf("register merger: %v", err) t.Fatalf("register merger: %v", err)
} }
@@ -191,8 +205,8 @@ func mustRegisterMerger(t *testing.T, registry *pipeline.MergerRegistry, spec pi
func mustRegisterNormalizer(t *testing.T, registry *pipeline.NormalizerRegistry, spec pipeline.ModuleSpec) { func mustRegisterNormalizer(t *testing.T, registry *pipeline.NormalizerRegistry, spec pipeline.ModuleSpec) {
t.Helper() t.Helper()
if err := registry.RegisterLegacyRawWithSpec(spec, func() (contracts.LegacyRawNormalizer, error) { if err := pipeline.RegisterNormalizer[seriatimArtifact](registry, spec, func() (contracts.Normalizer[seriatimArtifact], error) {
return noop.New(), nil return noop.NewTyped[seriatimArtifact](), nil
}); err != nil { }); err != nil {
t.Fatalf("register normalizer: %v", err) t.Fatalf("register normalizer: %v", err)
} }
@@ -223,8 +237,8 @@ func (fakeExtractor) Key() string { return "fake/extract" }
func (fakeExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil } func (fakeExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (fakeExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) { func (fakeExtractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[seriatimArtifact], error) {
return contracts.ExtractionResult{}, nil return contracts.TypedExtractionResult[seriatimArtifact]{}, nil
} }
type fakeOutput struct{} type fakeOutput struct{}
@@ -246,7 +260,30 @@ func withoutCapability(capabilities []string, capability string) []string {
} }
var ( var (
_ contracts.Chunker = fakeChunker{} _ contracts.Chunker = fakeChunker{}
_ contracts.LegacyRawExtractor = fakeExtractor{} _ contracts.Extractor[seriatimArtifact] = fakeExtractor{}
_ contracts.OutputEncoder = fakeOutput{} _ contracts.OutputEncoder = fakeOutput{}
) )
const seriatimArtifactKind contracts.ArtifactKind = "test/seriatim-event"
type seriatimArtifact struct {
Value string `json:"value"`
SourceRefs []source.SourceRef `json:"source_refs"`
}
type seriatimArtifactCodec struct{}
func (seriatimArtifactCodec) Kind() contracts.ArtifactKind { return seriatimArtifactKind }
func (seriatimArtifactCodec) Schema() contracts.ArtifactSchema {
return contracts.ArtifactSchema{ID: "fake.event", Name: "fake_event", Version: "v1", JSONSchema: []byte(`{"type":"object"}`)}
}
func (seriatimArtifactCodec) MediaType() string { return "application/json" }
func (seriatimArtifactCodec) Encode(value seriatimArtifact) ([]byte, error) {
return json.Marshal(value)
}
func (seriatimArtifactCodec) Decode(content []byte) (seriatimArtifact, error) {
var value seriatimArtifact
err := json.Unmarshal(content, &value)
return value, err
}

View File

@@ -55,16 +55,16 @@ func TestRunnerProcessesSeriatimInputWithFakeModules(t *testing.T) {
t.Fatalf("len(NormalizeOutputs) = %d, want 1", len(output.NormalizeOutputs)) t.Fatalf("len(NormalizeOutputs) = %d, want 1", len(output.NormalizeOutputs))
} }
rawOutput := output.NormalizeOutputs[0] serializedOutput := output.NormalizeOutputs[0]
if rawOutput.LaneID != "events" || rawOutput.NormalizerKey != pipeline.DefaultNormalizeModule || rawOutput.Artifact.Schema.ID != "fake.event" || rawOutput.Artifact.Schema.Version != "v1" { if serializedOutput.LaneID != "events" || serializedOutput.NormalizerKey != pipeline.DefaultNormalizeModule || serializedOutput.Artifact.Schema.ID != "fake.event" || serializedOutput.Artifact.Schema.Version != "v1" {
t.Fatalf("raw output envelope = %#v, want fake extractor envelope", rawOutput) t.Fatalf("serialized output envelope = %#v, want fake extractor envelope", serializedOutput)
} }
var payload struct { var payload struct {
Value string `json:"value"` Value string `json:"value"`
SourceRefs []source.SourceRef `json:"source_refs"` SourceRefs []source.SourceRef `json:"source_refs"`
} }
if err := json.Unmarshal(rawOutput.Artifact.Content, &payload); err != nil { if err := json.Unmarshal(serializedOutput.Artifact.Content, &payload); err != nil {
t.Fatalf("Unmarshal(raw output) error = %v, want nil", err) t.Fatalf("Unmarshal(serialized output) error = %v, want nil", err)
} }
if len(payload.SourceRefs) != 1 { if len(payload.SourceRefs) != 1 {
t.Fatalf("len(SourceRefs) = %d, want 1", len(payload.SourceRefs)) t.Fatalf("len(SourceRefs) = %d, want 1", len(payload.SourceRefs))
@@ -115,7 +115,7 @@ func configResolveInput(t *testing.T) config.ResolveInput {
} }
} }
func seriatimRunnerRegistries(t *testing.T, extractor contracts.LegacyRawExtractor) pipeline.Registries { func seriatimRunnerRegistries(t *testing.T, extractor contracts.Extractor[seriatimArtifact]) pipeline.Registries {
t.Helper() t.Helper()
inputs := pipeline.NewInputAdapterRegistry() inputs := pipeline.NewInputAdapterRegistry()
@@ -133,19 +133,20 @@ func seriatimRunnerRegistries(t *testing.T, extractor contracts.LegacyRawExtract
}); err != nil { }); err != nil {
t.Fatalf("register chunker: %v", err) t.Fatalf("register chunker: %v", err)
} }
if err := extractors.RegisterLegacyRaw("fake/extract", func() (contracts.LegacyRawExtractor, error) { if err := pipeline.RegisterExtractor[seriatimArtifact](extractors, pipeline.ModuleSpec{Key: "fake/extract", Stage: pipeline.StageExtract, ArtifactKind: seriatimArtifactKind, Requires: []string{"chunks", "transcript.speaker", "transcript.timestamps"}, Provides: []string{"fake.artifacts"}}, func() (contracts.Extractor[seriatimArtifact], error) {
return extractor, nil return extractor, nil
}); err != nil { }); err != nil {
t.Fatalf("register extractor: %v", err) t.Fatalf("register extractor: %v", err)
} }
if err := mergers.RegisterLegacyRaw(pipeline.DefaultMergeModule, func() (contracts.LegacyRawMerger, error) { if err := appendorder.RegisterTyped(mergers, seriatimArtifactKind, func(values []seriatimArtifact) (seriatimArtifact, error) {
return appendorder.New(), nil if len(values) == 0 {
return seriatimArtifact{}, nil
}
return values[0], nil
}); err != nil { }); err != nil {
t.Fatalf("register merger: %v", err) t.Fatalf("register merger: %v", err)
} }
if err := normalizers.RegisterLegacyRaw(pipeline.DefaultNormalizeModule, func() (contracts.LegacyRawNormalizer, error) { if err := noop.RegisterTyped[seriatimArtifact](normalizers, seriatimArtifactKind); err != nil {
return noop.New(), nil
}); err != nil {
t.Fatalf("register normalizer: %v", err) t.Fatalf("register normalizer: %v", err)
} }
if err := outputs.Register(pipeline.DefaultOutputModule, func() (contracts.OutputEncoder, error) { if err := outputs.Register(pipeline.DefaultOutputModule, func() (contracts.OutputEncoder, error) {
@@ -154,10 +155,14 @@ func seriatimRunnerRegistries(t *testing.T, extractor contracts.LegacyRawExtract
t.Fatalf("register output: %v", err) t.Fatalf("register output: %v", err)
} }
codecs := pipeline.NewArtifactCodecRegistry()
if err := pipeline.RegisterArtifactCodec(codecs, seriatimArtifactCodec{}); err != nil {
t.Fatalf("register artifact codec: %v", err)
}
return pipeline.Registries{ return pipeline.Registries{
Inputs: inputs, Inputs: inputs,
Chunkers: chunkers, Chunkers: chunkers,
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), ArtifactCodecs: codecs,
Extractors: extractors, Extractors: extractors,
Mergers: mergers, Mergers: mergers,
Normalizers: normalizers, Normalizers: normalizers,
@@ -203,36 +208,33 @@ func (e *runnerSeriatimExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil return nil
} }
func (e *runnerSeriatimExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) { func (e *runnerSeriatimExtractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[seriatimArtifact], error) {
e.calls++ e.calls++
if req.Source == nil { if req.Source == nil {
return contracts.ExtractionResult{}, fmt.Errorf("source must not be nil") return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("source must not be nil")
} }
if req.Chunk == nil { if req.Chunk == nil {
return contracts.ExtractionResult{}, fmt.Errorf("chunk must not be nil") return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("chunk must not be nil")
} }
if got := unitIDs(req.Source.Units); !equalInts(got, []int{1, 2}) { if got := unitIDs(req.Source.Units); !equalInts(got, []int{1, 2}) {
return contracts.ExtractionResult{}, fmt.Errorf("source unit IDs = %#v, want Seriatim segment IDs", got) return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("source unit IDs = %#v, want Seriatim segment IDs", got)
} }
if got := unitIDs(req.Chunk.Units); !equalInts(got, []int{1, 2}) { if got := unitIDs(req.Chunk.Units); !equalInts(got, []int{1, 2}) {
return contracts.ExtractionResult{}, fmt.Errorf("chunk unit IDs = %#v, want Seriatim segment IDs", got) return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("chunk unit IDs = %#v, want Seriatim segment IDs", got)
} }
for _, unit := range req.Chunk.Units { for _, unit := range req.Chunk.Units {
if speaker, ok := Speaker(unit); !ok || speaker == "" { if speaker, ok := Speaker(unit); !ok || speaker == "" {
return contracts.ExtractionResult{}, fmt.Errorf("unit %d missing speaker metadata", unit.ID) return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("unit %d missing speaker metadata", unit.ID)
} }
if _, ok := Start(unit); !ok { if _, ok := Start(unit); !ok {
return contracts.ExtractionResult{}, fmt.Errorf("unit %d missing start metadata", unit.ID) return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("unit %d missing start metadata", unit.ID)
} }
if _, ok := End(unit); !ok { if _, ok := End(unit); !ok {
return contracts.ExtractionResult{}, fmt.Errorf("unit %d missing end metadata", unit.ID) return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("unit %d missing end metadata", unit.ID)
} }
} }
payload, err := json.Marshal(struct { return contracts.TypedExtractionResult[seriatimArtifact]{Value: seriatimArtifact{
Value string `json:"value"`
SourceRefs []source.SourceRef `json:"source_refs"`
}{
Value: "seriatim-source-ref", Value: "seriatim-source-ref",
SourceRefs: []source.SourceRef{ SourceRefs: []source.SourceRef{
{ {
@@ -241,20 +243,7 @@ func (e *runnerSeriatimExtractor) Extract(ctx context.Context, req contracts.Ext
EndUnitID: req.Chunk.Units[len(req.Chunk.Units)-1].ID, EndUnitID: req.Chunk.Units[len(req.Chunk.Units)-1].ID,
}, },
}, },
}) }}, nil
if err != nil {
return contracts.ExtractionResult{}, err
}
return contracts.ExtractionResult{
Output: contracts.ExtractOutput{
Schema: contracts.ResponseSchema{ID: "fake.event", Name: "fake_event", Version: "v1"},
Payload: contracts.RawPayload{
Content: payload,
MediaType: "application/json",
},
},
}, nil
} }
type runnerSeriatimOutput struct{} type runnerSeriatimOutput struct{}
@@ -292,7 +281,7 @@ func equalInts(a, b []int) bool {
} }
var ( var (
_ contracts.Chunker = runnerSeriatimChunker{} _ contracts.Chunker = runnerSeriatimChunker{}
_ contracts.LegacyRawExtractor = (*runnerSeriatimExtractor)(nil) _ contracts.Extractor[seriatimArtifact] = (*runnerSeriatimExtractor)(nil)
_ contracts.OutputEncoder = runnerSeriatimOutput{} _ contracts.OutputEncoder = runnerSeriatimOutput{}
) )