eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement raw module output contracts

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-07 18:58:23 +00:00
parent 9e3f8809b3
commit c05ecb58d8
33 changed files with 1166 additions and 1780 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
internal/cli/run.go
View File

@@ -275,7 +275,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
PipelineID: effective.PipelineID,
OutputPath: runOutputDir,
DiagnosticsPath: runDir.Path(),
ApprovedCount: len(output.Approved),
OutputCount: len(output.NormalizeOutputs),
RejectedCount: len(output.Rejected),
WarningCount: len(output.Warnings),
ValidationStatus: output.Manifest.ValidationStatus,
@@ -293,7 +293,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("apply diagnostics retention: %w", err))
}
fmt.Fprintf(stdout, "pipeline %q complete: approved=%d rejected=%d output=%s\n", effective.PipelineID, len(output.Approved), len(output.Rejected), runOutputDir)
fmt.Fprintf(stdout, "pipeline %q complete: outputs=%d rejected=%d output=%s\n", effective.PipelineID, len(output.NormalizeOutputs), len(output.Rejected), runOutputDir)
if len(output.Warnings) > 0 {
fmt.Fprintf(stderr, "notarius: run completed with %d warning(s)\n", len(output.Warnings))
}
@@ -305,7 +305,7 @@ type runReport struct {
PipelineID string `json:"pipeline_id"`
OutputPath string `json:"output_path"`
DiagnosticsPath string `json:"diagnostics_path,omitempty"`
ApprovedCount int `json:"approved_count"`
OutputCount int `json:"output_count"`
RejectedCount int `json:"rejected_count"`
WarningCount int `json:"warning_count"`
ValidationStatus string `json:"validation_status,omitempty"`

104
internal/cli/run_test.go
View File

@@ -690,7 +690,7 @@ func TestRunPipelineSuccessUsesProductionRegistriesAndFakeLLM(t *testing.T) {
if client.calls != 1 {
t.Fatalf("LLM calls = %d, want 1", client.calls)
}
for _, want := range []string{"dnd-session", "approved=1", "rejected=0", outputDir} {
for _, want := range []string{"dnd-session", "outputs=1", "rejected=0", outputDir} {
if !strings.Contains(stdout.String(), want) {
t.Fatalf("stdout = %q, want substring %q", stdout.String(), want)
}
@@ -719,8 +719,8 @@ func TestRunPipelineOnlySelectsRequestedLane(t *testing.T) {
if client.calls != 1 {
t.Fatalf("LLM calls = %d, want only selected lane to run once", client.calls)
}
if !strings.Contains(stdout.String(), "approved=1") {
t.Fatalf("stdout = %q, want approved count", stdout.String())
if !strings.Contains(stdout.String(), "outputs=1") {
t.Fatalf("stdout = %q, want output count", stdout.String())
}
}
@@ -743,7 +743,7 @@ func TestRunPipelineLLMFactoryFailure(t *testing.T) {
}
}
func TestRunPipelineValidationRejectionCompletesSuccessfully(t *testing.T) {
func TestRunPipelineCarriesInvalidLLMSourceRefsAsRawOutput(t *testing.T) {
configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "dnd/spells"))
inputPath := writeSeriatimInput(t)
outputDir := t.TempDir()
@@ -759,8 +759,8 @@ func TestRunPipelineValidationRejectionCompletesSuccessfully(t *testing.T) {
if code != 0 {
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
}
if !strings.Contains(stdout.String(), "approved=0") || !strings.Contains(stdout.String(), "rejected=1") {
t.Fatalf("stdout = %q, want rejection counts", stdout.String())
if !strings.Contains(stdout.String(), "outputs=1") || !strings.Contains(stdout.String(), "rejected=0") {
t.Fatalf("stdout = %q, want raw output count", stdout.String())
}
}
@@ -1644,7 +1644,7 @@ func TestRunPipelineWritesDurableOutputFiles(t *testing.T) {
for _, name := range []string{
"index.json",
"manifest.json",
"artifacts/dnd.spell_cast.json",
"outputs/spells.json",
"rejected.json",
"warnings.json",
} {
@@ -1819,7 +1819,7 @@ func TestRunPipelineWritesDiagnosticsArtifactsOnSuccess(t *testing.T) {
}
}
report := string(readFile(t, filepath.Join(runDir, diagnostics.ArtifactRunReport)))
if !strings.Contains(report, `"approved_count": 1`) || !strings.Contains(report, `"validation_status": "approved"`) || !strings.Contains(report, outputDir) {
if !strings.Contains(report, `"output_count": 1`) || !strings.Contains(report, `"validation_status": "approved"`) || !strings.Contains(report, outputDir) {
t.Fatalf("unexpected run report: %s", report)
}
}
@@ -1989,29 +1989,26 @@ func TestExampleFixtureRunWritesExpectedJSON(t *testing.T) {
t.Fatalf("extractor metadata = %#v, want prompt/schema identifiers", extractorMetadata)
}
var artifactFile struct {
ArtifactType string `json:"artifact_type"`
Artifacts []artifacts.Artifact `json:"artifacts"`
var spellOutput struct {
SpellCasts []struct {
Caster string `json:"caster"`
Spell string `json:"spell"`
Effect string `json:"effect"`
SourceRefs []source.SourceRef `json:"source_refs"`
} `json:"spell_casts"`
}
readJSONFile(t, filepath.Join(runOutputDir, "artifacts", "dnd.spell_cast.json"), &artifactFile)
if artifactFile.ArtifactType != "dnd.spell_cast" || len(artifactFile.Artifacts) != 1 {
t.Fatalf("artifact file = %#v, want one spell artifact", artifactFile)
}
var payload struct {
Caster string `json:"caster"`
Spell string `json:"spell"`
Effect string `json:"effect"`
}
if err := json.Unmarshal(artifactFile.Artifacts[0].Payload, &payload); err != nil {
t.Fatalf("unmarshal spell payload: %v", err)
readJSONFile(t, filepath.Join(runOutputDir, "outputs", "spells.json"), &spellOutput)
if len(spellOutput.SpellCasts) != 1 {
t.Fatalf("spell output = %#v, want one spell cast", spellOutput)
}
payload := spellOutput.SpellCasts[0]
if payload.Caster != "Aria" || payload.Spell != "Cure Wounds" || payload.Effect == "" {
t.Fatalf("payload = %#v, want deterministic spell output", payload)
}
if len(artifactFile.Artifacts[0].SourceRefs) != 1 {
t.Fatalf("source refs = %#v, want one source ref", artifactFile.Artifacts[0].SourceRefs)
if len(payload.SourceRefs) != 1 {
t.Fatalf("source refs = %#v, want one source ref", payload.SourceRefs)
}
ref := artifactFile.Artifacts[0].SourceRefs[0]
ref := payload.SourceRefs[0]
if ref.SourceID != "session-alpha" || ref.StartUnitID != 1 || ref.EndUnitID != 1 {
t.Fatalf("source ref = %#v, want fixture source ref", ref)
}
@@ -2175,11 +2172,11 @@ func TestExampleFixtureFailureCoverage(t *testing.T) {
wantStderr: "spell_casts",
},
{
name: "invalid source reference rejection",
name: "invalid source reference raw output",
args: []string{"run", "dnd-session", "--config", configPath, "--input", inputPath},
factory: fakeLLMFactory(newFakeRunLLMClient(true), nil),
wantCode: 0,
wantOutputStatus: "rejected",
wantOutputStatus: "approved",
},
}
@@ -2213,10 +2210,6 @@ func TestExampleFixtureFailureCoverage(t *testing.T) {
if manifest.ValidationStatus != test.wantOutputStatus {
t.Fatalf("validation status = %q, want %q", manifest.ValidationStatus, test.wantOutputStatus)
}
rejected := string(readFile(t, filepath.Join(runOutputDir, "rejected.json")))
if !strings.Contains(rejected, "invalid_source_ref") {
t.Fatalf("rejected output = %s, want invalid source ref rejection", rejected)
}
}
})
}
@@ -2682,30 +2675,17 @@ func (fakeRunExtractor) Key() string {
return "fake/extract"
}
func (fakeRunExtractor) ArtifactType() string {
return "fake.artifact"
}
func (fakeRunExtractor) SchemaVersion() string {
return "v1"
}
func (fakeRunExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return []contracts.ReferenceSlot{{Name: "roster"}}
}
func (fakeRunExtractor) Validators() []contracts.Validator {
return nil
}
func (fakeRunExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
return contracts.ExtractionResult{
Candidates: []artifacts.ArtifactCandidate{
{
Payload: []byte(`{"value":true}`),
SourceRefs: []source.SourceRef{
{SourceID: "source", StartUnitID: 1, EndUnitID: 1},
},
Output: contracts.ExtractOutput{
Schema: contracts.ResponseSchema{ID: "fake.artifact", Name: "fake_artifact", Version: "v1"},
Payload: contracts.RawPayload{
Content: []byte(`{"value":true}`),
MediaType: "application/json",
},
},
}, nil
@@ -2718,11 +2698,20 @@ func (fakeRunMerger) Key() string {
}
func (fakeRunMerger) Merge(ctx context.Context, req contracts.MergeRequest) (contracts.MergeResult, error) {
var candidates []artifacts.ArtifactCandidate
for _, chunkArtifacts := range req.ChunkArtifacts {
candidates = append(candidates, chunkArtifacts.Candidates...)
output := contracts.MergeOutput{
LaneID: req.LaneID,
SourceID: req.Source.ID,
Schema: contracts.ResponseSchema{ID: "fake.artifact", Name: "fake_artifact", Version: "v1"},
Payload: contracts.RawPayload{
Content: []byte(`{"merged":true}`),
MediaType: "application/json",
},
}
return contracts.MergeResult{Candidates: candidates}, nil
if len(req.ExtractOutputs) > 0 {
output.Schema = req.ExtractOutputs[0].Schema
output.Payload = req.ExtractOutputs[0].Payload
}
return contracts.MergeResult{Output: output}, nil
}
type fakeRunNormalizer struct{}
@@ -2736,7 +2725,14 @@ func (fakeRunNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
}
func (fakeRunNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
return contracts.NormalizeResult{Candidates: append([]artifacts.ArtifactCandidate(nil), req.Candidates...)}, nil
return contracts.NormalizeResult{
Output: contracts.NormalizeOutput{
LaneID: req.LaneID,
SourceID: req.MergeOutput.SourceID,
Schema: req.MergeOutput.Schema,
Payload: req.MergeOutput.Payload,
},
}, nil
}
func onlyChildDir(t *testing.T, root string) string {

147
internal/framework/contracts/composition_test.go
View File

@@ -27,7 +27,6 @@ func TestContractsComposeAcrossPackages(t *testing.T) {
extractor := compositionExtractor{}
merger := compositionMerger{}
normalizer := compositionNormalizer{}
validator := compositionValidator{}
encoder := compositionOutputEncoder{}
doc, err := adapter.Parse(ctx, contracts.ParseRequest{SourceID: "source-1"})
@@ -58,70 +57,37 @@ func TestContractsComposeAcrossPackages(t *testing.T) {
if err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
if len(extraction.Candidates) != 1 {
t.Fatalf("len(Candidates) = %d, want 1", len(extraction.Candidates))
}
candidate := extraction.Candidates[0]
for _, ref := range candidate.SourceRefs {
if err := source.ValidateRef(doc, ref); err != nil {
t.Fatalf("ValidateRef() error = %v, want nil", err)
}
if extraction.Output.Payload.MediaType != "application/json" {
t.Fatalf("extract media type = %q, want application/json", extraction.Output.Payload.MediaType)
}
merge, err := merger.Merge(ctx, contracts.MergeRequest{
Source: doc,
LaneID: candidate.ArtifactType,
ChunkArtifacts: []contracts.ChunkArtifacts{
{
Chunk: chunking.Chunks[0],
Candidates: extraction.Candidates,
},
},
Source: doc,
LaneID: "generic-lane",
ExtractOutputs: []contracts.ExtractOutput{extraction.Output},
})
if err != nil {
t.Fatalf("Merge() error = %v, want nil", err)
}
if len(merge.Candidates) != 1 {
t.Fatalf("len(merge.Candidates) = %d, want 1", len(merge.Candidates))
if string(merge.Output.Payload.Content) != `{"value":"example"}` {
t.Fatalf("merge output = %s, want extract payload", merge.Output.Payload.Content)
}
normalize, err := normalizer.Normalize(ctx, contracts.NormalizeRequest{
Source: doc,
LaneID: candidate.ArtifactType,
Candidates: merge.Candidates,
Source: doc,
LaneID: "generic-lane",
MergeOutput: merge.Output,
})
if err != nil {
t.Fatalf("Normalize() error = %v, want nil", err)
}
if len(normalize.Candidates) != 1 {
t.Fatalf("len(normalize.Candidates) = %d, want 1", len(normalize.Candidates))
}
validation, err := validator.Validate(ctx, contracts.ValidationRequest{
Source: doc,
Candidates: normalize.Candidates,
})
if err != nil {
t.Fatalf("Validate() error = %v, want nil", err)
}
if len(validation.Decisions) != 1 {
t.Fatalf("len(Decisions) = %d, want 1", len(validation.Decisions))
}
decision := validation.Decisions[0]
if !decision.Approved {
t.Fatal("Approved = false, want true")
}
if decision.CandidateIndex != candidate.Index {
t.Fatalf("CandidateIndex = %d, want %d", decision.CandidateIndex, candidate.Index)
if string(normalize.Output.Payload.Content) != `{"value":"example"}` {
t.Fatalf("normalize output = %s, want merge payload", normalize.Output.Payload.Content)
}
output, err := encoder.Encode(ctx, contracts.OutputRequest{
Manifest: artifacts.RunManifest{RunID: "run-1"},
Approved: []artifacts.Artifact{
artifacts.ArtifactFromCandidate(normalize.Candidates[0]),
},
Manifest: artifacts.RunManifest{RunID: "run-1"},
NormalizeOutputs: []contracts.NormalizeOutput{normalize.Output},
})
if err != nil {
t.Fatalf("Encode() error = %v, want nil", err)
@@ -203,49 +169,24 @@ func (extractor compositionExtractor) Key() string {
return "generic-extractor"
}
func (extractor compositionExtractor) ArtifactType() string {
return "generic-artifact"
}
func (extractor compositionExtractor) SchemaVersion() string {
return "v1"
}
func (extractor compositionExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (extractor compositionExtractor) Validators() []contracts.Validator {
return []contracts.Validator{compositionValidator{}}
}
func (extractor compositionExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
if req.Source == nil {
return contracts.ExtractionResult{}, errors.New("source document is required")
}
units := req.Source.Units
if req.Chunk != nil {
units = req.Chunk.Units
}
if req.AmbientContext["synopsis"] == "" {
return contracts.ExtractionResult{}, errors.New("ambient synopsis is required")
}
return contracts.ExtractionResult{
Candidates: []artifacts.ArtifactCandidate{
{
Index: 0,
ExtractorKey: extractor.Key(),
ArtifactType: extractor.ArtifactType(),
SchemaVersion: extractor.SchemaVersion(),
Payload: json.RawMessage(`{"value":"example"}`),
SourceRefs: []source.SourceRef{
{
SourceID: req.Source.ID,
StartUnitID: units[0].ID,
EndUnitID: units[len(units)-1].ID,
},
},
Output: contracts.ExtractOutput{
Schema: contracts.ResponseSchema{ID: "schema-id", Name: "schema-name", Version: "v1"},
Payload: contracts.RawPayload{
Content: []byte(`{"value":"example"}`),
MediaType: "application/json",
},
},
}, nil
@@ -258,12 +199,14 @@ func (merger compositionMerger) Key() string {
}
func (merger compositionMerger) Merge(ctx context.Context, req contracts.MergeRequest) (contracts.MergeResult, error) {
var candidates []artifacts.ArtifactCandidate
for _, chunkArtifacts := range req.ChunkArtifacts {
candidates = append(candidates, chunkArtifacts.Candidates...)
}
return contracts.MergeResult{Candidates: candidates}, nil
output := req.ExtractOutputs[0]
return contracts.MergeResult{Output: contracts.MergeOutput{
LaneID: req.LaneID,
MergerKey: merger.Key(),
SourceID: output.SourceID,
Schema: output.Schema,
Payload: cloneCompositionPayload(output.Payload),
}}, nil
}
type compositionNormalizer struct{}
@@ -277,7 +220,33 @@ func (normalizer compositionNormalizer) ReferenceSlots() []contracts.ReferenceSl
}
func (normalizer compositionNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
return contracts.NormalizeResult{Candidates: req.Candidates}, nil
return contracts.NormalizeResult{Output: contracts.NormalizeOutput{
LaneID: req.LaneID,
NormalizerKey: normalizer.Key(),
SourceID: req.MergeOutput.SourceID,
Schema: req.MergeOutput.Schema,
Payload: cloneCompositionPayload(req.MergeOutput.Payload),
}}, nil
}
func cloneCompositionPayload(payload contracts.RawPayload) contracts.RawPayload {
return contracts.RawPayload{
Content: append([]byte(nil), payload.Content...),
MediaType: payload.MediaType,
Metadata: cloneCompositionMetadata(payload.Metadata),
Warnings: append([]contracts.Warning(nil), payload.Warnings...),
}
}
func cloneCompositionMetadata(metadata map[string]any) map[string]any {
if len(metadata) == 0 {
return nil
}
out := make(map[string]any, len(metadata))
for key, value := range metadata {
out[key] = value
}
return out
}
type compositionValidator struct{}
@@ -311,11 +280,11 @@ func (encoder compositionOutputEncoder) Key() string {
func (encoder compositionOutputEncoder) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
payload := struct {
RunID string `json:"run_id"`
ApprovedCount int `json:"approved_count"`
RunID string `json:"run_id"`
OutputCount int `json:"output_count"`
}{
RunID: req.Manifest.RunID,
ApprovedCount: len(req.Approved),
RunID: req.Manifest.RunID,
OutputCount: len(req.NormalizeOutputs),
}
encoded, err := json.Marshal(payload)
if err != nil {

104
internal/framework/contracts/contracts.go
View File

@@ -186,36 +186,59 @@ type ExtractionRequest struct {
}
type ExtractionResult struct {
Candidates []artifacts.ArtifactCandidate `json:"candidates,omitempty"`
Warnings []Warning `json:"warnings,omitempty"`
Output ExtractOutput `json:"output"`
Warnings []Warning `json:"warnings,omitempty"`
}
type Extractor interface {
Key() string
ArtifactType() string
SchemaVersion() string
ReferenceSlots() []ReferenceSlot
Validators() []Validator
Extract(ctx context.Context, req ExtractionRequest) (ExtractionResult, error)
}
type ChunkArtifacts struct {
Chunk SourceChunk `json:"chunk"`
Candidates []artifacts.ArtifactCandidate `json:"candidates"`
type RawPayload struct {
Content []byte `json:"-"`
MediaType string `json:"media_type"`
Metadata map[string]any `json:"metadata,omitempty"`
Warnings []Warning `json:"warnings,omitempty"`
}
type ResponseSchema struct {
ID string `json:"id,omitempty"`
Name string `json:"name,omitempty"`
Version string `json:"version,omitempty"`
}
type ExtractOutput struct {
LaneID string `json:"lane_id"`
ExtractorKey string `json:"extractor_key"`
SourceID string `json:"source_id"`
ChunkID string `json:"chunk_id"`
ChunkIndex int `json:"chunk_index"`
Schema ResponseSchema `json:"schema,omitempty"`
Payload RawPayload `json:"payload"`
}
type MergeRequest struct {
Source *source.SourceDocument `json:"-"`
LaneID string `json:"lane_id"`
ChunkArtifacts []ChunkArtifacts `json:"chunk_artifacts"`
ExtractOutputs []ExtractOutput `json:"extract_outputs"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type MergeResult struct {
Candidates []artifacts.ArtifactCandidate `json:"candidates"`
Warnings []Warning `json:"warnings,omitempty"`
Output MergeOutput `json:"output"`
Warnings []Warning `json:"warnings,omitempty"`
}
type MergeOutput struct {
LaneID string `json:"lane_id"`
MergerKey string `json:"merger_key"`
SourceID string `json:"source_id,omitempty"`
Schema ResponseSchema `json:"schema,omitempty"`
Payload RawPayload `json:"payload"`
}
type Merger interface {
@@ -224,21 +247,29 @@ type Merger interface {
}
type NormalizeRequest struct {
Source *source.SourceDocument `json:"-"`
LaneID string `json:"lane_id"`
Candidates []artifacts.ArtifactCandidate `json:"candidates"`
SourceInput LLMInputMaterial `json:"source_input,omitempty"`
SessionID string `json:"session_id,omitempty"`
References ReferenceSet `json:"references,omitempty"`
LLMClient StructuredLLMClient `json:"-"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
Source *source.SourceDocument `json:"-"`
LaneID string `json:"lane_id"`
MergeOutput MergeOutput `json:"merge_output"`
SourceInput LLMInputMaterial `json:"source_input,omitempty"`
SessionID string `json:"session_id,omitempty"`
References ReferenceSet `json:"references,omitempty"`
LLMClient StructuredLLMClient `json:"-"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type NormalizeResult struct {
Candidates []artifacts.ArtifactCandidate `json:"candidates"`
Warnings []Warning `json:"warnings,omitempty"`
Output NormalizeOutput `json:"output"`
Warnings []Warning `json:"warnings,omitempty"`
}
type NormalizeOutput struct {
LaneID string `json:"lane_id"`
NormalizerKey string `json:"normalizer_key"`
SourceID string `json:"source_id,omitempty"`
Schema ResponseSchema `json:"schema,omitempty"`
Payload RawPayload `json:"payload"`
}
type Normalizer interface {
@@ -281,13 +312,13 @@ type Warning struct {
}
type OutputRequest struct {
Manifest artifacts.RunManifest `json:"manifest"`
Approved []artifacts.Artifact `json:"approved,omitempty"`
Rejected []artifacts.RejectedArtifact `json:"rejected,omitempty"`
Warnings []Warning `json:"warnings,omitempty"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
Manifest artifacts.RunManifest `json:"manifest"`
NormalizeOutputs []NormalizeOutput `json:"normalize_outputs,omitempty"`
Rejected []RejectedOutput `json:"rejected,omitempty"`
Warnings []Warning `json:"warnings,omitempty"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type OutputFile struct {
@@ -306,6 +337,19 @@ type OutputEncoder interface {
Encode(ctx context.Context, req OutputRequest) (OutputResult, error)
}
type RejectedOutput struct {
Stage string `json:"stage"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key,omitempty"`
ChunkID string `json:"chunk_id,omitempty"`
ChunkIndex int `json:"chunk_index,omitempty"`
ValidatorName string `json:"validator_name,omitempty"`
ReasonCode string `json:"reason_code,omitempty"`
Message string `json:"message"`
AttemptCount int `json:"attempt_count,omitempty"`
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
}
type ManifestMetadataProvider interface {
ManifestMetadata() map[string]any
}

203
internal/framework/contracts/contracts_test.go
View File

@@ -19,13 +19,9 @@ var _ Validator = fakeValidator{}
var _ StructuredLLMClient = fakeLLMClient{}
var _ OutputEncoder = fakeOutputEncoder{}
func TestFakeExtractorReturnsCandidateAndValidator(t *testing.T) {
validator := fakeValidator{name: "generic-validator"}
func TestFakeExtractorReturnsRawOutput(t *testing.T) {
extractor := fakeExtractor{
key: "generic-extractor",
artifactType: "generic-artifact",
schemaVersion: "v1",
validators: []Validator{validator},
key: "generic-extractor",
}
doc := &source.SourceDocument{
ID: "source-1",
@@ -45,37 +41,14 @@ func TestFakeExtractorReturnsCandidateAndValidator(t *testing.T) {
if extractor.Key() != "generic-extractor" {
t.Fatalf("Key() = %q, want generic-extractor", extractor.Key())
}
if extractor.ArtifactType() != "generic-artifact" {
t.Fatalf("ArtifactType() = %q, want generic-artifact", extractor.ArtifactType())
if result.Output.ExtractorKey != "" {
t.Fatalf("ExtractorKey = %q, want runner-owned empty value", result.Output.ExtractorKey)
}
if extractor.SchemaVersion() != "v1" {
t.Fatalf("SchemaVersion() = %q, want v1", extractor.SchemaVersion())
if result.Output.Schema.Version != "v1" {
t.Fatalf("Schema.Version = %q, want v1", result.Output.Schema.Version)
}
if len(extractor.Validators()) != 1 {
t.Fatalf("len(Validators()) = %d, want 1", len(extractor.Validators()))
}
if extractor.Validators()[0].Name() != "generic-validator" {
t.Fatalf("Validators()[0].Name() = %q, want generic-validator", extractor.Validators()[0].Name())
}
if len(result.Candidates) != 1 {
t.Fatalf("len(Candidates) = %d, want 1", len(result.Candidates))
}
candidate := result.Candidates[0]
if candidate.Index != 0 {
t.Fatalf("ArtifactCandidate.Index = %d, want 0", candidate.Index)
}
if candidate.ExtractorKey != extractor.Key() {
t.Fatalf("ArtifactCandidate.ExtractorKey = %q, want %q", candidate.ExtractorKey, extractor.Key())
}
if candidate.ArtifactType != extractor.ArtifactType() {
t.Fatalf("ArtifactCandidate.ArtifactType = %q, want %q", candidate.ArtifactType, extractor.ArtifactType())
}
if candidate.SchemaVersion != extractor.SchemaVersion() {
t.Fatalf("ArtifactCandidate.SchemaVersion = %q, want %q", candidate.SchemaVersion, extractor.SchemaVersion())
}
if string(candidate.Payload) != `{"value":"example"}` {
t.Fatalf("ArtifactCandidate.Payload = %s, want example payload", candidate.Payload)
if result.Output.Payload.MediaType != "application/json" || string(result.Output.Payload.Content) != `{"value":"example"}` {
t.Fatalf("payload = %q %s, want JSON raw output", result.Output.Payload.MediaType, result.Output.Payload.Content)
}
}
@@ -146,11 +119,7 @@ func TestFakeChunkerReceivesLLMClient(t *testing.T) {
}
func TestFakeExtractorReceivesChunkAndAmbientContext(t *testing.T) {
extractor := fakeExtractor{
key: "generic-extractor",
artifactType: "generic-artifact",
schemaVersion: "v1",
}
extractor := fakeExtractor{key: "generic-extractor"}
doc := &source.SourceDocument{
ID: "source-1",
Kind: "document",
@@ -180,20 +149,11 @@ func TestFakeExtractorReceivesChunkAndAmbientContext(t *testing.T) {
if err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
if len(result.Candidates) != 1 {
t.Fatalf("len(Candidates) = %d, want 1", len(result.Candidates))
if result.Output.ChunkID != "" || result.Output.ChunkIndex != 0 {
t.Fatalf("chunk provenance = %q/%d, want runner-owned zero values", result.Output.ChunkID, result.Output.ChunkIndex)
}
candidate := result.Candidates[0]
if string(candidate.Payload) != `{"value":"chunked"}` {
t.Fatalf("ArtifactCandidate.Payload = %s, want chunked payload", candidate.Payload)
}
if len(candidate.SourceRefs) != 1 {
t.Fatalf("len(SourceRefs) = %d, want 1", len(candidate.SourceRefs))
}
ref := candidate.SourceRefs[0]
if ref.StartUnitID != 2 || ref.EndUnitID != 2 {
t.Fatalf("SourceRef = %+v, want unit 2 range", ref)
if string(result.Output.Payload.Content) != `{"value":"chunked"}` {
t.Fatalf("Payload.Content = %s, want chunked payload", result.Output.Payload.Content)
}
}
@@ -363,19 +323,17 @@ func TestLLMInputSetCloneCopiesContent(t *testing.T) {
}
func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
candidate := artifacts.ArtifactCandidate{
Index: 0,
ExtractorKey: "generic-extractor",
ArtifactType: "generic-artifact",
SchemaVersion: "v1",
Payload: json.RawMessage(`{"value":"example"}`),
}
chunk := SourceChunk{
ID: "source-1:chunk:0",
SourceID: "source-1",
Index: 0,
Units: []source.SourceUnit{
{ID: 1, Kind: "section", Text: "Source text."},
extractOutput := ExtractOutput{
LaneID: "generic-lane",
ExtractorKey: "generic-extractor",
SourceID: "source-1",
ChunkID: "source-1:chunk:0",
ChunkIndex: 0,
Schema: ResponseSchema{ID: "schema-id", Name: "schema-name", Version: "v1"},
Payload: RawPayload{
Content: []byte(`{"value":"example"}`),
MediaType: "application/json",
Metadata: map[string]any{"confidence": 0.75},
},
}
merger := fakeMerger{key: "generic-merger"}
@@ -383,13 +341,8 @@ func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
encoder := fakeOutputEncoder{key: "generic-output"}
merged, err := merger.Merge(context.Background(), MergeRequest{
LaneID: "generic-artifact",
ChunkArtifacts: []ChunkArtifacts{
{
Chunk: chunk,
Candidates: []artifacts.ArtifactCandidate{candidate},
},
},
LaneID: "generic-lane",
ExtractOutputs: []ExtractOutput{extractOutput},
})
if err != nil {
t.Fatalf("Merge() error = %v, want nil", err)
@@ -397,13 +350,13 @@ func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
if merger.Key() != "generic-merger" {
t.Fatalf("Merger.Key() = %q, want generic-merger", merger.Key())
}
if len(merged.Candidates) != 1 {
t.Fatalf("len(merged.Candidates) = %d, want 1", len(merged.Candidates))
if string(merged.Output.Payload.Content) != `{"value":"example"}` {
t.Fatalf("merged content = %s, want raw extract content", merged.Output.Payload.Content)
}
normalized, err := normalizer.Normalize(context.Background(), NormalizeRequest{
LaneID: "generic-artifact",
Candidates: merged.Candidates,
LaneID: "generic-lane",
MergeOutput: merged.Output,
})
if err != nil {
t.Fatalf("Normalize() error = %v, want nil", err)
@@ -411,15 +364,13 @@ func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
if normalizer.Key() != "generic-normalizer" {
t.Fatalf("Normalizer.Key() = %q, want generic-normalizer", normalizer.Key())
}
if len(normalized.Candidates) != 1 {
t.Fatalf("len(normalized.Candidates) = %d, want 1", len(normalized.Candidates))
if string(normalized.Output.Payload.Content) != `{"value":"example"}` {
t.Fatalf("normalized content = %s, want raw merge content", normalized.Output.Payload.Content)
}
encoded, err := encoder.Encode(context.Background(), OutputRequest{
Manifest: artifacts.RunManifest{RunID: "run-1"},
Approved: []artifacts.Artifact{
artifacts.ArtifactFromCandidate(normalized.Candidates[0]),
},
Manifest: artifacts.RunManifest{RunID: "run-1"},
NormalizeOutputs: []NormalizeOutput{normalized.Output},
})
if err != nil {
t.Fatalf("Encode() error = %v, want nil", err)
@@ -433,7 +384,7 @@ func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
if encoded.Files[0].ContentType != "application/json" {
t.Fatalf("ContentType = %q, want application/json", encoded.Files[0].ContentType)
}
if string(encoded.Files[0].Bytes) != `{"run_id":"run-1","approved_count":1}` {
if string(encoded.Files[0].Bytes) != `{"run_id":"run-1","output_count":1}` {
t.Fatalf("Bytes = %s, want encoded output", encoded.Files[0].Bytes)
}
}
@@ -529,57 +480,29 @@ func (chunker *recordingChunker) Chunk(ctx context.Context, req ChunkRequest) (C
}
type fakeExtractor struct {
key string
artifactType string
schemaVersion string
validators []Validator
key string
}
func (extractor fakeExtractor) Key() string {
return extractor.key
}
func (extractor fakeExtractor) ArtifactType() string {
return extractor.artifactType
}
func (extractor fakeExtractor) SchemaVersion() string {
return extractor.schemaVersion
}
func (extractor fakeExtractor) ReferenceSlots() []ReferenceSlot {
return nil
}
func (extractor fakeExtractor) Validators() []Validator {
return extractor.validators
}
func (extractor fakeExtractor) Extract(ctx context.Context, req ExtractionRequest) (ExtractionResult, error) {
units := req.Source.Units
if req.Chunk != nil {
units = req.Chunk.Units
}
payload := json.RawMessage(`{"value":"example"}`)
if req.AmbientContext["mode"] == "chunked" {
payload = json.RawMessage(`{"value":"chunked"}`)
}
return ExtractionResult{
Candidates: []artifacts.ArtifactCandidate{
{
Index: 0,
ExtractorKey: extractor.key,
ArtifactType: extractor.artifactType,
SchemaVersion: extractor.schemaVersion,
Payload: payload,
SourceRefs: []source.SourceRef{
{
SourceID: req.Source.ID,
StartUnitID: units[0].ID,
EndUnitID: units[len(units)-1].ID,
},
},
Output: ExtractOutput{
Schema: ResponseSchema{ID: "schema-id", Name: "schema-name", Version: "v1"},
Payload: RawPayload{
Content: append([]byte(nil), payload...),
MediaType: "application/json",
},
},
}, nil
@@ -594,12 +517,14 @@ func (merger fakeMerger) Key() string {
}
func (merger fakeMerger) Merge(ctx context.Context, req MergeRequest) (MergeResult, error) {
var candidates []artifacts.ArtifactCandidate
for _, chunkArtifacts := range req.ChunkArtifacts {
candidates = append(candidates, chunkArtifacts.Candidates...)
}
return MergeResult{Candidates: candidates}, nil
output := req.ExtractOutputs[0]
return MergeResult{Output: MergeOutput{
LaneID: req.LaneID,
MergerKey: merger.key,
SourceID: output.SourceID,
Schema: output.Schema,
Payload: cloneTestRawPayload(output.Payload),
}}, nil
}
type fakeNormalizer struct {
@@ -615,7 +540,33 @@ func (normalizer fakeNormalizer) ReferenceSlots() []ReferenceSlot {
}
func (normalizer fakeNormalizer) Normalize(ctx context.Context, req NormalizeRequest) (NormalizeResult, error) {
return NormalizeResult{Candidates: req.Candidates}, nil
return NormalizeResult{Output: NormalizeOutput{
LaneID: req.LaneID,
NormalizerKey: normalizer.key,
SourceID: req.MergeOutput.SourceID,
Schema: req.MergeOutput.Schema,
Payload: cloneTestRawPayload(req.MergeOutput.Payload),
}}, nil
}
func cloneTestRawPayload(payload RawPayload) RawPayload {
return RawPayload{
Content: append([]byte(nil), payload.Content...),
MediaType: payload.MediaType,
Metadata: cloneTestMetadata(payload.Metadata),
Warnings: append([]Warning(nil), payload.Warnings...),
}
}
func cloneTestMetadata(metadata map[string]any) map[string]any {
if len(metadata) == 0 {
return nil
}
out := make(map[string]any, len(metadata))
for key, value := range metadata {
out[key] = value
}
return out
}
type fakeValidator struct {
@@ -665,7 +616,7 @@ func (encoder fakeOutputEncoder) Encode(ctx context.Context, req OutputRequest)
{
Name: "artifacts/generic.json",
ContentType: "application/json",
Bytes: []byte(`{"run_id":"` + req.Manifest.RunID + `","approved_count":1}`),
Bytes: []byte(`{"run_id":"` + req.Manifest.RunID + `","output_count":1}`),
},
},
}, nil

6
internal/framework/pipeline/default_modules_test.go
View File

@@ -113,14 +113,8 @@ type defaultExtractor struct{}
func (defaultExtractor) Key() string { return "extract" }
func (defaultExtractor) ArtifactType() string { return "record" }
func (defaultExtractor) SchemaVersion() string { return "v1" }
func (defaultExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (defaultExtractor) Validators() []contracts.Validator { return nil }
func (defaultExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
return contracts.ExtractionResult{}, nil
}

12
internal/framework/pipeline/extractor_registry_test.go
View File

@@ -367,22 +367,10 @@ func (extractor registryFakeExtractor) Key() string {
return extractor.key
}
func (extractor registryFakeExtractor) ArtifactType() string {
return "generic-artifact"
}
func (extractor registryFakeExtractor) SchemaVersion() string {
return "v1"
}
func (extractor registryFakeExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (extractor registryFakeExtractor) Validators() []contracts.Validator {
return nil
}
func (extractor registryFakeExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
return contracts.ExtractionResult{}, nil
}

114
internal/framework/pipeline/registry_integration_test.go
View File

@@ -5,11 +5,8 @@ import (
"reflect"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
validate "gitea.maximumdirect.net/eric/notarius/internal/framework/validate"
)
func TestRunnerUsesRegistries(t *testing.T) {
@@ -33,11 +30,11 @@ func TestRunnerUsesRegistries(t *testing.T) {
if !reflect.DeepEqual(executed, []string{"extract-first:chunk-0", "extract-second:chunk-0"}) {
t.Fatalf("executed = %#v, want extractor chunk execution", executed)
}
if got := artifactKeys(output.Approved); !reflect.DeepEqual(got, []string{"extract-first"}) {
t.Fatalf("approved keys = %#v, want [extract-first]", got)
if got := normalizeOutputKeys(output.NormalizeOutputs); !reflect.DeepEqual(got, []string{"normalize", "normalize"}) {
t.Fatalf("normalize output keys = %#v, want one output from each lane", got)
}
if got := rejectedKeys(output.Rejected); !reflect.DeepEqual(got, []string{"extract-second"}) {
t.Fatalf("rejected keys = %#v, want [extract-second]", got)
if len(output.Rejected) != 0 {
t.Fatalf("len(Rejected) = %d, want none", len(output.Rejected))
}
}
@@ -64,12 +61,8 @@ func integrationRegistries(t *testing.T, built, executed *[]string) Registries {
}); err != nil {
t.Fatalf("register chunker: %v", err)
}
registerIntegrationExtractor(t, registries.Extractors, "extract-first", built, executed, []contracts.Validator{
integrationValidator{name: "approve-first", approve: true},
})
registerIntegrationExtractor(t, registries.Extractors, "extract-second", built, executed, []contracts.Validator{
integrationValidator{name: "reject-second", approve: false},
})
registerIntegrationExtractor(t, registries.Extractors, "extract-first", built, executed)
registerIntegrationExtractor(t, registries.Extractors, "extract-second", built, executed)
if err := registries.Mergers.Register("merge", func() (contracts.Merger, error) {
*built = append(*built, "merge")
return integrationMerger{}, nil
@@ -91,12 +84,12 @@ func integrationRegistries(t *testing.T, built, executed *[]string) Registries {
return registries
}
func registerIntegrationExtractor(t *testing.T, registry *ExtractorRegistry, key string, built, executed *[]string, validators []contracts.Validator) {
func registerIntegrationExtractor(t *testing.T, registry *ExtractorRegistry, key string, built, executed *[]string) {
t.Helper()
if err := registry.Register(key, func() (contracts.Extractor, error) {
*built = append(*built, key)
return integrationExtractor{key: key, executed: executed, validators: validators}, nil
return integrationExtractor{key: key, executed: executed}, nil
}); err != nil {
t.Fatalf("Register(%q) error = %v, want nil", key, err)
}
@@ -140,36 +133,27 @@ func (chunker integrationChunker) Chunk(ctx context.Context, req contracts.Chunk
}
type integrationExtractor struct {
key string
executed *[]string
validators []contracts.Validator
key string
executed *[]string
}
func (extractor integrationExtractor) Key() string {
return extractor.key
}
func (extractor integrationExtractor) ArtifactType() string {
return "generic-artifact"
}
func (extractor integrationExtractor) SchemaVersion() string {
return "v1"
}
func (extractor integrationExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (extractor integrationExtractor) Validators() []contracts.Validator {
return extractor.validators
}
func (extractor integrationExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
*extractor.executed = append(*extractor.executed, extractor.key+":"+req.Chunk.ID)
return contracts.ExtractionResult{
Candidates: []artifacts.ArtifactCandidate{
{Payload: []byte(`{"value":true}`)},
Output: contracts.ExtractOutput{
Schema: contracts.ResponseSchema{ID: "integration", Name: "integration", Version: "v1"},
Payload: contracts.RawPayload{
Content: []byte(`{"value":true}`),
MediaType: "application/json",
},
},
}, nil
}
@@ -183,11 +167,20 @@ func (merger integrationMerger) Key() string {
}
func (merger integrationMerger) Merge(ctx context.Context, req contracts.MergeRequest) (contracts.MergeResult, error) {
var candidates []artifacts.ArtifactCandidate
for _, chunkArtifacts := range req.ChunkArtifacts {
candidates = append(candidates, chunkArtifacts.Candidates...)
output := contracts.MergeOutput{
LaneID: req.LaneID,
Schema: contracts.ResponseSchema{ID: "integration", Name: "integration", Version: "v1"},
Payload: contracts.RawPayload{
Content: []byte(`{"merged":true}`),
MediaType: "application/json",
},
}
return contracts.MergeResult{Candidates: candidates}, nil
if len(req.ExtractOutputs) > 0 {
output.SourceID = req.ExtractOutputs[0].SourceID
output.Schema = req.ExtractOutputs[0].Schema
output.Payload = req.ExtractOutputs[0].Payload
}
return contracts.MergeResult{Output: output}, nil
}
func (normalizer integrationNormalizer) Key() string {
@@ -199,7 +192,14 @@ func (normalizer integrationNormalizer) ReferenceSlots() []contracts.ReferenceSl
}
func (normalizer integrationNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
return contracts.NormalizeResult{Candidates: req.Candidates}, nil
return contracts.NormalizeResult{
Output: contracts.NormalizeOutput{
LaneID: req.LaneID,
SourceID: req.MergeOutput.SourceID,
Schema: req.MergeOutput.Schema,
Payload: req.MergeOutput.Payload,
},
}, nil
}
type integrationOutput struct{}
@@ -216,30 +216,6 @@ func (output integrationOutput) Encode(ctx context.Context, req contracts.Output
}, nil
}
type integrationValidator struct {
name string
approve bool
}
func (validator integrationValidator) Name() string {
return validator.name
}
func (validator integrationValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
decisions := make([]contracts.ValidationDecision, 0, len(req.Candidates))
for _, candidate := range req.Candidates {
if validator.approve {
decisions = append(decisions, validate.Approved(candidate.Index))
} else {
decisions = append(decisions, validate.Rejected(candidate.Index, "invalid", "not accepted"))
}
}
return contracts.ValidationResult{
ValidatorName: validator.name,
Decisions: decisions,
}, nil
}
func integrationPipeline() ResolvedPipeline {
return ResolvedPipeline{
ID: "pipeline-1",
@@ -276,18 +252,10 @@ func integrationSourceDocument() *source.SourceDocument {
}
}
func artifactKeys(approved []artifacts.Artifact) []string {
keys := make([]string, 0, len(approved))
for _, artifact := range approved {
keys = append(keys, artifact.ExtractorKey)
}
return keys
}
func rejectedKeys(rejected []artifacts.RejectedArtifact) []string {
keys := make([]string, 0, len(rejected))
for _, artifact := range rejected {
keys = append(keys, artifact.Candidate.ExtractorKey)
func normalizeOutputKeys(outputs []contracts.NormalizeOutput) []string {
keys := make([]string, 0, len(outputs))
for _, output := range outputs {
keys = append(keys, output.NormalizerKey)
}
return keys
}

251
internal/framework/pipeline/runner.go
View File

@@ -15,7 +15,6 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/validate"
)
type Registries struct {
@@ -51,11 +50,11 @@ type RunInput struct {
}
type RunOutput struct {
Manifest artifacts.RunManifest `json:"manifest"`
Approved []artifacts.Artifact `json:"approved,omitempty"`
Rejected []artifacts.RejectedArtifact `json:"rejected,omitempty"`
Warnings []contracts.Warning `json:"warnings,omitempty"`
OutputFiles []contracts.OutputFile `json:"-"`
Manifest artifacts.RunManifest `json:"manifest"`
NormalizeOutputs []contracts.NormalizeOutput `json:"normalize_outputs,omitempty"`
Rejected []contracts.RejectedOutput `json:"rejected,omitempty"`
Warnings []contracts.Warning `json:"warnings,omitempty"`
OutputFiles []contracts.OutputFile `json:"-"`
}
func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err error) {
@@ -126,9 +125,8 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
return failOutput(output), fmt.Errorf("validate chunks from chunker %q: %w", chunker.Key(), err)
}
nextCandidateIndex := 0
for _, lane := range input.Pipeline.ArtifactLanes {
if err := r.runLane(ctx, input, doc, sourceInput, sessionID, canonicalChunks, lane, &output, &nextCandidateIndex); err != nil {
if err := r.runLane(ctx, input, doc, sourceInput, sessionID, canonicalChunks, lane, &output); err != nil {
return failOutput(output), err
}
}
@@ -146,13 +144,13 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
}
attachModuleManifestMetadata(&output, "output", encoder)
encoded, err := encoder.Encode(ctx, contracts.OutputRequest{
Manifest: output.Manifest,
Approved: output.Approved,
Rejected: output.Rejected,
Warnings: output.Warnings,
LLMProfile: input.Pipeline.Output.LLMProfile,
Options: cloneOptions(input.Pipeline.Output.Options),
Metadata: input.Metadata,
Manifest: output.Manifest,
NormalizeOutputs: cloneNormalizeOutputs(output.NormalizeOutputs),
Rejected: cloneRejectedOutputs(output.Rejected),
Warnings: output.Warnings,
LLMProfile: input.Pipeline.Output.LLMProfile,
Options: cloneOptions(input.Pipeline.Output.Options),
Metadata: input.Metadata,
})
output.Warnings = append(output.Warnings, encoded.Warnings...)
if err != nil {
@@ -167,7 +165,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
return output, nil
}
func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []contracts.SourceChunk, lane ResolvedArtifactLane, output *RunOutput, nextCandidateIndex *int) error {
func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []contracts.SourceChunk, lane ResolvedArtifactLane, output *RunOutput) error {
extractor, err := r.registries.Extractors.Build(lane.Extract.Module)
if err != nil {
return fmt.Errorf("build extractor %q for lane %q: %w", lane.Extract.Module, lane.ID, err)
@@ -182,19 +180,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
}
setLaneManifestMetadata(output, lane.ID, extractor, merger, normalizer)
var validators []validatorExecution
if len(lane.Validators) > 0 {
validators, err = r.buildConfiguredValidators(lane)
if err != nil {
return err
}
} else {
for _, validator := range extractor.Validators() {
validators = append(validators, validatorExecution{validator: validator})
}
}
chunkArtifacts := make([]contracts.ChunkArtifacts, 0, len(chunks))
extractOutputs := make([]contracts.ExtractOutput, 0, len(chunks))
for index := range chunks {
chunk := chunks[index]
result, err := extractor.Extract(ctx, contracts.ExtractionRequest{
@@ -212,21 +198,20 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
if err != nil {
return fmt.Errorf("extract lane %q chunk %q with extractor %q: %w", lane.ID, chunk.ID, extractor.Key(), err)
}
candidates, err := normalizeCandidates(extractor, result.Candidates, nextCandidateIndex)
if err != nil {
return err
}
chunkArtifacts = append(chunkArtifacts, contracts.ChunkArtifacts{
Chunk: chunk,
Candidates: candidates,
})
extractOutput := result.Output
extractOutput.LaneID = lane.ID
extractOutput.ExtractorKey = extractor.Key()
extractOutput.SourceID = doc.ID
extractOutput.ChunkID = chunk.ID
extractOutput.ChunkIndex = chunk.Index
extractOutput.Payload.Warnings = append(extractOutput.Payload.Warnings, result.Warnings...)
extractOutputs = append(extractOutputs, cloneExtractOutput(extractOutput))
}
mergeResult, err := merger.Merge(ctx, contracts.MergeRequest{
Source: doc,
LaneID: lane.ID,
ChunkArtifacts: chunkArtifacts,
ExtractOutputs: cloneExtractOutputs(extractOutputs),
LLMProfile: lane.Merge.LLMProfile,
Options: cloneOptions(lane.Merge.Options),
Metadata: input.Metadata,
@@ -239,7 +224,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
normalizeResult, err := normalizer.Normalize(ctx, contracts.NormalizeRequest{
Source: doc,
LaneID: lane.ID,
Candidates: mergeResult.Candidates,
MergeOutput: cloneMergeOutput(mergeResult.Output),
SourceInput: sourceInput.Clone(),
SessionID: sessionID,
References: CloneReferenceSet(lane.NormalizeReferences.ReferenceSet),
@@ -252,21 +237,12 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
if err != nil {
return fmt.Errorf("normalize lane %q with normalizer %q: %w", lane.ID, normalizer.Key(), err)
}
if err := validateCandidateEnvelope(extractor, normalizeResult.Candidates); err != nil {
return fmt.Errorf("validate normalized candidates for lane %q: %w", lane.ID, err)
}
approved, rejected, warnings, err := runValidators(ctx, extractor.Key(), validators, doc, normalizeResult.Candidates, input.Metadata)
output.Warnings = append(output.Warnings, warnings...)
output.Rejected = append(output.Rejected, rejected...)
if err != nil {
return err
}
for _, candidate := range approved {
output.Approved = append(output.Approved, artifacts.ArtifactFromCandidate(candidate))
}
normalizeOutput := normalizeResult.Output
normalizeOutput.LaneID = lane.ID
normalizeOutput.NormalizerKey = normalizer.Key()
normalizeOutput.SourceID = doc.ID
normalizeOutput.Payload.Warnings = append(normalizeOutput.Payload.Warnings, normalizeResult.Warnings...)
output.NormalizeOutputs = append(output.NormalizeOutputs, cloneNormalizeOutput(normalizeOutput))
return nil
}
@@ -628,6 +604,59 @@ func cloneWarnings(warnings []contracts.Warning) []contracts.Warning {
return append([]contracts.Warning(nil), warnings...)
}
func cloneRawPayload(payload contracts.RawPayload) contracts.RawPayload {
return contracts.RawPayload{
Content: append([]byte(nil), payload.Content...),
MediaType: payload.MediaType,
Metadata: cloneMetadata(payload.Metadata),
Warnings: cloneWarnings(payload.Warnings),
}
}
func cloneExtractOutput(output contracts.ExtractOutput) contracts.ExtractOutput {
output.Payload = cloneRawPayload(output.Payload)
return output
}
func cloneExtractOutputs(outputs []contracts.ExtractOutput) []contracts.ExtractOutput {
if len(outputs) == 0 {
return nil
}
out := make([]contracts.ExtractOutput, 0, len(outputs))
for _, output := range outputs {
out = append(out, cloneExtractOutput(output))
}
return out
}
func cloneMergeOutput(output contracts.MergeOutput) contracts.MergeOutput {
output.Payload = cloneRawPayload(output.Payload)
return output
}
func cloneNormalizeOutput(output contracts.NormalizeOutput) contracts.NormalizeOutput {
output.Payload = cloneRawPayload(output.Payload)
return output
}
func cloneNormalizeOutputs(outputs []contracts.NormalizeOutput) []contracts.NormalizeOutput {
if len(outputs) == 0 {
return nil
}
out := make([]contracts.NormalizeOutput, 0, len(outputs))
for _, output := range outputs {
out = append(out, cloneNormalizeOutput(output))
}
return out
}
func cloneRejectedOutputs(rejected []contracts.RejectedOutput) []contracts.RejectedOutput {
if len(rejected) == 0 {
return nil
}
return append([]contracts.RejectedOutput(nil), rejected...)
}
func timePtr(t time.Time) *time.Time {
return &t
}
@@ -640,115 +669,3 @@ func pipelineUsesConfiguredValidators(pipeline ResolvedPipeline) bool {
}
return false
}
func normalizeCandidates(extractor contracts.Extractor, candidates []artifacts.ArtifactCandidate, nextIndex *int) ([]artifacts.ArtifactCandidate, error) {
normalized := make([]artifacts.ArtifactCandidate, 0, len(candidates))
for _, candidate := range candidates {
candidate.Index = *nextIndex
*nextIndex = *nextIndex + 1
if candidate.ExtractorKey == "" {
candidate.ExtractorKey = extractor.Key()
} else if candidate.ExtractorKey != extractor.Key() {
return nil, fmt.Errorf("candidate extractor_key %q does not match extractor %q", candidate.ExtractorKey, extractor.Key())
}
if candidate.ArtifactType == "" {
candidate.ArtifactType = extractor.ArtifactType()
} else if candidate.ArtifactType != extractor.ArtifactType() {
return nil, fmt.Errorf("candidate artifact_type %q does not match extractor %q artifact type %q", candidate.ArtifactType, extractor.Key(), extractor.ArtifactType())
}
if candidate.SchemaVersion == "" {
candidate.SchemaVersion = extractor.SchemaVersion()
} else if candidate.SchemaVersion != extractor.SchemaVersion() {
return nil, fmt.Errorf("candidate schema_version %q does not match extractor %q schema version %q", candidate.SchemaVersion, extractor.Key(), extractor.SchemaVersion())
}
normalized = append(normalized, candidate)
}
return normalized, nil
}
func validateCandidateEnvelope(extractor contracts.Extractor, candidates []artifacts.ArtifactCandidate) error {
seen := make(map[int]struct{}, len(candidates))
for _, candidate := range candidates {
if _, ok := seen[candidate.Index]; ok {
return fmt.Errorf("candidate index %d is duplicated", candidate.Index)
}
seen[candidate.Index] = struct{}{}
if candidate.ExtractorKey == "" {
return fmt.Errorf("candidate index %d extractor_key must not be empty", candidate.Index)
}
if candidate.ExtractorKey != extractor.Key() {
return fmt.Errorf("candidate index %d extractor_key %q does not match extractor %q", candidate.Index, candidate.ExtractorKey, extractor.Key())
}
if candidate.ArtifactType == "" {
return fmt.Errorf("candidate index %d artifact_type must not be empty", candidate.Index)
}
if candidate.ArtifactType != extractor.ArtifactType() {
return fmt.Errorf("candidate index %d artifact_type %q does not match extractor %q artifact type %q", candidate.Index, candidate.ArtifactType, extractor.Key(), extractor.ArtifactType())
}
if candidate.SchemaVersion == "" {
return fmt.Errorf("candidate index %d schema_version must not be empty", candidate.Index)
}
if candidate.SchemaVersion != extractor.SchemaVersion() {
return fmt.Errorf("candidate index %d schema_version %q does not match extractor %q schema version %q", candidate.Index, candidate.SchemaVersion, extractor.Key(), extractor.SchemaVersion())
}
}
return nil
}
func runValidators(ctx context.Context, extractorKey string, validators []validatorExecution, doc *source.SourceDocument, candidates []artifacts.ArtifactCandidate, metadata map[string]any) ([]artifacts.ArtifactCandidate, []artifacts.RejectedArtifact, []contracts.Warning, error) {
eligible := candidates
var rejected []artifacts.RejectedArtifact
var warnings []contracts.Warning
for validatorIndex, execution := range validators {
validator := execution.validator
if validator == nil {
return nil, rejected, warnings, fmt.Errorf("extractor %q validator[%d] must not be nil", extractorKey, validatorIndex)
}
result, err := validator.Validate(ctx, contracts.ValidationRequest{
Source: doc,
Candidates: eligible,
LLMProfile: execution.binding.LLMProfile,
Options: cloneOptions(execution.binding.Options),
Metadata: metadata,
})
warnings = append(warnings, result.Warnings...)
if err != nil {
return nil, rejected, warnings, fmt.Errorf("validate extractor %q with validator %q: %w", extractorKey, validator.Name(), err)
}
if result.ValidatorName != validator.Name() {
return nil, rejected, warnings, fmt.Errorf("validator %q returned result for %q", validator.Name(), result.ValidatorName)
}
if err := validate.EnforceDecisionCardinality(eligible, result.Decisions); err != nil {
return nil, rejected, warnings, fmt.Errorf("validate extractor %q with validator %q: %w", extractorKey, validator.Name(), err)
}
decisions := make(map[int]contracts.ValidationDecision, len(result.Decisions))
for _, decision := range result.Decisions {
decisions[decision.CandidateIndex] = decision
}
nextEligible := make([]artifacts.ArtifactCandidate, 0, len(eligible))
for _, candidate := range eligible {
decision := decisions[candidate.Index]
if decision.Approved {
nextEligible = append(nextEligible, candidate)
continue
}
rejected = append(rejected, artifacts.RejectedArtifact{
Candidate: candidate,
ValidatorName: result.ValidatorName,
ReasonCode: decision.ReasonCode,
Message: decision.Message,
})
}
eligible = nextEligible
}
return eligible, rejected, warnings, nil
}

391
internal/framework/pipeline/runner_test.go
View File

@@ -11,7 +11,6 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
validate "gitea.maximumdirect.net/eric/notarius/internal/framework/validate"
)
func TestNewAndDataTypes(t *testing.T) {
@@ -29,17 +28,17 @@ func TestNewAndDataTypes(t *testing.T) {
Metadata: map[string]any{"request": "test"},
}
output := RunOutput{
Manifest: artifacts.RunManifest{PipelineID: "pipeline-1"},
Approved: []artifacts.Artifact{{ExtractorKey: "extract-alpha"}},
Rejected: []artifacts.RejectedArtifact{{ValidatorName: "validator"}},
Warnings: []contracts.Warning{{ReasonCode: "note", Message: "message"}},
OutputFiles: []contracts.OutputFile{{Name: "artifacts/generic.json", ContentType: "application/json", Bytes: []byte(`{}`)}},
Manifest: artifacts.RunManifest{PipelineID: "pipeline-1"},
NormalizeOutputs: []contracts.NormalizeOutput{{NormalizerKey: "normalize"}},
Rejected: []contracts.RejectedOutput{{ValidatorName: "validator"}},
Warnings: []contracts.Warning{{ReasonCode: "note", Message: "message"}},
OutputFiles: []contracts.OutputFile{{Name: "outputs/generic.json", ContentType: "application/json", Bytes: []byte(`{}`)}},
}
if input.Pipeline.ID != "pipeline-1" || input.SourceID != "source-1" {
t.Fatalf("RunInput = %#v, want constructed fields", input)
}
if output.Manifest.PipelineID != "pipeline-1" || len(output.Approved) != 1 || len(output.Rejected) != 1 || len(output.Warnings) != 1 || len(output.OutputFiles) != 1 {
if output.Manifest.PipelineID != "pipeline-1" || len(output.NormalizeOutputs) != 1 || len(output.Rejected) != 1 || len(output.Warnings) != 1 || len(output.OutputFiles) != 1 {
t.Fatalf("RunOutput = %#v, want constructed fields", output)
}
}
@@ -361,8 +360,8 @@ func TestRunAllowsPartialCoverageAndOverlappingChunks(t *testing.T) {
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.Approved) != 2 {
t.Fatalf("len(Approved) = %d, want one candidate per accepted chunk", len(output.Approved))
if len(output.NormalizeOutputs) != 1 {
t.Fatalf("len(NormalizeOutputs) = %d, want one lane output", len(output.NormalizeOutputs))
}
}
@@ -397,8 +396,8 @@ func TestRunCanonicalizesChunkUnitsBeforeExtraction(t *testing.T) {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.Approved) != 1 {
t.Fatalf("len(Approved) = %d, want 1", len(output.Approved))
if len(output.NormalizeOutputs) != 1 {
t.Fatalf("len(NormalizeOutputs) = %d, want 1", len(output.NormalizeOutputs))
}
extractor := modules.extractors["extract-alpha"]
@@ -504,8 +503,8 @@ func TestRunExecutesChunksAndPassesChunkAndLLMClient(t *testing.T) {
if extractor.seenMetadata[0]["request"] != "test" {
t.Fatalf("seen metadata = %#v, want request metadata", extractor.seenMetadata)
}
if len(output.Approved) != 2 {
t.Fatalf("len(Approved) = %d, want 2", len(output.Approved))
if len(output.NormalizeOutputs) != 1 {
t.Fatalf("len(NormalizeOutputs) = %d, want one lane output", len(output.NormalizeOutputs))
}
}
@@ -660,12 +659,6 @@ func TestRunPassesModuleBindingConfigToStageRequests(t *testing.T) {
if got := modules.normalizers["normalize"].requests[0].Options["normalize_option"]; got != "normalize-value" {
t.Fatalf("normalize Options = %#v, want normalize option", modules.normalizers["normalize"].requests[0].Options)
}
if got := modules.validators["configured"].requests[0].LLMProfile; got != "validator-profile" {
t.Fatalf("validator LLMProfile = %q, want validator-profile", got)
}
if got := modules.validators["configured"].requests[0].Options["validator_option"]; got != "validator-value" {
t.Fatalf("validator Options = %#v, want validator option", modules.validators["configured"].requests[0].Options)
}
if got := modules.output.requests[0].LLMProfile; got != "output-profile" {
t.Fatalf("output LLMProfile = %q, want output-profile", got)
}
@@ -801,7 +794,7 @@ func TestRunRecordsTopLevelModuleMetadataForSingletonModules(t *testing.T) {
}
}
func TestRunPassesPerChunkCandidatesToMergeAndNormalize(t *testing.T) {
func TestRunPassesPerChunkRawOutputsToMergeAndNormalize(t *testing.T) {
modules := defaultRunnerModules()
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
@@ -813,111 +806,31 @@ func TestRunPassesPerChunkCandidatesToMergeAndNormalize(t *testing.T) {
if len(merger.requests) != 1 {
t.Fatalf("len(merge requests) = %d, want 1", len(merger.requests))
}
chunkArtifacts := merger.requests[0].ChunkArtifacts
if len(chunkArtifacts) != 2 {
t.Fatalf("len(ChunkArtifacts) = %d, want 2", len(chunkArtifacts))
extractOutputs := merger.requests[0].ExtractOutputs
if len(extractOutputs) != 2 {
t.Fatalf("len(ExtractOutputs) = %d, want 2", len(extractOutputs))
}
if chunkArtifacts[0].Chunk.ID != "chunk-0" || chunkArtifacts[1].Chunk.ID != "chunk-1" {
t.Fatalf("merge chunks = %#v, want chunk order", chunkArtifacts)
if extractOutputs[0].ChunkID != "chunk-0" || extractOutputs[1].ChunkID != "chunk-1" {
t.Fatalf("merge chunks = %#v, want chunk order", extractOutputs)
}
if got := candidateIndices(chunkArtifacts[0].Candidates); !reflect.DeepEqual(got, []int{0}) {
t.Fatalf("first chunk candidate indices = %#v, want [0]", got)
if extractOutputs[0].ChunkIndex != 0 || string(extractOutputs[0].Payload.Content) != `{"chunk":"chunk-0"}` {
t.Fatalf("first extract output = %#v, want first chunk payload", extractOutputs[0])
}
if got := candidateIndices(chunkArtifacts[1].Candidates); !reflect.DeepEqual(got, []int{1}) {
t.Fatalf("second chunk candidate indices = %#v, want [1]", got)
if extractOutputs[1].ChunkIndex != 1 || string(extractOutputs[1].Payload.Content) != `{"chunk":"chunk-1"}` {
t.Fatalf("second extract output = %#v, want second chunk payload", extractOutputs[1])
}
normalizer := modules.normalizers["normalize"]
if len(normalizer.requests) != 1 {
t.Fatalf("len(normalize requests) = %d, want 1", len(normalizer.requests))
}
if got := candidateIndices(normalizer.requests[0].Candidates); !reflect.DeepEqual(got, []int{0, 1}) {
t.Fatalf("normalize candidate indices = %#v, want merged candidates", got)
if string(normalizer.requests[0].MergeOutput.Payload.Content) != `{"merged":true}` {
t.Fatalf("normalize merge output = %#v, want merged raw output", normalizer.requests[0].MergeOutput)
}
}
func TestRunRejectsInvalidPostNormalizeCandidateEnvelope(t *testing.T) {
tests := []struct {
name string
candidates []artifacts.ArtifactCandidate
want string
}{
{
name: "duplicate index",
candidates: []artifacts.ArtifactCandidate{
runnerCandidate(0),
runnerCandidate(0),
},
want: "duplicated",
},
{
name: "missing extractor key",
candidates: []artifacts.ArtifactCandidate{
{Index: 0, ArtifactType: "artifact", SchemaVersion: "v1", Payload: []byte(`{"value":true}`)},
},
want: "extractor_key",
},
{
name: "mismatched schema version",
candidates: []artifacts.ArtifactCandidate{
{Index: 0, ExtractorKey: "extract-alpha", ArtifactType: "artifact", SchemaVersion: "other", Payload: []byte(`{"value":true}`)},
},
want: "schema_version",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
modules := defaultRunnerModules()
modules.normalizers["normalize"].result = test.candidates
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
assertRunError(t, err, test.want)
if output.Manifest.ValidationStatus != "failed" {
t.Fatalf("ValidationStatus = %q, want failed", output.Manifest.ValidationStatus)
}
if len(output.Approved) != 0 {
t.Fatalf("len(Approved) = %d, want no approved artifacts", len(output.Approved))
}
})
}
}
func TestRunValidatorApprovalAndRejection(t *testing.T) {
func TestRunRecordsConfiguredValidatorsInManifest(t *testing.T) {
modules := defaultRunnerModules()
rejectFirst := &runnerValidator{
name: "default-validator",
decisions: func(candidates []artifacts.ArtifactCandidate) []contracts.ValidationDecision {
return []contracts.ValidationDecision{
validate.Rejected(candidates[0].Index, "invalid", "not accepted"),
validate.Approved(candidates[1].Index),
}
},
}
modules.extractors["extract-alpha"].validators = []contracts.Validator{rejectFirst}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if output.Manifest.ValidationStatus != "rejected" {
t.Fatalf("ValidationStatus = %q, want rejected", output.Manifest.ValidationStatus)
}
if len(output.Approved) != 1 || len(output.Rejected) != 1 {
t.Fatalf("approved/rejected = %d/%d, want 1/1", len(output.Approved), len(output.Rejected))
}
if output.Rejected[0].ValidatorName != "default-validator" || output.Rejected[0].ReasonCode != "invalid" {
t.Fatalf("Rejected[0] = %#v, want rejection details", output.Rejected[0])
}
}
func TestRunUsesConfiguredValidatorsInLaneOrder(t *testing.T) {
modules := defaultRunnerModules()
var order []string
modules.validators["configured"] = &runnerValidator{name: "configured", decisions: approveAll, order: &order}
modules.validators["second-validator"] = &runnerValidator{name: "second-validator", decisions: approveAll, order: &order}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{
Pipeline: resolvedPipelineWithValidators("configured", "second-validator"),
@@ -926,93 +839,17 @@ func TestRunUsesConfiguredValidatorsInLaneOrder(t *testing.T) {
t.Fatalf("Run() error = %v, want nil", err)
}
if !reflect.DeepEqual(order, []string{"configured", "second-validator"}) {
t.Fatalf("validator order = %#v, want configured order", order)
}
if got := output.Manifest.ArtifactLanes[0].Validators; !reflect.DeepEqual(got, []string{"configured", "second-validator"}) {
t.Fatalf("manifest validators = %#v, want configured validators", got)
}
}
func TestRunUsesDefaultValidatorsWhenLaneDoesNotConfigureValidators(t *testing.T) {
modules := defaultRunnerModules()
defaultValidator := &runnerValidator{name: "default-validator", decisions: approveAll}
configuredValidator := &runnerValidator{name: "configured", decisions: approveAll}
modules.extractors["extract-alpha"].validators = []contracts.Validator{defaultValidator}
modules.validators["configured"] = configuredValidator
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if defaultValidator.calls != 1 {
t.Fatalf("default validator calls = %d, want 1", defaultValidator.calls)
}
if configuredValidator.calls != 0 {
t.Fatalf("configured validator calls = %d, want 0", configuredValidator.calls)
}
}
func TestRunConfiguredValidatorsReplaceExtractorDefaults(t *testing.T) {
modules := defaultRunnerModules()
defaultValidator := &runnerValidator{name: "default-validator", decisions: approveAll}
configuredValidator := &runnerValidator{name: "configured", decisions: approveAll}
modules.extractors["extract-alpha"].validators = []contracts.Validator{defaultValidator}
modules.validators["configured"] = configuredValidator
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipelineWithValidators("configured")})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if defaultValidator.calls != 0 {
t.Fatalf("default validator calls = %d, want 0", defaultValidator.calls)
}
if configuredValidator.calls != 1 {
t.Fatalf("configured validator calls = %d, want 1", configuredValidator.calls)
}
}
func TestRunAssignsGlobalCandidateIndicesAcrossLanesAndChunks(t *testing.T) {
modules := defaultRunnerModules()
var seenIndices []int
recordIndices := func(candidates []artifacts.ArtifactCandidate) []contracts.ValidationDecision {
seenIndices = append(seenIndices, candidateIndices(candidates)...)
return approveAll(candidates)
}
modules.extractors["extract-alpha"].validators = []contracts.Validator{&runnerValidator{name: "alpha-validator", decisions: recordIndices}}
modules.extractors["extract-beta"] = &runnerExtractor{key: "extract-beta", artifactType: "artifact", schemaVersion: "v1", validators: []contracts.Validator{&runnerValidator{name: "beta-validator", decisions: recordIndices}}}
pipeline := resolvedPipeline()
pipeline.ArtifactLanes = append(pipeline.ArtifactLanes, ResolvedArtifactLane{
ID: "beta",
Extract: Binding("extract-beta"),
Merge: Binding("merge"),
Normalize: Binding("normalize"),
})
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if !reflect.DeepEqual(seenIndices, []int{0, 1, 2, 3}) {
t.Fatalf("seen indices = %#v, want global indices", seenIndices)
}
if len(output.Approved) != 4 {
t.Fatalf("len(Approved) = %d, want 4", len(output.Approved))
}
}
func TestRunCollectsStageWarnings(t *testing.T) {
modules := defaultRunnerModules()
modules.chunker.warnings = []contracts.Warning{{ReasonCode: "chunk-warning", Message: "chunk warning"}}
modules.extractors["extract-alpha"].warnings = []contracts.Warning{{ReasonCode: "extract-warning", Message: "extract warning"}}
modules.mergers["merge"].warnings = []contracts.Warning{{ReasonCode: "merge-warning", Message: "merge warning"}}
modules.normalizers["normalize"].warnings = []contracts.Warning{{ReasonCode: "normalize-warning", Message: "normalize warning"}}
modules.extractors["extract-alpha"].validators = []contracts.Validator{&runnerValidator{
name: "default-validator",
decisions: approveAll,
warnings: []contracts.Warning{{ReasonCode: "validator-warning", Message: "validator warning"}},
}}
modules.output.warnings = []contracts.Warning{{ReasonCode: "output-warning", Message: "output warning"}}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
@@ -1020,13 +857,13 @@ func TestRunCollectsStageWarnings(t *testing.T) {
t.Fatalf("Run() error = %v, want nil", err)
}
want := []string{"chunk-warning", "extract-warning", "extract-warning", "merge-warning", "normalize-warning", "validator-warning", "output-warning"}
want := []string{"chunk-warning", "extract-warning", "extract-warning", "merge-warning", "normalize-warning", "output-warning"}
if got := warningReasons(output.Warnings); !reflect.DeepEqual(got, want) {
t.Fatalf("warning reasons = %#v, want %#v", got, want)
}
}
func TestRunOutputEncoderReceivesManifestAndArtifacts(t *testing.T) {
func TestRunOutputEncoderReceivesManifestAndRawOutputs(t *testing.T) {
modules := defaultRunnerModules()
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
@@ -1038,8 +875,8 @@ func TestRunOutputEncoderReceivesManifestAndArtifacts(t *testing.T) {
t.Fatalf("len(OutputFiles) = %d, want 1", len(output.OutputFiles))
}
file := output.OutputFiles[0]
if file.Name != "artifacts/generic.json" {
t.Fatalf("OutputFiles[0].Name = %q, want artifacts/generic.json", file.Name)
if file.Name != "outputs/generic.json" {
t.Fatalf("OutputFiles[0].Name = %q, want outputs/generic.json", file.Name)
}
if file.ContentType != "application/json" {
t.Fatalf("ContentType = %q, want application/json", file.ContentType)
@@ -1054,8 +891,11 @@ func TestRunOutputEncoderReceivesManifestAndArtifacts(t *testing.T) {
if req.Manifest.PipelineID != "pipeline-1" || req.Manifest.PipelineDigest != "sha256:pipeline" {
t.Fatalf("output manifest = %#v, want pipeline details", req.Manifest)
}
if len(req.Approved) != 2 {
t.Fatalf("len(output approved) = %d, want 2", len(req.Approved))
if len(req.NormalizeOutputs) != 1 {
t.Fatalf("len(output NormalizeOutputs) = %d, want 1", len(req.NormalizeOutputs))
}
if req.NormalizeOutputs[0].LaneID != "alpha" || req.NormalizeOutputs[0].NormalizerKey != "normalize" {
t.Fatalf("NormalizeOutputs[0] = %#v, want normalized alpha output", req.NormalizeOutputs[0])
}
}
@@ -1066,9 +906,9 @@ func TestRunRejectsUnsafeOutputFileNames(t *testing.T) {
}{
{name: "empty", fileName: ""},
{name: "absolute", fileName: "/tmp/output.json"},
{name: "parent", fileName: "artifacts/../manifest.json"},
{name: "backslash", fileName: `artifacts\manifest.json`},
{name: "unclean", fileName: "artifacts//manifest.json"},
{name: "parent", fileName: "outputs/../manifest.json"},
{name: "backslash", fileName: `outputs\manifest.json`},
{name: "unclean", fileName: "outputs//manifest.json"},
}
for _, test := range tests {
@@ -1101,8 +941,8 @@ func TestRunReturnsFailedManifestWhenOutputEncoderFails(t *testing.T) {
if output.Manifest.CompletedAt == nil {
t.Fatal("CompletedAt = nil, want failed run completion timestamp")
}
if len(output.Approved) != 2 {
t.Fatalf("len(Approved) = %d, want partial approved output", len(output.Approved))
if len(output.NormalizeOutputs) != 1 {
t.Fatalf("len(NormalizeOutputs) = %d, want partial normalized output", len(output.NormalizeOutputs))
}
}
@@ -1313,7 +1153,7 @@ func TestRunManifestIncludesExtractorMetadata(t *testing.T) {
func TestRunReturnsPartialOutputWhenLaterLaneFails(t *testing.T) {
modules := defaultRunnerModules()
modules.extractors["extract-beta"] = &runnerExtractor{key: "extract-beta", artifactType: "artifact", schemaVersion: "v1", err: errors.New("extract failed")}
modules.extractors["extract-beta"] = &runnerExtractor{key: "extract-beta", err: errors.New("extract failed")}
pipeline := resolvedPipeline()
pipeline.ArtifactLanes = append(pipeline.ArtifactLanes, ResolvedArtifactLane{
ID: "beta",
@@ -1328,46 +1168,8 @@ func TestRunReturnsPartialOutputWhenLaterLaneFails(t *testing.T) {
if output.Manifest.ValidationStatus != "failed" {
t.Fatalf("ValidationStatus = %q, want failed", output.Manifest.ValidationStatus)
}
if len(output.Approved) != 2 {
t.Fatalf("len(Approved) = %d, want first lane approved output", len(output.Approved))
}
}
func TestRunSurfacesValidatorErrors(t *testing.T) {
tests := []struct {
name string
validator *runnerValidator
want string
}{
{name: "name mismatch", validator: &runnerValidator{name: "default-validator", resultName: "other", decisions: approveAll}, want: "returned result"},
{name: "cardinality", validator: &runnerValidator{name: "default-validator", decisions: func(candidates []artifacts.ArtifactCandidate) []contracts.ValidationDecision { return nil }}, want: "0 decisions"},
{name: "error", validator: &runnerValidator{name: "default-validator", decisions: approveAll, err: errors.New("validator failed")}, want: "validator failed"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
modules := defaultRunnerModules()
modules.extractors["extract-alpha"].validators = []contracts.Validator{test.validator}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
assertRunError(t, err, test.want)
if output.Manifest.ValidationStatus != "failed" {
t.Fatalf("ValidationStatus = %q, want failed", output.Manifest.ValidationStatus)
}
})
}
}
func TestRunRejectsNilDefaultValidator(t *testing.T) {
modules := defaultRunnerModules()
modules.extractors["extract-alpha"].validators = []contracts.Validator{nil}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
assertRunError(t, err, "validator[0] must not be nil")
if output.Manifest.ValidationStatus != "failed" {
t.Fatalf("ValidationStatus = %q, want failed", output.Manifest.ValidationStatus)
if len(output.NormalizeOutputs) != 1 {
t.Fatalf("len(NormalizeOutputs) = %d, want first lane output", len(output.NormalizeOutputs))
}
}
@@ -1438,7 +1240,7 @@ func defaultRunnerModules() *runnerModules {
input: &runnerInputAdapter{key: "input", doc: validSourceDocument()},
chunker: &runnerChunker{key: "chunk", chunks: []contracts.SourceChunk{sourceChunkWithID("chunk-0", 0), sourceChunkWithID("chunk-1", 1)}},
extractors: map[string]*runnerExtractor{
"extract-alpha": {key: "extract-alpha", artifactType: "artifact", schemaVersion: "v1"},
"extract-alpha": {key: "extract-alpha"},
},
mergers: map[string]*runnerMerger{
"merge": {key: "merge"},
@@ -1447,13 +1249,13 @@ func defaultRunnerModules() *runnerModules {
"normalize": {key: "normalize"},
},
validators: map[string]*runnerValidator{
"configured": {name: "configured", decisions: approveAll},
"second-validator": {name: "second-validator", decisions: approveAll},
"configured": {name: "configured"},
"second-validator": {name: "second-validator"},
},
output: &runnerOutputEncoder{
key: "output",
files: []contracts.OutputFile{
{Name: "artifacts/generic.json", ContentType: "application/json", Bytes: []byte(`{"encoded":true}`)},
{Name: "outputs/generic.json", ContentType: "application/json", Bytes: []byte(`{"encoded":true}`)},
},
},
}
@@ -1572,11 +1374,8 @@ func (chunker *runnerChunker) ManifestMetadata() map[string]any {
type runnerExtractor struct {
key string
artifactType string
schemaVersion string
manifestMetadata map[string]any
candidates []artifacts.ArtifactCandidate
validators []contracts.Validator
output *contracts.ExtractOutput
warnings []contracts.Warning
err error
requests []contracts.ExtractionRequest
@@ -1589,14 +1388,6 @@ func (extractor *runnerExtractor) Key() string {
return extractor.key
}
func (extractor *runnerExtractor) ArtifactType() string {
return extractor.artifactType
}
func (extractor *runnerExtractor) SchemaVersion() string {
return extractor.schemaVersion
}
func (extractor *runnerExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
@@ -1605,10 +1396,6 @@ func (extractor *runnerExtractor) ManifestMetadata() map[string]any {
return extractor.manifestMetadata
}
func (extractor *runnerExtractor) Validators() []contracts.Validator {
return extractor.validators
}
func (extractor *runnerExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
extractor.requests = append(extractor.requests, req)
if req.Chunk != nil {
@@ -1617,19 +1404,28 @@ func (extractor *runnerExtractor) Extract(ctx context.Context, req contracts.Ext
extractor.seenLLMClients = append(extractor.seenLLMClients, req.LLMClient)
extractor.seenMetadata = append(extractor.seenMetadata, req.Metadata)
candidates := append([]artifacts.ArtifactCandidate(nil), extractor.candidates...)
if len(candidates) == 0 {
candidates = []artifacts.ArtifactCandidate{{Payload: []byte(`{"value":true}`)}}
output := contracts.ExtractOutput{
Schema: contracts.ResponseSchema{ID: "runner.raw", Name: "runner_raw", Version: "v1"},
Payload: contracts.RawPayload{
Content: []byte(`{"value":true}`),
MediaType: "application/json",
},
}
if req.Chunk != nil {
output.Payload.Content = []byte(`{"chunk":"` + req.Chunk.ID + `"}`)
}
if extractor.output != nil {
output = *extractor.output
}
return contracts.ExtractionResult{
Candidates: candidates,
Warnings: extractor.warnings,
Output: output,
Warnings: extractor.warnings,
}, extractor.err
}
type runnerMerger struct {
key string
result []artifacts.ArtifactCandidate
result *contracts.MergeOutput
warnings []contracts.Warning
err error
requests []contracts.MergeRequest
@@ -1641,21 +1437,27 @@ func (merger *runnerMerger) Key() string {
func (merger *runnerMerger) Merge(ctx context.Context, req contracts.MergeRequest) (contracts.MergeResult, error) {
merger.requests = append(merger.requests, req)
candidates := append([]artifacts.ArtifactCandidate(nil), merger.result...)
if candidates == nil {
for _, chunkArtifacts := range req.ChunkArtifacts {
candidates = append(candidates, chunkArtifacts.Candidates...)
}
output := contracts.MergeOutput{
LaneID: req.LaneID,
SourceID: req.Source.ID,
Schema: contracts.ResponseSchema{ID: "runner.raw", Name: "runner_raw", Version: "v1"},
Payload: contracts.RawPayload{
Content: []byte(`{"merged":true}`),
MediaType: "application/json",
},
}
if merger.result != nil {
output = *merger.result
}
return contracts.MergeResult{
Candidates: candidates,
Warnings: merger.warnings,
Output: output,
Warnings: merger.warnings,
}, merger.err
}
type runnerNormalizer struct {
key string
result []artifacts.ArtifactCandidate
result *contracts.NormalizeOutput
warnings []contracts.Warning
err error
requests []contracts.NormalizeRequest
@@ -1671,13 +1473,18 @@ func (normalizer *runnerNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
func (normalizer *runnerNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
normalizer.requests = append(normalizer.requests, req)
candidates := append([]artifacts.ArtifactCandidate(nil), normalizer.result...)
if candidates == nil {
candidates = append(candidates, req.Candidates...)
output := contracts.NormalizeOutput{
LaneID: req.LaneID,
SourceID: req.MergeOutput.SourceID,
Schema: req.MergeOutput.Schema,
Payload: req.MergeOutput.Payload,
}
if normalizer.result != nil {
output = *normalizer.result
}
return contracts.NormalizeResult{
Candidates: candidates,
Warnings: normalizer.warnings,
Output: output,
Warnings: normalizer.warnings,
}, normalizer.err
}
@@ -1757,14 +1564,6 @@ func (client manifestReportingLLMClient) LLMProfileManifests() []artifacts.LLMPr
return append([]artifacts.LLMProfileManifest(nil), client.profiles...)
}
func approveAll(candidates []artifacts.ArtifactCandidate) []contracts.ValidationDecision {
decisions := make([]contracts.ValidationDecision, 0, len(candidates))
for _, candidate := range candidates {
decisions = append(decisions, validate.Approved(candidate.Index))
}
return decisions
}
func validSourceDocument() *source.SourceDocument {
return &source.SourceDocument{
ID: "source-1",
@@ -1866,24 +1665,6 @@ func warningReasons(warnings []contracts.Warning) []string {
return reasons
}
func candidateIndices(candidates []artifacts.ArtifactCandidate) []int {
indices := make([]int, 0, len(candidates))
for _, candidate := range candidates {
indices = append(indices, candidate.Index)
}
return indices
}
func runnerCandidate(index int) artifacts.ArtifactCandidate {
return artifacts.ArtifactCandidate{
Index: index,
ExtractorKey: "extract-alpha",
ArtifactType: "artifact",
SchemaVersion: "v1",
Payload: []byte(`{"value":true}`),
}
}
func assertRunError(t *testing.T, err error, want string) {
t.Helper()

55
internal/framework/pipeline/testdata/walking_skeleton_output.json vendored
View File

@@ -12,40 +12,31 @@
}
]
},
"approved": [
"normalize_outputs": [
{
"extractor_key": "fake/extract",
"artifact_type": "fake_event",
"schema_version": "v1",
"payload": {
"chunk_id": "fixture-source:chunk:0",
"llm_call": 1,
"text": "First event. Second event."
"lane_id": "events",
"normalizer_key": "noop",
"source_id": "fixture-source",
"schema": {
"id": "fake_event",
"name": "fake_event",
"version": "v1"
},
"source_refs": [
{
"source_id": "fixture-source",
"start_unit_id": 1,
"end_unit_id": 2
}
]
},
{
"extractor_key": "fake/extract",
"artifact_type": "fake_event",
"schema_version": "v1",
"payload": {
"chunk_id": "fixture-source:chunk:1",
"llm_call": 2,
"text": "Third event."
},
"source_refs": [
{
"source_id": "fixture-source",
"start_unit_id": 3,
"end_unit_id": 3
}
]
"media_type": "application/json",
"content": {
"outputs": [
{
"chunk_id": "fixture-source:chunk:0",
"llm_call": 1,
"text": "First event. Second event."
},
{
"chunk_id": "fixture-source:chunk:1",
"llm_call": 2,
"text": "Third event."
}
]
}
}
]
}

83
internal/framework/pipeline/walking_skeleton_test.go
View File

@@ -256,22 +256,10 @@ func (extractor walkingSkeletonExtractor) Key() string {
return "fake/extract"
}
func (extractor walkingSkeletonExtractor) ArtifactType() string {
return "fake_event"
}
func (extractor walkingSkeletonExtractor) SchemaVersion() string {
return "v1"
}
func (extractor walkingSkeletonExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (extractor walkingSkeletonExtractor) Validators() []contracts.Validator {
return nil
}
func (extractor walkingSkeletonExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
var response struct {
Call int `json:"call"`
@@ -294,16 +282,11 @@ func (extractor walkingSkeletonExtractor) Extract(ctx context.Context, req contr
}
return contracts.ExtractionResult{
Candidates: []artifacts.ArtifactCandidate{
{
Payload: payload,
SourceRefs: []source.SourceRef{
{
SourceID: req.Source.ID,
StartUnitID: req.Chunk.Units[0].ID,
EndUnitID: req.Chunk.Units[len(req.Chunk.Units)-1].ID,
},
},
Output: contracts.ExtractOutput{
Schema: contracts.ResponseSchema{ID: "fake_event", Name: "fake_event", Version: "v1"},
Payload: contracts.RawPayload{
Content: payload,
MediaType: "application/json",
},
},
}, nil
@@ -336,11 +319,25 @@ func (merger walkingSkeletonMerger) Key() string {
}
func (merger walkingSkeletonMerger) Merge(ctx context.Context, req contracts.MergeRequest) (contracts.MergeResult, error) {
var candidates []artifacts.ArtifactCandidate
for _, chunkArtifacts := range req.ChunkArtifacts {
candidates = append(candidates, chunkArtifacts.Candidates...)
outputs := make([]json.RawMessage, 0, len(req.ExtractOutputs))
for _, output := range req.ExtractOutputs {
outputs = append(outputs, json.RawMessage(output.Payload.Content))
}
return contracts.MergeResult{Candidates: candidates}, nil
content, err := json.Marshal(map[string]any{"outputs": outputs})
if err != nil {
return contracts.MergeResult{}, err
}
return contracts.MergeResult{
Output: contracts.MergeOutput{
LaneID: req.LaneID,
SourceID: req.Source.ID,
Schema: contracts.ResponseSchema{ID: "fake_event", Name: "fake_event", Version: "v1"},
Payload: contracts.RawPayload{
Content: content,
MediaType: "application/json",
},
},
}, nil
}
type walkingSkeletonNormalizer struct{}
@@ -364,7 +361,14 @@ func (normalizer walkingSkeletonNormalizer) Normalize(ctx context.Context, req c
}, &response); err != nil {
return contracts.NormalizeResult{}, err
}
return contracts.NormalizeResult{Candidates: req.Candidates}, nil
return contracts.NormalizeResult{
Output: contracts.NormalizeOutput{
LaneID: req.LaneID,
SourceID: req.MergeOutput.SourceID,
Schema: req.MergeOutput.Schema,
Payload: req.MergeOutput.Payload,
},
}, nil
}
type walkingSkeletonOutput struct{}
@@ -374,9 +378,28 @@ func (output walkingSkeletonOutput) Key() string {
}
func (output walkingSkeletonOutput) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
type rawOutput struct {
LaneID string `json:"lane_id"`
NormalizerKey string `json:"normalizer_key"`
SourceID string `json:"source_id"`
Schema contracts.ResponseSchema `json:"schema"`
MediaType string `json:"media_type"`
Content json.RawMessage `json:"content"`
}
rawOutputs := make([]rawOutput, 0, len(req.NormalizeOutputs))
for _, output := range req.NormalizeOutputs {
rawOutputs = append(rawOutputs, rawOutput{
LaneID: output.LaneID,
NormalizerKey: output.NormalizerKey,
SourceID: output.SourceID,
Schema: output.Schema,
MediaType: output.Payload.MediaType,
Content: json.RawMessage(output.Payload.Content),
})
}
encoded, err := json.Marshal(struct {
Manifest artifacts.RunManifest `json:"manifest"`
Approved []artifacts.Artifact `json:"approved"`
Manifest artifacts.RunManifest `json:"manifest"`
NormalizeOutputs []rawOutput `json:"normalize_outputs"`
}{
Manifest: artifacts.RunManifest{
PipelineID: req.Manifest.PipelineID,
@@ -384,7 +407,7 @@ func (output walkingSkeletonOutput) Encode(ctx context.Context, req contracts.Ou
ArtifactLanes: req.Manifest.ArtifactLanes,
ValidationStatus: req.Manifest.ValidationStatus,
},
Approved: req.Approved,
NormalizeOutputs: rawOutputs,
})
if err != nil {
return contracts.OutputResult{}, err

72
internal/modules/extract/dnd/spells/extractor.go
View File

@@ -4,10 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/sharedassets/dnd"
@@ -45,14 +42,6 @@ func (e *Extractor) Key() string {
return Key
}
func (e *Extractor) ArtifactType() string {
return ArtifactType
}
func (e *Extractor) SchemaVersion() string {
return SchemaVersion
}
func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot {
return dnd.ReferenceSlots(referenceSlotDescriptions)
}
@@ -77,13 +66,6 @@ func (e *Extractor) ManifestMetadata() map[string]any {
return metadata
}
func (e *Extractor) Validators() []contracts.Validator {
return []contracts.Validator{
ShapeValidator{},
SourceRefValidator{},
}
}
func (e *Extractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
if e == nil {
return contracts.ExtractionResult{}, extractorErrorf("extractor must not be nil")
@@ -121,22 +103,26 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.ExtractionRequest
if response.SpellCasts == nil {
return contracts.ExtractionResult{}, extractorErrorf("malformed structured output: spell_casts must be present")
}
if len(response.SpellCasts) == 0 {
return contracts.ExtractionResult{}, nil
content, err := json.Marshal(response)
if err != nil {
return contracts.ExtractionResult{}, extractorErrorf("marshal raw output: %w", err)
}
candidates := make([]artifacts.ArtifactCandidate, 0, len(response.SpellCasts))
for i, spellCast := range response.SpellCasts {
payload, err := spellCastPayload(spellCast)
if err != nil {
return contracts.ExtractionResult{}, extractorErrorf("marshal spell cast[%d]: %w", i, err)
}
candidates = append(candidates, artifacts.ArtifactCandidate{
Payload: payload,
SourceRefs: sourceRefCandidates(req.Source, spellCast.SourceRefs),
})
}
return contracts.ExtractionResult{Candidates: candidates}, nil
return contracts.ExtractionResult{
Output: contracts.ExtractOutput{
Schema: contracts.ResponseSchema{
ID: ResponseSchemaID,
Name: ResponseSchemaName,
Version: SchemaVersion,
},
Payload: contracts.RawPayload{
Content: content,
MediaType: "application/json",
Metadata: map[string]any{
"spell_cast_count": len(response.SpellCasts),
},
},
},
}, nil
}
func ModuleSpec() pipeline.ModuleSpec {
@@ -155,26 +141,6 @@ func Register(registry *pipeline.ExtractorRegistry) error {
})
}
func spellCastPayload(spellCast spellCastResponse) (json.RawMessage, error) {
return json.Marshal(SpellCast{
Caster: strings.TrimSpace(spellCast.Caster),
Spell: strings.TrimSpace(spellCast.Spell),
Effect: strings.TrimSpace(spellCast.Effect),
NarrativeDescription: strings.TrimSpace(spellCast.NarrativeDescription),
})
}
func sourceRefCandidates(doc *source.SourceDocument, refs []dnd.SourceRefResponse) []source.SourceRef {
if len(refs) == 0 {
return nil
}
out := make([]source.SourceRef, 0, len(refs))
for _, ref := range refs {
out = append(out, dnd.SourceRefCandidate(doc, ref))
}
return out
}
func extractorErrorf(format string, args ...any) error {
return fmt.Errorf("dnd spells extractor: "+format, args...)
}

70
internal/modules/extract/dnd/spells/extractor_test.go
View File

@@ -7,12 +7,11 @@ import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/modules/sharedassets/dnd"
)
func TestExtractReturnsSpellCandidateFromStructuredOutput(t *testing.T) {
func TestExtractReturnsRawOutputFromStructuredResponse(t *testing.T) {
client := &fakeSpellsLLMClient{
response: extractionResponse{
SpellCasts: []spellCastResponse{
@@ -53,29 +52,19 @@ func TestExtractReturnsSpellCandidateFromStructuredOutput(t *testing.T) {
t.Fatalf("transcript content = %q, want original source input", got)
}
if len(result.Candidates) != 1 {
t.Fatalf("len(Candidates) = %d, want 1", len(result.Candidates))
if result.Output.Payload.MediaType != "application/json" {
t.Fatalf("MediaType = %q, want application/json", result.Output.Payload.MediaType)
}
candidate := result.Candidates[0]
if candidate.Index != 0 || candidate.ExtractorKey != "" || candidate.ArtifactType != "" || candidate.SchemaVersion != "" {
t.Fatalf("candidate envelope fields = %#v, want runner-normalized zero values", candidate)
if result.Output.Schema.ID != ResponseSchemaID || result.Output.Schema.Name != ResponseSchemaName || result.Output.Schema.Version != SchemaVersion {
t.Fatalf("schema = %#v, want response schema provenance", result.Output.Schema)
}
var payload SpellCast
if err := json.Unmarshal(candidate.Payload, &payload); err != nil {
t.Fatalf("Unmarshal(Payload) error = %v, want nil", err)
var payload extractionResponse
if err := json.Unmarshal(result.Output.Payload.Content, &payload); err != nil {
t.Fatalf("Unmarshal(Content) error = %v, want nil", err)
}
wantPayload := SpellCast{
Caster: "Aria",
Spell: "Cure Wounds",
Effect: "Heals an injured ally.",
NarrativeDescription: "Aria restores the fighter after the fight.",
}
if payload != wantPayload {
t.Fatalf("payload = %#v, want %#v", payload, wantPayload)
}
wantRef := source.SourceRef{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 2}
if len(candidate.SourceRefs) != 1 || candidate.SourceRefs[0] != wantRef {
t.Fatalf("SourceRefs = %#v, want %#v", candidate.SourceRefs, []source.SourceRef{wantRef})
if len(payload.SpellCasts) != 1 || payload.SpellCasts[0].Spell != " Cure Wounds " {
t.Fatalf("payload = %#v, want raw structured response", payload)
}
}
@@ -174,15 +163,19 @@ func TestPromptInputsMapLegacyRosterReferenceToParty(t *testing.T) {
}
}
func TestExtractReturnsNoCandidatesForEmptyResponse(t *testing.T) {
func TestExtractReturnsRawOutputForEmptyResponse(t *testing.T) {
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}}
result, err := New().Extract(context.Background(), extractionRequestWithClient(client))
if err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
if len(result.Candidates) != 0 {
t.Fatalf("Candidates = %#v, want none", result.Candidates)
var payload extractionResponse
if err := json.Unmarshal(result.Output.Payload.Content, &payload); err != nil {
t.Fatalf("Unmarshal(Content) error = %v, want nil", err)
}
if len(payload.SpellCasts) != 0 {
t.Fatalf("SpellCasts = %#v, want none", payload.SpellCasts)
}
}
@@ -271,23 +264,16 @@ func TestExtractPreservesResponseOrder(t *testing.T) {
if err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
if len(result.Candidates) != 2 {
t.Fatalf("len(Candidates) = %d, want 2", len(result.Candidates))
var payload extractionResponse
if err := json.Unmarshal(result.Output.Payload.Content, &payload); err != nil {
t.Fatalf("Unmarshal(Content) error = %v, want nil", err)
}
var first, second SpellCast
if err := json.Unmarshal(result.Candidates[0].Payload, &first); err != nil {
t.Fatalf("Unmarshal(first) error = %v, want nil", err)
}
if err := json.Unmarshal(result.Candidates[1].Payload, &second); err != nil {
t.Fatalf("Unmarshal(second) error = %v, want nil", err)
}
if first.Spell != "Cure Wounds" || second.Spell != "Fire Bolt" {
t.Fatalf("candidate order = %q, %q; want response order", first.Spell, second.Spell)
if len(payload.SpellCasts) != 2 || payload.SpellCasts[0].Spell != "Cure Wounds" || payload.SpellCasts[1].Spell != "Fire Bolt" {
t.Fatalf("spell order = %#v, want response order", payload.SpellCasts)
}
}
func TestExtractCopiesCandidateSourceRefs(t *testing.T) {
func TestExtractDefensivelyCopiesRawContent(t *testing.T) {
client := &fakeSpellsLLMClient{
response: extractionResponse{
SpellCasts: []spellCastResponse{
@@ -308,8 +294,12 @@ func TestExtractCopiesCandidateSourceRefs(t *testing.T) {
}
client.response.SpellCasts[0].SourceRefs[0].StartUnitID = dnd.UnitRefFromInt(99)
if got := result.Candidates[0].SourceRefs[0].StartUnitID; got != 1 {
t.Fatalf("candidate source ref start = %d, want copied 1", got)
var payload extractionResponse
if err := json.Unmarshal(result.Output.Payload.Content, &payload); err != nil {
t.Fatalf("Unmarshal(Content) error = %v, want nil", err)
}
if got := payload.SpellCasts[0].SourceRefs[0].StartUnitID.String(); got != "1" {
t.Fatalf("source ref start = %q, want copied 1", got)
}
}

6
internal/modules/extract/dnd/spells/registry_test.go
View File

@@ -17,12 +17,6 @@ func TestNewReturnsExtractorWithMetadata(t *testing.T) {
if extractor.Key() != Key {
t.Fatalf("extractor.Key() = %q, want %q", extractor.Key(), Key)
}
if extractor.ArtifactType() != ArtifactType {
t.Fatalf("extractor.ArtifactType() = %q, want %q", extractor.ArtifactType(), ArtifactType)
}
if extractor.SchemaVersion() != SchemaVersion {
t.Fatalf("extractor.SchemaVersion() = %q, want %q", extractor.SchemaVersion(), SchemaVersion)
}
}
func TestModuleSpec(t *testing.T) {

79
internal/modules/extract/dnd/spells/runner_test.go
View File

@@ -48,31 +48,30 @@ func TestRunnerProcessesSeriatimInputWithDNDSpellsExtractor(t *testing.T) {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.Approved) != 2 {
t.Fatalf("len(Approved) = %d, want 2", len(output.Approved))
if len(output.NormalizeOutputs) != 1 {
t.Fatalf("len(NormalizeOutputs) = %d, want 1", len(output.NormalizeOutputs))
}
var first, second SpellCast
if err := json.Unmarshal(output.Approved[0].Payload, &first); err != nil {
t.Fatalf("Unmarshal(first payload) error = %v, want nil", err)
rawOutput := output.NormalizeOutputs[0]
if rawOutput.LaneID != "spells" || rawOutput.Schema.ID != ResponseSchemaID || rawOutput.Schema.Version != SchemaVersion {
t.Fatalf("raw output envelope = %#v, want dnd spells schema on spells lane", rawOutput)
}
if err := json.Unmarshal(output.Approved[1].Payload, &second); err != nil {
t.Fatalf("Unmarshal(second payload) error = %v, want nil", err)
response := decodeRunnerSpellResponse(t, rawOutput.Payload.Content)
if len(response.SpellCasts) != 2 {
t.Fatalf("len(spell_casts) = %d, want 2", len(response.SpellCasts))
}
first, second := response.SpellCasts[0], response.SpellCasts[1]
if first.Spell != "Cure Wounds" || second.Spell != "Fire Bolt" {
t.Fatalf("approved spell order = %q, %q; want response order", first.Spell, second.Spell)
t.Fatalf("spell order = %q, %q; want response order", first.Spell, second.Spell)
}
if first.Caster != "Aria" || second.Caster != "Borin" {
t.Fatalf("approved casters = %q, %q; want spell data", first.Caster, second.Caster)
t.Fatalf("casters = %q, %q; want spell data", first.Caster, second.Caster)
}
for _, artifact := range output.Approved {
if artifact.ExtractorKey != Key || artifact.ArtifactType != ArtifactType || artifact.SchemaVersion != SchemaVersion {
t.Fatalf("approved artifact envelope = %#v, want dnd spells envelope", artifact)
for _, spell := range response.SpellCasts {
if len(spell.SourceRefs) != 1 {
t.Fatalf("len(SourceRefs) = %d, want 1", len(spell.SourceRefs))
}
if len(artifact.SourceRefs) != 1 {
t.Fatalf("len(SourceRefs) = %d, want 1", len(artifact.SourceRefs))
}
if err := source.ValidateRef(expectedDoc, artifact.SourceRefs[0]); err != nil {
t.Fatalf("ValidateRef() error = %v, want nil", err)
if spell.SourceRefs[0].SourceID != expectedDoc.ID {
t.Fatalf("SourceID = %q, want fixture document ID", spell.SourceRefs[0].SourceID)
}
}
@@ -137,8 +136,8 @@ func TestRunnerPassesPartyAndGlossaryReferencesToDNDSpellsPrompt(t *testing.T) {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.Approved) != 1 {
t.Fatalf("len(Approved) = %d, want 1", len(output.Approved))
if len(output.NormalizeOutputs) != 1 {
t.Fatalf("len(NormalizeOutputs) = %d, want 1", len(output.NormalizeOutputs))
}
if len(output.Manifest.References) != 2 {
t.Fatalf("manifest references = %#v, want party and glossary provenance", output.Manifest.References)
@@ -178,8 +177,12 @@ func TestRunnerDoesNotExtractSpellMentionedOnlyInPartyReference(t *testing.T) {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.Approved) != 0 {
t.Fatalf("approved artifacts = %#v, want no party-reference-only spell casts", output.Approved)
if len(output.NormalizeOutputs) != 1 {
t.Fatalf("len(NormalizeOutputs) = %d, want empty spell response output", len(output.NormalizeOutputs))
}
response := decodeRunnerSpellResponse(t, output.NormalizeOutputs[0].Payload.Content)
if len(response.SpellCasts) != 0 {
t.Fatalf("spell_casts = %#v, want no party-reference-only spell casts", response.SpellCasts)
}
if len(llmClient.requests) != 1 {
t.Fatalf("LLM calls = %d, want 1", len(llmClient.requests))
@@ -196,7 +199,7 @@ func TestRunnerDoesNotExtractSpellMentionedOnlyInPartyReference(t *testing.T) {
}
}
func TestRunnerRejectsDNDSpellCastWithInvalidSourceRef(t *testing.T) {
func TestRunnerCarriesDNDSpellCastWithInvalidSourceRefAsRawOutput(t *testing.T) {
raw := readDNDSpellsFixture(t)
resolved := resolveDNDSpellsPipeline(t)
llmClient := &fakeSpellsLLMClient{
@@ -221,21 +224,21 @@ func TestRunnerRejectsDNDSpellCastWithInvalidSourceRef(t *testing.T) {
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.Approved) != 0 {
t.Fatalf("len(Approved) = %d, want 0", len(output.Approved))
if len(output.NormalizeOutputs) != 1 {
t.Fatalf("len(NormalizeOutputs) = %d, want 1", len(output.NormalizeOutputs))
}
if len(output.Rejected) != 1 {
t.Fatalf("len(Rejected) = %d, want 1", len(output.Rejected))
response := decodeRunnerSpellResponse(t, output.NormalizeOutputs[0].Payload.Content)
if len(response.SpellCasts) != 1 {
t.Fatalf("len(spell_casts) = %d, want 1", len(response.SpellCasts))
}
rejected := output.Rejected[0]
if rejected.ValidatorName != sourceRefValidatorName {
t.Fatalf("ValidatorName = %q, want %q", rejected.ValidatorName, sourceRefValidatorName)
if response.SpellCasts[0].SourceRefs[0].SourceID != "spell-session" {
t.Fatalf("SourceID = %q, want raw invalid source ref preserved", response.SpellCasts[0].SourceRefs[0].SourceID)
}
if rejected.ReasonCode != reasonInvalidSourceRef {
t.Fatalf("ReasonCode = %q, want %q", rejected.ReasonCode, reasonInvalidSourceRef)
if len(output.Rejected) != 0 {
t.Fatalf("len(Rejected) = %d, want 0", len(output.Rejected))
}
if output.Manifest.ValidationStatus != "rejected" {
t.Fatalf("ValidationStatus = %q, want rejected", output.Manifest.ValidationStatus)
if output.Manifest.ValidationStatus != "approved" {
t.Fatalf("ValidationStatus = %q, want approved", output.Manifest.ValidationStatus)
}
}
@@ -345,3 +348,13 @@ func parseDNDSpellsFixture(t *testing.T, raw []byte) *source.SourceDocument {
}
return doc
}
func decodeRunnerSpellResponse(t *testing.T, raw []byte) extractionResponse {
t.Helper()
var response extractionResponse
if err := json.Unmarshal(raw, &response); err != nil {
t.Fatalf("Unmarshal(raw output) error = %v, want nil", err)
}
return response
}

20
internal/modules/extract/dnd/spells/validator_test.go
View File

@@ -12,26 +12,6 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/validate"
)
func TestExtractorValidatorsReturnsExpectedChain(t *testing.T) {
validators := New().Validators()
if len(validators) != 2 {
t.Fatalf("len(Validators()) = %d, want 2", len(validators))
}
if validators[0].Name() != shapeValidatorName {
t.Fatalf("Validators()[0].Name() = %q, want %q", validators[0].Name(), shapeValidatorName)
}
if validators[1].Name() != sourceRefValidatorName {
t.Fatalf("Validators()[1].Name() = %q, want %q", validators[1].Name(), sourceRefValidatorName)
}
validators[0] = nil
again := New().Validators()
if len(again) != 2 || again[0] == nil || again[0].Name() != shapeValidatorName {
t.Fatalf("Validators() after caller mutation = %#v, want fresh validators", again)
}
}
func TestValidatorsApproveValidCandidate(t *testing.T) {
candidate := validSpellCandidate(7)

6
internal/modules/input/seriatim/config_test.go
View File

@@ -219,14 +219,8 @@ type fakeExtractor struct{}
func (fakeExtractor) Key() string { return "fake/extract" }
func (fakeExtractor) ArtifactType() string { return "fake" }
func (fakeExtractor) SchemaVersion() string { return "v1" }
func (fakeExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (fakeExtractor) Validators() []contracts.Validator { return nil }
func (fakeExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
return contracts.ExtractionResult{}, nil
}

72
internal/modules/input/seriatim/runner_test.go
View File

@@ -7,7 +7,6 @@ import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
@@ -43,22 +42,29 @@ func TestRunnerProcessesSeriatimInputWithFakeModules(t *testing.T) {
if got := output.Manifest.SourceDigests; len(got) != 1 || got[0] != expectedDoc.Digest {
t.Fatalf("manifest source digests = %#v, want %q", got, expectedDoc.Digest)
}
if len(output.Approved) != 1 {
t.Fatalf("len(Approved) = %d, want 1", len(output.Approved))
if len(output.NormalizeOutputs) != 1 {
t.Fatalf("len(NormalizeOutputs) = %d, want 1", len(output.NormalizeOutputs))
}
artifact := output.Approved[0]
if artifact.ExtractorKey != "fake/extract" || artifact.ArtifactType != "fake.event" || artifact.SchemaVersion != "v1" {
t.Fatalf("approved artifact envelope = %#v, want fake extractor envelope", artifact)
rawOutput := output.NormalizeOutputs[0]
if rawOutput.LaneID != "events" || rawOutput.NormalizerKey != pipeline.DefaultNormalizeModule || rawOutput.Schema.ID != "fake.event" || rawOutput.Schema.Version != "v1" {
t.Fatalf("raw output envelope = %#v, want fake extractor envelope", rawOutput)
}
if len(artifact.SourceRefs) != 1 {
t.Fatalf("len(SourceRefs) = %d, want 1", len(artifact.SourceRefs))
var payload struct {
Value string `json:"value"`
SourceRefs []source.SourceRef `json:"source_refs"`
}
if err := source.ValidateRef(expectedDoc, artifact.SourceRefs[0]); err != nil {
if err := json.Unmarshal(rawOutput.Payload.Content, &payload); err != nil {
t.Fatalf("Unmarshal(raw output) error = %v, want nil", err)
}
if len(payload.SourceRefs) != 1 {
t.Fatalf("len(SourceRefs) = %d, want 1", len(payload.SourceRefs))
}
if err := source.ValidateRef(expectedDoc, payload.SourceRefs[0]); err != nil {
t.Fatalf("ValidateRef() error = %v, want nil", err)
}
if artifact.SourceRefs[0].StartUnitID != 1 || artifact.SourceRefs[0].EndUnitID != 2 {
t.Fatalf("SourceRefs[0] = %#v, want Seriatim unit IDs", artifact.SourceRefs[0])
if payload.SourceRefs[0].StartUnitID != 1 || payload.SourceRefs[0].EndUnitID != 2 {
t.Fatalf("SourceRefs[0] = %#v, want Seriatim unit IDs", payload.SourceRefs[0])
}
if extractor.calls != 1 {
t.Fatalf("extractor calls = %d, want 1", extractor.calls)
@@ -186,22 +192,10 @@ func (e *runnerSeriatimExtractor) Key() string {
return "fake/extract"
}
func (e *runnerSeriatimExtractor) ArtifactType() string {
return "fake.event"
}
func (e *runnerSeriatimExtractor) SchemaVersion() string {
return "v1"
}
func (e *runnerSeriatimExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (e *runnerSeriatimExtractor) Validators() []contracts.Validator {
return nil
}
func (e *runnerSeriatimExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
e.calls++
if req.Source == nil {
@@ -228,17 +222,29 @@ func (e *runnerSeriatimExtractor) Extract(ctx context.Context, req contracts.Ext
}
}
return contracts.ExtractionResult{
Candidates: []artifacts.ArtifactCandidate{
payload, err := json.Marshal(struct {
Value string `json:"value"`
SourceRefs []source.SourceRef `json:"source_refs"`
}{
Value: "seriatim-source-ref",
SourceRefs: []source.SourceRef{
{
Payload: json.RawMessage(`{"value":"seriatim-source-ref"}`),
SourceRefs: []source.SourceRef{
{
SourceID: req.Source.ID,
StartUnitID: req.Chunk.Units[0].ID,
EndUnitID: req.Chunk.Units[len(req.Chunk.Units)-1].ID,
},
},
SourceID: req.Source.ID,
StartUnitID: req.Chunk.Units[0].ID,
EndUnitID: req.Chunk.Units[len(req.Chunk.Units)-1].ID,
},
},
})
if err != nil {
return contracts.ExtractionResult{}, err
}
return contracts.ExtractionResult{
Output: contracts.ExtractOutput{
Schema: contracts.ResponseSchema{ID: "fake.event", Name: "fake_event", Version: "v1"},
Payload: contracts.RawPayload{
Content: payload,
MediaType: "application/json",
},
},
}, nil

85
internal/modules/merge/appendorder/merger.go
View File

@@ -5,8 +5,6 @@ import (
"encoding/json"
"fmt"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
@@ -36,11 +34,34 @@ func (m *Merger) Merge(ctx context.Context, req contracts.MergeRequest) (contrac
return contracts.MergeResult{}, mergerErrorf("context error before merge: %w", err)
}
var candidates []artifacts.ArtifactCandidate
for _, chunkArtifacts := range req.ChunkArtifacts {
candidates = append(candidates, cloneCandidates(chunkArtifacts.Candidates)...)
if len(req.ExtractOutputs) == 1 {
payload := cloneRawPayload(req.ExtractOutputs[0].Payload)
return contracts.MergeResult{
Output: contracts.MergeOutput{
LaneID: req.LaneID,
MergerKey: Key,
SourceID: req.ExtractOutputs[0].SourceID,
Schema: req.ExtractOutputs[0].Schema,
Payload: payload,
},
}, nil
}
return contracts.MergeResult{Candidates: candidates}, nil
content, err := orderedContent(req.ExtractOutputs)
if err != nil {
return contracts.MergeResult{}, err
}
return contracts.MergeResult{
Output: contracts.MergeOutput{
LaneID: req.LaneID,
MergerKey: Key,
SourceID: sourceID(req.ExtractOutputs),
Payload: contracts.RawPayload{
Content: content,
MediaType: "application/json",
},
},
}, nil
}
func ModuleSpec() pipeline.ModuleSpec {
@@ -57,27 +78,45 @@ func Register(registry *pipeline.MergerRegistry) error {
})
}
func cloneCandidates(candidates []artifacts.ArtifactCandidate) []artifacts.ArtifactCandidate {
if len(candidates) == 0 {
return nil
func orderedContent(outputs []contracts.ExtractOutput) ([]byte, error) {
items := make([]map[string]any, 0, len(outputs))
for _, output := range outputs {
item := map[string]any{
"chunk_id": output.ChunkID,
"chunk_index": output.ChunkIndex,
"media_type": output.Payload.MediaType,
}
if json.Valid(output.Payload.Content) {
item["content"] = json.RawMessage(append([]byte(nil), output.Payload.Content...))
} else {
item["content"] = string(output.Payload.Content)
}
items = append(items, item)
}
out := make([]artifacts.ArtifactCandidate, 0, len(candidates))
for _, candidate := range candidates {
out = append(out, cloneCandidate(candidate))
content, err := json.Marshal(struct {
Outputs []map[string]any `json:"outputs"`
}{Outputs: items})
if err != nil {
return nil, mergerErrorf("encode merged output: %w", err)
}
return out
return content, nil
}
func cloneCandidate(candidate artifacts.ArtifactCandidate) artifacts.ArtifactCandidate {
return artifacts.ArtifactCandidate{
Index: candidate.Index,
ExtractorKey: candidate.ExtractorKey,
ArtifactType: candidate.ArtifactType,
SchemaVersion: candidate.SchemaVersion,
Payload: append(json.RawMessage(nil), candidate.Payload...),
SourceRefs: append([]source.SourceRef(nil), candidate.SourceRefs...),
Metadata: cloneMetadata(candidate.Metadata),
func sourceID(outputs []contracts.ExtractOutput) string {
for _, output := range outputs {
if output.SourceID != "" {
return output.SourceID
}
}
return ""
}
func cloneRawPayload(payload contracts.RawPayload) contracts.RawPayload {
return contracts.RawPayload{
Content: append([]byte(nil), payload.Content...),
MediaType: payload.MediaType,
Metadata: cloneMetadata(payload.Metadata),
Warnings: append([]contracts.Warning(nil), payload.Warnings...),
}
}

149
internal/modules/merge/appendorder/merger_test.go
View File

@@ -6,8 +6,6 @@ import (
"reflect"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
@@ -35,113 +33,102 @@ func TestModuleSpecAndRegister(t *testing.T) {
}
}
func TestMergePreservesChunkAndCandidateOrder(t *testing.T) {
func TestMergePassesThroughSingleExtractOutput(t *testing.T) {
input := extractOutput("chunk-0", 0, `{"name":"original"}`)
result, err := New().Merge(context.Background(), contracts.MergeRequest{
ChunkArtifacts: []contracts.ChunkArtifacts{
{
Chunk: sourceChunk(0),
Candidates: []artifacts.ArtifactCandidate{candidate(2, "first-b"), candidate(1, "first-a")},
},
{
Chunk: sourceChunk(1),
Candidates: []artifacts.ArtifactCandidate{candidate(4, "second-b"), candidate(3, "second-a")},
},
},
LaneID: "events",
ExtractOutputs: []contracts.ExtractOutput{input},
})
if err != nil {
t.Fatalf("Merge() error = %v, want nil", err)
}
got := candidateNames(result.Candidates)
want := []string{"first-b", "first-a", "second-b", "second-a"}
if !reflect.DeepEqual(got, want) {
t.Fatalf("candidate order = %#v, want %#v", got, want)
if result.Output.LaneID != "events" || result.Output.MergerKey != Key {
t.Fatalf("output provenance = %#v, want lane and merger", result.Output)
}
if len(result.Warnings) != 0 {
t.Fatalf("Warnings = %#v, want none", result.Warnings)
if string(result.Output.Payload.Content) != `{"name":"original"}` {
t.Fatalf("content = %s, want original content", result.Output.Payload.Content)
}
if result.Output.Payload.Metadata["name"] != "chunk-0" {
t.Fatalf("metadata = %#v, want original metadata", result.Output.Payload.Metadata)
}
}
func TestMergeDefensivelyCopiesCandidates(t *testing.T) {
input := []contracts.ChunkArtifacts{
{
Chunk: sourceChunk(0),
Candidates: []artifacts.ArtifactCandidate{candidate(1, "original")},
},
}
func TestMergeDefensivelyCopiesRawPayload(t *testing.T) {
input := extractOutput("chunk-0", 0, `{"name":"original"}`)
result, err := New().Merge(context.Background(), contracts.MergeRequest{ChunkArtifacts: input})
result, err := New().Merge(context.Background(), contracts.MergeRequest{
LaneID: "events",
ExtractOutputs: []contracts.ExtractOutput{input},
})
if err != nil {
t.Fatalf("Merge() error = %v, want nil", err)
}
if len(result.Candidates) != 1 {
t.Fatalf("len(Candidates) = %d, want 1", len(result.Candidates))
input.Payload.Content[0] = '['
input.Payload.Metadata["name"] = "changed"
if string(result.Output.Payload.Content) != `{"name":"original"}` {
t.Fatalf("content changed after input mutation: %s", result.Output.Payload.Content)
}
if result.Output.Payload.Metadata["name"] != "chunk-0" {
t.Fatalf("metadata changed after input mutation: %#v", result.Output.Payload.Metadata)
}
}
func TestMergeWrapsMultipleOutputsInChunkOrder(t *testing.T) {
result, err := New().Merge(context.Background(), contracts.MergeRequest{
LaneID: "events",
ExtractOutputs: []contracts.ExtractOutput{
extractOutput("chunk-0", 0, `{"name":"first"}`),
extractOutput("chunk-1", 1, `{"name":"second"}`),
},
})
if err != nil {
t.Fatalf("Merge() error = %v, want nil", err)
}
if result.Output.Payload.MediaType != "application/json" {
t.Fatalf("MediaType = %q, want application/json", result.Output.Payload.MediaType)
}
input[0].Candidates[0].Index = 99
input[0].Candidates[0].Payload[0] = '['
input[0].Candidates[0].SourceRefs[0].StartUnitID = 99
input[0].Candidates[0].Metadata["name"] = "changed"
got := result.Candidates[0]
if got.Index != 1 {
t.Fatalf("Index = %d, want 1", got.Index)
var decoded struct {
Outputs []struct {
ChunkID string `json:"chunk_id"`
ChunkIndex int `json:"chunk_index"`
Content json.RawMessage `json:"content"`
} `json:"outputs"`
}
if string(got.Payload) != `{"name":"original"}` {
t.Fatalf("Payload = %s, want original payload", got.Payload)
if err := json.Unmarshal(result.Output.Payload.Content, &decoded); err != nil {
t.Fatalf("Unmarshal() error = %v, want nil", err)
}
if got.SourceRefs[0].StartUnitID != 1 {
t.Fatalf("SourceRefs = %#v, want original source ref", got.SourceRefs)
}
if got.Metadata["name"] != "original" {
t.Fatalf("Metadata = %#v, want original metadata", got.Metadata)
if len(decoded.Outputs) != 2 || decoded.Outputs[0].ChunkID != "chunk-0" || decoded.Outputs[1].ChunkID != "chunk-1" {
t.Fatalf("outputs = %#v, want chunk order", decoded.Outputs)
}
}
func TestMergeHandlesEmptyInput(t *testing.T) {
result, err := New().Merge(context.Background(), contracts.MergeRequest{})
result, err := New().Merge(context.Background(), contracts.MergeRequest{LaneID: "events"})
if err != nil {
t.Fatalf("Merge() error = %v, want nil", err)
}
if len(result.Candidates) != 0 {
t.Fatalf("len(Candidates) = %d, want 0", len(result.Candidates))
}
if len(result.Warnings) != 0 {
t.Fatalf("Warnings = %#v, want none", result.Warnings)
if string(result.Output.Payload.Content) != `{"outputs":[]}` {
t.Fatalf("content = %s, want empty outputs", result.Output.Payload.Content)
}
}
func candidate(index int, name string) artifacts.ArtifactCandidate {
return artifacts.ArtifactCandidate{
Index: index,
ExtractorKey: "generic-extractor",
ArtifactType: "generic-artifact",
SchemaVersion: "v1",
Payload: json.RawMessage(`{"name":"` + name + `"}`),
SourceRefs: []source.SourceRef{
{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1},
},
Metadata: map[string]any{
"name": name,
},
}
}
func candidateNames(candidates []artifacts.ArtifactCandidate) []string {
names := make([]string, 0, len(candidates))
for _, candidate := range candidates {
names = append(names, candidate.Metadata["name"].(string))
}
return names
}
func sourceChunk(index int) contracts.SourceChunk {
return contracts.SourceChunk{
ID: "chunk",
SourceID: "source-1",
Index: index,
Units: []source.SourceUnit{
{ID: 1, Kind: "unit", Text: "Source unit."},
func extractOutput(chunkID string, chunkIndex int, content string) contracts.ExtractOutput {
return contracts.ExtractOutput{
LaneID: "events",
ExtractorKey: "extract",
SourceID: "source-1",
ChunkID: chunkID,
ChunkIndex: chunkIndex,
Schema: contracts.ResponseSchema{ID: "schema-id", Name: "schema-name", Version: "v1"},
Payload: contracts.RawPayload{
Content: []byte(content),
MediaType: "application/json",
Metadata: map[string]any{"name": chunkID},
},
}
}

36
internal/modules/normalize/noop/normalizer.go
View File

@@ -2,11 +2,8 @@ package noop
import (
"context"
"encoding/json"
"fmt"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
@@ -39,7 +36,15 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.NormalizeReque
if err := ctx.Err(); err != nil {
return contracts.NormalizeResult{}, normalizerErrorf("context error before normalize: %w", err)
}
return contracts.NormalizeResult{Candidates: cloneCandidates(req.Candidates)}, nil
return contracts.NormalizeResult{
Output: contracts.NormalizeOutput{
LaneID: req.LaneID,
NormalizerKey: Key,
SourceID: req.MergeOutput.SourceID,
Schema: req.MergeOutput.Schema,
Payload: cloneRawPayload(req.MergeOutput.Payload),
},
}, nil
}
func ModuleSpec() pipeline.ModuleSpec {
@@ -57,24 +62,13 @@ func Register(registry *pipeline.NormalizerRegistry) error {
})
}
func cloneCandidates(candidates []artifacts.ArtifactCandidate) []artifacts.ArtifactCandidate {
if len(candidates) == 0 {
return nil
func cloneRawPayload(payload contracts.RawPayload) contracts.RawPayload {
return contracts.RawPayload{
Content: append([]byte(nil), payload.Content...),
MediaType: payload.MediaType,
Metadata: cloneMetadata(payload.Metadata),
Warnings: append([]contracts.Warning(nil), payload.Warnings...),
}
out := make([]artifacts.ArtifactCandidate, 0, len(candidates))
for _, candidate := range candidates {
out = append(out, artifacts.ArtifactCandidate{
Index: candidate.Index,
ExtractorKey: candidate.ExtractorKey,
ArtifactType: candidate.ArtifactType,
SchemaVersion: candidate.SchemaVersion,
Payload: append(json.RawMessage(nil), candidate.Payload...),
SourceRefs: append([]source.SourceRef(nil), candidate.SourceRefs...),
Metadata: cloneMetadata(candidate.Metadata),
})
}
return out
}
func cloneMetadata(metadata map[string]any) map[string]any {

109
internal/modules/normalize/noop/normalizer_test.go
View File

@@ -2,12 +2,9 @@ package noop
import (
"context"
"encoding/json"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
@@ -43,98 +40,60 @@ func TestModuleSpecAndRegister(t *testing.T) {
}
}
func TestNormalizePassesThroughOrderAndValues(t *testing.T) {
input := []artifacts.ArtifactCandidate{
candidate(3, "third"),
candidate(1, "first"),
candidate(2, "second"),
}
func TestNormalizePassesThroughMergeOutput(t *testing.T) {
input := mergeOutput(`{"name":"original"}`)
result, err := New().Normalize(context.Background(), contracts.NormalizeRequest{Candidates: input})
result, err := New().Normalize(context.Background(), contracts.NormalizeRequest{
LaneID: "events",
MergeOutput: input,
})
if err != nil {
t.Fatalf("Normalize() error = %v, want nil", err)
}
if len(result.Warnings) != 0 {
t.Fatalf("Warnings = %#v, want none", result.Warnings)
}
got := candidateNames(result.Candidates)
want := []string{"third", "first", "second"}
if !reflect.DeepEqual(got, want) {
t.Fatalf("candidate order = %#v, want %#v", got, want)
if result.Output.LaneID != "events" || result.Output.NormalizerKey != Key {
t.Fatalf("output provenance = %#v, want lane and normalizer", result.Output)
}
if !reflect.DeepEqual(result.Candidates[0].SourceRefs, input[0].SourceRefs) {
t.Fatalf("SourceRefs = %#v, want %#v", result.Candidates[0].SourceRefs, input[0].SourceRefs)
if string(result.Output.Payload.Content) != `{"name":"original"}` {
t.Fatalf("content = %s, want original content", result.Output.Payload.Content)
}
if !reflect.DeepEqual(result.Candidates[0].Metadata, input[0].Metadata) {
t.Fatalf("Metadata = %#v, want %#v", result.Candidates[0].Metadata, input[0].Metadata)
if result.Output.Payload.Metadata["name"] != "original" {
t.Fatalf("metadata = %#v, want original metadata", result.Output.Payload.Metadata)
}
}
func TestNormalizeDefensivelyCopiesCandidates(t *testing.T) {
input := []artifacts.ArtifactCandidate{candidate(1, "original")}
func TestNormalizeDefensivelyCopiesRawPayload(t *testing.T) {
input := mergeOutput(`{"name":"original"}`)
result, err := New().Normalize(context.Background(), contracts.NormalizeRequest{Candidates: input})
result, err := New().Normalize(context.Background(), contracts.NormalizeRequest{
LaneID: "events",
MergeOutput: input,
})
if err != nil {
t.Fatalf("Normalize() error = %v, want nil", err)
}
if len(result.Candidates) != 1 {
t.Fatalf("len(Candidates) = %d, want 1", len(result.Candidates))
}
input[0].Index = 99
input[0].Payload[0] = '['
input[0].SourceRefs[0].EndUnitID = 99
input[0].Metadata["name"] = "changed"
input.Payload.Content[0] = '['
input.Payload.Metadata["name"] = "changed"
got := result.Candidates[0]
if got.Index != 1 {
t.Fatalf("Index = %d, want 1", got.Index)
if string(result.Output.Payload.Content) != `{"name":"original"}` {
t.Fatalf("content changed after input mutation: %s", result.Output.Payload.Content)
}
if string(got.Payload) != `{"name":"original"}` {
t.Fatalf("Payload = %s, want original payload", got.Payload)
}
if got.SourceRefs[0].EndUnitID != 1 {
t.Fatalf("SourceRefs = %#v, want original source ref", got.SourceRefs)
}
if got.Metadata["name"] != "original" {
t.Fatalf("Metadata = %#v, want original metadata", got.Metadata)
if result.Output.Payload.Metadata["name"] != "original" {
t.Fatalf("metadata changed after input mutation: %#v", result.Output.Payload.Metadata)
}
}
func TestNormalizeHandlesEmptyInput(t *testing.T) {
result, err := New().Normalize(context.Background(), contracts.NormalizeRequest{})
if err != nil {
t.Fatalf("Normalize() error = %v, want nil", err)
}
if len(result.Candidates) != 0 {
t.Fatalf("len(Candidates) = %d, want 0", len(result.Candidates))
}
if len(result.Warnings) != 0 {
t.Fatalf("Warnings = %#v, want none", result.Warnings)
}
}
func candidate(index int, name string) artifacts.ArtifactCandidate {
return artifacts.ArtifactCandidate{
Index: index,
ExtractorKey: "generic-extractor",
ArtifactType: "generic-artifact",
SchemaVersion: "v1",
Payload: json.RawMessage(`{"name":"` + name + `"}`),
SourceRefs: []source.SourceRef{
{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1},
},
Metadata: map[string]any{
"name": name,
func mergeOutput(content string) contracts.MergeOutput {
return contracts.MergeOutput{
LaneID: "events",
MergerKey: "merge",
SourceID: "source-1",
Schema: contracts.ResponseSchema{ID: "schema-id", Name: "schema-name", Version: "v1"},
Payload: contracts.RawPayload{
Content: []byte(content),
MediaType: "application/json",
Metadata: map[string]any{"name": "original"},
},
}
}
func candidateNames(candidates []artifacts.ArtifactCandidate) []string {
names := make([]string, 0, len(candidates))
for _, candidate := range candidates {
names = append(names, candidate.Metadata["name"].(string))
}
return names
}

169
internal/modules/output/json/encoder.go
View File

@@ -8,8 +8,6 @@ import (
"sort"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
@@ -18,7 +16,7 @@ const Key = "json"
const contentTypeJSON = "application/json"
var safeArtifactFileChar = regexp.MustCompile(`[^A-Za-z0-9._-]`)
var safeOutputFileChar = regexp.MustCompile(`[^A-Za-z0-9._-]`)
var _ contracts.OutputEncoder = (*Encoder)(nil)
@@ -66,24 +64,24 @@ func Register(registry *pipeline.OutputEncoderRegistry) error {
}
type indexFile struct {
ManifestFile string `json:"manifest_file"`
ArtifactFiles []artifactFileIndex `json:"artifact_files"`
RejectedFile string `json:"rejected_file"`
WarningsFile string `json:"warnings_file"`
ManifestFile string `json:"manifest_file"`
OutputFiles []outputFileIndex `json:"output_files"`
RejectedFile string `json:"rejected_file"`
WarningsFile string `json:"warnings_file"`
}
type artifactFileIndex struct {
ArtifactType string `json:"artifact_type"`
File string `json:"file"`
}
type artifactFile struct {
ArtifactType string `json:"artifact_type"`
Artifacts []artifacts.Artifact `json:"artifacts"`
type outputFileIndex struct {
LaneID string `json:"lane_id"`
MediaType string `json:"media_type,omitempty"`
File string `json:"file"`
ModuleKey string `json:"module_key,omitempty"`
SchemaID string `json:"schema_id,omitempty"`
SchemaName string `json:"schema_name,omitempty"`
SchemaVer string `json:"schema_version,omitempty"`
}
type rejectedFile struct {
Rejected []artifacts.RejectedArtifact `json:"rejected"`
Rejected []contracts.RejectedOutput `json:"rejected"`
}
type warningsFile struct {
@@ -91,43 +89,39 @@ type warningsFile struct {
}
func logicalFiles(req contracts.OutputRequest) ([]contracts.OutputFile, error) {
artifactsByType := make(map[string][]artifacts.Artifact)
for _, artifact := range req.Approved {
artifactsByType[artifact.ArtifactType] = append(artifactsByType[artifact.ArtifactType], cloneArtifact(artifact))
}
outputs := cloneNormalizeOutputs(req.NormalizeOutputs)
sort.SliceStable(outputs, func(i, j int) bool {
return outputs[i].LaneID < outputs[j].LaneID
})
artifactTypes := make([]string, 0, len(artifactsByType))
for artifactType := range artifactsByType {
artifactTypes = append(artifactTypes, artifactType)
}
sort.Strings(artifactTypes)
artifactIndexes := make([]artifactFileIndex, 0, len(artifactTypes))
files := make([]contracts.OutputFile, 0, len(artifactTypes)+4)
outputIndexes := make([]outputFileIndex, 0, len(outputs))
files := make([]contracts.OutputFile, 0, len(outputs)+4)
manifestFile, err := jsonFile("manifest.json", req.Manifest)
if err != nil {
return nil, err
}
files = append(files, manifestFile)
usedArtifactFiles := make(map[string]string, len(artifactTypes))
for _, artifactType := range artifactTypes {
name, err := artifactFileName(artifactType)
usedOutputFiles := make(map[string]string, len(outputs))
for _, output := range outputs {
name, err := outputFileName(output.LaneID)
if err != nil {
return nil, err
}
if existingType, ok := usedArtifactFiles[name]; ok {
return nil, encoderErrorf("artifact types %q and %q produce duplicate output file %q", existingType, artifactType, name)
if existingLane, ok := usedOutputFiles[name]; ok {
return nil, encoderErrorf("lanes %q and %q produce duplicate output file %q", existingLane, output.LaneID, name)
}
usedArtifactFiles[name] = artifactType
artifactIndexes = append(artifactIndexes, artifactFileIndex{
ArtifactType: artifactType,
File: name,
})
file, err := jsonFile(name, artifactFile{
ArtifactType: artifactType,
Artifacts: artifactsByType[artifactType],
usedOutputFiles[name] = output.LaneID
outputIndexes = append(outputIndexes, outputFileIndex{
LaneID: output.LaneID,
MediaType: output.Payload.MediaType,
File: name,
ModuleKey: output.NormalizerKey,
SchemaID: output.Schema.ID,
SchemaName: output.Schema.Name,
SchemaVer: output.Schema.Version,
})
file, err := rawOutputFile(name, output.Payload)
if err != nil {
return nil, err
}
@@ -135,10 +129,10 @@ func logicalFiles(req contracts.OutputRequest) ([]contracts.OutputFile, error) {
}
index := indexFile{
ManifestFile: "manifest.json",
ArtifactFiles: artifactIndexes,
RejectedFile: "rejected.json",
WarningsFile: "warnings.json",
ManifestFile: "manifest.json",
OutputFiles: outputIndexes,
RejectedFile: "rejected.json",
WarningsFile: "warnings.json",
}
indexOutput, err := jsonFile("index.json", index)
if err != nil {
@@ -159,6 +153,32 @@ func logicalFiles(req contracts.OutputRequest) ([]contracts.OutputFile, error) {
return files, nil
}
func rawOutputFile(name string, payload contracts.RawPayload) (contracts.OutputFile, error) {
content := append([]byte(nil), payload.Content...)
if len(content) == 0 {
content = []byte("null")
}
if payload.MediaType == contentTypeJSON && stdjson.Valid(content) {
var decoded any
if err := stdjson.Unmarshal(content, &decoded); err == nil {
pretty, err := marshalPretty(decoded)
if err != nil {
return contracts.OutputFile{}, err
}
content = pretty
}
}
mediaType := strings.TrimSpace(payload.MediaType)
if mediaType == "" {
mediaType = "application/octet-stream"
}
return contracts.OutputFile{
Name: name,
ContentType: mediaType,
Bytes: append([]byte(nil), content...),
}, nil
}
func jsonFile(name string, value any) (contracts.OutputFile, error) {
data, err := marshalPretty(value)
if err != nil {
@@ -179,57 +199,46 @@ func marshalPretty(value any) ([]byte, error) {
return append(data, '\n'), nil
}
func artifactFileName(artifactType string) (string, error) {
sanitized := safeArtifactFileChar.ReplaceAllString(strings.TrimSpace(artifactType), "_")
func outputFileName(laneID string) (string, error) {
sanitized := safeOutputFileChar.ReplaceAllString(strings.TrimSpace(laneID), "_")
for strings.Contains(sanitized, "..") {
sanitized = strings.ReplaceAll(sanitized, "..", "__")
}
sanitized = strings.Trim(sanitized, "._")
if sanitized == "" {
return "", encoderErrorf("artifact type %q cannot produce a safe file name", artifactType)
return "", encoderErrorf("lane id %q cannot produce a safe file name", laneID)
}
return "artifacts/" + sanitized + ".json", nil
return "outputs/" + sanitized + ".json", nil
}
func cloneArtifact(artifact artifacts.Artifact) artifacts.Artifact {
return artifacts.Artifact{
ExtractorKey: artifact.ExtractorKey,
ArtifactType: artifact.ArtifactType,
SchemaVersion: artifact.SchemaVersion,
Payload: append(stdjson.RawMessage(nil), artifact.Payload...),
SourceRefs: append([]source.SourceRef(nil), artifact.SourceRefs...),
Metadata: cloneMetadata(artifact.Metadata),
func cloneNormalizeOutputs(outputs []contracts.NormalizeOutput) []contracts.NormalizeOutput {
if len(outputs) == 0 {
return nil
}
}
func cloneRejected(rejected []artifacts.RejectedArtifact) []artifacts.RejectedArtifact {
if len(rejected) == 0 {
return []artifacts.RejectedArtifact{}
}
out := make([]artifacts.RejectedArtifact, 0, len(rejected))
for _, item := range rejected {
out = append(out, artifacts.RejectedArtifact{
Candidate: cloneCandidate(item.Candidate),
ValidatorName: item.ValidatorName,
ReasonCode: item.ReasonCode,
Message: item.Message,
})
out := make([]contracts.NormalizeOutput, 0, len(outputs))
for _, output := range outputs {
output.Payload = cloneRawPayload(output.Payload)
out = append(out, output)
}
return out
}
func cloneCandidate(candidate artifacts.ArtifactCandidate) artifacts.ArtifactCandidate {
return artifacts.ArtifactCandidate{
Index: candidate.Index,
ExtractorKey: candidate.ExtractorKey,
ArtifactType: candidate.ArtifactType,
SchemaVersion: candidate.SchemaVersion,
Payload: append(stdjson.RawMessage(nil), candidate.Payload...),
SourceRefs: append([]source.SourceRef(nil), candidate.SourceRefs...),
Metadata: cloneMetadata(candidate.Metadata),
func cloneRawPayload(payload contracts.RawPayload) contracts.RawPayload {
return contracts.RawPayload{
Content: append([]byte(nil), payload.Content...),
MediaType: payload.MediaType,
Metadata: cloneMetadata(payload.Metadata),
Warnings: append([]contracts.Warning(nil), payload.Warnings...),
}
}
func cloneRejected(rejected []contracts.RejectedOutput) []contracts.RejectedOutput {
if len(rejected) == 0 {
return []contracts.RejectedOutput{}
}
return append([]contracts.RejectedOutput(nil), rejected...)
}
func cloneWarnings(warnings []contracts.Warning) []contracts.Warning {
if len(warnings) == 0 {
return []contracts.Warning{}

153
internal/modules/output/json/encoder_test.go
View File

@@ -8,7 +8,6 @@ import (
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
@@ -37,20 +36,24 @@ func TestModuleSpecAndRegister(t *testing.T) {
}
}
func TestEncodeReturnsLogicalFilesGroupedByArtifactType(t *testing.T) {
func TestEncodeReturnsLogicalFilesForNormalizedOutputs(t *testing.T) {
req := contracts.OutputRequest{
Manifest: artifacts.RunManifest{RunID: "run-1", PipelineID: "pipeline-1"},
Approved: []artifacts.Artifact{
artifact("dnd.spell-cast", "first"),
artifact("notes/item", "item"),
artifact("dnd.spell-cast", "second"),
NormalizeOutputs: []contracts.NormalizeOutput{
normalizeOutput("spells", `{"spell_casts":[{"spell":"Cure Wounds"}]}`),
normalizeOutput("notes/items", `{"items":[{"name":"Torch"}]}`),
},
Rejected: []artifacts.RejectedArtifact{
Rejected: []contracts.RejectedOutput{
{
Candidate: candidate("bad type", "bad"),
ValidatorName: "validator",
Stage: "extract",
LaneID: "spells",
ModuleKey: "dnd/spells",
ChunkID: "chunk-1",
ChunkIndex: 1,
ReasonCode: "invalid",
Message: "not accepted",
AttemptCount: 1,
ValidatorName: "validator",
},
},
Warnings: []contracts.Warning{{ReasonCode: "warning", Message: "check source"}},
@@ -62,51 +65,46 @@ func TestEncodeReturnsLogicalFilesGroupedByArtifactType(t *testing.T) {
}
wantNames := []string{
"artifacts/dnd.spell-cast.json",
"artifacts/notes_item.json",
"index.json",
"manifest.json",
"outputs/notes_items.json",
"outputs/spells.json",
"rejected.json",
"warnings.json",
}
if got := outputFileNames(result.Files); !reflect.DeepEqual(got, wantNames) {
t.Fatalf("file names = %#v, want %#v", got, wantNames)
}
for _, file := range result.Files {
if file.ContentType != contentTypeJSON {
t.Fatalf("%s ContentType = %q, want %q", file.Name, file.ContentType, contentTypeJSON)
}
if !strings.HasSuffix(string(file.Bytes), "\n") {
t.Fatalf("%s does not end with newline: %q", file.Name, string(file.Bytes))
}
if !stdjson.Valid(file.Bytes) {
if file.ContentType == contentTypeJSON && !stdjson.Valid(file.Bytes) {
t.Fatalf("%s has invalid JSON: %s", file.Name, file.Bytes)
}
}
spellFile := decodeObject(t, fileBytes(t, result.Files, "artifacts/dnd.spell-cast.json"))
if spellFile["artifact_type"] != "dnd.spell-cast" {
t.Fatalf("artifact_type = %#v, want dnd.spell-cast", spellFile["artifact_type"])
}
spells := spellFile["artifacts"].([]any)
if len(spells) != 2 {
t.Fatalf("len(spells) = %d, want 2", len(spells))
}
firstPayload := spells[0].(map[string]any)["payload"].(map[string]any)
secondPayload := spells[1].(map[string]any)["payload"].(map[string]any)
if firstPayload["name"] != "first" || secondPayload["name"] != "second" {
t.Fatalf("spell order payloads = %#v then %#v, want runner order", firstPayload, secondPayload)
spells := decodeObject(t, fileBytes(t, result.Files, "outputs/spells.json"))
spellCasts := spells["spell_casts"].([]any)
if spellCasts[0].(map[string]any)["spell"] != "Cure Wounds" {
t.Fatalf("spells output = %#v, want raw normalized content", spells)
}
index := decodeObject(t, fileBytes(t, result.Files, "index.json"))
artifactFiles := index["artifact_files"].([]any)
if len(artifactFiles) != 2 {
t.Fatalf("len(index artifact_files) = %d, want 2", len(artifactFiles))
outputFiles := index["output_files"].([]any)
if len(outputFiles) != 2 {
t.Fatalf("len(index output_files) = %d, want 2", len(outputFiles))
}
firstIndex := artifactFiles[0].(map[string]any)
secondIndex := artifactFiles[1].(map[string]any)
if firstIndex["artifact_type"] != "dnd.spell-cast" || secondIndex["artifact_type"] != "notes/item" {
t.Fatalf("artifact_files = %#v, want sorted by artifact type", artifactFiles)
firstIndex := outputFiles[0].(map[string]any)
secondIndex := outputFiles[1].(map[string]any)
if firstIndex["lane_id"] != "notes/items" || secondIndex["lane_id"] != "spells" {
t.Fatalf("output_files = %#v, want sorted by lane id", outputFiles)
}
rejected := decodeObject(t, fileBytes(t, result.Files, "rejected.json"))
if got := rejected["rejected"].([]any); len(got) != 1 {
t.Fatalf("rejected = %#v, want one rejected output", got)
}
}
@@ -179,12 +177,12 @@ func TestEncodeIncludesManifestReferences(t *testing.T) {
}
}
func TestEncodeRejectsArtifactTypeWithoutSafeFileName(t *testing.T) {
func TestEncodeRejectsLaneIDWithoutSafeFileName(t *testing.T) {
_, err := New().Encode(context.Background(), contracts.OutputRequest{
Approved: []artifacts.Artifact{artifact("///", "unsafe")},
NormalizeOutputs: []contracts.NormalizeOutput{normalizeOutput("///", `{"value":true}`)},
})
if err == nil {
t.Fatal("Encode() error = nil, want unsafe artifact type error")
t.Fatal("Encode() error = nil, want unsafe lane id error")
}
if !strings.Contains(err.Error(), "json output encoder") || !strings.Contains(err.Error(), "safe file name") {
t.Fatalf("Encode() error = %q, want safe file name context", err.Error())
@@ -193,22 +191,22 @@ func TestEncodeRejectsArtifactTypeWithoutSafeFileName(t *testing.T) {
func TestEncodeSanitizesParentPathSequences(t *testing.T) {
result, err := New().Encode(context.Background(), contracts.OutputRequest{
Approved: []artifacts.Artifact{artifact("dnd..spell.", "spell")},
NormalizeOutputs: []contracts.NormalizeOutput{normalizeOutput("dnd..spell.", `{"value":true}`)},
})
if err != nil {
t.Fatalf("Encode() error = %v, want nil", err)
}
if got := outputFileNames(result.Files); !containsString(got, "artifacts/dnd__spell.json") {
t.Fatalf("file names = %#v, want sanitized artifact filename", got)
if got := outputFileNames(result.Files); !containsString(got, "outputs/dnd__spell.json") {
t.Fatalf("file names = %#v, want sanitized output filename", got)
}
}
func TestEncodeRejectsSanitizedFilenameCollisions(t *testing.T) {
_, err := New().Encode(context.Background(), contracts.OutputRequest{
Approved: []artifacts.Artifact{
artifact("a/b", "slash"),
artifact("a?b", "question"),
NormalizeOutputs: []contracts.NormalizeOutput{
normalizeOutput("a/b", `{"value":"slash"}`),
normalizeOutput("a?b", `{"value":"question"}`),
},
})
if err == nil {
@@ -222,16 +220,11 @@ func TestEncodeRejectsSanitizedFilenameCollisions(t *testing.T) {
func TestEncodeDoesNotMutateInputs(t *testing.T) {
req := contracts.OutputRequest{
Manifest: artifacts.RunManifest{RunID: "run-1"},
Approved: []artifacts.Artifact{
artifact("dnd.spell", "original"),
NormalizeOutputs: []contracts.NormalizeOutput{
normalizeOutput("spells", `{"name":"original"}`),
},
Rejected: []artifacts.RejectedArtifact{
{
Candidate: candidate("bad", "rejected"),
ValidatorName: "validator",
ReasonCode: "invalid",
Message: "not accepted",
},
Rejected: []contracts.RejectedOutput{
{Stage: "extract", LaneID: "spells", Message: "not accepted"},
},
Warnings: []contracts.Warning{{ReasonCode: "warning", Message: "message"}},
}
@@ -246,14 +239,13 @@ func TestEncodeDoesNotMutateInputs(t *testing.T) {
t.Fatalf("request mutated:\nbefore: %s\nafter: %s", before, after)
}
req.Approved[0].Payload[0] = '['
req.Approved[0].SourceRefs[0].StartUnitID = 99
req.Approved[0].Metadata["name"] = "changed"
req.Rejected[0].Candidate.Payload[0] = '['
req.NormalizeOutputs[0].Payload.Content[0] = '['
req.NormalizeOutputs[0].Payload.Metadata["name"] = "changed"
req.Rejected[0].Message = "changed"
req.Warnings[0].Message = "changed"
if !stdjson.Valid(fileBytes(t, result.Files, "artifacts/dnd.spell.json")) {
t.Fatal("artifact output changed after request mutation")
if !stdjson.Valid(fileBytes(t, result.Files, "outputs/spells.json")) {
t.Fatal("output changed after request mutation")
}
warnings := decodeObject(t, fileBytes(t, result.Files, "warnings.json"))
gotWarnings := warnings["warnings"].([]any)
@@ -262,9 +254,9 @@ func TestEncodeDoesNotMutateInputs(t *testing.T) {
}
}
func TestArtifactFilesDoNotContainWarnings(t *testing.T) {
func TestOutputFilesDoNotContainWarnings(t *testing.T) {
result, err := New().Encode(context.Background(), contracts.OutputRequest{
Approved: []artifacts.Artifact{artifact("dnd.spell", "spell")},
NormalizeOutputs: []contracts.NormalizeOutput{normalizeOutput("spells", `{"spell":"Shield"}`)},
Warnings: []contracts.Warning{
{ReasonCode: "pipeline-warning", Message: "warning"},
},
@@ -273,36 +265,27 @@ func TestArtifactFilesDoNotContainWarnings(t *testing.T) {
t.Fatalf("Encode() error = %v, want nil", err)
}
artifactFile := decodeObject(t, fileBytes(t, result.Files, "artifacts/dnd.spell.json"))
if _, ok := artifactFile["warnings"]; ok {
t.Fatalf("artifact file contains warnings: %#v", artifactFile)
outputFile := decodeObject(t, fileBytes(t, result.Files, "outputs/spells.json"))
if _, ok := outputFile["warnings"]; ok {
t.Fatalf("output file contains warnings: %#v", outputFile)
}
}
func artifact(artifactType, name string) artifacts.Artifact {
return artifacts.Artifact{
ExtractorKey: "extractor",
ArtifactType: artifactType,
SchemaVersion: "v1",
Payload: stdjson.RawMessage(`{"name":"` + name + `"}`),
SourceRefs: []source.SourceRef{
{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1},
func normalizeOutput(laneID string, content string) contracts.NormalizeOutput {
return contracts.NormalizeOutput{
LaneID: laneID,
NormalizerKey: "noop",
SourceID: "source-1",
Schema: contracts.ResponseSchema{
ID: "schema-id",
Name: "schema-name",
Version: "v1",
},
Metadata: map[string]any{"name": name},
}
}
func candidate(artifactType, name string) artifacts.ArtifactCandidate {
return artifacts.ArtifactCandidate{
Index: 1,
ExtractorKey: "extractor",
ArtifactType: artifactType,
SchemaVersion: "v1",
Payload: stdjson.RawMessage(`{"name":"` + name + `"}`),
SourceRefs: []source.SourceRef{
{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1},
Payload: contracts.RawPayload{
Content: []byte(content),
MediaType: contentTypeJSON,
Metadata: map[string]any{"name": laneID},
},
Metadata: map[string]any{"name": name},
}
}