eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement raw module output contracts

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-07 18:58:23 +00:00
parent 9e3f8809b3
commit c05ecb58d8
33 changed files with 1166 additions and 1780 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
internal/modules/extract/dnd/spells/extractor.go
View File

@@ -4,10 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/sharedassets/dnd"
@@ -45,14 +42,6 @@ func (e *Extractor) Key() string {
return Key
}
func (e *Extractor) ArtifactType() string {
return ArtifactType
}
func (e *Extractor) SchemaVersion() string {
return SchemaVersion
}
func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot {
return dnd.ReferenceSlots(referenceSlotDescriptions)
}
@@ -77,13 +66,6 @@ func (e *Extractor) ManifestMetadata() map[string]any {
return metadata
}
func (e *Extractor) Validators() []contracts.Validator {
return []contracts.Validator{
ShapeValidator{},
SourceRefValidator{},
}
}
func (e *Extractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
if e == nil {
return contracts.ExtractionResult{}, extractorErrorf("extractor must not be nil")
@@ -121,22 +103,26 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.ExtractionRequest
if response.SpellCasts == nil {
return contracts.ExtractionResult{}, extractorErrorf("malformed structured output: spell_casts must be present")
}
if len(response.SpellCasts) == 0 {
return contracts.ExtractionResult{}, nil
content, err := json.Marshal(response)
if err != nil {
return contracts.ExtractionResult{}, extractorErrorf("marshal raw output: %w", err)
}
candidates := make([]artifacts.ArtifactCandidate, 0, len(response.SpellCasts))
for i, spellCast := range response.SpellCasts {
payload, err := spellCastPayload(spellCast)
if err != nil {
return contracts.ExtractionResult{}, extractorErrorf("marshal spell cast[%d]: %w", i, err)
}
candidates = append(candidates, artifacts.ArtifactCandidate{
Payload: payload,
SourceRefs: sourceRefCandidates(req.Source, spellCast.SourceRefs),
})
}
return contracts.ExtractionResult{Candidates: candidates}, nil
return contracts.ExtractionResult{
Output: contracts.ExtractOutput{
Schema: contracts.ResponseSchema{
ID: ResponseSchemaID,
Name: ResponseSchemaName,
Version: SchemaVersion,
},
Payload: contracts.RawPayload{
Content: content,
MediaType: "application/json",
Metadata: map[string]any{
"spell_cast_count": len(response.SpellCasts),
},
},
},
}, nil
}
func ModuleSpec() pipeline.ModuleSpec {
@@ -155,26 +141,6 @@ func Register(registry *pipeline.ExtractorRegistry) error {
})
}
func spellCastPayload(spellCast spellCastResponse) (json.RawMessage, error) {
return json.Marshal(SpellCast{
Caster: strings.TrimSpace(spellCast.Caster),
Spell: strings.TrimSpace(spellCast.Spell),
Effect: strings.TrimSpace(spellCast.Effect),
NarrativeDescription: strings.TrimSpace(spellCast.NarrativeDescription),
})
}
func sourceRefCandidates(doc *source.SourceDocument, refs []dnd.SourceRefResponse) []source.SourceRef {
if len(refs) == 0 {
return nil
}
out := make([]source.SourceRef, 0, len(refs))
for _, ref := range refs {
out = append(out, dnd.SourceRefCandidate(doc, ref))
}
return out
}
func extractorErrorf(format string, args ...any) error {
return fmt.Errorf("dnd spells extractor: "+format, args...)
}

70
internal/modules/extract/dnd/spells/extractor_test.go
View File

@@ -7,12 +7,11 @@ import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/modules/sharedassets/dnd"
)
func TestExtractReturnsSpellCandidateFromStructuredOutput(t *testing.T) {
func TestExtractReturnsRawOutputFromStructuredResponse(t *testing.T) {
client := &fakeSpellsLLMClient{
response: extractionResponse{
SpellCasts: []spellCastResponse{
@@ -53,29 +52,19 @@ func TestExtractReturnsSpellCandidateFromStructuredOutput(t *testing.T) {
t.Fatalf("transcript content = %q, want original source input", got)
}
if len(result.Candidates) != 1 {
t.Fatalf("len(Candidates) = %d, want 1", len(result.Candidates))
if result.Output.Payload.MediaType != "application/json" {
t.Fatalf("MediaType = %q, want application/json", result.Output.Payload.MediaType)
}
candidate := result.Candidates[0]
if candidate.Index != 0 || candidate.ExtractorKey != "" || candidate.ArtifactType != "" || candidate.SchemaVersion != "" {
t.Fatalf("candidate envelope fields = %#v, want runner-normalized zero values", candidate)
if result.Output.Schema.ID != ResponseSchemaID || result.Output.Schema.Name != ResponseSchemaName || result.Output.Schema.Version != SchemaVersion {
t.Fatalf("schema = %#v, want response schema provenance", result.Output.Schema)
}
var payload SpellCast
if err := json.Unmarshal(candidate.Payload, &payload); err != nil {
t.Fatalf("Unmarshal(Payload) error = %v, want nil", err)
var payload extractionResponse
if err := json.Unmarshal(result.Output.Payload.Content, &payload); err != nil {
t.Fatalf("Unmarshal(Content) error = %v, want nil", err)
}
wantPayload := SpellCast{
Caster: "Aria",
Spell: "Cure Wounds",
Effect: "Heals an injured ally.",
NarrativeDescription: "Aria restores the fighter after the fight.",
}
if payload != wantPayload {
t.Fatalf("payload = %#v, want %#v", payload, wantPayload)
}
wantRef := source.SourceRef{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 2}
if len(candidate.SourceRefs) != 1 || candidate.SourceRefs[0] != wantRef {
t.Fatalf("SourceRefs = %#v, want %#v", candidate.SourceRefs, []source.SourceRef{wantRef})
if len(payload.SpellCasts) != 1 || payload.SpellCasts[0].Spell != " Cure Wounds " {
t.Fatalf("payload = %#v, want raw structured response", payload)
}
}
@@ -174,15 +163,19 @@ func TestPromptInputsMapLegacyRosterReferenceToParty(t *testing.T) {
}
}
func TestExtractReturnsNoCandidatesForEmptyResponse(t *testing.T) {
func TestExtractReturnsRawOutputForEmptyResponse(t *testing.T) {
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}}
result, err := New().Extract(context.Background(), extractionRequestWithClient(client))
if err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
if len(result.Candidates) != 0 {
t.Fatalf("Candidates = %#v, want none", result.Candidates)
var payload extractionResponse
if err := json.Unmarshal(result.Output.Payload.Content, &payload); err != nil {
t.Fatalf("Unmarshal(Content) error = %v, want nil", err)
}
if len(payload.SpellCasts) != 0 {
t.Fatalf("SpellCasts = %#v, want none", payload.SpellCasts)
}
}
@@ -271,23 +264,16 @@ func TestExtractPreservesResponseOrder(t *testing.T) {
if err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
if len(result.Candidates) != 2 {
t.Fatalf("len(Candidates) = %d, want 2", len(result.Candidates))
var payload extractionResponse
if err := json.Unmarshal(result.Output.Payload.Content, &payload); err != nil {
t.Fatalf("Unmarshal(Content) error = %v, want nil", err)
}
var first, second SpellCast
if err := json.Unmarshal(result.Candidates[0].Payload, &first); err != nil {
t.Fatalf("Unmarshal(first) error = %v, want nil", err)
}
if err := json.Unmarshal(result.Candidates[1].Payload, &second); err != nil {
t.Fatalf("Unmarshal(second) error = %v, want nil", err)
}
if first.Spell != "Cure Wounds" || second.Spell != "Fire Bolt" {
t.Fatalf("candidate order = %q, %q; want response order", first.Spell, second.Spell)
if len(payload.SpellCasts) != 2 || payload.SpellCasts[0].Spell != "Cure Wounds" || payload.SpellCasts[1].Spell != "Fire Bolt" {
t.Fatalf("spell order = %#v, want response order", payload.SpellCasts)
}
}
func TestExtractCopiesCandidateSourceRefs(t *testing.T) {
func TestExtractDefensivelyCopiesRawContent(t *testing.T) {
client := &fakeSpellsLLMClient{
response: extractionResponse{
SpellCasts: []spellCastResponse{
@@ -308,8 +294,12 @@ func TestExtractCopiesCandidateSourceRefs(t *testing.T) {
}
client.response.SpellCasts[0].SourceRefs[0].StartUnitID = dnd.UnitRefFromInt(99)
if got := result.Candidates[0].SourceRefs[0].StartUnitID; got != 1 {
t.Fatalf("candidate source ref start = %d, want copied 1", got)
var payload extractionResponse
if err := json.Unmarshal(result.Output.Payload.Content, &payload); err != nil {
t.Fatalf("Unmarshal(Content) error = %v, want nil", err)
}
if got := payload.SpellCasts[0].SourceRefs[0].StartUnitID.String(); got != "1" {
t.Fatalf("source ref start = %q, want copied 1", got)
}
}

6
internal/modules/extract/dnd/spells/registry_test.go
View File

@@ -17,12 +17,6 @@ func TestNewReturnsExtractorWithMetadata(t *testing.T) {
if extractor.Key() != Key {
t.Fatalf("extractor.Key() = %q, want %q", extractor.Key(), Key)
}
if extractor.ArtifactType() != ArtifactType {
t.Fatalf("extractor.ArtifactType() = %q, want %q", extractor.ArtifactType(), ArtifactType)
}
if extractor.SchemaVersion() != SchemaVersion {
t.Fatalf("extractor.SchemaVersion() = %q, want %q", extractor.SchemaVersion(), SchemaVersion)
}
}
func TestModuleSpec(t *testing.T) {

79
internal/modules/extract/dnd/spells/runner_test.go
View File

@@ -48,31 +48,30 @@ func TestRunnerProcessesSeriatimInputWithDNDSpellsExtractor(t *testing.T) {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.Approved) != 2 {
t.Fatalf("len(Approved) = %d, want 2", len(output.Approved))
if len(output.NormalizeOutputs) != 1 {
t.Fatalf("len(NormalizeOutputs) = %d, want 1", len(output.NormalizeOutputs))
}
var first, second SpellCast
if err := json.Unmarshal(output.Approved[0].Payload, &first); err != nil {
t.Fatalf("Unmarshal(first payload) error = %v, want nil", err)
rawOutput := output.NormalizeOutputs[0]
if rawOutput.LaneID != "spells" || rawOutput.Schema.ID != ResponseSchemaID || rawOutput.Schema.Version != SchemaVersion {
t.Fatalf("raw output envelope = %#v, want dnd spells schema on spells lane", rawOutput)
}
if err := json.Unmarshal(output.Approved[1].Payload, &second); err != nil {
t.Fatalf("Unmarshal(second payload) error = %v, want nil", err)
response := decodeRunnerSpellResponse(t, rawOutput.Payload.Content)
if len(response.SpellCasts) != 2 {
t.Fatalf("len(spell_casts) = %d, want 2", len(response.SpellCasts))
}
first, second := response.SpellCasts[0], response.SpellCasts[1]
if first.Spell != "Cure Wounds" || second.Spell != "Fire Bolt" {
t.Fatalf("approved spell order = %q, %q; want response order", first.Spell, second.Spell)
t.Fatalf("spell order = %q, %q; want response order", first.Spell, second.Spell)
}
if first.Caster != "Aria" || second.Caster != "Borin" {
t.Fatalf("approved casters = %q, %q; want spell data", first.Caster, second.Caster)
t.Fatalf("casters = %q, %q; want spell data", first.Caster, second.Caster)
}
for _, artifact := range output.Approved {
if artifact.ExtractorKey != Key || artifact.ArtifactType != ArtifactType || artifact.SchemaVersion != SchemaVersion {
t.Fatalf("approved artifact envelope = %#v, want dnd spells envelope", artifact)
for _, spell := range response.SpellCasts {
if len(spell.SourceRefs) != 1 {
t.Fatalf("len(SourceRefs) = %d, want 1", len(spell.SourceRefs))
}
if len(artifact.SourceRefs) != 1 {
t.Fatalf("len(SourceRefs) = %d, want 1", len(artifact.SourceRefs))
}
if err := source.ValidateRef(expectedDoc, artifact.SourceRefs[0]); err != nil {
t.Fatalf("ValidateRef() error = %v, want nil", err)
if spell.SourceRefs[0].SourceID != expectedDoc.ID {
t.Fatalf("SourceID = %q, want fixture document ID", spell.SourceRefs[0].SourceID)
}
}
@@ -137,8 +136,8 @@ func TestRunnerPassesPartyAndGlossaryReferencesToDNDSpellsPrompt(t *testing.T) {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.Approved) != 1 {
t.Fatalf("len(Approved) = %d, want 1", len(output.Approved))
if len(output.NormalizeOutputs) != 1 {
t.Fatalf("len(NormalizeOutputs) = %d, want 1", len(output.NormalizeOutputs))
}
if len(output.Manifest.References) != 2 {
t.Fatalf("manifest references = %#v, want party and glossary provenance", output.Manifest.References)
@@ -178,8 +177,12 @@ func TestRunnerDoesNotExtractSpellMentionedOnlyInPartyReference(t *testing.T) {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.Approved) != 0 {
t.Fatalf("approved artifacts = %#v, want no party-reference-only spell casts", output.Approved)
if len(output.NormalizeOutputs) != 1 {
t.Fatalf("len(NormalizeOutputs) = %d, want empty spell response output", len(output.NormalizeOutputs))
}
response := decodeRunnerSpellResponse(t, output.NormalizeOutputs[0].Payload.Content)
if len(response.SpellCasts) != 0 {
t.Fatalf("spell_casts = %#v, want no party-reference-only spell casts", response.SpellCasts)
}
if len(llmClient.requests) != 1 {
t.Fatalf("LLM calls = %d, want 1", len(llmClient.requests))
@@ -196,7 +199,7 @@ func TestRunnerDoesNotExtractSpellMentionedOnlyInPartyReference(t *testing.T) {
}
}
func TestRunnerRejectsDNDSpellCastWithInvalidSourceRef(t *testing.T) {
func TestRunnerCarriesDNDSpellCastWithInvalidSourceRefAsRawOutput(t *testing.T) {
raw := readDNDSpellsFixture(t)
resolved := resolveDNDSpellsPipeline(t)
llmClient := &fakeSpellsLLMClient{
@@ -221,21 +224,21 @@ func TestRunnerRejectsDNDSpellCastWithInvalidSourceRef(t *testing.T) {
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.Approved) != 0 {
t.Fatalf("len(Approved) = %d, want 0", len(output.Approved))
if len(output.NormalizeOutputs) != 1 {
t.Fatalf("len(NormalizeOutputs) = %d, want 1", len(output.NormalizeOutputs))
}
if len(output.Rejected) != 1 {
t.Fatalf("len(Rejected) = %d, want 1", len(output.Rejected))
response := decodeRunnerSpellResponse(t, output.NormalizeOutputs[0].Payload.Content)
if len(response.SpellCasts) != 1 {
t.Fatalf("len(spell_casts) = %d, want 1", len(response.SpellCasts))
}
rejected := output.Rejected[0]
if rejected.ValidatorName != sourceRefValidatorName {
t.Fatalf("ValidatorName = %q, want %q", rejected.ValidatorName, sourceRefValidatorName)
if response.SpellCasts[0].SourceRefs[0].SourceID != "spell-session" {
t.Fatalf("SourceID = %q, want raw invalid source ref preserved", response.SpellCasts[0].SourceRefs[0].SourceID)
}
if rejected.ReasonCode != reasonInvalidSourceRef {
t.Fatalf("ReasonCode = %q, want %q", rejected.ReasonCode, reasonInvalidSourceRef)
if len(output.Rejected) != 0 {
t.Fatalf("len(Rejected) = %d, want 0", len(output.Rejected))
}
if output.Manifest.ValidationStatus != "rejected" {
t.Fatalf("ValidationStatus = %q, want rejected", output.Manifest.ValidationStatus)
if output.Manifest.ValidationStatus != "approved" {
t.Fatalf("ValidationStatus = %q, want approved", output.Manifest.ValidationStatus)
}
}
@@ -345,3 +348,13 @@ func parseDNDSpellsFixture(t *testing.T, raw []byte) *source.SourceDocument {
}
return doc
}
func decodeRunnerSpellResponse(t *testing.T, raw []byte) extractionResponse {
t.Helper()
var response extractionResponse
if err := json.Unmarshal(raw, &response); err != nil {
t.Fatalf("Unmarshal(raw output) error = %v, want nil", err)
}
return response
}

20
internal/modules/extract/dnd/spells/validator_test.go
View File

@@ -12,26 +12,6 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/validate"
)
func TestExtractorValidatorsReturnsExpectedChain(t *testing.T) {
validators := New().Validators()
if len(validators) != 2 {
t.Fatalf("len(Validators()) = %d, want 2", len(validators))
}
if validators[0].Name() != shapeValidatorName {
t.Fatalf("Validators()[0].Name() = %q, want %q", validators[0].Name(), shapeValidatorName)
}
if validators[1].Name() != sourceRefValidatorName {
t.Fatalf("Validators()[1].Name() = %q, want %q", validators[1].Name(), sourceRefValidatorName)
}
validators[0] = nil
again := New().Validators()
if len(again) != 2 || again[0] == nil || again[0].Name() != shapeValidatorName {
t.Fatalf("Validators() after caller mutation = %#v, want fresh validators", again)
}
}
func TestValidatorsApproveValidCandidate(t *testing.T) {
candidate := validSpellCandidate(7)

6
internal/modules/input/seriatim/config_test.go
View File

@@ -219,14 +219,8 @@ type fakeExtractor struct{}
func (fakeExtractor) Key() string { return "fake/extract" }
func (fakeExtractor) ArtifactType() string { return "fake" }
func (fakeExtractor) SchemaVersion() string { return "v1" }
func (fakeExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (fakeExtractor) Validators() []contracts.Validator { return nil }
func (fakeExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
return contracts.ExtractionResult{}, nil
}

72
internal/modules/input/seriatim/runner_test.go
View File

@@ -7,7 +7,6 @@ import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
@@ -43,22 +42,29 @@ func TestRunnerProcessesSeriatimInputWithFakeModules(t *testing.T) {
if got := output.Manifest.SourceDigests; len(got) != 1 || got[0] != expectedDoc.Digest {
t.Fatalf("manifest source digests = %#v, want %q", got, expectedDoc.Digest)
}
if len(output.Approved) != 1 {
t.Fatalf("len(Approved) = %d, want 1", len(output.Approved))
if len(output.NormalizeOutputs) != 1 {
t.Fatalf("len(NormalizeOutputs) = %d, want 1", len(output.NormalizeOutputs))
}
artifact := output.Approved[0]
if artifact.ExtractorKey != "fake/extract" || artifact.ArtifactType != "fake.event" || artifact.SchemaVersion != "v1" {
t.Fatalf("approved artifact envelope = %#v, want fake extractor envelope", artifact)
rawOutput := output.NormalizeOutputs[0]
if rawOutput.LaneID != "events" || rawOutput.NormalizerKey != pipeline.DefaultNormalizeModule || rawOutput.Schema.ID != "fake.event" || rawOutput.Schema.Version != "v1" {
t.Fatalf("raw output envelope = %#v, want fake extractor envelope", rawOutput)
}
if len(artifact.SourceRefs) != 1 {
t.Fatalf("len(SourceRefs) = %d, want 1", len(artifact.SourceRefs))
var payload struct {
Value string `json:"value"`
SourceRefs []source.SourceRef `json:"source_refs"`
}
if err := source.ValidateRef(expectedDoc, artifact.SourceRefs[0]); err != nil {
if err := json.Unmarshal(rawOutput.Payload.Content, &payload); err != nil {
t.Fatalf("Unmarshal(raw output) error = %v, want nil", err)
}
if len(payload.SourceRefs) != 1 {
t.Fatalf("len(SourceRefs) = %d, want 1", len(payload.SourceRefs))
}
if err := source.ValidateRef(expectedDoc, payload.SourceRefs[0]); err != nil {
t.Fatalf("ValidateRef() error = %v, want nil", err)
}
if artifact.SourceRefs[0].StartUnitID != 1 || artifact.SourceRefs[0].EndUnitID != 2 {
t.Fatalf("SourceRefs[0] = %#v, want Seriatim unit IDs", artifact.SourceRefs[0])
if payload.SourceRefs[0].StartUnitID != 1 || payload.SourceRefs[0].EndUnitID != 2 {
t.Fatalf("SourceRefs[0] = %#v, want Seriatim unit IDs", payload.SourceRefs[0])
}
if extractor.calls != 1 {
t.Fatalf("extractor calls = %d, want 1", extractor.calls)
@@ -186,22 +192,10 @@ func (e *runnerSeriatimExtractor) Key() string {
return "fake/extract"
}
func (e *runnerSeriatimExtractor) ArtifactType() string {
return "fake.event"
}
func (e *runnerSeriatimExtractor) SchemaVersion() string {
return "v1"
}
func (e *runnerSeriatimExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (e *runnerSeriatimExtractor) Validators() []contracts.Validator {
return nil
}
func (e *runnerSeriatimExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
e.calls++
if req.Source == nil {
@@ -228,17 +222,29 @@ func (e *runnerSeriatimExtractor) Extract(ctx context.Context, req contracts.Ext
}
}
return contracts.ExtractionResult{
Candidates: []artifacts.ArtifactCandidate{
payload, err := json.Marshal(struct {
Value string `json:"value"`
SourceRefs []source.SourceRef `json:"source_refs"`
}{
Value: "seriatim-source-ref",
SourceRefs: []source.SourceRef{
{
Payload: json.RawMessage(`{"value":"seriatim-source-ref"}`),
SourceRefs: []source.SourceRef{
{
SourceID: req.Source.ID,
StartUnitID: req.Chunk.Units[0].ID,
EndUnitID: req.Chunk.Units[len(req.Chunk.Units)-1].ID,
},
},
SourceID: req.Source.ID,
StartUnitID: req.Chunk.Units[0].ID,
EndUnitID: req.Chunk.Units[len(req.Chunk.Units)-1].ID,
},
},
})
if err != nil {
return contracts.ExtractionResult{}, err
}
return contracts.ExtractionResult{
Output: contracts.ExtractOutput{
Schema: contracts.ResponseSchema{ID: "fake.event", Name: "fake_event", Version: "v1"},
Payload: contracts.RawPayload{
Content: payload,
MediaType: "application/json",
},
},
}, nil

85
internal/modules/merge/appendorder/merger.go
View File

@@ -5,8 +5,6 @@ import (
"encoding/json"
"fmt"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
@@ -36,11 +34,34 @@ func (m *Merger) Merge(ctx context.Context, req contracts.MergeRequest) (contrac
return contracts.MergeResult{}, mergerErrorf("context error before merge: %w", err)
}
var candidates []artifacts.ArtifactCandidate
for _, chunkArtifacts := range req.ChunkArtifacts {
candidates = append(candidates, cloneCandidates(chunkArtifacts.Candidates)...)
if len(req.ExtractOutputs) == 1 {
payload := cloneRawPayload(req.ExtractOutputs[0].Payload)
return contracts.MergeResult{
Output: contracts.MergeOutput{
LaneID: req.LaneID,
MergerKey: Key,
SourceID: req.ExtractOutputs[0].SourceID,
Schema: req.ExtractOutputs[0].Schema,
Payload: payload,
},
}, nil
}
return contracts.MergeResult{Candidates: candidates}, nil
content, err := orderedContent(req.ExtractOutputs)
if err != nil {
return contracts.MergeResult{}, err
}
return contracts.MergeResult{
Output: contracts.MergeOutput{
LaneID: req.LaneID,
MergerKey: Key,
SourceID: sourceID(req.ExtractOutputs),
Payload: contracts.RawPayload{
Content: content,
MediaType: "application/json",
},
},
}, nil
}
func ModuleSpec() pipeline.ModuleSpec {
@@ -57,27 +78,45 @@ func Register(registry *pipeline.MergerRegistry) error {
})
}
func cloneCandidates(candidates []artifacts.ArtifactCandidate) []artifacts.ArtifactCandidate {
if len(candidates) == 0 {
return nil
func orderedContent(outputs []contracts.ExtractOutput) ([]byte, error) {
items := make([]map[string]any, 0, len(outputs))
for _, output := range outputs {
item := map[string]any{
"chunk_id": output.ChunkID,
"chunk_index": output.ChunkIndex,
"media_type": output.Payload.MediaType,
}
if json.Valid(output.Payload.Content) {
item["content"] = json.RawMessage(append([]byte(nil), output.Payload.Content...))
} else {
item["content"] = string(output.Payload.Content)
}
items = append(items, item)
}
out := make([]artifacts.ArtifactCandidate, 0, len(candidates))
for _, candidate := range candidates {
out = append(out, cloneCandidate(candidate))
content, err := json.Marshal(struct {
Outputs []map[string]any `json:"outputs"`
}{Outputs: items})
if err != nil {
return nil, mergerErrorf("encode merged output: %w", err)
}
return out
return content, nil
}
func cloneCandidate(candidate artifacts.ArtifactCandidate) artifacts.ArtifactCandidate {
return artifacts.ArtifactCandidate{
Index: candidate.Index,
ExtractorKey: candidate.ExtractorKey,
ArtifactType: candidate.ArtifactType,
SchemaVersion: candidate.SchemaVersion,
Payload: append(json.RawMessage(nil), candidate.Payload...),
SourceRefs: append([]source.SourceRef(nil), candidate.SourceRefs...),
Metadata: cloneMetadata(candidate.Metadata),
func sourceID(outputs []contracts.ExtractOutput) string {
for _, output := range outputs {
if output.SourceID != "" {
return output.SourceID
}
}
return ""
}
func cloneRawPayload(payload contracts.RawPayload) contracts.RawPayload {
return contracts.RawPayload{
Content: append([]byte(nil), payload.Content...),
MediaType: payload.MediaType,
Metadata: cloneMetadata(payload.Metadata),
Warnings: append([]contracts.Warning(nil), payload.Warnings...),
}
}

149
internal/modules/merge/appendorder/merger_test.go
View File

@@ -6,8 +6,6 @@ import (
"reflect"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
@@ -35,113 +33,102 @@ func TestModuleSpecAndRegister(t *testing.T) {
}
}
func TestMergePreservesChunkAndCandidateOrder(t *testing.T) {
func TestMergePassesThroughSingleExtractOutput(t *testing.T) {
input := extractOutput("chunk-0", 0, `{"name":"original"}`)
result, err := New().Merge(context.Background(), contracts.MergeRequest{
ChunkArtifacts: []contracts.ChunkArtifacts{
{
Chunk: sourceChunk(0),
Candidates: []artifacts.ArtifactCandidate{candidate(2, "first-b"), candidate(1, "first-a")},
},
{
Chunk: sourceChunk(1),
Candidates: []artifacts.ArtifactCandidate{candidate(4, "second-b"), candidate(3, "second-a")},
},
},
LaneID: "events",
ExtractOutputs: []contracts.ExtractOutput{input},
})
if err != nil {
t.Fatalf("Merge() error = %v, want nil", err)
}
got := candidateNames(result.Candidates)
want := []string{"first-b", "first-a", "second-b", "second-a"}
if !reflect.DeepEqual(got, want) {
t.Fatalf("candidate order = %#v, want %#v", got, want)
if result.Output.LaneID != "events" || result.Output.MergerKey != Key {
t.Fatalf("output provenance = %#v, want lane and merger", result.Output)
}
if len(result.Warnings) != 0 {
t.Fatalf("Warnings = %#v, want none", result.Warnings)
if string(result.Output.Payload.Content) != `{"name":"original"}` {
t.Fatalf("content = %s, want original content", result.Output.Payload.Content)
}
if result.Output.Payload.Metadata["name"] != "chunk-0" {
t.Fatalf("metadata = %#v, want original metadata", result.Output.Payload.Metadata)
}
}
func TestMergeDefensivelyCopiesCandidates(t *testing.T) {
input := []contracts.ChunkArtifacts{
{
Chunk: sourceChunk(0),
Candidates: []artifacts.ArtifactCandidate{candidate(1, "original")},
},
}
func TestMergeDefensivelyCopiesRawPayload(t *testing.T) {
input := extractOutput("chunk-0", 0, `{"name":"original"}`)
result, err := New().Merge(context.Background(), contracts.MergeRequest{ChunkArtifacts: input})
result, err := New().Merge(context.Background(), contracts.MergeRequest{
LaneID: "events",
ExtractOutputs: []contracts.ExtractOutput{input},
})
if err != nil {
t.Fatalf("Merge() error = %v, want nil", err)
}
if len(result.Candidates) != 1 {
t.Fatalf("len(Candidates) = %d, want 1", len(result.Candidates))
input.Payload.Content[0] = '['
input.Payload.Metadata["name"] = "changed"
if string(result.Output.Payload.Content) != `{"name":"original"}` {
t.Fatalf("content changed after input mutation: %s", result.Output.Payload.Content)
}
if result.Output.Payload.Metadata["name"] != "chunk-0" {
t.Fatalf("metadata changed after input mutation: %#v", result.Output.Payload.Metadata)
}
}
func TestMergeWrapsMultipleOutputsInChunkOrder(t *testing.T) {
result, err := New().Merge(context.Background(), contracts.MergeRequest{
LaneID: "events",
ExtractOutputs: []contracts.ExtractOutput{
extractOutput("chunk-0", 0, `{"name":"first"}`),
extractOutput("chunk-1", 1, `{"name":"second"}`),
},
})
if err != nil {
t.Fatalf("Merge() error = %v, want nil", err)
}
if result.Output.Payload.MediaType != "application/json" {
t.Fatalf("MediaType = %q, want application/json", result.Output.Payload.MediaType)
}
input[0].Candidates[0].Index = 99
input[0].Candidates[0].Payload[0] = '['
input[0].Candidates[0].SourceRefs[0].StartUnitID = 99
input[0].Candidates[0].Metadata["name"] = "changed"
got := result.Candidates[0]
if got.Index != 1 {
t.Fatalf("Index = %d, want 1", got.Index)
var decoded struct {
Outputs []struct {
ChunkID string `json:"chunk_id"`
ChunkIndex int `json:"chunk_index"`
Content json.RawMessage `json:"content"`
} `json:"outputs"`
}
if string(got.Payload) != `{"name":"original"}` {
t.Fatalf("Payload = %s, want original payload", got.Payload)
if err := json.Unmarshal(result.Output.Payload.Content, &decoded); err != nil {
t.Fatalf("Unmarshal() error = %v, want nil", err)
}
if got.SourceRefs[0].StartUnitID != 1 {
t.Fatalf("SourceRefs = %#v, want original source ref", got.SourceRefs)
}
if got.Metadata["name"] != "original" {
t.Fatalf("Metadata = %#v, want original metadata", got.Metadata)
if len(decoded.Outputs) != 2 || decoded.Outputs[0].ChunkID != "chunk-0" || decoded.Outputs[1].ChunkID != "chunk-1" {
t.Fatalf("outputs = %#v, want chunk order", decoded.Outputs)
}
}
func TestMergeHandlesEmptyInput(t *testing.T) {
result, err := New().Merge(context.Background(), contracts.MergeRequest{})
result, err := New().Merge(context.Background(), contracts.MergeRequest{LaneID: "events"})
if err != nil {
t.Fatalf("Merge() error = %v, want nil", err)
}
if len(result.Candidates) != 0 {
t.Fatalf("len(Candidates) = %d, want 0", len(result.Candidates))
}
if len(result.Warnings) != 0 {
t.Fatalf("Warnings = %#v, want none", result.Warnings)
if string(result.Output.Payload.Content) != `{"outputs":[]}` {
t.Fatalf("content = %s, want empty outputs", result.Output.Payload.Content)
}
}
func candidate(index int, name string) artifacts.ArtifactCandidate {
return artifacts.ArtifactCandidate{
Index: index,
ExtractorKey: "generic-extractor",
ArtifactType: "generic-artifact",
SchemaVersion: "v1",
Payload: json.RawMessage(`{"name":"` + name + `"}`),
SourceRefs: []source.SourceRef{
{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1},
},
Metadata: map[string]any{
"name": name,
},
}
}
func candidateNames(candidates []artifacts.ArtifactCandidate) []string {
names := make([]string, 0, len(candidates))
for _, candidate := range candidates {
names = append(names, candidate.Metadata["name"].(string))
}
return names
}
func sourceChunk(index int) contracts.SourceChunk {
return contracts.SourceChunk{
ID: "chunk",
SourceID: "source-1",
Index: index,
Units: []source.SourceUnit{
{ID: 1, Kind: "unit", Text: "Source unit."},
func extractOutput(chunkID string, chunkIndex int, content string) contracts.ExtractOutput {
return contracts.ExtractOutput{
LaneID: "events",
ExtractorKey: "extract",
SourceID: "source-1",
ChunkID: chunkID,
ChunkIndex: chunkIndex,
Schema: contracts.ResponseSchema{ID: "schema-id", Name: "schema-name", Version: "v1"},
Payload: contracts.RawPayload{
Content: []byte(content),
MediaType: "application/json",
Metadata: map[string]any{"name": chunkID},
},
}
}

36
internal/modules/normalize/noop/normalizer.go
View File

@@ -2,11 +2,8 @@ package noop
import (
"context"
"encoding/json"
"fmt"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
@@ -39,7 +36,15 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.NormalizeReque
if err := ctx.Err(); err != nil {
return contracts.NormalizeResult{}, normalizerErrorf("context error before normalize: %w", err)
}
return contracts.NormalizeResult{Candidates: cloneCandidates(req.Candidates)}, nil
return contracts.NormalizeResult{
Output: contracts.NormalizeOutput{
LaneID: req.LaneID,
NormalizerKey: Key,
SourceID: req.MergeOutput.SourceID,
Schema: req.MergeOutput.Schema,
Payload: cloneRawPayload(req.MergeOutput.Payload),
},
}, nil
}
func ModuleSpec() pipeline.ModuleSpec {
@@ -57,24 +62,13 @@ func Register(registry *pipeline.NormalizerRegistry) error {
})
}
func cloneCandidates(candidates []artifacts.ArtifactCandidate) []artifacts.ArtifactCandidate {
if len(candidates) == 0 {
return nil
func cloneRawPayload(payload contracts.RawPayload) contracts.RawPayload {
return contracts.RawPayload{
Content: append([]byte(nil), payload.Content...),
MediaType: payload.MediaType,
Metadata: cloneMetadata(payload.Metadata),
Warnings: append([]contracts.Warning(nil), payload.Warnings...),
}
out := make([]artifacts.ArtifactCandidate, 0, len(candidates))
for _, candidate := range candidates {
out = append(out, artifacts.ArtifactCandidate{
Index: candidate.Index,
ExtractorKey: candidate.ExtractorKey,
ArtifactType: candidate.ArtifactType,
SchemaVersion: candidate.SchemaVersion,
Payload: append(json.RawMessage(nil), candidate.Payload...),
SourceRefs: append([]source.SourceRef(nil), candidate.SourceRefs...),
Metadata: cloneMetadata(candidate.Metadata),
})
}
return out
}
func cloneMetadata(metadata map[string]any) map[string]any {

109
internal/modules/normalize/noop/normalizer_test.go
View File

@@ -2,12 +2,9 @@ package noop
import (
"context"
"encoding/json"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
@@ -43,98 +40,60 @@ func TestModuleSpecAndRegister(t *testing.T) {
}
}
func TestNormalizePassesThroughOrderAndValues(t *testing.T) {
input := []artifacts.ArtifactCandidate{
candidate(3, "third"),
candidate(1, "first"),
candidate(2, "second"),
}
func TestNormalizePassesThroughMergeOutput(t *testing.T) {
input := mergeOutput(`{"name":"original"}`)
result, err := New().Normalize(context.Background(), contracts.NormalizeRequest{Candidates: input})
result, err := New().Normalize(context.Background(), contracts.NormalizeRequest{
LaneID: "events",
MergeOutput: input,
})
if err != nil {
t.Fatalf("Normalize() error = %v, want nil", err)
}
if len(result.Warnings) != 0 {
t.Fatalf("Warnings = %#v, want none", result.Warnings)
}
got := candidateNames(result.Candidates)
want := []string{"third", "first", "second"}
if !reflect.DeepEqual(got, want) {
t.Fatalf("candidate order = %#v, want %#v", got, want)
if result.Output.LaneID != "events" || result.Output.NormalizerKey != Key {
t.Fatalf("output provenance = %#v, want lane and normalizer", result.Output)
}
if !reflect.DeepEqual(result.Candidates[0].SourceRefs, input[0].SourceRefs) {
t.Fatalf("SourceRefs = %#v, want %#v", result.Candidates[0].SourceRefs, input[0].SourceRefs)
if string(result.Output.Payload.Content) != `{"name":"original"}` {
t.Fatalf("content = %s, want original content", result.Output.Payload.Content)
}
if !reflect.DeepEqual(result.Candidates[0].Metadata, input[0].Metadata) {
t.Fatalf("Metadata = %#v, want %#v", result.Candidates[0].Metadata, input[0].Metadata)
if result.Output.Payload.Metadata["name"] != "original" {
t.Fatalf("metadata = %#v, want original metadata", result.Output.Payload.Metadata)
}
}
func TestNormalizeDefensivelyCopiesCandidates(t *testing.T) {
input := []artifacts.ArtifactCandidate{candidate(1, "original")}
func TestNormalizeDefensivelyCopiesRawPayload(t *testing.T) {
input := mergeOutput(`{"name":"original"}`)
result, err := New().Normalize(context.Background(), contracts.NormalizeRequest{Candidates: input})
result, err := New().Normalize(context.Background(), contracts.NormalizeRequest{
LaneID: "events",
MergeOutput: input,
})
if err != nil {
t.Fatalf("Normalize() error = %v, want nil", err)
}
if len(result.Candidates) != 1 {
t.Fatalf("len(Candidates) = %d, want 1", len(result.Candidates))
}
input[0].Index = 99
input[0].Payload[0] = '['
input[0].SourceRefs[0].EndUnitID = 99
input[0].Metadata["name"] = "changed"
input.Payload.Content[0] = '['
input.Payload.Metadata["name"] = "changed"
got := result.Candidates[0]
if got.Index != 1 {
t.Fatalf("Index = %d, want 1", got.Index)
if string(result.Output.Payload.Content) != `{"name":"original"}` {
t.Fatalf("content changed after input mutation: %s", result.Output.Payload.Content)
}
if string(got.Payload) != `{"name":"original"}` {
t.Fatalf("Payload = %s, want original payload", got.Payload)
}
if got.SourceRefs[0].EndUnitID != 1 {
t.Fatalf("SourceRefs = %#v, want original source ref", got.SourceRefs)
}
if got.Metadata["name"] != "original" {
t.Fatalf("Metadata = %#v, want original metadata", got.Metadata)
if result.Output.Payload.Metadata["name"] != "original" {
t.Fatalf("metadata changed after input mutation: %#v", result.Output.Payload.Metadata)
}
}
func TestNormalizeHandlesEmptyInput(t *testing.T) {
result, err := New().Normalize(context.Background(), contracts.NormalizeRequest{})
if err != nil {
t.Fatalf("Normalize() error = %v, want nil", err)
}
if len(result.Candidates) != 0 {
t.Fatalf("len(Candidates) = %d, want 0", len(result.Candidates))
}
if len(result.Warnings) != 0 {
t.Fatalf("Warnings = %#v, want none", result.Warnings)
}
}
func candidate(index int, name string) artifacts.ArtifactCandidate {
return artifacts.ArtifactCandidate{
Index: index,
ExtractorKey: "generic-extractor",
ArtifactType: "generic-artifact",
SchemaVersion: "v1",
Payload: json.RawMessage(`{"name":"` + name + `"}`),
SourceRefs: []source.SourceRef{
{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1},
},
Metadata: map[string]any{
"name": name,
func mergeOutput(content string) contracts.MergeOutput {
return contracts.MergeOutput{
LaneID: "events",
MergerKey: "merge",
SourceID: "source-1",
Schema: contracts.ResponseSchema{ID: "schema-id", Name: "schema-name", Version: "v1"},
Payload: contracts.RawPayload{
Content: []byte(content),
MediaType: "application/json",
Metadata: map[string]any{"name": "original"},
},
}
}
func candidateNames(candidates []artifacts.ArtifactCandidate) []string {
names := make([]string, 0, len(candidates))
for _, candidate := range candidates {
names = append(names, candidate.Metadata["name"].(string))
}
return names
}

169
internal/modules/output/json/encoder.go
View File

@@ -8,8 +8,6 @@ import (
"sort"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
@@ -18,7 +16,7 @@ const Key = "json"
const contentTypeJSON = "application/json"
var safeArtifactFileChar = regexp.MustCompile(`[^A-Za-z0-9._-]`)
var safeOutputFileChar = regexp.MustCompile(`[^A-Za-z0-9._-]`)
var _ contracts.OutputEncoder = (*Encoder)(nil)
@@ -66,24 +64,24 @@ func Register(registry *pipeline.OutputEncoderRegistry) error {
}
type indexFile struct {
ManifestFile string `json:"manifest_file"`
ArtifactFiles []artifactFileIndex `json:"artifact_files"`
RejectedFile string `json:"rejected_file"`
WarningsFile string `json:"warnings_file"`
ManifestFile string `json:"manifest_file"`
OutputFiles []outputFileIndex `json:"output_files"`
RejectedFile string `json:"rejected_file"`
WarningsFile string `json:"warnings_file"`
}
type artifactFileIndex struct {
ArtifactType string `json:"artifact_type"`
File string `json:"file"`
}
type artifactFile struct {
ArtifactType string `json:"artifact_type"`
Artifacts []artifacts.Artifact `json:"artifacts"`
type outputFileIndex struct {
LaneID string `json:"lane_id"`
MediaType string `json:"media_type,omitempty"`
File string `json:"file"`
ModuleKey string `json:"module_key,omitempty"`
SchemaID string `json:"schema_id,omitempty"`
SchemaName string `json:"schema_name,omitempty"`
SchemaVer string `json:"schema_version,omitempty"`
}
type rejectedFile struct {
Rejected []artifacts.RejectedArtifact `json:"rejected"`
Rejected []contracts.RejectedOutput `json:"rejected"`
}
type warningsFile struct {
@@ -91,43 +89,39 @@ type warningsFile struct {
}
func logicalFiles(req contracts.OutputRequest) ([]contracts.OutputFile, error) {
artifactsByType := make(map[string][]artifacts.Artifact)
for _, artifact := range req.Approved {
artifactsByType[artifact.ArtifactType] = append(artifactsByType[artifact.ArtifactType], cloneArtifact(artifact))
}
outputs := cloneNormalizeOutputs(req.NormalizeOutputs)
sort.SliceStable(outputs, func(i, j int) bool {
return outputs[i].LaneID < outputs[j].LaneID
})
artifactTypes := make([]string, 0, len(artifactsByType))
for artifactType := range artifactsByType {
artifactTypes = append(artifactTypes, artifactType)
}
sort.Strings(artifactTypes)
artifactIndexes := make([]artifactFileIndex, 0, len(artifactTypes))
files := make([]contracts.OutputFile, 0, len(artifactTypes)+4)
outputIndexes := make([]outputFileIndex, 0, len(outputs))
files := make([]contracts.OutputFile, 0, len(outputs)+4)
manifestFile, err := jsonFile("manifest.json", req.Manifest)
if err != nil {
return nil, err
}
files = append(files, manifestFile)
usedArtifactFiles := make(map[string]string, len(artifactTypes))
for _, artifactType := range artifactTypes {
name, err := artifactFileName(artifactType)
usedOutputFiles := make(map[string]string, len(outputs))
for _, output := range outputs {
name, err := outputFileName(output.LaneID)
if err != nil {
return nil, err
}
if existingType, ok := usedArtifactFiles[name]; ok {
return nil, encoderErrorf("artifact types %q and %q produce duplicate output file %q", existingType, artifactType, name)
if existingLane, ok := usedOutputFiles[name]; ok {
return nil, encoderErrorf("lanes %q and %q produce duplicate output file %q", existingLane, output.LaneID, name)
}
usedArtifactFiles[name] = artifactType
artifactIndexes = append(artifactIndexes, artifactFileIndex{
ArtifactType: artifactType,
File: name,
})
file, err := jsonFile(name, artifactFile{
ArtifactType: artifactType,
Artifacts: artifactsByType[artifactType],
usedOutputFiles[name] = output.LaneID
outputIndexes = append(outputIndexes, outputFileIndex{
LaneID: output.LaneID,
MediaType: output.Payload.MediaType,
File: name,
ModuleKey: output.NormalizerKey,
SchemaID: output.Schema.ID,
SchemaName: output.Schema.Name,
SchemaVer: output.Schema.Version,
})
file, err := rawOutputFile(name, output.Payload)
if err != nil {
return nil, err
}
@@ -135,10 +129,10 @@ func logicalFiles(req contracts.OutputRequest) ([]contracts.OutputFile, error) {
}
index := indexFile{
ManifestFile: "manifest.json",
ArtifactFiles: artifactIndexes,
RejectedFile: "rejected.json",
WarningsFile: "warnings.json",
ManifestFile: "manifest.json",
OutputFiles: outputIndexes,
RejectedFile: "rejected.json",
WarningsFile: "warnings.json",
}
indexOutput, err := jsonFile("index.json", index)
if err != nil {
@@ -159,6 +153,32 @@ func logicalFiles(req contracts.OutputRequest) ([]contracts.OutputFile, error) {
return files, nil
}
func rawOutputFile(name string, payload contracts.RawPayload) (contracts.OutputFile, error) {
content := append([]byte(nil), payload.Content...)
if len(content) == 0 {
content = []byte("null")
}
if payload.MediaType == contentTypeJSON && stdjson.Valid(content) {
var decoded any
if err := stdjson.Unmarshal(content, &decoded); err == nil {
pretty, err := marshalPretty(decoded)
if err != nil {
return contracts.OutputFile{}, err
}
content = pretty
}
}
mediaType := strings.TrimSpace(payload.MediaType)
if mediaType == "" {
mediaType = "application/octet-stream"
}
return contracts.OutputFile{
Name: name,
ContentType: mediaType,
Bytes: append([]byte(nil), content...),
}, nil
}
func jsonFile(name string, value any) (contracts.OutputFile, error) {
data, err := marshalPretty(value)
if err != nil {
@@ -179,57 +199,46 @@ func marshalPretty(value any) ([]byte, error) {
return append(data, '\n'), nil
}
func artifactFileName(artifactType string) (string, error) {
sanitized := safeArtifactFileChar.ReplaceAllString(strings.TrimSpace(artifactType), "_")
func outputFileName(laneID string) (string, error) {
sanitized := safeOutputFileChar.ReplaceAllString(strings.TrimSpace(laneID), "_")
for strings.Contains(sanitized, "..") {
sanitized = strings.ReplaceAll(sanitized, "..", "__")
}
sanitized = strings.Trim(sanitized, "._")
if sanitized == "" {
return "", encoderErrorf("artifact type %q cannot produce a safe file name", artifactType)
return "", encoderErrorf("lane id %q cannot produce a safe file name", laneID)
}
return "artifacts/" + sanitized + ".json", nil
return "outputs/" + sanitized + ".json", nil
}
func cloneArtifact(artifact artifacts.Artifact) artifacts.Artifact {
return artifacts.Artifact{
ExtractorKey: artifact.ExtractorKey,
ArtifactType: artifact.ArtifactType,
SchemaVersion: artifact.SchemaVersion,
Payload: append(stdjson.RawMessage(nil), artifact.Payload...),
SourceRefs: append([]source.SourceRef(nil), artifact.SourceRefs...),
Metadata: cloneMetadata(artifact.Metadata),
func cloneNormalizeOutputs(outputs []contracts.NormalizeOutput) []contracts.NormalizeOutput {
if len(outputs) == 0 {
return nil
}
}
func cloneRejected(rejected []artifacts.RejectedArtifact) []artifacts.RejectedArtifact {
if len(rejected) == 0 {
return []artifacts.RejectedArtifact{}
}
out := make([]artifacts.RejectedArtifact, 0, len(rejected))
for _, item := range rejected {
out = append(out, artifacts.RejectedArtifact{
Candidate: cloneCandidate(item.Candidate),
ValidatorName: item.ValidatorName,
ReasonCode: item.ReasonCode,
Message: item.Message,
})
out := make([]contracts.NormalizeOutput, 0, len(outputs))
for _, output := range outputs {
output.Payload = cloneRawPayload(output.Payload)
out = append(out, output)
}
return out
}
func cloneCandidate(candidate artifacts.ArtifactCandidate) artifacts.ArtifactCandidate {
return artifacts.ArtifactCandidate{
Index: candidate.Index,
ExtractorKey: candidate.ExtractorKey,
ArtifactType: candidate.ArtifactType,
SchemaVersion: candidate.SchemaVersion,
Payload: append(stdjson.RawMessage(nil), candidate.Payload...),
SourceRefs: append([]source.SourceRef(nil), candidate.SourceRefs...),
Metadata: cloneMetadata(candidate.Metadata),
func cloneRawPayload(payload contracts.RawPayload) contracts.RawPayload {
return contracts.RawPayload{
Content: append([]byte(nil), payload.Content...),
MediaType: payload.MediaType,
Metadata: cloneMetadata(payload.Metadata),
Warnings: append([]contracts.Warning(nil), payload.Warnings...),
}
}
func cloneRejected(rejected []contracts.RejectedOutput) []contracts.RejectedOutput {
if len(rejected) == 0 {
return []contracts.RejectedOutput{}
}
return append([]contracts.RejectedOutput(nil), rejected...)
}
func cloneWarnings(warnings []contracts.Warning) []contracts.Warning {
if len(warnings) == 0 {
return []contracts.Warning{}

153
internal/modules/output/json/encoder_test.go
View File

@@ -8,7 +8,6 @@ import (
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
@@ -37,20 +36,24 @@ func TestModuleSpecAndRegister(t *testing.T) {
}
}
func TestEncodeReturnsLogicalFilesGroupedByArtifactType(t *testing.T) {
func TestEncodeReturnsLogicalFilesForNormalizedOutputs(t *testing.T) {
req := contracts.OutputRequest{
Manifest: artifacts.RunManifest{RunID: "run-1", PipelineID: "pipeline-1"},
Approved: []artifacts.Artifact{
artifact("dnd.spell-cast", "first"),
artifact("notes/item", "item"),
artifact("dnd.spell-cast", "second"),
NormalizeOutputs: []contracts.NormalizeOutput{
normalizeOutput("spells", `{"spell_casts":[{"spell":"Cure Wounds"}]}`),
normalizeOutput("notes/items", `{"items":[{"name":"Torch"}]}`),
},
Rejected: []artifacts.RejectedArtifact{
Rejected: []contracts.RejectedOutput{
{
Candidate: candidate("bad type", "bad"),
ValidatorName: "validator",
Stage: "extract",
LaneID: "spells",
ModuleKey: "dnd/spells",
ChunkID: "chunk-1",
ChunkIndex: 1,
ReasonCode: "invalid",
Message: "not accepted",
AttemptCount: 1,
ValidatorName: "validator",
},
},
Warnings: []contracts.Warning{{ReasonCode: "warning", Message: "check source"}},
@@ -62,51 +65,46 @@ func TestEncodeReturnsLogicalFilesGroupedByArtifactType(t *testing.T) {
}
wantNames := []string{
"artifacts/dnd.spell-cast.json",
"artifacts/notes_item.json",
"index.json",
"manifest.json",
"outputs/notes_items.json",
"outputs/spells.json",
"rejected.json",
"warnings.json",
}
if got := outputFileNames(result.Files); !reflect.DeepEqual(got, wantNames) {
t.Fatalf("file names = %#v, want %#v", got, wantNames)
}
for _, file := range result.Files {
if file.ContentType != contentTypeJSON {
t.Fatalf("%s ContentType = %q, want %q", file.Name, file.ContentType, contentTypeJSON)
}
if !strings.HasSuffix(string(file.Bytes), "\n") {
t.Fatalf("%s does not end with newline: %q", file.Name, string(file.Bytes))
}
if !stdjson.Valid(file.Bytes) {
if file.ContentType == contentTypeJSON && !stdjson.Valid(file.Bytes) {
t.Fatalf("%s has invalid JSON: %s", file.Name, file.Bytes)
}
}
spellFile := decodeObject(t, fileBytes(t, result.Files, "artifacts/dnd.spell-cast.json"))
if spellFile["artifact_type"] != "dnd.spell-cast" {
t.Fatalf("artifact_type = %#v, want dnd.spell-cast", spellFile["artifact_type"])
}
spells := spellFile["artifacts"].([]any)
if len(spells) != 2 {
t.Fatalf("len(spells) = %d, want 2", len(spells))
}
firstPayload := spells[0].(map[string]any)["payload"].(map[string]any)
secondPayload := spells[1].(map[string]any)["payload"].(map[string]any)
if firstPayload["name"] != "first" || secondPayload["name"] != "second" {
t.Fatalf("spell order payloads = %#v then %#v, want runner order", firstPayload, secondPayload)
spells := decodeObject(t, fileBytes(t, result.Files, "outputs/spells.json"))
spellCasts := spells["spell_casts"].([]any)
if spellCasts[0].(map[string]any)["spell"] != "Cure Wounds" {
t.Fatalf("spells output = %#v, want raw normalized content", spells)
}
index := decodeObject(t, fileBytes(t, result.Files, "index.json"))
artifactFiles := index["artifact_files"].([]any)
if len(artifactFiles) != 2 {
t.Fatalf("len(index artifact_files) = %d, want 2", len(artifactFiles))
outputFiles := index["output_files"].([]any)
if len(outputFiles) != 2 {
t.Fatalf("len(index output_files) = %d, want 2", len(outputFiles))
}
firstIndex := artifactFiles[0].(map[string]any)
secondIndex := artifactFiles[1].(map[string]any)
if firstIndex["artifact_type"] != "dnd.spell-cast" || secondIndex["artifact_type"] != "notes/item" {
t.Fatalf("artifact_files = %#v, want sorted by artifact type", artifactFiles)
firstIndex := outputFiles[0].(map[string]any)
secondIndex := outputFiles[1].(map[string]any)
if firstIndex["lane_id"] != "notes/items" || secondIndex["lane_id"] != "spells" {
t.Fatalf("output_files = %#v, want sorted by lane id", outputFiles)
}
rejected := decodeObject(t, fileBytes(t, result.Files, "rejected.json"))
if got := rejected["rejected"].([]any); len(got) != 1 {
t.Fatalf("rejected = %#v, want one rejected output", got)
}
}
@@ -179,12 +177,12 @@ func TestEncodeIncludesManifestReferences(t *testing.T) {
}
}
func TestEncodeRejectsArtifactTypeWithoutSafeFileName(t *testing.T) {
func TestEncodeRejectsLaneIDWithoutSafeFileName(t *testing.T) {
_, err := New().Encode(context.Background(), contracts.OutputRequest{
Approved: []artifacts.Artifact{artifact("///", "unsafe")},
NormalizeOutputs: []contracts.NormalizeOutput{normalizeOutput("///", `{"value":true}`)},
})
if err == nil {
t.Fatal("Encode() error = nil, want unsafe artifact type error")
t.Fatal("Encode() error = nil, want unsafe lane id error")
}
if !strings.Contains(err.Error(), "json output encoder") || !strings.Contains(err.Error(), "safe file name") {
t.Fatalf("Encode() error = %q, want safe file name context", err.Error())
@@ -193,22 +191,22 @@ func TestEncodeRejectsArtifactTypeWithoutSafeFileName(t *testing.T) {
func TestEncodeSanitizesParentPathSequences(t *testing.T) {
result, err := New().Encode(context.Background(), contracts.OutputRequest{
Approved: []artifacts.Artifact{artifact("dnd..spell.", "spell")},
NormalizeOutputs: []contracts.NormalizeOutput{normalizeOutput("dnd..spell.", `{"value":true}`)},
})
if err != nil {
t.Fatalf("Encode() error = %v, want nil", err)
}
if got := outputFileNames(result.Files); !containsString(got, "artifacts/dnd__spell.json") {
t.Fatalf("file names = %#v, want sanitized artifact filename", got)
if got := outputFileNames(result.Files); !containsString(got, "outputs/dnd__spell.json") {
t.Fatalf("file names = %#v, want sanitized output filename", got)
}
}
func TestEncodeRejectsSanitizedFilenameCollisions(t *testing.T) {
_, err := New().Encode(context.Background(), contracts.OutputRequest{
Approved: []artifacts.Artifact{
artifact("a/b", "slash"),
artifact("a?b", "question"),
NormalizeOutputs: []contracts.NormalizeOutput{
normalizeOutput("a/b", `{"value":"slash"}`),
normalizeOutput("a?b", `{"value":"question"}`),
},
})
if err == nil {
@@ -222,16 +220,11 @@ func TestEncodeRejectsSanitizedFilenameCollisions(t *testing.T) {
func TestEncodeDoesNotMutateInputs(t *testing.T) {
req := contracts.OutputRequest{
Manifest: artifacts.RunManifest{RunID: "run-1"},
Approved: []artifacts.Artifact{
artifact("dnd.spell", "original"),
NormalizeOutputs: []contracts.NormalizeOutput{
normalizeOutput("spells", `{"name":"original"}`),
},
Rejected: []artifacts.RejectedArtifact{
{
Candidate: candidate("bad", "rejected"),
ValidatorName: "validator",
ReasonCode: "invalid",
Message: "not accepted",
},
Rejected: []contracts.RejectedOutput{
{Stage: "extract", LaneID: "spells", Message: "not accepted"},
},
Warnings: []contracts.Warning{{ReasonCode: "warning", Message: "message"}},
}
@@ -246,14 +239,13 @@ func TestEncodeDoesNotMutateInputs(t *testing.T) {
t.Fatalf("request mutated:\nbefore: %s\nafter: %s", before, after)
}
req.Approved[0].Payload[0] = '['
req.Approved[0].SourceRefs[0].StartUnitID = 99
req.Approved[0].Metadata["name"] = "changed"
req.Rejected[0].Candidate.Payload[0] = '['
req.NormalizeOutputs[0].Payload.Content[0] = '['
req.NormalizeOutputs[0].Payload.Metadata["name"] = "changed"
req.Rejected[0].Message = "changed"
req.Warnings[0].Message = "changed"
if !stdjson.Valid(fileBytes(t, result.Files, "artifacts/dnd.spell.json")) {
t.Fatal("artifact output changed after request mutation")
if !stdjson.Valid(fileBytes(t, result.Files, "outputs/spells.json")) {
t.Fatal("output changed after request mutation")
}
warnings := decodeObject(t, fileBytes(t, result.Files, "warnings.json"))
gotWarnings := warnings["warnings"].([]any)
@@ -262,9 +254,9 @@ func TestEncodeDoesNotMutateInputs(t *testing.T) {
}
}
func TestArtifactFilesDoNotContainWarnings(t *testing.T) {
func TestOutputFilesDoNotContainWarnings(t *testing.T) {
result, err := New().Encode(context.Background(), contracts.OutputRequest{
Approved: []artifacts.Artifact{artifact("dnd.spell", "spell")},
NormalizeOutputs: []contracts.NormalizeOutput{normalizeOutput("spells", `{"spell":"Shield"}`)},
Warnings: []contracts.Warning{
{ReasonCode: "pipeline-warning", Message: "warning"},
},
@@ -273,36 +265,27 @@ func TestArtifactFilesDoNotContainWarnings(t *testing.T) {
t.Fatalf("Encode() error = %v, want nil", err)
}
artifactFile := decodeObject(t, fileBytes(t, result.Files, "artifacts/dnd.spell.json"))
if _, ok := artifactFile["warnings"]; ok {
t.Fatalf("artifact file contains warnings: %#v", artifactFile)
outputFile := decodeObject(t, fileBytes(t, result.Files, "outputs/spells.json"))
if _, ok := outputFile["warnings"]; ok {
t.Fatalf("output file contains warnings: %#v", outputFile)
}
}
func artifact(artifactType, name string) artifacts.Artifact {
return artifacts.Artifact{
ExtractorKey: "extractor",
ArtifactType: artifactType,
SchemaVersion: "v1",
Payload: stdjson.RawMessage(`{"name":"` + name + `"}`),
SourceRefs: []source.SourceRef{
{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1},
func normalizeOutput(laneID string, content string) contracts.NormalizeOutput {
return contracts.NormalizeOutput{
LaneID: laneID,
NormalizerKey: "noop",
SourceID: "source-1",
Schema: contracts.ResponseSchema{
ID: "schema-id",
Name: "schema-name",
Version: "v1",
},
Metadata: map[string]any{"name": name},
}
}
func candidate(artifactType, name string) artifacts.ArtifactCandidate {
return artifacts.ArtifactCandidate{
Index: 1,
ExtractorKey: "extractor",
ArtifactType: artifactType,
SchemaVersion: "v1",
Payload: stdjson.RawMessage(`{"name":"` + name + `"}`),
SourceRefs: []source.SourceRef{
{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1},
Payload: contracts.RawPayload{
Content: []byte(content),
MediaType: contentTypeJSON,
Metadata: map[string]any{"name": laneID},
},
Metadata: map[string]any{"name": name},
}
}