eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update raw output files and manifests

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-07 19:27:28 +00:00
parent aa14faa3cb
commit 7c95791e94
10 changed files with 253 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
internal/modules/output/json/encoder.go
View File

@@ -4,6 +4,7 @@ import (
"context"
stdjson "encoding/json"
"fmt"
"mime"
"regexp"
"sort"
"strings"
@@ -158,27 +159,36 @@ func rawOutputFile(name string, payload contracts.RawPayload) (contracts.OutputF
if len(content) == 0 {
content = []byte("null")
}
if payload.MediaType == contentTypeJSON && stdjson.Valid(content) {
var decoded any
if err := stdjson.Unmarshal(content, &decoded); err == nil {
pretty, err := marshalPretty(decoded)
if err != nil {
return contracts.OutputFile{}, err
}
content = pretty
}
}
mediaType := strings.TrimSpace(payload.MediaType)
if mediaType == "" {
mediaType = "application/octet-stream"
}
if !isJSONMediaType(mediaType) {
return contracts.OutputFile{}, encoderErrorf("normalized output %q has unsupported media type %q", name, mediaType)
}
var decoded any
if err := stdjson.Unmarshal(content, &decoded); err != nil {
return contracts.OutputFile{}, encoderErrorf("normalized output %q contains invalid JSON: %w", name, err)
}
pretty, err := marshalPretty(decoded)
if err != nil {
return contracts.OutputFile{}, err
}
return contracts.OutputFile{
Name: name,
ContentType: mediaType,
Bytes: append([]byte(nil), content...),
Bytes: pretty,
}, nil
}
func isJSONMediaType(mediaType string) bool {
base, _, err := mime.ParseMediaType(strings.TrimSpace(mediaType))
if err != nil {
base = strings.TrimSpace(mediaType)
}
return strings.EqualFold(base, contentTypeJSON)
}
func jsonFile(name string, value any) (contracts.OutputFile, error) {
data, err := marshalPretty(value)
if err != nil {
@@ -208,7 +218,7 @@ func outputFileName(laneID string) (string, error) {
if sanitized == "" {
return "", encoderErrorf("lane id %q cannot produce a safe file name", laneID)
}
return "outputs/" + sanitized + ".json", nil
return "lanes/" + sanitized + ".json", nil
}
func cloneNormalizeOutputs(outputs []contracts.NormalizeOutput) []contracts.NormalizeOutput {

101
internal/modules/output/json/encoder_test.go
View File

@@ -66,9 +66,9 @@ func TestEncodeReturnsLogicalFilesForNormalizedOutputs(t *testing.T) {
wantNames := []string{
"index.json",
"lanes/notes_items.json",
"lanes/spells.json",
"manifest.json",
"outputs/notes_items.json",
"outputs/spells.json",
"rejected.json",
"warnings.json",
}
@@ -85,7 +85,7 @@ func TestEncodeReturnsLogicalFilesForNormalizedOutputs(t *testing.T) {
}
}
spells := decodeObject(t, fileBytes(t, result.Files, "outputs/spells.json"))
spells := decodeObject(t, fileBytes(t, result.Files, "lanes/spells.json"))
spellCasts := spells["spell_casts"].([]any)
if spellCasts[0].(map[string]any)["spell"] != "Cure Wounds" {
t.Fatalf("spells output = %#v, want raw normalized content", spells)
@@ -177,6 +177,58 @@ func TestEncodeIncludesManifestReferences(t *testing.T) {
}
}
func TestEncodeIncludesManifestRawOutputProvenance(t *testing.T) {
result, err := New().Encode(context.Background(), contracts.OutputRequest{
Manifest: artifacts.RunManifest{
RunID: "run-1",
NormalizedOutputs: []artifacts.NormalizedOutputManifest{
{
LaneID: "spells",
ModuleKey: "noop",
SourceID: "source-1",
MediaType: contentTypeJSON,
Schema: artifacts.OutputSchemaProvenance{
ID: "schema-id",
Name: "schema-name",
Version: "v1",
},
},
},
RejectedOutputs: []artifacts.RejectedOutputManifest{
{
Stage: "extract",
LaneID: "spells",
ModuleKey: "dnd/spells",
ChunkID: "chunk-0",
ReasonCode: "raw_output_rejected",
AttemptCount: 2,
},
},
},
})
if err != nil {
t.Fatalf("Encode() error = %v, want nil", err)
}
manifest := decodeObject(t, fileBytes(t, result.Files, "manifest.json"))
normalized := manifest["normalized_outputs"].([]any)
if len(normalized) != 1 {
t.Fatalf("normalized_outputs = %#v, want one entry", normalized)
}
normalizedEntry := normalized[0].(map[string]any)
if normalizedEntry["lane_id"] != "spells" || normalizedEntry["media_type"] != contentTypeJSON {
t.Fatalf("normalized output manifest = %#v, want lane and media type", normalizedEntry)
}
rejected := manifest["rejected_outputs"].([]any)
if len(rejected) != 1 {
t.Fatalf("rejected_outputs = %#v, want one entry", rejected)
}
rejectedEntry := rejected[0].(map[string]any)
if rejectedEntry["attempt_count"] != float64(2) || rejectedEntry["chunk_id"] != "chunk-0" {
t.Fatalf("rejected output manifest = %#v, want attempt count and chunk", rejectedEntry)
}
}
func TestEncodeRejectsLaneIDWithoutSafeFileName(t *testing.T) {
_, err := New().Encode(context.Background(), contracts.OutputRequest{
NormalizeOutputs: []contracts.NormalizeOutput{normalizeOutput("///", `{"value":true}`)},
@@ -197,11 +249,48 @@ func TestEncodeSanitizesParentPathSequences(t *testing.T) {
t.Fatalf("Encode() error = %v, want nil", err)
}
if got := outputFileNames(result.Files); !containsString(got, "outputs/dnd__spell.json") {
if got := outputFileNames(result.Files); !containsString(got, "lanes/dnd__spell.json") {
t.Fatalf("file names = %#v, want sanitized output filename", got)
}
}
func TestEncodeRejectsInvalidJSONAndUnsupportedMediaTypes(t *testing.T) {
tests := []struct {
name string
output contracts.NormalizeOutput
want string
}{
{
name: "invalid JSON",
output: normalizeOutput("spells", `{"spell_casts":[`),
want: "invalid JSON",
},
{
name: "unsupported media type",
output: func() contracts.NormalizeOutput {
output := normalizeOutput("spells", `{"spell_casts":[]}`)
output.Payload.MediaType = "text/plain"
return output
}(),
want: "unsupported media type",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
_, err := New().Encode(context.Background(), contracts.OutputRequest{
NormalizeOutputs: []contracts.NormalizeOutput{test.output},
})
if err == nil {
t.Fatal("Encode() error = nil, want error")
}
if !strings.Contains(err.Error(), test.want) {
t.Fatalf("Encode() error = %q, want %q", err.Error(), test.want)
}
})
}
}
func TestEncodeRejectsSanitizedFilenameCollisions(t *testing.T) {
_, err := New().Encode(context.Background(), contracts.OutputRequest{
NormalizeOutputs: []contracts.NormalizeOutput{
@@ -244,7 +333,7 @@ func TestEncodeDoesNotMutateInputs(t *testing.T) {
req.Rejected[0].Message = "changed"
req.Warnings[0].Message = "changed"
if !stdjson.Valid(fileBytes(t, result.Files, "outputs/spells.json")) {
if !stdjson.Valid(fileBytes(t, result.Files, "lanes/spells.json")) {
t.Fatal("output changed after request mutation")
}
warnings := decodeObject(t, fileBytes(t, result.Files, "warnings.json"))
@@ -265,7 +354,7 @@ func TestOutputFilesDoNotContainWarnings(t *testing.T) {
t.Fatalf("Encode() error = %v, want nil", err)
}
outputFile := decodeObject(t, fileBytes(t, result.Files, "outputs/spells.json"))
outputFile := decodeObject(t, fileBytes(t, result.Files, "lanes/spells.json"))
if _, ok := outputFile["warnings"]; ok {
t.Fatalf("output file contains warnings: %#v", outputFile)
}