eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement raw module output contracts

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-07 18:58:23 +00:00
parent 9e3f8809b3
commit c05ecb58d8
33 changed files with 1166 additions and 1780 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
internal/cli/run.go
View File

@@ -275,7 +275,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
PipelineID: effective.PipelineID,
OutputPath: runOutputDir,
DiagnosticsPath: runDir.Path(),
ApprovedCount: len(output.Approved),
OutputCount: len(output.NormalizeOutputs),
RejectedCount: len(output.Rejected),
WarningCount: len(output.Warnings),
ValidationStatus: output.Manifest.ValidationStatus,
@@ -293,7 +293,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("apply diagnostics retention: %w", err))
}
fmt.Fprintf(stdout, "pipeline %q complete: approved=%d rejected=%d output=%s\n", effective.PipelineID, len(output.Approved), len(output.Rejected), runOutputDir)
fmt.Fprintf(stdout, "pipeline %q complete: outputs=%d rejected=%d output=%s\n", effective.PipelineID, len(output.NormalizeOutputs), len(output.Rejected), runOutputDir)
if len(output.Warnings) > 0 {
fmt.Fprintf(stderr, "notarius: run completed with %d warning(s)\n", len(output.Warnings))
}
@@ -305,7 +305,7 @@ type runReport struct {
PipelineID string `json:"pipeline_id"`
OutputPath string `json:"output_path"`
DiagnosticsPath string `json:"diagnostics_path,omitempty"`
ApprovedCount int `json:"approved_count"`
OutputCount int `json:"output_count"`
RejectedCount int `json:"rejected_count"`
WarningCount int `json:"warning_count"`
ValidationStatus string `json:"validation_status,omitempty"`

104
internal/cli/run_test.go
View File

@@ -690,7 +690,7 @@ func TestRunPipelineSuccessUsesProductionRegistriesAndFakeLLM(t *testing.T) {
if client.calls != 1 {
t.Fatalf("LLM calls = %d, want 1", client.calls)
}
for _, want := range []string{"dnd-session", "approved=1", "rejected=0", outputDir} {
for _, want := range []string{"dnd-session", "outputs=1", "rejected=0", outputDir} {
if !strings.Contains(stdout.String(), want) {
t.Fatalf("stdout = %q, want substring %q", stdout.String(), want)
}
@@ -719,8 +719,8 @@ func TestRunPipelineOnlySelectsRequestedLane(t *testing.T) {
if client.calls != 1 {
t.Fatalf("LLM calls = %d, want only selected lane to run once", client.calls)
}
if !strings.Contains(stdout.String(), "approved=1") {
t.Fatalf("stdout = %q, want approved count", stdout.String())
if !strings.Contains(stdout.String(), "outputs=1") {
t.Fatalf("stdout = %q, want output count", stdout.String())
}
}
@@ -743,7 +743,7 @@ func TestRunPipelineLLMFactoryFailure(t *testing.T) {
}
}
func TestRunPipelineValidationRejectionCompletesSuccessfully(t *testing.T) {
func TestRunPipelineCarriesInvalidLLMSourceRefsAsRawOutput(t *testing.T) {
configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "dnd/spells"))
inputPath := writeSeriatimInput(t)
outputDir := t.TempDir()
@@ -759,8 +759,8 @@ func TestRunPipelineValidationRejectionCompletesSuccessfully(t *testing.T) {
if code != 0 {
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
}
if !strings.Contains(stdout.String(), "approved=0") || !strings.Contains(stdout.String(), "rejected=1") {
t.Fatalf("stdout = %q, want rejection counts", stdout.String())
if !strings.Contains(stdout.String(), "outputs=1") || !strings.Contains(stdout.String(), "rejected=0") {
t.Fatalf("stdout = %q, want raw output count", stdout.String())
}
}
@@ -1644,7 +1644,7 @@ func TestRunPipelineWritesDurableOutputFiles(t *testing.T) {
for _, name := range []string{
"index.json",
"manifest.json",
"artifacts/dnd.spell_cast.json",
"outputs/spells.json",
"rejected.json",
"warnings.json",
} {
@@ -1819,7 +1819,7 @@ func TestRunPipelineWritesDiagnosticsArtifactsOnSuccess(t *testing.T) {
}
}
report := string(readFile(t, filepath.Join(runDir, diagnostics.ArtifactRunReport)))
if !strings.Contains(report, `"approved_count": 1`) || !strings.Contains(report, `"validation_status": "approved"`) || !strings.Contains(report, outputDir) {
if !strings.Contains(report, `"output_count": 1`) || !strings.Contains(report, `"validation_status": "approved"`) || !strings.Contains(report, outputDir) {
t.Fatalf("unexpected run report: %s", report)
}
}
@@ -1989,29 +1989,26 @@ func TestExampleFixtureRunWritesExpectedJSON(t *testing.T) {
t.Fatalf("extractor metadata = %#v, want prompt/schema identifiers", extractorMetadata)
}
var artifactFile struct {
ArtifactType string `json:"artifact_type"`
Artifacts []artifacts.Artifact `json:"artifacts"`
var spellOutput struct {
SpellCasts []struct {
Caster string `json:"caster"`
Spell string `json:"spell"`
Effect string `json:"effect"`
SourceRefs []source.SourceRef `json:"source_refs"`
} `json:"spell_casts"`
}
readJSONFile(t, filepath.Join(runOutputDir, "artifacts", "dnd.spell_cast.json"), &artifactFile)
if artifactFile.ArtifactType != "dnd.spell_cast" || len(artifactFile.Artifacts) != 1 {
t.Fatalf("artifact file = %#v, want one spell artifact", artifactFile)
}
var payload struct {
Caster string `json:"caster"`
Spell string `json:"spell"`
Effect string `json:"effect"`
}
if err := json.Unmarshal(artifactFile.Artifacts[0].Payload, &payload); err != nil {
t.Fatalf("unmarshal spell payload: %v", err)
readJSONFile(t, filepath.Join(runOutputDir, "outputs", "spells.json"), &spellOutput)
if len(spellOutput.SpellCasts) != 1 {
t.Fatalf("spell output = %#v, want one spell cast", spellOutput)
}
payload := spellOutput.SpellCasts[0]
if payload.Caster != "Aria" || payload.Spell != "Cure Wounds" || payload.Effect == "" {
t.Fatalf("payload = %#v, want deterministic spell output", payload)
}
if len(artifactFile.Artifacts[0].SourceRefs) != 1 {
t.Fatalf("source refs = %#v, want one source ref", artifactFile.Artifacts[0].SourceRefs)
if len(payload.SourceRefs) != 1 {
t.Fatalf("source refs = %#v, want one source ref", payload.SourceRefs)
}
ref := artifactFile.Artifacts[0].SourceRefs[0]
ref := payload.SourceRefs[0]
if ref.SourceID != "session-alpha" || ref.StartUnitID != 1 || ref.EndUnitID != 1 {
t.Fatalf("source ref = %#v, want fixture source ref", ref)
}
@@ -2175,11 +2172,11 @@ func TestExampleFixtureFailureCoverage(t *testing.T) {
wantStderr: "spell_casts",
},
{
name: "invalid source reference rejection",
name: "invalid source reference raw output",
args: []string{"run", "dnd-session", "--config", configPath, "--input", inputPath},
factory: fakeLLMFactory(newFakeRunLLMClient(true), nil),
wantCode: 0,
wantOutputStatus: "rejected",
wantOutputStatus: "approved",
},
}
@@ -2213,10 +2210,6 @@ func TestExampleFixtureFailureCoverage(t *testing.T) {
if manifest.ValidationStatus != test.wantOutputStatus {
t.Fatalf("validation status = %q, want %q", manifest.ValidationStatus, test.wantOutputStatus)
}
rejected := string(readFile(t, filepath.Join(runOutputDir, "rejected.json")))
if !strings.Contains(rejected, "invalid_source_ref") {
t.Fatalf("rejected output = %s, want invalid source ref rejection", rejected)
}
}
})
}
@@ -2682,30 +2675,17 @@ func (fakeRunExtractor) Key() string {
return "fake/extract"
}
func (fakeRunExtractor) ArtifactType() string {
return "fake.artifact"
}
func (fakeRunExtractor) SchemaVersion() string {
return "v1"
}
func (fakeRunExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return []contracts.ReferenceSlot{{Name: "roster"}}
}
func (fakeRunExtractor) Validators() []contracts.Validator {
return nil
}
func (fakeRunExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
return contracts.ExtractionResult{
Candidates: []artifacts.ArtifactCandidate{
{
Payload: []byte(`{"value":true}`),
SourceRefs: []source.SourceRef{
{SourceID: "source", StartUnitID: 1, EndUnitID: 1},
},
Output: contracts.ExtractOutput{
Schema: contracts.ResponseSchema{ID: "fake.artifact", Name: "fake_artifact", Version: "v1"},
Payload: contracts.RawPayload{
Content: []byte(`{"value":true}`),
MediaType: "application/json",
},
},
}, nil
@@ -2718,11 +2698,20 @@ func (fakeRunMerger) Key() string {
}
func (fakeRunMerger) Merge(ctx context.Context, req contracts.MergeRequest) (contracts.MergeResult, error) {
var candidates []artifacts.ArtifactCandidate
for _, chunkArtifacts := range req.ChunkArtifacts {
candidates = append(candidates, chunkArtifacts.Candidates...)
output := contracts.MergeOutput{
LaneID: req.LaneID,
SourceID: req.Source.ID,
Schema: contracts.ResponseSchema{ID: "fake.artifact", Name: "fake_artifact", Version: "v1"},
Payload: contracts.RawPayload{
Content: []byte(`{"merged":true}`),
MediaType: "application/json",
},
}
return contracts.MergeResult{Candidates: candidates}, nil
if len(req.ExtractOutputs) > 0 {
output.Schema = req.ExtractOutputs[0].Schema
output.Payload = req.ExtractOutputs[0].Payload
}
return contracts.MergeResult{Output: output}, nil
}
type fakeRunNormalizer struct{}
@@ -2736,7 +2725,14 @@ func (fakeRunNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
}
func (fakeRunNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
return contracts.NormalizeResult{Candidates: append([]artifacts.ArtifactCandidate(nil), req.Candidates...)}, nil
return contracts.NormalizeResult{
Output: contracts.NormalizeOutput{
LaneID: req.LaneID,
SourceID: req.MergeOutput.SourceID,
Schema: req.MergeOutput.Schema,
Payload: req.MergeOutput.Payload,
},
}, nil
}
func onlyChildDir(t *testing.T, root string) string {