eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Write run outputs and diagnostics

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-04 01:11:57 +00:00
parent ae218d7c57
commit 7de41eb3bd
4 changed files with 595 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

333
internal/cli/run_test.go
View File

@@ -13,6 +13,7 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/generic"
@@ -546,10 +547,11 @@ func TestRunPipelineRejectsUnknownFlag(t *testing.T) {
func TestRunPipelineUnknownPipeline(t *testing.T) {
configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "dnd/spells"))
inputPath := writeSeriatimInput(t)
diagnosticsDir := t.TempDir()
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{"run", "missing", "--config", configPath, "--input", inputPath}, &stdout, &stderr, Options{
code := RunWithOptions([]string{"run", "missing", "--config", configPath, "--input", inputPath, "--diagnostics-dir", diagnosticsDir}, &stdout, &stderr, Options{
LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil),
})
@@ -564,10 +566,11 @@ func TestRunPipelineUnknownPipeline(t *testing.T) {
func TestRunPipelineUnknownOnlyLane(t *testing.T) {
configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "dnd/spells"))
inputPath := writeSeriatimInput(t)
diagnosticsDir := t.TempDir()
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--only", "missing"}, &stdout, &stderr, Options{
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--only", "missing", "--diagnostics-dir", diagnosticsDir}, &stdout, &stderr, Options{
LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil),
})
@@ -582,10 +585,11 @@ func TestRunPipelineUnknownOnlyLane(t *testing.T) {
func TestRunPipelineInvalidInputPath(t *testing.T) {
configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "dnd/spells"))
inputPath := filepath.Join(t.TempDir(), "missing.json")
diagnosticsDir := t.TempDir()
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath}, &stdout, &stderr, Options{
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--diagnostics-dir", diagnosticsDir}, &stdout, &stderr, Options{
LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil),
})
@@ -600,11 +604,13 @@ func TestRunPipelineInvalidInputPath(t *testing.T) {
func TestRunPipelineSuccessUsesProductionRegistriesAndFakeLLM(t *testing.T) {
configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "dnd/spells"))
inputPath := writeSeriatimInput(t)
outputDir := t.TempDir()
diagnosticsDir := t.TempDir()
client := newFakeRunLLMClient(false)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath}, &stdout, &stderr, Options{
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir, "--diagnostics-dir", diagnosticsDir}, &stdout, &stderr, Options{
LLMClientFactory: fakeLLMFactory(client, nil),
})
@@ -614,7 +620,7 @@ func TestRunPipelineSuccessUsesProductionRegistriesAndFakeLLM(t *testing.T) {
if client.calls != 1 {
t.Fatalf("LLM calls = %d, want 1", client.calls)
}
for _, want := range []string{"dnd-session", "approved=1", "rejected=0"} {
for _, want := range []string{"dnd-session", "approved=1", "rejected=0", outputDir} {
if !strings.Contains(stdout.String(), want) {
t.Fatalf("stdout = %q, want substring %q", stdout.String(), want)
}
@@ -627,11 +633,13 @@ func TestRunPipelineSuccessUsesProductionRegistriesAndFakeLLM(t *testing.T) {
func TestRunPipelineOnlySelectsRequestedLane(t *testing.T) {
configPath := writeTestConfig(t, mvpConfigYAMLForLanes("dnd-session", "spells", "rituals"))
inputPath := writeSeriatimInput(t)
outputDir := t.TempDir()
diagnosticsDir := t.TempDir()
client := newFakeRunLLMClient(false)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--only", "spells"}, &stdout, &stderr, Options{
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--only", "spells", "--output-dir", outputDir, "--diagnostics-dir", diagnosticsDir}, &stdout, &stderr, Options{
LLMClientFactory: fakeLLMFactory(client, nil),
})
@@ -649,10 +657,11 @@ func TestRunPipelineOnlySelectsRequestedLane(t *testing.T) {
func TestRunPipelineLLMFactoryFailure(t *testing.T) {
configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "dnd/spells"))
inputPath := writeSeriatimInput(t)
diagnosticsDir := t.TempDir()
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath}, &stdout, &stderr, Options{
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--diagnostics-dir", diagnosticsDir}, &stdout, &stderr, Options{
LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), errors.New("factory unavailable")),
})
@@ -667,11 +676,13 @@ func TestRunPipelineLLMFactoryFailure(t *testing.T) {
func TestRunPipelineValidationRejectionCompletesSuccessfully(t *testing.T) {
configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "dnd/spells"))
inputPath := writeSeriatimInput(t)
outputDir := t.TempDir()
diagnosticsDir := t.TempDir()
client := newFakeRunLLMClient(true)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath}, &stdout, &stderr, Options{
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir, "--diagnostics-dir", diagnosticsDir}, &stdout, &stderr, Options{
LLMClientFactory: fakeLLMFactory(client, nil),
})
@@ -686,12 +697,14 @@ func TestRunPipelineValidationRejectionCompletesSuccessfully(t *testing.T) {
func TestRunPipelineLLMProfileOverrideSelectsFactoryProfile(t *testing.T) {
configPath := writeTestConfig(t, mvpConfigYAMLWithProfiles("dnd-session"))
inputPath := writeSeriatimInput(t)
outputDir := t.TempDir()
diagnosticsDir := t.TempDir()
client := newFakeRunLLMClient(false)
factory := &recordingLLMFactory{client: client}
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--llm-profile", "runtime"}, &stdout, &stderr, Options{
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--llm-profile", "runtime", "--output-dir", outputDir, "--diagnostics-dir", diagnosticsDir}, &stdout, &stderr, Options{
LLMClientFactory: factory.build,
})
@@ -703,6 +716,190 @@ func TestRunPipelineLLMProfileOverrideSelectsFactoryProfile(t *testing.T) {
}
}
func TestRunPipelineWritesDurableOutputFiles(t *testing.T) {
diagnosticsDir := t.TempDir()
outputDir := t.TempDir()
configPath := writeTestConfig(t, mvpConfigYAMLWithDiagnostics("dnd-session", diagnosticsDir, "always"))
inputPath := writeSeriatimInput(t)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir}, &stdout, &stderr, Options{
LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil),
})
if code != 0 {
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
}
runOutputDir := onlyChildDir(t, outputDir)
for _, name := range []string{
"index.json",
"manifest.json",
"artifacts/dnd.spell_cast.json",
"rejected.json",
"warnings.json",
} {
if _, err := os.Stat(filepath.Join(runOutputDir, filepath.FromSlash(name))); err != nil {
t.Fatalf("expected output file %q: %v", name, err)
}
}
if !strings.Contains(stdout.String(), runOutputDir) {
t.Fatalf("stdout = %q, want output path %q", stdout.String(), runOutputDir)
}
assertNoTemporaryFiles(t, runOutputDir)
}
func TestRunPipelineRejectsUnsafeOutputFileName(t *testing.T) {
diagnosticsDir := t.TempDir()
outputDir := t.TempDir()
configPath := writeTestConfig(t, mvpConfigYAMLWithDiagnostics("dnd-session", diagnosticsDir, "always"))
inputPath := writeSeriatimInput(t)
registries := registriesWithOutput(t, unsafeOutputEncoder{})
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir}, &stdout, &stderr, Options{
Registries: registries,
LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil),
})
if code != 1 {
t.Fatalf("RunWithOptions() code = %d, want 1", code)
}
if !strings.Contains(stderr.String(), "output file name") {
t.Fatalf("stderr = %q, want unsafe output file error", stderr.String())
}
runDir := onlyChildDir(t, diagnosticsDir)
if got := string(readFile(t, filepath.Join(runDir, diagnostics.ArtifactErrorLog))); !strings.Contains(got, "output file name") {
t.Fatalf("error log = %q, want unsafe output file error", got)
}
}
func TestRunPipelineWritesDiagnosticsArtifactsOnSuccess(t *testing.T) {
diagnosticsDir := t.TempDir()
outputDir := t.TempDir()
configPath := writeTestConfig(t, mvpConfigYAMLWithDiagnostics("dnd-session", diagnosticsDir, "always"))
inputPath := writeSeriatimInput(t)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir}, &stdout, &stderr, Options{
LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil),
})
if code != 0 {
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
}
runDir := onlyChildDir(t, diagnosticsDir)
for _, name := range []string{
diagnostics.ArtifactInvocationMetadata,
diagnostics.ArtifactEffectiveConfig,
diagnostics.ArtifactResolvedPipeline,
diagnostics.ArtifactRunManifest,
diagnostics.ArtifactRunReport,
diagnostics.ArtifactWarnings,
} {
if _, err := os.Stat(filepath.Join(runDir, name)); err != nil {
t.Fatalf("expected diagnostics artifact %q: %v", name, err)
}
}
report := string(readFile(t, filepath.Join(runDir, diagnostics.ArtifactRunReport)))
if !strings.Contains(report, `"approved_count": 1`) || !strings.Contains(report, `"validation_status": "approved"`) || !strings.Contains(report, outputDir) {
t.Fatalf("unexpected run report: %s", report)
}
}
func TestRunPipelineWritesErrorLogAfterDiagnosticsCreation(t *testing.T) {
diagnosticsDir := t.TempDir()
configPath := writeTestConfig(t, mvpConfigYAMLWithDiagnostics("dnd-session", diagnosticsDir, "always"))
inputPath := writeSeriatimInput(t)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath}, &stdout, &stderr, Options{
LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), errors.New("factory unavailable")),
})
if code != 1 {
t.Fatalf("RunWithOptions() code = %d, want 1", code)
}
runDir := onlyChildDir(t, diagnosticsDir)
if got := string(readFile(t, filepath.Join(runDir, diagnostics.ArtifactErrorLog))); !strings.Contains(got, "factory unavailable") {
t.Fatalf("error log = %q, want factory error", got)
}
}
func TestRunPipelineRetentionNeverRemovesSuccessfulWarningFreeDiagnostics(t *testing.T) {
diagnosticsDir := t.TempDir()
outputDir := t.TempDir()
configPath := writeTestConfig(t, mvpConfigYAMLWithDiagnostics("dnd-session", diagnosticsDir, "never"))
inputPath := writeSeriatimInput(t)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir}, &stdout, &stderr, Options{
LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil),
})
if code != 0 {
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
}
if entries := childDirs(t, diagnosticsDir); len(entries) != 0 {
t.Fatalf("diagnostics run dirs = %v, want none", entries)
}
}
func TestRunPipelineWarningsAreDiagnosedAndReported(t *testing.T) {
diagnosticsDir := t.TempDir()
outputDir := t.TempDir()
configPath := writeTestConfig(t, mvpConfigYAMLWithDiagnostics("dnd-session", diagnosticsDir, "auto"))
inputPath := writeSeriatimInput(t)
registries := registriesWithOutput(t, warningOutputEncoder{})
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir}, &stdout, &stderr, Options{
Registries: registries,
LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil),
})
if code != 0 {
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
}
if !strings.Contains(stderr.String(), "1 warning") {
t.Fatalf("stderr = %q, want warning count", stderr.String())
}
runDir := onlyChildDir(t, diagnosticsDir)
warnings := string(readFile(t, filepath.Join(runDir, diagnostics.ArtifactWarnings)))
if !strings.Contains(warnings, "synthetic_warning") {
t.Fatalf("warnings artifact = %q, want synthetic warning", warnings)
}
}
func TestRunPipelineDiagnosticsDirFlagOverridesConfig(t *testing.T) {
configDiagnosticsDir := t.TempDir()
overrideDiagnosticsDir := t.TempDir()
outputDir := t.TempDir()
configPath := writeTestConfig(t, mvpConfigYAMLWithDiagnostics("dnd-session", configDiagnosticsDir, "always"))
inputPath := writeSeriatimInput(t)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir, "--diagnostics-dir", overrideDiagnosticsDir}, &stdout, &stderr, Options{
LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil),
})
if code != 0 {
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
}
if entries := childDirs(t, configDiagnosticsDir); len(entries) != 0 {
t.Fatalf("config diagnostics dir entries = %v, want none", entries)
}
if entries := childDirs(t, overrideDiagnosticsDir); len(entries) != 1 {
t.Fatalf("override diagnostics dir entries = %v, want one run dir", entries)
}
}
func writeTestConfig(t *testing.T, content string) string {
t.Helper()
return writeFile(t, "config.yml", content)
@@ -778,6 +975,20 @@ pipelines:
`
}
func mvpConfigYAMLWithDiagnostics(pipelineID, diagnosticsDir, retention string) string {
return `version: 1
diagnostics:
work_dir: ` + diagnosticsDir + `
retention: ` + retention + `
pipelines:
` + pipelineID + `:
input: seriatim
artifacts:
spells:
extract: dnd/spells
`
}
func writeSeriatimInput(t *testing.T) string {
t.Helper()
return writeFile(t, "source.json", `{
@@ -857,6 +1068,110 @@ func (factory *recordingLLMFactory) build(ctx context.Context, cfg config.Config
return factory.client, []artifacts.LLMProfileManifest{{ID: strings.TrimSpace(profileID)}}, nil
}
type unsafeOutputEncoder struct{}
func (unsafeOutputEncoder) Key() string {
return "json"
}
func (unsafeOutputEncoder) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
return contracts.OutputResult{
Files: []contracts.OutputFile{
{
Name: "../escape.json",
ContentType: "application/json",
Bytes: []byte("{}\n"),
},
},
}, nil
}
type warningOutputEncoder struct{}
func (warningOutputEncoder) Key() string {
return "json"
}
func (warningOutputEncoder) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
result, err := jsonoutput.New().Encode(ctx, req)
if err != nil {
return contracts.OutputResult{}, err
}
result.Warnings = append(result.Warnings, contracts.Warning{
Scope: "output",
ReasonCode: "synthetic_warning",
Message: "synthetic output warning",
})
return result, nil
}
func registriesWithOutput(t *testing.T, encoder contracts.OutputEncoder) pipeline.Registries {
t.Helper()
registries, err := productionRegistries()
if err != nil {
t.Fatalf("productionRegistries: %v", err)
}
outputs := pipeline.NewOutputEncoderRegistry()
if err := outputs.RegisterWithSpec(jsonoutput.ModuleSpec(), func() (contracts.OutputEncoder, error) {
return encoder, nil
}); err != nil {
t.Fatalf("register test output encoder: %v", err)
}
registries.Outputs = outputs
return registries
}
func onlyChildDir(t *testing.T, root string) string {
t.Helper()
children := childDirs(t, root)
if len(children) != 1 {
t.Fatalf("child dirs under %q = %v, want one", root, children)
}
return children[0]
}
func childDirs(t *testing.T, root string) []string {
t.Helper()
entries, err := os.ReadDir(root)
if err != nil {
if os.IsNotExist(err) {
return nil
}
t.Fatalf("read dir %q: %v", root, err)
}
var dirs []string
for _, entry := range entries {
if entry.IsDir() {
dirs = append(dirs, filepath.Join(root, entry.Name()))
}
}
return dirs
}
func readFile(t *testing.T, path string) []byte {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read %q: %v", path, err)
}
return data
}
func assertNoTemporaryFiles(t *testing.T, root string) {
t.Helper()
if err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error {
if err != nil {
return err
}
if strings.Contains(entry.Name(), ".tmp-") {
t.Fatalf("temporary file remains after success: %s", path)
}
return nil
}); err != nil {
t.Fatalf("walk output dir %q: %v", root, err)
}
}
func fakeCatalog(t *testing.T) pipeline.ModuleCatalog {
t.Helper()
inputs := pipeline.NewInputAdapterRegistry()