eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Write workspace debug artifacts during runs

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-08 03:14:59 +00:00
parent ae9c2e1d5e
commit a5bbfea9b9
10 changed files with 1134 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

211
internal/cli/run_test.go
View File

@@ -3,12 +3,14 @@ package cli
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"errors"
"io/fs"
"os"
"path/filepath"
"reflect"
"regexp"
"sort"
"strings"
"testing"
@@ -2219,6 +2221,121 @@ func TestRunPipelineWritesCheckpointsWhenWorkspaceResumeEnabled(t *testing.T) {
assertPathNotExist(t, filepath.Join(workspaceDir, "debug"))
}
func TestRunPipelineWritesDebugWhenWorkspaceDebugEnabled(t *testing.T) {
workspaceDir := filepath.Join(t.TempDir(), "workspace")
outputDir := t.TempDir()
configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceDebugEnabled("dnd-session", workspaceDir, "always"))
inputPath := writeSeriatimInput(t)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir}, &stdout, &stderr, Options{
LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil),
})
if code != 0 {
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
}
debugDir := onlyChildDir(t, filepath.Join(workspaceDir, "debug"))
for _, name := range []string{
"run.json",
"source/input.json",
"source/output.json",
"chunk/input.json",
"chunk/output.json",
"extract/spells/input.json",
"extract/spells/output.json",
"merge/spells/input.json",
"merge/spells/output.json",
"normalize/spells/input.json",
"normalize/spells/output.json",
"output/input.json",
"output/output.json",
"llm/call-0001.json",
} {
if _, err := os.Stat(filepath.Join(debugDir, name)); err != nil {
t.Fatalf("expected debug artifact %q: %v", name, err)
}
}
assertPathNotExist(t, filepath.Join(workspaceDir, "checkpoints"))
}
func TestRunPipelineDebugAndResumeCanBeEnabledIndependently(t *testing.T) {
workspaceDir := filepath.Join(t.TempDir(), "workspace")
outputDir := t.TempDir()
configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceDiagnosticsAndStateEnabled("dnd-session", workspaceDir, "always"))
inputPath := writeSeriatimInput(t)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir}, &stdout, &stderr, Options{
LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil),
})
if code != 0 {
t.Fatalf("seed RunWithOptions() code = %d, stderr=%q", code, stderr.String())
}
code = RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir, "--resume"}, &stdout, &stderr, Options{
LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil),
})
if code != 0 {
t.Fatalf("resume RunWithOptions() code = %d, stderr=%q", code, stderr.String())
}
if entries := childDirs(t, filepath.Join(workspaceDir, "checkpoints")); len(entries) != 1 {
t.Fatalf("workspace checkpoint pipeline dirs = %v, want one", entries)
}
if entries := childDirs(t, filepath.Join(workspaceDir, "debug")); len(entries) != 2 {
t.Fatalf("workspace debug run dirs = %v, want two", entries)
}
}
func TestRunPipelineDebugRedactsObviousSecrets(t *testing.T) {
workspaceDir := filepath.Join(t.TempDir(), "workspace")
outputDir := t.TempDir()
configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceDebugEnabled("dnd-session", workspaceDir, "always"))
inputPath := writeSeriatimInput(t)
client := newFakeRunLLMClient(false)
client.payload = map[string]any{
"spell_casts": []map[string]any{
{
"caster": "Aria",
"spell": "sk-secretvalue",
"effect": "Bearer secretvalue",
"narrative_description": "Aria casts a spell.",
"source_refs": []map[string]any{
{"source_id": "session-alpha", "start_unit_id": 1, "end_unit_id": 1},
},
},
},
}
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir}, &stdout, &stderr, Options{
LLMClientFactory: fakeLLMFactory(client, nil),
})
if code != 0 {
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
}
assertDebugTreeDoesNotContain(t, onlyChildDir(t, filepath.Join(workspaceDir, "debug")), "sk-secretvalue", "Bearer secretvalue")
}
func TestWorkspaceStateRootsDoNotOverlap(t *testing.T) {
workspaceDir := filepath.Join(t.TempDir(), "workspace")
outputDir := t.TempDir()
configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceDiagnosticsAndStateEnabled("dnd-session", workspaceDir, "always"))
inputPath := writeSeriatimInput(t)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir}, &stdout, &stderr, Options{
LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil),
})
if code != 0 {
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
}
assertDistinctRoots(t, filepath.Join(workspaceDir, "diagnostics"), filepath.Join(workspaceDir, "checkpoints"), filepath.Join(workspaceDir, "debug"))
}
func TestRunPipelineResumeRequiresWorkspaceResumeEnabled(t *testing.T) {
workspaceDir := filepath.Join(t.TempDir(), "workspace")
outputDir := t.TempDir()
@@ -2504,7 +2621,9 @@ func TestRunPipelineDiagnosticsDirFlagOverridesWorkspaceDiagnosticsOnly(t *testi
if entries := childDirs(t, filepath.Join(workspaceDir, "checkpoints")); len(entries) != 1 {
t.Fatalf("workspace checkpoint pipeline dirs = %v, want one", entries)
}
assertPathNotExist(t, filepath.Join(workspaceDir, "debug"))
if entries := childDirs(t, filepath.Join(workspaceDir, "debug")); len(entries) != 1 {
t.Fatalf("workspace debug run dirs = %v, want one", entries)
}
}
func TestExampleFixtureConfigValidateAndPipelinesList(t *testing.T) {
@@ -3054,6 +3173,24 @@ pipelines:
`
}
func mvpConfigYAMLWithWorkspaceDebugEnabled(pipelineID, workspaceDir, retention string) string {
return `version: 2
workspace:
directory: ` + workspaceDir + `
diagnostics:
enabled: true
retention: ` + retention + `
debug:
enabled: true
pipelines:
` + pipelineID + `:
input: seriatim
artifacts:
spells:
extract: dnd/spells
`
}
func mvpConfigYAMLWithWorkspaceResumeAndChunker(pipelineID, workspaceDir, retention, chunker string) string {
return `version: 2
workspace:
@@ -3523,6 +3660,78 @@ func anyDiagnosticsFileContains(t *testing.T, runDirs []string, name string, wan
return false
}
func assertDistinctRoots(t *testing.T, roots ...string) {
t.Helper()
for i, first := range roots {
for _, second := range roots[i+1:] {
firstAbs, err := filepath.Abs(first)
if err != nil {
t.Fatalf("resolve %q: %v", first, err)
}
secondAbs, err := filepath.Abs(second)
if err != nil {
t.Fatalf("resolve %q: %v", second, err)
}
if firstAbs == secondAbs {
t.Fatalf("workspace roots overlap exactly: %q", firstAbs)
}
firstRel, err := filepath.Rel(firstAbs, secondAbs)
if err != nil {
t.Fatalf("rel %q %q: %v", firstAbs, secondAbs, err)
}
secondRel, err := filepath.Rel(secondAbs, firstAbs)
if err != nil {
t.Fatalf("rel %q %q: %v", secondAbs, firstAbs, err)
}
if !strings.HasPrefix(firstRel, ".."+string(filepath.Separator)) && firstRel != ".." {
t.Fatalf("workspace root %q contains %q", firstAbs, secondAbs)
}
if !strings.HasPrefix(secondRel, ".."+string(filepath.Separator)) && secondRel != ".." {
t.Fatalf("workspace root %q contains %q", secondAbs, firstAbs)
}
}
}
}
var debugBase64FieldPattern = regexp.MustCompile(`"(?:content_base64|content)"\s*:\s*"([^"]*)"`)
func assertDebugTreeDoesNotContain(t *testing.T, root string, forbidden ...string) {
t.Helper()
if err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error {
if err != nil {
return err
}
if entry.IsDir() {
return nil
}
data, err := os.ReadFile(path)
if err != nil {
return err
}
text := string(data)
for _, value := range forbidden {
if strings.Contains(text, value) {
t.Fatalf("debug artifact %q contains forbidden value %q", path, value)
}
}
for _, match := range debugBase64FieldPattern.FindAllStringSubmatch(text, -1) {
decoded, err := base64.StdEncoding.DecodeString(match[1])
if err != nil {
continue
}
decodedText := string(decoded)
for _, value := range forbidden {
if strings.Contains(decodedText, value) {
t.Fatalf("debug artifact %q decoded content contains forbidden value %q", path, value)
}
}
}
return nil
}); err != nil {
t.Fatalf("walk debug tree %q: %v", root, err)
}
}
func seedWorkspaceCheckpoint(t *testing.T, configPath string, inputPath string, extraArgs []string) {
t.Helper()
client := newFakeRunLLMClient(false)