Complete Phase 18 operational hardening
This commit is contained in:
@@ -2,13 +2,18 @@ package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/cli"
|
||||
)
|
||||
@@ -29,6 +34,7 @@ func TestHelperProcess(t *testing.T) {
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
cli.ConfigureSubprocessTestHooksFromEnv()
|
||||
code := cli.Run(os.Args[sep+1:], os.Stdout, os.Stderr)
|
||||
os.Exit(code)
|
||||
}
|
||||
@@ -125,6 +131,115 @@ func TestProcessFailureMalformedJSONSubprocess(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureMissingTranscriptFileSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
filepath.Join(t.TempDir(), "missing-transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "transcript_read") {
|
||||
t.Fatalf("expected transcript_read failure, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureMissingGlossaryFileSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
filepath.Join(t.TempDir(), "missing-glossary.yaml"),
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "glossary_read") {
|
||||
t.Fatalf("expected glossary_read failure, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureTranscriptSchemaSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
schemaFixturePath("transcript_empty_speaker.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "transcript_schema") {
|
||||
t.Fatalf("expected transcript_schema failure, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureMalformedGlossaryYAMLSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
schemaFixturePath("glossary_malformed.yaml"),
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "glossary_schema") {
|
||||
t.Fatalf("expected glossary_schema failure, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureUnreadableTranscriptSubprocess(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("portable unreadable-file permissions are not reliable on windows")
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
transcriptPath := filepath.Join(dir, "transcript.json")
|
||||
if err := os.WriteFile(transcriptPath, []byte(`[]`), 0o000); err != nil {
|
||||
t.Fatalf("write unreadable transcript: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = os.Chmod(transcriptPath, 0o644) })
|
||||
if _, err := os.ReadFile(transcriptPath); err == nil {
|
||||
t.Skip("unable to make transcript unreadable on this platform/user")
|
||||
}
|
||||
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
transcriptPath,
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "transcript_read") {
|
||||
t.Fatalf("expected transcript_read failure, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureUnwritableOutputSubprocess(t *testing.T) {
|
||||
outputDir := t.TempDir()
|
||||
result := runCLISubprocess(
|
||||
@@ -147,6 +262,269 @@ func TestProcessFailureUnwritableOutputSubprocess(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureUnwritableReportJSONSubprocess(t *testing.T) {
|
||||
reportDir := t.TempDir()
|
||||
outputPath := filepath.Join(t.TempDir(), "out.json")
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--output",
|
||||
outputPath,
|
||||
"--report-json",
|
||||
reportDir,
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "failed to write report JSON file") {
|
||||
t.Fatalf("expected report write failure message, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessSuccessReportJSONSubprocess(t *testing.T) {
|
||||
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--report-json",
|
||||
reportPath,
|
||||
)
|
||||
if result.exitCode != 0 {
|
||||
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
|
||||
}
|
||||
if result.stderr != "" {
|
||||
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
|
||||
}
|
||||
if !json.Valid([]byte(result.stdout)) {
|
||||
t.Fatalf("expected transcript JSON only on stdout, got %q", result.stdout)
|
||||
}
|
||||
report := readFile(t, reportPath)
|
||||
if !json.Valid(report) {
|
||||
t.Fatalf("expected valid report JSON, got %q", string(report))
|
||||
}
|
||||
// Ensure report JSON is not printed to stdout.
|
||||
if strings.Contains(result.stdout, `"phase16-default-pipeline-integration"`) {
|
||||
t.Fatalf("report JSON leaked to stdout: %q", result.stdout)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessSuccessLargeTranscriptSubprocess(t *testing.T) {
|
||||
transcriptPath := writeLargeTranscriptFixture(t, 320)
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
transcriptPath,
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
)
|
||||
if result.exitCode != 0 {
|
||||
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
|
||||
}
|
||||
if result.stderr != "" {
|
||||
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
|
||||
}
|
||||
if !json.Valid([]byte(result.stdout)) {
|
||||
t.Fatalf("expected valid transcript JSON on stdout")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureMalformedStructuredLLMResponseViaSubprocessHook(t *testing.T) {
|
||||
workDir := t.TempDir()
|
||||
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||
result := runCLISubprocessWithEnv(t,
|
||||
map[string]string{"AUDITA_SUBPROCESS_TEST_LLM_MODE": "malformed_structured"},
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--modules",
|
||||
"grammar",
|
||||
"--report-json",
|
||||
reportPath,
|
||||
"--work-dir",
|
||||
workDir,
|
||||
"--work-dir-retention",
|
||||
"always",
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "runner_execution") {
|
||||
t.Fatalf("expected runner_execution failure, got %q", result.stderr)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "diagnostics:") {
|
||||
t.Fatalf("expected diagnostics path in stderr, got %q", result.stderr)
|
||||
}
|
||||
report := readFile(t, reportPath)
|
||||
if !json.Valid(report) {
|
||||
t.Fatalf("expected valid failure report JSON")
|
||||
}
|
||||
runDir := onlyRunDir(t, workDir)
|
||||
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
|
||||
t.Fatalf("expected error.log, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureBackendLLMViaSubprocessHook(t *testing.T) {
|
||||
workDir := t.TempDir()
|
||||
result := runCLISubprocessWithEnv(t,
|
||||
map[string]string{"AUDITA_SUBPROCESS_TEST_LLM_MODE": "backend_error"},
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--modules",
|
||||
"grammar",
|
||||
"--work-dir",
|
||||
workDir,
|
||||
"--work-dir-retention",
|
||||
"always",
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "synthetic backend failure") {
|
||||
t.Fatalf("expected backend failure details, got %q", result.stderr)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "diagnostics:") {
|
||||
t.Fatalf("expected diagnostics path in stderr, got %q", result.stderr)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(onlyRunDir(t, workDir), "error.log")); err != nil {
|
||||
t.Fatalf("expected error.log in retained failed run: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureMidPipelinePreservesPartialReportsSubprocess(t *testing.T) {
|
||||
workDir := t.TempDir()
|
||||
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||
result := runCLISubprocessWithEnv(t,
|
||||
map[string]string{"AUDITA_SUBPROCESS_TEST_LLM_MODE": "mid_pipeline_fail"},
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--modules",
|
||||
"glossary,homophones,glossary,spoken_word,grammar",
|
||||
"--report-json",
|
||||
reportPath,
|
||||
"--work-dir",
|
||||
workDir,
|
||||
"--work-dir-retention",
|
||||
"always",
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
reportRaw := readFile(t, reportPath)
|
||||
var report struct {
|
||||
Status string `json:"status"`
|
||||
ErrorPhase string `json:"error_phase"`
|
||||
ModuleResults []struct {
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
Status string `json:"status"`
|
||||
} `json:"module_results"`
|
||||
}
|
||||
if err := json.Unmarshal(reportRaw, &report); err != nil {
|
||||
t.Fatalf("unmarshal report: %v", err)
|
||||
}
|
||||
if report.Status != "failed" || report.ErrorPhase != "runner_execution" {
|
||||
t.Fatalf("expected failed runner_execution report, got %+v", report)
|
||||
}
|
||||
if len(report.ModuleResults) == 0 {
|
||||
t.Fatalf("expected partial module results in failure report")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessCancellationViaSubprocessTimeoutHook(t *testing.T) {
|
||||
workDir := t.TempDir()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
|
||||
result := runCLISubprocessContext(t, ctx,
|
||||
map[string]string{
|
||||
"AUDITA_SUBPROCESS_TEST_LLM_MODE": "block_until_cancel",
|
||||
"AUDITA_SUBPROCESS_TEST_RUN_TIMEOUT_MS": "120",
|
||||
},
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--modules",
|
||||
"grammar",
|
||||
"--work-dir",
|
||||
workDir,
|
||||
"--work-dir-retention",
|
||||
"always",
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "context deadline exceeded") {
|
||||
t.Fatalf("expected context deadline error, got %q", result.stderr)
|
||||
}
|
||||
runDir := onlyRunDir(t, workDir)
|
||||
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
|
||||
t.Fatalf("expected error.log for canceled run: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(runDir, "report.json")); err != nil {
|
||||
t.Fatalf("expected report.json for canceled run: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessSubprocessNoSecretLeakInOutputsAndDiagnostics(t *testing.T) {
|
||||
secret := "phase18-subprocess-secret"
|
||||
workDir := t.TempDir()
|
||||
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||
outputPath := filepath.Join(t.TempDir(), "out.json")
|
||||
result := runCLISubprocessWithEnv(t,
|
||||
map[string]string{
|
||||
"AUDITA_LLM_API_KEY": secret,
|
||||
"AUDITA_VALIDATION_LLM_API_KEY": secret,
|
||||
},
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--output",
|
||||
outputPath,
|
||||
"--report-json",
|
||||
reportPath,
|
||||
"--work-dir",
|
||||
workDir,
|
||||
"--work-dir-retention",
|
||||
"always",
|
||||
)
|
||||
if result.exitCode != 0 {
|
||||
t.Fatalf("expected success, got %d stderr=%q", result.exitCode, result.stderr)
|
||||
}
|
||||
if strings.Contains(result.stdout, secret) || strings.Contains(result.stderr, secret) {
|
||||
t.Fatalf("secret leaked in subprocess stdio")
|
||||
}
|
||||
assertNoSecretInFile(t, reportPath, secret)
|
||||
assertNoSecretInTree(t, onlyRunDir(t, workDir), secret)
|
||||
}
|
||||
|
||||
type subprocessResult struct {
|
||||
stdout string
|
||||
stderr string
|
||||
@@ -155,10 +533,23 @@ type subprocessResult struct {
|
||||
|
||||
func runCLISubprocess(t *testing.T, args ...string) subprocessResult {
|
||||
t.Helper()
|
||||
return runCLISubprocessWithEnv(t, nil, args...)
|
||||
}
|
||||
|
||||
func runCLISubprocessWithEnv(t *testing.T, extraEnv map[string]string, args ...string) subprocessResult {
|
||||
t.Helper()
|
||||
return runCLISubprocessContext(t, context.Background(), extraEnv, args...)
|
||||
}
|
||||
|
||||
func runCLISubprocessContext(t *testing.T, ctx context.Context, extraEnv map[string]string, args ...string) subprocessResult {
|
||||
t.Helper()
|
||||
cmdArgs := append([]string{"-test.run=TestHelperProcess", "--"}, args...)
|
||||
cmd := exec.Command(os.Args[0], cmdArgs...)
|
||||
cmd.Env = append(filterAuditaEnv(os.Environ()), "GO_WANT_HELPER_PROCESS=1")
|
||||
cmd := exec.CommandContext(ctx, os.Args[0], cmdArgs...)
|
||||
env := append(filterAuditaEnv(os.Environ()), "GO_WANT_HELPER_PROCESS=1")
|
||||
for k, v := range extraEnv {
|
||||
env = append(env, fmt.Sprintf("%s=%s", k, v))
|
||||
}
|
||||
cmd.Env = env
|
||||
var stdoutBuf bytes.Buffer
|
||||
var stderrBuf bytes.Buffer
|
||||
cmd.Stdout = &stdoutBuf
|
||||
@@ -201,6 +592,10 @@ func fixturePath(name string) string {
|
||||
return filepath.Join("..", "..", "internal", "cli", "testdata", name)
|
||||
}
|
||||
|
||||
func schemaFixturePath(name string) string {
|
||||
return filepath.Join("..", "..", "internal", "core", "schema", "testdata", name)
|
||||
}
|
||||
|
||||
func readFile(t *testing.T, path string) []byte {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
@@ -228,3 +623,63 @@ func assertJSONSemanticallyEqual(t *testing.T, expected []byte, actual []byte) {
|
||||
t.Fatalf("JSON content mismatch: expected %q got %q", string(expected), string(actual))
|
||||
}
|
||||
}
|
||||
|
||||
func writeLargeTranscriptFixture(t *testing.T, segments int) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "large-transcript.json")
|
||||
rows := make([]string, 0, segments)
|
||||
for i := 0; i < segments; i++ {
|
||||
rows = append(rows, fmt.Sprintf(`{"id":%d,"speaker":"Speaker%d","start":%s,"end":%s,"text":"Segment %d has enough words to exercise stdout and pipe buffering safely."}`,
|
||||
i+1,
|
||||
(i%4)+1,
|
||||
strconv.FormatFloat(float64(i)*1.1, 'f', 1, 64),
|
||||
strconv.FormatFloat(float64(i)*1.1+1.0, 'f', 1, 64),
|
||||
i+1,
|
||||
))
|
||||
}
|
||||
payload := "[\n " + strings.Join(rows, ",\n ") + "\n]\n"
|
||||
if err := os.WriteFile(path, []byte(payload), 0o644); err != nil {
|
||||
t.Fatalf("write large transcript fixture: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func onlyRunDir(t *testing.T, workDir string) string {
|
||||
t.Helper()
|
||||
entries, err := os.ReadDir(workDir)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read work dir %q: %v", workDir, err)
|
||||
}
|
||||
dirs := make([]string, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
if e.IsDir() {
|
||||
dirs = append(dirs, filepath.Join(workDir, e.Name()))
|
||||
}
|
||||
}
|
||||
if len(dirs) != 1 {
|
||||
t.Fatalf("expected exactly one run dir in %q, found %d", workDir, len(dirs))
|
||||
}
|
||||
return dirs[0]
|
||||
}
|
||||
|
||||
func assertNoSecretInFile(t *testing.T, path, secret string) {
|
||||
t.Helper()
|
||||
raw := string(readFile(t, path))
|
||||
if strings.Contains(raw, secret) {
|
||||
t.Fatalf("secret leaked in %s", path)
|
||||
}
|
||||
}
|
||||
|
||||
func assertNoSecretInTree(t *testing.T, root, secret string) {
|
||||
t.Helper()
|
||||
_ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil || d == nil || d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
raw, readErr := os.ReadFile(path)
|
||||
if readErr == nil && strings.Contains(string(raw), secret) {
|
||||
t.Fatalf("secret leaked in %s", path)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user