From 68e2d9b54956b8c1d5b2f64170c6b223ce1663e0 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 12 May 2026 13:32:35 +0000 Subject: [PATCH] Complete Phase 18 operational hardening --- cmd/audita/main_integration_test.go | 459 +++++++++++++++++++++++++- docs/architecture.md | 34 +- docs/rewrite-notes.md | 41 ++- docs/subprocess-operations.md | 85 +++++ internal/cli/run.go | 10 +- internal/cli/subprocess_test_hooks.go | 112 +++++++ 6 files changed, 719 insertions(+), 22 deletions(-) create mode 100644 docs/subprocess-operations.md create mode 100644 internal/cli/subprocess_test_hooks.go diff --git a/cmd/audita/main_integration_test.go b/cmd/audita/main_integration_test.go index ecc4ec0..274a99d 100644 --- a/cmd/audita/main_integration_test.go +++ b/cmd/audita/main_integration_test.go @@ -2,13 +2,18 @@ package main import ( "bytes" + "context" "encoding/json" + "fmt" "os" "os/exec" "path/filepath" "reflect" + "runtime" + "strconv" "strings" "testing" + "time" "gitea.maximumdirect.net/eric/audita/internal/cli" ) @@ -29,6 +34,7 @@ func TestHelperProcess(t *testing.T) { os.Exit(2) } + cli.ConfigureSubprocessTestHooksFromEnv() code := cli.Run(os.Args[sep+1:], os.Stdout, os.Stderr) os.Exit(code) } @@ -125,6 +131,115 @@ func TestProcessFailureMalformedJSONSubprocess(t *testing.T) { } } +func TestProcessFailureMissingTranscriptFileSubprocess(t *testing.T) { + result := runCLISubprocess( + t, + "process", + filepath.Join(t.TempDir(), "missing-transcript.json"), + "--glossary", + fixturePath("tiny_glossary.yaml"), + ) + if result.exitCode == 0 { + t.Fatalf("expected nonzero exit code") + } + if result.stdout != "" { + t.Fatalf("expected empty stdout on failure, got %q", result.stdout) + } + if !strings.Contains(result.stderr, "transcript_read") { + t.Fatalf("expected transcript_read failure, got %q", result.stderr) + } +} + +func TestProcessFailureMissingGlossaryFileSubprocess(t *testing.T) { + result := runCLISubprocess( + t, + "process", + fixturePath("tiny_transcript.json"), + "--glossary", + filepath.Join(t.TempDir(), "missing-glossary.yaml"), + ) + if result.exitCode == 0 { + t.Fatalf("expected nonzero exit code") + } + if result.stdout != "" { + t.Fatalf("expected empty stdout on failure, got %q", result.stdout) + } + if !strings.Contains(result.stderr, "glossary_read") { + t.Fatalf("expected glossary_read failure, got %q", result.stderr) + } +} + +func TestProcessFailureTranscriptSchemaSubprocess(t *testing.T) { + result := runCLISubprocess( + t, + "process", + schemaFixturePath("transcript_empty_speaker.json"), + "--glossary", + fixturePath("tiny_glossary.yaml"), + ) + if result.exitCode == 0 { + t.Fatalf("expected nonzero exit code") + } + if result.stdout != "" { + t.Fatalf("expected empty stdout on failure, got %q", result.stdout) + } + if !strings.Contains(result.stderr, "transcript_schema") { + t.Fatalf("expected transcript_schema failure, got %q", result.stderr) + } +} + +func TestProcessFailureMalformedGlossaryYAMLSubprocess(t *testing.T) { + result := runCLISubprocess( + t, + "process", + fixturePath("tiny_transcript.json"), + "--glossary", + schemaFixturePath("glossary_malformed.yaml"), + ) + if result.exitCode == 0 { + t.Fatalf("expected nonzero exit code") + } + if result.stdout != "" { + t.Fatalf("expected empty stdout on failure, got %q", result.stdout) + } + if !strings.Contains(result.stderr, "glossary_schema") { + t.Fatalf("expected glossary_schema failure, got %q", result.stderr) + } +} + +func TestProcessFailureUnreadableTranscriptSubprocess(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("portable unreadable-file permissions are not reliable on windows") + } + + dir := t.TempDir() + transcriptPath := filepath.Join(dir, "transcript.json") + if err := os.WriteFile(transcriptPath, []byte(`[]`), 0o000); err != nil { + t.Fatalf("write unreadable transcript: %v", err) + } + t.Cleanup(func() { _ = os.Chmod(transcriptPath, 0o644) }) + if _, err := os.ReadFile(transcriptPath); err == nil { + t.Skip("unable to make transcript unreadable on this platform/user") + } + + result := runCLISubprocess( + t, + "process", + transcriptPath, + "--glossary", + fixturePath("tiny_glossary.yaml"), + ) + if result.exitCode == 0 { + t.Fatalf("expected nonzero exit code") + } + if result.stdout != "" { + t.Fatalf("expected empty stdout on failure, got %q", result.stdout) + } + if !strings.Contains(result.stderr, "transcript_read") { + t.Fatalf("expected transcript_read failure, got %q", result.stderr) + } +} + func TestProcessFailureUnwritableOutputSubprocess(t *testing.T) { outputDir := t.TempDir() result := runCLISubprocess( @@ -147,6 +262,269 @@ func TestProcessFailureUnwritableOutputSubprocess(t *testing.T) { } } +func TestProcessFailureUnwritableReportJSONSubprocess(t *testing.T) { + reportDir := t.TempDir() + outputPath := filepath.Join(t.TempDir(), "out.json") + result := runCLISubprocess( + t, + "process", + fixturePath("tiny_transcript.json"), + "--glossary", + fixturePath("tiny_glossary.yaml"), + "--output", + outputPath, + "--report-json", + reportDir, + ) + if result.exitCode == 0 { + t.Fatalf("expected nonzero exit code") + } + if result.stdout != "" { + t.Fatalf("expected empty stdout on failure, got %q", result.stdout) + } + if !strings.Contains(result.stderr, "failed to write report JSON file") { + t.Fatalf("expected report write failure message, got %q", result.stderr) + } +} + +func TestProcessSuccessReportJSONSubprocess(t *testing.T) { + reportPath := filepath.Join(t.TempDir(), "report.json") + result := runCLISubprocess( + t, + "process", + fixturePath("tiny_transcript.json"), + "--glossary", + fixturePath("tiny_glossary.yaml"), + "--report-json", + reportPath, + ) + if result.exitCode != 0 { + t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr) + } + if result.stderr != "" { + t.Fatalf("expected empty stderr on success, got %q", result.stderr) + } + if !json.Valid([]byte(result.stdout)) { + t.Fatalf("expected transcript JSON only on stdout, got %q", result.stdout) + } + report := readFile(t, reportPath) + if !json.Valid(report) { + t.Fatalf("expected valid report JSON, got %q", string(report)) + } + // Ensure report JSON is not printed to stdout. + if strings.Contains(result.stdout, `"phase16-default-pipeline-integration"`) { + t.Fatalf("report JSON leaked to stdout: %q", result.stdout) + } +} + +func TestProcessSuccessLargeTranscriptSubprocess(t *testing.T) { + transcriptPath := writeLargeTranscriptFixture(t, 320) + result := runCLISubprocess( + t, + "process", + transcriptPath, + "--glossary", + fixturePath("tiny_glossary.yaml"), + ) + if result.exitCode != 0 { + t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr) + } + if result.stderr != "" { + t.Fatalf("expected empty stderr on success, got %q", result.stderr) + } + if !json.Valid([]byte(result.stdout)) { + t.Fatalf("expected valid transcript JSON on stdout") + } +} + +func TestProcessFailureMalformedStructuredLLMResponseViaSubprocessHook(t *testing.T) { + workDir := t.TempDir() + reportPath := filepath.Join(t.TempDir(), "report.json") + result := runCLISubprocessWithEnv(t, + map[string]string{"AUDITA_SUBPROCESS_TEST_LLM_MODE": "malformed_structured"}, + "process", + fixturePath("tiny_transcript.json"), + "--glossary", + fixturePath("tiny_glossary.yaml"), + "--modules", + "grammar", + "--report-json", + reportPath, + "--work-dir", + workDir, + "--work-dir-retention", + "always", + ) + if result.exitCode == 0 { + t.Fatalf("expected nonzero exit code") + } + if result.stdout != "" { + t.Fatalf("expected empty stdout on failure, got %q", result.stdout) + } + if !strings.Contains(result.stderr, "runner_execution") { + t.Fatalf("expected runner_execution failure, got %q", result.stderr) + } + if !strings.Contains(result.stderr, "diagnostics:") { + t.Fatalf("expected diagnostics path in stderr, got %q", result.stderr) + } + report := readFile(t, reportPath) + if !json.Valid(report) { + t.Fatalf("expected valid failure report JSON") + } + runDir := onlyRunDir(t, workDir) + if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil { + t.Fatalf("expected error.log, got: %v", err) + } +} + +func TestProcessFailureBackendLLMViaSubprocessHook(t *testing.T) { + workDir := t.TempDir() + result := runCLISubprocessWithEnv(t, + map[string]string{"AUDITA_SUBPROCESS_TEST_LLM_MODE": "backend_error"}, + "process", + fixturePath("tiny_transcript.json"), + "--glossary", + fixturePath("tiny_glossary.yaml"), + "--modules", + "grammar", + "--work-dir", + workDir, + "--work-dir-retention", + "always", + ) + if result.exitCode == 0 { + t.Fatalf("expected nonzero exit code") + } + if result.stdout != "" { + t.Fatalf("expected empty stdout on failure, got %q", result.stdout) + } + if !strings.Contains(result.stderr, "synthetic backend failure") { + t.Fatalf("expected backend failure details, got %q", result.stderr) + } + if !strings.Contains(result.stderr, "diagnostics:") { + t.Fatalf("expected diagnostics path in stderr, got %q", result.stderr) + } + if _, err := os.Stat(filepath.Join(onlyRunDir(t, workDir), "error.log")); err != nil { + t.Fatalf("expected error.log in retained failed run: %v", err) + } +} + +func TestProcessFailureMidPipelinePreservesPartialReportsSubprocess(t *testing.T) { + workDir := t.TempDir() + reportPath := filepath.Join(t.TempDir(), "report.json") + result := runCLISubprocessWithEnv(t, + map[string]string{"AUDITA_SUBPROCESS_TEST_LLM_MODE": "mid_pipeline_fail"}, + "process", + fixturePath("tiny_transcript.json"), + "--glossary", + fixturePath("tiny_glossary.yaml"), + "--modules", + "glossary,homophones,glossary,spoken_word,grammar", + "--report-json", + reportPath, + "--work-dir", + workDir, + "--work-dir-retention", + "always", + ) + if result.exitCode == 0 { + t.Fatalf("expected nonzero exit code") + } + if result.stdout != "" { + t.Fatalf("expected empty stdout on failure, got %q", result.stdout) + } + reportRaw := readFile(t, reportPath) + var report struct { + Status string `json:"status"` + ErrorPhase string `json:"error_phase"` + ModuleResults []struct { + ModuleInstance string `json:"module_instance"` + Status string `json:"status"` + } `json:"module_results"` + } + if err := json.Unmarshal(reportRaw, &report); err != nil { + t.Fatalf("unmarshal report: %v", err) + } + if report.Status != "failed" || report.ErrorPhase != "runner_execution" { + t.Fatalf("expected failed runner_execution report, got %+v", report) + } + if len(report.ModuleResults) == 0 { + t.Fatalf("expected partial module results in failure report") + } +} + +func TestProcessCancellationViaSubprocessTimeoutHook(t *testing.T) { + workDir := t.TempDir() + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + + result := runCLISubprocessContext(t, ctx, + map[string]string{ + "AUDITA_SUBPROCESS_TEST_LLM_MODE": "block_until_cancel", + "AUDITA_SUBPROCESS_TEST_RUN_TIMEOUT_MS": "120", + }, + "process", + fixturePath("tiny_transcript.json"), + "--glossary", + fixturePath("tiny_glossary.yaml"), + "--modules", + "grammar", + "--work-dir", + workDir, + "--work-dir-retention", + "always", + ) + if result.exitCode == 0 { + t.Fatalf("expected nonzero exit code") + } + if result.stdout != "" { + t.Fatalf("expected empty stdout on failure, got %q", result.stdout) + } + if !strings.Contains(result.stderr, "context deadline exceeded") { + t.Fatalf("expected context deadline error, got %q", result.stderr) + } + runDir := onlyRunDir(t, workDir) + if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil { + t.Fatalf("expected error.log for canceled run: %v", err) + } + if _, err := os.Stat(filepath.Join(runDir, "report.json")); err != nil { + t.Fatalf("expected report.json for canceled run: %v", err) + } +} + +func TestProcessSubprocessNoSecretLeakInOutputsAndDiagnostics(t *testing.T) { + secret := "phase18-subprocess-secret" + workDir := t.TempDir() + reportPath := filepath.Join(t.TempDir(), "report.json") + outputPath := filepath.Join(t.TempDir(), "out.json") + result := runCLISubprocessWithEnv(t, + map[string]string{ + "AUDITA_LLM_API_KEY": secret, + "AUDITA_VALIDATION_LLM_API_KEY": secret, + }, + "process", + fixturePath("tiny_transcript.json"), + "--glossary", + fixturePath("tiny_glossary.yaml"), + "--output", + outputPath, + "--report-json", + reportPath, + "--work-dir", + workDir, + "--work-dir-retention", + "always", + ) + if result.exitCode != 0 { + t.Fatalf("expected success, got %d stderr=%q", result.exitCode, result.stderr) + } + if strings.Contains(result.stdout, secret) || strings.Contains(result.stderr, secret) { + t.Fatalf("secret leaked in subprocess stdio") + } + assertNoSecretInFile(t, reportPath, secret) + assertNoSecretInTree(t, onlyRunDir(t, workDir), secret) +} + type subprocessResult struct { stdout string stderr string @@ -155,10 +533,23 @@ type subprocessResult struct { func runCLISubprocess(t *testing.T, args ...string) subprocessResult { t.Helper() + return runCLISubprocessWithEnv(t, nil, args...) +} +func runCLISubprocessWithEnv(t *testing.T, extraEnv map[string]string, args ...string) subprocessResult { + t.Helper() + return runCLISubprocessContext(t, context.Background(), extraEnv, args...) +} + +func runCLISubprocessContext(t *testing.T, ctx context.Context, extraEnv map[string]string, args ...string) subprocessResult { + t.Helper() cmdArgs := append([]string{"-test.run=TestHelperProcess", "--"}, args...) - cmd := exec.Command(os.Args[0], cmdArgs...) - cmd.Env = append(filterAuditaEnv(os.Environ()), "GO_WANT_HELPER_PROCESS=1") + cmd := exec.CommandContext(ctx, os.Args[0], cmdArgs...) + env := append(filterAuditaEnv(os.Environ()), "GO_WANT_HELPER_PROCESS=1") + for k, v := range extraEnv { + env = append(env, fmt.Sprintf("%s=%s", k, v)) + } + cmd.Env = env var stdoutBuf bytes.Buffer var stderrBuf bytes.Buffer cmd.Stdout = &stdoutBuf @@ -201,6 +592,10 @@ func fixturePath(name string) string { return filepath.Join("..", "..", "internal", "cli", "testdata", name) } +func schemaFixturePath(name string) string { + return filepath.Join("..", "..", "internal", "core", "schema", "testdata", name) +} + func readFile(t *testing.T, path string) []byte { t.Helper() data, err := os.ReadFile(path) @@ -228,3 +623,63 @@ func assertJSONSemanticallyEqual(t *testing.T, expected []byte, actual []byte) { t.Fatalf("JSON content mismatch: expected %q got %q", string(expected), string(actual)) } } + +func writeLargeTranscriptFixture(t *testing.T, segments int) string { + t.Helper() + path := filepath.Join(t.TempDir(), "large-transcript.json") + rows := make([]string, 0, segments) + for i := 0; i < segments; i++ { + rows = append(rows, fmt.Sprintf(`{"id":%d,"speaker":"Speaker%d","start":%s,"end":%s,"text":"Segment %d has enough words to exercise stdout and pipe buffering safely."}`, + i+1, + (i%4)+1, + strconv.FormatFloat(float64(i)*1.1, 'f', 1, 64), + strconv.FormatFloat(float64(i)*1.1+1.0, 'f', 1, 64), + i+1, + )) + } + payload := "[\n " + strings.Join(rows, ",\n ") + "\n]\n" + if err := os.WriteFile(path, []byte(payload), 0o644); err != nil { + t.Fatalf("write large transcript fixture: %v", err) + } + return path +} + +func onlyRunDir(t *testing.T, workDir string) string { + t.Helper() + entries, err := os.ReadDir(workDir) + if err != nil { + t.Fatalf("failed to read work dir %q: %v", workDir, err) + } + dirs := make([]string, 0, len(entries)) + for _, e := range entries { + if e.IsDir() { + dirs = append(dirs, filepath.Join(workDir, e.Name())) + } + } + if len(dirs) != 1 { + t.Fatalf("expected exactly one run dir in %q, found %d", workDir, len(dirs)) + } + return dirs[0] +} + +func assertNoSecretInFile(t *testing.T, path, secret string) { + t.Helper() + raw := string(readFile(t, path)) + if strings.Contains(raw, secret) { + t.Fatalf("secret leaked in %s", path) + } +} + +func assertNoSecretInTree(t *testing.T, root, secret string) { + t.Helper() + _ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil || d == nil || d.IsDir() { + return nil + } + raw, readErr := os.ReadFile(path) + if readErr == nil && strings.Contains(string(raw), secret) { + t.Fatalf("secret leaked in %s", path) + } + return nil + }) +} diff --git a/docs/architecture.md b/docs/architecture.md index dfba01c..7713405 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -43,8 +43,7 @@ Implemented today: - Explicit runtime support for `--modules spoken_word` through the production runner path. Not implemented in CLI runtime path today: -- Operational hardening tasks beyond current runtime/reporting/diagnostics behavior. -- Rollout and Python retirement work. +- Rollout and Python retirement work (Phase 19). Current reality: - all production modules exist and are wired into the default runtime path. @@ -66,7 +65,8 @@ Phase sequencing note: - Phase 15 spoken-word module implementation and explicit runtime wiring are complete; - Phase 16 default full pipeline integration is complete; - Phase 17 parity fixture suite is complete; -- next recommended phase is Phase 18 (operational hardening and subprocess integration). +- Phase 18 operational hardening and subprocess integration are complete; +- next recommended phase is Phase 19 (documentation, rollout, and Python retirement). ## Actual Go package layout @@ -492,17 +492,23 @@ Implemented tests currently cover: - production homophones module prompt constraints, proposal mapping, validator-chain behavior, confidence-threshold enforcement, diagnostics redaction, protected-term behavior, and explicit CLI/runtime integration (`internal/modules/homophones/*_test.go`, `internal/cli/run_test.go`, `internal/framework/runner/*_test.go`) - production spoken_word module prompt constraints, proposal mapping, validator-chain behavior, semantic guardrail behavior, confidence-threshold enforcement, diagnostics redaction, protected-term behavior, and explicit CLI/runtime integration (`internal/modules/spoken_word/*_test.go`, `internal/cli/run_test.go`, `internal/framework/runner/*_test.go`) - glossary-derived protected-term extraction and stable behavior (`internal/framework/validators/protected_terms_test.go`) +- default full-pipeline runtime shape and ordering (`internal/cli/run_test.go`, `cmd/audita/main_integration_test.go`, `internal/cli/parity_test.go`) +- subprocess operational hardening behavior including large-input, failure-mode, timeout/cancellation, backend-failure, and partial-progress paths (`cmd/audita/main_integration_test.go`) +- report/diagnostics redaction and artifact-shape behavior across success and failure paths (`internal/cli/run_test.go`, `cmd/audita/main_integration_test.go`) -Not covered yet (because not implemented): full default-sequence transcript-polishing runtime behavior as a single default path. +## Operational hardening status (Phase 18) +The runtime now includes hardened subprocess behavior for parent-process callers: +- deterministic success/failure exit codes; +- strict stdout/stderr separation suitable for machine orchestration; +- failure stderr summaries that include diagnostics location when available; +- retained failure diagnostics (`report.json`, `error.log`, and artifacts written before failure); +- deterministic timeout/cancellation behavior in tests; +- redaction coverage for API keys/secrets across reports, diagnostics artifacts, and surfaced errors. -## Intended final architecture (not yet implemented) -The intended end-state still matches the rewrite plan: -- sequential module pipeline over a mutable working transcript -- default full module-sequence integration in the standard runtime path -- structured LLM proposal generation -- deterministic and LLM validators -- validator cardinality enforcement in pipeline execution -- proposal application integrated per module stage -- prompt/response diagnostics for LLM/module stages +Operational caller guidance is documented in [`docs/subprocess-operations.md`](docs/subprocess-operations.md). -Until those phases are implemented, documentation and external descriptions should treat the current Go CLI as deterministic preprocessing/reporting infrastructure, not a full LLM transcript polisher. +## Remaining work (Phase 19) +Remaining rewrite work is documentation/rollout/retirement-oriented: +- repository-level rollout documentation and migration guidance; +- Python retirement/archival decisions and related documentation cleanup; +- final release-facing communication of Go-primary operational guidance. diff --git a/docs/rewrite-notes.md b/docs/rewrite-notes.md index a4fe6fc..1c34469 100644 --- a/docs/rewrite-notes.md +++ b/docs/rewrite-notes.md @@ -31,7 +31,7 @@ The Go rewrite is complete when both of the following are true: ## Current implementation status -The Go rewrite is currently in a deterministic foundation stage. +The Go rewrite now has feature-complete runtime behavior for the module pipeline plus Phase 18 operational hardening coverage. Implemented: - CLI command surface for `audita process`. @@ -97,9 +97,8 @@ Implemented: - validation override behavior when validation fields are set. - Generic JSON diagnostics primitives for LLM interactions (request metadata, request payload, response payload, optional error payload) with secret redaction. -Not yet implemented in runtime pipeline: -- Operational hardening beyond current Phase 16 runtime/reporting/diagnostics scope. -- Rollout/Python retirement work. +Not yet implemented: +- Phase 19 rollout and Python retirement/documentation transition work. ## Completed phases @@ -237,7 +236,7 @@ Not implemented in Phase 8 (by design): ## Remaining work plan -Next recommended phase: **Phase 18 (operational hardening and subprocess integration)**. +Next recommended phase: **Phase 19 (documentation, rollout, and Python retirement)**. ## Phase 9: Structured LLM client and scheduler infrastructure @@ -628,6 +627,38 @@ The Go binary should be safe to call from other Go applications and should not r - Operational docs are accurate. - `go test ./...` passes. +### Phase 18 completion status + +Completed. + +Implemented: +- Expanded subprocess integration coverage in `cmd/audita/main_integration_test.go` for: + - successful default full-pipeline runs with `--output` and without `--output`; + - successful `--report-json` writes; + - large-transcript subprocess behavior; + - missing/unreadable transcript and missing/malformed glossary failures (portable handling where required); + - unwritable output and unwritable report-json failure behavior (portable handling where required); + - malformed structured LLM response failure behavior; + - synthetic backend LLM failure behavior; + - timeout/cancellation behavior with deterministic context cancellation hooks; + - mid-pipeline failure with partial module progress preserved in reports. +- Hardened subprocess failure stderr output to include diagnostics path when available. +- Added deterministic test-only subprocess LLM/runtime hooks used only in helper-process tests: + - backend failure mode; + - malformed structured-output mode; + - block-until-cancel mode; + - mid-pipeline fail mode. +- Added regression coverage for secret redaction across subprocess stdout/stderr/report/diagnostics artifacts. +- Confirmed existing LLM adapter and scheduler tests continue to cover: + - timeout and context cancellation propagation; + - retry behavior; + - malformed output safety; + - permit release on error/cancellation. +- Added focused subprocess-caller operational guidance in `docs/subprocess-operations.md`. + +Intentionally deferred: +- Phase 19 documentation/rollout/Python-retirement transitions. + ## Phase 19: Documentation, rollout, and Python retirement ### Purpose diff --git a/docs/subprocess-operations.md b/docs/subprocess-operations.md new file mode 100644 index 0000000..d2917d5 --- /dev/null +++ b/docs/subprocess-operations.md @@ -0,0 +1,85 @@ +# Audita Go Subprocess Operations + +This document describes how parent processes should invoke `audita process` safely in production orchestration. + +## Recommended command form + +Use explicit file outputs for orchestrated runs: + +```sh +audita process \ + --glossary \ + --output \ + --report-json +``` + +Recommended additions: +- `--work-dir ` to control diagnostics location. +- `--work-dir-retention ` to control retained run directories. +- `--modules ...` only when intentionally overriding the default full sequence. + +## Stdout behavior + +- With `--output`: stdout is expected to be empty on success. +- Without `--output`: stdout contains transcript JSON only on success. +- Report JSON is never written to stdout. + +## Stderr behavior + +- Success path should be quiet or minimal human-readable logs. +- Failure path writes concise human-readable errors. +- When a diagnostics run directory exists, failure stderr includes its path. +- Prompt/response diagnostic payloads are not streamed to stderr. + +## Output file behavior + +- `--output` writes transcript JSON to the provided path. +- Output write failures return nonzero and surface actionable errors. +- The command does not silently ignore output write errors. + +## Report JSON behavior + +- `--report-json` writes a machine-readable process report to the requested path. +- Run-directory `report.json` is written independently under diagnostics. +- Best-effort failure reports are emitted when possible without masking the primary failure. +- Report write failures return nonzero with clear stderr messaging. + +## Diagnostics directory behavior + +- Each run creates (when possible) a per-run diagnostics directory. +- Typical artifacts include transcript, normalization, chunking, invocation, effective config, LLM diagnostics, `report.json`, and `error.log` on failure. +- Failed runs retain diagnostics. +- Under `auto` retention, successful runs with skipped/rejected corrections are retained; clean successful runs may be removed. + +## Exit codes + +- `0`: success. +- Nonzero: failure (input/schema/config/module/LLM/runtime/output/report/diagnostics errors). + +Treat any nonzero as a failed subprocess invocation. + +## Timeout and cancellation + +- Runtime operations propagate context cancellation and request timeouts through LLM/scheduler paths. +- On cancellation or timeout, the process exits nonzero and should not hang. +- If diagnostics were initialized before failure, failure artifacts remain available for debugging. + +## Secret redaction expectations + +API keys and configured secret values are redacted from: +- reports (`--report-json` and run-dir `report.json`); +- diagnostics artifacts (including effective config and LLM interaction artifacts); +- surfaced adapter/runtime errors; +- test fixtures and regression outputs. + +Parent-process logs should still avoid printing raw environment variables. + +## Parent-process pipe guidance + +To avoid deadlocks in orchestrators: +- always read both stdout and stderr concurrently when invoking as a subprocess; +- prefer file outputs (`--output`, `--report-json`) for machine workflows; +- treat stderr as human-readable diagnostics, not structured data; +- parse structured results from output/report files. + +For Go callers, prefer `exec.CommandContext` with explicit timeout/cancellation and buffered/streamed readers for both pipes. diff --git a/internal/cli/run.go b/internal/cli/run.go index bd16b0f..cd4e5b5 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -58,6 +58,9 @@ var processProposalLLMClient contracts.StructuredLLMClient var processProposalLLMScheduler runner.ValidationScheduler var processValidationLLMClient contracts.StructuredLLMClient var processValidationLLMScheduler runner.ValidationScheduler +var processRunnerContext = func() (context.Context, context.CancelFunc) { + return context.Background(), func() {} +} var processRunner = func(inv processInvocation, stdout io.Writer) (*normalization.NormalizationSummary, *chunking.Summary, *runner.RunOutput, *diagnostics.RunDirectory, error) { runDir, err := diagnostics.NewRunDirectory(inv.Config.WorkDir, string(inv.Config.WorkDirRetention)) @@ -214,7 +217,9 @@ var processRunner = func(inv processInvocation, stdout io.Writer) (*normalizatio return fail("runner_setup", err, nil) } - runnerResult, runErr := runner.New(moduleFactory).Run(context.Background(), runner.RunInput{ + runCtx, cancelRun := processRunnerContext() + defer cancelRun() + runnerResult, runErr := runner.New(moduleFactory).Run(runCtx, runner.RunInput{ Config: &inv.Config, Transcript: normalizedTranscript, Glossary: glossary, @@ -434,6 +439,9 @@ func runProcess(args []string, stdout, stderr io.Writer) int { } fmt.Fprintf(stderr, "audita process: %v\n", runErr) + if runDir != nil { + fmt.Fprintf(stderr, "audita process: diagnostics: %s\n", runDir.Path()) + } return 1 } diff --git a/internal/cli/subprocess_test_hooks.go b/internal/cli/subprocess_test_hooks.go new file mode 100644 index 0000000..a3c8e02 --- /dev/null +++ b/internal/cli/subprocess_test_hooks.go @@ -0,0 +1,112 @@ +package cli + +import ( + "context" + "errors" + "os" + "strconv" + "strings" + "sync" + "time" + + "gitea.maximumdirect.net/eric/audita/internal/framework/contracts" + "gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation" + "gitea.maximumdirect.net/eric/audita/internal/framework/validators" +) + +const ( + subprocessTestLLMModeEnv = "AUDITA_SUBPROCESS_TEST_LLM_MODE" + subprocessTestRunTimeoutMSEnv = "AUDITA_SUBPROCESS_TEST_RUN_TIMEOUT_MS" +) + +// ConfigureSubprocessTestHooksFromEnv enables deterministic test-only hooks for +// subprocess integration tests that run through the Go test binary helper path. +func ConfigureSubprocessTestHooksFromEnv() { + if !shouldUseNoOpLLMClientForTests() { + return + } + + mode := strings.TrimSpace(os.Getenv(subprocessTestLLMModeEnv)) + if mode != "" { + client := &subprocessTestLLMClient{mode: mode} + processProposalLLMClient = client + processValidationLLMClient = client + } + + timeoutMSRaw := strings.TrimSpace(os.Getenv(subprocessTestRunTimeoutMSEnv)) + if timeoutMSRaw == "" { + return + } + timeoutMS, err := strconv.Atoi(timeoutMSRaw) + if err != nil || timeoutMS <= 0 { + return + } + processRunnerContext = func() (context.Context, context.CancelFunc) { + return context.WithTimeout(context.Background(), time.Duration(timeoutMS)*time.Millisecond) + } +} + +type subprocessTestLLMClient struct { + mode string + mu sync.Mutex + proposals int +} + +func (c *subprocessTestLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { + _ = req + + switch c.mode { + case "backend_error": + return contracts.StructuredCompletionResponse{}, errors.New("synthetic backend failure") + case "block_until_cancel": + <-ctx.Done() + return contracts.StructuredCompletionResponse{}, ctx.Err() + case "malformed_structured": + switch target := out.(type) { + case *proposal_generation.StructuredCorrectionSet: + *target = proposal_generation.StructuredCorrectionSet{ + Corrections: []proposal_generation.StructuredCorrectionProposal{ + {TargetSegmentID: 0, OriginalText: "x", CorrectedText: "y", Confidence: 0.99}, + }, + } + case *validators.LLMValidationResponse: + *target = validators.LLMValidationResponse{ + Validations: []validators.LLMValidationDecision{ + {CorrectionIndex: 999, Approved: true, Confidence: 0.9, Reason: "bad index"}, + }, + } + } + case "mid_pipeline_fail": + switch target := out.(type) { + case *proposal_generation.StructuredCorrectionSet: + c.mu.Lock() + c.proposals++ + proposalCall := c.proposals + c.mu.Unlock() + + if proposalCall >= 3 { + *target = proposal_generation.StructuredCorrectionSet{ + Corrections: []proposal_generation.StructuredCorrectionProposal{ + {TargetSegmentID: 0, OriginalText: "x", CorrectedText: "y", Confidence: 0.99}, + }, + } + } else { + *target = proposal_generation.StructuredCorrectionSet{ + Corrections: []proposal_generation.StructuredCorrectionProposal{ + {TargetSegmentID: 1, OriginalText: "Segment", CorrectedText: "Segment", Confidence: 0.99}, + }, + } + } + case *validators.LLMValidationResponse: + *target = validators.LLMValidationResponse{Validations: nil} + } + default: + switch target := out.(type) { + case *proposal_generation.StructuredCorrectionSet: + *target = proposal_generation.StructuredCorrectionSet{Corrections: nil} + case *validators.LLMValidationResponse: + *target = validators.LLMValidationResponse{Validations: nil} + } + } + return contracts.StructuredCompletionResponse{}, nil +}