Adopt Audita v2 configuration and subprocess contract
This commit is contained in:
@@ -2,630 +2,262 @@ package audita
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
||||
)
|
||||
|
||||
func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
||||
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "success")
|
||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||
|
||||
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||
|
||||
wrapper := writeAuditaHelperWrapper(t)
|
||||
totalLLMConcurrency := 3
|
||||
proposalLLMConcurrency := 2
|
||||
validationLLMConcurrency := 2
|
||||
runner, err := NewSubprocessRunner(SubprocessRunnerConfig{
|
||||
Binary: wrapper,
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "super-secret",
|
||||
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary", "homophones", "glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
TranscriptDescription: "Campaign Session 42",
|
||||
ConfigPath: "/etc/audita/config.yml",
|
||||
OutputSchema: "audita-v1",
|
||||
WorkDirRetention: "auto",
|
||||
TotalLLMConcurrency: &totalLLMConcurrency,
|
||||
ProposalLLMConcurrency: &proposalLLMConcurrency,
|
||||
ValidationModel: "openrouter/google/gemma-4-31b-it",
|
||||
ValidationLLMConcurrency: &validationLLMConcurrency,
|
||||
Report: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewSubprocessRunner() error = %v", err)
|
||||
}
|
||||
t.Setenv("OPENAI_KEY_SOURCE", "changed-after-resolution")
|
||||
|
||||
func auditaTestRequest(t *testing.T) PolishRequest {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
req := PolishRequest{
|
||||
GeneratedConfigPath: filepath.Join(dir, "audita.generated.yml"),
|
||||
MergedTranscriptPath: filepath.Join(dir, "base.json"),
|
||||
return PolishRequest{
|
||||
MergedTranscriptPath: filepath.Join(dir, "merged.json"),
|
||||
GlossaryPath: filepath.Join(dir, "glossary.yml"),
|
||||
OutputProcessedPath: filepath.Join(dir, "polished.json"),
|
||||
ReportPath: filepath.Join(dir, "audita.report.json"),
|
||||
WorkDir: filepath.Join(dir, "artifacts", "audita-work"),
|
||||
StdoutLogPath: filepath.Join(dir, "audita.stdout.log"),
|
||||
StderrLogPath: filepath.Join(dir, "audita.stderr.log"),
|
||||
}
|
||||
writeAuditaTestFile(t, req.MergedTranscriptPath, `{"segments":[]}`)
|
||||
writeAuditaTestFile(t, req.GlossaryPath, "terms: []\n")
|
||||
|
||||
res, err := runner.Run(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if res.ProcessedTranscriptPath != req.OutputProcessedPath {
|
||||
t.Fatalf("ProcessedTranscriptPath = %q, want %q", res.ProcessedTranscriptPath, req.OutputProcessedPath)
|
||||
}
|
||||
if res.ReportPath != req.ReportPath {
|
||||
t.Fatalf("ReportPath = %q, want %q", res.ReportPath, req.ReportPath)
|
||||
}
|
||||
if res.WorkDir != req.WorkDir {
|
||||
t.Fatalf("WorkDir = %q, want %q", res.WorkDir, req.WorkDir)
|
||||
}
|
||||
if res.InvokedBinary != wrapper {
|
||||
t.Fatalf("InvokedBinary = %q, want %q", res.InvokedBinary, wrapper)
|
||||
}
|
||||
if res.ExitCode != 0 {
|
||||
t.Fatalf("ExitCode = %d, want 0", res.ExitCode)
|
||||
}
|
||||
if res.Duration <= 0 {
|
||||
t.Fatalf("Duration = %s, want > 0", res.Duration)
|
||||
}
|
||||
assertJSONFileAudita(t, req.OutputProcessedPath)
|
||||
assertJSONFileAudita(t, req.ReportPath)
|
||||
if _, err := os.Stat(req.StdoutLogPath); err != nil {
|
||||
t.Fatalf("stdout log missing: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(req.StderrLogPath); err != nil {
|
||||
t.Fatalf("stderr log missing: %v", err)
|
||||
}
|
||||
|
||||
rec := readAuditaHelperRecord(t, recordPath)
|
||||
wantArgs := []string{
|
||||
"process", req.MergedTranscriptPath,
|
||||
"--glossary", req.GlossaryPath,
|
||||
"--output", req.OutputProcessedPath,
|
||||
"--work-dir", req.WorkDir,
|
||||
"--base-url", "https://openrouter.ai/api/v1",
|
||||
"--model", "openrouter/google/gemma-4-31b-it",
|
||||
"--modules", "glossary,homophones,glossary",
|
||||
"--report-json", req.ReportPath,
|
||||
"--transcript-description", "Campaign Session 42",
|
||||
"--config", "/etc/audita/config.yml",
|
||||
"--output-schema", "audita-v1",
|
||||
"--work-dir-retention", "auto",
|
||||
"--total-llm-concurrency", "3",
|
||||
"--proposal-llm-concurrency", "2",
|
||||
"--validation-model", "openrouter/google/gemma-4-31b-it",
|
||||
"--validation-llm-concurrency", "2",
|
||||
}
|
||||
if strings.Join(rec.Args, "\n") != strings.Join(wantArgs, "\n") {
|
||||
t.Fatalf("args = %#v, want %#v", rec.Args, wantArgs)
|
||||
}
|
||||
if rec.Env["AUDITA_LLM_API_KEY"] != "super-secret" {
|
||||
t.Fatalf("AUDITA_LLM_API_KEY = %q, want propagated secret", rec.Env["AUDITA_LLM_API_KEY"])
|
||||
}
|
||||
if rec.Env["AUDITA_LLM_CONCURRENCY"] != "" {
|
||||
t.Fatalf("AUDITA_LLM_CONCURRENCY = %q, want empty/omitted", rec.Env["AUDITA_LLM_CONCURRENCY"])
|
||||
}
|
||||
|
||||
cfgData, err := os.ReadFile(req.GeneratedConfigPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read generated config: %v", err)
|
||||
}
|
||||
if strings.Contains(string(cfgData), "super-secret") {
|
||||
t.Fatalf("generated config must not contain credential value")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerRejectsIncompleteResolvedCredential(t *testing.T) {
|
||||
_, err := NewSubprocessRunner(SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyReference: "MISSING_AUDITA_KEY",
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "credential value and reference must be provided together") {
|
||||
t.Fatalf("NewSubprocessRunner() error = %v, want credential pair error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerUnconfiguredCredentialEnvOmitsCredential(t *testing.T) {
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "success")
|
||||
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
|
||||
res, err := runner.Run(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if res.Metadata["credential_present"] != false {
|
||||
t.Fatalf("credential_present = %#v, want false", res.Metadata["credential_present"])
|
||||
}
|
||||
rec := readAuditaHelperRecord(t, recordPath)
|
||||
if rec.Env["AUDITA_LLM_API_KEY"] != "" {
|
||||
t.Fatalf("AUDITA_LLM_API_KEY = %q, want omitted/empty", rec.Env["AUDITA_LLM_API_KEY"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerOmitsUnspecifiedParentEnvironment(t *testing.T) {
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "success")
|
||||
t.Setenv("AUDITA_INHERITED_MARKER", "inherited-from-parent")
|
||||
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
LLMAPIKey: "",
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
if _, err := runner.Run(context.Background(), req); err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
|
||||
rec := readAuditaHelperRecord(t, recordPath)
|
||||
if rec.Env["AUDITA_INHERITED_MARKER"] != "" {
|
||||
t.Fatalf("AUDITA_INHERITED_MARKER = %q, want omitted from the child environment", rec.Env["AUDITA_INHERITED_MARKER"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerOmitsModulesFlagWhenNotConfigured(t *testing.T) {
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "success")
|
||||
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "",
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
if _, err := runner.Run(context.Background(), req); err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
rec := readAuditaHelperRecord(t, recordPath)
|
||||
for i := 0; i < len(rec.Args); i++ {
|
||||
if rec.Args[i] == "--modules" {
|
||||
t.Fatalf("args contained --modules unexpectedly: %#v", rec.Args)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerOmitsBaseURLAndModelFlagsWhenNotConfigured(t *testing.T) {
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "success")
|
||||
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "",
|
||||
Report: false,
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
if _, err := runner.Run(context.Background(), req); err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
rec := readAuditaHelperRecord(t, recordPath)
|
||||
for i := 0; i < len(rec.Args); i++ {
|
||||
if rec.Args[i] == "--base-url" {
|
||||
t.Fatalf("args contained --base-url unexpectedly: %#v", rec.Args)
|
||||
}
|
||||
if rec.Args[i] == "--model" {
|
||||
t.Fatalf("args contained --model unexpectedly: %#v", rec.Args)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "fail")
|
||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "super-secret",
|
||||
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: true,
|
||||
})
|
||||
req := auditaReqForTest(t, true)
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "run audita process") {
|
||||
t.Fatalf("error = %q, want subprocess context", err.Error())
|
||||
}
|
||||
if !strings.Contains(err.Error(), "exit code") {
|
||||
t.Fatalf("error = %q, want exit code context", err.Error())
|
||||
}
|
||||
if !strings.Contains(err.Error(), req.StdoutLogPath) || !strings.Contains(err.Error(), req.StderrLogPath) {
|
||||
t.Fatalf("error = %q, want stdout/stderr log paths", err.Error())
|
||||
}
|
||||
if strings.Contains(err.Error(), "super-secret") {
|
||||
t.Fatalf("failure diagnostic exposed credential value: %q", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerSubprocessFailureAddsStderrDescriptorHint(t *testing.T) {
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "fail_bad_fd")
|
||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "super-secret",
|
||||
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: true,
|
||||
})
|
||||
req := auditaReqForTest(t, true)
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "hint=audita child process may have started with invalid stderr/stdout descriptors") {
|
||||
t.Fatalf("error = %q, want stderr descriptor hint", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerMissingOutputFails(t *testing.T) {
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "missing_output")
|
||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "super-secret",
|
||||
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "validate audita processed output") {
|
||||
t.Fatalf("error = %q, want output validation context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) {
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "invalid_output")
|
||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "super-secret",
|
||||
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "parse json") {
|
||||
t.Fatalf("error = %q, want parse json context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerSegmentsMissingFails(t *testing.T) {
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "segments_missing")
|
||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "super-secret",
|
||||
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "segments") {
|
||||
t.Fatalf("error = %q, want segments validation context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "invalid_report")
|
||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "super-secret",
|
||||
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: true,
|
||||
})
|
||||
req := auditaReqForTest(t, true)
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "validate audita report output") {
|
||||
t.Fatalf("error = %q, want report validation context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerConstructorValidation(t *testing.T) {
|
||||
_, err := NewSubprocessRunnerFromConfigValues("", "3h", "synthetic-secret", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", "", "", "", "", nil, nil, "", nil, true)
|
||||
if err == nil {
|
||||
t.Fatal("expected binary validation error")
|
||||
}
|
||||
_, err = NewSubprocessRunnerFromConfigValues("audita", "bad", "synthetic-secret", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", "", "", "", "", nil, nil, "", nil, true)
|
||||
if err == nil {
|
||||
t.Fatal("expected timeout parse error")
|
||||
}
|
||||
_, err = NewSubprocessRunner(SubprocessRunnerConfig{
|
||||
Binary: "audita",
|
||||
Timeout: mustParseAuditaDuration(t, "1s"),
|
||||
LLMAPIKey: "synthetic-secret",
|
||||
LLMAPIKeyReference: "AUDITA_LLM_API_KEY",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "://",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected base url validation error")
|
||||
}
|
||||
}
|
||||
|
||||
type auditaHelperRecord struct {
|
||||
Args []string `json:"args"`
|
||||
Env map[string]string `json:"env"`
|
||||
}
|
||||
|
||||
func TestAuditaSubprocessHelper(t *testing.T) {
|
||||
if os.Getenv("GO_WANT_AUDITA_HELPER") != "1" {
|
||||
return
|
||||
}
|
||||
|
||||
args := os.Args
|
||||
start := -1
|
||||
for i := range args {
|
||||
if args[i] == "--" {
|
||||
start = i + 1
|
||||
break
|
||||
}
|
||||
}
|
||||
if start < 0 || start >= len(args) {
|
||||
_, _ = os.Stderr.WriteString("missing -- args separator\n")
|
||||
os.Exit(2)
|
||||
}
|
||||
procArgs := args[start:]
|
||||
outPath := auditaFlagValue(procArgs, "--output")
|
||||
reportPath := auditaFlagValue(procArgs, "--report-json")
|
||||
recordPath := os.Getenv("AUDITA_HELPER_RECORD_PATH")
|
||||
if strings.TrimSpace(recordPath) != "" {
|
||||
rec := auditaHelperRecord{
|
||||
Args: procArgs,
|
||||
Env: map[string]string{
|
||||
"AUDITA_LLM_API_KEY": os.Getenv("AUDITA_LLM_API_KEY"),
|
||||
"AUDITA_LLM_CONCURRENCY": os.Getenv("AUDITA_LLM_CONCURRENCY"),
|
||||
"AUDITA_INHERITED_MARKER": os.Getenv("AUDITA_INHERITED_MARKER"),
|
||||
},
|
||||
}
|
||||
data, _ := json.Marshal(rec)
|
||||
_ = os.MkdirAll(filepath.Dir(recordPath), 0o755)
|
||||
_ = os.WriteFile(recordPath, data, 0o644)
|
||||
}
|
||||
|
||||
mode := os.Getenv("AUDITA_HELPER_MODE")
|
||||
switch mode {
|
||||
case "success":
|
||||
writeAuditaHelperFile(outPath, `{"schema":"audita.processed.v1","segments":[]}`)
|
||||
if reportPath != "" {
|
||||
writeAuditaHelperFile(reportPath, `{"schema":"audita.report.v1","steps":[]}`)
|
||||
}
|
||||
_, _ = os.Stdout.WriteString("audita helper success stdout\n")
|
||||
_, _ = os.Stderr.WriteString("audita helper success stderr\n")
|
||||
os.Exit(0)
|
||||
case "fail":
|
||||
_, _ = os.Stderr.WriteString("audita helper failure\n")
|
||||
os.Exit(8)
|
||||
case "fail_bad_fd":
|
||||
_, _ = os.Stderr.WriteString("OSError: [Errno 9] Bad file descriptor\n")
|
||||
os.Exit(120)
|
||||
case "missing_output":
|
||||
if reportPath != "" {
|
||||
writeAuditaHelperFile(reportPath, `{"schema":"audita.report.v1","steps":[]}`)
|
||||
}
|
||||
os.Exit(0)
|
||||
case "invalid_output":
|
||||
writeAuditaHelperFile(outPath, `not-json`)
|
||||
if reportPath != "" {
|
||||
writeAuditaHelperFile(reportPath, `{"schema":"audita.report.v1","steps":[]}`)
|
||||
}
|
||||
os.Exit(0)
|
||||
case "segments_missing":
|
||||
writeAuditaHelperFile(outPath, `{"schema":"audita.processed.v1"}`)
|
||||
if reportPath != "" {
|
||||
writeAuditaHelperFile(reportPath, `{"schema":"audita.report.v1","steps":[]}`)
|
||||
}
|
||||
os.Exit(0)
|
||||
case "invalid_report":
|
||||
writeAuditaHelperFile(outPath, `{"schema":"audita.processed.v1","segments":[]}`)
|
||||
if reportPath != "" {
|
||||
writeAuditaHelperFile(reportPath, `not-json`)
|
||||
}
|
||||
os.Exit(0)
|
||||
default:
|
||||
_, _ = os.Stderr.WriteString(fmt.Sprintf("unknown helper mode %q\n", mode))
|
||||
os.Exit(2)
|
||||
}
|
||||
}
|
||||
|
||||
func mustAuditaRunner(t *testing.T, cfg SubprocessRunnerConfig) *SubprocessRunner {
|
||||
t.Helper()
|
||||
r, err := NewSubprocessRunner(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewSubprocessRunner() error = %v", err)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func auditaReqForTest(t *testing.T, withReport bool) PolishRequest {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
merged := filepath.Join(dir, "base.json")
|
||||
glossary := filepath.Join(dir, "glossary.yml")
|
||||
writeAuditaTestFile(t, merged, `{"segments":[]}`)
|
||||
writeAuditaTestFile(t, glossary, "terms: []\n")
|
||||
req := PolishRequest{
|
||||
GeneratedConfigPath: filepath.Join(dir, "audita.generated.yml"),
|
||||
MergedTranscriptPath: merged,
|
||||
GlossaryPath: glossary,
|
||||
OutputProcessedPath: filepath.Join(dir, "polished.json"),
|
||||
WorkDir: filepath.Join(dir, "artifacts", "audita-work"),
|
||||
StdoutLogPath: filepath.Join(dir, "audita.stdout.log"),
|
||||
StderrLogPath: filepath.Join(dir, "audita.stderr.log"),
|
||||
StdoutLogPath: filepath.Join(dir, "stdout.log"),
|
||||
StderrLogPath: filepath.Join(dir, "stderr.log"),
|
||||
}
|
||||
if withReport {
|
||||
req.ReportPath = filepath.Join(dir, "audita.report.json")
|
||||
}
|
||||
return req
|
||||
}
|
||||
|
||||
func writeAuditaHelperWrapper(t *testing.T) string {
|
||||
func auditaTestRunner(t *testing.T, cfg SubprocessRunnerConfig) *SubprocessRunner {
|
||||
t.Helper()
|
||||
exe, err := os.Executable()
|
||||
if cfg.Binary == "" {
|
||||
cfg.Binary = "audita"
|
||||
}
|
||||
if cfg.Timeout == 0 {
|
||||
cfg.Timeout = time.Second
|
||||
}
|
||||
runner, err := NewSubprocessRunner(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("os.Executable() error = %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := filepath.Join(t.TempDir(), "audita-helper-wrapper.sh")
|
||||
content := "#!/bin/sh\nexec \"" + exe + "\" -test.run=TestAuditaSubprocessHelper -- \"$@\"\n"
|
||||
if err := os.WriteFile(path, []byte(content), 0o755); err != nil {
|
||||
t.Fatalf("WriteFile(%q) error = %v", path, err)
|
||||
}
|
||||
return path
|
||||
return runner
|
||||
}
|
||||
|
||||
func writeAuditaTestFile(t *testing.T, path, contents string) {
|
||||
func auditaTestSecret(t *testing.T, name, value string) configkit.Secret {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll(%q): %v", path, err)
|
||||
t.Setenv(name, value)
|
||||
type loaded struct {
|
||||
Credential configkit.Secret `yaml:"credential"`
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(contents), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(%q): %v", path, err)
|
||||
result, err := configkit.Load(configkit.Bytes("credential.yml", []byte("credential: "+name+"\n")), configkit.Options[loaded]{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return result.Config.Credential
|
||||
}
|
||||
|
||||
func writeAuditaTestFile(t *testing.T, path, value string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(path, []byte(value), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func writeAuditaHelperFile(path, contents string) {
|
||||
if strings.TrimSpace(path) == "" {
|
||||
return
|
||||
}
|
||||
_ = os.MkdirAll(filepath.Dir(path), 0o755)
|
||||
_ = os.WriteFile(path, []byte(contents), 0o644)
|
||||
}
|
||||
|
||||
func auditaFlagValue(args []string, name string) string {
|
||||
for i := 0; i < len(args)-1; i++ {
|
||||
if args[i] == name {
|
||||
return args[i+1]
|
||||
func TestSubprocessRunnerUsesV2InvocationAndCapturedCredentials(t *testing.T) {
|
||||
const first = "NARRATIO_AUDITA_CUSTOM_ONE"
|
||||
const second = "NARRATIO_AUDITA_CUSTOM_TWO"
|
||||
firstSecret := auditaTestSecret(t, first, "loaded-one")
|
||||
secondSecret := auditaTestSecret(t, second, "loaded-two")
|
||||
t.Setenv(first, "parent-changed")
|
||||
t.Setenv(second, "parent-changed-too")
|
||||
t.Setenv("NARRATIO_AUDITA_UNSELECTED", "not-forwarded")
|
||||
runner := auditaTestRunner(t, SubprocessRunnerConfig{
|
||||
Credentials: []configkit.Secret{firstSecret, secondSecret},
|
||||
Modules: []string{"grammar", "glossary", "grammar"},
|
||||
Overlay: "testing", ProposalProfile: "proposal", ValidationProfile: "validation",
|
||||
TranscriptDescription: "campaign", ConfigPath: "/etc/audita/config.yml", OutputSchema: "audita-v1",
|
||||
})
|
||||
req := auditaTestRequest(t)
|
||||
var captured subprocess.RunRequest
|
||||
runner.run = func(_ context.Context, processReq subprocess.RunRequest) (subprocess.RunResult, error) {
|
||||
captured = processReq
|
||||
writeAuditaTestFile(t, req.OutputProcessedPath, `{"segments":[]}`)
|
||||
for _, name := range []string{"errors.yml", "recoveries.yml", "report.md"} {
|
||||
writeAuditaTestFile(t, filepath.Join(filepath.Dir(req.OutputProcessedPath), name), "recoverable diagnostic\n")
|
||||
}
|
||||
return subprocess.RunResult{}, nil
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func readAuditaHelperRecord(t *testing.T, path string) auditaHelperRecord {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
result, err := runner.Run(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(%q): %v", path, err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
var rec auditaHelperRecord
|
||||
if err := json.Unmarshal(data, &rec); err != nil {
|
||||
t.Fatalf("json unmarshal helper record: %v", err)
|
||||
want := []string{"process", req.MergedTranscriptPath, "--glossary", req.GlossaryPath, "--output", req.OutputProcessedPath,
|
||||
"--modules", "grammar,glossary,grammar", "--config", "/etc/audita/config.yml", "--overlay", "testing",
|
||||
"--proposal-profile", "proposal", "--validation-profile", "validation", "--transcript-description", "campaign", "--output-schema", "audita-v1"}
|
||||
if !reflect.DeepEqual(captured.Args, want) {
|
||||
t.Fatalf("args = %#v, want %#v", captured.Args, want)
|
||||
}
|
||||
return rec
|
||||
}
|
||||
|
||||
func assertJSONFileAudita(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if len(captured.EnvOverrides) != 2 || captured.EnvOverrides[first] != "loaded-one" || captured.EnvOverrides[second] != "loaded-two" || captured.EnvOverrides["NARRATIO_AUDITA_UNSELECTED"] != "" {
|
||||
t.Fatalf("credential forwarding = %#v", captured.EnvOverrides)
|
||||
}
|
||||
if !reflect.DeepEqual(captured.SensitiveEnvNames, []string{first, second}) {
|
||||
t.Fatalf("sensitive names = %#v", captured.SensitiveEnvNames)
|
||||
}
|
||||
if result.Reports.ErrorsPath == "" || result.Reports.RecoveriesPath == "" || result.Reports.OverviewPath == "" {
|
||||
t.Fatalf("reports = %#v", result.Reports)
|
||||
}
|
||||
generated, err := os.ReadFile(req.GeneratedConfigPath)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(%q): %v", path, err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
var v any
|
||||
if err := json.Unmarshal(data, &v); err != nil {
|
||||
t.Fatalf("json unmarshal %q: %v", path, err)
|
||||
if !strings.Contains(string(generated), "audita.generated.v2") || !strings.Contains(string(generated), first) || strings.Contains(string(generated), "loaded-one") {
|
||||
t.Fatalf("unsafe or incomplete generated invocation: %s", generated)
|
||||
}
|
||||
}
|
||||
|
||||
func mustParseAuditaDuration(t *testing.T, value string) time.Duration {
|
||||
t.Helper()
|
||||
d, err := time.ParseDuration(value)
|
||||
if err != nil {
|
||||
t.Fatalf("time.ParseDuration(%q) error = %v", value, err)
|
||||
func TestSubprocessRunnerModulePresenceAndReportAbsence(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
configured []string
|
||||
requested []string
|
||||
wantFlag bool
|
||||
wantValue string
|
||||
}{
|
||||
{"omitted", nil, nil, false, ""},
|
||||
{"configured", []string{"grammar", "grammar"}, nil, true, "grammar,grammar"},
|
||||
{"empty override", []string{"grammar"}, []string{}, true, ""},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
runner := auditaTestRunner(t, SubprocessRunnerConfig{Modules: test.configured})
|
||||
req := auditaTestRequest(t)
|
||||
req.Modules = test.requested
|
||||
runner.run = func(_ context.Context, processReq subprocess.RunRequest) (subprocess.RunResult, error) {
|
||||
index := -1
|
||||
for i, arg := range processReq.Args {
|
||||
if arg == "--modules" {
|
||||
index = i
|
||||
}
|
||||
if strings.HasPrefix(arg, "--report") || arg == "--debug" || arg == "--work-dir" {
|
||||
t.Fatalf("retired or unwanted flag %q", arg)
|
||||
}
|
||||
}
|
||||
if (index >= 0) != test.wantFlag || (index >= 0 && processReq.Args[index+1] != test.wantValue) {
|
||||
t.Fatalf("args = %#v", processReq.Args)
|
||||
}
|
||||
writeAuditaTestFile(t, req.OutputProcessedPath, `{"segments":[]}`)
|
||||
return subprocess.RunResult{}, nil
|
||||
}
|
||||
result, err := runner.Run(context.Background(), req)
|
||||
if err != nil || result.Reports != (ReportBundle{}) {
|
||||
t.Fatalf("result = %#v, error = %v", result, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerRejectsMissingOrMalformedTranscript(t *testing.T) {
|
||||
for _, output := range []string{"", `{"segments":"wrong"}`} {
|
||||
t.Run(output, func(t *testing.T) {
|
||||
runner := auditaTestRunner(t, SubprocessRunnerConfig{})
|
||||
req := auditaTestRequest(t)
|
||||
runner.run = func(context.Context, subprocess.RunRequest) (subprocess.RunResult, error) {
|
||||
if output != "" {
|
||||
writeAuditaTestFile(t, req.OutputProcessedPath, output)
|
||||
}
|
||||
return subprocess.RunResult{}, nil
|
||||
}
|
||||
if _, err := runner.Run(context.Background(), req); err == nil {
|
||||
t.Fatal("missing or malformed transcript was accepted")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerPreservesFailureAndAvailableReports(t *testing.T) {
|
||||
runner := auditaTestRunner(t, SubprocessRunnerConfig{})
|
||||
req := auditaTestRequest(t)
|
||||
processFailure := errors.New("audita exited after writing output")
|
||||
runner.run = func(context.Context, subprocess.RunRequest) (subprocess.RunResult, error) {
|
||||
writeAuditaTestFile(t, req.OutputProcessedPath, `{"segments":[]}`)
|
||||
writeAuditaTestFile(t, filepath.Join(filepath.Dir(req.OutputProcessedPath), "errors.yml"), "status: failed\n")
|
||||
return subprocess.RunResult{ExitCode: 7}, processFailure
|
||||
}
|
||||
result, err := runner.Run(context.Background(), req)
|
||||
if !errors.Is(err, processFailure) || !strings.Contains(err.Error(), filepath.Dir(req.OutputProcessedPath)) {
|
||||
t.Fatalf("error = %v", err)
|
||||
}
|
||||
if result.Reports.ErrorsPath == "" || result.Reports.RecoveriesPath != "" || result.Reports.OverviewPath != "" {
|
||||
t.Fatalf("reports = %#v", result.Reports)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerDoesNotMaskProcessFailureWithReportError(t *testing.T) {
|
||||
runner := auditaTestRunner(t, SubprocessRunnerConfig{})
|
||||
req := auditaTestRequest(t)
|
||||
processFailure := errors.New("audita process failed")
|
||||
runner.run = func(context.Context, subprocess.RunRequest) (subprocess.RunResult, error) {
|
||||
path := filepath.Join(filepath.Dir(req.OutputProcessedPath), "errors.yml")
|
||||
if err := os.WriteFile(path, make([]byte, MaxReportOutputBytes+1), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return subprocess.RunResult{ExitCode: 7}, processFailure
|
||||
}
|
||||
if _, err := runner.Run(context.Background(), req); !errors.Is(err, processFailure) {
|
||||
t.Fatalf("error = %v, want original process error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerBoundsAdjacentReports(t *testing.T) {
|
||||
runner := auditaTestRunner(t, SubprocessRunnerConfig{})
|
||||
req := auditaTestRequest(t)
|
||||
runner.run = func(context.Context, subprocess.RunRequest) (subprocess.RunResult, error) {
|
||||
writeAuditaTestFile(t, req.OutputProcessedPath, `{"segments":[]}`)
|
||||
path := filepath.Join(filepath.Dir(req.OutputProcessedPath), "errors.yml")
|
||||
if err := os.WriteFile(path, make([]byte, MaxReportOutputBytes+1), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return subprocess.RunResult{}, nil
|
||||
}
|
||||
if _, err := runner.Run(context.Background(), req); err == nil || !strings.Contains(err.Error(), "errors.yml") || !strings.Contains(err.Error(), "16777216") {
|
||||
t.Fatalf("error = %v, want named report limit", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerRedactsCustomCredentialStreams(t *testing.T) {
|
||||
const first = "NARRATIO_AUDITA_STREAM_ONE"
|
||||
const second = "NARRATIO_AUDITA_STREAM_TWO"
|
||||
for _, fail := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "success", true: "failure"}[fail], func(t *testing.T) {
|
||||
firstSecret := auditaTestSecret(t, first, "audita-first-secret")
|
||||
secondSecret := auditaTestSecret(t, second, "audita-second-secret")
|
||||
req := auditaTestRequest(t)
|
||||
body := "#!/bin/sh\nprintf '%s\\n' \"$" + first + "\"\nprintf '%s\\n' \"$" + second + "\" >&2\n"
|
||||
if fail {
|
||||
body += "exit 7\n"
|
||||
} else {
|
||||
body += "printf '{\"segments\":[]}' > \"" + req.OutputProcessedPath + "\"\n"
|
||||
}
|
||||
binary := filepath.Join(t.TempDir(), "audita-helper")
|
||||
writeAuditaTestFile(t, binary, body)
|
||||
if err := os.Chmod(binary, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runner := auditaTestRunner(t, SubprocessRunnerConfig{Binary: binary, Credentials: []configkit.Secret{firstSecret, secondSecret}})
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
if (err != nil) != fail {
|
||||
t.Fatalf("error = %v, fail = %v", err, fail)
|
||||
}
|
||||
for _, path := range []string{req.StdoutLogPath, req.StderrLogPath} {
|
||||
data, readErr := os.ReadFile(path)
|
||||
if readErr != nil {
|
||||
t.Fatal(readErr)
|
||||
}
|
||||
if strings.Contains(string(data), "audita-first-secret") || strings.Contains(string(data), "audita-second-secret") || !strings.Contains(string(data), "<redacted>") {
|
||||
t.Fatalf("unredacted stream %q: %s", path, data)
|
||||
}
|
||||
}
|
||||
if fail && (strings.Contains(err.Error(), "audita-first-secret") || strings.Contains(err.Error(), "audita-second-secret")) {
|
||||
t.Fatalf("unredacted failure: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user