package cli import ( "bytes" "encoding/json" "os" "path/filepath" "reflect" "strings" "testing" "gitea.maximumdirect.net/eric/audita/internal/core/reporting" ) type parityFixtureCase struct { Name string `json:"name"` TranscriptFile string `json:"transcript_file"` GlossaryFile string `json:"glossary_file"` ModulesCSV string `json:"modules_csv,omitempty"` ProposalResponsesFile string `json:"proposal_responses_file,omitempty"` ValidationResponsesFile string `json:"validation_responses_file,omitempty"` Env map[string]string `json:"env,omitempty"` Expect parityExpectation `json:"expect"` } type parityExpectation struct { ExitCode int `json:"exit_code"` Status string `json:"status,omitempty"` ErrorPhase string `json:"error_phase,omitempty"` StdoutMode string `json:"stdout_mode,omitempty"` // empty|json StderrContains string `json:"stderr_contains,omitempty"` OutputTranscriptFile string `json:"output_transcript_file,omitempty"` ModuleInstances []string `json:"module_instances,omitempty"` ModuleCount int `json:"module_count,omitempty"` TotalAppliedChanges int `json:"total_applied_changes,omitempty"` TotalSkippedChanges int `json:"total_skipped_changes,omitempty"` FailedModuleInstance string `json:"failed_module_instance,omitempty"` ValidatorRejectedReasonCodes []string `json:"validator_rejected_reason_codes,omitempty"` ApplicationSkipReasonCodes []string `json:"application_skip_reason_codes,omitempty"` RequireErrorLog bool `json:"require_error_log,omitempty"` SecretMarkers []string `json:"secret_markers,omitempty"` ExpectedProposalCalls []string `json:"expected_proposal_calls,omitempty"` ExpectedValidationCalls []string `json:"expected_validation_calls,omitempty"` ModuleAppliedCounts []int `json:"module_applied_counts,omitempty"` ModuleRejectedCounts []int `json:"module_rejected_counts,omitempty"` ModuleSkipCounts []int `json:"module_skip_counts,omitempty"` MinResponsePayloadArtifacts int `json:"min_response_payload_artifacts,omitempty"` } func TestParityFixtures(t *testing.T) { casePaths, err := filepath.Glob(parityFixturePath("*.case.json")) if err != nil { t.Fatalf("glob parity fixtures: %v", err) } if len(casePaths) == 0 { t.Fatal("expected at least one parity fixture case") } for _, casePath := range casePaths { fx := loadParityFixtureCase(t, casePath) t.Run(fx.Name, func(t *testing.T) { runParityFixtureCase(t, filepath.Dir(casePath), fx) }) } } func loadParityFixtureCase(t *testing.T, casePath string) parityFixtureCase { t.Helper() var fx parityFixtureCase raw := readFile(t, casePath) if err := json.Unmarshal(raw, &fx); err != nil { t.Fatalf("parse parity case %q: %v", casePath, err) } if strings.TrimSpace(fx.Name) == "" { t.Fatalf("parity case %q missing name", casePath) } return fx } func runParityFixtureCase(t *testing.T, caseDir string, fx parityFixtureCase) { t.Helper() for k, v := range fx.Env { t.Setenv(k, v) } proposalClient := &fakeStructuredLLMClient{} validationClient := &fakeStructuredLLMClient{} if strings.TrimSpace(fx.ProposalResponsesFile) != "" { raw := readFile(t, filepath.Join(caseDir, fx.ProposalResponsesFile)) if err := json.Unmarshal(raw, &proposalClient.proposalResponses); err != nil { t.Fatalf("parse proposal responses: %v", err) } processProposalLLMClient = proposalClient } if strings.TrimSpace(fx.ValidationResponsesFile) != "" { raw := readFile(t, filepath.Join(caseDir, fx.ValidationResponsesFile)) if err := json.Unmarshal(raw, &validationClient.validationResponses); err != nil { t.Fatalf("parse validation responses: %v", err) } processValidationLLMClient = validationClient } t.Cleanup(func() { processProposalLLMClient = nil processValidationLLMClient = nil }) var stdout, stderr bytes.Buffer workDir := t.TempDir() outputPath := filepath.Join(t.TempDir(), "out.json") reportPath := filepath.Join(t.TempDir(), "report.json") args := []string{ "process", filepath.Join(caseDir, fx.TranscriptFile), "--glossary", filepath.Join(caseDir, fx.GlossaryFile), "--report-json", reportPath, "--work-dir", workDir, "--work-dir-retention", "always", } // Keep stdout shape deterministic for parity tests. if fx.Expect.StdoutMode != "json" { args = append(args, "--output", outputPath) } if strings.TrimSpace(fx.ModulesCSV) != "" { args = append(args, "--modules", fx.ModulesCSV) } exitCode := Run(args, &stdout, &stderr) if exitCode != fx.Expect.ExitCode { t.Fatalf("expected exit code %d, got %d stderr=%q", fx.Expect.ExitCode, exitCode, stderr.String()) } switch fx.Expect.StdoutMode { case "json": if _, err := json.Marshal(stdout.String()); err != nil { t.Fatalf("unexpected stdout marshal error: %v", err) } if !json.Valid(stdout.Bytes()) { t.Fatalf("expected JSON stdout, got %q", stdout.String()) } default: if stdout.Len() != 0 { t.Fatalf("expected empty stdout, got %q", stdout.String()) } } if fx.Expect.StderrContains != "" && !strings.Contains(stderr.String(), fx.Expect.StderrContains) { t.Fatalf("expected stderr to contain %q, got %q", fx.Expect.StderrContains, stderr.String()) } report := readProcessReport(t, reportPath) assertParityReport(t, report, fx.Expect) runDir := onlyRunDir(t, workDir) runDirReport := readProcessReport(t, filepath.Join(runDir, "report.json")) assertParityReport(t, runDirReport, fx.Expect) if fx.Expect.RequireErrorLog { if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil { t.Fatalf("expected error.log: %v", err) } } if strings.TrimSpace(fx.Expect.OutputTranscriptFile) != "" && fx.Expect.ExitCode == 0 { got := readFile(t, outputPath) want := readFile(t, filepath.Join(caseDir, fx.Expect.OutputTranscriptFile)) assertJSONSemanticEqual(t, want, got) } if len(fx.Expect.ExpectedProposalCalls) > 0 && !reflect.DeepEqual(proposalClient.calls, fx.Expect.ExpectedProposalCalls) { t.Fatalf("unexpected proposal calls: got %v want %v", proposalClient.calls, fx.Expect.ExpectedProposalCalls) } if len(fx.Expect.ExpectedValidationCalls) > 0 && !reflect.DeepEqual(validationClient.calls, fx.Expect.ExpectedValidationCalls) { t.Fatalf("unexpected validation calls: got %v want %v", validationClient.calls, fx.Expect.ExpectedValidationCalls) } if len(fx.Expect.SecretMarkers) > 0 { assertNoSecretMarkers(t, reportPath, fx.Expect.SecretMarkers) assertNoSecretMarkersInTree(t, runDir, fx.Expect.SecretMarkers) } if fx.Expect.MinResponsePayloadArtifacts > 0 { matches, err := filepath.Glob(filepath.Join(runDir, "*", "*response-payload.json")) if err != nil { t.Fatalf("glob response payload artifacts: %v", err) } if len(matches) < fx.Expect.MinResponsePayloadArtifacts { t.Fatalf("expected at least %d response payload artifacts, got %d", fx.Expect.MinResponsePayloadArtifacts, len(matches)) } } } func assertParityReport(t *testing.T, report reporting.ProcessReport, exp parityExpectation) { t.Helper() if exp.Status != "" && report.Status != exp.Status { t.Fatalf("expected report status %q, got %q", exp.Status, report.Status) } if exp.ErrorPhase != "" && report.ErrorPhase != exp.ErrorPhase { t.Fatalf("expected report error_phase %q, got %q", exp.ErrorPhase, report.ErrorPhase) } if len(exp.ModuleInstances) > 0 { got := make([]string, 0, len(report.ModuleResults)) for _, mr := range report.ModuleResults { got = append(got, mr.ModuleInstance) } if !reflect.DeepEqual(got, exp.ModuleInstances) { t.Fatalf("unexpected module instances: got %v want %v", got, exp.ModuleInstances) } } if exp.ModuleCount > 0 { if report.ModulesSummary == nil || report.ModulesSummary.ModuleCount != exp.ModuleCount { t.Fatalf("expected module_count=%d, got %+v", exp.ModuleCount, report.ModulesSummary) } } if exp.TotalAppliedChanges > 0 { if report.ModulesSummary == nil || report.ModulesSummary.TotalAppliedChanges != exp.TotalAppliedChanges { t.Fatalf("expected total_applied_changes=%d, got %+v", exp.TotalAppliedChanges, report.ModulesSummary) } } if exp.TotalSkippedChanges > 0 { if report.ModulesSummary == nil || report.ModulesSummary.TotalSkippedChanges != exp.TotalSkippedChanges { t.Fatalf("expected total_skipped_changes=%d, got %+v", exp.TotalSkippedChanges, report.ModulesSummary) } } if exp.FailedModuleInstance != "" { if report.ModulesSummary == nil || report.ModulesSummary.FailedModuleInstance != exp.FailedModuleInstance { t.Fatalf("expected failed_module_instance=%q, got %+v", exp.FailedModuleInstance, report.ModulesSummary) } } if len(exp.ValidatorRejectedReasonCodes) > 0 { got := collectValidatorRejectedReasonCodes(report.ModuleResults) if !reflect.DeepEqual(got, exp.ValidatorRejectedReasonCodes) { t.Fatalf("unexpected validator rejected reason codes: got %v want %v", got, exp.ValidatorRejectedReasonCodes) } } if len(exp.ApplicationSkipReasonCodes) > 0 { got := collectApplicationSkipReasonCodes(report.ModuleResults) if !reflect.DeepEqual(got, exp.ApplicationSkipReasonCodes) { t.Fatalf("unexpected application skip reason codes: got %v want %v", got, exp.ApplicationSkipReasonCodes) } } if len(exp.ModuleAppliedCounts) > 0 { got := make([]int, 0, len(report.ModuleResults)) for _, mr := range report.ModuleResults { got = append(got, len(mr.AppliedChanges)) } if !reflect.DeepEqual(got, exp.ModuleAppliedCounts) { t.Fatalf("unexpected per-module applied counts: got %v want %v", got, exp.ModuleAppliedCounts) } } if len(exp.ModuleRejectedCounts) > 0 { got := make([]int, 0, len(report.ModuleResults)) for _, mr := range report.ModuleResults { got = append(got, len(mr.ValidatorRejected)) } if !reflect.DeepEqual(got, exp.ModuleRejectedCounts) { t.Fatalf("unexpected per-module rejected counts: got %v want %v", got, exp.ModuleRejectedCounts) } } if len(exp.ModuleSkipCounts) > 0 { got := make([]int, 0, len(report.ModuleResults)) for _, mr := range report.ModuleResults { got = append(got, len(mr.SkippedChanges)) } if !reflect.DeepEqual(got, exp.ModuleSkipCounts) { t.Fatalf("unexpected per-module skip counts: got %v want %v", got, exp.ModuleSkipCounts) } } } func collectValidatorRejectedReasonCodes(results []reporting.ModuleReport) []string { out := make([]string, 0) for _, mr := range results { for _, vr := range mr.ValidatorRejected { out = append(out, vr.ReasonCode) } } return out } func collectApplicationSkipReasonCodes(results []reporting.ModuleReport) []string { out := make([]string, 0) for _, mr := range results { for _, sk := range mr.SkippedChanges { out = append(out, string(sk.SkipReason)) } } return out } func assertJSONSemanticEqual(t *testing.T, expected []byte, actual []byte) { t.Helper() var exp any var act any if err := json.Unmarshal(expected, &exp); err != nil { t.Fatalf("unmarshal expected json: %v", err) } if err := json.Unmarshal(actual, &act); err != nil { t.Fatalf("unmarshal actual json: %v", err) } if !reflect.DeepEqual(exp, act) { t.Fatalf("JSON mismatch\nexpected=%s\nactual=%s", string(expected), string(actual)) } } func assertNoSecretMarkers(t *testing.T, filePath string, markers []string) { t.Helper() raw := string(readFile(t, filePath)) for _, marker := range markers { if marker != "" && strings.Contains(raw, marker) { t.Fatalf("secret marker %q leaked in %s", marker, filePath) } } } func assertNoSecretMarkersInTree(t *testing.T, root string, markers []string) { t.Helper() _ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { if err != nil || d == nil || d.IsDir() { return nil } raw := string(readFile(t, path)) for _, marker := range markers { if marker != "" && strings.Contains(raw, marker) { t.Fatalf("secret marker %q leaked in %s", marker, path) } } return nil }) } func parityFixturePath(name string) string { return filepath.Join("testdata", "parity", name) }