339 lines
12 KiB
Go
339 lines
12 KiB
Go
package cli
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
|
)
|
|
|
|
type parityFixtureCase struct {
|
|
Name string `json:"name"`
|
|
TranscriptFile string `json:"transcript_file"`
|
|
GlossaryFile string `json:"glossary_file"`
|
|
ModulesCSV string `json:"modules_csv,omitempty"`
|
|
ProposalResponsesFile string `json:"proposal_responses_file,omitempty"`
|
|
ValidationResponsesFile string `json:"validation_responses_file,omitempty"`
|
|
Env map[string]string `json:"env,omitempty"`
|
|
Expect parityExpectation `json:"expect"`
|
|
}
|
|
|
|
type parityExpectation struct {
|
|
ExitCode int `json:"exit_code"`
|
|
Status string `json:"status,omitempty"`
|
|
ErrorPhase string `json:"error_phase,omitempty"`
|
|
StdoutMode string `json:"stdout_mode,omitempty"` // empty|json
|
|
StderrContains string `json:"stderr_contains,omitempty"`
|
|
OutputTranscriptFile string `json:"output_transcript_file,omitempty"`
|
|
ModuleInstances []string `json:"module_instances,omitempty"`
|
|
ModuleCount int `json:"module_count,omitempty"`
|
|
TotalAppliedChanges int `json:"total_applied_changes,omitempty"`
|
|
TotalSkippedChanges int `json:"total_skipped_changes,omitempty"`
|
|
FailedModuleInstance string `json:"failed_module_instance,omitempty"`
|
|
ValidatorRejectedReasonCodes []string `json:"validator_rejected_reason_codes,omitempty"`
|
|
ApplicationSkipReasonCodes []string `json:"application_skip_reason_codes,omitempty"`
|
|
RequireErrorLog bool `json:"require_error_log,omitempty"`
|
|
SecretMarkers []string `json:"secret_markers,omitempty"`
|
|
ExpectedProposalCalls []string `json:"expected_proposal_calls,omitempty"`
|
|
ExpectedValidationCalls []string `json:"expected_validation_calls,omitempty"`
|
|
ModuleAppliedCounts []int `json:"module_applied_counts,omitempty"`
|
|
ModuleRejectedCounts []int `json:"module_rejected_counts,omitempty"`
|
|
ModuleSkipCounts []int `json:"module_skip_counts,omitempty"`
|
|
MinResponsePayloadArtifacts int `json:"min_response_payload_artifacts,omitempty"`
|
|
}
|
|
|
|
func TestParityFixtures(t *testing.T) {
|
|
casePaths, err := filepath.Glob(parityFixturePath("*.case.json"))
|
|
if err != nil {
|
|
t.Fatalf("glob parity fixtures: %v", err)
|
|
}
|
|
if len(casePaths) == 0 {
|
|
t.Fatal("expected at least one parity fixture case")
|
|
}
|
|
|
|
for _, casePath := range casePaths {
|
|
fx := loadParityFixtureCase(t, casePath)
|
|
t.Run(fx.Name, func(t *testing.T) {
|
|
runParityFixtureCase(t, filepath.Dir(casePath), fx)
|
|
})
|
|
}
|
|
}
|
|
|
|
func loadParityFixtureCase(t *testing.T, casePath string) parityFixtureCase {
|
|
t.Helper()
|
|
var fx parityFixtureCase
|
|
raw := readFile(t, casePath)
|
|
if err := json.Unmarshal(raw, &fx); err != nil {
|
|
t.Fatalf("parse parity case %q: %v", casePath, err)
|
|
}
|
|
if strings.TrimSpace(fx.Name) == "" {
|
|
t.Fatalf("parity case %q missing name", casePath)
|
|
}
|
|
return fx
|
|
}
|
|
|
|
func runParityFixtureCase(t *testing.T, caseDir string, fx parityFixtureCase) {
|
|
t.Helper()
|
|
for k, v := range fx.Env {
|
|
t.Setenv(k, v)
|
|
}
|
|
|
|
proposalClient := &fakeStructuredLLMClient{}
|
|
validationClient := &fakeStructuredLLMClient{}
|
|
|
|
if strings.TrimSpace(fx.ProposalResponsesFile) != "" {
|
|
raw := readFile(t, filepath.Join(caseDir, fx.ProposalResponsesFile))
|
|
if err := json.Unmarshal(raw, &proposalClient.proposalResponses); err != nil {
|
|
t.Fatalf("parse proposal responses: %v", err)
|
|
}
|
|
processProposalLLMClient = proposalClient
|
|
}
|
|
if strings.TrimSpace(fx.ValidationResponsesFile) != "" {
|
|
raw := readFile(t, filepath.Join(caseDir, fx.ValidationResponsesFile))
|
|
if err := json.Unmarshal(raw, &validationClient.validationResponses); err != nil {
|
|
t.Fatalf("parse validation responses: %v", err)
|
|
}
|
|
processValidationLLMClient = validationClient
|
|
}
|
|
t.Cleanup(func() {
|
|
processProposalLLMClient = nil
|
|
processValidationLLMClient = nil
|
|
})
|
|
|
|
var stdout, stderr bytes.Buffer
|
|
workDir := t.TempDir()
|
|
outputPath := filepath.Join(t.TempDir(), "out.json")
|
|
reportPath := filepath.Join(t.TempDir(), "report.json")
|
|
args := []string{
|
|
"process",
|
|
filepath.Join(caseDir, fx.TranscriptFile),
|
|
"--glossary",
|
|
filepath.Join(caseDir, fx.GlossaryFile),
|
|
"--report-json",
|
|
reportPath,
|
|
"--work-dir",
|
|
workDir,
|
|
"--work-dir-retention",
|
|
"always",
|
|
}
|
|
// Keep stdout shape deterministic for parity tests.
|
|
if fx.Expect.StdoutMode != "json" {
|
|
args = append(args, "--output", outputPath)
|
|
}
|
|
if strings.TrimSpace(fx.ModulesCSV) != "" {
|
|
args = append(args, "--modules", fx.ModulesCSV)
|
|
}
|
|
|
|
exitCode := Run(args, &stdout, &stderr)
|
|
if exitCode != fx.Expect.ExitCode {
|
|
t.Fatalf("expected exit code %d, got %d stderr=%q", fx.Expect.ExitCode, exitCode, stderr.String())
|
|
}
|
|
|
|
switch fx.Expect.StdoutMode {
|
|
case "json":
|
|
if _, err := json.Marshal(stdout.String()); err != nil {
|
|
t.Fatalf("unexpected stdout marshal error: %v", err)
|
|
}
|
|
if !json.Valid(stdout.Bytes()) {
|
|
t.Fatalf("expected JSON stdout, got %q", stdout.String())
|
|
}
|
|
default:
|
|
if stdout.Len() != 0 {
|
|
t.Fatalf("expected empty stdout, got %q", stdout.String())
|
|
}
|
|
}
|
|
|
|
if fx.Expect.StderrContains != "" && !strings.Contains(stderr.String(), fx.Expect.StderrContains) {
|
|
t.Fatalf("expected stderr to contain %q, got %q", fx.Expect.StderrContains, stderr.String())
|
|
}
|
|
|
|
report := readProcessReport(t, reportPath)
|
|
assertParityReport(t, report, fx.Expect)
|
|
|
|
runDir := onlyRunDir(t, workDir)
|
|
runDirReport := readProcessReport(t, filepath.Join(runDir, "report.json"))
|
|
assertParityReport(t, runDirReport, fx.Expect)
|
|
|
|
if fx.Expect.RequireErrorLog {
|
|
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
|
|
t.Fatalf("expected error.log: %v", err)
|
|
}
|
|
}
|
|
|
|
if strings.TrimSpace(fx.Expect.OutputTranscriptFile) != "" && fx.Expect.ExitCode == 0 {
|
|
got := readFile(t, outputPath)
|
|
want := readFile(t, filepath.Join(caseDir, fx.Expect.OutputTranscriptFile))
|
|
assertJSONSemanticEqual(t, want, got)
|
|
}
|
|
|
|
if len(fx.Expect.ExpectedProposalCalls) > 0 && !reflect.DeepEqual(proposalClient.calls, fx.Expect.ExpectedProposalCalls) {
|
|
t.Fatalf("unexpected proposal calls: got %v want %v", proposalClient.calls, fx.Expect.ExpectedProposalCalls)
|
|
}
|
|
if len(fx.Expect.ExpectedValidationCalls) > 0 && !reflect.DeepEqual(validationClient.calls, fx.Expect.ExpectedValidationCalls) {
|
|
t.Fatalf("unexpected validation calls: got %v want %v", validationClient.calls, fx.Expect.ExpectedValidationCalls)
|
|
}
|
|
|
|
if len(fx.Expect.SecretMarkers) > 0 {
|
|
assertNoSecretMarkers(t, reportPath, fx.Expect.SecretMarkers)
|
|
assertNoSecretMarkersInTree(t, runDir, fx.Expect.SecretMarkers)
|
|
}
|
|
if fx.Expect.MinResponsePayloadArtifacts > 0 {
|
|
matches, err := filepath.Glob(filepath.Join(runDir, "*", "*response-payload.json"))
|
|
if err != nil {
|
|
t.Fatalf("glob response payload artifacts: %v", err)
|
|
}
|
|
if len(matches) < fx.Expect.MinResponsePayloadArtifacts {
|
|
t.Fatalf("expected at least %d response payload artifacts, got %d", fx.Expect.MinResponsePayloadArtifacts, len(matches))
|
|
}
|
|
}
|
|
}
|
|
|
|
func assertParityReport(t *testing.T, report reporting.ProcessReport, exp parityExpectation) {
|
|
t.Helper()
|
|
if exp.Status != "" && report.Status != exp.Status {
|
|
t.Fatalf("expected report status %q, got %q", exp.Status, report.Status)
|
|
}
|
|
if exp.ErrorPhase != "" && report.ErrorPhase != exp.ErrorPhase {
|
|
t.Fatalf("expected report error_phase %q, got %q", exp.ErrorPhase, report.ErrorPhase)
|
|
}
|
|
if len(exp.ModuleInstances) > 0 {
|
|
got := make([]string, 0, len(report.ModuleResults))
|
|
for _, mr := range report.ModuleResults {
|
|
got = append(got, mr.ModuleInstance)
|
|
}
|
|
if !reflect.DeepEqual(got, exp.ModuleInstances) {
|
|
t.Fatalf("unexpected module instances: got %v want %v", got, exp.ModuleInstances)
|
|
}
|
|
}
|
|
if exp.ModuleCount > 0 {
|
|
if report.ModulesSummary == nil || report.ModulesSummary.ModuleCount != exp.ModuleCount {
|
|
t.Fatalf("expected module_count=%d, got %+v", exp.ModuleCount, report.ModulesSummary)
|
|
}
|
|
}
|
|
if exp.TotalAppliedChanges > 0 {
|
|
if report.ModulesSummary == nil || report.ModulesSummary.TotalAppliedChanges != exp.TotalAppliedChanges {
|
|
t.Fatalf("expected total_applied_changes=%d, got %+v", exp.TotalAppliedChanges, report.ModulesSummary)
|
|
}
|
|
}
|
|
if exp.TotalSkippedChanges > 0 {
|
|
if report.ModulesSummary == nil || report.ModulesSummary.TotalSkippedChanges != exp.TotalSkippedChanges {
|
|
t.Fatalf("expected total_skipped_changes=%d, got %+v", exp.TotalSkippedChanges, report.ModulesSummary)
|
|
}
|
|
}
|
|
if exp.FailedModuleInstance != "" {
|
|
if report.ModulesSummary == nil || report.ModulesSummary.FailedModuleInstance != exp.FailedModuleInstance {
|
|
t.Fatalf("expected failed_module_instance=%q, got %+v", exp.FailedModuleInstance, report.ModulesSummary)
|
|
}
|
|
}
|
|
|
|
if len(exp.ValidatorRejectedReasonCodes) > 0 {
|
|
got := collectValidatorRejectedReasonCodes(report.ModuleResults)
|
|
if !reflect.DeepEqual(got, exp.ValidatorRejectedReasonCodes) {
|
|
t.Fatalf("unexpected validator rejected reason codes: got %v want %v", got, exp.ValidatorRejectedReasonCodes)
|
|
}
|
|
}
|
|
if len(exp.ApplicationSkipReasonCodes) > 0 {
|
|
got := collectApplicationSkipReasonCodes(report.ModuleResults)
|
|
if !reflect.DeepEqual(got, exp.ApplicationSkipReasonCodes) {
|
|
t.Fatalf("unexpected application skip reason codes: got %v want %v", got, exp.ApplicationSkipReasonCodes)
|
|
}
|
|
}
|
|
if len(exp.ModuleAppliedCounts) > 0 {
|
|
got := make([]int, 0, len(report.ModuleResults))
|
|
for _, mr := range report.ModuleResults {
|
|
got = append(got, len(mr.AppliedChanges))
|
|
}
|
|
if !reflect.DeepEqual(got, exp.ModuleAppliedCounts) {
|
|
t.Fatalf("unexpected per-module applied counts: got %v want %v", got, exp.ModuleAppliedCounts)
|
|
}
|
|
}
|
|
if len(exp.ModuleRejectedCounts) > 0 {
|
|
got := make([]int, 0, len(report.ModuleResults))
|
|
for _, mr := range report.ModuleResults {
|
|
got = append(got, len(mr.ValidatorRejected))
|
|
}
|
|
if !reflect.DeepEqual(got, exp.ModuleRejectedCounts) {
|
|
t.Fatalf("unexpected per-module rejected counts: got %v want %v", got, exp.ModuleRejectedCounts)
|
|
}
|
|
}
|
|
if len(exp.ModuleSkipCounts) > 0 {
|
|
got := make([]int, 0, len(report.ModuleResults))
|
|
for _, mr := range report.ModuleResults {
|
|
got = append(got, len(mr.SkippedChanges))
|
|
}
|
|
if !reflect.DeepEqual(got, exp.ModuleSkipCounts) {
|
|
t.Fatalf("unexpected per-module skip counts: got %v want %v", got, exp.ModuleSkipCounts)
|
|
}
|
|
}
|
|
}
|
|
|
|
func collectValidatorRejectedReasonCodes(results []reporting.ModuleReport) []string {
|
|
out := make([]string, 0)
|
|
for _, mr := range results {
|
|
for _, vr := range mr.ValidatorRejected {
|
|
out = append(out, vr.ReasonCode)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func collectApplicationSkipReasonCodes(results []reporting.ModuleReport) []string {
|
|
out := make([]string, 0)
|
|
for _, mr := range results {
|
|
for _, sk := range mr.SkippedChanges {
|
|
out = append(out, string(sk.SkipReason))
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func assertJSONSemanticEqual(t *testing.T, expected []byte, actual []byte) {
|
|
t.Helper()
|
|
var exp any
|
|
var act any
|
|
if err := json.Unmarshal(expected, &exp); err != nil {
|
|
t.Fatalf("unmarshal expected json: %v", err)
|
|
}
|
|
if err := json.Unmarshal(actual, &act); err != nil {
|
|
t.Fatalf("unmarshal actual json: %v", err)
|
|
}
|
|
if !reflect.DeepEqual(exp, act) {
|
|
t.Fatalf("JSON mismatch\nexpected=%s\nactual=%s", string(expected), string(actual))
|
|
}
|
|
}
|
|
|
|
func assertNoSecretMarkers(t *testing.T, filePath string, markers []string) {
|
|
t.Helper()
|
|
raw := string(readFile(t, filePath))
|
|
for _, marker := range markers {
|
|
if marker != "" && strings.Contains(raw, marker) {
|
|
t.Fatalf("secret marker %q leaked in %s", marker, filePath)
|
|
}
|
|
}
|
|
}
|
|
|
|
func assertNoSecretMarkersInTree(t *testing.T, root string, markers []string) {
|
|
t.Helper()
|
|
_ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
|
if err != nil || d == nil || d.IsDir() {
|
|
return nil
|
|
}
|
|
raw := string(readFile(t, path))
|
|
for _, marker := range markers {
|
|
if marker != "" && strings.Contains(raw, marker) {
|
|
t.Fatalf("secret marker %q leaked in %s", marker, path)
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func parityFixturePath(name string) string {
|
|
return filepath.Join("testdata", "parity", name)
|
|
}
|