package cli import ( "bytes" "encoding/json" "os" "path/filepath" "reflect" "strings" "testing" "gitea.maximumdirect.net/eric/audita/internal/core/reporting" "gitea.maximumdirect.net/eric/audita/internal/core/schema" "gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation" "gitea.maximumdirect.net/eric/audita/internal/framework/validators" ) type releaseFixtureExpectations struct { MustApplyTexts []string `json:"must_apply_texts"` MustNotApplyTexts []string `json:"must_not_apply_texts"` ProtectedTerms []string `json:"protected_terms"` ExpectedModuleInstance []string `json:"expected_module_instances"` MinimumCounts struct { Applied int `json:"applied"` Rejected int `json:"rejected"` Skipped int `json:"skipped"` } `json:"minimum_counts"` } func TestReleaseFixtureDefaultPipelineReadiness(t *testing.T) { base := fixturePath(filepath.Join("release", "default-release")) var expectations releaseFixtureExpectations if err := json.Unmarshal(readFile(t, base+".expectations.json"), &expectations); err != nil { t.Fatalf("unmarshal release expectations: %v", err) } proposalResponses := readProposalResponses(t, base+".proposals.json") validationResponses := readValidationResponses(t, base+".validations.json") // First pass: default full pipeline with deterministic fake LLM responses. first := runReleaseFixturePass(t, releaseRunConfig{ transcriptPath: base + ".transcript.json", glossaryPath: base + ".glossary.yaml", outputSchema: "bare-segments", proposalResponses: proposalResponses, validationResponses: validationResponses, expectedProposalCalls: []string{"glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", "grammar:proposal"}, reportSchemaName: reporting.DefaultProcessReportSchemaName, reportSchemaVersion: reporting.DefaultProcessReportSchemaVersion, expectedOutputSchema: "bare-segments", expectModuleInstances: expectations.ExpectedModuleInstance, expectUtilizationPaths: true, }) gotTranscript := mustReadTranscript(t, first.outputPath) expectFinalTranscriptContains(t, gotTranscript, expectations.MustApplyTexts) expectFinalTranscriptDoesNotContain(t, gotTranscript, expectations.MustNotApplyTexts) expectFinalTranscriptContains(t, gotTranscript, expectations.ProtectedTerms) assertReleaseCounts(t, first.report, expectations) assertPromptAndSchemaMetadataPresent(t, first.runDir) assertReleaseLedgerShape(t, first.report) assertReleaseUtilizationShape(t, first.report) assertStableValidatorKeysPresent(t, first.report) assertStdoutStderrContract(t, first.stdout, first.stderr) assertNoSecretMarkersInTree(t, first.runDir, []string{"release-secret"}) assertNoSecretMarkers(t, first.reportPath, []string{"release-secret"}) // Output schema check: audita-v1 object payload. auditaV1 := runReleaseFixturePass(t, releaseRunConfig{ transcriptPath: base + ".transcript.json", glossaryPath: base + ".glossary.yaml", outputSchema: "audita-v1", proposalResponses: proposalResponses, validationResponses: validationResponses, expectedProposalCalls: []string{"glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", "grammar:proposal"}, reportSchemaName: reporting.DefaultProcessReportSchemaName, reportSchemaVersion: reporting.DefaultProcessReportSchemaVersion, expectedOutputSchema: "audita-v1", expectModuleInstances: expectations.ExpectedModuleInstance, expectUtilizationPaths: true, }) assertAuditaV1OutputShape(t, auditaV1.outputPath) // Idempotence-oriented second pass: // run again on first output with deterministic no-op responses. noOpProposals := make([]proposal_generation.StructuredCorrectionSet, 5) for i := range noOpProposals { noOpProposals[i] = proposal_generation.StructuredCorrectionSet{Corrections: nil} } second := runReleaseFixturePass(t, releaseRunConfig{ transcriptPath: first.outputPath, glossaryPath: base + ".glossary.yaml", outputSchema: "bare-segments", proposalResponses: noOpProposals, validationResponses: nil, expectedProposalCalls: []string{"glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", "grammar:proposal"}, reportSchemaName: reporting.DefaultProcessReportSchemaName, reportSchemaVersion: reporting.DefaultProcessReportSchemaVersion, expectedOutputSchema: "bare-segments", expectModuleInstances: expectations.ExpectedModuleInstance, expectUtilizationPaths: true, }) firstSegments := mustReadTranscript(t, first.outputPath) secondSegments := mustReadTranscript(t, second.outputPath) if !reflect.DeepEqual(firstSegments, secondSegments) { t.Fatalf("expected idempotent second pass transcript; first=%+v second=%+v", firstSegments, secondSegments) } if second.report.ModulesSummary == nil { t.Fatalf("expected modules summary on second pass") } if second.report.ModulesSummary.TotalAppliedChanges != 0 { t.Fatalf("expected no-op second pass (0 applied), got %+v", second.report.ModulesSummary) } } type releaseRunConfig struct { transcriptPath string glossaryPath string outputSchema string proposalResponses []proposal_generation.StructuredCorrectionSet validationResponses []validators.LLMValidationResponse expectedProposalCalls []string reportSchemaName string reportSchemaVersion string expectedOutputSchema string expectModuleInstances []string expectUtilizationPaths bool } type releaseRunResult struct { stdout string stderr string outputPath string reportPath string report reporting.ProcessReport runDir string } func runReleaseFixturePass(t *testing.T, cfg releaseRunConfig) releaseRunResult { t.Helper() processProposalLLMClient = &fakeStructuredLLMClient{proposalResponses: append([]proposal_generation.StructuredCorrectionSet(nil), cfg.proposalResponses...)} processValidationLLMClient = &fakeStructuredLLMClient{validationResponses: append([]validators.LLMValidationResponse(nil), cfg.validationResponses...)} t.Cleanup(func() { processProposalLLMClient = nil processValidationLLMClient = nil }) workDir := t.TempDir() reportPath := filepath.Join(t.TempDir(), "report.json") outputPath := filepath.Join(t.TempDir(), "out.json") configPath := writeFile(t, "release-config.yml", "version: 1\n") args := []string{ "process", cfg.transcriptPath, "--glossary", cfg.glossaryPath, "--config", configPath, "--output", outputPath, "--output-schema", cfg.outputSchema, "--report-json", reportPath, "--work-dir", workDir, "--work-dir-retention", "always", } var stdout, stderr bytes.Buffer exitCode := Run(args, &stdout, &stderr) if exitCode != 0 { t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String()) } report := readProcessReport(t, reportPath) if report.ReportMetadata.ReportSchemaName != cfg.reportSchemaName || report.ReportMetadata.ReportSchemaVersion != cfg.reportSchemaVersion { t.Fatalf("unexpected report schema metadata: %+v", report.ReportMetadata) } if report.ReportMetadata.OutputSchema != cfg.expectedOutputSchema { t.Fatalf("unexpected output schema metadata: got %q want %q", report.ReportMetadata.OutputSchema, cfg.expectedOutputSchema) } if len(cfg.expectModuleInstances) > 0 { got := make([]string, 0, len(report.ModuleResults)) for _, mr := range report.ModuleResults { got = append(got, mr.ModuleInstance) } if !reflect.DeepEqual(got, cfg.expectModuleInstances) { t.Fatalf("unexpected module instances: got %v want %v", got, cfg.expectModuleInstances) } } if report.Diagnostics == nil { t.Fatalf("expected diagnostics metadata") } if cfg.expectUtilizationPaths { if report.Diagnostics.UtilizationSummaryPath == "" || report.Diagnostics.CorrectionLedgerPath == "" { t.Fatalf("expected utilization/ledger artifact paths in report diagnostics: %+v", report.Diagnostics) } } runDir := onlyRunDir(t, workDir) if _, err := os.Stat(filepath.Join(runDir, "report.json")); err != nil { t.Fatalf("expected run-dir report: %v", err) } if c, ok := processProposalLLMClient.(*fakeStructuredLLMClient); ok { if !reflect.DeepEqual(c.calls, cfg.expectedProposalCalls) { t.Fatalf("unexpected proposal call order: got %v want %v", c.calls, cfg.expectedProposalCalls) } } return releaseRunResult{ stdout: stdout.String(), stderr: stderr.String(), outputPath: outputPath, reportPath: reportPath, report: report, runDir: runDir, } } func readProposalResponses(t *testing.T, path string) []proposal_generation.StructuredCorrectionSet { t.Helper() var out []proposal_generation.StructuredCorrectionSet if err := json.Unmarshal(readFile(t, path), &out); err != nil { t.Fatalf("unmarshal proposal responses: %v", err) } return out } func readValidationResponses(t *testing.T, path string) []validators.LLMValidationResponse { t.Helper() var out []validators.LLMValidationResponse if err := json.Unmarshal(readFile(t, path), &out); err != nil { t.Fatalf("unmarshal validation responses: %v", err) } return out } func mustReadTranscript(t *testing.T, path string) []schema.Segment { t.Helper() transcript, err := schema.ParseTranscriptJSON(readFile(t, path)) if err != nil { t.Fatalf("parse transcript output: %v", err) } return transcript.Segments } func expectFinalTranscriptContains(t *testing.T, segments []schema.Segment, needles []string) { t.Helper() joined := flattenTranscriptText(segments) for _, needle := range needles { if !strings.Contains(joined, needle) { t.Fatalf("expected transcript to contain %q, got %q", needle, joined) } } } func expectFinalTranscriptDoesNotContain(t *testing.T, segments []schema.Segment, needles []string) { t.Helper() joined := flattenTranscriptText(segments) for _, needle := range needles { if strings.Contains(joined, needle) { t.Fatalf("expected transcript to not contain %q, got %q", needle, joined) } } } func flattenTranscriptText(segments []schema.Segment) string { parts := make([]string, 0, len(segments)) for _, s := range segments { parts = append(parts, s.Text) } return strings.Join(parts, "\n") } func assertReleaseCounts(t *testing.T, report reporting.ProcessReport, exp releaseFixtureExpectations) { t.Helper() if report.ModulesSummary == nil { t.Fatalf("expected modules_summary") } if report.ModulesSummary.TotalAppliedChanges < exp.MinimumCounts.Applied { t.Fatalf("expected at least %d applied changes, got %+v", exp.MinimumCounts.Applied, report.ModulesSummary) } validatorRejected := 0 skipped := 0 for _, mr := range report.ModuleResults { validatorRejected += len(mr.ValidatorRejected) skipped += len(mr.SkippedChanges) } if validatorRejected < exp.MinimumCounts.Rejected { t.Fatalf("expected at least %d validator rejections, got %d", exp.MinimumCounts.Rejected, validatorRejected) } if skipped < exp.MinimumCounts.Skipped { t.Fatalf("expected at least %d application skips, got %d", exp.MinimumCounts.Skipped, skipped) } } func assertReleaseUtilizationShape(t *testing.T, report reporting.ProcessReport) { t.Helper() var payload struct { EffectiveConcurrency struct { TotalLLM int `json:"total_llm"` } `json:"effective_concurrency"` RunTiming struct { SchedulerQueueWaitMS int64 `json:"scheduler_queue_wait_ms"` LLMExecutionTimeMS int64 `json:"llm_execution_time_ms"` DeterministicValidationMS int64 `json:"deterministic_validation_time_ms"` } `json:"run_timing"` Modules []map[string]any `json:"modules"` Validators []map[string]any `json:"validators"` } if err := json.Unmarshal(readFile(t, report.Diagnostics.UtilizationSummaryPath), &payload); err != nil { t.Fatalf("unmarshal utilization diagnostics: %v", err) } if payload.EffectiveConcurrency.TotalLLM <= 0 { t.Fatalf("expected positive total llm concurrency, got %+v", payload.EffectiveConcurrency) } if payload.RunTiming.SchedulerQueueWaitMS < 0 || payload.RunTiming.LLMExecutionTimeMS < 0 || payload.RunTiming.DeterministicValidationMS < 0 { t.Fatalf("expected non-negative run timing values, got %+v", payload.RunTiming) } if len(payload.Modules) == 0 { t.Fatalf("expected module timing summaries") } if len(payload.Validators) == 0 { t.Fatalf("expected validator timing summaries") } } func assertReleaseLedgerShape(t *testing.T, report reporting.ProcessReport) { t.Helper() var entries []struct { ModuleKey string `json:"module_key"` ModuleInstance string `json:"module_instance"` ProposalIndex int `json:"proposal_index"` Disposition string `json:"disposition"` DispositionReason string `json:"disposition_reason_code"` OriginalText string `json:"original_text"` ProposedCorrected string `json:"proposed_corrected_text"` ReplacementPolicy string `json:"replacement_policy"` DeterministicResults []struct { ValidatorKey string `json:"validator_key"` } `json:"deterministic_validator_decisions"` LLMResults []struct { ValidatorKey string `json:"validator_key"` } `json:"llm_validator_decisions"` } if err := json.Unmarshal(readFile(t, report.Diagnostics.CorrectionLedgerPath), &entries); err != nil { t.Fatalf("unmarshal correction ledger: %v", err) } if len(entries) == 0 { t.Fatalf("expected correction ledger entries") } hasApplied := false hasRejected := false hasSkipped := false for _, entry := range entries { if entry.ModuleInstance == "" || entry.ModuleKey == "" { t.Fatalf("expected module identity in ledger entry: %+v", entry) } switch entry.Disposition { case "applied": hasApplied = true case "rejected": hasRejected = true case "skipped": hasSkipped = true } } if !hasApplied || !hasRejected { t.Fatalf("expected applied and rejected entries in correction ledger, got %+v", entries) } if !hasSkipped { // Some deterministic fixture paths do not trigger apply-time skips; // rejections are still captured separately from application skips. } } func assertPromptAndSchemaMetadataPresent(t *testing.T, runDir string) { t.Helper() metadataPaths, err := filepath.Glob(filepath.Join(runDir, "*", "*request-metadata.json")) if err != nil { t.Fatalf("glob request metadata artifacts: %v", err) } if len(metadataPaths) == 0 { t.Fatalf("expected request metadata artifacts with prompt metadata") } foundPromptMetadata := false foundSchemaMetadata := false for _, path := range metadataPaths { var payload map[string]any if err := json.Unmarshal(readFile(t, path), &payload); err != nil { t.Fatalf("unmarshal request metadata artifact %q: %v", path, err) } if pm, ok := payload["prompt_metadata"].(map[string]any); ok { if pm["prompt_id"] != nil && pm["prompt_version"] != nil && pm["sha256"] != nil { foundPromptMetadata = true } } if sm, ok := payload["response_schema"].(map[string]any); ok { if sm["id"] != nil && sm["version"] != nil && sm["name"] != nil && sm["sha256"] != nil { foundSchemaMetadata = true } } } if !foundPromptMetadata { t.Fatalf("expected prompt metadata in request metadata artifacts") } if !foundSchemaMetadata { t.Fatalf("expected structured response schema metadata in request metadata artifacts") } } func assertStableValidatorKeysPresent(t *testing.T, report reporting.ProcessReport) { t.Helper() seen := map[string]bool{} for _, module := range report.ModuleResults { for _, decision := range module.ValidatorDecisions { seen[decision.ValidatorName] = true } for _, rejected := range module.ValidatorRejected { seen[rejected.ValidatorName] = true } } expectedAny := []string{ "confidence_threshold", "original_text_presence", "no_effect", } for _, key := range expectedAny { if !seen[key] { t.Fatalf("expected stable validator key %q in report decisions/rejections; seen=%v", key, seen) } } } func assertStdoutStderrContract(t *testing.T, stdout, stderr string) { t.Helper() if stdout != "" { t.Fatalf("expected empty stdout with --output, got %q", stdout) } if strings.Contains(stderr, `"module_results"`) || strings.Contains(stderr, `"report_metadata"`) { t.Fatalf("stderr should remain human-readable, not report JSON: %q", stderr) } } func assertAuditaV1OutputShape(t *testing.T, outputPath string) { t.Helper() var payload struct { Schema string `json:"schema"` Version string `json:"version"` Segments []schema.Segment `json:"segments"` } if err := json.Unmarshal(readFile(t, outputPath), &payload); err != nil { t.Fatalf("unmarshal audita-v1 output: %v", err) } if payload.Schema != "audita-v1" || payload.Version != "v1" { t.Fatalf("unexpected audita-v1 metadata: %+v", payload) } if len(payload.Segments) == 0 { t.Fatalf("expected non-empty audita-v1 segments") } }