Complete pre-1.0 release hardening

internal/cli/release_fixtures_test.go

@@ -0,0 +1,469 @@
+package cli
+
+import (
+	"bytes"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
+	"gitea.maximumdirect.net/eric/audita/internal/core/schema"
+	"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
+	"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
+)
+
+type releaseFixtureExpectations struct {
+	MustApplyTexts         []string `json:"must_apply_texts"`
+	MustNotApplyTexts      []string `json:"must_not_apply_texts"`
+	ProtectedTerms         []string `json:"protected_terms"`
+	ExpectedModuleInstance []string `json:"expected_module_instances"`
+	MinimumCounts          struct {
+		Applied  int `json:"applied"`
+		Rejected int `json:"rejected"`
+		Skipped  int `json:"skipped"`
+	} `json:"minimum_counts"`
+}
+
+func TestReleaseFixtureDefaultPipelineReadiness(t *testing.T) {
+	base := fixturePath(filepath.Join("release", "default-release"))
+
+	var expectations releaseFixtureExpectations
+	if err := json.Unmarshal(readFile(t, base+".expectations.json"), &expectations); err != nil {
+		t.Fatalf("unmarshal release expectations: %v", err)
+	}
+
+	proposalResponses := readProposalResponses(t, base+".proposals.json")
+	validationResponses := readValidationResponses(t, base+".validations.json")
+
+	// First pass: default full pipeline with deterministic fake LLM responses.
+	first := runReleaseFixturePass(t, releaseRunConfig{
+		transcriptPath:         base + ".transcript.json",
+		glossaryPath:           base + ".glossary.yaml",
+		outputSchema:           "bare-segments",
+		proposalResponses:      proposalResponses,
+		validationResponses:    validationResponses,
+		expectedProposalCalls:  []string{"glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", "grammar:proposal"},
+		reportSchemaName:       reporting.DefaultProcessReportSchemaName,
+		reportSchemaVersion:    reporting.DefaultProcessReportSchemaVersion,
+		expectedOutputSchema:   "bare-segments",
+		expectModuleInstances:  expectations.ExpectedModuleInstance,
+		expectUtilizationPaths: true,
+	})
+
+	gotTranscript := mustReadTranscript(t, first.outputPath)
+	expectFinalTranscriptContains(t, gotTranscript, expectations.MustApplyTexts)
+	expectFinalTranscriptDoesNotContain(t, gotTranscript, expectations.MustNotApplyTexts)
+	expectFinalTranscriptContains(t, gotTranscript, expectations.ProtectedTerms)
+
+	assertReleaseCounts(t, first.report, expectations)
+	assertPromptAndSchemaMetadataPresent(t, first.runDir)
+	assertReleaseLedgerShape(t, first.report)
+	assertReleaseUtilizationShape(t, first.report)
+	assertStableValidatorKeysPresent(t, first.report)
+	assertStdoutStderrContract(t, first.stdout, first.stderr)
+	assertNoSecretMarkersInTree(t, first.runDir, []string{"release-secret"})
+	assertNoSecretMarkers(t, first.reportPath, []string{"release-secret"})
+
+	// Output schema check: audita-v1 object payload.
+	auditaV1 := runReleaseFixturePass(t, releaseRunConfig{
+		transcriptPath:         base + ".transcript.json",
+		glossaryPath:           base + ".glossary.yaml",
+		outputSchema:           "audita-v1",
+		proposalResponses:      proposalResponses,
+		validationResponses:    validationResponses,
+		expectedProposalCalls:  []string{"glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", "grammar:proposal"},
+		reportSchemaName:       reporting.DefaultProcessReportSchemaName,
+		reportSchemaVersion:    reporting.DefaultProcessReportSchemaVersion,
+		expectedOutputSchema:   "audita-v1",
+		expectModuleInstances:  expectations.ExpectedModuleInstance,
+		expectUtilizationPaths: true,
+	})
+	assertAuditaV1OutputShape(t, auditaV1.outputPath)
+
+	// Idempotence-oriented second pass:
+	// run again on first output with deterministic no-op responses.
+	noOpProposals := make([]proposal_generation.StructuredCorrectionSet, 5)
+	for i := range noOpProposals {
+		noOpProposals[i] = proposal_generation.StructuredCorrectionSet{Corrections: nil}
+	}
+	second := runReleaseFixturePass(t, releaseRunConfig{
+		transcriptPath:         first.outputPath,
+		glossaryPath:           base + ".glossary.yaml",
+		outputSchema:           "bare-segments",
+		proposalResponses:      noOpProposals,
+		validationResponses:    nil,
+		expectedProposalCalls:  []string{"glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", "grammar:proposal"},
+		reportSchemaName:       reporting.DefaultProcessReportSchemaName,
+		reportSchemaVersion:    reporting.DefaultProcessReportSchemaVersion,
+		expectedOutputSchema:   "bare-segments",
+		expectModuleInstances:  expectations.ExpectedModuleInstance,
+		expectUtilizationPaths: true,
+	})
+	firstSegments := mustReadTranscript(t, first.outputPath)
+	secondSegments := mustReadTranscript(t, second.outputPath)
+	if !reflect.DeepEqual(firstSegments, secondSegments) {
+		t.Fatalf("expected idempotent second pass transcript; first=%+v second=%+v", firstSegments, secondSegments)
+	}
+	if second.report.ModulesSummary == nil {
+		t.Fatalf("expected modules summary on second pass")
+	}
+	if second.report.ModulesSummary.TotalAppliedChanges != 0 {
+		t.Fatalf("expected no-op second pass (0 applied), got %+v", second.report.ModulesSummary)
+	}
+}
+
+type releaseRunConfig struct {
+	transcriptPath         string
+	glossaryPath           string
+	outputSchema           string
+	proposalResponses      []proposal_generation.StructuredCorrectionSet
+	validationResponses    []validators.LLMValidationResponse
+	expectedProposalCalls  []string
+	reportSchemaName       string
+	reportSchemaVersion    string
+	expectedOutputSchema   string
+	expectModuleInstances  []string
+	expectUtilizationPaths bool
+}
+
+type releaseRunResult struct {
+	stdout     string
+	stderr     string
+	outputPath string
+	reportPath string
+	report     reporting.ProcessReport
+	runDir     string
+}
+
+func runReleaseFixturePass(t *testing.T, cfg releaseRunConfig) releaseRunResult {
+	t.Helper()
+
+	processProposalLLMClient = &fakeStructuredLLMClient{proposalResponses: append([]proposal_generation.StructuredCorrectionSet(nil), cfg.proposalResponses...)}
+	processValidationLLMClient = &fakeStructuredLLMClient{validationResponses: append([]validators.LLMValidationResponse(nil), cfg.validationResponses...)}
+	t.Cleanup(func() {
+		processProposalLLMClient = nil
+		processValidationLLMClient = nil
+	})
+
+	workDir := t.TempDir()
+	reportPath := filepath.Join(t.TempDir(), "report.json")
+	outputPath := filepath.Join(t.TempDir(), "out.json")
+	configPath := writeFile(t, "release-config.yml", "version: 1\n")
+
+	args := []string{
+		"process",
+		cfg.transcriptPath,
+		"--glossary",
+		cfg.glossaryPath,
+		"--config",
+		configPath,
+		"--output",
+		outputPath,
+		"--output-schema",
+		cfg.outputSchema,
+		"--report-json",
+		reportPath,
+		"--work-dir",
+		workDir,
+		"--work-dir-retention",
+		"always",
+	}
+
+	var stdout, stderr bytes.Buffer
+	exitCode := Run(args, &stdout, &stderr)
+	if exitCode != 0 {
+		t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
+	}
+
+	report := readProcessReport(t, reportPath)
+	if report.ReportMetadata.ReportSchemaName != cfg.reportSchemaName || report.ReportMetadata.ReportSchemaVersion != cfg.reportSchemaVersion {
+		t.Fatalf("unexpected report schema metadata: %+v", report.ReportMetadata)
+	}
+	if report.ReportMetadata.OutputSchema != cfg.expectedOutputSchema {
+		t.Fatalf("unexpected output schema metadata: got %q want %q", report.ReportMetadata.OutputSchema, cfg.expectedOutputSchema)
+	}
+	if len(cfg.expectModuleInstances) > 0 {
+		got := make([]string, 0, len(report.ModuleResults))
+		for _, mr := range report.ModuleResults {
+			got = append(got, mr.ModuleInstance)
+		}
+		if !reflect.DeepEqual(got, cfg.expectModuleInstances) {
+			t.Fatalf("unexpected module instances: got %v want %v", got, cfg.expectModuleInstances)
+		}
+	}
+	if report.Diagnostics == nil {
+		t.Fatalf("expected diagnostics metadata")
+	}
+	if cfg.expectUtilizationPaths {
+		if report.Diagnostics.UtilizationSummaryPath == "" || report.Diagnostics.CorrectionLedgerPath == "" {
+			t.Fatalf("expected utilization/ledger artifact paths in report diagnostics: %+v", report.Diagnostics)
+		}
+	}
+
+	runDir := onlyRunDir(t, workDir)
+	if _, err := os.Stat(filepath.Join(runDir, "report.json")); err != nil {
+		t.Fatalf("expected run-dir report: %v", err)
+	}
+
+	if c, ok := processProposalLLMClient.(*fakeStructuredLLMClient); ok {
+		if !reflect.DeepEqual(c.calls, cfg.expectedProposalCalls) {
+			t.Fatalf("unexpected proposal call order: got %v want %v", c.calls, cfg.expectedProposalCalls)
+		}
+	}
+
+	return releaseRunResult{
+		stdout:     stdout.String(),
+		stderr:     stderr.String(),
+		outputPath: outputPath,
+		reportPath: reportPath,
+		report:     report,
+		runDir:     runDir,
+	}
+}
+
+func readProposalResponses(t *testing.T, path string) []proposal_generation.StructuredCorrectionSet {
+	t.Helper()
+	var out []proposal_generation.StructuredCorrectionSet
+	if err := json.Unmarshal(readFile(t, path), &out); err != nil {
+		t.Fatalf("unmarshal proposal responses: %v", err)
+	}
+	return out
+}
+
+func readValidationResponses(t *testing.T, path string) []validators.LLMValidationResponse {
+	t.Helper()
+	var out []validators.LLMValidationResponse
+	if err := json.Unmarshal(readFile(t, path), &out); err != nil {
+		t.Fatalf("unmarshal validation responses: %v", err)
+	}
+	return out
+}
+
+func mustReadTranscript(t *testing.T, path string) []schema.Segment {
+	t.Helper()
+	transcript, err := schema.ParseTranscriptJSON(readFile(t, path))
+	if err != nil {
+		t.Fatalf("parse transcript output: %v", err)
+	}
+	return transcript.Segments
+}
+
+func expectFinalTranscriptContains(t *testing.T, segments []schema.Segment, needles []string) {
+	t.Helper()
+	joined := flattenTranscriptText(segments)
+	for _, needle := range needles {
+		if !strings.Contains(joined, needle) {
+			t.Fatalf("expected transcript to contain %q, got %q", needle, joined)
+		}
+	}
+}
+
+func expectFinalTranscriptDoesNotContain(t *testing.T, segments []schema.Segment, needles []string) {
+	t.Helper()
+	joined := flattenTranscriptText(segments)
+	for _, needle := range needles {
+		if strings.Contains(joined, needle) {
+			t.Fatalf("expected transcript to not contain %q, got %q", needle, joined)
+		}
+	}
+}
+
+func flattenTranscriptText(segments []schema.Segment) string {
+	parts := make([]string, 0, len(segments))
+	for _, s := range segments {
+		parts = append(parts, s.Text)
+	}
+	return strings.Join(parts, "\n")
+}
+
+func assertReleaseCounts(t *testing.T, report reporting.ProcessReport, exp releaseFixtureExpectations) {
+	t.Helper()
+	if report.ModulesSummary == nil {
+		t.Fatalf("expected modules_summary")
+	}
+	if report.ModulesSummary.TotalAppliedChanges < exp.MinimumCounts.Applied {
+		t.Fatalf("expected at least %d applied changes, got %+v", exp.MinimumCounts.Applied, report.ModulesSummary)
+	}
+	validatorRejected := 0
+	skipped := 0
+	for _, mr := range report.ModuleResults {
+		validatorRejected += len(mr.ValidatorRejected)
+		skipped += len(mr.SkippedChanges)
+	}
+	if validatorRejected < exp.MinimumCounts.Rejected {
+		t.Fatalf("expected at least %d validator rejections, got %d", exp.MinimumCounts.Rejected, validatorRejected)
+	}
+	if skipped < exp.MinimumCounts.Skipped {
+		t.Fatalf("expected at least %d application skips, got %d", exp.MinimumCounts.Skipped, skipped)
+	}
+}
+
+func assertReleaseUtilizationShape(t *testing.T, report reporting.ProcessReport) {
+	t.Helper()
+	var payload struct {
+		EffectiveConcurrency struct {
+			TotalLLM int `json:"total_llm"`
+		} `json:"effective_concurrency"`
+		RunTiming struct {
+			SchedulerQueueWaitMS      int64 `json:"scheduler_queue_wait_ms"`
+			LLMExecutionTimeMS        int64 `json:"llm_execution_time_ms"`
+			DeterministicValidationMS int64 `json:"deterministic_validation_time_ms"`
+		} `json:"run_timing"`
+		Modules    []map[string]any `json:"modules"`
+		Validators []map[string]any `json:"validators"`
+	}
+	if err := json.Unmarshal(readFile(t, report.Diagnostics.UtilizationSummaryPath), &payload); err != nil {
+		t.Fatalf("unmarshal utilization diagnostics: %v", err)
+	}
+	if payload.EffectiveConcurrency.TotalLLM <= 0 {
+		t.Fatalf("expected positive total llm concurrency, got %+v", payload.EffectiveConcurrency)
+	}
+	if payload.RunTiming.SchedulerQueueWaitMS < 0 || payload.RunTiming.LLMExecutionTimeMS < 0 || payload.RunTiming.DeterministicValidationMS < 0 {
+		t.Fatalf("expected non-negative run timing values, got %+v", payload.RunTiming)
+	}
+	if len(payload.Modules) == 0 {
+		t.Fatalf("expected module timing summaries")
+	}
+	if len(payload.Validators) == 0 {
+		t.Fatalf("expected validator timing summaries")
+	}
+}
+
+func assertReleaseLedgerShape(t *testing.T, report reporting.ProcessReport) {
+	t.Helper()
+	var entries []struct {
+		ModuleKey            string `json:"module_key"`
+		ModuleInstance       string `json:"module_instance"`
+		ProposalIndex        int    `json:"proposal_index"`
+		Disposition          string `json:"disposition"`
+		DispositionReason    string `json:"disposition_reason_code"`
+		OriginalText         string `json:"original_text"`
+		ProposedCorrected    string `json:"proposed_corrected_text"`
+		ReplacementPolicy    string `json:"replacement_policy"`
+		DeterministicResults []struct {
+			ValidatorKey string `json:"validator_key"`
+		} `json:"deterministic_validator_decisions"`
+		LLMResults []struct {
+			ValidatorKey string `json:"validator_key"`
+		} `json:"llm_validator_decisions"`
+	}
+	if err := json.Unmarshal(readFile(t, report.Diagnostics.CorrectionLedgerPath), &entries); err != nil {
+		t.Fatalf("unmarshal correction ledger: %v", err)
+	}
+	if len(entries) == 0 {
+		t.Fatalf("expected correction ledger entries")
+	}
+	hasApplied := false
+	hasRejected := false
+	hasSkipped := false
+	for _, entry := range entries {
+		if entry.ModuleInstance == "" || entry.ModuleKey == "" {
+			t.Fatalf("expected module identity in ledger entry: %+v", entry)
+		}
+		switch entry.Disposition {
+		case "applied":
+			hasApplied = true
+		case "rejected":
+			hasRejected = true
+		case "skipped":
+			hasSkipped = true
+		}
+	}
+	if !hasApplied || !hasRejected {
+		t.Fatalf("expected applied and rejected entries in correction ledger, got %+v", entries)
+	}
+	if !hasSkipped {
+		// Some deterministic fixture paths do not trigger apply-time skips;
+		// rejections are still captured separately from application skips.
+	}
+}
+
+func assertPromptAndSchemaMetadataPresent(t *testing.T, runDir string) {
+	t.Helper()
+	metadataPaths, err := filepath.Glob(filepath.Join(runDir, "*", "*request-metadata.json"))
+	if err != nil {
+		t.Fatalf("glob request metadata artifacts: %v", err)
+	}
+	if len(metadataPaths) == 0 {
+		t.Fatalf("expected request metadata artifacts with prompt metadata")
+	}
+
+	foundPromptMetadata := false
+	foundSchemaMetadata := false
+	for _, path := range metadataPaths {
+		var payload map[string]any
+		if err := json.Unmarshal(readFile(t, path), &payload); err != nil {
+			t.Fatalf("unmarshal request metadata artifact %q: %v", path, err)
+		}
+		if pm, ok := payload["prompt_metadata"].(map[string]any); ok {
+			if pm["prompt_id"] != nil && pm["prompt_version"] != nil && pm["sha256"] != nil {
+				foundPromptMetadata = true
+			}
+		}
+		if sm, ok := payload["response_schema"].(map[string]any); ok {
+			if sm["id"] != nil && sm["version"] != nil && sm["name"] != nil && sm["sha256"] != nil {
+				foundSchemaMetadata = true
+			}
+		}
+	}
+	if !foundPromptMetadata {
+		t.Fatalf("expected prompt metadata in request metadata artifacts")
+	}
+	if !foundSchemaMetadata {
+		t.Fatalf("expected structured response schema metadata in request metadata artifacts")
+	}
+}
+
+func assertStableValidatorKeysPresent(t *testing.T, report reporting.ProcessReport) {
+	t.Helper()
+	seen := map[string]bool{}
+	for _, module := range report.ModuleResults {
+		for _, decision := range module.ValidatorDecisions {
+			seen[decision.ValidatorName] = true
+		}
+		for _, rejected := range module.ValidatorRejected {
+			seen[rejected.ValidatorName] = true
+		}
+	}
+	expectedAny := []string{
+		"confidence_threshold",
+		"original_text_presence",
+		"no_effect",
+	}
+	for _, key := range expectedAny {
+		if !seen[key] {
+			t.Fatalf("expected stable validator key %q in report decisions/rejections; seen=%v", key, seen)
+		}
+	}
+}
+
+func assertStdoutStderrContract(t *testing.T, stdout, stderr string) {
+	t.Helper()
+	if stdout != "" {
+		t.Fatalf("expected empty stdout with --output, got %q", stdout)
+	}
+	if strings.Contains(stderr, `"module_results"`) || strings.Contains(stderr, `"report_metadata"`) {
+		t.Fatalf("stderr should remain human-readable, not report JSON: %q", stderr)
+	}
+}
+
+func assertAuditaV1OutputShape(t *testing.T, outputPath string) {
+	t.Helper()
+	var payload struct {
+		Schema   string           `json:"schema"`
+		Version  string           `json:"version"`
+		Segments []schema.Segment `json:"segments"`
+	}
+	if err := json.Unmarshal(readFile(t, outputPath), &payload); err != nil {
+		t.Fatalf("unmarshal audita-v1 output: %v", err)
+	}
+	if payload.Schema != "audita-v1" || payload.Version != "v1" {
+		t.Fatalf("unexpected audita-v1 metadata: %+v", payload)
+	}
+	if len(payload.Segments) == 0 {
+		t.Fatalf("expected non-empty audita-v1 segments")
+	}
+}

internal/cli/testdata/release/default-release.expectations.json

@@ -0,0 +1,24 @@
+{
+  "must_apply_texts": [
+    "Hello, there were Jesters hmm"
+  ],
+  "must_not_apply_texts": [
+    "JESTERX",
+    "there were gestures"
+  ],
+  "protected_terms": [
+    "Jesters"
+  ],
+  "expected_module_instances": [
+    "glossary_1",
+    "homophones",
+    "glossary_2",
+    "spoken_word",
+    "grammar"
+  ],
+  "minimum_counts": {
+    "applied": 1,
+    "rejected": 1,
+    "skipped": 0
+  }
+}

internal/cli/testdata/release/default-release.expected-transcript.json

@@ -0,0 +1,9 @@
+[
+  {
+    "id": 1,
+    "speaker": "Alice",
+    "start": 0,
+    "end": 1,
+    "text": "Hello, there were Jesters hmm"
+  }
+]

internal/cli/testdata/release/default-release.glossary.yaml

@@ -0,0 +1,7 @@
+glossary:
+  - name: Jesters
+    aliases:
+      - jester
+    plural: jesters
+    category: faction
+    summary: A protected in-world faction term.

internal/cli/testdata/release/default-release.proposals.json

@@ -0,0 +1,28 @@
+[
+  {
+    "corrections": [
+      {"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99}
+    ]
+  },
+  {
+    "corrections": [
+      {"id": 1, "original_text": "Jesters", "corrected_text": "jesters", "confidence": 0.99},
+      {"id": 1, "original_text": "Jesters", "corrected_text": "JESTERX", "confidence": 0.99}
+    ]
+  },
+  {
+    "corrections": [
+      {"id": 1, "original_text": "jesters", "corrected_text": "JESTERS", "confidence": 0.99}
+    ]
+  },
+  {
+    "corrections": [
+      {"id": 1, "original_text": "uh", "corrected_text": "hmm", "confidence": 0.99}
+    ]
+  },
+  {
+    "corrections": [
+      {"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99}
+    ]
+  }
+]

internal/cli/testdata/release/default-release.transcript.json

@@ -0,0 +1,3 @@
+[
+  {"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , there were gestures uh"}
+]

internal/cli/testdata/release/default-release.validations.json

@@ -0,0 +1,12 @@
+[
+  {"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
+  {"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
+  {"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
+  {"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
+  {"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
+  {"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
+  {"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
+  {"validations": [{"correction_index": 0, "approved": false, "confidence": 0.99, "reason": "reject cleanup"}]},
+  {"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "release-secret"}]},
+  {"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
+]