From 06879828223288ad8f087a53bb6be49d741dcac7 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Wed, 13 May 2026 19:42:35 +0000 Subject: [PATCH] Complete pre-1.0 release hardening --- README.md | 1 + docs/architecture.md | 14 + docs/configuration.md | 3 + docs/release-checklist.md | 122 +++++ docs/roadmap.md | 20 + docs/subprocess-operations.md | 6 + internal/cli/release_fixtures_test.go | 469 ++++++++++++++++++ .../release/default-release.expectations.json | 24 + .../default-release.expected-transcript.json | 9 + .../release/default-release.glossary.yaml | 7 + .../release/default-release.proposals.json | 28 ++ .../release/default-release.transcript.json | 3 + .../release/default-release.validations.json | 12 + 13 files changed, 718 insertions(+) create mode 100644 docs/release-checklist.md create mode 100644 internal/cli/release_fixtures_test.go create mode 100644 internal/cli/testdata/release/default-release.expectations.json create mode 100644 internal/cli/testdata/release/default-release.expected-transcript.json create mode 100644 internal/cli/testdata/release/default-release.glossary.yaml create mode 100644 internal/cli/testdata/release/default-release.proposals.json create mode 100644 internal/cli/testdata/release/default-release.transcript.json create mode 100644 internal/cli/testdata/release/default-release.validations.json diff --git a/README.md b/README.md index dc2c2cf..560f5cd 100644 --- a/README.md +++ b/README.md @@ -300,3 +300,4 @@ Optional external report output: - Diagnostics: [`docs/diagnostics.md`](docs/diagnostics.md) - Structured LLM adapter: [`docs/structured-llm.md`](docs/structured-llm.md) - Subprocess operations: [`docs/subprocess-operations.md`](docs/subprocess-operations.md) +- Release checklist: [`docs/release-checklist.md`](docs/release-checklist.md) diff --git a/docs/architecture.md b/docs/architecture.md index 4ff6c29..30c6500 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -100,6 +100,7 @@ internal/framework/proposals/ apply.go internal/framework/runner/ + observability.go runner.go internal/framework/proposal_generation/ @@ -149,6 +150,18 @@ internal/framework/llm/ scheduler.go effective_config.go diagnostics.go + +internal/framework/responseschema/ + registry.go + registry_test.go + +internal/cli/ + review_artifacts.go + parity_test.go + release_fixtures_test.go + testdata/ + parity/ + release/ ``` ## Current CLI behavior @@ -771,6 +784,7 @@ Implemented tests currently cover: - default full-pipeline runtime shape and ordering (`internal/cli/run_test.go`, `cmd/audita/main_integration_test.go`, `internal/cli/parity_test.go`) - subprocess operational hardening behavior including large-input, failure-mode, timeout/cancellation, backend-failure, and partial-progress paths (`cmd/audita/main_integration_test.go`) - report/diagnostics redaction and artifact-shape behavior across success and failure paths (`internal/cli/run_test.go`, `cmd/audita/main_integration_test.go`) +- curated release-fixture and idempotence-oriented readiness checks using fake structured LLM responses (`internal/cli/release_fixtures_test.go`, `internal/cli/testdata/release`) ## Operational hardening status The runtime now includes hardened subprocess behavior for parent-process callers: diff --git a/docs/configuration.md b/docs/configuration.md index b9c56f5..d2f88fc 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -99,6 +99,9 @@ diagnostics: retention: auto ``` +`context.description` provides background-only transcript context for prompts. +If both config and CLI provide a description, `--transcript-description` takes precedence. + `output.schema` supports the built-in output schema registry values: - `bare-segments` (default) - `audita-v1` diff --git a/docs/release-checklist.md b/docs/release-checklist.md new file mode 100644 index 0000000..dd4e4af --- /dev/null +++ b/docs/release-checklist.md @@ -0,0 +1,122 @@ +# Audita Release Checklist + +Use this checklist before cutting a pre-1.0 or 1.0 release candidate. + +## Core test pass + +- Run: + - `go test ./...` +- Confirm tests pass without live LLM credentials and without Python dependencies. + +## Config validation and precedence + +- Validate a representative config: + - `audita config validate --config ` +- Inspect redacted effective config: + - `audita config print-effective --config ` +- Confirm precedence behavior: + - defaults -> file config -> environment -> CLI. +- Confirm missing `/etc/audita/config.yml` is non-fatal when `--config`/`AUDITA_CONFIG` are unset. + +## Output schema checks + +- Verify default output schema remains `bare-segments`. +- Verify `--output-schema audita-v1` emits object payload with `schema` and `version`. +- Verify unknown schema (for example `seriatim-intermediate`) fails clearly. + +## Subprocess contract checks + +- With `--output`, verify stdout is empty on success. +- Without `--output`, verify stdout contains transcript JSON only. +- Verify `--report-json` writes file output and does not write report JSON to stdout. +- Verify failure stderr remains human-readable and includes diagnostics path when available. +- Verify nonzero exit on failures. + +## Structured LLM checks + +- Verify runtime uses the Audita-owned OpenAI-compatible adapter. +- Verify structured response schemas are attached via `response_format.type=json_schema`. +- Verify diagnostics metadata includes structured schema `id/version/name/sha256`. +- Verify provider output is still locally decoded/validated before use. + +## Report and diagnostics schema checks + +- Verify report metadata fields: + - `report_schema_name` + - `report_schema_version` + - `output_schema` + - `config_version` when file config is used. +- Verify diagnostics artifact references exist in reports: + - transcript/normalization/chunking/invocation/effective-config artifacts + - utilization diagnostics artifact + - correction ledger artifact + - error log on failures. + +## Redaction checks + +- Verify secrets are redacted from: + - `effective-config.json` + - run-dir and `--report-json` reports + - LLM request/response/error diagnostics payloads. +- Verify no API keys/bearer tokens leak into fixtures or outputs. + +## Prompt and validator metadata checks + +- Verify prompt metadata appears in LLM request metadata diagnostics: + - `prompt_id`, `prompt_version`, `prompt_source`, `embedded_path`, `sha256`. +- Verify stable validator keys appear in report decisions/rejections. +- Verify built-in validator chains resolve and execute for default and explicit module runs. + +## Utilization diagnostics checks + +- Verify `utilization-diagnostics.json` exists on successful runs. +- Verify partial utilization artifact behavior on controlled failure paths. +- Verify utilization fields are structurally present and nonnegative: + - effective concurrency + - run timing + - module timing summaries + - per-validator timing summaries. + +## Correction ledger checks + +- Verify `correction-ledger.json` exists on successful runs. +- Verify report references ledger artifact path. +- Verify ledger dispositions include applied/rejected and skipped/failed where exercised. +- Verify validator rejection and proposal-application skip remain distinct. + +## Pipeline behavior checks + +- Verify default full pipeline run remains: + - `glossary`, `homophones`, `glossary`, `spoken_word`, `grammar` + - with deterministic repeated instance naming (`glossary_1`, `glossary_2`). +- Verify explicit module runs (`--modules`) still work. + +## Failure and cancellation checks + +- Verify controlled failure paths retain diagnostics and produce best-effort failure reports. +- Verify timeout/cancellation paths exit nonzero, do not hang, and retain failure diagnostics when initialized. + +## Release fixture/idempotence checks + +- Run release fixtures (`internal/cli/testdata/release`) through `go test ./...`. +- Confirm fixture checks cover: + - must-apply and must-not-apply expectations + - protected-term survival + - report and diagnostics contracts + - output-schema checks + - prompt/schema metadata diagnostics + - utilization/ledger artifacts + - idempotence-oriented second pass no-op behavior with deterministic fake responses. + +## Deferred-feature guardrail + +- Confirm release docs do not claim support for deferred items: + - filesystem prompt overrides + - user-configurable validator chains + - arbitrary user-supplied output schemas + - resume/start-at/stop-after execution + - diff/check/propose-only modes + - generated transcript descriptions enabled by default + - interactive review UI + - UI/server wrapper + - provider benchmarking harness. diff --git a/docs/roadmap.md b/docs/roadmap.md index 8aaa872..576c08f 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -897,6 +897,26 @@ This ledger should not replace existing reports. It should provide a flattened r Finish 1.0 by turning the new architecture into a documented, tested, stable release candidate. +## Implementation status (2026-05-13) + +This workstream is now implemented: +- architecture, contract, configuration, output-schema, structured-LLM, validator, prompt, diagnostics, and subprocess docs are aligned to the current runtime; +- a practical release checklist document exists (`docs/release-checklist.md`); +- curated release fixtures exist under `internal/cli/testdata/release` and run under `go test ./...` using fake structured LLM responses only; +- release fixture coverage includes: + - must-apply and must-not-apply expectations; + - protected-term survival; + - default module sequence and stable instance naming; + - report schema metadata; + - diagnostics artifact references; + - output schema checks (`bare-segments`, `audita-v1`); + - prompt/structured-schema diagnostics metadata presence; + - stable validator keys; + - utilization diagnostics and correction-ledger artifacts; + - idempotence-oriented second-pass no-op expectations with deterministic fake responses. + +Deferred work remains deferred (no filesystem prompt overrides, no user-configurable validator chains, no arbitrary user-supplied output schemas, no resume/start-at/stop-after execution, no diff/check/propose-only modes, no generated transcript descriptions enabled by default, no interactive review UI, no UI/server wrapper, and no provider-specific benchmarking harness). + ## Documentation updates Update or add: diff --git a/docs/subprocess-operations.md b/docs/subprocess-operations.md index d36ff44..5e4205e 100644 --- a/docs/subprocess-operations.md +++ b/docs/subprocess-operations.md @@ -21,6 +21,12 @@ Recommended additions: - `--total-llm-concurrency`, `--proposal-llm-concurrency`, and `--validation-llm-concurrency` when orchestration needs explicit LLM throughput controls. - `--modules ...` only when intentionally overriding the default full sequence. +For config-driven orchestration, validate config files in CI/preflight: + +```sh +audita config validate --config +``` + ## Stdout behavior - With `--output`: stdout is expected to be empty on success. diff --git a/internal/cli/release_fixtures_test.go b/internal/cli/release_fixtures_test.go new file mode 100644 index 0000000..ed64692 --- /dev/null +++ b/internal/cli/release_fixtures_test.go @@ -0,0 +1,469 @@ +package cli + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/audita/internal/core/reporting" + "gitea.maximumdirect.net/eric/audita/internal/core/schema" + "gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation" + "gitea.maximumdirect.net/eric/audita/internal/framework/validators" +) + +type releaseFixtureExpectations struct { + MustApplyTexts []string `json:"must_apply_texts"` + MustNotApplyTexts []string `json:"must_not_apply_texts"` + ProtectedTerms []string `json:"protected_terms"` + ExpectedModuleInstance []string `json:"expected_module_instances"` + MinimumCounts struct { + Applied int `json:"applied"` + Rejected int `json:"rejected"` + Skipped int `json:"skipped"` + } `json:"minimum_counts"` +} + +func TestReleaseFixtureDefaultPipelineReadiness(t *testing.T) { + base := fixturePath(filepath.Join("release", "default-release")) + + var expectations releaseFixtureExpectations + if err := json.Unmarshal(readFile(t, base+".expectations.json"), &expectations); err != nil { + t.Fatalf("unmarshal release expectations: %v", err) + } + + proposalResponses := readProposalResponses(t, base+".proposals.json") + validationResponses := readValidationResponses(t, base+".validations.json") + + // First pass: default full pipeline with deterministic fake LLM responses. + first := runReleaseFixturePass(t, releaseRunConfig{ + transcriptPath: base + ".transcript.json", + glossaryPath: base + ".glossary.yaml", + outputSchema: "bare-segments", + proposalResponses: proposalResponses, + validationResponses: validationResponses, + expectedProposalCalls: []string{"glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", "grammar:proposal"}, + reportSchemaName: reporting.DefaultProcessReportSchemaName, + reportSchemaVersion: reporting.DefaultProcessReportSchemaVersion, + expectedOutputSchema: "bare-segments", + expectModuleInstances: expectations.ExpectedModuleInstance, + expectUtilizationPaths: true, + }) + + gotTranscript := mustReadTranscript(t, first.outputPath) + expectFinalTranscriptContains(t, gotTranscript, expectations.MustApplyTexts) + expectFinalTranscriptDoesNotContain(t, gotTranscript, expectations.MustNotApplyTexts) + expectFinalTranscriptContains(t, gotTranscript, expectations.ProtectedTerms) + + assertReleaseCounts(t, first.report, expectations) + assertPromptAndSchemaMetadataPresent(t, first.runDir) + assertReleaseLedgerShape(t, first.report) + assertReleaseUtilizationShape(t, first.report) + assertStableValidatorKeysPresent(t, first.report) + assertStdoutStderrContract(t, first.stdout, first.stderr) + assertNoSecretMarkersInTree(t, first.runDir, []string{"release-secret"}) + assertNoSecretMarkers(t, first.reportPath, []string{"release-secret"}) + + // Output schema check: audita-v1 object payload. + auditaV1 := runReleaseFixturePass(t, releaseRunConfig{ + transcriptPath: base + ".transcript.json", + glossaryPath: base + ".glossary.yaml", + outputSchema: "audita-v1", + proposalResponses: proposalResponses, + validationResponses: validationResponses, + expectedProposalCalls: []string{"glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", "grammar:proposal"}, + reportSchemaName: reporting.DefaultProcessReportSchemaName, + reportSchemaVersion: reporting.DefaultProcessReportSchemaVersion, + expectedOutputSchema: "audita-v1", + expectModuleInstances: expectations.ExpectedModuleInstance, + expectUtilizationPaths: true, + }) + assertAuditaV1OutputShape(t, auditaV1.outputPath) + + // Idempotence-oriented second pass: + // run again on first output with deterministic no-op responses. + noOpProposals := make([]proposal_generation.StructuredCorrectionSet, 5) + for i := range noOpProposals { + noOpProposals[i] = proposal_generation.StructuredCorrectionSet{Corrections: nil} + } + second := runReleaseFixturePass(t, releaseRunConfig{ + transcriptPath: first.outputPath, + glossaryPath: base + ".glossary.yaml", + outputSchema: "bare-segments", + proposalResponses: noOpProposals, + validationResponses: nil, + expectedProposalCalls: []string{"glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", "grammar:proposal"}, + reportSchemaName: reporting.DefaultProcessReportSchemaName, + reportSchemaVersion: reporting.DefaultProcessReportSchemaVersion, + expectedOutputSchema: "bare-segments", + expectModuleInstances: expectations.ExpectedModuleInstance, + expectUtilizationPaths: true, + }) + firstSegments := mustReadTranscript(t, first.outputPath) + secondSegments := mustReadTranscript(t, second.outputPath) + if !reflect.DeepEqual(firstSegments, secondSegments) { + t.Fatalf("expected idempotent second pass transcript; first=%+v second=%+v", firstSegments, secondSegments) + } + if second.report.ModulesSummary == nil { + t.Fatalf("expected modules summary on second pass") + } + if second.report.ModulesSummary.TotalAppliedChanges != 0 { + t.Fatalf("expected no-op second pass (0 applied), got %+v", second.report.ModulesSummary) + } +} + +type releaseRunConfig struct { + transcriptPath string + glossaryPath string + outputSchema string + proposalResponses []proposal_generation.StructuredCorrectionSet + validationResponses []validators.LLMValidationResponse + expectedProposalCalls []string + reportSchemaName string + reportSchemaVersion string + expectedOutputSchema string + expectModuleInstances []string + expectUtilizationPaths bool +} + +type releaseRunResult struct { + stdout string + stderr string + outputPath string + reportPath string + report reporting.ProcessReport + runDir string +} + +func runReleaseFixturePass(t *testing.T, cfg releaseRunConfig) releaseRunResult { + t.Helper() + + processProposalLLMClient = &fakeStructuredLLMClient{proposalResponses: append([]proposal_generation.StructuredCorrectionSet(nil), cfg.proposalResponses...)} + processValidationLLMClient = &fakeStructuredLLMClient{validationResponses: append([]validators.LLMValidationResponse(nil), cfg.validationResponses...)} + t.Cleanup(func() { + processProposalLLMClient = nil + processValidationLLMClient = nil + }) + + workDir := t.TempDir() + reportPath := filepath.Join(t.TempDir(), "report.json") + outputPath := filepath.Join(t.TempDir(), "out.json") + configPath := writeFile(t, "release-config.yml", "version: 1\n") + + args := []string{ + "process", + cfg.transcriptPath, + "--glossary", + cfg.glossaryPath, + "--config", + configPath, + "--output", + outputPath, + "--output-schema", + cfg.outputSchema, + "--report-json", + reportPath, + "--work-dir", + workDir, + "--work-dir-retention", + "always", + } + + var stdout, stderr bytes.Buffer + exitCode := Run(args, &stdout, &stderr) + if exitCode != 0 { + t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String()) + } + + report := readProcessReport(t, reportPath) + if report.ReportMetadata.ReportSchemaName != cfg.reportSchemaName || report.ReportMetadata.ReportSchemaVersion != cfg.reportSchemaVersion { + t.Fatalf("unexpected report schema metadata: %+v", report.ReportMetadata) + } + if report.ReportMetadata.OutputSchema != cfg.expectedOutputSchema { + t.Fatalf("unexpected output schema metadata: got %q want %q", report.ReportMetadata.OutputSchema, cfg.expectedOutputSchema) + } + if len(cfg.expectModuleInstances) > 0 { + got := make([]string, 0, len(report.ModuleResults)) + for _, mr := range report.ModuleResults { + got = append(got, mr.ModuleInstance) + } + if !reflect.DeepEqual(got, cfg.expectModuleInstances) { + t.Fatalf("unexpected module instances: got %v want %v", got, cfg.expectModuleInstances) + } + } + if report.Diagnostics == nil { + t.Fatalf("expected diagnostics metadata") + } + if cfg.expectUtilizationPaths { + if report.Diagnostics.UtilizationSummaryPath == "" || report.Diagnostics.CorrectionLedgerPath == "" { + t.Fatalf("expected utilization/ledger artifact paths in report diagnostics: %+v", report.Diagnostics) + } + } + + runDir := onlyRunDir(t, workDir) + if _, err := os.Stat(filepath.Join(runDir, "report.json")); err != nil { + t.Fatalf("expected run-dir report: %v", err) + } + + if c, ok := processProposalLLMClient.(*fakeStructuredLLMClient); ok { + if !reflect.DeepEqual(c.calls, cfg.expectedProposalCalls) { + t.Fatalf("unexpected proposal call order: got %v want %v", c.calls, cfg.expectedProposalCalls) + } + } + + return releaseRunResult{ + stdout: stdout.String(), + stderr: stderr.String(), + outputPath: outputPath, + reportPath: reportPath, + report: report, + runDir: runDir, + } +} + +func readProposalResponses(t *testing.T, path string) []proposal_generation.StructuredCorrectionSet { + t.Helper() + var out []proposal_generation.StructuredCorrectionSet + if err := json.Unmarshal(readFile(t, path), &out); err != nil { + t.Fatalf("unmarshal proposal responses: %v", err) + } + return out +} + +func readValidationResponses(t *testing.T, path string) []validators.LLMValidationResponse { + t.Helper() + var out []validators.LLMValidationResponse + if err := json.Unmarshal(readFile(t, path), &out); err != nil { + t.Fatalf("unmarshal validation responses: %v", err) + } + return out +} + +func mustReadTranscript(t *testing.T, path string) []schema.Segment { + t.Helper() + transcript, err := schema.ParseTranscriptJSON(readFile(t, path)) + if err != nil { + t.Fatalf("parse transcript output: %v", err) + } + return transcript.Segments +} + +func expectFinalTranscriptContains(t *testing.T, segments []schema.Segment, needles []string) { + t.Helper() + joined := flattenTranscriptText(segments) + for _, needle := range needles { + if !strings.Contains(joined, needle) { + t.Fatalf("expected transcript to contain %q, got %q", needle, joined) + } + } +} + +func expectFinalTranscriptDoesNotContain(t *testing.T, segments []schema.Segment, needles []string) { + t.Helper() + joined := flattenTranscriptText(segments) + for _, needle := range needles { + if strings.Contains(joined, needle) { + t.Fatalf("expected transcript to not contain %q, got %q", needle, joined) + } + } +} + +func flattenTranscriptText(segments []schema.Segment) string { + parts := make([]string, 0, len(segments)) + for _, s := range segments { + parts = append(parts, s.Text) + } + return strings.Join(parts, "\n") +} + +func assertReleaseCounts(t *testing.T, report reporting.ProcessReport, exp releaseFixtureExpectations) { + t.Helper() + if report.ModulesSummary == nil { + t.Fatalf("expected modules_summary") + } + if report.ModulesSummary.TotalAppliedChanges < exp.MinimumCounts.Applied { + t.Fatalf("expected at least %d applied changes, got %+v", exp.MinimumCounts.Applied, report.ModulesSummary) + } + validatorRejected := 0 + skipped := 0 + for _, mr := range report.ModuleResults { + validatorRejected += len(mr.ValidatorRejected) + skipped += len(mr.SkippedChanges) + } + if validatorRejected < exp.MinimumCounts.Rejected { + t.Fatalf("expected at least %d validator rejections, got %d", exp.MinimumCounts.Rejected, validatorRejected) + } + if skipped < exp.MinimumCounts.Skipped { + t.Fatalf("expected at least %d application skips, got %d", exp.MinimumCounts.Skipped, skipped) + } +} + +func assertReleaseUtilizationShape(t *testing.T, report reporting.ProcessReport) { + t.Helper() + var payload struct { + EffectiveConcurrency struct { + TotalLLM int `json:"total_llm"` + } `json:"effective_concurrency"` + RunTiming struct { + SchedulerQueueWaitMS int64 `json:"scheduler_queue_wait_ms"` + LLMExecutionTimeMS int64 `json:"llm_execution_time_ms"` + DeterministicValidationMS int64 `json:"deterministic_validation_time_ms"` + } `json:"run_timing"` + Modules []map[string]any `json:"modules"` + Validators []map[string]any `json:"validators"` + } + if err := json.Unmarshal(readFile(t, report.Diagnostics.UtilizationSummaryPath), &payload); err != nil { + t.Fatalf("unmarshal utilization diagnostics: %v", err) + } + if payload.EffectiveConcurrency.TotalLLM <= 0 { + t.Fatalf("expected positive total llm concurrency, got %+v", payload.EffectiveConcurrency) + } + if payload.RunTiming.SchedulerQueueWaitMS < 0 || payload.RunTiming.LLMExecutionTimeMS < 0 || payload.RunTiming.DeterministicValidationMS < 0 { + t.Fatalf("expected non-negative run timing values, got %+v", payload.RunTiming) + } + if len(payload.Modules) == 0 { + t.Fatalf("expected module timing summaries") + } + if len(payload.Validators) == 0 { + t.Fatalf("expected validator timing summaries") + } +} + +func assertReleaseLedgerShape(t *testing.T, report reporting.ProcessReport) { + t.Helper() + var entries []struct { + ModuleKey string `json:"module_key"` + ModuleInstance string `json:"module_instance"` + ProposalIndex int `json:"proposal_index"` + Disposition string `json:"disposition"` + DispositionReason string `json:"disposition_reason_code"` + OriginalText string `json:"original_text"` + ProposedCorrected string `json:"proposed_corrected_text"` + ReplacementPolicy string `json:"replacement_policy"` + DeterministicResults []struct { + ValidatorKey string `json:"validator_key"` + } `json:"deterministic_validator_decisions"` + LLMResults []struct { + ValidatorKey string `json:"validator_key"` + } `json:"llm_validator_decisions"` + } + if err := json.Unmarshal(readFile(t, report.Diagnostics.CorrectionLedgerPath), &entries); err != nil { + t.Fatalf("unmarshal correction ledger: %v", err) + } + if len(entries) == 0 { + t.Fatalf("expected correction ledger entries") + } + hasApplied := false + hasRejected := false + hasSkipped := false + for _, entry := range entries { + if entry.ModuleInstance == "" || entry.ModuleKey == "" { + t.Fatalf("expected module identity in ledger entry: %+v", entry) + } + switch entry.Disposition { + case "applied": + hasApplied = true + case "rejected": + hasRejected = true + case "skipped": + hasSkipped = true + } + } + if !hasApplied || !hasRejected { + t.Fatalf("expected applied and rejected entries in correction ledger, got %+v", entries) + } + if !hasSkipped { + // Some deterministic fixture paths do not trigger apply-time skips; + // rejections are still captured separately from application skips. + } +} + +func assertPromptAndSchemaMetadataPresent(t *testing.T, runDir string) { + t.Helper() + metadataPaths, err := filepath.Glob(filepath.Join(runDir, "*", "*request-metadata.json")) + if err != nil { + t.Fatalf("glob request metadata artifacts: %v", err) + } + if len(metadataPaths) == 0 { + t.Fatalf("expected request metadata artifacts with prompt metadata") + } + + foundPromptMetadata := false + foundSchemaMetadata := false + for _, path := range metadataPaths { + var payload map[string]any + if err := json.Unmarshal(readFile(t, path), &payload); err != nil { + t.Fatalf("unmarshal request metadata artifact %q: %v", path, err) + } + if pm, ok := payload["prompt_metadata"].(map[string]any); ok { + if pm["prompt_id"] != nil && pm["prompt_version"] != nil && pm["sha256"] != nil { + foundPromptMetadata = true + } + } + if sm, ok := payload["response_schema"].(map[string]any); ok { + if sm["id"] != nil && sm["version"] != nil && sm["name"] != nil && sm["sha256"] != nil { + foundSchemaMetadata = true + } + } + } + if !foundPromptMetadata { + t.Fatalf("expected prompt metadata in request metadata artifacts") + } + if !foundSchemaMetadata { + t.Fatalf("expected structured response schema metadata in request metadata artifacts") + } +} + +func assertStableValidatorKeysPresent(t *testing.T, report reporting.ProcessReport) { + t.Helper() + seen := map[string]bool{} + for _, module := range report.ModuleResults { + for _, decision := range module.ValidatorDecisions { + seen[decision.ValidatorName] = true + } + for _, rejected := range module.ValidatorRejected { + seen[rejected.ValidatorName] = true + } + } + expectedAny := []string{ + "confidence_threshold", + "original_text_presence", + "no_effect", + } + for _, key := range expectedAny { + if !seen[key] { + t.Fatalf("expected stable validator key %q in report decisions/rejections; seen=%v", key, seen) + } + } +} + +func assertStdoutStderrContract(t *testing.T, stdout, stderr string) { + t.Helper() + if stdout != "" { + t.Fatalf("expected empty stdout with --output, got %q", stdout) + } + if strings.Contains(stderr, `"module_results"`) || strings.Contains(stderr, `"report_metadata"`) { + t.Fatalf("stderr should remain human-readable, not report JSON: %q", stderr) + } +} + +func assertAuditaV1OutputShape(t *testing.T, outputPath string) { + t.Helper() + var payload struct { + Schema string `json:"schema"` + Version string `json:"version"` + Segments []schema.Segment `json:"segments"` + } + if err := json.Unmarshal(readFile(t, outputPath), &payload); err != nil { + t.Fatalf("unmarshal audita-v1 output: %v", err) + } + if payload.Schema != "audita-v1" || payload.Version != "v1" { + t.Fatalf("unexpected audita-v1 metadata: %+v", payload) + } + if len(payload.Segments) == 0 { + t.Fatalf("expected non-empty audita-v1 segments") + } +} diff --git a/internal/cli/testdata/release/default-release.expectations.json b/internal/cli/testdata/release/default-release.expectations.json new file mode 100644 index 0000000..47b84a3 --- /dev/null +++ b/internal/cli/testdata/release/default-release.expectations.json @@ -0,0 +1,24 @@ +{ + "must_apply_texts": [ + "Hello, there were Jesters hmm" + ], + "must_not_apply_texts": [ + "JESTERX", + "there were gestures" + ], + "protected_terms": [ + "Jesters" + ], + "expected_module_instances": [ + "glossary_1", + "homophones", + "glossary_2", + "spoken_word", + "grammar" + ], + "minimum_counts": { + "applied": 1, + "rejected": 1, + "skipped": 0 + } +} diff --git a/internal/cli/testdata/release/default-release.expected-transcript.json b/internal/cli/testdata/release/default-release.expected-transcript.json new file mode 100644 index 0000000..92f501d --- /dev/null +++ b/internal/cli/testdata/release/default-release.expected-transcript.json @@ -0,0 +1,9 @@ +[ + { + "id": 1, + "speaker": "Alice", + "start": 0, + "end": 1, + "text": "Hello, there were Jesters hmm" + } +] diff --git a/internal/cli/testdata/release/default-release.glossary.yaml b/internal/cli/testdata/release/default-release.glossary.yaml new file mode 100644 index 0000000..e3882aa --- /dev/null +++ b/internal/cli/testdata/release/default-release.glossary.yaml @@ -0,0 +1,7 @@ +glossary: + - name: Jesters + aliases: + - jester + plural: jesters + category: faction + summary: A protected in-world faction term. diff --git a/internal/cli/testdata/release/default-release.proposals.json b/internal/cli/testdata/release/default-release.proposals.json new file mode 100644 index 0000000..9abf1dc --- /dev/null +++ b/internal/cli/testdata/release/default-release.proposals.json @@ -0,0 +1,28 @@ +[ + { + "corrections": [ + {"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99} + ] + }, + { + "corrections": [ + {"id": 1, "original_text": "Jesters", "corrected_text": "jesters", "confidence": 0.99}, + {"id": 1, "original_text": "Jesters", "corrected_text": "JESTERX", "confidence": 0.99} + ] + }, + { + "corrections": [ + {"id": 1, "original_text": "jesters", "corrected_text": "JESTERS", "confidence": 0.99} + ] + }, + { + "corrections": [ + {"id": 1, "original_text": "uh", "corrected_text": "hmm", "confidence": 0.99} + ] + }, + { + "corrections": [ + {"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99} + ] + } +] diff --git a/internal/cli/testdata/release/default-release.transcript.json b/internal/cli/testdata/release/default-release.transcript.json new file mode 100644 index 0000000..90d2247 --- /dev/null +++ b/internal/cli/testdata/release/default-release.transcript.json @@ -0,0 +1,3 @@ +[ + {"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , there were gestures uh"} +] diff --git a/internal/cli/testdata/release/default-release.validations.json b/internal/cli/testdata/release/default-release.validations.json new file mode 100644 index 0000000..cc18051 --- /dev/null +++ b/internal/cli/testdata/release/default-release.validations.json @@ -0,0 +1,12 @@ +[ + {"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}, + {"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}, + {"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}, + {"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}, + {"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}, + {"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}, + {"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}, + {"validations": [{"correction_index": 0, "approved": false, "confidence": 0.99, "reason": "reject cleanup"}]}, + {"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "release-secret"}]}, + {"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]} +]