470 lines
17 KiB
Go
470 lines
17 KiB
Go
package cli
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
|
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
|
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
|
)
|
|
|
|
type releaseFixtureExpectations struct {
|
|
MustApplyTexts []string `json:"must_apply_texts"`
|
|
MustNotApplyTexts []string `json:"must_not_apply_texts"`
|
|
ProtectedTerms []string `json:"protected_terms"`
|
|
ExpectedModuleInstance []string `json:"expected_module_instances"`
|
|
MinimumCounts struct {
|
|
Applied int `json:"applied"`
|
|
Rejected int `json:"rejected"`
|
|
Skipped int `json:"skipped"`
|
|
} `json:"minimum_counts"`
|
|
}
|
|
|
|
func TestReleaseFixtureDefaultPipelineReadiness(t *testing.T) {
|
|
base := fixturePath(filepath.Join("release", "default-release"))
|
|
|
|
var expectations releaseFixtureExpectations
|
|
if err := json.Unmarshal(readFile(t, base+".expectations.json"), &expectations); err != nil {
|
|
t.Fatalf("unmarshal release expectations: %v", err)
|
|
}
|
|
|
|
proposalResponses := readProposalResponses(t, base+".proposals.json")
|
|
validationResponses := readValidationResponses(t, base+".validations.json")
|
|
|
|
// First pass: default full pipeline with deterministic fake LLM responses.
|
|
first := runReleaseFixturePass(t, releaseRunConfig{
|
|
transcriptPath: base + ".transcript.json",
|
|
glossaryPath: base + ".glossary.yaml",
|
|
outputSchema: "bare-segments",
|
|
proposalResponses: proposalResponses,
|
|
validationResponses: validationResponses,
|
|
expectedProposalCalls: []string{"glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", "grammar:proposal"},
|
|
reportSchemaName: reporting.DefaultProcessReportSchemaName,
|
|
reportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
|
|
expectedOutputSchema: "bare-segments",
|
|
expectModuleInstances: expectations.ExpectedModuleInstance,
|
|
expectUtilizationPaths: true,
|
|
})
|
|
|
|
gotTranscript := mustReadTranscript(t, first.outputPath)
|
|
expectFinalTranscriptContains(t, gotTranscript, expectations.MustApplyTexts)
|
|
expectFinalTranscriptDoesNotContain(t, gotTranscript, expectations.MustNotApplyTexts)
|
|
expectFinalTranscriptContains(t, gotTranscript, expectations.ProtectedTerms)
|
|
|
|
assertReleaseCounts(t, first.report, expectations)
|
|
assertPromptAndSchemaMetadataPresent(t, first.runDir)
|
|
assertReleaseLedgerShape(t, first.report)
|
|
assertReleaseUtilizationShape(t, first.report)
|
|
assertStableValidatorKeysPresent(t, first.report)
|
|
assertStdoutStderrContract(t, first.stdout, first.stderr)
|
|
assertNoSecretMarkersInTree(t, first.runDir, []string{"release-secret"})
|
|
assertNoSecretMarkers(t, first.reportPath, []string{"release-secret"})
|
|
|
|
// Output schema check: audita-v1 object payload.
|
|
auditaV1 := runReleaseFixturePass(t, releaseRunConfig{
|
|
transcriptPath: base + ".transcript.json",
|
|
glossaryPath: base + ".glossary.yaml",
|
|
outputSchema: "audita-v1",
|
|
proposalResponses: proposalResponses,
|
|
validationResponses: validationResponses,
|
|
expectedProposalCalls: []string{"glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", "grammar:proposal"},
|
|
reportSchemaName: reporting.DefaultProcessReportSchemaName,
|
|
reportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
|
|
expectedOutputSchema: "audita-v1",
|
|
expectModuleInstances: expectations.ExpectedModuleInstance,
|
|
expectUtilizationPaths: true,
|
|
})
|
|
assertAuditaV1OutputShape(t, auditaV1.outputPath)
|
|
|
|
// Idempotence-oriented second pass:
|
|
// run again on first output with deterministic no-op responses.
|
|
noOpProposals := make([]proposal_generation.StructuredCorrectionSet, 5)
|
|
for i := range noOpProposals {
|
|
noOpProposals[i] = proposal_generation.StructuredCorrectionSet{Corrections: nil}
|
|
}
|
|
second := runReleaseFixturePass(t, releaseRunConfig{
|
|
transcriptPath: first.outputPath,
|
|
glossaryPath: base + ".glossary.yaml",
|
|
outputSchema: "bare-segments",
|
|
proposalResponses: noOpProposals,
|
|
validationResponses: nil,
|
|
expectedProposalCalls: []string{"glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", "grammar:proposal"},
|
|
reportSchemaName: reporting.DefaultProcessReportSchemaName,
|
|
reportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
|
|
expectedOutputSchema: "bare-segments",
|
|
expectModuleInstances: expectations.ExpectedModuleInstance,
|
|
expectUtilizationPaths: true,
|
|
})
|
|
firstSegments := mustReadTranscript(t, first.outputPath)
|
|
secondSegments := mustReadTranscript(t, second.outputPath)
|
|
if !reflect.DeepEqual(firstSegments, secondSegments) {
|
|
t.Fatalf("expected idempotent second pass transcript; first=%+v second=%+v", firstSegments, secondSegments)
|
|
}
|
|
if second.report.ModulesSummary == nil {
|
|
t.Fatalf("expected modules summary on second pass")
|
|
}
|
|
if second.report.ModulesSummary.TotalAppliedChanges != 0 {
|
|
t.Fatalf("expected no-op second pass (0 applied), got %+v", second.report.ModulesSummary)
|
|
}
|
|
}
|
|
|
|
type releaseRunConfig struct {
|
|
transcriptPath string
|
|
glossaryPath string
|
|
outputSchema string
|
|
proposalResponses []proposal_generation.StructuredCorrectionSet
|
|
validationResponses []validators.LLMValidationResponse
|
|
expectedProposalCalls []string
|
|
reportSchemaName string
|
|
reportSchemaVersion string
|
|
expectedOutputSchema string
|
|
expectModuleInstances []string
|
|
expectUtilizationPaths bool
|
|
}
|
|
|
|
type releaseRunResult struct {
|
|
stdout string
|
|
stderr string
|
|
outputPath string
|
|
reportPath string
|
|
report reporting.ProcessReport
|
|
runDir string
|
|
}
|
|
|
|
func runReleaseFixturePass(t *testing.T, cfg releaseRunConfig) releaseRunResult {
|
|
t.Helper()
|
|
|
|
processProposalLLMClient = &fakeStructuredLLMClient{proposalResponses: append([]proposal_generation.StructuredCorrectionSet(nil), cfg.proposalResponses...)}
|
|
processValidationLLMClient = &fakeStructuredLLMClient{validationResponses: append([]validators.LLMValidationResponse(nil), cfg.validationResponses...)}
|
|
t.Cleanup(func() {
|
|
processProposalLLMClient = nil
|
|
processValidationLLMClient = nil
|
|
})
|
|
|
|
workDir := t.TempDir()
|
|
reportPath := filepath.Join(t.TempDir(), "report.json")
|
|
outputPath := filepath.Join(t.TempDir(), "out.json")
|
|
configPath := writeFile(t, "release-config.yml", "version: 1\n")
|
|
|
|
args := []string{
|
|
"process",
|
|
cfg.transcriptPath,
|
|
"--glossary",
|
|
cfg.glossaryPath,
|
|
"--config",
|
|
configPath,
|
|
"--output",
|
|
outputPath,
|
|
"--output-schema",
|
|
cfg.outputSchema,
|
|
"--report-json",
|
|
reportPath,
|
|
"--work-dir",
|
|
workDir,
|
|
"--work-dir-retention",
|
|
"always",
|
|
}
|
|
|
|
var stdout, stderr bytes.Buffer
|
|
exitCode := Run(args, &stdout, &stderr)
|
|
if exitCode != 0 {
|
|
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
|
|
}
|
|
|
|
report := readProcessReport(t, reportPath)
|
|
if report.ReportMetadata.ReportSchemaName != cfg.reportSchemaName || report.ReportMetadata.ReportSchemaVersion != cfg.reportSchemaVersion {
|
|
t.Fatalf("unexpected report schema metadata: %+v", report.ReportMetadata)
|
|
}
|
|
if report.ReportMetadata.OutputSchema != cfg.expectedOutputSchema {
|
|
t.Fatalf("unexpected output schema metadata: got %q want %q", report.ReportMetadata.OutputSchema, cfg.expectedOutputSchema)
|
|
}
|
|
if len(cfg.expectModuleInstances) > 0 {
|
|
got := make([]string, 0, len(report.ModuleResults))
|
|
for _, mr := range report.ModuleResults {
|
|
got = append(got, mr.ModuleInstance)
|
|
}
|
|
if !reflect.DeepEqual(got, cfg.expectModuleInstances) {
|
|
t.Fatalf("unexpected module instances: got %v want %v", got, cfg.expectModuleInstances)
|
|
}
|
|
}
|
|
if report.Diagnostics == nil {
|
|
t.Fatalf("expected diagnostics metadata")
|
|
}
|
|
if cfg.expectUtilizationPaths {
|
|
if report.Diagnostics.UtilizationSummaryPath == "" || report.Diagnostics.CorrectionLedgerPath == "" {
|
|
t.Fatalf("expected utilization/ledger artifact paths in report diagnostics: %+v", report.Diagnostics)
|
|
}
|
|
}
|
|
|
|
runDir := onlyRunDir(t, workDir)
|
|
if _, err := os.Stat(filepath.Join(runDir, "report.json")); err != nil {
|
|
t.Fatalf("expected run-dir report: %v", err)
|
|
}
|
|
|
|
if c, ok := processProposalLLMClient.(*fakeStructuredLLMClient); ok {
|
|
if !reflect.DeepEqual(c.calls, cfg.expectedProposalCalls) {
|
|
t.Fatalf("unexpected proposal call order: got %v want %v", c.calls, cfg.expectedProposalCalls)
|
|
}
|
|
}
|
|
|
|
return releaseRunResult{
|
|
stdout: stdout.String(),
|
|
stderr: stderr.String(),
|
|
outputPath: outputPath,
|
|
reportPath: reportPath,
|
|
report: report,
|
|
runDir: runDir,
|
|
}
|
|
}
|
|
|
|
func readProposalResponses(t *testing.T, path string) []proposal_generation.StructuredCorrectionSet {
|
|
t.Helper()
|
|
var out []proposal_generation.StructuredCorrectionSet
|
|
if err := json.Unmarshal(readFile(t, path), &out); err != nil {
|
|
t.Fatalf("unmarshal proposal responses: %v", err)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func readValidationResponses(t *testing.T, path string) []validators.LLMValidationResponse {
|
|
t.Helper()
|
|
var out []validators.LLMValidationResponse
|
|
if err := json.Unmarshal(readFile(t, path), &out); err != nil {
|
|
t.Fatalf("unmarshal validation responses: %v", err)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func mustReadTranscript(t *testing.T, path string) []schema.Segment {
|
|
t.Helper()
|
|
transcript, err := schema.ParseTranscriptJSON(readFile(t, path))
|
|
if err != nil {
|
|
t.Fatalf("parse transcript output: %v", err)
|
|
}
|
|
return transcript.Segments
|
|
}
|
|
|
|
func expectFinalTranscriptContains(t *testing.T, segments []schema.Segment, needles []string) {
|
|
t.Helper()
|
|
joined := flattenTranscriptText(segments)
|
|
for _, needle := range needles {
|
|
if !strings.Contains(joined, needle) {
|
|
t.Fatalf("expected transcript to contain %q, got %q", needle, joined)
|
|
}
|
|
}
|
|
}
|
|
|
|
func expectFinalTranscriptDoesNotContain(t *testing.T, segments []schema.Segment, needles []string) {
|
|
t.Helper()
|
|
joined := flattenTranscriptText(segments)
|
|
for _, needle := range needles {
|
|
if strings.Contains(joined, needle) {
|
|
t.Fatalf("expected transcript to not contain %q, got %q", needle, joined)
|
|
}
|
|
}
|
|
}
|
|
|
|
func flattenTranscriptText(segments []schema.Segment) string {
|
|
parts := make([]string, 0, len(segments))
|
|
for _, s := range segments {
|
|
parts = append(parts, s.Text)
|
|
}
|
|
return strings.Join(parts, "\n")
|
|
}
|
|
|
|
func assertReleaseCounts(t *testing.T, report reporting.ProcessReport, exp releaseFixtureExpectations) {
|
|
t.Helper()
|
|
if report.ModulesSummary == nil {
|
|
t.Fatalf("expected modules_summary")
|
|
}
|
|
if report.ModulesSummary.TotalAppliedChanges < exp.MinimumCounts.Applied {
|
|
t.Fatalf("expected at least %d applied changes, got %+v", exp.MinimumCounts.Applied, report.ModulesSummary)
|
|
}
|
|
validatorRejected := 0
|
|
skipped := 0
|
|
for _, mr := range report.ModuleResults {
|
|
validatorRejected += len(mr.ValidatorRejected)
|
|
skipped += len(mr.SkippedChanges)
|
|
}
|
|
if validatorRejected < exp.MinimumCounts.Rejected {
|
|
t.Fatalf("expected at least %d validator rejections, got %d", exp.MinimumCounts.Rejected, validatorRejected)
|
|
}
|
|
if skipped < exp.MinimumCounts.Skipped {
|
|
t.Fatalf("expected at least %d application skips, got %d", exp.MinimumCounts.Skipped, skipped)
|
|
}
|
|
}
|
|
|
|
func assertReleaseUtilizationShape(t *testing.T, report reporting.ProcessReport) {
|
|
t.Helper()
|
|
var payload struct {
|
|
EffectiveConcurrency struct {
|
|
TotalLLM int `json:"total_llm"`
|
|
} `json:"effective_concurrency"`
|
|
RunTiming struct {
|
|
SchedulerQueueWaitMS int64 `json:"scheduler_queue_wait_ms"`
|
|
LLMExecutionTimeMS int64 `json:"llm_execution_time_ms"`
|
|
DeterministicValidationMS int64 `json:"deterministic_validation_time_ms"`
|
|
} `json:"run_timing"`
|
|
Modules []map[string]any `json:"modules"`
|
|
Validators []map[string]any `json:"validators"`
|
|
}
|
|
if err := json.Unmarshal(readFile(t, report.Diagnostics.UtilizationSummaryPath), &payload); err != nil {
|
|
t.Fatalf("unmarshal utilization diagnostics: %v", err)
|
|
}
|
|
if payload.EffectiveConcurrency.TotalLLM <= 0 {
|
|
t.Fatalf("expected positive total llm concurrency, got %+v", payload.EffectiveConcurrency)
|
|
}
|
|
if payload.RunTiming.SchedulerQueueWaitMS < 0 || payload.RunTiming.LLMExecutionTimeMS < 0 || payload.RunTiming.DeterministicValidationMS < 0 {
|
|
t.Fatalf("expected non-negative run timing values, got %+v", payload.RunTiming)
|
|
}
|
|
if len(payload.Modules) == 0 {
|
|
t.Fatalf("expected module timing summaries")
|
|
}
|
|
if len(payload.Validators) == 0 {
|
|
t.Fatalf("expected validator timing summaries")
|
|
}
|
|
}
|
|
|
|
func assertReleaseLedgerShape(t *testing.T, report reporting.ProcessReport) {
|
|
t.Helper()
|
|
var entries []struct {
|
|
ModuleKey string `json:"module_key"`
|
|
ModuleInstance string `json:"module_instance"`
|
|
ProposalIndex int `json:"proposal_index"`
|
|
Disposition string `json:"disposition"`
|
|
DispositionReason string `json:"disposition_reason_code"`
|
|
OriginalText string `json:"original_text"`
|
|
ProposedCorrected string `json:"proposed_corrected_text"`
|
|
ReplacementPolicy string `json:"replacement_policy"`
|
|
DeterministicResults []struct {
|
|
ValidatorKey string `json:"validator_key"`
|
|
} `json:"deterministic_validator_decisions"`
|
|
LLMResults []struct {
|
|
ValidatorKey string `json:"validator_key"`
|
|
} `json:"llm_validator_decisions"`
|
|
}
|
|
if err := json.Unmarshal(readFile(t, report.Diagnostics.CorrectionLedgerPath), &entries); err != nil {
|
|
t.Fatalf("unmarshal correction ledger: %v", err)
|
|
}
|
|
if len(entries) == 0 {
|
|
t.Fatalf("expected correction ledger entries")
|
|
}
|
|
hasApplied := false
|
|
hasRejected := false
|
|
hasSkipped := false
|
|
for _, entry := range entries {
|
|
if entry.ModuleInstance == "" || entry.ModuleKey == "" {
|
|
t.Fatalf("expected module identity in ledger entry: %+v", entry)
|
|
}
|
|
switch entry.Disposition {
|
|
case "applied":
|
|
hasApplied = true
|
|
case "rejected":
|
|
hasRejected = true
|
|
case "skipped":
|
|
hasSkipped = true
|
|
}
|
|
}
|
|
if !hasApplied || !hasRejected {
|
|
t.Fatalf("expected applied and rejected entries in correction ledger, got %+v", entries)
|
|
}
|
|
if !hasSkipped {
|
|
// Some deterministic fixture paths do not trigger apply-time skips;
|
|
// rejections are still captured separately from application skips.
|
|
}
|
|
}
|
|
|
|
func assertPromptAndSchemaMetadataPresent(t *testing.T, runDir string) {
|
|
t.Helper()
|
|
metadataPaths, err := filepath.Glob(filepath.Join(runDir, "*", "*request-metadata.json"))
|
|
if err != nil {
|
|
t.Fatalf("glob request metadata artifacts: %v", err)
|
|
}
|
|
if len(metadataPaths) == 0 {
|
|
t.Fatalf("expected request metadata artifacts with prompt metadata")
|
|
}
|
|
|
|
foundPromptMetadata := false
|
|
foundSchemaMetadata := false
|
|
for _, path := range metadataPaths {
|
|
var payload map[string]any
|
|
if err := json.Unmarshal(readFile(t, path), &payload); err != nil {
|
|
t.Fatalf("unmarshal request metadata artifact %q: %v", path, err)
|
|
}
|
|
if pm, ok := payload["prompt_metadata"].(map[string]any); ok {
|
|
if pm["prompt_id"] != nil && pm["prompt_version"] != nil && pm["sha256"] != nil {
|
|
foundPromptMetadata = true
|
|
}
|
|
}
|
|
if sm, ok := payload["response_schema"].(map[string]any); ok {
|
|
if sm["id"] != nil && sm["version"] != nil && sm["name"] != nil && sm["sha256"] != nil {
|
|
foundSchemaMetadata = true
|
|
}
|
|
}
|
|
}
|
|
if !foundPromptMetadata {
|
|
t.Fatalf("expected prompt metadata in request metadata artifacts")
|
|
}
|
|
if !foundSchemaMetadata {
|
|
t.Fatalf("expected structured response schema metadata in request metadata artifacts")
|
|
}
|
|
}
|
|
|
|
func assertStableValidatorKeysPresent(t *testing.T, report reporting.ProcessReport) {
|
|
t.Helper()
|
|
seen := map[string]bool{}
|
|
for _, module := range report.ModuleResults {
|
|
for _, decision := range module.ValidatorDecisions {
|
|
seen[decision.ValidatorName] = true
|
|
}
|
|
for _, rejected := range module.ValidatorRejected {
|
|
seen[rejected.ValidatorName] = true
|
|
}
|
|
}
|
|
expectedAny := []string{
|
|
"confidence_threshold",
|
|
"original_text_presence",
|
|
"no_effect",
|
|
}
|
|
for _, key := range expectedAny {
|
|
if !seen[key] {
|
|
t.Fatalf("expected stable validator key %q in report decisions/rejections; seen=%v", key, seen)
|
|
}
|
|
}
|
|
}
|
|
|
|
func assertStdoutStderrContract(t *testing.T, stdout, stderr string) {
|
|
t.Helper()
|
|
if stdout != "" {
|
|
t.Fatalf("expected empty stdout with --output, got %q", stdout)
|
|
}
|
|
if strings.Contains(stderr, `"module_results"`) || strings.Contains(stderr, `"report_metadata"`) {
|
|
t.Fatalf("stderr should remain human-readable, not report JSON: %q", stderr)
|
|
}
|
|
}
|
|
|
|
func assertAuditaV1OutputShape(t *testing.T, outputPath string) {
|
|
t.Helper()
|
|
var payload struct {
|
|
Schema string `json:"schema"`
|
|
Version string `json:"version"`
|
|
Segments []schema.Segment `json:"segments"`
|
|
}
|
|
if err := json.Unmarshal(readFile(t, outputPath), &payload); err != nil {
|
|
t.Fatalf("unmarshal audita-v1 output: %v", err)
|
|
}
|
|
if payload.Schema != "audita-v1" || payload.Version != "v1" {
|
|
t.Fatalf("unexpected audita-v1 metadata: %+v", payload)
|
|
}
|
|
if len(payload.Segments) == 0 {
|
|
t.Fatalf("expected non-empty audita-v1 segments")
|
|
}
|
|
}
|