From a355f05a273d6ac9d577eab8d30caa60cb74eee8 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Mon, 16 Nov 2026 15:19:59 +0000 Subject: [PATCH] Verify the reliability sprint and guard release checkout validation --- .woodpecker/release.yml | 3 +- cmd/audita/main.go | 7 +- cmd/audita/main_integration_test.go | 50 ++----- docs/internal/llm-runtime.md | 3 +- docs/migrations/v2.md | 3 + docs/roadmap/implementation.md | 67 ++++++++- .../cli/process_ordered_integration_test.go | 67 +++++++-- internal/cli/runtime_errors_test.go | 25 +++- .../processreport/correction_ledger.go | 6 +- .../framework/responseschema/registry_test.go | 11 -- internal/framework/runner/runner_test.go | 83 ----------- internal/prompts/registry_test.go | 28 ++-- internal/testsupport/files.go | 57 -------- internal/validators/registry_test.go | 117 ++++------------ scripts/release_test.go | 129 +++++++++++++++--- 15 files changed, 322 insertions(+), 334 deletions(-) delete mode 100644 internal/testsupport/files.go diff --git a/.woodpecker/release.yml b/.woodpecker/release.yml index 0887346..82bd8ba 100644 --- a/.woodpecker/release.yml +++ b/.woodpecker/release.yml @@ -24,7 +24,8 @@ steps: test "$(printf '%s' "$${tag_commit}" | tr '[:upper:]' '[:lower:]')" = "$${expected_commit}" git checkout --detach "$${tag_commit}" test "$(git rev-parse HEAD | tr '[:upper:]' '[:lower:]')" = "$${expected_commit}" - test -z "$(git status --porcelain --untracked-files=all)" + checkout_status=$(git status --porcelain --untracked-files=all) || exit $? + test -z "$${checkout_status}" bootstrap_dir=$(mktemp -d "$${CI_WORKSPACE%/*}/.releasekit-bootstrap.XXXXXX") trap 'rm -rf "$${bootstrap_dir}"' EXIT diff --git a/cmd/audita/main.go b/cmd/audita/main.go index 8f1b96e..6d351f8 100644 --- a/cmd/audita/main.go +++ b/cmd/audita/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "io" "os" "os/signal" "syscall" @@ -10,11 +11,11 @@ import ( ) func main() { - os.Exit(run()) + os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) } -func run() int { +func run(args []string, stdout, stderr io.Writer) int { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() - return cli.Run(ctx, os.Args[1:], os.Stdout, os.Stderr) + return cli.Run(ctx, args, stdout, stderr) } diff --git a/cmd/audita/main_integration_test.go b/cmd/audita/main_integration_test.go index 7656183..6812714 100644 --- a/cmd/audita/main_integration_test.go +++ b/cmd/audita/main_integration_test.go @@ -12,7 +12,6 @@ import ( "net/http/httptest" "os" "os/exec" - "os/signal" "path/filepath" "reflect" "strconv" @@ -41,10 +40,7 @@ func TestProcessSubprocessHelper(t *testing.T) { } syscall.Umask(int(mask)) } - ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) - code := cli.Run(ctx, os.Args[i+1:], os.Stdout, os.Stderr) - stop() - os.Exit(code) + os.Exit(run(os.Args[i+1:], os.Stdout, os.Stderr)) } } os.Exit(2) @@ -237,7 +233,6 @@ type processFixture struct { dir string config string transcript string - glossary string output string provider *fixtureProvider server *httptest.Server @@ -282,11 +277,8 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture { if err := os.WriteFile(transcript, []byte(`[{"id":1,"speaker":"A","start":0,"end":1,"text":"hello world"}]`), 0600); err != nil { t.Fatal(err) } - glossary := filepath.Join(dir, "glossary.yaml") - if err := os.WriteFile(glossary, []byte("glossary:\n - name: Example\n category: product\n summary: A sample.\n"), 0600); err != nil { - t.Fatal(err) - } - return processFixture{dir: dir, config: configPath, transcript: transcript, glossary: glossary, + + return processFixture{dir: dir, config: configPath, transcript: transcript, output: filepath.Join(dir, "output.json"), provider: p, server: server} } @@ -314,25 +306,6 @@ func (f processFixture) run(t *testing.T, includeOutput bool, extra ...string) ( return code, stdout.String(), stderr.String() } -func (f processFixture) runConfigView(t *testing.T, view string) (int, string, string) { - t.Helper() - command := exec.Command(os.Args[0], "-test.run=^TestProcessSubprocessHelper$", "--", "config", "print-effective", "--config", f.config, "--view", view) - command.Dir = f.dir - command.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1", "AUDITA_TEST_PROVIDER_KEY="+f.provider.secret) - var stdout, stderr bytes.Buffer - command.Stdout, command.Stderr = &stdout, &stderr - err := command.Run() - code := 0 - if err != nil { - if exit, ok := err.(*exec.ExitError); ok { - code = exit.ExitCode() - } else { - t.Fatal(err) - } - } - return code, stdout.String(), stderr.String() -} - func (f processFixture) errors(t *testing.T) reporting.ErrorsDocument { t.Helper() raw, err := os.ReadFile(filepath.Join(f.dir, "errors.yml")) @@ -879,7 +852,7 @@ func TestSubprocessReportingAndDebugSelections(t *testing.T) { for _, selection := range []struct{ reporting, debug bool }{ {false, false}, {false, true}, {true, false}, {true, true}, } { - f := newProcessFixture(t, "repair-filter-later-failure", "[grammar, grammar]") + f := newProcessFixture(t, "normal", "[grammar]") reportDir := filepath.Join(f.dir, "missing") extra := []string{"--verbose", "--reporting=false", "--report-dir", reportDir} @@ -894,7 +867,7 @@ func TestSubprocessReportingAndDebugSelections(t *testing.T) { extra = append(extra, "--debug") } code, stdout, stderr := f.run(t, true, extra...) - if code != 0 || stdout != "" || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors; 1 recoveries") { + if code != 0 || stdout != "" || !strings.Contains(stderr, "1 corrections applied; 0 work units with errors; 0 recoveries") { t.Fatalf("selection=%+v code=%d stdout=%q stderr=%q", selection, code, stdout, stderr) } output, err := os.ReadFile(f.output) @@ -1072,8 +1045,9 @@ func TestSubprocessCancellationLeavesFailedReportsWithoutTranscript(t *testing.T t.Fatal(err) } err := command.Wait() - if ctx.Err() != nil || err == nil { - t.Fatalf("cancellation did not stop run: %v %v stderr=%q", err, ctx.Err(), stderr.String()) + var exit *exec.ExitError + if ctx.Err() != nil || !errors.As(err, &exit) || exit.ExitCode() != 1 { + t.Fatalf("interrupt did not reach the failure exit: %v %v stderr=%q", err, ctx.Err(), stderr.String()) } if document := f.errors(t); document.Status != reporting.StatusFailed || len(document.Errors) == 0 { t.Fatalf("cancellation report: %+v", document) @@ -1556,11 +1530,13 @@ func TestSubprocessEmitsProgressBeforeProviderReturns(t *testing.T) { case <-ctx.Done(): t.Fatal("proposal progress did not reach stderr while the provider was blocked") } - if err := cmd.Process.Signal(os.Interrupt); err != nil { + if err := cmd.Process.Signal(syscall.SIGTERM); err != nil { t.Fatal(err) } - if err := cmd.Wait(); err == nil || ctx.Err() != nil { - t.Fatalf("interrupt did not complete the invocation: %v", err) + err = cmd.Wait() + var exit *exec.ExitError + if !errors.As(err, &exit) || exit.ExitCode() != 1 || ctx.Err() != nil { + t.Fatalf("termination did not reach the failure exit: %v", err) } } diff --git a/docs/internal/llm-runtime.md b/docs/internal/llm-runtime.md index 3725538..5e86211 100644 --- a/docs/internal/llm-runtime.md +++ b/docs/internal/llm-runtime.md @@ -16,7 +16,8 @@ repair. An expired parent context stops processing and retains any native cause. With an active parent, PromptKit's operation deadline remains recoverable even when a preparation error wraps it. A native transport timeout affects the HTTP attempt and is recorded as a transport failure. The prepared notification -provides the actual request snapshot; failures before notification have no snapshot. +supplies prompt metadata to observers. With debug capture enabled, it also +supplies the actual request messages when preparation reaches that point. PromptKit owns backend and credential selection. When a required credential is absent, the CLI identifies the role, profile, and environment variable using the native diagnostic. See the [integration contract](../integrations/openai-compatible-llm.md) diff --git a/docs/migrations/v2.md b/docs/migrations/v2.md index 33b6ee8..0fb601b 100644 --- a/docs/migrations/v2.md +++ b/docs/migrations/v2.md @@ -1,5 +1,8 @@ # Migrating to Audita v2 +This guide describes the historical v2 release. For current commands, +configuration, and example projects, use the [v3 migration guide](v3.md). + Audita v2 adopts ConfigKit for configuration and PromptKit for LLM execution. Version-1 configuration files, environment overlays, and retired flags are not accepted. Copy a maintained [local](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.1.1/examples/minimal-config.yml) or diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index dccf160..6527e6f 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -52,7 +52,7 @@ record the mismatch explicitly. A test that asserts only nonzero status does not verify this fix. **Complete.** The upstream handoff is recorded below. The exit-status fix -remains outstanding for the final verification pass. +remains outstanding after the final verification pass. Build a temporary executable with `go build -o /tmp/audita-inspection ./cmd/audita`. In a scratch directory, put `version: 3` in `config.yml`, `profiles: [` in the @@ -747,3 +747,68 @@ Audit #28, #29, #58, #60–62 and sprint closure. The completed sprint should leave the target architecture reachable through `audita process` and the standard delegated commands. Report checks and measurements actually performed, including any outstanding external dependency. + +**Complete.** The integrated Audita work and release bootstrap passed checks. +The delegated inspection exit status remains an outstanding upstream item. + +The bootstrap rejects a dirty checkout and a failed status command with no +output. A clean fixture reaches ReleaseKit with the prepared checkout and +expected commit arguments. Process fixtures share the production entry point +and check interrupt and termination exit status. Registry coverage checks +effective roles, required prompt identities and preparation, schema identity, +and independent copies. The ordered processing fixture decodes debug artifact +coordinates. The full repair/filter/later-failure case remains separate from +the smaller output-choice matrix. Unused fixture helpers and the legacy +glossary input were removed. + +### Delegated inspection status + +The [CommandKit upstream tags](https://forge.maximumdirect.net/base-layer/commandkit/tags) +confirmed `v0.7.0` is the latest release of both `ckprompt` and `ckconfig`. +The built Audita executable reproduced the Stage 1 mismatch: + +| Invalid input | Path | Observed status and diagnostic | +| --- | --- | --- | +| Malformed automatic `prompts.yml` | `process` | 2, `yaml.invalid` | +| Malformed automatic `prompts.yml` | `promptkit list prompts` | 1, `yaml.invalid` | +| Malformed automatic `prompts.yml` | `config show/sources/validate/explain/diff --scope promptkit` | 1, `yaml.invalid` | +| Malformed explicit app main file | `config show/sources/validate/explain/diff --scope app` | 1, `yaml.invalid` | + +Caller cancellation remains status 1 in the shared CLI and subprocess tests. +The CommandKit integrations still own the fix to return status 2 for typed +operator-input failures. Audita retains its native delegations. The expected +upstream change and reproduction remain in the Stage 1 handoff. + +### Catalog closure + +The table identifies the implementing stages and exceptions without changing +the historical audit observations. Each stage records its verification above. + +| Catalog findings | Implementing stages or disposition | +| --- | --- | +| #1, #6–8, #16, #26, #42 | Stage 4. Only cooperative cancellation boundaries from #6 were selected. | +| #2, #4 | No implementation. The existing catalog dispositions stand. | +| #3, #9, #63–65 | Stages 2 and 14. | +| #5, #57 | Stage 19. | +| #10–11, #27, #40, #49 | Stages 9, 11, and 17. | +| #12, #18–19, #38, #47 | Stage 8. Only punctuation-edge handling from #18 was selected. | +| #13, #31, #51–52, #54 | Stages 2 and 5. | +| #14–15, #22, #30–32, #39, #41 | Stages 5, 12, and 15–18. | +| #17, #21, #35, #53, #59, #62 | Stages 5 and 6, with final caller checks in Stage 20. | +| #20, #24 | Stages 7 and 11 for processing; Stage 1 and the upstream disposition above for delegated inspection. | +| #23, #25, #44 | Stages 12, 18, and 19. | +| #28–29, #58, #60–61 | Stage 20. | +| #33, #43 | Stages 3 and 18. | +| #34, #45, #48 | Stage 14. | +| #36 | Stage 10. | +| #37, #50, #55–56 | Stage 11. | +| #46 | Stage 13. | + +Verified with Go 1.27.3 by building the CLI, running `go test -count=1 ./...` +offline, and running targeted race checks for the runner, LLM adapter, +validators, reports, diagnostics, glossary, and CLI. The final review +checked Go formatting, whitespace, and 278 relative documentation links +and exercised the example configurations. Measurements above cover merge +and glossary allocation reductions, validation context, repair request and +artifact costs, bounded section contention, and retained evidence by output +mode. These are synthetic measurements. No release was published. diff --git a/internal/cli/process_ordered_integration_test.go b/internal/cli/process_ordered_integration_test.go index 6575b76..7e03e72 100644 --- a/internal/cli/process_ordered_integration_test.go +++ b/internal/cli/process_ordered_integration_test.go @@ -16,7 +16,6 @@ import ( "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" "gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics" - "gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm" commandkit "go.maximumdirect.net/commandkit" @@ -138,21 +137,63 @@ pipeline: if len(result.Snapshot.Recoveries) != 1 || result.Snapshot.Recoveries[0].StageIndex != 0 || result.Snapshot.Recoveries[0].StageID == nil || *result.Snapshot.Recoveries[0].StageID != "party/../|" { t.Fatalf("recovery attribution: %+v", result.Snapshot.Recoveries) } - markdown := string(reporting.Markdown(result.Snapshot)) - if !strings.Contains(markdown, `1: glossary (party/../\|<tag>)`) || !strings.Contains(markdown, "2: glossary (spell)") || !strings.Contains(markdown, "3: grammar") { - t.Fatalf("report labels: %s", markdown) + type coordinates struct { + StageIndex int `json:"stage_index"` + StageID *string `json:"stage_id"` } - for _, artifact := range []string{diagnostics.ArtifactCorrectionLedger, diagnostics.ArtifactLLMExecution} { - raw, readErr := os.ReadFile(filepath.Join(prepared.DebugPath(), artifact)) - if readErr != nil { - t.Fatal(readErr) + expected := map[int]string{0: "party/../|", 1: "spell", 2: ""} + check := func(name string, entries []coordinates) { + t.Helper() + seen := make(map[int]bool) + for _, entry := range entries { + id, ok := expected[entry.StageIndex] + if !ok || (id == "" && entry.StageID != nil) || (id != "" && (entry.StageID == nil || *entry.StageID != id)) { + t.Fatalf("%s has wrong stage association: %+v", name, entry) + } + seen[entry.StageIndex] = true } - var compact bytes.Buffer - if err := json.Compact(&compact, raw); err != nil { - t.Fatalf("decode %s: %v", artifact, err) + if len(seen) != len(expected) { + t.Fatalf("%s missing stage associations: %v", name, seen) } - if text := compact.String(); !strings.Contains(text, `"stage_index":0`) || !strings.Contains(text, `"stage_id":"party/../|\u003ctag\u003e"`) || !strings.Contains(text, `"stage_index":1`) || !strings.Contains(text, `"stage_id":"spell"`) || !strings.Contains(text, `"stage_index":2`) { - t.Fatalf("%s missing attribution: %s", artifact, raw) + } + raw, err := os.ReadFile(filepath.Join(prepared.DebugPath(), diagnostics.ArtifactCorrectionLedger)) + if err != nil { + t.Fatal(err) + } + var ledger []coordinates + if err := json.Unmarshal(raw, &ledger); err != nil { + t.Fatal(err) + } + check("ledger", ledger) + raw, err = os.ReadFile(filepath.Join(prepared.DebugPath(), diagnostics.ArtifactLLMExecution)) + if err != nil { + t.Fatal(err) + } + var calls struct { + Calls []struct { + Context struct { + coordinates + Role string + } + } + } + if err := json.Unmarshal(raw, &calls); err != nil { + t.Fatal(err) + } + var locations []coordinates + roles := make(map[int]map[string]bool) + for _, call := range calls.Calls { + location := call.Context.coordinates + locations = append(locations, location) + if roles[location.StageIndex] == nil { + roles[location.StageIndex] = make(map[string]bool) + } + roles[location.StageIndex][call.Context.Role] = true + } + check("LLM observations", locations) + for index := range expected { + if !roles[index]["proposal"] || !roles[index]["validation"] { + t.Fatalf("stage %d missing role evidence: %v", index, roles[index]) } } if !reflect.DeepEqual(models, []string{"model-one", "model-one", "review", "review", "model-two", "review", "review", "model-three", "review", "review"}) { diff --git a/internal/cli/runtime_errors_test.go b/internal/cli/runtime_errors_test.go index c52a336..7318050 100644 --- a/internal/cli/runtime_errors_test.go +++ b/internal/cli/runtime_errors_test.go @@ -6,6 +6,7 @@ import ( "errors" "os" "path/filepath" + "strings" "testing" commandkit "go.maximumdirect.net/commandkit" @@ -23,7 +24,7 @@ func writeRuntimeErrorConfig(t *testing.T, operatorDocument string) string { return main } -func TestResourceCommandsClassifyOperatorPromptErrorsAsInputErrors(t *testing.T) { +func TestResourceCommandsRejectInvalidOperatorPrompts(t *testing.T) { definitions := map[string]string{ "malformed operator YAML": "profiles: [\n", "invalid operator template": "collections:\n audita:\n prompts:\n - id: modules.grammar.proposal\n messages:\n - role: user\n content: '{{'\n output:\n format: text\n validation_mode: none\n", @@ -40,6 +41,28 @@ func TestResourceCommandsClassifyOperatorPromptErrorsAsInputErrors(t *testing.T) } } +func TestMalformedCompanionKeepsDelegatedExitStatusVisible(t *testing.T) { + main := writeRuntimeErrorConfig(t, "profiles: [\n") + transcript := filepath.Join(filepath.Dir(main), "transcript.json") + if err := os.WriteFile(transcript, []byte("[]\n"), 0600); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + args []string + status int + }{ + {[]string{"promptkit", "list", "prompts"}, 1}, + {[]string{"config", "show", "--scope", "promptkit"}, 1}, + {[]string{"process", transcript}, 2}, + } { + var stdout, stderr bytes.Buffer + code := Run(context.Background(), append(tc.args, "--config", main), &stdout, &stderr) + if code != tc.status || !strings.Contains(stderr.String(), "yaml.invalid") || stdout.Len() != 0 { + t.Fatalf("%v: status=%d want=%d stdout=%q stderr=%q", tc.args, code, tc.status, stdout.String(), stderr.String()) + } + } +} + func TestResourceCommandKeepsCallerCancellationAsFailure(t *testing.T) { main := writeRuntimeErrorConfig(t, "profiles:\n - id: proposal\n model: local-model\n") transcript := filepath.Join(filepath.Dir(main), "transcript.json") diff --git a/internal/framework/processreport/correction_ledger.go b/internal/framework/processreport/correction_ledger.go index 9b1ab91..eda2aa2 100644 --- a/internal/framework/processreport/correction_ledger.go +++ b/internal/framework/processreport/correction_ledger.go @@ -234,7 +234,11 @@ func recordLedgerDecision(decision runner.ValidatorDecisionRecord, deterministic ArtifactPath: decision.DiagnosticArtifactPath, ValidatorKey: decision.ValidatorName, Approved: decision.Approved, Disposition: decision.Disposition, ReasonCode: decision.ReasonCode, Message: decision.Message, } - if decision.ExecutionClass == validatormetadata.ExecutionClassLLMBacked { + class := decision.ExecutionClass + if class == "" { + class = validatormetadata.ClassForKey(decision.ValidatorName) + } + if class == validatormetadata.ExecutionClassLLMBacked { *llm = append(*llm, record) } else { *deterministic = append(*deterministic, record) diff --git a/internal/framework/responseschema/registry_test.go b/internal/framework/responseschema/registry_test.go index 9417a21..982bdf6 100644 --- a/internal/framework/responseschema/registry_test.go +++ b/internal/framework/responseschema/registry_test.go @@ -68,17 +68,6 @@ func TestLookupUnknownSchema(t *testing.T) { } } -func TestRegisteredSchemasLoadReadableEmbeddedJSON(t *testing.T) { - for _, schema := range Registered() { - if len(schema.JSONSchema) == 0 { - t.Fatalf("expected non-empty JSON schema for %q", schema.ID) - } - if !json.Valid(schema.JSONSchema) { - t.Fatalf("expected valid JSON schema for %q", schema.ID) - } - } -} - func TestSchemaHashesMatchRegisteredJSON(t *testing.T) { for _, key := range []Key{CorrectionSetKey, CorrectionRepairKey, ValidatorDecisionSetKey} { schema, ok := Lookup(key) diff --git a/internal/framework/runner/runner_test.go b/internal/framework/runner/runner_test.go index 1c41b90..a7c5682 100644 --- a/internal/framework/runner/runner_test.go +++ b/internal/framework/runner/runner_test.go @@ -7,7 +7,6 @@ import ( "os" "path/filepath" "reflect" - "strconv" "strings" "sync" "testing" @@ -451,34 +450,6 @@ func TestRunnerProposalsExecutePerChunkSection(t *testing.T) { } } -func waitForRunnerEntries(t *testing.T, entered <-chan struct{}, want int, label string) { - t.Helper() - deadline := time.After(350 * time.Millisecond) - got := 0 - for got < want { - select { - case <-entered: - got++ - case <-deadline: - t.Fatalf("timed out waiting for %d entries (%s), got %d", want, label, got) - } - } -} - -func waitForRunnerSectionIDs(t *testing.T, started <-chan int, want map[int]struct{}) { - t.Helper() - deadline := time.After(350 * time.Millisecond) - seen := map[int]struct{}{} - for len(seen) < len(want) { - select { - case id := <-started: - seen[id] = struct{}{} - case <-deadline: - t.Fatalf("timed out waiting for section IDs %v, got %v", mapKeys(want), mapKeys(seen)) - } - } -} - func mapKeys(values map[int]struct{}) []int { keys := make([]int, 0, len(values)) for key := range values { @@ -563,60 +534,6 @@ func TestRunnerValidationStartsBeforeAllSectionProposalsComplete(t *testing.T) { } } -type stageAwareStructuredClient struct { - startedSection chan int - releaseValidation <-chan struct{} - eventSink chan<- string -} - -type captureContractStructuredClient struct { - lastRequest contracts.StructuredCompletionRequest -} - -func (c *captureContractStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { - _ = ctx - c.lastRequest = req - if target, ok := out.(*validators.LLMValidationResponse); ok { - *target = validators.LLMValidationResponse{ - Validations: []validators.LLMValidationDecision{ - {CorrectionIndex: "1", Approved: true, Reason: "ok"}, - }, - } - } - return contracts.StructuredCompletionResponse{}, nil -} - -func (c *stageAwareStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { - section := 0 - if req.Context.SectionIndex != nil { - section = *req.Context.SectionIndex - } - if c.startedSection != nil { - select { - case c.startedSection <- section: - default: - } - } - if c.eventSink != nil { - c.eventSink <- "v" + strconv.Itoa(section) - } - if c.releaseValidation != nil { - select { - case <-c.releaseValidation: - case <-ctx.Done(): - return contracts.StructuredCompletionResponse{}, ctx.Err() - } - } - - target := out.(*validators.LLMValidationResponse) - *target = validators.LLMValidationResponse{ - Validations: []validators.LLMValidationDecision{ - {CorrectionIndex: "1", Approved: true, Reason: "ok"}, - }, - } - return contracts.StructuredCompletionResponse{}, nil -} - func TestRunnerSkippedRecorded(t *testing.T) { transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "word word"}}} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ diff --git a/internal/prompts/registry_test.go b/internal/prompts/registry_test.go index fb3c6d8..08bfa4f 100644 --- a/internal/prompts/registry_test.go +++ b/internal/prompts/registry_test.go @@ -76,19 +76,27 @@ func TestEmbeddedDefinitionsPrepareThroughPromptKit(t *testing.T) { } } -func TestRegistryMetadataMatchesEmbeddedDefinitions(t *testing.T) { - all := RegisteredMetadata() - if len(all) != 11 { - t.Fatalf("prompt count: %d", len(all)) +func TestRegistryMetadataIdentifiesRequiredPrompts(t *testing.T) { + required := []string{ + "audita/modules.glossary.proposal", "audita/modules.glossary.repair", + "audita/modules.grammar.proposal", "audita/modules.grammar.repair", + "audita/modules.homophones.proposal", "audita/modules.homophones.repair", + "audita/modules.spoken_word.proposal", "audita/modules.spoken_word.repair", + "audita/validators.editorial_review", "audita/validators.meaning_reversal_review", + "audita/validators.spoken_form_plausibility", } - for _, meta := range all { - if meta.PromptSource != SourceBuiltin || meta.EmbeddedPath == "" { - t.Fatalf("metadata: %+v", meta) - } - if _, err := Filesystem().Open(meta.EmbeddedPath); err != nil { - t.Fatal(err) + for _, id := range required { + metadata, ok := LookupMetadata(id) + if !ok || metadata.PromptID != id || metadata.PromptSource != SourceBuiltin || + metadata.EmbeddedPath != AssetRoot+"/"+strings.TrimPrefix(id, "audita/")+".yaml" { + t.Fatalf("required prompt metadata for %q: %+v", id, metadata) } } + all := RegisteredMetadata() + all[0].PromptID = "changed" + if again := RegisteredMetadata(); again[0].PromptID == "changed" { + t.Fatal("metadata enumeration shares mutable state") + } } func TestRepairDefinitionsPrepareThroughPromptKit(t *testing.T) { diff --git a/internal/testsupport/files.go b/internal/testsupport/files.go deleted file mode 100644 index d366394..0000000 --- a/internal/testsupport/files.go +++ /dev/null @@ -1,57 +0,0 @@ -package testsupport - -import ( - "os" - "path/filepath" - "strings" - "testing" -) - -func ReadFile(t *testing.T, path string) []byte { - t.Helper() - data, err := os.ReadFile(path) - if err != nil { - t.Fatalf("failed to read file %q: %v", path, err) - } - return data -} - -func OnlyRunDir(t *testing.T, workDir string) string { - t.Helper() - entries, err := os.ReadDir(workDir) - if err != nil { - t.Fatalf("failed to read work dir %q: %v", workDir, err) - } - dirs := make([]string, 0, len(entries)) - for _, e := range entries { - if e.IsDir() { - dirs = append(dirs, filepath.Join(workDir, e.Name())) - } - } - if len(dirs) != 1 { - t.Fatalf("expected exactly one run dir in %q, found %d", workDir, len(dirs)) - } - return dirs[0] -} - -func AssertNoSecretInFile(t *testing.T, path, secret string) { - t.Helper() - raw := string(ReadFile(t, path)) - if strings.Contains(raw, secret) { - t.Fatalf("secret leaked in %s", path) - } -} - -func AssertNoSecretInTree(t *testing.T, root, secret string) { - t.Helper() - _ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { - if err != nil || d == nil || d.IsDir() { - return nil - } - raw, readErr := os.ReadFile(path) - if readErr == nil && strings.Contains(string(raw), secret) { - t.Fatalf("secret leaked in %s", path) - } - return nil - }) -} diff --git a/internal/validators/registry_test.go b/internal/validators/registry_test.go index 795332e..196e66e 100644 --- a/internal/validators/registry_test.go +++ b/internal/validators/registry_test.go @@ -9,104 +9,33 @@ import ( "gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" - "gitea.maximumdirect.net/eric/audita/v3/internal/validators/editorial_review" - "gitea.maximumdirect.net/eric/audita/v3/internal/validators/meaning_reversal_review" validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata" - "gitea.maximumdirect.net/eric/audita/v3/internal/validators/no_effect" - "gitea.maximumdirect.net/eric/audita/v3/internal/validators/non_empty_resulting_segment" - "gitea.maximumdirect.net/eric/audita/v3/internal/validators/original_text_presence" - "gitea.maximumdirect.net/eric/audita/v3/internal/validators/proposal_shape" - "gitea.maximumdirect.net/eric/audita/v3/internal/validators/protected_terms" - "gitea.maximumdirect.net/eric/audita/v3/internal/validators/spoken_form_plausibility" ) -func TestBuiltInRegistryRegistersAllKeys(t *testing.T) { - r := NewBuiltInRegistry() - for _, key := range []string{ - KeyProposalShape, - KeyOriginalTextPresence, - KeyNonEmptyResultingSegment, - KeyNoEffect, - KeyProtectedTerms, - KeySpokenFormPlausibility, - KeyMeaningReversalReview, - KeyEditorialReview, +func TestRegisteredValidatorsUseDocumentedRoles(t *testing.T) { + registry := NewBuiltInRegistry() + for _, tc := range []struct { + key string + role validatormetadata.ExecutionClass + }{ + {KeyProposalShape, validatormetadata.ExecutionClassDeterministic}, + {KeyOriginalTextPresence, validatormetadata.ExecutionClassDeterministic}, + {KeyNonEmptyResultingSegment, validatormetadata.ExecutionClassDeterministic}, + {KeyNoEffect, validatormetadata.ExecutionClassDeterministic}, + {KeyProtectedTerms, validatormetadata.ExecutionClassDeterministic}, + {KeySpokenFormPlausibility, validatormetadata.ExecutionClassLLMBacked}, + {KeyMeaningReversalReview, validatormetadata.ExecutionClassLLMBacked}, + {KeyEditorialReview, validatormetadata.ExecutionClassLLMBacked}, } { - def, ok := r.Lookup(key) - if !ok { - t.Fatalf("expected validator key %q to be registered", key) - } - v, err := def.Build() - if err != nil { - t.Fatalf("build validator %q: %v", key, err) - } - if v.Name() != key { - t.Fatalf("expected validator name %q, got %q", key, v.Name()) - } - } -} - -func TestBuiltInValidatorPackagesConstruct(t *testing.T) { - type validatorCtor struct { - name string - key string - build func() (frameworkvalidators.Validator, error) - wantClass validatormetadata.ExecutionClass - } - cases := []validatorCtor{ - {name: "proposal_shape", key: KeyProposalShape, build: proposal_shape.New, wantClass: validatormetadata.ExecutionClassDeterministic}, - {name: "original_text_presence", key: KeyOriginalTextPresence, build: original_text_presence.New, wantClass: validatormetadata.ExecutionClassDeterministic}, - {name: "non_empty_resulting_segment", key: KeyNonEmptyResultingSegment, build: non_empty_resulting_segment.New, wantClass: validatormetadata.ExecutionClassDeterministic}, - {name: "no_effect", key: KeyNoEffect, build: no_effect.New, wantClass: validatormetadata.ExecutionClassDeterministic}, - {name: "protected_terms", key: KeyProtectedTerms, build: protected_terms.New, wantClass: validatormetadata.ExecutionClassDeterministic}, - {name: "spoken_form_plausibility", key: KeySpokenFormPlausibility, build: spoken_form_plausibility.New, wantClass: validatormetadata.ExecutionClassLLMBacked}, - {name: "meaning_reversal_review", key: KeyMeaningReversalReview, build: meaning_reversal_review.New, wantClass: validatormetadata.ExecutionClassLLMBacked}, - {name: "editorial_review", key: KeyEditorialReview, build: editorial_review.New, wantClass: validatormetadata.ExecutionClassLLMBacked}, - } - - for _, tc := range cases { - v, err := tc.build() - if err != nil { - t.Fatalf("%s: build: %v", tc.name, err) - } - if v.Name() != tc.key { - t.Fatalf("%s: expected key %q, got %q", tc.name, tc.key, v.Name()) - } - if got := validatormetadata.ClassOf(v); got != tc.wantClass { - t.Fatalf("%s: expected class %q, got %q", tc.name, tc.wantClass, got) - } - } -} - -func TestRegistryBuildsClassifiedValidators(t *testing.T) { - r := NewBuiltInRegistry() - for _, key := range r.RegisteredKeys() { - v, err := r.MustBuild(key) - if err != nil { - t.Fatalf("must build %q: %v", key, err) - } - if _, ok := v.(validatormetadata.ClassifiedValidator); !ok { - t.Fatalf("expected built validator %q to expose execution classification metadata", key) - } - want := validatormetadata.ClassForKey(key) - if got := validatormetadata.ClassOf(v); got != want { - t.Fatalf("expected class %q for %q, got %q", want, key, got) - } - } -} - -func TestExecutionClassResolvableByStableKeyAndByValidatorInstance(t *testing.T) { - r := NewBuiltInRegistry() - for _, key := range r.RegisteredKeys() { - v, err := r.MustBuild(key) - if err != nil { - t.Fatalf("must build %q: %v", key, err) - } - fromKey := validatormetadata.ClassForKey(key) - fromInstance := validatormetadata.ClassOf(v) - if fromInstance != fromKey { - t.Fatalf("class mismatch for %q: key=%q instance=%q", key, fromKey, fromInstance) - } + t.Run(tc.key, func(t *testing.T) { + v, err := registry.MustBuild(tc.key) + if err != nil { + t.Fatal(err) + } + if v.Name() != tc.key || validatormetadata.ClassOf(v) != tc.role { + t.Fatalf("registered name=%q role=%q, want %q and %q", v.Name(), validatormetadata.ClassOf(v), tc.key, tc.role) + } + }) } } diff --git a/scripts/release_test.go b/scripts/release_test.go index 0a33723..cff7f46 100644 --- a/scripts/release_test.go +++ b/scripts/release_test.go @@ -4,7 +4,12 @@ import ( "encoding/json" "maps" "os" + "os/exec" + "path/filepath" + "strings" "testing" + + "gopkg.in/yaml.v3" ) func TestReleaseKitProfileDeclaresProductAndDeliverySettings(t *testing.T) { @@ -60,27 +65,6 @@ func TestReleaseKitProfileDeclaresProductAndDeliverySettings(t *testing.T) { } } -func TestReleaseKitTargetMembership(t *testing.T) { - want := []string{"linux/amd64", "linux/arm64"} - for _, test := range []struct { - name string - targets []string - match bool - }{ - {"missing amd64", []string{"linux/arm64", "linux/arm64"}, false}, - {"missing arm64", []string{"linux/amd64", "linux/amd64"}, false}, - {"additional target", []string{"linux/amd64", "linux/arm64", "darwin/arm64"}, false}, - {"reordered", []string{"linux/arm64", "linux/amd64"}, true}, - {"repeated", []string{"linux/amd64", "linux/arm64", "linux/amd64"}, true}, - } { - t.Run(test.name, func(t *testing.T) { - if got := sameMembers(test.targets, want); got != test.match { - t.Fatalf("target membership for %q = %t, want %t", test.targets, got, test.match) - } - }) - } -} - func sameMembers(got, want []string) bool { gotSet := make(map[string]struct{}, len(got)) for _, value := range got { @@ -92,3 +76,106 @@ func sameMembers(got, want []string) bool { } return maps.Equal(gotSet, wantSet) } + +func TestReleaseBootstrapRequiresSuccessfulCleanStatus(t *testing.T) { + data, err := os.ReadFile("../.woodpecker/release.yml") + if err != nil { + t.Fatal(err) + } + var workflow struct { + Steps []struct { + Name string + Commands []string + } + } + if err := yaml.Unmarshal(data, &workflow); err != nil { + t.Fatal(err) + } + var script string + for _, step := range workflow.Steps { + if step.Name == "deliver" { + // Woodpecker removes one dollar from escaped shell variables. + script = strings.ReplaceAll(strings.Join(step.Commands, "\n"), "$${", "${") + } + } + if script == "" { + t.Fatal("delivery command is missing") + } + + for _, tc := range []struct { + name string + status string + exit int + }{ + {"clean", "clean", 0}, + {"dirty", "dirty", 1}, + {"status failed with no output", "failed", 93}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + workspace := filepath.Join(dir, "checkout") + if err := os.Mkdir(workspace, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(workspace, "go.mod"), []byte("module example.invalid/release\n"), 0600); err != nil { + t.Fatal(err) + } + writeCommand := func(name, script string) { + t.Helper() + if err := os.WriteFile(filepath.Join(dir, name), []byte("#!/bin/sh\nset -eu\n"+script), 0700); err != nil { + t.Fatal(err) + } + } + writeCommand("git", ` +case "$1" in + check-ref-format|fetch|checkout) ;; + rev-parse) printf '%s\n' "$CI_COMMIT_SHA" ;; + status) + test "$*" = 'status --porcelain --untracked-files=all' + case "$STATUS_FIXTURE" in + dirty) printf '%s\n' '?? untracked' ;; + failed) exit 93 ;; + esac + ;; + *) exit 99 ;; +esac +`) + writeCommand("go", ` +case "$*" in + 'env GOHOSTOS') printf '%s\n' linux ;; + 'env GOHOSTARCH') printf '%s\n' amd64 ;; +esac +printf '%s\n' "$*" >> "$GO_CALLS_FIXTURE" +`) + calls := filepath.Join(dir, "go-calls") + commit := strings.Repeat("a", 40) + command := exec.Command("sh", "-c", script) + command.Dir = workspace + command.Env = append(os.Environ(), "PATH="+dir+string(os.PathListSeparator)+os.Getenv("PATH"), + "CI_WORKSPACE="+workspace, "CI_COMMIT_TAG=v1.2.3", "CI_COMMIT_SHA="+commit, + "STATUS_FIXTURE="+tc.status, "GO_CALLS_FIXTURE="+calls) + output, err := command.CombinedOutput() + exit := 0 + if err != nil { + failure, ok := err.(*exec.ExitError) + if !ok { + t.Fatal(err) + } + exit = failure.ExitCode() + } + if exit != tc.exit { + t.Fatalf("exit=%d want %d: %s", exit, tc.exit, output) + } + transcript, err := os.ReadFile(calls) + if tc.exit != 0 { + if !os.IsNotExist(err) { + t.Fatalf("bootstrap reached Go after failed status: %s (%v)", transcript, err) + } + return + } + if err != nil || !strings.Contains(string(transcript), "ci-deliver --repo "+workspace+" --prepared --commit "+commit+" v1.2.3") { + t.Fatalf("prepared delivery was not invoked: %s (%v)", transcript, err) + } + }) + } +}