Fix the five resilience review findings

This commit is contained in:
2026-09-24 21:48:18 +00:00
parent 9fc4de22b7
commit ec78b5a5db
19 changed files with 550 additions and 132 deletions

View File

@@ -11,6 +11,7 @@ import (
"os/exec"
"os/signal"
"path/filepath"
"strconv"
"strings"
"sync"
"syscall"
@@ -28,6 +29,13 @@ func TestProcessSubprocessHelper(t *testing.T) {
}
for i, arg := range os.Args {
if arg == "--" {
if value := os.Getenv("AUDITA_TEST_UMASK"); value != "" {
mask, err := strconv.ParseUint(value, 8, 32)
if err != nil {
os.Exit(2)
}
syscall.Umask(int(mask))
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
code := cli.Run(ctx, os.Args[i+1:], os.Stdout, os.Stderr)
stop()
@@ -45,6 +53,7 @@ type fixtureProvider struct {
mode string
secret string
entered chan struct{}
beforeResponse func()
}
func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
@@ -71,7 +80,11 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
}
validationCall := p.validationCalls
proposalCall := p.proposalCalls
beforeResponse := p.beforeResponse
p.mu.Unlock()
if beforeResponse != nil {
beforeResponse()
}
if p.mode == "wait-cancel" {
select {
case p.entered <- struct{}{}:
@@ -166,6 +179,7 @@ func (f processFixture) run(t *testing.T, includeOutput bool, extra ...string) (
}
args = append(args, extra...)
command := exec.Command(os.Args[0], args...)
command.Dir = f.dir
command.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1", "AUDITA_TEST_PROVIDER_KEY="+f.provider.secret)
var stdout, stderr bytes.Buffer
command.Stdout, command.Stderr = &stdout, &stderr
@@ -486,3 +500,141 @@ func TestSubprocessTranscriptPreflightWithoutReporting(t *testing.T) {
t.Fatalf("transcript preflight: code=%d calls=%d stderr=%q", code, f.provider.callCount(), stderr)
}
}
func TestSubprocessOutputPermissionsHonorUmask(t *testing.T) {
for _, mask := range []string{"0077", "0022"} {
t.Run(mask, func(t *testing.T) {
t.Setenv("AUDITA_TEST_UMASK", mask)
f := newProcessFixture(t, "normal", "[]")
paths := []string{f.output, filepath.Join(f.dir, "errors.yml"), filepath.Join(f.dir, "recoveries.yml"), filepath.Join(f.dir, "report.md")}
parsed, _ := strconv.ParseUint(mask, 8, 32)
want := os.FileMode(0644 &^ parsed)
for attempt := 0; attempt < 2; attempt++ {
code, _, stderr := f.run(t, true)
if code != 0 {
t.Fatalf("run: %d %s", code, stderr)
}
for _, path := range paths {
info, err := os.Stat(path)
if err != nil {
t.Fatal(err)
}
if info.Mode().Perm() != want {
t.Errorf("%s mode=%o want=%o", filepath.Base(path), info.Mode().Perm(), want)
}
if err := os.Chmod(path, 0600); err != nil {
t.Fatal(err)
}
}
want = 0600 // Existing restrictive modes survive replacement under either mask.
}
})
}
}
func TestSubprocessWhitespaceOutputIsALiteralFile(t *testing.T) {
f := newProcessFixture(t, "normal", "[]")
f.output = " "
code, stdout, stderr := f.run(t, true)
if code != 0 || stdout != "" {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
output, err := os.ReadFile(filepath.Join(f.dir, " "))
if err != nil || !bytes.Contains(output, []byte("hello world")) {
t.Fatalf("missing transcript: %s %v", output, err)
}
if f.errors(t).Status != reporting.StatusCompleted {
t.Fatal("reports did not follow file output")
}
}
func TestSubprocessPathAnnouncementsAreRedacted(t *testing.T) {
for _, fail := range []bool{false, true} {
t.Run(fmt.Sprint(fail), func(t *testing.T) {
f := newProcessFixture(t, "normal", "[]")
selected, unused := "selected-path-secret", "unused-path-secret"
f.provider.secret = selected
t.Setenv("AUDITA_UNUSED_PATH_KEY", unused)
cfg, err := os.ReadFile(f.config)
if err != nil {
t.Fatal(err)
}
cfg = bytes.Replace(cfg, []byte("backends:\n"), []byte("backends:\n unused:\n endpoint: http://unused.invalid/v1\n credential: AUDITA_UNUSED_PATH_KEY\n"), 1)
if err := os.WriteFile(f.config, cfg, 0600); err != nil {
t.Fatal(err)
}
reportDir := filepath.Join(f.dir, unused)
if err := os.Mkdir(reportDir, 0700); err != nil {
t.Fatal(err)
}
f.output = filepath.Join(reportDir, "out.json")
debugParent := filepath.Join(f.dir, selected)
if fail {
if err := os.WriteFile(f.transcript, []byte("invalid"), 0600); err != nil {
t.Fatal(err)
}
}
code, _, stderr := f.run(t, true, "--debug", "--work-dir", debugParent)
if (!fail && code != 0) || (fail && code != 1) {
t.Fatalf("code=%d stderr=%q", code, stderr)
}
if strings.Contains(stderr, selected) || strings.Contains(stderr, unused) {
t.Fatalf("credential in stderr: %q", stderr)
}
if !strings.Contains(stderr, "[REDACTED]") {
t.Fatalf("missing redacted announcement: %q", stderr)
}
if _, err := os.Stat(filepath.Join(reportDir, "report.md")); err != nil {
t.Fatal(err)
}
dirs, err := os.ReadDir(debugParent)
if err != nil || len(dirs) != 1 {
t.Fatalf("debug location: %v %v", dirs, err)
}
if !fail {
if _, err := os.Stat(f.output); err != nil {
t.Fatal(err)
}
}
})
}
}
func TestSubprocessInteractionWriteFailureIsVisibleAndNonfatal(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
code, _, baselineSummary := f.run(t, true)
if code != 0 {
t.Fatal(baselineSummary)
}
baseline, err := os.ReadFile(f.output)
if err != nil {
t.Fatal(err)
}
f.provider.mu.Lock()
f.provider.beforeResponse = func() {
dirs, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*"))
if err != nil || len(dirs) != 1 {
t.Errorf("debug directory: %v %v", dirs, err)
return
}
if err := os.WriteFile(filepath.Join(dirs[0], "grammar"), []byte("block interaction writes"), 0600); err != nil {
t.Error(err)
}
}
f.provider.mu.Unlock()
code, _, stderr := f.run(t, true, "--debug")
if code != 0 || !strings.HasPrefix(stderr, strings.SplitN(baselineSummary, "\n", 2)[0]) {
t.Fatalf("changed outcome: %d %q", code, stderr)
}
if !strings.Contains(stderr, "debug proposal interaction") || !strings.Contains(stderr, "debug validation interaction") {
t.Fatalf("missing debug failures: %q", stderr)
}
output, err := os.ReadFile(f.output)
if err != nil || !bytes.Equal(output, baseline) {
t.Fatalf("changed transcript: %s %v", output, err)
}
document := f.errors(t)
if len(document.Errors) != 0 || *document.Summary.Recoveries != 0 {
t.Fatalf("debug failure classified as model incident: %+v", document)
}
}

View File

@@ -45,6 +45,8 @@ validation. The shared setting flags are:
| Debug | `--debug[=true\|false]`, `--work-dir DIR` |
`process` alone accepts `--glossary` (required) and `--output`.
An omitted or empty output value selects stdout. Nonempty output filenames are
used literally, including whitespace.
Reporting defaults to enabled. `--reporting=false` disables all three ordinary
reports; an explicit false overrides a true file or overlay setting. With file
output, reports go beside the transcript and `--report-dir` has no effect. With

View File

@@ -29,7 +29,10 @@ when debug is off. With debug on, source/normalized snapshots, invocation,
effective settings, origins, profiles, initial chunk summary, correction ledger,
indexed execution/utilization, and model interactions are written. Runner
candidate capture includes proposals later rejected or interrupted. Debug
write failures produce safe stderr messages and do not abort processing.
write failures reach the CLI through an explicit concurrency-safe diagnostic
callback and produce redacted stderr messages without aborting processing.
Completed validator decisions returned with a fatal batch error remain available
to the debug ledger; they never authorize applying an incompletely validated edit.
Every debug writer uses all configured backend secrets for redaction, including
unselected backend credentials and source snapshots. Debug can still contain
sensitive transcript and prompt content.

View File

@@ -14,9 +14,10 @@ formats; [configuration](../config.md) owns settings; the
4. Module instances execute in configured order. Sections may propose
concurrently, but proposal indices follow section order and response order.
5. Deterministic validators precede LLM-backed validators. Every submitted
candidate receives one decision per validator. Recoverable failed batches
mark their candidates unavailable; successful survivors continue through
all required validators.
candidate receives one decision per validator on a normal return. Fatal returns
may preserve a checked subset of completed decisions as diagnostic evidence.
Recoverable failed batches mark their candidates unavailable; successful
survivors continue through all required validators.
6. Only fully approved proposals reach application. Corrections apply in
proposal-index order within their owning section. Existing replacement
policies and apply-time safety checks resolve stale conflicts.

View File

@@ -28,11 +28,12 @@ Audita checks active destinations against input, configuration, and selected
profile files before model dispatch. It preflights file transcript output
without truncating an existing file. With reporting enabled, all three reports
are initialized as `in_progress` before input processing. Output is encoded
fully before publication. Each file is replaced through a temporary file in
its destination directory: transcript first, then `errors.yml`,
fully before publication. Each file is replaced through a temporary file on
its destination filesystem: transcript first, then `errors.yml`,
`recoveries.yml`, and `report.md`. This is per-file replacement, not a
multi-file transaction. A late report error may leave the transcript published;
the process exits nonzero and attempts failed-status reports. On fatal processing
multi-file transaction. New files honor the process umask; replacing an existing
file preserves its permission bits. A late report error may leave the transcript
published; the process exits nonzero and attempts failed-status reports. On fatal processing
or cancellation, no completed transcript is published, but partial reports are
finalized where possible. An uncatchable termination can leave `in_progress`.

View File

@@ -4,7 +4,8 @@
**Scope:** The fourteen implementation commits after `8ffbf41`, compared with
[implementation.md](implementation.md), [resilience.md](resilience.md), and the
current architecture and output contracts.
**Status:** Five open findings, R6–R10. This review implements no fixes.
**Status:** R6–R10 addressed and verified. The findings below describe the reviewed
source; their line references are historical. See the resolution record below.
The earlier library-adoption findings R1–R5 were resolved before this work.
Their findings and resolution evidence remain in the
@@ -192,8 +193,49 @@ one assembled CLI case is sufficient. Debug-off must still create no artifacts.
- A temporary runner probe reproduced R9; its expected evidence-preservation
assertion failed. The probe and all temporary artifacts were removed.
Passing retained tests do not cover the five failures above. Add proportionate
regression coverage with their fixes. This review changes only this document;
At review time, the retained tests did not cover the five failures above.
The initial review changed only this document;
application fixes, upstream changes, deferred feature work, and release publication
remain outside scope. Target-platform release builds were not repeated for this
review; the native temporary CLI build succeeded.
## Resolution Record
All five findings are addressed without changing retry budgets, library ownership,
correction acceptance, or the independent reporting/debug selections.
- **R6:** Per-file replacement uses ordinary file creation inside a private
temporary directory in the destination parent. Creation applies the umask;
existing permission bits are restored only for existing destinations. Temporary
files/directories are removed on success and failure. The subprocess permission
test covers both restrictive and ordinary masks and subsequent replacement of
restrictive files; existing output-session failure/cleanup tests still pass.
- **R7:** File selection uses the literal nonempty output path throughout report
placement and output preparation. Final CLI stream selection uses the prepared
output session. `TestSubprocessWhitespaceOutputIsALiteralFile` proves the output
and reports are produced; the existing stdout integration test remains passing.
- **R8:** Every CLI report/debug location announcement uses the shared configured-
secret string redactor. `TestSubprocessPathAnnouncementsAreRedacted` covers
success and failure with selected and unselected credentials in real paths,
while confirming that the real destinations are used.
- **R9:** Fatal validator returns use partial identity/cardinality validation,
retain completed decisions and rejection details, and never return application
approvals. `TestFatalBatchRetainsEarlierDecisionsInLedger` exercises the real
runner and batching validator with two completed batches and a third fatal
failure, retaining their decisions and all operation facts in the debug ledger.
A separate runner regression rejects unknown/duplicate partial identities while
retaining the originating failure. Existing cancellation/joining tests pass.
- **R10:** An explicit diagnostic-error callback connects proposal and validator
writers to the CLI's synchronized, redacted nonfatal diagnostics. Origin/profile
serialization failures use the same path. Narrow proposal/validator tests and
`TestSubprocessInteractionWriteFailureIsVisibleAndNonfatal` verify visible
failures, unchanged transcript/counts/exit status, and no fabricated artifact
paths. Existing reporting/debug-selection tests verify debug-off persistence
remains disabled.
Validation completed: full `go test ./...`, `go vet ./...`, focused race tests for
runner, validators, proposal generation, report assembly, diagnostics/output, CLI,
and subprocess integration; required Linux amd64/arm64 builds in a temporary
output directory; and documentation link/whitespace checks. No live provider,
upstream change, tag, or publication was involved.

View File

@@ -212,7 +212,7 @@ func selectedReportDir(cfg config.Config, outputPath string) (string, error) {
if !cfg.Reporting.Enabled {
return "", nil
}
if strings.TrimSpace(outputPath) != "" {
if outputPath != "" {
return filepath.Dir(outputPath), nil
}
if strings.TrimSpace(cfg.Reporting.StdoutDir) == "" {

View File

@@ -10,6 +10,7 @@ import (
"io"
"path/filepath"
"strings"
"sync"
"time"
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
@@ -18,6 +19,7 @@ import (
coreio "gitea.maximumdirect.net/eric/audita/internal/core/io"
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
"gitea.maximumdirect.net/eric/audita/internal/core/outputschema"
"gitea.maximumdirect.net/eric/audita/internal/core/redaction"
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
@@ -131,6 +133,7 @@ type processState struct {
session *diagnostics.OutputSession
debugDir *diagnostics.RunDirectory
debugWarnings []string
debugMu sync.Mutex
runOutput *runner.RunOutput
}
@@ -183,7 +186,7 @@ func (s *processState) debugWrite(label string, write func(*diagnostics.RunDirec
return
}
if err := write(s.debugDir); err != nil {
s.debugWarnings = append(s.debugWarnings, "debug "+label+": "+llm.RedactError(err, s.cfg))
s.reportDiagnosticError(fmt.Errorf("%s: %w", label, err))
}
}
@@ -193,7 +196,7 @@ func (s *processState) startDebug() {
}
dir, err := diagnostics.NewRunDirectory(s.cfg.Diagnostics.WorkDir, s.runID, llm.ConfiguredSecrets(&s.cfg)...)
if err != nil {
s.debugWarnings = append(s.debugWarnings, "debug directory: "+llm.RedactError(err, s.cfg))
s.reportDiagnosticError(fmt.Errorf("directory: %w", err))
return
}
s.debugDir = dir
@@ -209,17 +212,21 @@ func (s *processState) startDebug() {
if err := s.selected.Result.WriteEffective(&effective); err == nil {
s.debugWrite("effective settings", func(d *diagnostics.RunDirectory) error { return d.WriteEffectiveConfig(effective.Bytes()) })
} else {
s.debugWarnings = append(s.debugWarnings, "debug effective settings: "+llm.RedactError(err, s.cfg))
s.reportDiagnosticError(fmt.Errorf("effective settings: %w", err))
}
if origins, err := llm.ExplanationYAML(s.selected.Result); err == nil {
s.debugWrite("configuration origins", func(d *diagnostics.RunDirectory) error {
return d.WriteYAMLArtifact(diagnostics.ArtifactConfigOrigins, origins)
})
} else {
s.reportDiagnosticError(fmt.Errorf("configuration origins: %w", err))
}
if profiles, err := s.runtime.SelectedProfilesYAML(); err == nil {
s.debugWrite("selected profiles", func(d *diagnostics.RunDirectory) error {
return d.WriteYAMLArtifact(diagnostics.ArtifactSelectedProfiles, profiles)
})
} else {
s.reportDiagnosticError(fmt.Errorf("selected profiles: %w", err))
}
}
@@ -272,7 +279,8 @@ func (s *processState) execute() int {
Config: &s.cfg, Transcript: normalized, Glossary: glossary, ModuleSpecs: s.specs,
ProposalLLMClient: proposalClient, ProposalDiagnosticsDir: s.debugPath(),
ValidationLLMClient: validationClient, ValidationDiagnosticsDir: s.debugPath(),
CaptureCandidates: s.debugDir != nil,
CaptureCandidates: s.debugDir != nil,
ReportDiagnosticError: s.reportDiagnosticError,
})
result.Observations = observations.Snapshot()
result.ObservationsAvailable = true
@@ -362,7 +370,7 @@ func (s *processState) fail(phase string, reason outcomes.ReasonCode, cause erro
s.printFailures(writeFailures)
s.printDebugWarnings()
if s.debugDir != nil {
fmt.Fprintf(s.stderr, "audita process: diagnostics: %s\n", s.debugDir.Path())
fmt.Fprintf(s.stderr, "audita process: diagnostics: %s\n", s.safePath(s.debugDir.Path()))
}
return 1
}
@@ -374,7 +382,7 @@ func (s *processState) finish(outputBytes []byte) int {
if err != nil {
return s.fail("report_write", outcomes.ReasonInternalFailure, err)
}
if s.outputPath == "" {
if s.session.TranscriptPath() == "" {
if _, err := s.stdout.Write(outputBytes); err != nil {
return s.fail("transcript_write", outcomes.ReasonInternalFailure, err)
}
@@ -391,16 +399,16 @@ func (s *processState) finish(outputBytes []byte) int {
s.printFailures(writeFailures)
s.printDebugWarnings()
if s.debugDir != nil {
fmt.Fprintf(s.stderr, "audita process: diagnostics: %s\n", s.debugDir.Path())
fmt.Fprintf(s.stderr, "audita process: diagnostics: %s\n", s.safePath(s.debugDir.Path()))
}
return 1
}
fmt.Fprintln(s.stderr, reporting.CompletionLine(snapshot))
for _, path := range s.session.ReportPaths() {
fmt.Fprintf(s.stderr, "Report: %s\n", path)
fmt.Fprintf(s.stderr, "Report: %s\n", s.safePath(path))
}
if s.debugDir != nil {
fmt.Fprintf(s.stderr, "Debug: %s\n", s.debugDir.Path())
fmt.Fprintf(s.stderr, "Debug: %s\n", s.safePath(s.debugDir.Path()))
}
s.printDebugWarnings()
return 0
@@ -425,7 +433,24 @@ func (s *processState) printFailures(failures []diagnostics.FileFailure) {
}
func (s *processState) printDebugWarnings() {
s.debugMu.Lock()
defer s.debugMu.Unlock()
for _, warning := range s.debugWarnings {
fmt.Fprintf(s.stderr, "audita process: %s\n", warning)
}
}
// safePath sanitizes presentation only; filesystem destinations remain literal.
func (s *processState) safePath(path string) string {
return redaction.New(llm.ConfiguredSecrets(&s.cfg)).String(path)
}
func (s *processState) reportDiagnosticError(err error) {
if err == nil {
return
}
message := "debug " + llm.RedactError(err, s.cfg)
s.debugMu.Lock()
defer s.debugMu.Unlock()
s.debugWarnings = append(s.debugWarnings, message)
}

View File

@@ -50,7 +50,7 @@ type OutputSession struct {
func PrepareOutput(selection OutputSelection) (*OutputSession, error) {
session := &OutputSession{enabled: selection.ReportingEnabled, replace: atomicReplace}
active := make([]string, 0, 4)
if strings.TrimSpace(selection.TranscriptPath) != "" {
if selection.TranscriptPath != "" {
session.transcript = selection.TranscriptPath
active = append(active, session.transcript)
}
@@ -74,7 +74,7 @@ func PrepareOutput(selection OutputSelection) (*OutputSession, error) {
}
protected := make([]string, 0, len(selection.ProtectedPaths))
for _, path := range selection.ProtectedPaths {
if strings.TrimSpace(path) != "" {
if path != "" {
protected = append(protected, path)
}
}
@@ -251,18 +251,26 @@ func atomicReplace(path string, payload []byte) error {
if err != nil {
return err
}
parent := filepath.Dir(publishPath)
temp, err := os.CreateTemp(parent, ".audita-write-*")
// A private temporary directory lets ordinary file creation apply the umask
// without reading or changing process-global state. Rename stays on the same
// filesystem, and the incomplete payload is not accessible through the parent.
dir, err := os.MkdirTemp(filepath.Dir(publishPath), ".audita-write-*")
if err != nil {
return err
}
name := temp.Name()
defer os.Remove(name)
mode := os.FileMode(0644)
if info, err := os.Stat(publishPath); err == nil {
mode = info.Mode().Perm()
defer os.Remove(dir)
name := filepath.Join(dir, "payload")
temp, err := os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644)
if err != nil {
return err
}
if err := temp.Chmod(mode); err != nil {
defer os.Remove(name)
if info, err := os.Stat(publishPath); err == nil {
if err := temp.Chmod(info.Mode().Perm()); err != nil {
_ = temp.Close()
return err
}
} else if !errors.Is(err, os.ErrNotExist) {
_ = temp.Close()
return err
}

View File

@@ -61,11 +61,13 @@ func SectionMetadataFromSection(section chunking.Section) SectionMetadata {
// ExecutionContext carries shared execution state for proposal generation and validation.
type ExecutionContext struct {
Config *config.Config `json:"-"`
WorkingTranscript *schema.Transcript `json:"-"`
Glossary *schema.Glossary `json:"-"`
Section *SectionMetadata `json:"section,omitempty"`
DiagnosticsDir string `json:"diagnostics_dir,omitempty"`
// ReportDiagnosticError may be called concurrently; it must not affect processing.
ReportDiagnosticError func(error) `json:"-"`
Config *config.Config `json:"-"`
WorkingTranscript *schema.Transcript `json:"-"`
Glossary *schema.Glossary `json:"-"`
Section *SectionMetadata `json:"section,omitempty"`
DiagnosticsDir string `json:"diagnostics_dir,omitempty"`
}
// ProposalRequest is the input to module proposal generation.

View File

@@ -0,0 +1,99 @@
package processreport
import (
"context"
"errors"
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
)
type partialModule struct{ validator contracts.Validator }
func (m partialModule) Key() string { return "grammar" }
func (m partialModule) ReplacementPolicy() proposals.ReplacementPolicy {
return proposals.ReplacementPolicyRequireUnique
}
func (m partialModule) Validators() []contracts.Validator { return []contracts.Validator{m.validator} }
func (m partialModule) ModuleForSpec(contracts.ModuleRunSpec) (contracts.TranscriptModule, error) {
return m, nil
}
func (m partialModule) Propose(context.Context, contracts.ProposalRequest) (contracts.ProposalResult, error) {
result := contracts.ProposalResult{}
for _, text := range []string{"Hello", "Greetings", "Hi"} {
result.Proposals = append(result.Proposals, proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "hello", CorrectedText: text, Confidence: 1})
}
return result, nil
}
type oneItemEstimator struct{}
func (oneItemEstimator) EstimateTokens(payload string) int {
return 100 * strings.Count(payload, `"correction_index"`)
}
type interruptedValidationClient struct {
cause error
calls int
}
func (c *interruptedValidationClient) CompleteStructured(_ context.Context, req structuredoutput.Request, target any) (structuredoutput.Response, error) {
c.calls++
if *req.Context.BatchIndex == 2 {
return structuredoutput.Response{}, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "validation interrupted", Cause: c.cause}
}
index := *req.Context.BatchIndex
*target.(*validators.LLMValidationResponse) = validators.LLMValidationResponse{Validations: []validators.LLMValidationDecision{{CorrectionIndex: index, Approved: index == 1, Confidence: 1, Reason: "completed assessment"}}}
return structuredoutput.Response{}, nil
}
func TestFatalBatchRetainsEarlierDecisionsInLedger(t *testing.T) {
validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
if err != nil {
t.Fatal(err)
}
validator.SetTokenEstimator(oneItemEstimator{})
origin := errors.New("later batch failed")
client := &interruptedValidationClient{cause: origin}
cfg := config.Config{Chunking: config.Chunking{MaxSectionTokens: 8192, MinSectionTokens: 1}, Validation: config.Validation{MaxPromptTokens: 150}}
result, err := runner.New(partialModule{validator}).Run(context.Background(), runner.RunInput{
Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}}, ValidationLLMClient: client, CaptureCandidates: true,
})
if !errors.Is(err, origin) || client.calls != 3 {
t.Fatalf("fatal cause/calls: %v %d", err, client.calls)
}
if result.FinalTranscript.Segments[0].Text != "hello" || len(result.ModuleResults[0].AppliedChanges) != 0 {
t.Fatal("fatal partial approvals changed transcript")
}
if len(result.WorkUnits[0].Operations) != 3 || result.WorkUnits[0].State != outcomes.WorkInterrupted {
t.Fatalf("partial facts: %+v", result.WorkUnits)
}
ledger := BuildCorrectionLedger(CorrectionLedgerInput{RunOutput: &result})
candidates := make(map[int]CorrectionLedgerEntry)
for _, entry := range ledger {
if entry.Disposition != correctionDispositionFailed {
candidates[entry.ProposalIndex] = entry
}
}
if len(candidates) != 3 {
t.Fatalf("candidate evidence: %+v", ledger)
}
for index := 0; index < 2; index++ {
entry := candidates[index]
if len(entry.LLMValidatorResults) != 1 || entry.LLMValidatorResults[0].Message != "completed assessment" || entry.LLMValidatorResults[0].Approved != (index == 1) {
t.Errorf("lost batch %d decision: %+v", index, entry)
}
}
if candidates[0].Disposition != correctionDispositionRejected || candidates[1].Disposition != correctionDispositionInterrupted || candidates[2].Disposition != correctionDispositionInterrupted {
t.Fatalf("wrong partial dispositions: %+v", ledger)
}
}

View File

@@ -50,21 +50,23 @@ type StructuredCorrectionSet struct {
// Request captures reusable proposal-generation inputs for future modules.
type Request struct {
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
WorkingTranscript *schema.Transcript `json:"-"`
Section *contracts.SectionMetadata `json:"section,omitempty"`
Glossary *schema.Glossary `json:"-"`
Config *config.Config `json:"-"`
PromptID string `json:"prompt_id"`
PromptVersion string `json:"prompt_version"`
Variables map[string]string `json:"variables"`
StageName string `json:"stage_name,omitempty"`
StartIndex int `json:"start_index"`
LLMClient contracts.StructuredLLMClient
DiagnosticsDir string
DiagnosticsWriter InteractionDiagnosticsWriter
// ReportDiagnosticError may be called concurrently; it must not affect processing.
ReportDiagnosticError func(error) `json:"-"`
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
WorkingTranscript *schema.Transcript `json:"-"`
Section *contracts.SectionMetadata `json:"section,omitempty"`
Glossary *schema.Glossary `json:"-"`
Config *config.Config `json:"-"`
PromptID string `json:"prompt_id"`
PromptVersion string `json:"prompt_version"`
Variables map[string]string `json:"variables"`
StageName string `json:"stage_name,omitempty"`
StartIndex int `json:"start_index"`
LLMClient contracts.StructuredLLMClient
DiagnosticsDir string
DiagnosticsWriter InteractionDiagnosticsWriter
}
// Result contains generated candidate proposals and optional diagnostics paths.
@@ -145,13 +147,17 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
requestMetadata["response_schema"] = responseschema.MustLookup(responseschema.CorrectionSetKey).DiagnosticsMap()
if writer != nil {
artifacts, _ = writer.WriteInteraction(
var diagnosticErr error
artifacts, diagnosticErr = writer.WriteInteraction(
stage,
requestMetadata,
completion.Prepared,
map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model},
errPayload(callErr),
)
if diagnosticErr != nil && req.ReportDiagnosticError != nil {
req.ReportDiagnosticError(fmt.Errorf("proposal interaction %s: %w", stage, diagnosticErr))
}
}
if callErr != nil {

View File

@@ -51,6 +51,7 @@ func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contr
}
type captureDiagnosticsWriter struct {
err error
lastStage string
lastRequestMetadata any
lastRequestPayload any
@@ -62,7 +63,7 @@ func (w *captureDiagnosticsWriter) WriteInteraction(stage string, requestMetadat
w.lastRequestPayload = requestPayload
_ = responsePayload
_ = errorPayload
return InteractionArtifacts{}, nil
return InteractionArtifacts{}, w.err
}
func defaultRequest(t *testing.T) Request {
@@ -383,3 +384,16 @@ func TestGenerateCandidatesClientError(t *testing.T) {
t.Fatalf("expected error diagnostics artifact under %s", req.DiagnosticsDir)
}
}
func TestProposalDiagnosticFailureDoesNotChangeOperation(t *testing.T) {
cause := errors.New("diagnostic storage unavailable")
req := defaultRequest(t)
req.LLMClient = &fakeStructuredClient{responses: []StructuredCorrectionSet{{}}}
req.DiagnosticsWriter = &captureDiagnosticsWriter{err: cause}
var reported error
req.ReportDiagnosticError = func(err error) { reported = err }
result, err := GenerateCandidates(context.Background(), req)
if err != nil || !errors.Is(reported, cause) || len(result.Operations) != 1 || !result.Operations[0].Succeeded || len(result.Operations[0].RecoveryReasons) != 0 || result.Artifacts.ResponsePayloadPath != "" {
t.Fatalf("debug failure changed outcome: %+v %v reported=%v", result, err, reported)
}
}

View File

@@ -55,20 +55,21 @@ func ExecuteModuleProposal(ctx context.Context, req ModuleProposalRequest) (cont
}
generated, err := GenerateCandidates(ctx, Request{
ModuleKey: req.ProposalRequest.RunSpec.ModuleKey,
ModuleInstance: req.ProposalRequest.RunSpec.InstanceName,
ReplacementPolicy: req.ProposalRequest.RunSpec.ReplacementPolicy,
WorkingTranscript: req.ProposalRequest.WorkingTranscript,
Section: req.ProposalRequest.Section,
Glossary: req.ProposalRequest.Glossary,
Config: req.ProposalRequest.Config,
PromptID: promptMetadata.PromptID,
PromptVersion: promptMetadata.PromptVersion,
Variables: variables,
StageName: stagename.ModuleProposal(req.ProposalRequest.RunSpec.InstanceName, sectionIndexPtr(req.ProposalRequest.Section)),
StartIndex: 0,
LLMClient: req.ProposalRequest.LLMClient,
DiagnosticsDir: req.ProposalRequest.DiagnosticsDir,
ModuleKey: req.ProposalRequest.RunSpec.ModuleKey,
ModuleInstance: req.ProposalRequest.RunSpec.InstanceName,
ReplacementPolicy: req.ProposalRequest.RunSpec.ReplacementPolicy,
WorkingTranscript: req.ProposalRequest.WorkingTranscript,
Section: req.ProposalRequest.Section,
Glossary: req.ProposalRequest.Glossary,
Config: req.ProposalRequest.Config,
PromptID: promptMetadata.PromptID,
PromptVersion: promptMetadata.PromptVersion,
Variables: variables,
StageName: stagename.ModuleProposal(req.ProposalRequest.RunSpec.InstanceName, sectionIndexPtr(req.ProposalRequest.Section)),
StartIndex: 0,
LLMClient: req.ProposalRequest.LLMClient,
DiagnosticsDir: req.ProposalRequest.DiagnosticsDir,
ReportDiagnosticError: req.ProposalRequest.ReportDiagnosticError,
})
if err != nil {
return contracts.ProposalResult{Operations: generated.Operations}, err

View File

@@ -2,6 +2,7 @@ package runner
import (
"context"
"errors"
"fmt"
"path/filepath"
"sort"
@@ -86,6 +87,8 @@ type ValidatorRejectedChange struct {
// RunInput is the deterministic runner input.
type RunInput struct {
// ReportDiagnosticError may be called concurrently; it must not affect processing.
ReportDiagnosticError func(error) `json:"-"`
Config *config.Config
Transcript *schema.Transcript
Glossary *schema.Glossary
@@ -188,6 +191,7 @@ func (r *Runner) run(ctx context.Context, input RunInput) (RunOutput, error) {
ValidationClient: input.ValidationLLMClient,
ValidationDiagnosticsDir: input.ValidationDiagnosticsDir,
Timings: input.timings,
ReportDiagnosticError: input.ReportDiagnosticError,
})
if input.CaptureCandidates {
candidates = append(candidates, pipelineResult.Candidates...)
@@ -293,6 +297,8 @@ func applyWithinSections(sections []chunking.Section, approved []proposals.Enric
}
type collectSectionProposalsInput struct {
// ReportDiagnosticError may be called concurrently; it must not affect processing.
ReportDiagnosticError func(error) `json:"-"`
Module contracts.TranscriptModule
Spec contracts.ModuleRunSpec
Policy proposals.ReplacementPolicy
@@ -356,11 +362,12 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
meta := contracts.SectionMetadataFromSection(section)
proposalResult, err := input.Module.Propose(runCtx, contracts.ProposalRequest{
ExecutionContext: contracts.ExecutionContext{
Config: input.Config,
WorkingTranscript: transcriptFromSection(section),
Glossary: input.Glossary,
Section: &meta,
DiagnosticsDir: input.DiagnosticsDir,
Config: input.Config,
WorkingTranscript: transcriptFromSection(section),
Glossary: input.Glossary,
Section: &meta,
DiagnosticsDir: input.DiagnosticsDir,
ReportDiagnosticError: input.ReportDiagnosticError,
},
RunSpec: contracts.ModuleRunSpec{
ModuleKey: input.Spec.ModuleKey,
@@ -495,17 +502,18 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
go func(sectionPos int, enriched []proposals.EnrichedCorrectionProposal, sectionMetadata contracts.SectionMetadata) {
defer vwg.Done()
validated, err := validateSectionCandidates(runCtx, validateSectionCandidatesInput{
Spec: input.Spec,
Policy: input.Policy,
Glossary: input.Glossary,
Config: input.Config,
WorkingTranscript: transcriptFromSection(sectionResult.section),
SectionIndex: &sectionMetadata.Index,
Validators: validatorsOrdered,
SectionEnriched: enriched,
ValidationLLMClient: input.ValidationClient,
DiagnosticsDir: input.ValidationDiagnosticsDir,
Timings: input.Timings,
Spec: input.Spec,
Policy: input.Policy,
Glossary: input.Glossary,
Config: input.Config,
WorkingTranscript: transcriptFromSection(sectionResult.section),
SectionIndex: &sectionMetadata.Index,
Validators: validatorsOrdered,
SectionEnriched: enriched,
ValidationLLMClient: input.ValidationClient,
DiagnosticsDir: input.ValidationDiagnosticsDir,
Timings: input.Timings,
ReportDiagnosticError: input.ReportDiagnosticError,
})
setErr(err)
validationResults <- sectionValidationResult{
@@ -578,17 +586,19 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
}
type validateSectionCandidatesInput struct {
Spec contracts.ModuleRunSpec
Policy proposals.ReplacementPolicy
Glossary *schema.Glossary
Config *config.Config
WorkingTranscript *schema.Transcript
SectionIndex *int
Validators []contracts.Validator
SectionEnriched []proposals.EnrichedCorrectionProposal
ValidationLLMClient contracts.StructuredLLMClient
DiagnosticsDir string
Timings *validatorTimingCollector
// ReportDiagnosticError may be called concurrently; it must not affect processing.
ReportDiagnosticError func(error) `json:"-"`
Spec contracts.ModuleRunSpec
Policy proposals.ReplacementPolicy
Glossary *schema.Glossary
Config *config.Config
WorkingTranscript *schema.Transcript
SectionIndex *int
Validators []contracts.Validator
SectionEnriched []proposals.EnrichedCorrectionProposal
ValidationLLMClient contracts.StructuredLLMClient
DiagnosticsDir string
Timings *validatorTimingCollector
}
type validateSectionCandidatesResult struct {
@@ -622,38 +632,34 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
validatorStarted := time.Now()
vResult, err := validator.Validate(ctx, contracts.ValidationRequest{
WorkingTranscript: input.WorkingTranscript,
CandidateProposal: eligible,
ModuleKey: input.Spec.ModuleKey,
ModuleInstance: input.Spec.InstanceName,
SectionIndex: input.SectionIndex,
ValidatorOrder: &validatorOrder,
ReplacementPolicy: input.Policy,
Glossary: input.Glossary,
Config: input.Config,
LLMClient: input.ValidationLLMClient,
DiagnosticsWriter: diagnosticsWriter,
WorkingTranscript: input.WorkingTranscript,
CandidateProposal: eligible,
ModuleKey: input.Spec.ModuleKey,
ModuleInstance: input.Spec.InstanceName,
SectionIndex: input.SectionIndex,
ValidatorOrder: &validatorOrder,
ReplacementPolicy: input.Policy,
Glossary: input.Glossary,
Config: input.Config,
LLMClient: input.ValidationLLMClient,
DiagnosticsWriter: diagnosticsWriter,
ReportDiagnosticError: input.ReportDiagnosticError,
})
if input.Timings != nil {
input.Timings.record(input.Spec.ModuleKey, input.Spec.InstanceName, validator.Name(), validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked, time.Since(validatorStarted))
}
operations = append(operations, vResult.Operations...)
var decisionErr error
if err != nil {
return validateSectionCandidatesResult{
decisions: decisions,
rejected: rejected,
operations: operations,
filtered: filtered,
}, fmt.Errorf("validator %q failed: %w", validator.Name(), err)
decisionErr = validators.EnforcePartialDecisionCardinality(eligible, vResult.Decisions)
} else {
decisionErr = validators.EnforceDecisionCardinality(eligible, vResult.Decisions)
}
if err := validators.EnforceDecisionCardinality(eligible, vResult.Decisions); err != nil {
if decisionErr != nil {
return validateSectionCandidatesResult{
decisions: decisions,
rejected: rejected,
operations: operations,
filtered: filtered,
}, fmt.Errorf("validator %q cardinality failed: %w", validator.Name(), err)
decisions: decisions, rejected: rejected, operations: operations, filtered: filtered,
}, fmt.Errorf("validator %q cardinality failed: %w", validator.Name(), errors.Join(err, decisionErr))
}
nextEligible := make([]proposals.EnrichedCorrectionProposal, 0, len(eligible))
@@ -694,6 +700,13 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
})
}
if err != nil {
// Completed decisions are evidence only. A fatal return never grants
// approval to candidates that have not finished the full chain.
return validateSectionCandidatesResult{
decisions: decisions, rejected: rejected, operations: operations, filtered: filtered,
}, fmt.Errorf("validator %q failed: %w", validator.Name(), err)
}
eligible = nextEligible
}

View File

@@ -1280,3 +1280,20 @@ func TestRunnerGlossaryModuleUsesConfidenceThreshold(t *testing.T) {
t.Fatalf("expected low confidence reason, got %+v", out.ModuleResults[0].ValidatorRejected[0])
}
}
func TestFatalPartialDecisionsRejectUnknownAndDuplicateIdentity(t *testing.T) {
origin := errors.New("original fatal failure")
candidate := proposals.EnrichedCorrectionProposal{ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 7}}
for _, decisions := range [][]validators.Decision{
{{ProposalIndex: 8, Approved: true}},
{{ProposalIndex: 7, Approved: true}, {ProposalIndex: 7, Approved: true}},
} {
v := fakeValidator{name: "broken", validateF: func(contracts.ValidationRequest) (validators.Result, error) {
return validators.Result{Decisions: decisions}, origin
}}
result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{Validators: []contracts.Validator{v}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}})
if !errors.Is(err, origin) || len(result.decisions) != 0 || len(result.approved) != 0 {
t.Fatalf("invalid partial evidence accepted or origin lost: %+v %v", result, err)
}
}
}

View File

@@ -151,7 +151,8 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
artifacts := InteractionArtifacts{}
if req.DiagnosticsWriter != nil {
artifacts, _ = req.DiagnosticsWriter.WriteInteraction(
var diagnosticErr error
artifacts, diagnosticErr = req.DiagnosticsWriter.WriteInteraction(
stage,
map[string]any{
"validator_name": v.name,
@@ -164,6 +165,9 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model},
errPayload(err),
)
if diagnosticErr != nil && req.ReportDiagnosticError != nil {
req.ReportDiagnosticError(fmt.Errorf("validation interaction %s: %w", stage, diagnosticErr))
}
}
if err != nil {
var failure *structuredoutput.Failure

View File

@@ -23,6 +23,7 @@ type fakeStructuredLLMClient struct {
}
type captureValidationDiagnosticsWriter struct {
err error
lastRequestMetadata any
}
@@ -32,7 +33,7 @@ func (w *captureValidationDiagnosticsWriter) WriteInteraction(stage string, requ
_ = responsePayload
_ = errorPayload
w.lastRequestMetadata = requestMetadata
return InteractionArtifacts{}, nil
return InteractionArtifacts{}, w.err
}
func (f *fakeStructuredLLMClient) CompleteStructured(ctx context.Context, req StructuredCompletionRequest, out any) (StructuredCompletionResponse, error) {
@@ -321,3 +322,17 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) {
})
}
}
func TestValidationDiagnosticFailureDoesNotChangeDecision(t *testing.T) {
cause := errors.New("diagnostic storage unavailable")
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 1, Reason: "ok"}}}}}
req.DiagnosticsWriter = &captureValidationDiagnosticsWriter{err: cause}
var reported error
req.ReportDiagnosticError = func(err error) { reported = err }
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
result, err := validator.Validate(context.Background(), req)
if err != nil || !errors.Is(reported, cause) || len(result.Decisions) != 1 || !result.Decisions[0].Approved || result.Decisions[0].DiagnosticArtifactPath != "" || len(result.Operations) != 1 || !result.Operations[0].Succeeded || len(result.Operations[0].RecoveryReasons) != 0 {
t.Fatalf("debug failure changed outcome: %+v %v reported=%v", result, err, reported)
}
}

View File

@@ -29,17 +29,19 @@ const (
// Request is the runtime input shared by deterministic validators.
type Request struct {
SectionIndex *int `json:"section_index,omitempty"`
WorkingTranscript *schema.Transcript `json:"-"`
CandidateProposal []proposals.EnrichedCorrectionProposal `json:"candidate_proposals"`
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
ValidatorOrder *int `json:"validator_order,omitempty"`
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
Glossary *schema.Glossary `json:"-"`
Config *config.Config `json:"-"`
LLMClient StructuredLLMClient `json:"-"`
DiagnosticsWriter InteractionDiagnosticsWriter `json:"-"`
// ReportDiagnosticError may be called concurrently; it must not affect processing.
ReportDiagnosticError func(error) `json:"-"`
SectionIndex *int `json:"section_index,omitempty"`
WorkingTranscript *schema.Transcript `json:"-"`
CandidateProposal []proposals.EnrichedCorrectionProposal `json:"candidate_proposals"`
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
ValidatorOrder *int `json:"validator_order,omitempty"`
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
Glossary *schema.Glossary `json:"-"`
Config *config.Config `json:"-"`
LLMClient StructuredLLMClient `json:"-"`
DiagnosticsWriter InteractionDiagnosticsWriter `json:"-"`
}
// Decision is one validator decision for one proposal index.
@@ -62,7 +64,8 @@ const (
DispositionUnavailable DecisionDisposition = "unavailable"
)
// Result is one validator output containing exactly one decision per proposal index.
// Result contains one decision per proposal index on success. Fatal returns may
// include completed partial decisions as evidence, never as application approval.
type Result struct {
ValidatorName string `json:"validator_name"`
Decisions []Decision `json:"decisions"`
@@ -88,6 +91,16 @@ type Validator interface {
// EnforceDecisionCardinality verifies every candidate proposal index receives exactly one decision.
func EnforceDecisionCardinality(candidate []proposals.EnrichedCorrectionProposal, decisions []Decision) error {
return enforceDecisionCardinality(candidate, decisions, true)
}
// EnforcePartialDecisionCardinality accepts missing decisions on fatal returns,
// but still rejects duplicate and unknown proposal identities.
func EnforcePartialDecisionCardinality(candidate []proposals.EnrichedCorrectionProposal, decisions []Decision) error {
return enforceDecisionCardinality(candidate, decisions, false)
}
func enforceDecisionCardinality(candidate []proposals.EnrichedCorrectionProposal, decisions []Decision, requireAll bool) error {
expected := make(map[int]struct{}, len(candidate))
for _, c := range candidate {
expected[c.ProposalIndex] = struct{}{}
@@ -105,7 +118,7 @@ func EnforceDecisionCardinality(candidate []proposals.EnrichedCorrectionProposal
}
for idx := range expected {
if seen[idx] == 0 {
if requireAll && seen[idx] == 0 {
return fmt.Errorf("missing decision proposal index %d", idx)
}
}