Remove unnecessary output restrictions and simplify report tests

This commit is contained in:
2026-09-25 00:36:44 +00:00
parent efaefdbf3d
commit 0b6cd98262
13 changed files with 330 additions and 105 deletions

View File

@@ -638,3 +638,64 @@ func TestSubprocessInteractionWriteFailureIsVisibleAndNonfatal(t *testing.T) {
t.Fatalf("debug failure classified as model incident: %+v", document) t.Fatalf("debug failure classified as model incident: %+v", document)
} }
} }
func TestSubprocessInPlaceTranscript(t *testing.T) {
for _, invalid := range []bool{false, true} {
t.Run(fmt.Sprint(invalid), func(t *testing.T) {
f := newProcessFixture(t, "success", "[grammar]")
f.output = f.transcript
if invalid {
if err := os.WriteFile(f.transcript, []byte("invalid"), 0600); err != nil {
t.Fatal(err)
}
}
before, err := os.ReadFile(f.transcript)
if err != nil {
t.Fatal(err)
}
code, stdout, stderr := f.run(t, true)
after, err := os.ReadFile(f.transcript)
if err != nil {
t.Fatal(err)
}
if invalid {
if code != 1 || !bytes.Equal(before, after) {
t.Fatalf("failed processing changed input: code=%d %s", code, stderr)
}
} else if code != 0 || stdout != "" || !bytes.Contains(after, []byte("Hello world")) {
t.Fatalf("in-place publication failed: code=%d %s %s", code, stderr, after)
}
})
}
}
func TestSubprocessProfileSources(t *testing.T) {
f := newProcessFixture(t, "success", "[]")
// PromptKit uses profile IDs, not filename stems. A report-named profile is
// still an input and must not be overwritten during report initialization.
loaded := filepath.Join(f.dir, "profiles", "errors.yml")
if err := os.Rename(filepath.Join(f.dir, "profiles", "proposal.yaml"), loaded); err != nil {
t.Fatal(err)
}
before, err := os.ReadFile(loaded)
if err != nil {
t.Fatal(err)
}
f.output = filepath.Join(f.dir, "profiles", "proposal.yml")
code, _, stderr := f.run(t, true, "--reporting=false")
if code != 0 {
t.Fatalf("unused alternate rejected: %d %s", code, stderr)
}
// Remove the transcript from the profile directory before the next load.
if err := os.Remove(f.output); err != nil {
t.Fatal(err)
}
code, _, stderr = f.run(t, true)
after, err := os.ReadFile(loaded)
if err != nil {
t.Fatal(err)
}
if code != 2 || !bytes.Equal(before, after) || f.provider.callCount() != 0 {
t.Fatalf("loaded profile not protected: %d %s", code, stderr)
}
}

View File

@@ -74,7 +74,7 @@ matrices. Rows show every initial chunk with 1-based chunk number and initial
segment range; columns follow configured module-instance order, including segment range; columns follow configured module-instance order, including
repeats. Completed error cells are `0` or `1`; recovery cells are operation repeats. Completed error cells are `0` or `1`; recovery cells are operation
counts. `—` means not started and `n*` means interrupted with n facts recorded. counts. `—` means not started and `n*` means interrupted with n facts recorded.
An empty stage or chunk set says “No work units.” Short reason breakdowns count An empty stage or chunk set is identified as having no work units. Short reason breakdowns count
records containing each reason; a recovery can have several reasons, so those records containing each reason; a recovery can have several reasons, so those
counts need not sum to the recovery total. The report refers to both YAML counts need not sum to the recovery total. The report refers to both YAML
files. It does not include transcript excerpts or correction lists. files. It does not include transcript excerpts or correction lists.

View File

@@ -17,7 +17,11 @@ and boolean values retain their types. Ordinary reports omit raw causes,
transcript text, complete decisions, and token/timing payloads. transcript text, complete decisions, and token/timing payloads.
`internal/core/diagnostics.OutputSession` owns active destination checks, `internal/core/diagnostics.OutputSession` owns active destination checks,
transcript preflight, report initialization, and per-file final replacement. transcript preflight, report initialization, regular-file replacement, and direct
stream publication. Transcript input is protected from early report writes but
may be the final transcript destination. The LLM runtime records successful reads
through PromptKit's profile filesystem and supplies those source paths to CLI;
no second filename-based discovery rule is used.
CLI composes it after configuration/profile inspection and before reading CLI composes it after configuration/profile inspection and before reading
inputs or dispatching model work. It does not create ordinary report parents. inputs or dispatching model work. It does not create ordinary report parents.
On a final write error it attempts other enabled reports and one failed-status On a final write error it attempts other enabled reports and one failed-status

View File

@@ -23,13 +23,22 @@ overridden by `--report-dir`. `--reporting=false` suppresses all three reports
and removes their path requirements. It does not suppress the stderr summary or and removes their path requirements. It does not suppress the stderr summary or
fatal errors. Stdout contains only transcript JSON; file output leaves it empty. fatal errors. Stdout contains only transcript JSON; file output leaves it empty.
Existing report directories and transcript-output parents must be writable. Existing report directories and parents for regular transcript files must be
Audita checks active destinations against input, configuration, and selected writable. Audita prevents report writes from overwriting transcript, glossary,
profile files before model dispatch. It preflights file transcript output loaded configuration, or loaded profile sources, and rejects conflicting output
without truncating an existing file. With reporting enabled, all three reports destinations. The final transcript may intentionally replace its own input after
are initialized as `in_progress` before input processing. Output is encoded successful processing; other input/configuration/profile sources remain protected.
fully before publication. Each file is replaced through a temporary file on Preparation never truncates an existing transcript. Device and pipe destinations
its destination filesystem: transcript first, then `errors.yml`, (such as `/dev/null` and named pipes) receive ordinary writes at publication time;
they are not opened during preflight or replaced with regular files. A pipe write
can wait for its reader, and stream write errors are fatal. With reporting enabled,
reports still use the output path's parent; disable reporting when that placement
is unsuitable for a stream destination.
With reporting enabled, all three reports are initialized as `in_progress` before
input processing. Output is encoded fully before publication. Each regular output
file is replaced through a temporary file on its destination filesystem:
transcript first, then `errors.yml`,
`recoveries.yml`, and `report.md`. This is per-file replacement, not a `recoveries.yml`, and `report.md`. This is per-file replacement, not a
multi-file transaction. New files honor the process umask; replacing an existing multi-file transaction. New files honor the process umask; replacing an existing
file preserves its permission bits. A late report error may leave the transcript file preserves its permission bits. A late report error may leave the transcript

View File

@@ -5,6 +5,12 @@
**Status:** Implemented. All stages below are complete; completion notes record **Status:** Implemented. All stages below are complete; completion notes record
the checks performed for each unit of work. the checks performed for each unit of work.
**Post-implementation revisions:** [Remediation R11–R14](remediation.md) supersedes
the original blanket transcript/input collision prohibition, inferred profile
filenames, regular-file-only transcript handling, and whole-document Markdown
golden requirements below. These stages and completion notes are historical;
[operations](../operations.md) describes current output behavior.
## Scope and Execution Instructions ## Scope and Execution Instructions
Implement the target in [resilience.md](resilience.md). User intent and product Implement the target in [resilience.md](resilience.md). User intent and product

View File

@@ -5,8 +5,8 @@
invariants, contracts, guarantees, and test requirements, compared with invariants, contracts, guarantees, and test requirements, compared with
[resilience.md](resilience.md), [implementation.md](implementation.md), and the [resilience.md](resilience.md), [implementation.md](implementation.md), and the
[agent policy](../policy/agents.md). [agent policy](../policy/agents.md).
**Status:** R11–R14 remain open. This document records required follow-up work; **Status:** R11–R14 addressed. The findings below describe the reviewed source;
no application behavior has changed as part of this review. see the resolution record for the implemented changes and verification.
The earlier findings R1–R10 have been addressed. Their findings and resolution The earlier findings R1–R10 have been addressed. Their findings and resolution
records remain in Git history; R6–R10 were resolved in `ec78b5a`. records remain in Git history; R6–R10 were resolved in `ec78b5a`.
@@ -134,7 +134,7 @@ Harmless prose or whitespace edits outside the required layout no longer require
updating a whole-document fixture and overlapping assertions. Retain meaningful updating a whole-document fixture and overlapping assertions. Retain meaningful
agreement checks between YAML, Markdown, and CLI totals. agreement checks between YAML, Markdown, and CLI totals.
## Verification and Review Limits ## Original Review Verification and Limits
- This was a source and contract review; no application changes or new runtime - This was a source and contract review; no application changes or new runtime
reproductions were made during this pass. reproductions were made during this pass.
@@ -148,3 +148,40 @@ agreement checks between YAML, Markdown, and CLI totals.
subsequent edit solely because they were used there. subsequent edit solely because they were used there.
- Application fixes, upstream changes, deferred feature work, and release - Application fixes, upstream changes, deferred feature work, and release
publication remain outside this document update. publication remain outside this document update.
## Resolution Record
- **R11:** Output selection distinguishes the transcript input from other protected
sources. Final transcript publication may replace its input; early report writes
still cannot overwrite it, and other source/output collisions remain checked.
`TestSubprocessInPlaceTranscript` proves corrected output replaces the source on
success and an invalid source remains untouched on failure. Output-session tests
cover direct and alias acceptance, nontruncating preparation, and rejection of
report/input aliases and conflicting output destinations.
- **R12:** The runtime records successful reads through PromptKit's existing profile
filesystem and supplies actual loaded paths to CLI. It no longer guesses two
filenames per profile ID. `TestSubprocessProfileSources` proves an unused
alternate filename is allowed and a loaded profile with a different filename
remains protected from report initialization. Configuration sources continue to
come from ConfigKit's source records.
- **R13:** Regular transcript files retain per-file replacement. Existing device
and pipe destinations receive ordinary writes during finalization and are not
opened during preflight. `TestTranscriptStreamDestinations` verifies `/dev/null`
and named-pipe delivery without replacing either destination, including pipe
preparation before opening its reader. `TestTranscriptStreamWriteFailure`
checks a real stream write failure using `/dev/full` where available. Existing
directory, parent, permission, cleanup, and late-report-failure tests remain.
- **R14:** Removed the whole-document Markdown golden fixture and overlapping
matrix assertions. Tests compare complete ordered table rows, summary quantities,
reason counts, escaping/redaction, and YAML/CLI count agreement. Summary prose is
not snapshotted, and empty-work handling no longer requires an exact sentence.
No renderer or output schema changed.
Current operations and internal documentation describe the revised boundaries.
The completed implementation plan explicitly identifies its superseded choices.
Validation: `go test ./...` and `go vet ./...` passed, as did focused race checks
for the LLM runtime, diagnostics/output, and CLI subprocess packages. The refined
Markdown assertions passed their package tests. Changed-document links and
`git diff --check` passed. No live provider, dependency change, tag, or publication
was involved.

View File

@@ -8,7 +8,6 @@ import (
"errors" "errors"
"fmt" "fmt"
"io" "io"
"path/filepath"
"strings" "strings"
"sync" "sync"
"time" "time"
@@ -82,14 +81,14 @@ func runProcess(ctx context.Context, args []string, stdout, stderr io.Writer) in
state := &processState{cfg: cfg, runtime: runtime, specs: specs, runID: runID, started: started, state := &processState{cfg: cfg, runtime: runtime, specs: specs, runID: runID, started: started,
transcriptPath: input.TranscriptPath, glossaryPath: input.GlossaryPath, outputPath: input.OutputPath, transcriptPath: input.TranscriptPath, glossaryPath: input.GlossaryPath, outputPath: input.OutputPath,
selected: selected, overlay: input.Overlay, stderr: stderr, stdout: stdout, ctx: ctx} selected: selected, overlay: input.Overlay, stderr: stderr, stdout: stdout, ctx: ctx}
protected := []string{input.TranscriptPath, input.GlossaryPath} protected := []string{input.GlossaryPath}
for _, source := range selected.Result.Sources() { for _, source := range selected.Result.Sources() {
if source.Path != "" { if source.Path != "" {
protected = append(protected, source.Path) protected = append(protected, source.Path)
} }
} }
protected = append(protected, selectedProfilePaths(cfg, selected.Path)...) protected = append(protected, runtime.ProfilePaths...)
session, err := diagnostics.PrepareOutput(diagnostics.OutputSelection{TranscriptPath: input.OutputPath, session, err := diagnostics.PrepareOutput(diagnostics.OutputSelection{InputTranscriptPath: input.TranscriptPath, TranscriptPath: input.OutputPath,
ReportDir: reportDir, ReportingEnabled: cfg.Reporting.Enabled, ProtectedPaths: protected}) ReportDir: reportDir, ReportingEnabled: cfg.Reporting.Enabled, ProtectedPaths: protected})
if err != nil { if err != nil {
fmt.Fprintf(stderr, "audita process: %s\n", llm.RedactError(err, cfg)) fmt.Fprintf(stderr, "audita process: %s\n", llm.RedactError(err, cfg))
@@ -145,19 +144,6 @@ func generateRunID() (string, error) {
return "run-" + hex.EncodeToString(random[:]), nil return "run-" + hex.EncodeToString(random[:]), nil
} }
func selectedProfilePaths(cfg config.Config, configPath string) []string {
dir := cfg.ProfilesDir
if !filepath.IsAbs(dir) {
dir = filepath.Join(filepath.Dir(configPath), dir)
}
ids := []string{cfg.LLM.ProposalProfile, cfg.LLM.ValidationProfile}
paths := make([]string, 0, len(ids)*2)
for _, id := range ids {
paths = append(paths, filepath.Join(dir, id+".yaml"), filepath.Join(dir, id+".yml"))
}
return paths
}
func (s *processState) snapshot(status string, completed *time.Time, failures []processreport.RunFailure) reporting.Snapshot { func (s *processState) snapshot(status string, completed *time.Time, failures []processreport.RunFailure) reporting.Snapshot {
return processreport.Build(processreport.BuildInput{RunID: s.runID, Status: status, StartedAt: s.started, return processreport.Build(processreport.BuildInput{RunID: s.runID, Status: status, StartedAt: s.started,
CompletedAt: completed, TranscriptPath: s.transcriptPath, GlossaryPath: s.glossaryPath, CompletedAt: completed, TranscriptPath: s.transcriptPath, GlossaryPath: s.glossaryPath,

View File

@@ -16,7 +16,9 @@ const (
// OutputSelection contains only active destinations. ProtectedPaths are inputs // OutputSelection contains only active destinations. ProtectedPaths are inputs
// and loaded configuration/profile sources that must never be overwritten. // and loaded configuration/profile sources that must never be overwritten.
// InputTranscriptPath may be replaced by final transcript publication, but not reports.
type OutputSelection struct { type OutputSelection struct {
InputTranscriptPath string
TranscriptPath string TranscriptPath string
ReportDir string ReportDir string
ReportingEnabled bool ReportingEnabled bool
@@ -88,7 +90,11 @@ func PrepareOutput(selection OutputSelection) (*OutputSession, error) {
return nil, fmt.Errorf("report output %q: %w", path, err) return nil, fmt.Errorf("report output %q: %w", path, err)
} }
} }
for _, source := range protected { sources := protected
if path != session.transcript && selection.InputTranscriptPath != "" {
sources = append(append([]string(nil), protected...), selection.InputTranscriptPath)
}
for _, source := range sources {
collision, err := sameDestination(path, source) collision, err := sameDestination(path, source)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -157,7 +163,7 @@ func (s *OutputSession) Finalize(transcript []byte, reports ReportBundle, failed
} }
failures := make([]FileFailure, 0) failures := make([]FileFailure, 0)
if transcript != nil && s.transcript != "" { if transcript != nil && s.transcript != "" {
if err := s.replace(s.transcript, transcript); err != nil { if err := s.writeTranscript(transcript); err != nil {
failures = append(failures, FileFailure{Path: s.transcript, Kind: "transcript write", Err: err}) failures = append(failures, FileFailure{Path: s.transcript, Kind: "transcript write", Err: err})
} }
} }
@@ -176,6 +182,24 @@ func (s *OutputSession) Finalize(transcript []byte, reports ReportBundle, failed
return failures return failures
} }
// Stream destinations use ordinary writes; replacing their directory entry would
// destroy the device or pipe. Regular files retain per-file replacement.
func (s *OutputSession) writeTranscript(payload []byte) error {
info, err := os.Stat(s.transcript)
if err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
if err == nil && !info.Mode().IsRegular() {
file, err := os.OpenFile(s.transcript, os.O_WRONLY, 0)
if err != nil {
return err
}
_, writeErr := file.Write(payload)
return errors.Join(writeErr, file.Close())
}
return s.replace(s.transcript, payload)
}
func bundleParts(bundle ReportBundle) [3][]byte { func bundleParts(bundle ReportBundle) [3][]byte {
return [3][]byte{bundle.Errors, bundle.Recoveries, bundle.Markdown} return [3][]byte{bundle.Errors, bundle.Recoveries, bundle.Markdown}
} }
@@ -183,8 +207,12 @@ func bundleParts(bundle ReportBundle) [3][]byte {
func preflightTranscript(path string) error { func preflightTranscript(path string) error {
info, err := os.Stat(path) info, err := os.Stat(path)
if err == nil { if err == nil {
if info.IsDir() {
return fmt.Errorf("transcript output %q is a directory", path)
}
if !info.Mode().IsRegular() { if !info.Mode().IsRegular() {
return fmt.Errorf("transcript output %q is not a regular file", path) // Opening a pipe here can block before its reader is ready.
return nil
} }
file, openErr := os.OpenFile(path, os.O_WRONLY, 0) file, openErr := os.OpenFile(path, os.O_WRONLY, 0)
if openErr != nil { if openErr != nil {

View File

@@ -8,7 +8,7 @@ import (
"testing" "testing"
) )
func TestPrepareOutputRejectsDirectAndAliasCollisions(t *testing.T) { func TestPrepareOutputAllowsInPlaceTranscriptButRejectsReportCollisions(t *testing.T) {
dir := t.TempDir() dir := t.TempDir()
input := filepath.Join(dir, "input.json") input := filepath.Join(dir, "input.json")
if err := os.WriteFile(input, []byte("source"), 0600); err != nil { if err := os.WriteFile(input, []byte("source"), 0600); err != nil {
@@ -23,8 +23,8 @@ func TestPrepareOutputRejectsDirectAndAliasCollisions(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
for _, output := range []string{input, alias, hardlink} { for _, output := range []string{input, alias, hardlink} {
if _, err := PrepareOutput(OutputSelection{TranscriptPath: output, ProtectedPaths: []string{input}}); err == nil { if _, err := PrepareOutput(OutputSelection{TranscriptPath: output, InputTranscriptPath: input}); err != nil {
t.Fatalf("collision accepted: %s", output) t.Fatalf("in-place output rejected: %s: %v", output, err)
} }
} }
if _, err := PrepareOutput(OutputSelection{TranscriptPath: filepath.Join(dir, ErrorsFile), ReportDir: dir, ReportingEnabled: true}); err == nil { if _, err := PrepareOutput(OutputSelection{TranscriptPath: filepath.Join(dir, ErrorsFile), ReportDir: dir, ReportingEnabled: true}); err == nil {
@@ -33,6 +33,15 @@ func TestPrepareOutputRejectsDirectAndAliasCollisions(t *testing.T) {
if _, err := PrepareOutput(OutputSelection{ReportDir: dir, ReportingEnabled: true, ProtectedPaths: []string{filepath.Join(dir, ErrorsFile)}}); err == nil { if _, err := PrepareOutput(OutputSelection{ReportDir: dir, ReportingEnabled: true, ProtectedPaths: []string{filepath.Join(dir, ErrorsFile)}}); err == nil {
t.Fatal("report/source collision accepted") t.Fatal("report/source collision accepted")
} }
if err := os.Symlink(input, filepath.Join(dir, ErrorsFile)); err != nil {
t.Fatal(err)
}
if _, err := PrepareOutput(OutputSelection{ReportDir: dir, ReportingEnabled: true, InputTranscriptPath: input}); err == nil {
t.Fatal("report/input alias accepted")
}
if err := os.Remove(filepath.Join(dir, ErrorsFile)); err != nil {
t.Fatal(err)
}
if err := os.Mkdir(filepath.Join(dir, MarkdownFile), 0700); err != nil { if err := os.Mkdir(filepath.Join(dir, MarkdownFile), 0700); err != nil {
t.Fatal(err) t.Fatal(err)
} }

View File

@@ -0,0 +1,69 @@
//go:build linux || darwin
package diagnostics
import (
"io"
"os"
"path/filepath"
"syscall"
"testing"
)
func TestTranscriptStreamDestinations(t *testing.T) {
fifo := filepath.Join(t.TempDir(), "transcript.pipe")
if err := syscall.Mkfifo(fifo, 0600); err != nil {
t.Fatal(err)
}
for _, path := range []string{os.DevNull, fifo} {
t.Run(filepath.Base(path), func(t *testing.T) {
before, err := os.Stat(path)
if err != nil {
t.Fatal(err)
}
// Preparation must finish even though the pipe has no reader yet.
session, err := PrepareOutput(OutputSelection{TranscriptPath: path})
if err != nil {
t.Fatal(err)
}
var reader *os.File
if path == fifo {
// Keep both ends open in the test so publication and reading can be
// sequential; the small payload fits in the pipe buffer.
reader, err = os.OpenFile(path, os.O_RDWR|syscall.O_NONBLOCK, 0)
if err != nil {
t.Fatal(err)
}
defer reader.Close()
}
payload := []byte("transcript")
if failures := session.Finalize(payload, ReportBundle{}, nil); len(failures) != 0 {
t.Fatal(failures)
}
after, err := os.Stat(path)
if err != nil || !os.SameFile(before, after) || before.Mode() != after.Mode() {
t.Fatalf("stream replaced: %v", err)
}
if reader != nil {
got := make([]byte, len(payload))
if _, err := io.ReadFull(reader, got); err != nil || string(got) != string(payload) {
t.Fatalf("pipe delivery: %q %v", got, err)
}
}
})
}
}
func TestTranscriptStreamWriteFailure(t *testing.T) {
if _, err := os.Stat("/dev/full"); os.IsNotExist(err) {
t.Skip("/dev/full unavailable")
}
session, err := PrepareOutput(OutputSelection{TranscriptPath: "/dev/full"})
if err != nil {
t.Fatal(err)
}
failures := session.Finalize([]byte("transcript"), ReportBundle{}, nil)
if len(failures) != 1 || failures[0].Kind != "transcript write" {
t.Fatalf("write failure lost: %+v", failures)
}
}

View File

@@ -1,9 +1,8 @@
package reporting package reporting
import ( import (
"bytes" "reflect"
"os" "regexp"
"path/filepath"
"strings" "strings"
"testing" "testing"
"time" "time"
@@ -30,25 +29,46 @@ func TestMarkdownFixedLayoutAndCounts(t *testing.T) {
Recoveries: []RecoveryRecord{{Reasons: []string{"output_repaired", "proposals_filtered"}}, {Reasons: []string{"output_repaired"}}, {Reasons: []string{"service_recovered"}}}, Recoveries: []RecoveryRecord{{Reasons: []string{"output_repaired", "proposals_filtered"}}, {Reasons: []string{"output_repaired"}}, {Reasons: []string{"service_recovered"}}},
} }
got := Markdown(snapshot, "secret") got := Markdown(snapshot, "secret")
fixture := filepath.Join("testdata", "report.md")
if os.Getenv("UPDATE_REPORT_GOLDEN") == "1" {
if err := os.WriteFile(fixture, got, 0600); err != nil {
t.Fatal(err)
}
}
want, err := os.ReadFile(fixture)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got, want) {
t.Fatalf("Markdown differs from %s:\n%s", fixture, got)
}
text := string(got) text := string(got)
for _, fragment := range []string{"| grammar_1 | grammar_2 |", "| Chunk 1 (segments 1–2) | 1 | 0* |", "| Chunk 1 (segments 1–2) | 1 | 2* |", "| Chunk 2 (segments 3–4) | 0 | — |", "in\\|put.json", "out<br>file.json", "glossary\\`[REDACTED]\\`.yaml", "- `transport_failed`: 1", "- `output_repaired`: 2"} { // Compare only complete table rows, preserving order and states while leaving
// prose and spacing outside the fixed matrices free to change.
wantRows := []string{
"| Chunk | grammar_1 | grammar_2 |",
"| --- | ---: | ---: |",
"| Chunk 1 (segments 1–2) | 1 | 0* |",
"| Chunk 2 (segments 3–4) | 0 | — |",
"| Chunk | grammar_1 | grammar_2 |",
"| --- | ---: | ---: |",
"| Chunk 1 (segments 1–2) | 1 | 2* |",
"| Chunk 2 (segments 3–4) | 0 | — |",
}
if rows := markdownTableRows(text); !reflect.DeepEqual(rows, wantRows) {
t.Fatalf("matrices: got %v; want %v", rows, wantRows)
}
for _, fragment := range []string{"in\\|put.json", "out<br>file.json", "glossary\\`[REDACTED]\\`.yaml", "`transport_failed`: 1", "`output_repaired`: 2", "`proposals_filtered`: 1", "`service_recovered`: 1"} {
if !strings.Contains(text, fragment) { if !strings.Contains(text, fragment) {
t.Fatalf("missing %q in:\n%s", fragment, text) t.Fatalf("missing %q in:\n%s", fragment, text)
} }
} }
// Assert summary quantities independently of the surrounding explanation.
numbers := regexp.MustCompile(`[0-9]+`)
for label, want := range map[string][]string{
"Corrections applied": {"1"},
"Errors": {"1", "0"},
"Recoveries": {"3", "4"},
} {
var got []string
for _, line := range strings.Split(text, "\n") {
if value, ok := strings.CutPrefix(strings.TrimSpace(line), label+":"); ok {
got = numbers.FindAllString(value, -1)
break
}
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("%s summary: got %v; want %v", label, got, want)
}
}
if strings.Contains(text, "secret") { if strings.Contains(text, "secret") {
t.Fatal("secret leaked") t.Fatal("secret leaked")
} }
@@ -69,7 +89,7 @@ func TestMarkdownFixedLayoutAndCounts(t *testing.T) {
func TestMarkdownNoWorkAndEscapedStageLabel(t *testing.T) { func TestMarkdownNoWorkAndEscapedStageLabel(t *testing.T) {
text := string(Markdown(Snapshot{RunID: "r", StageNames: []string{"bad|label`\nsecret"}}, "secret")) text := string(Markdown(Snapshot{RunID: "r", StageNames: []string{"bad|label`\nsecret"}}, "secret"))
if strings.Count(text, "No work units.") != 2 || strings.Contains(text, "secret") { if len(markdownTableRows(text)) != 0 || strings.Contains(text, "secret") {
t.Fatal(text) t.Fatal(text)
} }
text = string(Markdown(Snapshot{RunID: "r", StageNames: []string{"bad|label`\nsecret"}, WorkUnits: []WorkUnit{{StageIndex: 0, ChunkIndex: 0, State: "completed"}}}, "secret")) text = string(Markdown(Snapshot{RunID: "r", StageNames: []string{"bad|label`\nsecret"}, WorkUnits: []WorkUnit{{StageIndex: 0, ChunkIndex: 0, State: "completed"}}}, "secret"))
@@ -77,3 +97,13 @@ func TestMarkdownNoWorkAndEscapedStageLabel(t *testing.T) {
t.Fatal(text) t.Fatal(text)
} }
} }
func markdownTableRows(text string) []string {
var rows []string
for _, line := range strings.Split(text, "\n") {
if line = strings.TrimSpace(line); strings.HasPrefix(line, "|") {
rows = append(rows, line)
}
}
return rows
}

View File

@@ -1,43 +0,0 @@
# Audita report: run-123
Status: completed
Corrections applied: 1
Work units: 4 total; 2 completed; 1 interrupted; 1 not started
Errors: 1 work units; 0 run errors
Recoveries: 3 of 4 model operations
Proposal responses: 1; with filtering: 1
Transcript input: in\|put.json
Glossary input: glossary\`[REDACTED]\`.yaml
Transcript output: out<br>file.json
## Work-unit errors
| Chunk | grammar_1 | grammar_2 |
| --- | ---: | ---: |
| Chunk 1 (segments 1–2) | 1 | 0* |
| Chunk 2 (segments 3–4) | 0 | — |
## Recoveries
| Chunk | grammar_1 | grammar_2 |
| --- | ---: | ---: |
| Chunk 1 (segments 1–2) | 1 | 2* |
| Chunk 2 (segments 3–4) | 0 | — |
Legend: `—` = not started; `n*` = interrupted with n recorded errors or recoveries. Completed error cells are 0 or 1.
### Error reasons
- `transport_failed`: 1
### Recovery reasons
- `output_repaired`: 2
- `proposals_filtered`: 1
- `service_recovered`: 1
A recovery can have multiple reasons; reason counts need not sum to the recovery total.
Details: `errors.yml` and `recoveries.yml`.

View File

@@ -4,11 +4,13 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"io/fs"
"net/http" "net/http"
"os" "os"
"path/filepath" "path/filepath"
"sort" "sort"
"strings" "strings"
"sync"
"sync/atomic" "sync/atomic"
"gitea.maximumdirect.net/eric/audita/internal/core/config" "gitea.maximumdirect.net/eric/audita/internal/core/config"
@@ -24,6 +26,8 @@ type Runtime struct {
Config config.Config Config config.Config
Engine *pk.Engine Engine *pk.Engine
Profiles map[string]pk.ProfileInspection Profiles map[string]pk.ProfileInspection
// ProfilePaths contains filesystem sources actually read by PromptKit.
ProfilePaths []string
nextCallID atomic.Uint64 nextCallID atomic.Uint64
} }
@@ -39,10 +43,11 @@ func NewRuntime(ctx context.Context, c config.Config, mainPath string, client *h
if !filepath.IsAbs(profileDir) { if !filepath.IsAbs(profileDir) {
profileDir = filepath.Join(filepath.Dir(mainPath), profileDir) profileDir = filepath.Join(filepath.Dir(mainPath), profileDir)
} }
profileFS := &profileSourceFS{FS: os.DirFS(profileDir), paths: make(map[string]struct{})}
options := []pk.Option{ options := []pk.Option{
pk.WithPromptFS(prompts.Filesystem(), prompts.AssetRoot), pk.WithPromptFS(prompts.Filesystem(), prompts.AssetRoot),
pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot), pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot),
pk.WithProfileFS(os.DirFS(profileDir), "."), pk.WithProfileFS(profileFS, "."),
} }
ids := make([]string, 0, len(c.Backends)) ids := make([]string, 0, len(c.Backends))
for id := range c.Backends { for id := range c.Backends {
@@ -85,6 +90,12 @@ func NewRuntime(ctx context.Context, c config.Config, mainPath string, client *h
} }
runtime.Profiles[selection.role] = *profile runtime.Profiles[selection.role] = *profile
} }
profileFS.mu.Lock()
for name := range profileFS.paths {
runtime.ProfilePaths = append(runtime.ProfilePaths, filepath.Join(profileDir, filepath.FromSlash(name)))
}
profileFS.mu.Unlock()
sort.Strings(runtime.ProfilePaths)
return runtime, nil return runtime, nil
} }
@@ -98,3 +109,21 @@ func RedactError(err error, c config.Config) string {
} }
return redaction.New(ConfiguredSecrets(&c)).String(err.Error()) return redaction.New(ConfiguredSecrets(&c)).String(err.Error())
} }
// Record successful profile reads at the library boundary, including inherited
// and differently named sources, without duplicating PromptKit's discovery rules.
type profileSourceFS struct {
fs.FS
mu sync.Mutex
paths map[string]struct{}
}
func (f *profileSourceFS) ReadFile(name string) ([]byte, error) {
data, err := fs.ReadFile(f.FS, name)
if err == nil {
f.mu.Lock()
f.paths[name] = struct{}{}
f.mu.Unlock()
}
return data, err
}