Remove unnecessary output restrictions and simplify report tests
This commit is contained in:
@@ -638,3 +638,64 @@ func TestSubprocessInteractionWriteFailureIsVisibleAndNonfatal(t *testing.T) {
|
||||
t.Fatalf("debug failure classified as model incident: %+v", document)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessInPlaceTranscript(t *testing.T) {
|
||||
for _, invalid := range []bool{false, true} {
|
||||
t.Run(fmt.Sprint(invalid), func(t *testing.T) {
|
||||
f := newProcessFixture(t, "success", "[grammar]")
|
||||
f.output = f.transcript
|
||||
if invalid {
|
||||
if err := os.WriteFile(f.transcript, []byte("invalid"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
before, err := os.ReadFile(f.transcript)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
code, stdout, stderr := f.run(t, true)
|
||||
after, err := os.ReadFile(f.transcript)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if invalid {
|
||||
if code != 1 || !bytes.Equal(before, after) {
|
||||
t.Fatalf("failed processing changed input: code=%d %s", code, stderr)
|
||||
}
|
||||
} else if code != 0 || stdout != "" || !bytes.Contains(after, []byte("Hello world")) {
|
||||
t.Fatalf("in-place publication failed: code=%d %s %s", code, stderr, after)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessProfileSources(t *testing.T) {
|
||||
f := newProcessFixture(t, "success", "[]")
|
||||
// PromptKit uses profile IDs, not filename stems. A report-named profile is
|
||||
// still an input and must not be overwritten during report initialization.
|
||||
loaded := filepath.Join(f.dir, "profiles", "errors.yml")
|
||||
if err := os.Rename(filepath.Join(f.dir, "profiles", "proposal.yaml"), loaded); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before, err := os.ReadFile(loaded)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.output = filepath.Join(f.dir, "profiles", "proposal.yml")
|
||||
code, _, stderr := f.run(t, true, "--reporting=false")
|
||||
if code != 0 {
|
||||
t.Fatalf("unused alternate rejected: %d %s", code, stderr)
|
||||
}
|
||||
// Remove the transcript from the profile directory before the next load.
|
||||
if err := os.Remove(f.output); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
code, _, stderr = f.run(t, true)
|
||||
after, err := os.ReadFile(loaded)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if code != 2 || !bytes.Equal(before, after) || f.provider.callCount() != 0 {
|
||||
t.Fatalf("loaded profile not protected: %d %s", code, stderr)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -74,7 +74,7 @@ matrices. Rows show every initial chunk with 1-based chunk number and initial
|
||||
segment range; columns follow configured module-instance order, including
|
||||
repeats. Completed error cells are `0` or `1`; recovery cells are operation
|
||||
counts. `—` means not started and `n*` means interrupted with n facts recorded.
|
||||
An empty stage or chunk set says “No work units.” Short reason breakdowns count
|
||||
An empty stage or chunk set is identified as having no work units. Short reason breakdowns count
|
||||
records containing each reason; a recovery can have several reasons, so those
|
||||
counts need not sum to the recovery total. The report refers to both YAML
|
||||
files. It does not include transcript excerpts or correction lists.
|
||||
|
||||
@@ -17,7 +17,11 @@ and boolean values retain their types. Ordinary reports omit raw causes,
|
||||
transcript text, complete decisions, and token/timing payloads.
|
||||
|
||||
`internal/core/diagnostics.OutputSession` owns active destination checks,
|
||||
transcript preflight, report initialization, and per-file final replacement.
|
||||
transcript preflight, report initialization, regular-file replacement, and direct
|
||||
stream publication. Transcript input is protected from early report writes but
|
||||
may be the final transcript destination. The LLM runtime records successful reads
|
||||
through PromptKit's profile filesystem and supplies those source paths to CLI;
|
||||
no second filename-based discovery rule is used.
|
||||
CLI composes it after configuration/profile inspection and before reading
|
||||
inputs or dispatching model work. It does not create ordinary report parents.
|
||||
On a final write error it attempts other enabled reports and one failed-status
|
||||
|
||||
@@ -23,13 +23,22 @@ overridden by `--report-dir`. `--reporting=false` suppresses all three reports
|
||||
and removes their path requirements. It does not suppress the stderr summary or
|
||||
fatal errors. Stdout contains only transcript JSON; file output leaves it empty.
|
||||
|
||||
Existing report directories and transcript-output parents must be writable.
|
||||
Audita checks active destinations against input, configuration, and selected
|
||||
profile files before model dispatch. It preflights file transcript output
|
||||
without truncating an existing file. With reporting enabled, all three reports
|
||||
are initialized as `in_progress` before input processing. Output is encoded
|
||||
fully before publication. Each file is replaced through a temporary file on
|
||||
its destination filesystem: transcript first, then `errors.yml`,
|
||||
Existing report directories and parents for regular transcript files must be
|
||||
writable. Audita prevents report writes from overwriting transcript, glossary,
|
||||
loaded configuration, or loaded profile sources, and rejects conflicting output
|
||||
destinations. The final transcript may intentionally replace its own input after
|
||||
successful processing; other input/configuration/profile sources remain protected.
|
||||
Preparation never truncates an existing transcript. Device and pipe destinations
|
||||
(such as `/dev/null` and named pipes) receive ordinary writes at publication time;
|
||||
they are not opened during preflight or replaced with regular files. A pipe write
|
||||
can wait for its reader, and stream write errors are fatal. With reporting enabled,
|
||||
reports still use the output path's parent; disable reporting when that placement
|
||||
is unsuitable for a stream destination.
|
||||
|
||||
With reporting enabled, all three reports are initialized as `in_progress` before
|
||||
input processing. Output is encoded fully before publication. Each regular output
|
||||
file is replaced through a temporary file on its destination filesystem:
|
||||
transcript first, then `errors.yml`,
|
||||
`recoveries.yml`, and `report.md`. This is per-file replacement, not a
|
||||
multi-file transaction. New files honor the process umask; replacing an existing
|
||||
file preserves its permission bits. A late report error may leave the transcript
|
||||
|
||||
@@ -5,6 +5,12 @@
|
||||
**Status:** Implemented. All stages below are complete; completion notes record
|
||||
the checks performed for each unit of work.
|
||||
|
||||
**Post-implementation revisions:** [Remediation R11–R14](remediation.md) supersedes
|
||||
the original blanket transcript/input collision prohibition, inferred profile
|
||||
filenames, regular-file-only transcript handling, and whole-document Markdown
|
||||
golden requirements below. These stages and completion notes are historical;
|
||||
[operations](../operations.md) describes current output behavior.
|
||||
|
||||
## Scope and Execution Instructions
|
||||
|
||||
Implement the target in [resilience.md](resilience.md). User intent and product
|
||||
|
||||
@@ -5,8 +5,8 @@
|
||||
invariants, contracts, guarantees, and test requirements, compared with
|
||||
[resilience.md](resilience.md), [implementation.md](implementation.md), and the
|
||||
[agent policy](../policy/agents.md).
|
||||
**Status:** R11–R14 remain open. This document records required follow-up work;
|
||||
no application behavior has changed as part of this review.
|
||||
**Status:** R11–R14 addressed. The findings below describe the reviewed source;
|
||||
see the resolution record for the implemented changes and verification.
|
||||
|
||||
The earlier findings R1–R10 have been addressed. Their findings and resolution
|
||||
records remain in Git history; R6–R10 were resolved in `ec78b5a`.
|
||||
@@ -134,7 +134,7 @@ Harmless prose or whitespace edits outside the required layout no longer require
|
||||
updating a whole-document fixture and overlapping assertions. Retain meaningful
|
||||
agreement checks between YAML, Markdown, and CLI totals.
|
||||
|
||||
## Verification and Review Limits
|
||||
## Original Review Verification and Limits
|
||||
|
||||
- This was a source and contract review; no application changes or new runtime
|
||||
reproductions were made during this pass.
|
||||
@@ -148,3 +148,40 @@ agreement checks between YAML, Markdown, and CLI totals.
|
||||
subsequent edit solely because they were used there.
|
||||
- Application fixes, upstream changes, deferred feature work, and release
|
||||
publication remain outside this document update.
|
||||
|
||||
## Resolution Record
|
||||
|
||||
- **R11:** Output selection distinguishes the transcript input from other protected
|
||||
sources. Final transcript publication may replace its input; early report writes
|
||||
still cannot overwrite it, and other source/output collisions remain checked.
|
||||
`TestSubprocessInPlaceTranscript` proves corrected output replaces the source on
|
||||
success and an invalid source remains untouched on failure. Output-session tests
|
||||
cover direct and alias acceptance, nontruncating preparation, and rejection of
|
||||
report/input aliases and conflicting output destinations.
|
||||
- **R12:** The runtime records successful reads through PromptKit's existing profile
|
||||
filesystem and supplies actual loaded paths to CLI. It no longer guesses two
|
||||
filenames per profile ID. `TestSubprocessProfileSources` proves an unused
|
||||
alternate filename is allowed and a loaded profile with a different filename
|
||||
remains protected from report initialization. Configuration sources continue to
|
||||
come from ConfigKit's source records.
|
||||
- **R13:** Regular transcript files retain per-file replacement. Existing device
|
||||
and pipe destinations receive ordinary writes during finalization and are not
|
||||
opened during preflight. `TestTranscriptStreamDestinations` verifies `/dev/null`
|
||||
and named-pipe delivery without replacing either destination, including pipe
|
||||
preparation before opening its reader. `TestTranscriptStreamWriteFailure`
|
||||
checks a real stream write failure using `/dev/full` where available. Existing
|
||||
directory, parent, permission, cleanup, and late-report-failure tests remain.
|
||||
- **R14:** Removed the whole-document Markdown golden fixture and overlapping
|
||||
matrix assertions. Tests compare complete ordered table rows, summary quantities,
|
||||
reason counts, escaping/redaction, and YAML/CLI count agreement. Summary prose is
|
||||
not snapshotted, and empty-work handling no longer requires an exact sentence.
|
||||
No renderer or output schema changed.
|
||||
|
||||
Current operations and internal documentation describe the revised boundaries.
|
||||
The completed implementation plan explicitly identifies its superseded choices.
|
||||
|
||||
Validation: `go test ./...` and `go vet ./...` passed, as did focused race checks
|
||||
for the LLM runtime, diagnostics/output, and CLI subprocess packages. The refined
|
||||
Markdown assertions passed their package tests. Changed-document links and
|
||||
`git diff --check` passed. No live provider, dependency change, tag, or publication
|
||||
was involved.
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -82,14 +81,14 @@ func runProcess(ctx context.Context, args []string, stdout, stderr io.Writer) in
|
||||
state := &processState{cfg: cfg, runtime: runtime, specs: specs, runID: runID, started: started,
|
||||
transcriptPath: input.TranscriptPath, glossaryPath: input.GlossaryPath, outputPath: input.OutputPath,
|
||||
selected: selected, overlay: input.Overlay, stderr: stderr, stdout: stdout, ctx: ctx}
|
||||
protected := []string{input.TranscriptPath, input.GlossaryPath}
|
||||
protected := []string{input.GlossaryPath}
|
||||
for _, source := range selected.Result.Sources() {
|
||||
if source.Path != "" {
|
||||
protected = append(protected, source.Path)
|
||||
}
|
||||
}
|
||||
protected = append(protected, selectedProfilePaths(cfg, selected.Path)...)
|
||||
session, err := diagnostics.PrepareOutput(diagnostics.OutputSelection{TranscriptPath: input.OutputPath,
|
||||
protected = append(protected, runtime.ProfilePaths...)
|
||||
session, err := diagnostics.PrepareOutput(diagnostics.OutputSelection{InputTranscriptPath: input.TranscriptPath, TranscriptPath: input.OutputPath,
|
||||
ReportDir: reportDir, ReportingEnabled: cfg.Reporting.Enabled, ProtectedPaths: protected})
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "audita process: %s\n", llm.RedactError(err, cfg))
|
||||
@@ -145,19 +144,6 @@ func generateRunID() (string, error) {
|
||||
return "run-" + hex.EncodeToString(random[:]), nil
|
||||
}
|
||||
|
||||
func selectedProfilePaths(cfg config.Config, configPath string) []string {
|
||||
dir := cfg.ProfilesDir
|
||||
if !filepath.IsAbs(dir) {
|
||||
dir = filepath.Join(filepath.Dir(configPath), dir)
|
||||
}
|
||||
ids := []string{cfg.LLM.ProposalProfile, cfg.LLM.ValidationProfile}
|
||||
paths := make([]string, 0, len(ids)*2)
|
||||
for _, id := range ids {
|
||||
paths = append(paths, filepath.Join(dir, id+".yaml"), filepath.Join(dir, id+".yml"))
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
func (s *processState) snapshot(status string, completed *time.Time, failures []processreport.RunFailure) reporting.Snapshot {
|
||||
return processreport.Build(processreport.BuildInput{RunID: s.runID, Status: status, StartedAt: s.started,
|
||||
CompletedAt: completed, TranscriptPath: s.transcriptPath, GlossaryPath: s.glossaryPath,
|
||||
|
||||
@@ -16,11 +16,13 @@ const (
|
||||
|
||||
// OutputSelection contains only active destinations. ProtectedPaths are inputs
|
||||
// and loaded configuration/profile sources that must never be overwritten.
|
||||
// InputTranscriptPath may be replaced by final transcript publication, but not reports.
|
||||
type OutputSelection struct {
|
||||
TranscriptPath string
|
||||
ReportDir string
|
||||
ReportingEnabled bool
|
||||
ProtectedPaths []string
|
||||
InputTranscriptPath string
|
||||
TranscriptPath string
|
||||
ReportDir string
|
||||
ReportingEnabled bool
|
||||
ProtectedPaths []string
|
||||
}
|
||||
|
||||
type ReportBundle struct {
|
||||
@@ -88,7 +90,11 @@ func PrepareOutput(selection OutputSelection) (*OutputSession, error) {
|
||||
return nil, fmt.Errorf("report output %q: %w", path, err)
|
||||
}
|
||||
}
|
||||
for _, source := range protected {
|
||||
sources := protected
|
||||
if path != session.transcript && selection.InputTranscriptPath != "" {
|
||||
sources = append(append([]string(nil), protected...), selection.InputTranscriptPath)
|
||||
}
|
||||
for _, source := range sources {
|
||||
collision, err := sameDestination(path, source)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -157,7 +163,7 @@ func (s *OutputSession) Finalize(transcript []byte, reports ReportBundle, failed
|
||||
}
|
||||
failures := make([]FileFailure, 0)
|
||||
if transcript != nil && s.transcript != "" {
|
||||
if err := s.replace(s.transcript, transcript); err != nil {
|
||||
if err := s.writeTranscript(transcript); err != nil {
|
||||
failures = append(failures, FileFailure{Path: s.transcript, Kind: "transcript write", Err: err})
|
||||
}
|
||||
}
|
||||
@@ -176,6 +182,24 @@ func (s *OutputSession) Finalize(transcript []byte, reports ReportBundle, failed
|
||||
return failures
|
||||
}
|
||||
|
||||
// Stream destinations use ordinary writes; replacing their directory entry would
|
||||
// destroy the device or pipe. Regular files retain per-file replacement.
|
||||
func (s *OutputSession) writeTranscript(payload []byte) error {
|
||||
info, err := os.Stat(s.transcript)
|
||||
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
if err == nil && !info.Mode().IsRegular() {
|
||||
file, err := os.OpenFile(s.transcript, os.O_WRONLY, 0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, writeErr := file.Write(payload)
|
||||
return errors.Join(writeErr, file.Close())
|
||||
}
|
||||
return s.replace(s.transcript, payload)
|
||||
}
|
||||
|
||||
func bundleParts(bundle ReportBundle) [3][]byte {
|
||||
return [3][]byte{bundle.Errors, bundle.Recoveries, bundle.Markdown}
|
||||
}
|
||||
@@ -183,8 +207,12 @@ func bundleParts(bundle ReportBundle) [3][]byte {
|
||||
func preflightTranscript(path string) error {
|
||||
info, err := os.Stat(path)
|
||||
if err == nil {
|
||||
if info.IsDir() {
|
||||
return fmt.Errorf("transcript output %q is a directory", path)
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return fmt.Errorf("transcript output %q is not a regular file", path)
|
||||
// Opening a pipe here can block before its reader is ready.
|
||||
return nil
|
||||
}
|
||||
file, openErr := os.OpenFile(path, os.O_WRONLY, 0)
|
||||
if openErr != nil {
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPrepareOutputRejectsDirectAndAliasCollisions(t *testing.T) {
|
||||
func TestPrepareOutputAllowsInPlaceTranscriptButRejectsReportCollisions(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := filepath.Join(dir, "input.json")
|
||||
if err := os.WriteFile(input, []byte("source"), 0600); err != nil {
|
||||
@@ -23,8 +23,8 @@ func TestPrepareOutputRejectsDirectAndAliasCollisions(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, output := range []string{input, alias, hardlink} {
|
||||
if _, err := PrepareOutput(OutputSelection{TranscriptPath: output, ProtectedPaths: []string{input}}); err == nil {
|
||||
t.Fatalf("collision accepted: %s", output)
|
||||
if _, err := PrepareOutput(OutputSelection{TranscriptPath: output, InputTranscriptPath: input}); err != nil {
|
||||
t.Fatalf("in-place output rejected: %s: %v", output, err)
|
||||
}
|
||||
}
|
||||
if _, err := PrepareOutput(OutputSelection{TranscriptPath: filepath.Join(dir, ErrorsFile), ReportDir: dir, ReportingEnabled: true}); err == nil {
|
||||
@@ -33,6 +33,15 @@ func TestPrepareOutputRejectsDirectAndAliasCollisions(t *testing.T) {
|
||||
if _, err := PrepareOutput(OutputSelection{ReportDir: dir, ReportingEnabled: true, ProtectedPaths: []string{filepath.Join(dir, ErrorsFile)}}); err == nil {
|
||||
t.Fatal("report/source collision accepted")
|
||||
}
|
||||
if err := os.Symlink(input, filepath.Join(dir, ErrorsFile)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := PrepareOutput(OutputSelection{ReportDir: dir, ReportingEnabled: true, InputTranscriptPath: input}); err == nil {
|
||||
t.Fatal("report/input alias accepted")
|
||||
}
|
||||
if err := os.Remove(filepath.Join(dir, ErrorsFile)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Mkdir(filepath.Join(dir, MarkdownFile), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
69
internal/core/diagnostics/output_stream_test.go
Normal file
69
internal/core/diagnostics/output_stream_test.go
Normal file
@@ -0,0 +1,69 @@
|
||||
//go:build linux || darwin
|
||||
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTranscriptStreamDestinations(t *testing.T) {
|
||||
fifo := filepath.Join(t.TempDir(), "transcript.pipe")
|
||||
if err := syscall.Mkfifo(fifo, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, path := range []string{os.DevNull, fifo} {
|
||||
t.Run(filepath.Base(path), func(t *testing.T) {
|
||||
before, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Preparation must finish even though the pipe has no reader yet.
|
||||
session, err := PrepareOutput(OutputSelection{TranscriptPath: path})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var reader *os.File
|
||||
if path == fifo {
|
||||
// Keep both ends open in the test so publication and reading can be
|
||||
// sequential; the small payload fits in the pipe buffer.
|
||||
reader, err = os.OpenFile(path, os.O_RDWR|syscall.O_NONBLOCK, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer reader.Close()
|
||||
}
|
||||
payload := []byte("transcript")
|
||||
if failures := session.Finalize(payload, ReportBundle{}, nil); len(failures) != 0 {
|
||||
t.Fatal(failures)
|
||||
}
|
||||
after, err := os.Stat(path)
|
||||
if err != nil || !os.SameFile(before, after) || before.Mode() != after.Mode() {
|
||||
t.Fatalf("stream replaced: %v", err)
|
||||
}
|
||||
if reader != nil {
|
||||
got := make([]byte, len(payload))
|
||||
if _, err := io.ReadFull(reader, got); err != nil || string(got) != string(payload) {
|
||||
t.Fatalf("pipe delivery: %q %v", got, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranscriptStreamWriteFailure(t *testing.T) {
|
||||
if _, err := os.Stat("/dev/full"); os.IsNotExist(err) {
|
||||
t.Skip("/dev/full unavailable")
|
||||
}
|
||||
session, err := PrepareOutput(OutputSelection{TranscriptPath: "/dev/full"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
failures := session.Finalize([]byte("transcript"), ReportBundle{}, nil)
|
||||
if len(failures) != 1 || failures[0].Kind != "transcript write" {
|
||||
t.Fatalf("write failure lost: %+v", failures)
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,8 @@
|
||||
package reporting
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -30,25 +29,46 @@ func TestMarkdownFixedLayoutAndCounts(t *testing.T) {
|
||||
Recoveries: []RecoveryRecord{{Reasons: []string{"output_repaired", "proposals_filtered"}}, {Reasons: []string{"output_repaired"}}, {Reasons: []string{"service_recovered"}}},
|
||||
}
|
||||
got := Markdown(snapshot, "secret")
|
||||
fixture := filepath.Join("testdata", "report.md")
|
||||
if os.Getenv("UPDATE_REPORT_GOLDEN") == "1" {
|
||||
if err := os.WriteFile(fixture, got, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
want, err := os.ReadFile(fixture)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(got, want) {
|
||||
t.Fatalf("Markdown differs from %s:\n%s", fixture, got)
|
||||
}
|
||||
text := string(got)
|
||||
for _, fragment := range []string{"| grammar_1 | grammar_2 |", "| Chunk 1 (segments 1–2) | 1 | 0* |", "| Chunk 1 (segments 1–2) | 1 | 2* |", "| Chunk 2 (segments 3–4) | 0 | — |", "in\\|put.json", "out<br>file.json", "glossary\\`[REDACTED]\\`.yaml", "- `transport_failed`: 1", "- `output_repaired`: 2"} {
|
||||
// Compare only complete table rows, preserving order and states while leaving
|
||||
// prose and spacing outside the fixed matrices free to change.
|
||||
wantRows := []string{
|
||||
"| Chunk | grammar_1 | grammar_2 |",
|
||||
"| --- | ---: | ---: |",
|
||||
"| Chunk 1 (segments 1–2) | 1 | 0* |",
|
||||
"| Chunk 2 (segments 3–4) | 0 | — |",
|
||||
"| Chunk | grammar_1 | grammar_2 |",
|
||||
"| --- | ---: | ---: |",
|
||||
"| Chunk 1 (segments 1–2) | 1 | 2* |",
|
||||
"| Chunk 2 (segments 3–4) | 0 | — |",
|
||||
}
|
||||
if rows := markdownTableRows(text); !reflect.DeepEqual(rows, wantRows) {
|
||||
t.Fatalf("matrices: got %v; want %v", rows, wantRows)
|
||||
}
|
||||
for _, fragment := range []string{"in\\|put.json", "out<br>file.json", "glossary\\`[REDACTED]\\`.yaml", "`transport_failed`: 1", "`output_repaired`: 2", "`proposals_filtered`: 1", "`service_recovered`: 1"} {
|
||||
if !strings.Contains(text, fragment) {
|
||||
t.Fatalf("missing %q in:\n%s", fragment, text)
|
||||
}
|
||||
}
|
||||
|
||||
// Assert summary quantities independently of the surrounding explanation.
|
||||
numbers := regexp.MustCompile(`[0-9]+`)
|
||||
for label, want := range map[string][]string{
|
||||
"Corrections applied": {"1"},
|
||||
"Errors": {"1", "0"},
|
||||
"Recoveries": {"3", "4"},
|
||||
} {
|
||||
var got []string
|
||||
for _, line := range strings.Split(text, "\n") {
|
||||
if value, ok := strings.CutPrefix(strings.TrimSpace(line), label+":"); ok {
|
||||
got = numbers.FindAllString(value, -1)
|
||||
break
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("%s summary: got %v; want %v", label, got, want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(text, "secret") {
|
||||
t.Fatal("secret leaked")
|
||||
}
|
||||
@@ -69,7 +89,7 @@ func TestMarkdownFixedLayoutAndCounts(t *testing.T) {
|
||||
|
||||
func TestMarkdownNoWorkAndEscapedStageLabel(t *testing.T) {
|
||||
text := string(Markdown(Snapshot{RunID: "r", StageNames: []string{"bad|label`\nsecret"}}, "secret"))
|
||||
if strings.Count(text, "No work units.") != 2 || strings.Contains(text, "secret") {
|
||||
if len(markdownTableRows(text)) != 0 || strings.Contains(text, "secret") {
|
||||
t.Fatal(text)
|
||||
}
|
||||
text = string(Markdown(Snapshot{RunID: "r", StageNames: []string{"bad|label`\nsecret"}, WorkUnits: []WorkUnit{{StageIndex: 0, ChunkIndex: 0, State: "completed"}}}, "secret"))
|
||||
@@ -77,3 +97,13 @@ func TestMarkdownNoWorkAndEscapedStageLabel(t *testing.T) {
|
||||
t.Fatal(text)
|
||||
}
|
||||
}
|
||||
|
||||
func markdownTableRows(text string) []string {
|
||||
var rows []string
|
||||
for _, line := range strings.Split(text, "\n") {
|
||||
if line = strings.TrimSpace(line); strings.HasPrefix(line, "|") {
|
||||
rows = append(rows, line)
|
||||
}
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
43
internal/core/reporting/testdata/report.md
vendored
43
internal/core/reporting/testdata/report.md
vendored
@@ -1,43 +0,0 @@
|
||||
# Audita report: run-123
|
||||
|
||||
Status: completed
|
||||
|
||||
Corrections applied: 1
|
||||
Work units: 4 total; 2 completed; 1 interrupted; 1 not started
|
||||
Errors: 1 work units; 0 run errors
|
||||
Recoveries: 3 of 4 model operations
|
||||
Proposal responses: 1; with filtering: 1
|
||||
|
||||
Transcript input: in\|put.json
|
||||
Glossary input: glossary\`[REDACTED]\`.yaml
|
||||
Transcript output: out<br>file.json
|
||||
|
||||
## Work-unit errors
|
||||
|
||||
| Chunk | grammar_1 | grammar_2 |
|
||||
| --- | ---: | ---: |
|
||||
| Chunk 1 (segments 1–2) | 1 | 0* |
|
||||
| Chunk 2 (segments 3–4) | 0 | — |
|
||||
|
||||
## Recoveries
|
||||
|
||||
| Chunk | grammar_1 | grammar_2 |
|
||||
| --- | ---: | ---: |
|
||||
| Chunk 1 (segments 1–2) | 1 | 2* |
|
||||
| Chunk 2 (segments 3–4) | 0 | — |
|
||||
|
||||
Legend: `—` = not started; `n*` = interrupted with n recorded errors or recoveries. Completed error cells are 0 or 1.
|
||||
|
||||
### Error reasons
|
||||
|
||||
- `transport_failed`: 1
|
||||
|
||||
### Recovery reasons
|
||||
|
||||
- `output_repaired`: 2
|
||||
- `proposals_filtered`: 1
|
||||
- `service_recovered`: 1
|
||||
|
||||
A recovery can have multiple reasons; reason counts need not sum to the recovery total.
|
||||
|
||||
Details: `errors.yml` and `recoveries.yml`.
|
||||
@@ -4,11 +4,13 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
@@ -21,10 +23,12 @@ import (
|
||||
// Runtime owns one PromptKit engine and the validated selections for both roles.
|
||||
// The same engine is shared by the role-bound clients during processing.
|
||||
type Runtime struct {
|
||||
Config config.Config
|
||||
Engine *pk.Engine
|
||||
Profiles map[string]pk.ProfileInspection
|
||||
nextCallID atomic.Uint64
|
||||
Config config.Config
|
||||
Engine *pk.Engine
|
||||
Profiles map[string]pk.ProfileInspection
|
||||
// ProfilePaths contains filesystem sources actually read by PromptKit.
|
||||
ProfilePaths []string
|
||||
nextCallID atomic.Uint64
|
||||
}
|
||||
|
||||
// AssetError indicates a broken embedded prompt or schema rather than an
|
||||
@@ -39,10 +43,11 @@ func NewRuntime(ctx context.Context, c config.Config, mainPath string, client *h
|
||||
if !filepath.IsAbs(profileDir) {
|
||||
profileDir = filepath.Join(filepath.Dir(mainPath), profileDir)
|
||||
}
|
||||
profileFS := &profileSourceFS{FS: os.DirFS(profileDir), paths: make(map[string]struct{})}
|
||||
options := []pk.Option{
|
||||
pk.WithPromptFS(prompts.Filesystem(), prompts.AssetRoot),
|
||||
pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot),
|
||||
pk.WithProfileFS(os.DirFS(profileDir), "."),
|
||||
pk.WithProfileFS(profileFS, "."),
|
||||
}
|
||||
ids := make([]string, 0, len(c.Backends))
|
||||
for id := range c.Backends {
|
||||
@@ -85,6 +90,12 @@ func NewRuntime(ctx context.Context, c config.Config, mainPath string, client *h
|
||||
}
|
||||
runtime.Profiles[selection.role] = *profile
|
||||
}
|
||||
profileFS.mu.Lock()
|
||||
for name := range profileFS.paths {
|
||||
runtime.ProfilePaths = append(runtime.ProfilePaths, filepath.Join(profileDir, filepath.FromSlash(name)))
|
||||
}
|
||||
profileFS.mu.Unlock()
|
||||
sort.Strings(runtime.ProfilePaths)
|
||||
return runtime, nil
|
||||
}
|
||||
|
||||
@@ -98,3 +109,21 @@ func RedactError(err error, c config.Config) string {
|
||||
}
|
||||
return redaction.New(ConfiguredSecrets(&c)).String(err.Error())
|
||||
}
|
||||
|
||||
// Record successful profile reads at the library boundary, including inherited
|
||||
// and differently named sources, without duplicating PromptKit's discovery rules.
|
||||
type profileSourceFS struct {
|
||||
fs.FS
|
||||
mu sync.Mutex
|
||||
paths map[string]struct{}
|
||||
}
|
||||
|
||||
func (f *profileSourceFS) ReadFile(name string) ([]byte, error) {
|
||||
data, err := fs.ReadFile(f.FS, name)
|
||||
if err == nil {
|
||||
f.mu.Lock()
|
||||
f.paths[name] = struct{}{}
|
||||
f.mu.Unlock()
|
||||
}
|
||||
return data, err
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user