Remove unnecessary output restrictions and simplify report tests
This commit is contained in:
@@ -638,3 +638,64 @@ func TestSubprocessInteractionWriteFailureIsVisibleAndNonfatal(t *testing.T) {
|
|||||||
t.Fatalf("debug failure classified as model incident: %+v", document)
|
t.Fatalf("debug failure classified as model incident: %+v", document)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSubprocessInPlaceTranscript(t *testing.T) {
|
||||||
|
for _, invalid := range []bool{false, true} {
|
||||||
|
t.Run(fmt.Sprint(invalid), func(t *testing.T) {
|
||||||
|
f := newProcessFixture(t, "success", "[grammar]")
|
||||||
|
f.output = f.transcript
|
||||||
|
if invalid {
|
||||||
|
if err := os.WriteFile(f.transcript, []byte("invalid"), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
before, err := os.ReadFile(f.transcript)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
code, stdout, stderr := f.run(t, true)
|
||||||
|
after, err := os.ReadFile(f.transcript)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if invalid {
|
||||||
|
if code != 1 || !bytes.Equal(before, after) {
|
||||||
|
t.Fatalf("failed processing changed input: code=%d %s", code, stderr)
|
||||||
|
}
|
||||||
|
} else if code != 0 || stdout != "" || !bytes.Contains(after, []byte("Hello world")) {
|
||||||
|
t.Fatalf("in-place publication failed: code=%d %s %s", code, stderr, after)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessProfileSources(t *testing.T) {
|
||||||
|
f := newProcessFixture(t, "success", "[]")
|
||||||
|
// PromptKit uses profile IDs, not filename stems. A report-named profile is
|
||||||
|
// still an input and must not be overwritten during report initialization.
|
||||||
|
loaded := filepath.Join(f.dir, "profiles", "errors.yml")
|
||||||
|
if err := os.Rename(filepath.Join(f.dir, "profiles", "proposal.yaml"), loaded); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
before, err := os.ReadFile(loaded)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
f.output = filepath.Join(f.dir, "profiles", "proposal.yml")
|
||||||
|
code, _, stderr := f.run(t, true, "--reporting=false")
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("unused alternate rejected: %d %s", code, stderr)
|
||||||
|
}
|
||||||
|
// Remove the transcript from the profile directory before the next load.
|
||||||
|
if err := os.Remove(f.output); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
code, _, stderr = f.run(t, true)
|
||||||
|
after, err := os.ReadFile(loaded)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if code != 2 || !bytes.Equal(before, after) || f.provider.callCount() != 0 {
|
||||||
|
t.Fatalf("loaded profile not protected: %d %s", code, stderr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -74,7 +74,7 @@ matrices. Rows show every initial chunk with 1-based chunk number and initial
|
|||||||
segment range; columns follow configured module-instance order, including
|
segment range; columns follow configured module-instance order, including
|
||||||
repeats. Completed error cells are `0` or `1`; recovery cells are operation
|
repeats. Completed error cells are `0` or `1`; recovery cells are operation
|
||||||
counts. `—` means not started and `n*` means interrupted with n facts recorded.
|
counts. `—` means not started and `n*` means interrupted with n facts recorded.
|
||||||
An empty stage or chunk set says “No work units.” Short reason breakdowns count
|
An empty stage or chunk set is identified as having no work units. Short reason breakdowns count
|
||||||
records containing each reason; a recovery can have several reasons, so those
|
records containing each reason; a recovery can have several reasons, so those
|
||||||
counts need not sum to the recovery total. The report refers to both YAML
|
counts need not sum to the recovery total. The report refers to both YAML
|
||||||
files. It does not include transcript excerpts or correction lists.
|
files. It does not include transcript excerpts or correction lists.
|
||||||
|
|||||||
@@ -17,7 +17,11 @@ and boolean values retain their types. Ordinary reports omit raw causes,
|
|||||||
transcript text, complete decisions, and token/timing payloads.
|
transcript text, complete decisions, and token/timing payloads.
|
||||||
|
|
||||||
`internal/core/diagnostics.OutputSession` owns active destination checks,
|
`internal/core/diagnostics.OutputSession` owns active destination checks,
|
||||||
transcript preflight, report initialization, and per-file final replacement.
|
transcript preflight, report initialization, regular-file replacement, and direct
|
||||||
|
stream publication. Transcript input is protected from early report writes but
|
||||||
|
may be the final transcript destination. The LLM runtime records successful reads
|
||||||
|
through PromptKit's profile filesystem and supplies those source paths to CLI;
|
||||||
|
no second filename-based discovery rule is used.
|
||||||
CLI composes it after configuration/profile inspection and before reading
|
CLI composes it after configuration/profile inspection and before reading
|
||||||
inputs or dispatching model work. It does not create ordinary report parents.
|
inputs or dispatching model work. It does not create ordinary report parents.
|
||||||
On a final write error it attempts other enabled reports and one failed-status
|
On a final write error it attempts other enabled reports and one failed-status
|
||||||
|
|||||||
@@ -23,13 +23,22 @@ overridden by `--report-dir`. `--reporting=false` suppresses all three reports
|
|||||||
and removes their path requirements. It does not suppress the stderr summary or
|
and removes their path requirements. It does not suppress the stderr summary or
|
||||||
fatal errors. Stdout contains only transcript JSON; file output leaves it empty.
|
fatal errors. Stdout contains only transcript JSON; file output leaves it empty.
|
||||||
|
|
||||||
Existing report directories and transcript-output parents must be writable.
|
Existing report directories and parents for regular transcript files must be
|
||||||
Audita checks active destinations against input, configuration, and selected
|
writable. Audita prevents report writes from overwriting transcript, glossary,
|
||||||
profile files before model dispatch. It preflights file transcript output
|
loaded configuration, or loaded profile sources, and rejects conflicting output
|
||||||
without truncating an existing file. With reporting enabled, all three reports
|
destinations. The final transcript may intentionally replace its own input after
|
||||||
are initialized as `in_progress` before input processing. Output is encoded
|
successful processing; other input/configuration/profile sources remain protected.
|
||||||
fully before publication. Each file is replaced through a temporary file on
|
Preparation never truncates an existing transcript. Device and pipe destinations
|
||||||
its destination filesystem: transcript first, then `errors.yml`,
|
(such as `/dev/null` and named pipes) receive ordinary writes at publication time;
|
||||||
|
they are not opened during preflight or replaced with regular files. A pipe write
|
||||||
|
can wait for its reader, and stream write errors are fatal. With reporting enabled,
|
||||||
|
reports still use the output path's parent; disable reporting when that placement
|
||||||
|
is unsuitable for a stream destination.
|
||||||
|
|
||||||
|
With reporting enabled, all three reports are initialized as `in_progress` before
|
||||||
|
input processing. Output is encoded fully before publication. Each regular output
|
||||||
|
file is replaced through a temporary file on its destination filesystem:
|
||||||
|
transcript first, then `errors.yml`,
|
||||||
`recoveries.yml`, and `report.md`. This is per-file replacement, not a
|
`recoveries.yml`, and `report.md`. This is per-file replacement, not a
|
||||||
multi-file transaction. New files honor the process umask; replacing an existing
|
multi-file transaction. New files honor the process umask; replacing an existing
|
||||||
file preserves its permission bits. A late report error may leave the transcript
|
file preserves its permission bits. A late report error may leave the transcript
|
||||||
|
|||||||
@@ -5,6 +5,12 @@
|
|||||||
**Status:** Implemented. All stages below are complete; completion notes record
|
**Status:** Implemented. All stages below are complete; completion notes record
|
||||||
the checks performed for each unit of work.
|
the checks performed for each unit of work.
|
||||||
|
|
||||||
|
**Post-implementation revisions:** [Remediation R11–R14](remediation.md) supersedes
|
||||||
|
the original blanket transcript/input collision prohibition, inferred profile
|
||||||
|
filenames, regular-file-only transcript handling, and whole-document Markdown
|
||||||
|
golden requirements below. These stages and completion notes are historical;
|
||||||
|
[operations](../operations.md) describes current output behavior.
|
||||||
|
|
||||||
## Scope and Execution Instructions
|
## Scope and Execution Instructions
|
||||||
|
|
||||||
Implement the target in [resilience.md](resilience.md). User intent and product
|
Implement the target in [resilience.md](resilience.md). User intent and product
|
||||||
|
|||||||
@@ -5,8 +5,8 @@
|
|||||||
invariants, contracts, guarantees, and test requirements, compared with
|
invariants, contracts, guarantees, and test requirements, compared with
|
||||||
[resilience.md](resilience.md), [implementation.md](implementation.md), and the
|
[resilience.md](resilience.md), [implementation.md](implementation.md), and the
|
||||||
[agent policy](../policy/agents.md).
|
[agent policy](../policy/agents.md).
|
||||||
**Status:** R11–R14 remain open. This document records required follow-up work;
|
**Status:** R11–R14 addressed. The findings below describe the reviewed source;
|
||||||
no application behavior has changed as part of this review.
|
see the resolution record for the implemented changes and verification.
|
||||||
|
|
||||||
The earlier findings R1–R10 have been addressed. Their findings and resolution
|
The earlier findings R1–R10 have been addressed. Their findings and resolution
|
||||||
records remain in Git history; R6–R10 were resolved in `ec78b5a`.
|
records remain in Git history; R6–R10 were resolved in `ec78b5a`.
|
||||||
@@ -134,7 +134,7 @@ Harmless prose or whitespace edits outside the required layout no longer require
|
|||||||
updating a whole-document fixture and overlapping assertions. Retain meaningful
|
updating a whole-document fixture and overlapping assertions. Retain meaningful
|
||||||
agreement checks between YAML, Markdown, and CLI totals.
|
agreement checks between YAML, Markdown, and CLI totals.
|
||||||
|
|
||||||
## Verification and Review Limits
|
## Original Review Verification and Limits
|
||||||
|
|
||||||
- This was a source and contract review; no application changes or new runtime
|
- This was a source and contract review; no application changes or new runtime
|
||||||
reproductions were made during this pass.
|
reproductions were made during this pass.
|
||||||
@@ -148,3 +148,40 @@ agreement checks between YAML, Markdown, and CLI totals.
|
|||||||
subsequent edit solely because they were used there.
|
subsequent edit solely because they were used there.
|
||||||
- Application fixes, upstream changes, deferred feature work, and release
|
- Application fixes, upstream changes, deferred feature work, and release
|
||||||
publication remain outside this document update.
|
publication remain outside this document update.
|
||||||
|
|
||||||
|
## Resolution Record
|
||||||
|
|
||||||
|
- **R11:** Output selection distinguishes the transcript input from other protected
|
||||||
|
sources. Final transcript publication may replace its input; early report writes
|
||||||
|
still cannot overwrite it, and other source/output collisions remain checked.
|
||||||
|
`TestSubprocessInPlaceTranscript` proves corrected output replaces the source on
|
||||||
|
success and an invalid source remains untouched on failure. Output-session tests
|
||||||
|
cover direct and alias acceptance, nontruncating preparation, and rejection of
|
||||||
|
report/input aliases and conflicting output destinations.
|
||||||
|
- **R12:** The runtime records successful reads through PromptKit's existing profile
|
||||||
|
filesystem and supplies actual loaded paths to CLI. It no longer guesses two
|
||||||
|
filenames per profile ID. `TestSubprocessProfileSources` proves an unused
|
||||||
|
alternate filename is allowed and a loaded profile with a different filename
|
||||||
|
remains protected from report initialization. Configuration sources continue to
|
||||||
|
come from ConfigKit's source records.
|
||||||
|
- **R13:** Regular transcript files retain per-file replacement. Existing device
|
||||||
|
and pipe destinations receive ordinary writes during finalization and are not
|
||||||
|
opened during preflight. `TestTranscriptStreamDestinations` verifies `/dev/null`
|
||||||
|
and named-pipe delivery without replacing either destination, including pipe
|
||||||
|
preparation before opening its reader. `TestTranscriptStreamWriteFailure`
|
||||||
|
checks a real stream write failure using `/dev/full` where available. Existing
|
||||||
|
directory, parent, permission, cleanup, and late-report-failure tests remain.
|
||||||
|
- **R14:** Removed the whole-document Markdown golden fixture and overlapping
|
||||||
|
matrix assertions. Tests compare complete ordered table rows, summary quantities,
|
||||||
|
reason counts, escaping/redaction, and YAML/CLI count agreement. Summary prose is
|
||||||
|
not snapshotted, and empty-work handling no longer requires an exact sentence.
|
||||||
|
No renderer or output schema changed.
|
||||||
|
|
||||||
|
Current operations and internal documentation describe the revised boundaries.
|
||||||
|
The completed implementation plan explicitly identifies its superseded choices.
|
||||||
|
|
||||||
|
Validation: `go test ./...` and `go vet ./...` passed, as did focused race checks
|
||||||
|
for the LLM runtime, diagnostics/output, and CLI subprocess packages. The refined
|
||||||
|
Markdown assertions passed their package tests. Changed-document links and
|
||||||
|
`git diff --check` passed. No live provider, dependency change, tag, or publication
|
||||||
|
was involved.
|
||||||
|
|||||||
@@ -8,7 +8,6 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
@@ -82,14 +81,14 @@ func runProcess(ctx context.Context, args []string, stdout, stderr io.Writer) in
|
|||||||
state := &processState{cfg: cfg, runtime: runtime, specs: specs, runID: runID, started: started,
|
state := &processState{cfg: cfg, runtime: runtime, specs: specs, runID: runID, started: started,
|
||||||
transcriptPath: input.TranscriptPath, glossaryPath: input.GlossaryPath, outputPath: input.OutputPath,
|
transcriptPath: input.TranscriptPath, glossaryPath: input.GlossaryPath, outputPath: input.OutputPath,
|
||||||
selected: selected, overlay: input.Overlay, stderr: stderr, stdout: stdout, ctx: ctx}
|
selected: selected, overlay: input.Overlay, stderr: stderr, stdout: stdout, ctx: ctx}
|
||||||
protected := []string{input.TranscriptPath, input.GlossaryPath}
|
protected := []string{input.GlossaryPath}
|
||||||
for _, source := range selected.Result.Sources() {
|
for _, source := range selected.Result.Sources() {
|
||||||
if source.Path != "" {
|
if source.Path != "" {
|
||||||
protected = append(protected, source.Path)
|
protected = append(protected, source.Path)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
protected = append(protected, selectedProfilePaths(cfg, selected.Path)...)
|
protected = append(protected, runtime.ProfilePaths...)
|
||||||
session, err := diagnostics.PrepareOutput(diagnostics.OutputSelection{TranscriptPath: input.OutputPath,
|
session, err := diagnostics.PrepareOutput(diagnostics.OutputSelection{InputTranscriptPath: input.TranscriptPath, TranscriptPath: input.OutputPath,
|
||||||
ReportDir: reportDir, ReportingEnabled: cfg.Reporting.Enabled, ProtectedPaths: protected})
|
ReportDir: reportDir, ReportingEnabled: cfg.Reporting.Enabled, ProtectedPaths: protected})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintf(stderr, "audita process: %s\n", llm.RedactError(err, cfg))
|
fmt.Fprintf(stderr, "audita process: %s\n", llm.RedactError(err, cfg))
|
||||||
@@ -145,19 +144,6 @@ func generateRunID() (string, error) {
|
|||||||
return "run-" + hex.EncodeToString(random[:]), nil
|
return "run-" + hex.EncodeToString(random[:]), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func selectedProfilePaths(cfg config.Config, configPath string) []string {
|
|
||||||
dir := cfg.ProfilesDir
|
|
||||||
if !filepath.IsAbs(dir) {
|
|
||||||
dir = filepath.Join(filepath.Dir(configPath), dir)
|
|
||||||
}
|
|
||||||
ids := []string{cfg.LLM.ProposalProfile, cfg.LLM.ValidationProfile}
|
|
||||||
paths := make([]string, 0, len(ids)*2)
|
|
||||||
for _, id := range ids {
|
|
||||||
paths = append(paths, filepath.Join(dir, id+".yaml"), filepath.Join(dir, id+".yml"))
|
|
||||||
}
|
|
||||||
return paths
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *processState) snapshot(status string, completed *time.Time, failures []processreport.RunFailure) reporting.Snapshot {
|
func (s *processState) snapshot(status string, completed *time.Time, failures []processreport.RunFailure) reporting.Snapshot {
|
||||||
return processreport.Build(processreport.BuildInput{RunID: s.runID, Status: status, StartedAt: s.started,
|
return processreport.Build(processreport.BuildInput{RunID: s.runID, Status: status, StartedAt: s.started,
|
||||||
CompletedAt: completed, TranscriptPath: s.transcriptPath, GlossaryPath: s.glossaryPath,
|
CompletedAt: completed, TranscriptPath: s.transcriptPath, GlossaryPath: s.glossaryPath,
|
||||||
|
|||||||
@@ -16,7 +16,9 @@ const (
|
|||||||
|
|
||||||
// OutputSelection contains only active destinations. ProtectedPaths are inputs
|
// OutputSelection contains only active destinations. ProtectedPaths are inputs
|
||||||
// and loaded configuration/profile sources that must never be overwritten.
|
// and loaded configuration/profile sources that must never be overwritten.
|
||||||
|
// InputTranscriptPath may be replaced by final transcript publication, but not reports.
|
||||||
type OutputSelection struct {
|
type OutputSelection struct {
|
||||||
|
InputTranscriptPath string
|
||||||
TranscriptPath string
|
TranscriptPath string
|
||||||
ReportDir string
|
ReportDir string
|
||||||
ReportingEnabled bool
|
ReportingEnabled bool
|
||||||
@@ -88,7 +90,11 @@ func PrepareOutput(selection OutputSelection) (*OutputSession, error) {
|
|||||||
return nil, fmt.Errorf("report output %q: %w", path, err)
|
return nil, fmt.Errorf("report output %q: %w", path, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, source := range protected {
|
sources := protected
|
||||||
|
if path != session.transcript && selection.InputTranscriptPath != "" {
|
||||||
|
sources = append(append([]string(nil), protected...), selection.InputTranscriptPath)
|
||||||
|
}
|
||||||
|
for _, source := range sources {
|
||||||
collision, err := sameDestination(path, source)
|
collision, err := sameDestination(path, source)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -157,7 +163,7 @@ func (s *OutputSession) Finalize(transcript []byte, reports ReportBundle, failed
|
|||||||
}
|
}
|
||||||
failures := make([]FileFailure, 0)
|
failures := make([]FileFailure, 0)
|
||||||
if transcript != nil && s.transcript != "" {
|
if transcript != nil && s.transcript != "" {
|
||||||
if err := s.replace(s.transcript, transcript); err != nil {
|
if err := s.writeTranscript(transcript); err != nil {
|
||||||
failures = append(failures, FileFailure{Path: s.transcript, Kind: "transcript write", Err: err})
|
failures = append(failures, FileFailure{Path: s.transcript, Kind: "transcript write", Err: err})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -176,6 +182,24 @@ func (s *OutputSession) Finalize(transcript []byte, reports ReportBundle, failed
|
|||||||
return failures
|
return failures
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Stream destinations use ordinary writes; replacing their directory entry would
|
||||||
|
// destroy the device or pipe. Regular files retain per-file replacement.
|
||||||
|
func (s *OutputSession) writeTranscript(payload []byte) error {
|
||||||
|
info, err := os.Stat(s.transcript)
|
||||||
|
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err == nil && !info.Mode().IsRegular() {
|
||||||
|
file, err := os.OpenFile(s.transcript, os.O_WRONLY, 0)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, writeErr := file.Write(payload)
|
||||||
|
return errors.Join(writeErr, file.Close())
|
||||||
|
}
|
||||||
|
return s.replace(s.transcript, payload)
|
||||||
|
}
|
||||||
|
|
||||||
func bundleParts(bundle ReportBundle) [3][]byte {
|
func bundleParts(bundle ReportBundle) [3][]byte {
|
||||||
return [3][]byte{bundle.Errors, bundle.Recoveries, bundle.Markdown}
|
return [3][]byte{bundle.Errors, bundle.Recoveries, bundle.Markdown}
|
||||||
}
|
}
|
||||||
@@ -183,8 +207,12 @@ func bundleParts(bundle ReportBundle) [3][]byte {
|
|||||||
func preflightTranscript(path string) error {
|
func preflightTranscript(path string) error {
|
||||||
info, err := os.Stat(path)
|
info, err := os.Stat(path)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
|
if info.IsDir() {
|
||||||
|
return fmt.Errorf("transcript output %q is a directory", path)
|
||||||
|
}
|
||||||
if !info.Mode().IsRegular() {
|
if !info.Mode().IsRegular() {
|
||||||
return fmt.Errorf("transcript output %q is not a regular file", path)
|
// Opening a pipe here can block before its reader is ready.
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
file, openErr := os.OpenFile(path, os.O_WRONLY, 0)
|
file, openErr := os.OpenFile(path, os.O_WRONLY, 0)
|
||||||
if openErr != nil {
|
if openErr != nil {
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestPrepareOutputRejectsDirectAndAliasCollisions(t *testing.T) {
|
func TestPrepareOutputAllowsInPlaceTranscriptButRejectsReportCollisions(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
input := filepath.Join(dir, "input.json")
|
input := filepath.Join(dir, "input.json")
|
||||||
if err := os.WriteFile(input, []byte("source"), 0600); err != nil {
|
if err := os.WriteFile(input, []byte("source"), 0600); err != nil {
|
||||||
@@ -23,8 +23,8 @@ func TestPrepareOutputRejectsDirectAndAliasCollisions(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
for _, output := range []string{input, alias, hardlink} {
|
for _, output := range []string{input, alias, hardlink} {
|
||||||
if _, err := PrepareOutput(OutputSelection{TranscriptPath: output, ProtectedPaths: []string{input}}); err == nil {
|
if _, err := PrepareOutput(OutputSelection{TranscriptPath: output, InputTranscriptPath: input}); err != nil {
|
||||||
t.Fatalf("collision accepted: %s", output)
|
t.Fatalf("in-place output rejected: %s: %v", output, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if _, err := PrepareOutput(OutputSelection{TranscriptPath: filepath.Join(dir, ErrorsFile), ReportDir: dir, ReportingEnabled: true}); err == nil {
|
if _, err := PrepareOutput(OutputSelection{TranscriptPath: filepath.Join(dir, ErrorsFile), ReportDir: dir, ReportingEnabled: true}); err == nil {
|
||||||
@@ -33,6 +33,15 @@ func TestPrepareOutputRejectsDirectAndAliasCollisions(t *testing.T) {
|
|||||||
if _, err := PrepareOutput(OutputSelection{ReportDir: dir, ReportingEnabled: true, ProtectedPaths: []string{filepath.Join(dir, ErrorsFile)}}); err == nil {
|
if _, err := PrepareOutput(OutputSelection{ReportDir: dir, ReportingEnabled: true, ProtectedPaths: []string{filepath.Join(dir, ErrorsFile)}}); err == nil {
|
||||||
t.Fatal("report/source collision accepted")
|
t.Fatal("report/source collision accepted")
|
||||||
}
|
}
|
||||||
|
if err := os.Symlink(input, filepath.Join(dir, ErrorsFile)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := PrepareOutput(OutputSelection{ReportDir: dir, ReportingEnabled: true, InputTranscriptPath: input}); err == nil {
|
||||||
|
t.Fatal("report/input alias accepted")
|
||||||
|
}
|
||||||
|
if err := os.Remove(filepath.Join(dir, ErrorsFile)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
if err := os.Mkdir(filepath.Join(dir, MarkdownFile), 0700); err != nil {
|
if err := os.Mkdir(filepath.Join(dir, MarkdownFile), 0700); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
69
internal/core/diagnostics/output_stream_test.go
Normal file
69
internal/core/diagnostics/output_stream_test.go
Normal file
@@ -0,0 +1,69 @@
|
|||||||
|
//go:build linux || darwin
|
||||||
|
|
||||||
|
package diagnostics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"syscall"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestTranscriptStreamDestinations(t *testing.T) {
|
||||||
|
fifo := filepath.Join(t.TempDir(), "transcript.pipe")
|
||||||
|
if err := syscall.Mkfifo(fifo, 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, path := range []string{os.DevNull, fifo} {
|
||||||
|
t.Run(filepath.Base(path), func(t *testing.T) {
|
||||||
|
before, err := os.Stat(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// Preparation must finish even though the pipe has no reader yet.
|
||||||
|
session, err := PrepareOutput(OutputSelection{TranscriptPath: path})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var reader *os.File
|
||||||
|
if path == fifo {
|
||||||
|
// Keep both ends open in the test so publication and reading can be
|
||||||
|
// sequential; the small payload fits in the pipe buffer.
|
||||||
|
reader, err = os.OpenFile(path, os.O_RDWR|syscall.O_NONBLOCK, 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer reader.Close()
|
||||||
|
}
|
||||||
|
payload := []byte("transcript")
|
||||||
|
if failures := session.Finalize(payload, ReportBundle{}, nil); len(failures) != 0 {
|
||||||
|
t.Fatal(failures)
|
||||||
|
}
|
||||||
|
after, err := os.Stat(path)
|
||||||
|
if err != nil || !os.SameFile(before, after) || before.Mode() != after.Mode() {
|
||||||
|
t.Fatalf("stream replaced: %v", err)
|
||||||
|
}
|
||||||
|
if reader != nil {
|
||||||
|
got := make([]byte, len(payload))
|
||||||
|
if _, err := io.ReadFull(reader, got); err != nil || string(got) != string(payload) {
|
||||||
|
t.Fatalf("pipe delivery: %q %v", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTranscriptStreamWriteFailure(t *testing.T) {
|
||||||
|
if _, err := os.Stat("/dev/full"); os.IsNotExist(err) {
|
||||||
|
t.Skip("/dev/full unavailable")
|
||||||
|
}
|
||||||
|
session, err := PrepareOutput(OutputSelection{TranscriptPath: "/dev/full"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
failures := session.Finalize([]byte("transcript"), ReportBundle{}, nil)
|
||||||
|
if len(failures) != 1 || failures[0].Kind != "transcript write" {
|
||||||
|
t.Fatalf("write failure lost: %+v", failures)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,9 +1,8 @@
|
|||||||
package reporting
|
package reporting
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"reflect"
|
||||||
"os"
|
"regexp"
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -30,25 +29,46 @@ func TestMarkdownFixedLayoutAndCounts(t *testing.T) {
|
|||||||
Recoveries: []RecoveryRecord{{Reasons: []string{"output_repaired", "proposals_filtered"}}, {Reasons: []string{"output_repaired"}}, {Reasons: []string{"service_recovered"}}},
|
Recoveries: []RecoveryRecord{{Reasons: []string{"output_repaired", "proposals_filtered"}}, {Reasons: []string{"output_repaired"}}, {Reasons: []string{"service_recovered"}}},
|
||||||
}
|
}
|
||||||
got := Markdown(snapshot, "secret")
|
got := Markdown(snapshot, "secret")
|
||||||
fixture := filepath.Join("testdata", "report.md")
|
|
||||||
if os.Getenv("UPDATE_REPORT_GOLDEN") == "1" {
|
|
||||||
if err := os.WriteFile(fixture, got, 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
want, err := os.ReadFile(fixture)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !bytes.Equal(got, want) {
|
|
||||||
t.Fatalf("Markdown differs from %s:\n%s", fixture, got)
|
|
||||||
}
|
|
||||||
text := string(got)
|
text := string(got)
|
||||||
for _, fragment := range []string{"| grammar_1 | grammar_2 |", "| Chunk 1 (segments 1–2) | 1 | 0* |", "| Chunk 1 (segments 1–2) | 1 | 2* |", "| Chunk 2 (segments 3–4) | 0 | — |", "in\\|put.json", "out<br>file.json", "glossary\\`[REDACTED]\\`.yaml", "- `transport_failed`: 1", "- `output_repaired`: 2"} {
|
// Compare only complete table rows, preserving order and states while leaving
|
||||||
|
// prose and spacing outside the fixed matrices free to change.
|
||||||
|
wantRows := []string{
|
||||||
|
"| Chunk | grammar_1 | grammar_2 |",
|
||||||
|
"| --- | ---: | ---: |",
|
||||||
|
"| Chunk 1 (segments 1–2) | 1 | 0* |",
|
||||||
|
"| Chunk 2 (segments 3–4) | 0 | — |",
|
||||||
|
"| Chunk | grammar_1 | grammar_2 |",
|
||||||
|
"| --- | ---: | ---: |",
|
||||||
|
"| Chunk 1 (segments 1–2) | 1 | 2* |",
|
||||||
|
"| Chunk 2 (segments 3–4) | 0 | — |",
|
||||||
|
}
|
||||||
|
if rows := markdownTableRows(text); !reflect.DeepEqual(rows, wantRows) {
|
||||||
|
t.Fatalf("matrices: got %v; want %v", rows, wantRows)
|
||||||
|
}
|
||||||
|
for _, fragment := range []string{"in\\|put.json", "out<br>file.json", "glossary\\`[REDACTED]\\`.yaml", "`transport_failed`: 1", "`output_repaired`: 2", "`proposals_filtered`: 1", "`service_recovered`: 1"} {
|
||||||
if !strings.Contains(text, fragment) {
|
if !strings.Contains(text, fragment) {
|
||||||
t.Fatalf("missing %q in:\n%s", fragment, text)
|
t.Fatalf("missing %q in:\n%s", fragment, text)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Assert summary quantities independently of the surrounding explanation.
|
||||||
|
numbers := regexp.MustCompile(`[0-9]+`)
|
||||||
|
for label, want := range map[string][]string{
|
||||||
|
"Corrections applied": {"1"},
|
||||||
|
"Errors": {"1", "0"},
|
||||||
|
"Recoveries": {"3", "4"},
|
||||||
|
} {
|
||||||
|
var got []string
|
||||||
|
for _, line := range strings.Split(text, "\n") {
|
||||||
|
if value, ok := strings.CutPrefix(strings.TrimSpace(line), label+":"); ok {
|
||||||
|
got = numbers.FindAllString(value, -1)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("%s summary: got %v; want %v", label, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
if strings.Contains(text, "secret") {
|
if strings.Contains(text, "secret") {
|
||||||
t.Fatal("secret leaked")
|
t.Fatal("secret leaked")
|
||||||
}
|
}
|
||||||
@@ -69,7 +89,7 @@ func TestMarkdownFixedLayoutAndCounts(t *testing.T) {
|
|||||||
|
|
||||||
func TestMarkdownNoWorkAndEscapedStageLabel(t *testing.T) {
|
func TestMarkdownNoWorkAndEscapedStageLabel(t *testing.T) {
|
||||||
text := string(Markdown(Snapshot{RunID: "r", StageNames: []string{"bad|label`\nsecret"}}, "secret"))
|
text := string(Markdown(Snapshot{RunID: "r", StageNames: []string{"bad|label`\nsecret"}}, "secret"))
|
||||||
if strings.Count(text, "No work units.") != 2 || strings.Contains(text, "secret") {
|
if len(markdownTableRows(text)) != 0 || strings.Contains(text, "secret") {
|
||||||
t.Fatal(text)
|
t.Fatal(text)
|
||||||
}
|
}
|
||||||
text = string(Markdown(Snapshot{RunID: "r", StageNames: []string{"bad|label`\nsecret"}, WorkUnits: []WorkUnit{{StageIndex: 0, ChunkIndex: 0, State: "completed"}}}, "secret"))
|
text = string(Markdown(Snapshot{RunID: "r", StageNames: []string{"bad|label`\nsecret"}, WorkUnits: []WorkUnit{{StageIndex: 0, ChunkIndex: 0, State: "completed"}}}, "secret"))
|
||||||
@@ -77,3 +97,13 @@ func TestMarkdownNoWorkAndEscapedStageLabel(t *testing.T) {
|
|||||||
t.Fatal(text)
|
t.Fatal(text)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func markdownTableRows(text string) []string {
|
||||||
|
var rows []string
|
||||||
|
for _, line := range strings.Split(text, "\n") {
|
||||||
|
if line = strings.TrimSpace(line); strings.HasPrefix(line, "|") {
|
||||||
|
rows = append(rows, line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return rows
|
||||||
|
}
|
||||||
|
|||||||
43
internal/core/reporting/testdata/report.md
vendored
43
internal/core/reporting/testdata/report.md
vendored
@@ -1,43 +0,0 @@
|
|||||||
# Audita report: run-123
|
|
||||||
|
|
||||||
Status: completed
|
|
||||||
|
|
||||||
Corrections applied: 1
|
|
||||||
Work units: 4 total; 2 completed; 1 interrupted; 1 not started
|
|
||||||
Errors: 1 work units; 0 run errors
|
|
||||||
Recoveries: 3 of 4 model operations
|
|
||||||
Proposal responses: 1; with filtering: 1
|
|
||||||
|
|
||||||
Transcript input: in\|put.json
|
|
||||||
Glossary input: glossary\`[REDACTED]\`.yaml
|
|
||||||
Transcript output: out<br>file.json
|
|
||||||
|
|
||||||
## Work-unit errors
|
|
||||||
|
|
||||||
| Chunk | grammar_1 | grammar_2 |
|
|
||||||
| --- | ---: | ---: |
|
|
||||||
| Chunk 1 (segments 1–2) | 1 | 0* |
|
|
||||||
| Chunk 2 (segments 3–4) | 0 | — |
|
|
||||||
|
|
||||||
## Recoveries
|
|
||||||
|
|
||||||
| Chunk | grammar_1 | grammar_2 |
|
|
||||||
| --- | ---: | ---: |
|
|
||||||
| Chunk 1 (segments 1–2) | 1 | 2* |
|
|
||||||
| Chunk 2 (segments 3–4) | 0 | — |
|
|
||||||
|
|
||||||
Legend: `—` = not started; `n*` = interrupted with n recorded errors or recoveries. Completed error cells are 0 or 1.
|
|
||||||
|
|
||||||
### Error reasons
|
|
||||||
|
|
||||||
- `transport_failed`: 1
|
|
||||||
|
|
||||||
### Recovery reasons
|
|
||||||
|
|
||||||
- `output_repaired`: 2
|
|
||||||
- `proposals_filtered`: 1
|
|
||||||
- `service_recovered`: 1
|
|
||||||
|
|
||||||
A recovery can have multiple reasons; reason counts need not sum to the recovery total.
|
|
||||||
|
|
||||||
Details: `errors.yml` and `recoveries.yml`.
|
|
||||||
@@ -4,11 +4,13 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||||
@@ -24,6 +26,8 @@ type Runtime struct {
|
|||||||
Config config.Config
|
Config config.Config
|
||||||
Engine *pk.Engine
|
Engine *pk.Engine
|
||||||
Profiles map[string]pk.ProfileInspection
|
Profiles map[string]pk.ProfileInspection
|
||||||
|
// ProfilePaths contains filesystem sources actually read by PromptKit.
|
||||||
|
ProfilePaths []string
|
||||||
nextCallID atomic.Uint64
|
nextCallID atomic.Uint64
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -39,10 +43,11 @@ func NewRuntime(ctx context.Context, c config.Config, mainPath string, client *h
|
|||||||
if !filepath.IsAbs(profileDir) {
|
if !filepath.IsAbs(profileDir) {
|
||||||
profileDir = filepath.Join(filepath.Dir(mainPath), profileDir)
|
profileDir = filepath.Join(filepath.Dir(mainPath), profileDir)
|
||||||
}
|
}
|
||||||
|
profileFS := &profileSourceFS{FS: os.DirFS(profileDir), paths: make(map[string]struct{})}
|
||||||
options := []pk.Option{
|
options := []pk.Option{
|
||||||
pk.WithPromptFS(prompts.Filesystem(), prompts.AssetRoot),
|
pk.WithPromptFS(prompts.Filesystem(), prompts.AssetRoot),
|
||||||
pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot),
|
pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot),
|
||||||
pk.WithProfileFS(os.DirFS(profileDir), "."),
|
pk.WithProfileFS(profileFS, "."),
|
||||||
}
|
}
|
||||||
ids := make([]string, 0, len(c.Backends))
|
ids := make([]string, 0, len(c.Backends))
|
||||||
for id := range c.Backends {
|
for id := range c.Backends {
|
||||||
@@ -85,6 +90,12 @@ func NewRuntime(ctx context.Context, c config.Config, mainPath string, client *h
|
|||||||
}
|
}
|
||||||
runtime.Profiles[selection.role] = *profile
|
runtime.Profiles[selection.role] = *profile
|
||||||
}
|
}
|
||||||
|
profileFS.mu.Lock()
|
||||||
|
for name := range profileFS.paths {
|
||||||
|
runtime.ProfilePaths = append(runtime.ProfilePaths, filepath.Join(profileDir, filepath.FromSlash(name)))
|
||||||
|
}
|
||||||
|
profileFS.mu.Unlock()
|
||||||
|
sort.Strings(runtime.ProfilePaths)
|
||||||
return runtime, nil
|
return runtime, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -98,3 +109,21 @@ func RedactError(err error, c config.Config) string {
|
|||||||
}
|
}
|
||||||
return redaction.New(ConfiguredSecrets(&c)).String(err.Error())
|
return redaction.New(ConfiguredSecrets(&c)).String(err.Error())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Record successful profile reads at the library boundary, including inherited
|
||||||
|
// and differently named sources, without duplicating PromptKit's discovery rules.
|
||||||
|
type profileSourceFS struct {
|
||||||
|
fs.FS
|
||||||
|
mu sync.Mutex
|
||||||
|
paths map[string]struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *profileSourceFS) ReadFile(name string) ([]byte, error) {
|
||||||
|
data, err := fs.ReadFile(f.FS, name)
|
||||||
|
if err == nil {
|
||||||
|
f.mu.Lock()
|
||||||
|
f.paths[name] = struct{}{}
|
||||||
|
f.mu.Unlock()
|
||||||
|
}
|
||||||
|
return data, err
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user