eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Remove unnecessary output restrictions and simplify report tests

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-25 00:36:44 +00:00
parent efaefdbf3d
commit 0b6cd98262
13 changed files with 330 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
internal/cli/process.go
View File

@@ -8,7 +8,6 @@ import (
"errors"
"fmt"
"io"
"path/filepath"
"strings"
"sync"
"time"
@@ -82,14 +81,14 @@ func runProcess(ctx context.Context, args []string, stdout, stderr io.Writer) in
state := &processState{cfg: cfg, runtime: runtime, specs: specs, runID: runID, started: started,
transcriptPath: input.TranscriptPath, glossaryPath: input.GlossaryPath, outputPath: input.OutputPath,
selected: selected, overlay: input.Overlay, stderr: stderr, stdout: stdout, ctx: ctx}
protected := []string{input.TranscriptPath, input.GlossaryPath}
protected := []string{input.GlossaryPath}
for _, source := range selected.Result.Sources() {
if source.Path != "" {
protected = append(protected, source.Path)
}
}
protected = append(protected, selectedProfilePaths(cfg, selected.Path)...)
session, err := diagnostics.PrepareOutput(diagnostics.OutputSelection{TranscriptPath: input.OutputPath,
protected = append(protected, runtime.ProfilePaths...)
session, err := diagnostics.PrepareOutput(diagnostics.OutputSelection{InputTranscriptPath: input.TranscriptPath, TranscriptPath: input.OutputPath,
ReportDir: reportDir, ReportingEnabled: cfg.Reporting.Enabled, ProtectedPaths: protected})
if err != nil {
fmt.Fprintf(stderr, "audita process: %s\n", llm.RedactError(err, cfg))
@@ -145,19 +144,6 @@ func generateRunID() (string, error) {
return "run-" + hex.EncodeToString(random[:]), nil
}
func selectedProfilePaths(cfg config.Config, configPath string) []string {
dir := cfg.ProfilesDir
if !filepath.IsAbs(dir) {
dir = filepath.Join(filepath.Dir(configPath), dir)
}
ids := []string{cfg.LLM.ProposalProfile, cfg.LLM.ValidationProfile}
paths := make([]string, 0, len(ids)*2)
for _, id := range ids {
paths = append(paths, filepath.Join(dir, id+".yaml"), filepath.Join(dir, id+".yml"))
}
return paths
}
func (s *processState) snapshot(status string, completed *time.Time, failures []processreport.RunFailure) reporting.Snapshot {
return processreport.Build(processreport.BuildInput{RunID: s.runID, Status: status, StartedAt: s.started,
CompletedAt: completed, TranscriptPath: s.transcriptPath, GlossaryPath: s.glossaryPath,

42
internal/core/diagnostics/output_session.go
View File

@@ -16,11 +16,13 @@ const (
// OutputSelection contains only active destinations. ProtectedPaths are inputs
// and loaded configuration/profile sources that must never be overwritten.
// InputTranscriptPath may be replaced by final transcript publication, but not reports.
type OutputSelection struct {
TranscriptPath string
ReportDir string
ReportingEnabled bool
ProtectedPaths []string
InputTranscriptPath string
TranscriptPath string
ReportDir string
ReportingEnabled bool
ProtectedPaths []string
}
type ReportBundle struct {
@@ -88,7 +90,11 @@ func PrepareOutput(selection OutputSelection) (*OutputSession, error) {
return nil, fmt.Errorf("report output %q: %w", path, err)
}
}
for _, source := range protected {
sources := protected
if path != session.transcript && selection.InputTranscriptPath != "" {
sources = append(append([]string(nil), protected...), selection.InputTranscriptPath)
}
for _, source := range sources {
collision, err := sameDestination(path, source)
if err != nil {
return nil, err
@@ -157,7 +163,7 @@ func (s *OutputSession) Finalize(transcript []byte, reports ReportBundle, failed
}
failures := make([]FileFailure, 0)
if transcript != nil && s.transcript != "" {
if err := s.replace(s.transcript, transcript); err != nil {
if err := s.writeTranscript(transcript); err != nil {
failures = append(failures, FileFailure{Path: s.transcript, Kind: "transcript write", Err: err})
}
}
@@ -176,6 +182,24 @@ func (s *OutputSession) Finalize(transcript []byte, reports ReportBundle, failed
return failures
}
// Stream destinations use ordinary writes; replacing their directory entry would
// destroy the device or pipe. Regular files retain per-file replacement.
func (s *OutputSession) writeTranscript(payload []byte) error {
info, err := os.Stat(s.transcript)
if err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
if err == nil && !info.Mode().IsRegular() {
file, err := os.OpenFile(s.transcript, os.O_WRONLY, 0)
if err != nil {
return err
}
_, writeErr := file.Write(payload)
return errors.Join(writeErr, file.Close())
}
return s.replace(s.transcript, payload)
}
func bundleParts(bundle ReportBundle) [3][]byte {
return [3][]byte{bundle.Errors, bundle.Recoveries, bundle.Markdown}
}
@@ -183,8 +207,12 @@ func bundleParts(bundle ReportBundle) [3][]byte {
func preflightTranscript(path string) error {
info, err := os.Stat(path)
if err == nil {
if info.IsDir() {
return fmt.Errorf("transcript output %q is a directory", path)
}
if !info.Mode().IsRegular() {
return fmt.Errorf("transcript output %q is not a regular file", path)
// Opening a pipe here can block before its reader is ready.
return nil
}
file, openErr := os.OpenFile(path, os.O_WRONLY, 0)
if openErr != nil {

15
internal/core/diagnostics/output_session_test.go
View File

@@ -8,7 +8,7 @@ import (
"testing"
)
func TestPrepareOutputRejectsDirectAndAliasCollisions(t *testing.T) {
func TestPrepareOutputAllowsInPlaceTranscriptButRejectsReportCollisions(t *testing.T) {
dir := t.TempDir()
input := filepath.Join(dir, "input.json")
if err := os.WriteFile(input, []byte("source"), 0600); err != nil {
@@ -23,8 +23,8 @@ func TestPrepareOutputRejectsDirectAndAliasCollisions(t *testing.T) {
t.Fatal(err)
}
for _, output := range []string{input, alias, hardlink} {
if _, err := PrepareOutput(OutputSelection{TranscriptPath: output, ProtectedPaths: []string{input}}); err == nil {
t.Fatalf("collision accepted: %s", output)
if _, err := PrepareOutput(OutputSelection{TranscriptPath: output, InputTranscriptPath: input}); err != nil {
t.Fatalf("in-place output rejected: %s: %v", output, err)
}
}
if _, err := PrepareOutput(OutputSelection{TranscriptPath: filepath.Join(dir, ErrorsFile), ReportDir: dir, ReportingEnabled: true}); err == nil {
@@ -33,6 +33,15 @@ func TestPrepareOutputRejectsDirectAndAliasCollisions(t *testing.T) {
if _, err := PrepareOutput(OutputSelection{ReportDir: dir, ReportingEnabled: true, ProtectedPaths: []string{filepath.Join(dir, ErrorsFile)}}); err == nil {
t.Fatal("report/source collision accepted")
}
if err := os.Symlink(input, filepath.Join(dir, ErrorsFile)); err != nil {
t.Fatal(err)
}
if _, err := PrepareOutput(OutputSelection{ReportDir: dir, ReportingEnabled: true, InputTranscriptPath: input}); err == nil {
t.Fatal("report/input alias accepted")
}
if err := os.Remove(filepath.Join(dir, ErrorsFile)); err != nil {
t.Fatal(err)
}
if err := os.Mkdir(filepath.Join(dir, MarkdownFile), 0700); err != nil {
t.Fatal(err)
}

69
internal/core/diagnostics/output_stream_test.go Normal file
View File

@@ -0,0 +1,69 @@
//go:build linux || darwin
package diagnostics
import (
"io"
"os"
"path/filepath"
"syscall"
"testing"
)
func TestTranscriptStreamDestinations(t *testing.T) {
fifo := filepath.Join(t.TempDir(), "transcript.pipe")
if err := syscall.Mkfifo(fifo, 0600); err != nil {
t.Fatal(err)
}
for _, path := range []string{os.DevNull, fifo} {
t.Run(filepath.Base(path), func(t *testing.T) {
before, err := os.Stat(path)
if err != nil {
t.Fatal(err)
}
// Preparation must finish even though the pipe has no reader yet.
session, err := PrepareOutput(OutputSelection{TranscriptPath: path})
if err != nil {
t.Fatal(err)
}
var reader *os.File
if path == fifo {
// Keep both ends open in the test so publication and reading can be
// sequential; the small payload fits in the pipe buffer.
reader, err = os.OpenFile(path, os.O_RDWR|syscall.O_NONBLOCK, 0)
if err != nil {
t.Fatal(err)
}
defer reader.Close()
}
payload := []byte("transcript")
if failures := session.Finalize(payload, ReportBundle{}, nil); len(failures) != 0 {
t.Fatal(failures)
}
after, err := os.Stat(path)
if err != nil || !os.SameFile(before, after) || before.Mode() != after.Mode() {
t.Fatalf("stream replaced: %v", err)
}
if reader != nil {
got := make([]byte, len(payload))
if _, err := io.ReadFull(reader, got); err != nil || string(got) != string(payload) {
t.Fatalf("pipe delivery: %q %v", got, err)
}
}
})
}
}
func TestTranscriptStreamWriteFailure(t *testing.T) {
if _, err := os.Stat("/dev/full"); os.IsNotExist(err) {
t.Skip("/dev/full unavailable")
}
session, err := PrepareOutput(OutputSelection{TranscriptPath: "/dev/full"})
if err != nil {
t.Fatal(err)
}
failures := session.Finalize([]byte("transcript"), ReportBundle{}, nil)
if len(failures) != 1 || failures[0].Kind != "transcript write" {
t.Fatalf("write failure lost: %+v", failures)
}
}

66
internal/core/reporting/markdown_test.go
View File

@@ -1,9 +1,8 @@
package reporting
import (
"bytes"
"os"
"path/filepath"
"reflect"
"regexp"
"strings"
"testing"
"time"
@@ -30,25 +29,46 @@ func TestMarkdownFixedLayoutAndCounts(t *testing.T) {
Recoveries: []RecoveryRecord{{Reasons: []string{"output_repaired", "proposals_filtered"}}, {Reasons: []string{"output_repaired"}}, {Reasons: []string{"service_recovered"}}},
}
got := Markdown(snapshot, "secret")
fixture := filepath.Join("testdata", "report.md")
if os.Getenv("UPDATE_REPORT_GOLDEN") == "1" {
if err := os.WriteFile(fixture, got, 0600); err != nil {
t.Fatal(err)
}
}
want, err := os.ReadFile(fixture)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got, want) {
t.Fatalf("Markdown differs from %s:\n%s", fixture, got)
}
text := string(got)
for _, fragment := range []string{"| grammar_1 | grammar_2 |", "| Chunk 1 (segments 1–2) | 1 | 0* |", "| Chunk 1 (segments 1–2) | 1 | 2* |", "| Chunk 2 (segments 3–4) | 0 | — |", "in\\|put.json", "out<br>file.json", "glossary\\`[REDACTED]\\`.yaml", "- `transport_failed`: 1", "- `output_repaired`: 2"} {
// Compare only complete table rows, preserving order and states while leaving
// prose and spacing outside the fixed matrices free to change.
wantRows := []string{
"| Chunk | grammar_1 | grammar_2 |",
"| --- | ---: | ---: |",
"| Chunk 1 (segments 1–2) | 1 | 0* |",
"| Chunk 2 (segments 3–4) | 0 | — |",
"| Chunk | grammar_1 | grammar_2 |",
"| --- | ---: | ---: |",
"| Chunk 1 (segments 1–2) | 1 | 2* |",
"| Chunk 2 (segments 3–4) | 0 | — |",
}
if rows := markdownTableRows(text); !reflect.DeepEqual(rows, wantRows) {
t.Fatalf("matrices: got %v; want %v", rows, wantRows)
}
for _, fragment := range []string{"in\\|put.json", "out<br>file.json", "glossary\\`[REDACTED]\\`.yaml", "`transport_failed`: 1", "`output_repaired`: 2", "`proposals_filtered`: 1", "`service_recovered`: 1"} {
if !strings.Contains(text, fragment) {
t.Fatalf("missing %q in:\n%s", fragment, text)
}
}
// Assert summary quantities independently of the surrounding explanation.
numbers := regexp.MustCompile(`[0-9]+`)
for label, want := range map[string][]string{
"Corrections applied": {"1"},
"Errors": {"1", "0"},
"Recoveries": {"3", "4"},
} {
var got []string
for _, line := range strings.Split(text, "\n") {
if value, ok := strings.CutPrefix(strings.TrimSpace(line), label+":"); ok {
got = numbers.FindAllString(value, -1)
break
}
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("%s summary: got %v; want %v", label, got, want)
}
}
if strings.Contains(text, "secret") {
t.Fatal("secret leaked")
}
@@ -69,7 +89,7 @@ func TestMarkdownFixedLayoutAndCounts(t *testing.T) {
func TestMarkdownNoWorkAndEscapedStageLabel(t *testing.T) {
text := string(Markdown(Snapshot{RunID: "r", StageNames: []string{"bad|label`\nsecret"}}, "secret"))
if strings.Count(text, "No work units.") != 2 || strings.Contains(text, "secret") {
if len(markdownTableRows(text)) != 0 || strings.Contains(text, "secret") {
t.Fatal(text)
}
text = string(Markdown(Snapshot{RunID: "r", StageNames: []string{"bad|label`\nsecret"}, WorkUnits: []WorkUnit{{StageIndex: 0, ChunkIndex: 0, State: "completed"}}}, "secret"))
@@ -77,3 +97,13 @@ func TestMarkdownNoWorkAndEscapedStageLabel(t *testing.T) {
t.Fatal(text)
}
}
func markdownTableRows(text string) []string {
var rows []string
for _, line := range strings.Split(text, "\n") {
if line = strings.TrimSpace(line); strings.HasPrefix(line, "|") {
rows = append(rows, line)
}
}
return rows
}

43
internal/core/reporting/testdata/report.md vendored
View File

@@ -1,43 +0,0 @@
# Audita report: run-123
Status: completed
Corrections applied: 1
Work units: 4 total; 2 completed; 1 interrupted; 1 not started
Errors: 1 work units; 0 run errors
Recoveries: 3 of 4 model operations
Proposal responses: 1; with filtering: 1
Transcript input: in\|put.json
Glossary input: glossary\`[REDACTED]\`.yaml
Transcript output: out<br>file.json
## Work-unit errors
| Chunk | grammar_1 | grammar_2 |
| --- | ---: | ---: |
| Chunk 1 (segments 1–2) | 1 | 0* |
| Chunk 2 (segments 3–4) | 0 | — |
## Recoveries
| Chunk | grammar_1 | grammar_2 |
| --- | ---: | ---: |
| Chunk 1 (segments 1–2) | 1 | 2* |
| Chunk 2 (segments 3–4) | 0 | — |
Legend: `—` = not started; `n*` = interrupted with n recorded errors or recoveries. Completed error cells are 0 or 1.
### Error reasons
- `transport_failed`: 1
### Recovery reasons
- `output_repaired`: 2
- `proposals_filtered`: 1
- `service_recovered`: 1
A recovery can have multiple reasons; reason counts need not sum to the recovery total.
Details: `errors.yml` and `recoveries.yml`.

39
internal/framework/llm/runtime.go
View File

@@ -4,11 +4,13 @@ import (
"context"
"errors"
"fmt"
"io/fs"
"net/http"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"sync/atomic"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
@@ -21,10 +23,12 @@ import (
// Runtime owns one PromptKit engine and the validated selections for both roles.
// The same engine is shared by the role-bound clients during processing.
type Runtime struct {
Config config.Config
Engine *pk.Engine
Profiles map[string]pk.ProfileInspection
nextCallID atomic.Uint64
Config config.Config
Engine *pk.Engine
Profiles map[string]pk.ProfileInspection
// ProfilePaths contains filesystem sources actually read by PromptKit.
ProfilePaths []string
nextCallID atomic.Uint64
}
// AssetError indicates a broken embedded prompt or schema rather than an
@@ -39,10 +43,11 @@ func NewRuntime(ctx context.Context, c config.Config, mainPath string, client *h
if !filepath.IsAbs(profileDir) {
profileDir = filepath.Join(filepath.Dir(mainPath), profileDir)
}
profileFS := &profileSourceFS{FS: os.DirFS(profileDir), paths: make(map[string]struct{})}
options := []pk.Option{
pk.WithPromptFS(prompts.Filesystem(), prompts.AssetRoot),
pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot),
pk.WithProfileFS(os.DirFS(profileDir), "."),
pk.WithProfileFS(profileFS, "."),
}
ids := make([]string, 0, len(c.Backends))
for id := range c.Backends {
@@ -85,6 +90,12 @@ func NewRuntime(ctx context.Context, c config.Config, mainPath string, client *h
}
runtime.Profiles[selection.role] = *profile
}
profileFS.mu.Lock()
for name := range profileFS.paths {
runtime.ProfilePaths = append(runtime.ProfilePaths, filepath.Join(profileDir, filepath.FromSlash(name)))
}
profileFS.mu.Unlock()
sort.Strings(runtime.ProfilePaths)
return runtime, nil
}
@@ -98,3 +109,21 @@ func RedactError(err error, c config.Config) string {
}
return redaction.New(ConfiguredSecrets(&c)).String(err.Error())
}
// Record successful profile reads at the library boundary, including inherited
// and differently named sources, without duplicating PromptKit's discovery rules.
type profileSourceFS struct {
fs.FS
mu sync.Mutex
paths map[string]struct{}
}
func (f *profileSourceFS) ReadFile(name string) ([]byte, error) {
data, err := fs.ReadFile(f.FS, name)
if err == nil {
f.mu.Lock()
f.paths[name] = struct{}{}
f.mu.Unlock()
}
return data, err
}