eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Classify PromptKit asset and operator failures

Browse Source
This commit is contained in:
Eric Rakestraw
2026-10-04 00:45:23 +00:00
parent f5fa607c35
commit f8d00e2f20
5 changed files with 352 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
docs/roadmap/implementation.md
View File

@@ -1,7 +1,7 @@
# Implementation Plan: PromptKit v0.12.0 Adoption
**Last updated:** 2026-10-03
**Status:** In progress; Stages 1–3 completed.
**Status:** In progress; Stages 1–4 completed.
**Audience:** Luna coding agent.
## Scope and Execution
@@ -253,6 +253,14 @@ startup path needs existing exit behavior preserved.
`go test ./internal/cli`; check whitespace and commit. Preserve the original
failure cause where available rather than flattening it to a generic error.
**Completion marker:** Stage 4 complete. Embedded discovery and construction
failures are distinguished from operator inputs through an embedded-only
diagnostic construction; cancellation skips that diagnostic. Selected-profile
completeness and native directory validation are covered, and both config
commands preserve exit 1 for cancellation/assets and exit 2 for invalid
operator YAML, templates, and resources. Targeted runtime tests,
`go test ./internal/cli`, and `git diff --check` passed.
## Stage 5 — Protect Consumed Configuration and Message Files
**Primary files:** `internal/framework/llm/runtime.go` or a focused source-path

4
internal/cli/run.go
View File

@@ -97,7 +97,7 @@ func runConfigValidate(ctx context.Context, args []string, stdout, stderr io.Wri
}
if _, err := llm.NewRuntime(ctx, selected.Result.Config, selected.Path, nil); err != nil {
fmt.Fprintf(stderr, "audita config validate: %s\n", llm.RedactError(err, selected.Result.Config))
if llm.IsAssetError(err) {
if ctx.Err() != nil || llm.IsAssetError(err) {
return 1
}
return 2
@@ -124,7 +124,7 @@ func runConfigPrintEffective(ctx context.Context, args []string, stdout, stderr
llmRuntime, err := llm.NewRuntime(ctx, selected.Result.Config, selected.Path, nil)
if err != nil {
fmt.Fprintf(stderr, "audita config print-effective: %s\n", llm.RedactError(err, selected.Result.Config))
if llm.IsAssetError(err) {
if ctx.Err() != nil || llm.IsAssetError(err) {
return 1
}
return 2

76
internal/cli/runtime_errors_test.go Normal file
View File

@@ -0,0 +1,76 @@
package cli
import (
"bytes"
"context"
"os"
"path/filepath"
"strings"
"testing"
)
func writeRuntimeErrorConfig(t *testing.T, operatorDocument string) string {
t.Helper()
dir := t.TempDir()
profiles := filepath.Join(dir, "profiles")
if err := os.Mkdir(profiles, 0700); err != nil {
t.Fatal(err)
}
main := filepath.Join(dir, "config.yml")
configText := "version: 2\nprofiles_dir: profiles\nbackends:\n offline:\n endpoint: http://127.0.0.1:1/v1\n credential: AUDITA_TEST_PROVIDER_KEY\nllm:\n proposal_profile: proposal\n"
if err := os.WriteFile(main, []byte(configText), 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(profiles, "operator.yml"), []byte(operatorDocument), 0600); err != nil {
t.Fatal(err)
}
return main
}
func TestConfigCommandsClassifyOperatorPromptErrorsAsInputErrors(t *testing.T) {
t.Setenv("AUDITA_TEST_PROVIDER_KEY", "synthetic-cli-redaction-secret")
definitions := map[string]string{
"malformed operator YAML": "profiles:\n proposal: [\n",
"invalid operator template": "profiles:\n proposal:\n backend: offline\n model: local-model\nprompts:\n modules.grammar.proposal:\n v1:\n messages:\n - role: user\n content: '{{'\n",
"missing operator message resource": "profiles:\n proposal:\n backend: offline\n model: local-model\nprompts:\n modules.grammar.proposal:\n v1:\n messages:\n - role: user\n content_file: missing-instruction.md\n",
}
commands := []struct {
name string
args []string
}{
{name: "validate", args: []string{"config", "validate"}},
{name: "print effective", args: []string{"config", "print-effective", "--view", "profiles"}},
}
for definition, document := range definitions {
for _, command := range commands {
t.Run(definition+"/"+command.name, func(t *testing.T) {
main := writeRuntimeErrorConfig(t, document)
args := append(append([]string(nil), command.args...), "--config", main)
var stdout, stderr bytes.Buffer
if code := Run(context.Background(), args, &stdout, &stderr); code != 2 {
t.Fatalf("exit=%d, stdout=%q, stderr=%q", code, stdout.String(), stderr.String())
}
if strings.Contains(stderr.String(), "synthetic-cli-redaction-secret") {
t.Fatalf("configured secret leaked: %q", stderr.String())
}
})
}
}
}
func TestConfigCommandsKeepCallerCancellationAsRuntimeExit(t *testing.T) {
t.Setenv("AUDITA_TEST_PROVIDER_KEY", "synthetic-cli-redaction-secret")
main := writeRuntimeErrorConfig(t, "profiles:\n proposal:\n backend: offline\n model: local-model\n")
commands := [][]string{
{"config", "validate", "--config", main},
{"config", "print-effective", "--config", main, "--view", "profiles"},
}
for _, args := range commands {
ctx, cancel := context.WithCancel(context.Background())
cancel()
var stdout, stderr bytes.Buffer
if code := Run(ctx, args, &stdout, &stderr); code != 1 {
t.Fatalf("args=%v exit=%d, stdout=%q, stderr=%q", args, code, stdout.String(), stderr.String())
}
}
}

52
internal/framework/llm/runtime.go
View File

@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"io/fs"
"net/http"
"path/filepath"
"strings"
@@ -36,18 +37,34 @@ func (e *AssetError) Error() string { return e.Err.Error() }
func (e *AssetError) Unwrap() error { return e.Err }
func NewRuntime(ctx context.Context, c config.Config, mainPath string, client *http.Client) (*Runtime, error) {
return newRuntimeWithPromptFS(ctx, c, mainPath, client, prompts.Filesystem())
}
func newRuntimeWithPromptFS(ctx context.Context, c config.Config, mainPath string, client *http.Client, promptFS fs.FS) (*Runtime, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
profileDir := c.ProfilesDir
if !filepath.IsAbs(profileDir) {
profileDir = filepath.Join(filepath.Dir(mainPath), profileDir)
}
promptInputs, err := pk.FilesystemInputs("Audita embedded prompts", prompts.Filesystem(), prompts.AssetRoot)
promptInputs, err := pk.FilesystemInputs("Audita embedded prompts", promptFS, prompts.AssetRoot)
if err != nil {
return nil, fmt.Errorf("discover embedded prompts: %w", err)
if ctxErr := ctx.Err(); ctxErr != nil {
return nil, ctxErr
}
return nil, &AssetError{Err: fmt.Errorf("discover embedded prompts: %w", err)}
}
profileInputs, err := pk.DirectoryInputs(profileDir)
if err != nil {
if ctxErr := ctx.Err(); ctxErr != nil {
return nil, ctxErr
}
return nil, fmt.Errorf("discover operator definitions: %w", err)
}
if err := ctx.Err(); err != nil {
return nil, err
}
applicationBackends := make(map[string]any, len(c.Backends))
for id := range c.Backends {
backend := c.Backends[id]
@@ -71,11 +88,12 @@ func NewRuntime(ctx context.Context, c config.Config, mainPath string, client *h
TransientRetries: &c.LLM.TransientRetries,
}, pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot))
if err != nil {
return nil, fmt.Errorf("assemble LLM runtime: %w", err)
return nil, embeddedAssetOrInputError(ctx, promptInputs, fmt.Errorf("assemble LLM runtime: %w", err))
}
for _, meta := range prompts.RegisteredMetadata() {
if _, err := engine.InspectPrompt(ctx, meta.PromptID, meta.PromptVersion); err != nil {
return nil, &AssetError{Err: fmt.Errorf("inspect embedded prompt %s: %w", meta.PromptID, err)}
cause := fmt.Errorf("inspect prompt %s: %w", meta.PromptID, err)
return nil, embeddedAssetOrInputError(ctx, promptInputs, cause)
}
}
runtime := &Runtime{Config: c, Engine: engine, Profiles: make(map[string]pk.ProfileInspection, 2)}
@@ -110,6 +128,32 @@ func NewRuntime(ctx context.Context, c config.Config, mainPath string, client *h
return runtime, nil
}
func embeddedAssetOrInputError(ctx context.Context, promptInputs []ck.Input, cause error) error {
if err := ctx.Err(); err != nil {
return err
}
embeddedSources := pk.ConfigurationSources{
Underlays: []ck.Underlay{{Inputs: promptInputs}},
}
diagnosticEngine, err := pk.NewEngine(pk.Config{Sources: embeddedSources},
pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot))
if err == nil {
for _, meta := range prompts.RegisteredMetadata() {
if _, inspectErr := diagnosticEngine.InspectPrompt(ctx, meta.PromptID, meta.PromptVersion); inspectErr != nil {
err = fmt.Errorf("inspect embedded prompt %s: %w", meta.PromptID, inspectErr)
break
}
}
}
if ctxErr := ctx.Err(); ctxErr != nil {
return ctxErr
}
if err != nil {
return &AssetError{Err: fmt.Errorf("validate embedded prompt assets: %w", err)}
}
return cause
}
func IsAssetError(err error) bool { var target *AssetError; return errors.As(err, &target) }
// RedactError replaces every configured resolved backend secret before a

217
internal/framework/llm/runtime_assembly_test.go Normal file
View File

@@ -0,0 +1,217 @@
package llm
import (
"context"
"errors"
"io/fs"
"net/http"
"os"
"path"
"path/filepath"
"strings"
"testing"
"testing/fstest"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/prompts"
)
func runtimeAssemblyFixture(t *testing.T, selections string, documents map[string]string) (config.Config, string) {
t.Helper()
dir := t.TempDir()
profileDir := filepath.Join(dir, "profiles")
if err := os.Mkdir(profileDir, 0700); err != nil {
t.Fatal(err)
}
main := filepath.Join(dir, "config.yml")
text := "version: 2\nprofiles_dir: profiles\nbackends:\n offline:\n endpoint: http://127.0.0.1:1/v1\n concurrency: 1\n queue_capacity: 0\nllm:\n" + selections
if err := os.WriteFile(main, []byte(text), 0600); err != nil {
t.Fatal(err)
}
for name, content := range documents {
if err := os.WriteFile(filepath.Join(profileDir, name), []byte(content), 0600); err != nil {
t.Fatal(err)
}
}
loaded, err := config.Load(ck.File(main), "")
if err != nil {
t.Fatal(err)
}
return loaded.Config, main
}
func copyEmbeddedPromptFilesystem(t *testing.T) fstest.MapFS {
t.Helper()
source := prompts.Filesystem()
files := make(fstest.MapFS)
err := fs.WalkDir(source, prompts.AssetRoot, func(name string, entry fs.DirEntry, err error) error {
if err != nil || entry.IsDir() {
return err
}
data, err := fs.ReadFile(source, name)
if err != nil {
return err
}
files[name] = &fstest.MapFile{Data: data}
return nil
})
if err != nil {
t.Fatal(err)
}
return files
}
func TestNewRuntimeRequiresCompleteSelectedProfiles(t *testing.T) {
validProposal := " backend: offline\n model: proposal-model\n"
cases := []struct {
name string
selections string
profiles string
role string
field string
}{
{
name: "omitted proposal model with derived validation selection",
selections: " proposal_profile: proposal\n",
profiles: "profiles:\n proposal:\n backend: offline\n",
role: "proposal",
field: "model",
},
{
name: "empty validation model",
selections: " proposal_profile: proposal\n validation_profile: validation\n",
profiles: "profiles:\n proposal:\n" + validProposal + " validation:\n backend: offline\n model: \"\"\n",
role: "validation",
field: "model",
},
{
name: "null proposal model",
selections: " proposal_profile: proposal\n",
profiles: "profiles:\n proposal:\n backend: offline\n model: null\n",
role: "proposal",
field: "model",
},
{
name: "cleared validation endpoint",
selections: " proposal_profile: proposal\n validation_profile: validation\n",
profiles: "profiles:\n proposal:\n" + validProposal + " validation:\n backend: offline\n endpoint: null\n model: validation-model\n",
role: "validation",
field: "endpoint",
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
cfg, main := runtimeAssemblyFixture(t, tc.selections, map[string]string{"definitions.yml": tc.profiles})
_, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
if err == nil || !strings.Contains(err.Error(), tc.role) || !strings.Contains(err.Error(), tc.field) {
t.Fatalf("selected incomplete profile was not rejected for %s.%s: %v", tc.role, tc.field, err)
}
})
}
}
func TestNewRuntimeAllowsUnselectedPartialProfiles(t *testing.T) {
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n unfinished:\n backend: offline\n",
})
runtime, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
if err != nil {
t.Fatal(err)
}
if runtime.Profiles["proposal"].EffectiveModelParams.Model != "proposal-model" || runtime.Profiles["validation"].EffectiveModelParams.Model != "proposal-model" {
t.Fatalf("role profiles were not resolved: %+v", runtime.Profiles)
}
}
func TestMissingSelectedProfileDirectoryIsOperatorInputError(t *testing.T) {
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n",
})
if err := os.RemoveAll(filepath.Join(filepath.Dir(main), cfg.ProfilesDir)); err != nil {
t.Fatal(err)
}
_, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
if err == nil || IsAssetError(err) {
t.Fatalf("missing operator profile directory classification: %v", err)
}
}
func TestNewRuntimeRejectsMalformedUnselectedProfilesAndConflicts(t *testing.T) {
cases := []struct {
name string
document map[string]string
selection string
}{
{
name: "malformed unselected definition",
document: map[string]string{"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n unused:\n temperature: 9\n"},
selection: " proposal_profile: proposal\n",
},
{
name: "undefined selected backend",
document: map[string]string{"definitions.yml": "profiles:\n proposal:\n backend: missing\n model: proposal-model\n"},
selection: " proposal_profile: proposal\n",
},
{
name: "conflicting directory contributions",
document: map[string]string{
"a.yml": "profiles:\n proposal:\n backend: offline\n model: first-model\n",
"b.yml": "profiles:\n proposal:\n model: second-model\n",
},
selection: " proposal_profile: proposal\n",
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
cfg, main := runtimeAssemblyFixture(t, tc.selection, tc.document)
_, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
if err == nil || IsAssetError(err) {
t.Fatalf("operator definition classification: %v", err)
}
})
}
}
func TestRuntimeClassifiesEmbeddedAssetsAndHonorsCancellation(t *testing.T) {
_, err := newRuntimeWithPromptFS(context.Background(), config.Config{}, "config.yml", nil, fstest.MapFS{})
if err == nil || !IsAssetError(err) || !errors.Is(err, fs.ErrNotExist) {
t.Fatalf("embedded discovery error was not preserved as an asset error: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, err = newRuntimeWithPromptFS(ctx, config.Config{}, "config.yml", nil, fstest.MapFS{})
if !errors.Is(err, context.Canceled) || IsAssetError(err) {
t.Fatalf("caller cancellation classification: %v", err)
}
}
func TestBrokenEmbeddedPromptFailsEmbeddedOnlyConstruction(t *testing.T) {
promptFS := copyEmbeddedPromptFilesystem(t)
assetPath := path.Join(prompts.AssetRoot, prompts.PromptIDModuleGrammarProposal+".yaml")
promptFS[assetPath] = &fstest.MapFile{Data: []byte("prompts:\n modules.grammar.proposal:\n v1:\n messages: []\n output:\n format: json\n validation_mode: json_schema\n schema_path: correction_set.v1.json\n")}
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n",
})
_, err := newRuntimeWithPromptFS(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}, promptFS)
if err == nil || !IsAssetError(err) {
t.Fatalf("broken embedded prompt was not classified as an asset error: %v", err)
}
}
func TestOperatorPromptCanRepairInvalidEmbeddedPrompt(t *testing.T) {
promptFS := copyEmbeddedPromptFilesystem(t)
assetPath := path.Join(prompts.AssetRoot, prompts.PromptIDModuleGrammarProposal+".yaml")
promptFS[assetPath] = &fstest.MapFile{Data: []byte("prompts:\n modules.grammar.proposal:\n v1:\n messages: []\n output:\n format: json\n validation_mode: json_schema\n schema_path: correction_set.v1.json\n")}
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
"operator.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\nprompts:\n modules.grammar.proposal:\n v1:\n messages:\n - role: system\n content: operator system\n - role: user\n content: operator user\n",
})
runtime, err := newRuntimeWithPromptFS(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}, promptFS)
if err != nil {
t.Fatalf("valid final configuration was rejected with an invalid displaced embedded prompt: %v", err)
}
if runtime.Engine == nil {
t.Fatal("runtime engine was not retained")
}
}