eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Classify PromptKit asset and operator failures

Browse Source
This commit is contained in:
Eric Rakestraw
2026-10-04 00:45:23 +00:00
parent f5fa607c35
commit f8d00e2f20
5 changed files with 352 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
internal/framework/llm/runtime.go
View File

@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"io/fs"
"net/http"
"path/filepath"
"strings"
@@ -36,18 +37,34 @@ func (e *AssetError) Error() string { return e.Err.Error() }
func (e *AssetError) Unwrap() error { return e.Err }
func NewRuntime(ctx context.Context, c config.Config, mainPath string, client *http.Client) (*Runtime, error) {
return newRuntimeWithPromptFS(ctx, c, mainPath, client, prompts.Filesystem())
}
func newRuntimeWithPromptFS(ctx context.Context, c config.Config, mainPath string, client *http.Client, promptFS fs.FS) (*Runtime, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
profileDir := c.ProfilesDir
if !filepath.IsAbs(profileDir) {
profileDir = filepath.Join(filepath.Dir(mainPath), profileDir)
}
promptInputs, err := pk.FilesystemInputs("Audita embedded prompts", prompts.Filesystem(), prompts.AssetRoot)
promptInputs, err := pk.FilesystemInputs("Audita embedded prompts", promptFS, prompts.AssetRoot)
if err != nil {
return nil, fmt.Errorf("discover embedded prompts: %w", err)
if ctxErr := ctx.Err(); ctxErr != nil {
return nil, ctxErr
}
return nil, &AssetError{Err: fmt.Errorf("discover embedded prompts: %w", err)}
}
profileInputs, err := pk.DirectoryInputs(profileDir)
if err != nil {
if ctxErr := ctx.Err(); ctxErr != nil {
return nil, ctxErr
}
return nil, fmt.Errorf("discover operator definitions: %w", err)
}
if err := ctx.Err(); err != nil {
return nil, err
}
applicationBackends := make(map[string]any, len(c.Backends))
for id := range c.Backends {
backend := c.Backends[id]
@@ -71,11 +88,12 @@ func NewRuntime(ctx context.Context, c config.Config, mainPath string, client *h
TransientRetries: &c.LLM.TransientRetries,
}, pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot))
if err != nil {
return nil, fmt.Errorf("assemble LLM runtime: %w", err)
return nil, embeddedAssetOrInputError(ctx, promptInputs, fmt.Errorf("assemble LLM runtime: %w", err))
}
for _, meta := range prompts.RegisteredMetadata() {
if _, err := engine.InspectPrompt(ctx, meta.PromptID, meta.PromptVersion); err != nil {
return nil, &AssetError{Err: fmt.Errorf("inspect embedded prompt %s: %w", meta.PromptID, err)}
cause := fmt.Errorf("inspect prompt %s: %w", meta.PromptID, err)
return nil, embeddedAssetOrInputError(ctx, promptInputs, cause)
}
}
runtime := &Runtime{Config: c, Engine: engine, Profiles: make(map[string]pk.ProfileInspection, 2)}
@@ -110,6 +128,32 @@ func NewRuntime(ctx context.Context, c config.Config, mainPath string, client *h
return runtime, nil
}
func embeddedAssetOrInputError(ctx context.Context, promptInputs []ck.Input, cause error) error {
if err := ctx.Err(); err != nil {
return err
}
embeddedSources := pk.ConfigurationSources{
Underlays: []ck.Underlay{{Inputs: promptInputs}},
}
diagnosticEngine, err := pk.NewEngine(pk.Config{Sources: embeddedSources},
pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot))
if err == nil {
for _, meta := range prompts.RegisteredMetadata() {
if _, inspectErr := diagnosticEngine.InspectPrompt(ctx, meta.PromptID, meta.PromptVersion); inspectErr != nil {
err = fmt.Errorf("inspect embedded prompt %s: %w", meta.PromptID, inspectErr)
break
}
}
}
if ctxErr := ctx.Err(); ctxErr != nil {
return ctxErr
}
if err != nil {
return &AssetError{Err: fmt.Errorf("validate embedded prompt assets: %w", err)}
}
return cause
}
func IsAssetError(err error) bool { var target *AssetError; return errors.As(err, &target) }
// RedactError replaces every configured resolved backend secret before a

217
internal/framework/llm/runtime_assembly_test.go Normal file
View File

@@ -0,0 +1,217 @@
package llm
import (
"context"
"errors"
"io/fs"
"net/http"
"os"
"path"
"path/filepath"
"strings"
"testing"
"testing/fstest"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/prompts"
)
func runtimeAssemblyFixture(t *testing.T, selections string, documents map[string]string) (config.Config, string) {
t.Helper()
dir := t.TempDir()
profileDir := filepath.Join(dir, "profiles")
if err := os.Mkdir(profileDir, 0700); err != nil {
t.Fatal(err)
}
main := filepath.Join(dir, "config.yml")
text := "version: 2\nprofiles_dir: profiles\nbackends:\n offline:\n endpoint: http://127.0.0.1:1/v1\n concurrency: 1\n queue_capacity: 0\nllm:\n" + selections
if err := os.WriteFile(main, []byte(text), 0600); err != nil {
t.Fatal(err)
}
for name, content := range documents {
if err := os.WriteFile(filepath.Join(profileDir, name), []byte(content), 0600); err != nil {
t.Fatal(err)
}
}
loaded, err := config.Load(ck.File(main), "")
if err != nil {
t.Fatal(err)
}
return loaded.Config, main
}
func copyEmbeddedPromptFilesystem(t *testing.T) fstest.MapFS {
t.Helper()
source := prompts.Filesystem()
files := make(fstest.MapFS)
err := fs.WalkDir(source, prompts.AssetRoot, func(name string, entry fs.DirEntry, err error) error {
if err != nil || entry.IsDir() {
return err
}
data, err := fs.ReadFile(source, name)
if err != nil {
return err
}
files[name] = &fstest.MapFile{Data: data}
return nil
})
if err != nil {
t.Fatal(err)
}
return files
}
func TestNewRuntimeRequiresCompleteSelectedProfiles(t *testing.T) {
validProposal := " backend: offline\n model: proposal-model\n"
cases := []struct {
name string
selections string
profiles string
role string
field string
}{
{
name: "omitted proposal model with derived validation selection",
selections: " proposal_profile: proposal\n",
profiles: "profiles:\n proposal:\n backend: offline\n",
role: "proposal",
field: "model",
},
{
name: "empty validation model",
selections: " proposal_profile: proposal\n validation_profile: validation\n",
profiles: "profiles:\n proposal:\n" + validProposal + " validation:\n backend: offline\n model: \"\"\n",
role: "validation",
field: "model",
},
{
name: "null proposal model",
selections: " proposal_profile: proposal\n",
profiles: "profiles:\n proposal:\n backend: offline\n model: null\n",
role: "proposal",
field: "model",
},
{
name: "cleared validation endpoint",
selections: " proposal_profile: proposal\n validation_profile: validation\n",
profiles: "profiles:\n proposal:\n" + validProposal + " validation:\n backend: offline\n endpoint: null\n model: validation-model\n",
role: "validation",
field: "endpoint",
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
cfg, main := runtimeAssemblyFixture(t, tc.selections, map[string]string{"definitions.yml": tc.profiles})
_, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
if err == nil || !strings.Contains(err.Error(), tc.role) || !strings.Contains(err.Error(), tc.field) {
t.Fatalf("selected incomplete profile was not rejected for %s.%s: %v", tc.role, tc.field, err)
}
})
}
}
func TestNewRuntimeAllowsUnselectedPartialProfiles(t *testing.T) {
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n unfinished:\n backend: offline\n",
})
runtime, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
if err != nil {
t.Fatal(err)
}
if runtime.Profiles["proposal"].EffectiveModelParams.Model != "proposal-model" || runtime.Profiles["validation"].EffectiveModelParams.Model != "proposal-model" {
t.Fatalf("role profiles were not resolved: %+v", runtime.Profiles)
}
}
func TestMissingSelectedProfileDirectoryIsOperatorInputError(t *testing.T) {
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n",
})
if err := os.RemoveAll(filepath.Join(filepath.Dir(main), cfg.ProfilesDir)); err != nil {
t.Fatal(err)
}
_, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
if err == nil || IsAssetError(err) {
t.Fatalf("missing operator profile directory classification: %v", err)
}
}
func TestNewRuntimeRejectsMalformedUnselectedProfilesAndConflicts(t *testing.T) {
cases := []struct {
name string
document map[string]string
selection string
}{
{
name: "malformed unselected definition",
document: map[string]string{"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n unused:\n temperature: 9\n"},
selection: " proposal_profile: proposal\n",
},
{
name: "undefined selected backend",
document: map[string]string{"definitions.yml": "profiles:\n proposal:\n backend: missing\n model: proposal-model\n"},
selection: " proposal_profile: proposal\n",
},
{
name: "conflicting directory contributions",
document: map[string]string{
"a.yml": "profiles:\n proposal:\n backend: offline\n model: first-model\n",
"b.yml": "profiles:\n proposal:\n model: second-model\n",
},
selection: " proposal_profile: proposal\n",
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
cfg, main := runtimeAssemblyFixture(t, tc.selection, tc.document)
_, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
if err == nil || IsAssetError(err) {
t.Fatalf("operator definition classification: %v", err)
}
})
}
}
func TestRuntimeClassifiesEmbeddedAssetsAndHonorsCancellation(t *testing.T) {
_, err := newRuntimeWithPromptFS(context.Background(), config.Config{}, "config.yml", nil, fstest.MapFS{})
if err == nil || !IsAssetError(err) || !errors.Is(err, fs.ErrNotExist) {
t.Fatalf("embedded discovery error was not preserved as an asset error: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, err = newRuntimeWithPromptFS(ctx, config.Config{}, "config.yml", nil, fstest.MapFS{})
if !errors.Is(err, context.Canceled) || IsAssetError(err) {
t.Fatalf("caller cancellation classification: %v", err)
}
}
func TestBrokenEmbeddedPromptFailsEmbeddedOnlyConstruction(t *testing.T) {
promptFS := copyEmbeddedPromptFilesystem(t)
assetPath := path.Join(prompts.AssetRoot, prompts.PromptIDModuleGrammarProposal+".yaml")
promptFS[assetPath] = &fstest.MapFile{Data: []byte("prompts:\n modules.grammar.proposal:\n v1:\n messages: []\n output:\n format: json\n validation_mode: json_schema\n schema_path: correction_set.v1.json\n")}
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n",
})
_, err := newRuntimeWithPromptFS(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}, promptFS)
if err == nil || !IsAssetError(err) {
t.Fatalf("broken embedded prompt was not classified as an asset error: %v", err)
}
}
func TestOperatorPromptCanRepairInvalidEmbeddedPrompt(t *testing.T) {
promptFS := copyEmbeddedPromptFilesystem(t)
assetPath := path.Join(prompts.AssetRoot, prompts.PromptIDModuleGrammarProposal+".yaml")
promptFS[assetPath] = &fstest.MapFile{Data: []byte("prompts:\n modules.grammar.proposal:\n v1:\n messages: []\n output:\n format: json\n validation_mode: json_schema\n schema_path: correction_set.v1.json\n")}
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
"operator.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\nprompts:\n modules.grammar.proposal:\n v1:\n messages:\n - role: system\n content: operator system\n - role: user\n content: operator user\n",
})
runtime, err := newRuntimeWithPromptFS(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}, promptFS)
if err != nil {
t.Fatalf("valid final configuration was rejected with an invalid displaced embedded prompt: %v", err)
}
if runtime.Engine == nil {
t.Fatal("runtime engine was not retained")
}
}