diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 80b2854..228fa6d 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1,7 +1,7 @@ # Implementation Plan: PromptKit v0.12.0 Adoption **Last updated:** 2026-10-03 -**Status:** In progress; Stages 1–3 completed. +**Status:** In progress; Stages 1–4 completed. **Audience:** Luna coding agent. ## Scope and Execution @@ -253,6 +253,14 @@ startup path needs existing exit behavior preserved. `go test ./internal/cli`; check whitespace and commit. Preserve the original failure cause where available rather than flattening it to a generic error. +**Completion marker:** Stage 4 complete. Embedded discovery and construction +failures are distinguished from operator inputs through an embedded-only +diagnostic construction; cancellation skips that diagnostic. Selected-profile +completeness and native directory validation are covered, and both config +commands preserve exit 1 for cancellation/assets and exit 2 for invalid +operator YAML, templates, and resources. Targeted runtime tests, +`go test ./internal/cli`, and `git diff --check` passed. + ## Stage 5 — Protect Consumed Configuration and Message Files **Primary files:** `internal/framework/llm/runtime.go` or a focused source-path diff --git a/internal/cli/run.go b/internal/cli/run.go index c2cff64..f89dc88 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -97,7 +97,7 @@ func runConfigValidate(ctx context.Context, args []string, stdout, stderr io.Wri } if _, err := llm.NewRuntime(ctx, selected.Result.Config, selected.Path, nil); err != nil { fmt.Fprintf(stderr, "audita config validate: %s\n", llm.RedactError(err, selected.Result.Config)) - if llm.IsAssetError(err) { + if ctx.Err() != nil || llm.IsAssetError(err) { return 1 } return 2 @@ -124,7 +124,7 @@ func runConfigPrintEffective(ctx context.Context, args []string, stdout, stderr llmRuntime, err := llm.NewRuntime(ctx, selected.Result.Config, selected.Path, nil) if err != nil { fmt.Fprintf(stderr, "audita config print-effective: %s\n", llm.RedactError(err, selected.Result.Config)) - if llm.IsAssetError(err) { + if ctx.Err() != nil || llm.IsAssetError(err) { return 1 } return 2 diff --git a/internal/cli/runtime_errors_test.go b/internal/cli/runtime_errors_test.go new file mode 100644 index 0000000..4d334a3 --- /dev/null +++ b/internal/cli/runtime_errors_test.go @@ -0,0 +1,76 @@ +package cli + +import ( + "bytes" + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +func writeRuntimeErrorConfig(t *testing.T, operatorDocument string) string { + t.Helper() + dir := t.TempDir() + profiles := filepath.Join(dir, "profiles") + if err := os.Mkdir(profiles, 0700); err != nil { + t.Fatal(err) + } + main := filepath.Join(dir, "config.yml") + configText := "version: 2\nprofiles_dir: profiles\nbackends:\n offline:\n endpoint: http://127.0.0.1:1/v1\n credential: AUDITA_TEST_PROVIDER_KEY\nllm:\n proposal_profile: proposal\n" + if err := os.WriteFile(main, []byte(configText), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(profiles, "operator.yml"), []byte(operatorDocument), 0600); err != nil { + t.Fatal(err) + } + return main +} + +func TestConfigCommandsClassifyOperatorPromptErrorsAsInputErrors(t *testing.T) { + t.Setenv("AUDITA_TEST_PROVIDER_KEY", "synthetic-cli-redaction-secret") + definitions := map[string]string{ + "malformed operator YAML": "profiles:\n proposal: [\n", + "invalid operator template": "profiles:\n proposal:\n backend: offline\n model: local-model\nprompts:\n modules.grammar.proposal:\n v1:\n messages:\n - role: user\n content: '{{'\n", + "missing operator message resource": "profiles:\n proposal:\n backend: offline\n model: local-model\nprompts:\n modules.grammar.proposal:\n v1:\n messages:\n - role: user\n content_file: missing-instruction.md\n", + } + commands := []struct { + name string + args []string + }{ + {name: "validate", args: []string{"config", "validate"}}, + {name: "print effective", args: []string{"config", "print-effective", "--view", "profiles"}}, + } + for definition, document := range definitions { + for _, command := range commands { + t.Run(definition+"/"+command.name, func(t *testing.T) { + main := writeRuntimeErrorConfig(t, document) + args := append(append([]string(nil), command.args...), "--config", main) + var stdout, stderr bytes.Buffer + if code := Run(context.Background(), args, &stdout, &stderr); code != 2 { + t.Fatalf("exit=%d, stdout=%q, stderr=%q", code, stdout.String(), stderr.String()) + } + if strings.Contains(stderr.String(), "synthetic-cli-redaction-secret") { + t.Fatalf("configured secret leaked: %q", stderr.String()) + } + }) + } + } +} + +func TestConfigCommandsKeepCallerCancellationAsRuntimeExit(t *testing.T) { + t.Setenv("AUDITA_TEST_PROVIDER_KEY", "synthetic-cli-redaction-secret") + main := writeRuntimeErrorConfig(t, "profiles:\n proposal:\n backend: offline\n model: local-model\n") + commands := [][]string{ + {"config", "validate", "--config", main}, + {"config", "print-effective", "--config", main, "--view", "profiles"}, + } + for _, args := range commands { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + var stdout, stderr bytes.Buffer + if code := Run(ctx, args, &stdout, &stderr); code != 1 { + t.Fatalf("args=%v exit=%d, stdout=%q, stderr=%q", args, code, stdout.String(), stderr.String()) + } + } +} diff --git a/internal/framework/llm/runtime.go b/internal/framework/llm/runtime.go index 471d6b5..0b8ac53 100644 --- a/internal/framework/llm/runtime.go +++ b/internal/framework/llm/runtime.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "io/fs" "net/http" "path/filepath" "strings" @@ -36,18 +37,34 @@ func (e *AssetError) Error() string { return e.Err.Error() } func (e *AssetError) Unwrap() error { return e.Err } func NewRuntime(ctx context.Context, c config.Config, mainPath string, client *http.Client) (*Runtime, error) { + return newRuntimeWithPromptFS(ctx, c, mainPath, client, prompts.Filesystem()) +} + +func newRuntimeWithPromptFS(ctx context.Context, c config.Config, mainPath string, client *http.Client, promptFS fs.FS) (*Runtime, error) { + if err := ctx.Err(); err != nil { + return nil, err + } profileDir := c.ProfilesDir if !filepath.IsAbs(profileDir) { profileDir = filepath.Join(filepath.Dir(mainPath), profileDir) } - promptInputs, err := pk.FilesystemInputs("Audita embedded prompts", prompts.Filesystem(), prompts.AssetRoot) + promptInputs, err := pk.FilesystemInputs("Audita embedded prompts", promptFS, prompts.AssetRoot) if err != nil { - return nil, fmt.Errorf("discover embedded prompts: %w", err) + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, ctxErr + } + return nil, &AssetError{Err: fmt.Errorf("discover embedded prompts: %w", err)} } profileInputs, err := pk.DirectoryInputs(profileDir) if err != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, ctxErr + } return nil, fmt.Errorf("discover operator definitions: %w", err) } + if err := ctx.Err(); err != nil { + return nil, err + } applicationBackends := make(map[string]any, len(c.Backends)) for id := range c.Backends { backend := c.Backends[id] @@ -71,11 +88,12 @@ func NewRuntime(ctx context.Context, c config.Config, mainPath string, client *h TransientRetries: &c.LLM.TransientRetries, }, pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot)) if err != nil { - return nil, fmt.Errorf("assemble LLM runtime: %w", err) + return nil, embeddedAssetOrInputError(ctx, promptInputs, fmt.Errorf("assemble LLM runtime: %w", err)) } for _, meta := range prompts.RegisteredMetadata() { if _, err := engine.InspectPrompt(ctx, meta.PromptID, meta.PromptVersion); err != nil { - return nil, &AssetError{Err: fmt.Errorf("inspect embedded prompt %s: %w", meta.PromptID, err)} + cause := fmt.Errorf("inspect prompt %s: %w", meta.PromptID, err) + return nil, embeddedAssetOrInputError(ctx, promptInputs, cause) } } runtime := &Runtime{Config: c, Engine: engine, Profiles: make(map[string]pk.ProfileInspection, 2)} @@ -110,6 +128,32 @@ func NewRuntime(ctx context.Context, c config.Config, mainPath string, client *h return runtime, nil } +func embeddedAssetOrInputError(ctx context.Context, promptInputs []ck.Input, cause error) error { + if err := ctx.Err(); err != nil { + return err + } + embeddedSources := pk.ConfigurationSources{ + Underlays: []ck.Underlay{{Inputs: promptInputs}}, + } + diagnosticEngine, err := pk.NewEngine(pk.Config{Sources: embeddedSources}, + pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot)) + if err == nil { + for _, meta := range prompts.RegisteredMetadata() { + if _, inspectErr := diagnosticEngine.InspectPrompt(ctx, meta.PromptID, meta.PromptVersion); inspectErr != nil { + err = fmt.Errorf("inspect embedded prompt %s: %w", meta.PromptID, inspectErr) + break + } + } + } + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } + if err != nil { + return &AssetError{Err: fmt.Errorf("validate embedded prompt assets: %w", err)} + } + return cause +} + func IsAssetError(err error) bool { var target *AssetError; return errors.As(err, &target) } // RedactError replaces every configured resolved backend secret before a diff --git a/internal/framework/llm/runtime_assembly_test.go b/internal/framework/llm/runtime_assembly_test.go new file mode 100644 index 0000000..c98e09b --- /dev/null +++ b/internal/framework/llm/runtime_assembly_test.go @@ -0,0 +1,217 @@ +package llm + +import ( + "context" + "errors" + "io/fs" + "net/http" + "os" + "path" + "path/filepath" + "strings" + "testing" + "testing/fstest" + + ck "gitea.maximumdirect.net/eric/ConfigKit" + "gitea.maximumdirect.net/eric/audita/internal/core/config" + "gitea.maximumdirect.net/eric/audita/internal/prompts" +) + +func runtimeAssemblyFixture(t *testing.T, selections string, documents map[string]string) (config.Config, string) { + t.Helper() + dir := t.TempDir() + profileDir := filepath.Join(dir, "profiles") + if err := os.Mkdir(profileDir, 0700); err != nil { + t.Fatal(err) + } + main := filepath.Join(dir, "config.yml") + text := "version: 2\nprofiles_dir: profiles\nbackends:\n offline:\n endpoint: http://127.0.0.1:1/v1\n concurrency: 1\n queue_capacity: 0\nllm:\n" + selections + if err := os.WriteFile(main, []byte(text), 0600); err != nil { + t.Fatal(err) + } + for name, content := range documents { + if err := os.WriteFile(filepath.Join(profileDir, name), []byte(content), 0600); err != nil { + t.Fatal(err) + } + } + loaded, err := config.Load(ck.File(main), "") + if err != nil { + t.Fatal(err) + } + return loaded.Config, main +} + +func copyEmbeddedPromptFilesystem(t *testing.T) fstest.MapFS { + t.Helper() + source := prompts.Filesystem() + files := make(fstest.MapFS) + err := fs.WalkDir(source, prompts.AssetRoot, func(name string, entry fs.DirEntry, err error) error { + if err != nil || entry.IsDir() { + return err + } + data, err := fs.ReadFile(source, name) + if err != nil { + return err + } + files[name] = &fstest.MapFile{Data: data} + return nil + }) + if err != nil { + t.Fatal(err) + } + return files +} + +func TestNewRuntimeRequiresCompleteSelectedProfiles(t *testing.T) { + validProposal := " backend: offline\n model: proposal-model\n" + cases := []struct { + name string + selections string + profiles string + role string + field string + }{ + { + name: "omitted proposal model with derived validation selection", + selections: " proposal_profile: proposal\n", + profiles: "profiles:\n proposal:\n backend: offline\n", + role: "proposal", + field: "model", + }, + { + name: "empty validation model", + selections: " proposal_profile: proposal\n validation_profile: validation\n", + profiles: "profiles:\n proposal:\n" + validProposal + " validation:\n backend: offline\n model: \"\"\n", + role: "validation", + field: "model", + }, + { + name: "null proposal model", + selections: " proposal_profile: proposal\n", + profiles: "profiles:\n proposal:\n backend: offline\n model: null\n", + role: "proposal", + field: "model", + }, + { + name: "cleared validation endpoint", + selections: " proposal_profile: proposal\n validation_profile: validation\n", + profiles: "profiles:\n proposal:\n" + validProposal + " validation:\n backend: offline\n endpoint: null\n model: validation-model\n", + role: "validation", + field: "endpoint", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cfg, main := runtimeAssemblyFixture(t, tc.selections, map[string]string{"definitions.yml": tc.profiles}) + _, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}) + if err == nil || !strings.Contains(err.Error(), tc.role) || !strings.Contains(err.Error(), tc.field) { + t.Fatalf("selected incomplete profile was not rejected for %s.%s: %v", tc.role, tc.field, err) + } + }) + } +} + +func TestNewRuntimeAllowsUnselectedPartialProfiles(t *testing.T) { + cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{ + "definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n unfinished:\n backend: offline\n", + }) + runtime, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}) + if err != nil { + t.Fatal(err) + } + if runtime.Profiles["proposal"].EffectiveModelParams.Model != "proposal-model" || runtime.Profiles["validation"].EffectiveModelParams.Model != "proposal-model" { + t.Fatalf("role profiles were not resolved: %+v", runtime.Profiles) + } +} + +func TestMissingSelectedProfileDirectoryIsOperatorInputError(t *testing.T) { + cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{ + "definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n", + }) + if err := os.RemoveAll(filepath.Join(filepath.Dir(main), cfg.ProfilesDir)); err != nil { + t.Fatal(err) + } + _, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}) + if err == nil || IsAssetError(err) { + t.Fatalf("missing operator profile directory classification: %v", err) + } +} + +func TestNewRuntimeRejectsMalformedUnselectedProfilesAndConflicts(t *testing.T) { + cases := []struct { + name string + document map[string]string + selection string + }{ + { + name: "malformed unselected definition", + document: map[string]string{"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n unused:\n temperature: 9\n"}, + selection: " proposal_profile: proposal\n", + }, + { + name: "undefined selected backend", + document: map[string]string{"definitions.yml": "profiles:\n proposal:\n backend: missing\n model: proposal-model\n"}, + selection: " proposal_profile: proposal\n", + }, + { + name: "conflicting directory contributions", + document: map[string]string{ + "a.yml": "profiles:\n proposal:\n backend: offline\n model: first-model\n", + "b.yml": "profiles:\n proposal:\n model: second-model\n", + }, + selection: " proposal_profile: proposal\n", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cfg, main := runtimeAssemblyFixture(t, tc.selection, tc.document) + _, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}) + if err == nil || IsAssetError(err) { + t.Fatalf("operator definition classification: %v", err) + } + }) + } +} + +func TestRuntimeClassifiesEmbeddedAssetsAndHonorsCancellation(t *testing.T) { + _, err := newRuntimeWithPromptFS(context.Background(), config.Config{}, "config.yml", nil, fstest.MapFS{}) + if err == nil || !IsAssetError(err) || !errors.Is(err, fs.ErrNotExist) { + t.Fatalf("embedded discovery error was not preserved as an asset error: %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err = newRuntimeWithPromptFS(ctx, config.Config{}, "config.yml", nil, fstest.MapFS{}) + if !errors.Is(err, context.Canceled) || IsAssetError(err) { + t.Fatalf("caller cancellation classification: %v", err) + } +} + +func TestBrokenEmbeddedPromptFailsEmbeddedOnlyConstruction(t *testing.T) { + promptFS := copyEmbeddedPromptFilesystem(t) + assetPath := path.Join(prompts.AssetRoot, prompts.PromptIDModuleGrammarProposal+".yaml") + promptFS[assetPath] = &fstest.MapFile{Data: []byte("prompts:\n modules.grammar.proposal:\n v1:\n messages: []\n output:\n format: json\n validation_mode: json_schema\n schema_path: correction_set.v1.json\n")} + cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{ + "definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n", + }) + _, err := newRuntimeWithPromptFS(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}, promptFS) + if err == nil || !IsAssetError(err) { + t.Fatalf("broken embedded prompt was not classified as an asset error: %v", err) + } +} + +func TestOperatorPromptCanRepairInvalidEmbeddedPrompt(t *testing.T) { + promptFS := copyEmbeddedPromptFilesystem(t) + assetPath := path.Join(prompts.AssetRoot, prompts.PromptIDModuleGrammarProposal+".yaml") + promptFS[assetPath] = &fstest.MapFile{Data: []byte("prompts:\n modules.grammar.proposal:\n v1:\n messages: []\n output:\n format: json\n validation_mode: json_schema\n schema_path: correction_set.v1.json\n")} + cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{ + "operator.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\nprompts:\n modules.grammar.proposal:\n v1:\n messages:\n - role: system\n content: operator system\n - role: user\n content: operator user\n", + }) + runtime, err := newRuntimeWithPromptFS(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}, promptFS) + if err != nil { + t.Fatalf("valid final configuration was rejected with an invalid displaced embedded prompt: %v", err) + } + if runtime.Engine == nil { + t.Fatal("runtime engine was not retained") + } +}