Classify PromptKit asset and operator failures

docs/roadmap/implementation.md

@@ -1,7 +1,7 @@
 # Implementation Plan: PromptKit v0.12.0 Adoption
 
 **Last updated:** 2026-10-03
-**Status:** In progress; Stages 1–3 completed.
+**Status:** In progress; Stages 1–4 completed.
 **Audience:** Luna coding agent.
 
 ## Scope and Execution
@@ -253,6 +253,14 @@ startup path needs existing exit behavior preserved.
 `go test ./internal/cli`; check whitespace and commit. Preserve the original
 failure cause where available rather than flattening it to a generic error.
 
+**Completion marker:** Stage 4 complete. Embedded discovery and construction
+failures are distinguished from operator inputs through an embedded-only
+diagnostic construction; cancellation skips that diagnostic. Selected-profile
+completeness and native directory validation are covered, and both config
+commands preserve exit 1 for cancellation/assets and exit 2 for invalid
+operator YAML, templates, and resources. Targeted runtime tests,
+`go test ./internal/cli`, and `git diff --check` passed.
+
 ## Stage 5 — Protect Consumed Configuration and Message Files
 
 **Primary files:** `internal/framework/llm/runtime.go` or a focused source-path

internal/cli/run.go

@@ -97,7 +97,7 @@ func runConfigValidate(ctx context.Context, args []string, stdout, stderr io.Wri
 	}
 	if _, err := llm.NewRuntime(ctx, selected.Result.Config, selected.Path, nil); err != nil {
 		fmt.Fprintf(stderr, "audita config validate: %s\n", llm.RedactError(err, selected.Result.Config))
-		if llm.IsAssetError(err) {
+		if ctx.Err() != nil || llm.IsAssetError(err) {
 			return 1
 		}
 		return 2
@@ -124,7 +124,7 @@ func runConfigPrintEffective(ctx context.Context, args []string, stdout, stderr
 	llmRuntime, err := llm.NewRuntime(ctx, selected.Result.Config, selected.Path, nil)
 	if err != nil {
 		fmt.Fprintf(stderr, "audita config print-effective: %s\n", llm.RedactError(err, selected.Result.Config))
-		if llm.IsAssetError(err) {
+		if ctx.Err() != nil || llm.IsAssetError(err) {
 			return 1
 		}
 		return 2

internal/cli/runtime_errors_test.go

@@ -0,0 +1,76 @@
+package cli
+
+import (
+	"bytes"
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func writeRuntimeErrorConfig(t *testing.T, operatorDocument string) string {
+	t.Helper()
+	dir := t.TempDir()
+	profiles := filepath.Join(dir, "profiles")
+	if err := os.Mkdir(profiles, 0700); err != nil {
+		t.Fatal(err)
+	}
+	main := filepath.Join(dir, "config.yml")
+	configText := "version: 2\nprofiles_dir: profiles\nbackends:\n  offline:\n    endpoint: http://127.0.0.1:1/v1\n    credential: AUDITA_TEST_PROVIDER_KEY\nllm:\n  proposal_profile: proposal\n"
+	if err := os.WriteFile(main, []byte(configText), 0600); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(profiles, "operator.yml"), []byte(operatorDocument), 0600); err != nil {
+		t.Fatal(err)
+	}
+	return main
+}
+
+func TestConfigCommandsClassifyOperatorPromptErrorsAsInputErrors(t *testing.T) {
+	t.Setenv("AUDITA_TEST_PROVIDER_KEY", "synthetic-cli-redaction-secret")
+	definitions := map[string]string{
+		"malformed operator YAML":           "profiles:\n  proposal: [\n",
+		"invalid operator template":         "profiles:\n  proposal:\n    backend: offline\n    model: local-model\nprompts:\n  modules.grammar.proposal:\n    v1:\n      messages:\n        - role: user\n          content: '{{'\n",
+		"missing operator message resource": "profiles:\n  proposal:\n    backend: offline\n    model: local-model\nprompts:\n  modules.grammar.proposal:\n    v1:\n      messages:\n        - role: user\n          content_file: missing-instruction.md\n",
+	}
+	commands := []struct {
+		name string
+		args []string
+	}{
+		{name: "validate", args: []string{"config", "validate"}},
+		{name: "print effective", args: []string{"config", "print-effective", "--view", "profiles"}},
+	}
+	for definition, document := range definitions {
+		for _, command := range commands {
+			t.Run(definition+"/"+command.name, func(t *testing.T) {
+				main := writeRuntimeErrorConfig(t, document)
+				args := append(append([]string(nil), command.args...), "--config", main)
+				var stdout, stderr bytes.Buffer
+				if code := Run(context.Background(), args, &stdout, &stderr); code != 2 {
+					t.Fatalf("exit=%d, stdout=%q, stderr=%q", code, stdout.String(), stderr.String())
+				}
+				if strings.Contains(stderr.String(), "synthetic-cli-redaction-secret") {
+					t.Fatalf("configured secret leaked: %q", stderr.String())
+				}
+			})
+		}
+	}
+}
+
+func TestConfigCommandsKeepCallerCancellationAsRuntimeExit(t *testing.T) {
+	t.Setenv("AUDITA_TEST_PROVIDER_KEY", "synthetic-cli-redaction-secret")
+	main := writeRuntimeErrorConfig(t, "profiles:\n  proposal:\n    backend: offline\n    model: local-model\n")
+	commands := [][]string{
+		{"config", "validate", "--config", main},
+		{"config", "print-effective", "--config", main, "--view", "profiles"},
+	}
+	for _, args := range commands {
+		ctx, cancel := context.WithCancel(context.Background())
+		cancel()
+		var stdout, stderr bytes.Buffer
+		if code := Run(ctx, args, &stdout, &stderr); code != 1 {
+			t.Fatalf("args=%v exit=%d, stdout=%q, stderr=%q", args, code, stdout.String(), stderr.String())
+		}
+	}
+}

internal/framework/llm/runtime.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io/fs"
 	"net/http"
 	"path/filepath"
 	"strings"
@@ -36,18 +37,34 @@ func (e *AssetError) Error() string { return e.Err.Error() }
 func (e *AssetError) Unwrap() error { return e.Err }
 
 func NewRuntime(ctx context.Context, c config.Config, mainPath string, client *http.Client) (*Runtime, error) {
+	return newRuntimeWithPromptFS(ctx, c, mainPath, client, prompts.Filesystem())
+}
+
+func newRuntimeWithPromptFS(ctx context.Context, c config.Config, mainPath string, client *http.Client, promptFS fs.FS) (*Runtime, error) {
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
 	profileDir := c.ProfilesDir
 	if !filepath.IsAbs(profileDir) {
 		profileDir = filepath.Join(filepath.Dir(mainPath), profileDir)
 	}
-	promptInputs, err := pk.FilesystemInputs("Audita embedded prompts", prompts.Filesystem(), prompts.AssetRoot)
+	promptInputs, err := pk.FilesystemInputs("Audita embedded prompts", promptFS, prompts.AssetRoot)
 	if err != nil {
-		return nil, fmt.Errorf("discover embedded prompts: %w", err)
+		if ctxErr := ctx.Err(); ctxErr != nil {
+			return nil, ctxErr
+		}
+		return nil, &AssetError{Err: fmt.Errorf("discover embedded prompts: %w", err)}
 	}
 	profileInputs, err := pk.DirectoryInputs(profileDir)
 	if err != nil {
+		if ctxErr := ctx.Err(); ctxErr != nil {
+			return nil, ctxErr
+		}
 		return nil, fmt.Errorf("discover operator definitions: %w", err)
 	}
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
 	applicationBackends := make(map[string]any, len(c.Backends))
 	for id := range c.Backends {
 		backend := c.Backends[id]
@@ -71,11 +88,12 @@ func NewRuntime(ctx context.Context, c config.Config, mainPath string, client *h
 		TransientRetries: &c.LLM.TransientRetries,
 	}, pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot))
 	if err != nil {
-		return nil, fmt.Errorf("assemble LLM runtime: %w", err)
+		return nil, embeddedAssetOrInputError(ctx, promptInputs, fmt.Errorf("assemble LLM runtime: %w", err))
 	}
 	for _, meta := range prompts.RegisteredMetadata() {
 		if _, err := engine.InspectPrompt(ctx, meta.PromptID, meta.PromptVersion); err != nil {
-			return nil, &AssetError{Err: fmt.Errorf("inspect embedded prompt %s: %w", meta.PromptID, err)}
+			cause := fmt.Errorf("inspect prompt %s: %w", meta.PromptID, err)
+			return nil, embeddedAssetOrInputError(ctx, promptInputs, cause)
 		}
 	}
 	runtime := &Runtime{Config: c, Engine: engine, Profiles: make(map[string]pk.ProfileInspection, 2)}
@@ -110,6 +128,32 @@ func NewRuntime(ctx context.Context, c config.Config, mainPath string, client *h
 	return runtime, nil
 }
 
+func embeddedAssetOrInputError(ctx context.Context, promptInputs []ck.Input, cause error) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	embeddedSources := pk.ConfigurationSources{
+		Underlays: []ck.Underlay{{Inputs: promptInputs}},
+	}
+	diagnosticEngine, err := pk.NewEngine(pk.Config{Sources: embeddedSources},
+		pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot))
+	if err == nil {
+		for _, meta := range prompts.RegisteredMetadata() {
+			if _, inspectErr := diagnosticEngine.InspectPrompt(ctx, meta.PromptID, meta.PromptVersion); inspectErr != nil {
+				err = fmt.Errorf("inspect embedded prompt %s: %w", meta.PromptID, inspectErr)
+				break
+			}
+		}
+	}
+	if ctxErr := ctx.Err(); ctxErr != nil {
+		return ctxErr
+	}
+	if err != nil {
+		return &AssetError{Err: fmt.Errorf("validate embedded prompt assets: %w", err)}
+	}
+	return cause
+}
+
 func IsAssetError(err error) bool { var target *AssetError; return errors.As(err, &target) }
 
 // RedactError replaces every configured resolved backend secret before a

internal/framework/llm/runtime_assembly_test.go

@@ -0,0 +1,217 @@
+package llm
+
+import (
+	"context"
+	"errors"
+	"io/fs"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"testing"
+	"testing/fstest"
+
+	ck "gitea.maximumdirect.net/eric/ConfigKit"
+	"gitea.maximumdirect.net/eric/audita/internal/core/config"
+	"gitea.maximumdirect.net/eric/audita/internal/prompts"
+)
+
+func runtimeAssemblyFixture(t *testing.T, selections string, documents map[string]string) (config.Config, string) {
+	t.Helper()
+	dir := t.TempDir()
+	profileDir := filepath.Join(dir, "profiles")
+	if err := os.Mkdir(profileDir, 0700); err != nil {
+		t.Fatal(err)
+	}
+	main := filepath.Join(dir, "config.yml")
+	text := "version: 2\nprofiles_dir: profiles\nbackends:\n  offline:\n    endpoint: http://127.0.0.1:1/v1\n    concurrency: 1\n    queue_capacity: 0\nllm:\n" + selections
+	if err := os.WriteFile(main, []byte(text), 0600); err != nil {
+		t.Fatal(err)
+	}
+	for name, content := range documents {
+		if err := os.WriteFile(filepath.Join(profileDir, name), []byte(content), 0600); err != nil {
+			t.Fatal(err)
+		}
+	}
+	loaded, err := config.Load(ck.File(main), "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return loaded.Config, main
+}
+
+func copyEmbeddedPromptFilesystem(t *testing.T) fstest.MapFS {
+	t.Helper()
+	source := prompts.Filesystem()
+	files := make(fstest.MapFS)
+	err := fs.WalkDir(source, prompts.AssetRoot, func(name string, entry fs.DirEntry, err error) error {
+		if err != nil || entry.IsDir() {
+			return err
+		}
+		data, err := fs.ReadFile(source, name)
+		if err != nil {
+			return err
+		}
+		files[name] = &fstest.MapFile{Data: data}
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return files
+}
+
+func TestNewRuntimeRequiresCompleteSelectedProfiles(t *testing.T) {
+	validProposal := "    backend: offline\n    model: proposal-model\n"
+	cases := []struct {
+		name       string
+		selections string
+		profiles   string
+		role       string
+		field      string
+	}{
+		{
+			name:       "omitted proposal model with derived validation selection",
+			selections: "  proposal_profile: proposal\n",
+			profiles:   "profiles:\n  proposal:\n    backend: offline\n",
+			role:       "proposal",
+			field:      "model",
+		},
+		{
+			name:       "empty validation model",
+			selections: "  proposal_profile: proposal\n  validation_profile: validation\n",
+			profiles:   "profiles:\n  proposal:\n" + validProposal + "  validation:\n    backend: offline\n    model: \"\"\n",
+			role:       "validation",
+			field:      "model",
+		},
+		{
+			name:       "null proposal model",
+			selections: "  proposal_profile: proposal\n",
+			profiles:   "profiles:\n  proposal:\n    backend: offline\n    model: null\n",
+			role:       "proposal",
+			field:      "model",
+		},
+		{
+			name:       "cleared validation endpoint",
+			selections: "  proposal_profile: proposal\n  validation_profile: validation\n",
+			profiles:   "profiles:\n  proposal:\n" + validProposal + "  validation:\n    backend: offline\n    endpoint: null\n    model: validation-model\n",
+			role:       "validation",
+			field:      "endpoint",
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			cfg, main := runtimeAssemblyFixture(t, tc.selections, map[string]string{"definitions.yml": tc.profiles})
+			_, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
+			if err == nil || !strings.Contains(err.Error(), tc.role) || !strings.Contains(err.Error(), tc.field) {
+				t.Fatalf("selected incomplete profile was not rejected for %s.%s: %v", tc.role, tc.field, err)
+			}
+		})
+	}
+}
+
+func TestNewRuntimeAllowsUnselectedPartialProfiles(t *testing.T) {
+	cfg, main := runtimeAssemblyFixture(t, "  proposal_profile: proposal\n", map[string]string{
+		"definitions.yml": "profiles:\n  proposal:\n    backend: offline\n    model: proposal-model\n  unfinished:\n    backend: offline\n",
+	})
+	runtime, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if runtime.Profiles["proposal"].EffectiveModelParams.Model != "proposal-model" || runtime.Profiles["validation"].EffectiveModelParams.Model != "proposal-model" {
+		t.Fatalf("role profiles were not resolved: %+v", runtime.Profiles)
+	}
+}
+
+func TestMissingSelectedProfileDirectoryIsOperatorInputError(t *testing.T) {
+	cfg, main := runtimeAssemblyFixture(t, "  proposal_profile: proposal\n", map[string]string{
+		"definitions.yml": "profiles:\n  proposal:\n    backend: offline\n    model: proposal-model\n",
+	})
+	if err := os.RemoveAll(filepath.Join(filepath.Dir(main), cfg.ProfilesDir)); err != nil {
+		t.Fatal(err)
+	}
+	_, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
+	if err == nil || IsAssetError(err) {
+		t.Fatalf("missing operator profile directory classification: %v", err)
+	}
+}
+
+func TestNewRuntimeRejectsMalformedUnselectedProfilesAndConflicts(t *testing.T) {
+	cases := []struct {
+		name      string
+		document  map[string]string
+		selection string
+	}{
+		{
+			name:      "malformed unselected definition",
+			document:  map[string]string{"definitions.yml": "profiles:\n  proposal:\n    backend: offline\n    model: proposal-model\n  unused:\n    temperature: 9\n"},
+			selection: "  proposal_profile: proposal\n",
+		},
+		{
+			name:      "undefined selected backend",
+			document:  map[string]string{"definitions.yml": "profiles:\n  proposal:\n    backend: missing\n    model: proposal-model\n"},
+			selection: "  proposal_profile: proposal\n",
+		},
+		{
+			name: "conflicting directory contributions",
+			document: map[string]string{
+				"a.yml": "profiles:\n  proposal:\n    backend: offline\n    model: first-model\n",
+				"b.yml": "profiles:\n  proposal:\n    model: second-model\n",
+			},
+			selection: "  proposal_profile: proposal\n",
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			cfg, main := runtimeAssemblyFixture(t, tc.selection, tc.document)
+			_, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
+			if err == nil || IsAssetError(err) {
+				t.Fatalf("operator definition classification: %v", err)
+			}
+		})
+	}
+}
+
+func TestRuntimeClassifiesEmbeddedAssetsAndHonorsCancellation(t *testing.T) {
+	_, err := newRuntimeWithPromptFS(context.Background(), config.Config{}, "config.yml", nil, fstest.MapFS{})
+	if err == nil || !IsAssetError(err) || !errors.Is(err, fs.ErrNotExist) {
+		t.Fatalf("embedded discovery error was not preserved as an asset error: %v", err)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	_, err = newRuntimeWithPromptFS(ctx, config.Config{}, "config.yml", nil, fstest.MapFS{})
+	if !errors.Is(err, context.Canceled) || IsAssetError(err) {
+		t.Fatalf("caller cancellation classification: %v", err)
+	}
+}
+
+func TestBrokenEmbeddedPromptFailsEmbeddedOnlyConstruction(t *testing.T) {
+	promptFS := copyEmbeddedPromptFilesystem(t)
+	assetPath := path.Join(prompts.AssetRoot, prompts.PromptIDModuleGrammarProposal+".yaml")
+	promptFS[assetPath] = &fstest.MapFile{Data: []byte("prompts:\n  modules.grammar.proposal:\n    v1:\n      messages: []\n      output:\n        format: json\n        validation_mode: json_schema\n        schema_path: correction_set.v1.json\n")}
+	cfg, main := runtimeAssemblyFixture(t, "  proposal_profile: proposal\n", map[string]string{
+		"definitions.yml": "profiles:\n  proposal:\n    backend: offline\n    model: proposal-model\n",
+	})
+	_, err := newRuntimeWithPromptFS(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}, promptFS)
+	if err == nil || !IsAssetError(err) {
+		t.Fatalf("broken embedded prompt was not classified as an asset error: %v", err)
+	}
+}
+
+func TestOperatorPromptCanRepairInvalidEmbeddedPrompt(t *testing.T) {
+	promptFS := copyEmbeddedPromptFilesystem(t)
+	assetPath := path.Join(prompts.AssetRoot, prompts.PromptIDModuleGrammarProposal+".yaml")
+	promptFS[assetPath] = &fstest.MapFile{Data: []byte("prompts:\n  modules.grammar.proposal:\n    v1:\n      messages: []\n      output:\n        format: json\n        validation_mode: json_schema\n        schema_path: correction_set.v1.json\n")}
+	cfg, main := runtimeAssemblyFixture(t, "  proposal_profile: proposal\n", map[string]string{
+		"operator.yml": "profiles:\n  proposal:\n    backend: offline\n    model: proposal-model\nprompts:\n  modules.grammar.proposal:\n    v1:\n      messages:\n        - role: system\n          content: operator system\n        - role: user\n          content: operator user\n",
+	})
+	runtime, err := newRuntimeWithPromptFS(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}, promptFS)
+	if err != nil {
+		t.Fatalf("valid final configuration was rejected with an invalid displaced embedded prompt: %v", err)
+	}
+	if runtime.Engine == nil {
+		t.Fatal("runtime engine was not retained")
+	}
+}