Classify PromptKit asset and operator failures

internal/framework/llm/runtime.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io/fs"
 	"net/http"
 	"path/filepath"
 	"strings"
@@ -36,18 +37,34 @@ func (e *AssetError) Error() string { return e.Err.Error() }
 func (e *AssetError) Unwrap() error { return e.Err }
 
 func NewRuntime(ctx context.Context, c config.Config, mainPath string, client *http.Client) (*Runtime, error) {
+	return newRuntimeWithPromptFS(ctx, c, mainPath, client, prompts.Filesystem())
+}
+
+func newRuntimeWithPromptFS(ctx context.Context, c config.Config, mainPath string, client *http.Client, promptFS fs.FS) (*Runtime, error) {
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
 	profileDir := c.ProfilesDir
 	if !filepath.IsAbs(profileDir) {
 		profileDir = filepath.Join(filepath.Dir(mainPath), profileDir)
 	}
-	promptInputs, err := pk.FilesystemInputs("Audita embedded prompts", prompts.Filesystem(), prompts.AssetRoot)
+	promptInputs, err := pk.FilesystemInputs("Audita embedded prompts", promptFS, prompts.AssetRoot)
 	if err != nil {
-		return nil, fmt.Errorf("discover embedded prompts: %w", err)
+		if ctxErr := ctx.Err(); ctxErr != nil {
+			return nil, ctxErr
+		}
+		return nil, &AssetError{Err: fmt.Errorf("discover embedded prompts: %w", err)}
 	}
 	profileInputs, err := pk.DirectoryInputs(profileDir)
 	if err != nil {
+		if ctxErr := ctx.Err(); ctxErr != nil {
+			return nil, ctxErr
+		}
 		return nil, fmt.Errorf("discover operator definitions: %w", err)
 	}
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
 	applicationBackends := make(map[string]any, len(c.Backends))
 	for id := range c.Backends {
 		backend := c.Backends[id]
@@ -71,11 +88,12 @@ func NewRuntime(ctx context.Context, c config.Config, mainPath string, client *h
 		TransientRetries: &c.LLM.TransientRetries,
 	}, pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot))
 	if err != nil {
-		return nil, fmt.Errorf("assemble LLM runtime: %w", err)
+		return nil, embeddedAssetOrInputError(ctx, promptInputs, fmt.Errorf("assemble LLM runtime: %w", err))
 	}
 	for _, meta := range prompts.RegisteredMetadata() {
 		if _, err := engine.InspectPrompt(ctx, meta.PromptID, meta.PromptVersion); err != nil {
-			return nil, &AssetError{Err: fmt.Errorf("inspect embedded prompt %s: %w", meta.PromptID, err)}
+			cause := fmt.Errorf("inspect prompt %s: %w", meta.PromptID, err)
+			return nil, embeddedAssetOrInputError(ctx, promptInputs, cause)
 		}
 	}
 	runtime := &Runtime{Config: c, Engine: engine, Profiles: make(map[string]pk.ProfileInspection, 2)}
@@ -110,6 +128,32 @@ func NewRuntime(ctx context.Context, c config.Config, mainPath string, client *h
 	return runtime, nil
 }
 
+func embeddedAssetOrInputError(ctx context.Context, promptInputs []ck.Input, cause error) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	embeddedSources := pk.ConfigurationSources{
+		Underlays: []ck.Underlay{{Inputs: promptInputs}},
+	}
+	diagnosticEngine, err := pk.NewEngine(pk.Config{Sources: embeddedSources},
+		pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot))
+	if err == nil {
+		for _, meta := range prompts.RegisteredMetadata() {
+			if _, inspectErr := diagnosticEngine.InspectPrompt(ctx, meta.PromptID, meta.PromptVersion); inspectErr != nil {
+				err = fmt.Errorf("inspect embedded prompt %s: %w", meta.PromptID, inspectErr)
+				break
+			}
+		}
+	}
+	if ctxErr := ctx.Err(); ctxErr != nil {
+		return ctxErr
+	}
+	if err != nil {
+		return &AssetError{Err: fmt.Errorf("validate embedded prompt assets: %w", err)}
+	}
+	return cause
+}
+
 func IsAssetError(err error) bool { var target *AssetError; return errors.As(err, &target) }
 
 // RedactError replaces every configured resolved backend secret before a