Assemble PromptKit profiles for offline configuration inspection

internal/cli/config_input.go

@@ -19,6 +19,7 @@ type configInput struct {
 	ConfigPath     string
 	ConfigPathSet  bool
 	Overlay        string
+	View           string
 	TranscriptPath string
 	GlossaryPath   string
 	OutputPath     string
@@ -40,6 +41,9 @@ func parseConfigInput(command string, args []string, output io.Writer) (configIn
 	fs.SetOutput(output)
 	configPath := fs.String("config", "", "Main configuration YAML file")
 	overlay := fs.String("overlay", "", "Selected configuration overlay")
+	if command == "print-effective" {
+		fs.String("view", "settings", "Effective view: settings, profiles, or explain")
+	}
 	settingPaths := map[string]ck.Path{
 		"proposal-profile":                 ck.Keys("llm", "proposal_profile"),
 		"validation-profile":               ck.Keys("llm", "validation_profile"),
@@ -106,6 +110,14 @@ func parseConfigInput(command string, args []string, output io.Writer) (configIn
 		return configInput{}, false, fmt.Errorf("unexpected positional arguments")
 	}
 	input.ConfigPath, input.Overlay = *configPath, *overlay
+	if command == "print-effective" {
+		input.View = fs.Lookup("view").Value.String()
+		switch input.View {
+		case "settings", "profiles", "explain":
+		default:
+			return configInput{}, false, fmt.Errorf("unknown effective view %q", input.View)
+		}
+	}
 	fs.Visit(func(f *flag.Flag) {
 		if f.Name == "config" {
 			input.ConfigPathSet = true

internal/cli/config_input_test.go

@@ -108,6 +108,21 @@ func TestLegacyEnvironmentDoesNotSelectConfig(t *testing.T) {
 	}
 }
 
+func TestEffectiveViewParsing(t *testing.T) {
+	for _, view := range []string{"settings", "profiles", "explain"} {
+		input, _, err := parseConfigInput("print-effective", []string{"--view", view}, io.Discard)
+		if err != nil || input.View != view {
+			t.Fatalf("%q: %+v %v", view, input, err)
+		}
+	}
+	if _, _, err := parseConfigInput("print-effective", []string{"--view", "raw"}, io.Discard); err == nil {
+		t.Fatal("unknown view accepted")
+	}
+	if _, _, err := parseConfigInput("validate", []string{"--view", "profiles"}, io.Discard); err == nil {
+		t.Fatal("validate accepted inspection-only flag")
+	}
+}
+
 func TestSharedLoadingUsesSelectedFileOverlayAndVisitedOverrides(t *testing.T) {
 	dir := t.TempDir()
 	if err := os.Mkdir(filepath.Join(dir, "overlays"), 0700); err != nil {

internal/cli/run.go

@@ -52,6 +52,7 @@ type processInvocation struct {
 	ReportJSONPath  string
 	Config          config.Config
 	ConfigResult    *ck.Result[config.Config]
+	LLMRuntime      *llm.Runtime
 	ConfigPath      string
 	ConfigSource    string
 	ConfigVersion   *int
@@ -392,6 +393,14 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
 		fmt.Fprintf(stderr, "audita process: %s\n", configInputError(err))
 		return 2
 	}
+	llmRuntime, err := llm.NewRuntime(context.Background(), selected.Result.Config, selected.Path, nil)
+	if err != nil {
+		fmt.Fprintf(stderr, "audita process: %s\n", llm.RedactError(err, selected.Result.Config))
+		if llm.IsAssetError(err) {
+			return 1
+		}
+		return 2
+	}
 	version := selected.Result.Config.Version
 	explicitModules := false
 	for _, override := range input.Overrides {
@@ -407,6 +416,7 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
 		ReportJSONPath:  input.ReportPath,
 		Config:          selected.Result.Config,
 		ConfigResult:    selected.Result,
+		LLMRuntime:      llmRuntime,
 		ConfigPath:      selected.Path,
 		ConfigSource:    selected.Source,
 		ConfigVersion:   &version,
@@ -516,10 +526,18 @@ func runConfigValidate(args []string, stdout, stderr io.Writer) int {
 		writeConfigValidateUsage(stdout)
 		return 0
 	}
-	if _, err := loadConfigInput(input, "validate"); err != nil {
+	selected, err := loadConfigInput(input, "validate")
+	if err != nil {
 		fmt.Fprintf(stderr, "audita config validate: %s\n", configInputError(err))
 		return 2
 	}
+	if _, err := llm.NewRuntime(context.Background(), selected.Result.Config, selected.Path, nil); err != nil {
+		fmt.Fprintf(stderr, "audita config validate: %s\n", llm.RedactError(err, selected.Result.Config))
+		if llm.IsAssetError(err) {
+			return 1
+		}
+		return 2
+	}
 	fmt.Fprintln(stdout, "config is valid")
 	return 0
 }
@@ -539,7 +557,28 @@ func runConfigPrintEffective(args []string, stdout, stderr io.Writer) int {
 		fmt.Fprintf(stderr, "audita config print-effective: %s\n", configInputError(err))
 		return 2
 	}
-	if err := selected.Result.WriteEffective(stdout); err != nil {
+	llmRuntime, err := llm.NewRuntime(context.Background(), selected.Result.Config, selected.Path, nil)
+	if err != nil {
+		fmt.Fprintf(stderr, "audita config print-effective: %s\n", llm.RedactError(err, selected.Result.Config))
+		if llm.IsAssetError(err) {
+			return 1
+		}
+		return 2
+	}
+	if input.View == "settings" {
+		err = selected.Result.WriteEffective(stdout)
+	} else {
+		var output []byte
+		if input.View == "profiles" {
+			output, err = llmRuntime.SelectedProfilesYAML()
+		} else {
+			output, err = llm.ExplanationYAML(selected.Result)
+		}
+		if err == nil {
+			_, err = stdout.Write(output)
+		}
+	}
+	if err != nil {
 		fmt.Fprintf(stderr, "audita config print-effective: %v\n", err)
 		return 1
 	}

internal/framework/llm/runtime.go

@@ -0,0 +1,108 @@
+package llm
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"gitea.maximumdirect.net/eric/audita/internal/core/config"
+	"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
+	"gitea.maximumdirect.net/eric/audita/internal/prompts"
+	pk "gitea.maximumdirect.net/eric/promptkit"
+)
+
+// Runtime owns one PromptKit engine and the validated selections for both roles.
+// The same engine is shared by the role-bound clients during processing.
+type Runtime struct {
+	Config   config.Config
+	Engine   *pk.Engine
+	Profiles map[string]pk.ProfileInspection
+}
+
+// AssetError indicates a broken embedded prompt or schema rather than an
+// operator-provided setting or profile.
+type AssetError struct{ Err error }
+
+func (e *AssetError) Error() string { return e.Err.Error() }
+func (e *AssetError) Unwrap() error { return e.Err }
+
+func NewRuntime(ctx context.Context, c config.Config, mainPath string, client *http.Client) (*Runtime, error) {
+	profileDir := c.ProfilesDir
+	if !filepath.IsAbs(profileDir) {
+		profileDir = filepath.Join(filepath.Dir(mainPath), profileDir)
+	}
+	options := []pk.Option{
+		pk.WithPromptFS(prompts.Filesystem(), prompts.AssetRoot),
+		pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot),
+		pk.WithProfileFS(os.DirFS(profileDir), "."),
+	}
+	ids := make([]string, 0, len(c.Backends))
+	for id := range c.Backends {
+		ids = append(ids, id)
+	}
+	sort.Strings(ids)
+	for _, id := range ids {
+		backend := c.Backends[id]
+		options = append(options, pk.WithBackend(pk.Backend{ID: id, Endpoint: backend.Endpoint, ConcurrencyLimit: backend.Concurrency, QueueCapacity: &backend.QueueCapacity}))
+	}
+	engine, err := pk.NewEngine(pk.Config{HTTPClient: client, TransientRetries: &c.LLM.TransientRetries}, options...)
+	if err != nil {
+		return nil, fmt.Errorf("assemble LLM runtime: %w", err)
+	}
+	for _, meta := range prompts.RegisteredMetadata() {
+		if _, err := engine.InspectPrompt(ctx, meta.PromptID, meta.PromptVersion); err != nil {
+			return nil, &AssetError{Err: fmt.Errorf("inspect embedded prompt %s: %w", meta.PromptID, err)}
+		}
+	}
+	runtime := &Runtime{Config: c, Engine: engine, Profiles: make(map[string]pk.ProfileInspection, 2)}
+	for _, selection := range []struct{ role, id string }{{"proposal", c.LLM.ProposalProfile}, {"validation", c.LLM.ValidationProfile}} {
+		profile, err := engine.InspectProfile(ctx, selection.id)
+		if err != nil {
+			return nil, fmt.Errorf("inspect %s profile: %w", selection.role, err)
+		}
+		target := profile.EffectiveModelParams
+		backend, ok := c.Backends[target.BackendID]
+		if !ok {
+			return nil, fmt.Errorf("%s profile must select a registered application backend", selection.role)
+		}
+		endpoint := func(value string) string { return strings.TrimRight(strings.TrimSpace(value), "/") }
+		if endpoint(target.Endpoint) != endpoint(backend.Endpoint) {
+			return nil, fmt.Errorf("%s profile endpoint must match its registered backend", selection.role)
+		}
+		if target.APIKeyEnv != "" {
+			return nil, fmt.Errorf("%s profile must use the backend credential, not its own environment lookup", selection.role)
+		}
+		if profile.APIKeyRequired && backend.Credential == nil {
+			return nil, fmt.Errorf("%s profile requires a backend credential", selection.role)
+		}
+		runtime.Profiles[selection.role] = *profile
+	}
+	return runtime, nil
+}
+
+func IsAssetError(err error) bool { var target *AssetError; return errors.As(err, &target) }
+
+// RedactError replaces every configured resolved backend secret before a
+// library error is surfaced in CLI output or an artifact.
+func RedactError(err error, c config.Config) string {
+	if err == nil {
+		return ""
+	}
+	message := err.Error()
+	var secrets []string
+	for _, backend := range c.Backends {
+		if backend.Credential != nil && backend.Credential.Value() != "" {
+			secrets = append(secrets, backend.Credential.Value())
+		}
+	}
+	sort.Slice(secrets, func(i, j int) bool { return len(secrets[i]) > len(secrets[j]) })
+	for _, secret := range secrets {
+		message = strings.ReplaceAll(message, secret, "[REDACTED]")
+	}
+	return message
+}

internal/framework/llm/runtime_inspection_test.go

@@ -0,0 +1,168 @@
+package llm
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	ck "gitea.maximumdirect.net/eric/ConfigKit"
+	"gitea.maximumdirect.net/eric/audita/internal/core/config"
+)
+
+type rejectingTransport struct{ t *testing.T }
+
+func (r rejectingTransport) RoundTrip(*http.Request) (*http.Response, error) {
+	r.t.Error("profile inspection contacted a provider")
+	return nil, errors.New("provider access forbidden")
+}
+
+func profileLayout(t *testing.T, base, overlay string) string {
+	t.Helper()
+	dir := t.TempDir()
+	for _, name := range []string{"profiles", "overlays"} {
+		if err := os.Mkdir(filepath.Join(dir, name), 0700); err != nil {
+			t.Fatal(err)
+		}
+	}
+	for name, content := range map[string]string{
+		"base":    "id: base\nbackend: offline\nmodel: full-model\ntemperature: 0.3\nmax_tokens: 500\ntop_p: 0.8\ntimeout_seconds: 12\nservice_tier: priority\nreasoning_effort: medium\n",
+		"testing": "id: testing\nbackend: offline\nmodel: economy-model\n",
+	} {
+		if err := os.WriteFile(filepath.Join(dir, "profiles", name+".yaml"), []byte(content), 0600); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := os.WriteFile(filepath.Join(dir, "config.yml"), []byte(base), 0600); err != nil {
+		t.Fatal(err)
+	}
+	if overlay != "" {
+		if err := os.WriteFile(filepath.Join(dir, "overlays", "testing.yml"), []byte(overlay), 0600); err != nil {
+			t.Fatal(err)
+		}
+	}
+	return dir
+}
+
+const profileBase = "version: 2\nbackends:\n  offline:\n    endpoint: https://offline.invalid/v1/\n    credential: SYNTHETIC_KEY\nllm:\n  proposal_profile: base\n"
+
+func TestOfflineProfileAssemblyAndSafeViews(t *testing.T) {
+	t.Setenv("SYNTHETIC_KEY", "synthetic-value-never-print")
+	dir := profileLayout(t, profileBase, "llm:\n  proposal_profile: testing\n")
+	client := &http.Client{Transport: rejectingTransport{t}}
+	for _, tc := range []struct {
+		name, overlay string
+		overrides     []ck.Override
+		want          string
+	}{
+		{"base", "", nil, "full-model"},
+		{"overlay", "testing", nil, "economy-model"},
+		{"explicit override", "testing", []ck.Override{{Path: ck.Keys("llm", "proposal_profile"), Value: "base", Flag: "--proposal-profile"}}, "full-model"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			main := filepath.Join(dir, "config.yml")
+			loaded, err := config.Load(ck.File(main), tc.overlay, tc.overrides...)
+			if err != nil {
+				t.Fatal(err)
+			}
+			runtime, err := NewRuntime(context.Background(), loaded.Config, main, client)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if runtime.Engine == nil || runtime.Profiles["proposal"].EffectiveModelParams.Model != tc.want || runtime.Profiles["validation"].EffectiveModelParams.Model != tc.want {
+				t.Fatalf("roles not resolved: %+v", runtime.Profiles)
+			}
+			profiles, err := runtime.SelectedProfilesYAML()
+			if err != nil {
+				t.Fatal(err)
+			}
+			explain, err := ExplanationYAML(loaded)
+			if err != nil {
+				t.Fatal(err)
+			}
+			combined := string(profiles) + string(explain)
+			if !strings.Contains(combined, tc.want) || !strings.Contains(combined, "/llm/validation_profile") {
+				t.Fatalf("inspection missing selection/provenance: %s", combined)
+			}
+			if strings.Contains(combined, "synthetic-value-never-print") || strings.Contains(combined, "https://offline.invalid") {
+				t.Fatal("safe views exposed credentials or endpoint")
+			}
+			if tc.name == "overlay" && !strings.Contains(combined, "overlay") {
+				t.Fatal("overlay origin missing")
+			}
+			if tc.name == "explicit override" && !strings.Contains(combined, "cli") {
+				t.Fatal("CLI origin missing")
+			}
+			if !strings.Contains(string(explain), "validation uses the final proposal profile when omitted") {
+				t.Fatal("inherited validation derivation missing")
+			}
+			if tc.name == "overlay" && !strings.Contains(string(explain), "source: overlays/testing.yml\n  layer: overlay") {
+				t.Fatal("overlay field source missing")
+			}
+			if tc.name == "explicit override" && !strings.Contains(string(explain), "source: --proposal-profile\n  layer: cli") {
+				t.Fatal("explicit profile source missing")
+			}
+			if tc.name == "base" && (!strings.Contains(string(profiles), "temperature: 0.3") || !strings.Contains(string(profiles), "reasoning_effort: medium")) {
+				t.Fatal("selected controls missing")
+			}
+		})
+	}
+}
+
+func TestProfileAssemblyRejectsUnsafeSelections(t *testing.T) {
+	t.Setenv("SYNTHETIC_KEY", "synthetic-value-never-print")
+	dir := profileLayout(t, profileBase, "")
+	main := filepath.Join(dir, "config.yml")
+	for _, tc := range []struct{ name, profile string }{
+		{"endpoint only", "id: custom\nendpoint: https://offline.invalid/v1\nmodel: model\n"},
+		{"mismatched endpoint", "id: custom\nbackend: offline\nendpoint: https://elsewhere.invalid/v1\nmodel: model\n"},
+		{"profile credential", "id: custom\nbackend: offline\nmodel: model\napi_key_env: OTHER_KEY\n"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			if err := os.WriteFile(filepath.Join(dir, "profiles", "custom.yaml"), []byte(tc.profile), 0600); err != nil {
+				t.Fatal(err)
+			}
+			loaded, err := config.Load(ck.File(main), "", ck.Override{Path: ck.Keys("llm", "proposal_profile"), Value: "custom", Flag: "--proposal-profile"})
+			if err != nil {
+				t.Fatal(err)
+			}
+			if _, err := NewRuntime(context.Background(), loaded.Config, main, &http.Client{Transport: rejectingTransport{t}}); err == nil {
+				t.Fatal("unsafe profile accepted")
+			}
+		})
+	}
+	loaded, err := config.Load(ck.File(main), "", ck.Override{Path: ck.Keys("llm", "proposal_profile"), Value: "missing", Flag: "--proposal-profile"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := NewRuntime(context.Background(), loaded.Config, main, nil); err == nil {
+		t.Fatal("missing profile accepted")
+	}
+	if got := RedactError(errors.New("provider: synthetic-value-never-print"), loaded.Config); strings.Contains(got, "synthetic-value-never-print") {
+		t.Fatalf("secret leaked: %q", got)
+	}
+}
+
+func TestAbsoluteProfileDirectoryAndMissingSecret(t *testing.T) {
+	t.Setenv("SYNTHETIC_KEY", "synthetic-value-never-print")
+	dir := profileLayout(t, profileBase, "")
+	main := filepath.Join(dir, "config.yml")
+	absolute := strings.Replace(profileBase, "version: 2\n", "version: 2\nprofiles_dir: "+filepath.Join(dir, "profiles")+"\n", 1)
+	if err := os.WriteFile(main, []byte(absolute), 0600); err != nil {
+		t.Fatal(err)
+	}
+	loaded, err := config.Load(ck.File(main), "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := NewRuntime(context.Background(), loaded.Config, main, &http.Client{Transport: rejectingTransport{t}}); err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv("SYNTHETIC_KEY", "")
+	if _, err := config.Load(ck.File(main), ""); err == nil {
+		t.Fatal("missing surviving secret accepted")
+	}
+}

internal/framework/llm/safe_view.go

@@ -0,0 +1,111 @@
+package llm
+
+import (
+	"fmt"
+	"strings"
+
+	ck "gitea.maximumdirect.net/eric/ConfigKit"
+	"gitea.maximumdirect.net/eric/audita/internal/core/config"
+	"gopkg.in/yaml.v3"
+)
+
+type selectedProfileView struct {
+	ID              string  `yaml:"id"`
+	Backend         string  `yaml:"backend"`
+	Model           string  `yaml:"model"`
+	Temperature     float64 `yaml:"temperature"`
+	MaxTokens       int     `yaml:"max_tokens"`
+	TopP            float64 `yaml:"top_p"`
+	TimeoutSeconds  int     `yaml:"timeout_seconds"`
+	ServiceTier     string  `yaml:"service_tier,omitempty"`
+	ReasoningEffort string  `yaml:"reasoning_effort,omitempty"`
+}
+
+// SelectedProfilesYAML exposes selected controls without endpoint, credential,
+// or arbitrary extra provider parameters.
+func (r *Runtime) SelectedProfilesYAML() ([]byte, error) {
+	if r == nil {
+		return nil, fmt.Errorf("LLM runtime is unavailable")
+	}
+	view := func(role string) (selectedProfileView, error) {
+		profile, ok := r.Profiles[role]
+		if !ok {
+			return selectedProfileView{}, fmt.Errorf("%s profile is unavailable", role)
+		}
+		target := profile.EffectiveModelParams
+		return selectedProfileView{ID: profile.ProfileID, Backend: target.BackendID, Model: target.Model,
+			Temperature: target.Temperature, MaxTokens: target.MaxTokens, TopP: target.TopP,
+			TimeoutSeconds: target.TimeoutSeconds, ServiceTier: target.ServiceTier, ReasoningEffort: target.ReasoningEffort}, nil
+	}
+	proposal, err := view("proposal")
+	if err != nil {
+		return nil, err
+	}
+	validation, err := view("validation")
+	if err != nil {
+		return nil, err
+	}
+	return yaml.Marshal(struct {
+		Proposal   selectedProfileView `yaml:"proposal"`
+		Validation selectedProfileView `yaml:"validation"`
+	}{proposal, validation})
+}
+
+type derivationView struct {
+	Operation    ck.ChangeKind `yaml:"operation"`
+	Reason       string        `yaml:"reason"`
+	Dependencies []string      `yaml:"dependencies,omitempty"`
+}
+type explanationView struct {
+	Path        string           `yaml:"path"`
+	Source      string           `yaml:"source"`
+	Layer       ck.Layer         `yaml:"layer"`
+	Derivations []derivationView `yaml:"derivations,omitempty"`
+}
+
+// ExplanationYAML reports only origins and recorded resolution operations.
+// It intentionally excludes effective and historical values, including secrets.
+func ExplanationYAML(result *ck.Result[config.Config]) ([]byte, error) {
+	if result == nil {
+		return nil, fmt.Errorf("configuration result is unavailable")
+	}
+	metadata := result.Metadata()
+	sources := make(map[string]string)
+	for _, source := range result.Sources() {
+		sources[source.ID] = source.Label
+	}
+	records := make([]explanationView, 0)
+	paths := metadata.HistoricalPaths()
+	for i, path := range paths {
+		if path.String() == "" {
+			continue
+		}
+		if i+1 < len(paths) && strings.HasPrefix(paths[i+1].String(), path.String()+"/") {
+			continue
+		}
+		info, ok := metadata.Lookup(path)
+		if !ok {
+			continue
+		}
+		record := explanationView{Path: path.String(), Layer: info.Origin.Layer, Source: string(info.Origin.Layer)}
+		if len(info.Origin.Locations) > 0 {
+			if label := sources[info.Origin.Locations[0].SourceID]; label != "" {
+				record.Source = label
+			}
+		}
+		for _, change := range metadata.History(path) {
+			if change.Origin.Layer != ck.LayerResolution {
+				continue
+			}
+			for _, reason := range change.Reasons {
+				derivation := derivationView{Operation: change.Origin.Operation, Reason: reason}
+				for _, dependency := range change.Dependencies {
+					derivation.Dependencies = append(derivation.Dependencies, dependency.String())
+				}
+				record.Derivations = append(record.Derivations, derivation)
+			}
+		}
+		records = append(records, record)
+	}
+	return yaml.Marshal(records)
+}