From 764103a2e28f6f614a59bf49fc064be051866140 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 25 Aug 2026 01:41:21 +0000 Subject: [PATCH] Resolve inherited profiles in engine workflows --- docs/roadmap/implementation.md | 2 + engine.go | 2 +- profile_inheritance_contract_test.go | 243 +++++++++++++++++++++++++++ 3 files changed, 246 insertions(+), 1 deletion(-) create mode 100644 profile_inheritance_contract_test.go diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 43b0c87..77b7db9 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -387,6 +387,8 @@ Stage 3 is complete when the engine resolves aliases and refinements across all supported sources, public identities and errors match the roadmap, ordinary operations remain fresh, and prepared execution remains frozen. +**Status:** Complete. + ## Stage 4: Publish Canonical Documentation And Complete Validation ### Objective diff --git a/engine.go b/engine.go index 9b717b6..44a9638 100644 --- a/engine.go +++ b/engine.go @@ -458,7 +458,7 @@ func newProfileRepository(profileDir string, options engineOptions) profile.Repo repository = profile.NewOverlayRepository(options.memoryProfiles, repository) } - return repository + return profile.NewResolvingRepository(repository) } func fileSource(name string) (fs.FS, string, error) { diff --git a/profile_inheritance_contract_test.go b/profile_inheritance_contract_test.go new file mode 100644 index 0000000..7417506 --- /dev/null +++ b/profile_inheritance_contract_test.go @@ -0,0 +1,243 @@ +package promptkit_test + +import ( + "context" + "errors" + "io/fs" + "strings" + "sync" + "testing" + "testing/fstest" + + "gitea.maximumdirect.net/eric/promptkit" +) + +func TestProfileInheritanceBuiltInAliasWorkflow(t *testing.T) { + engine, err := promptkit.NewEngine(promptkit.Config{ + PromptDir: frameworkPromptDir, + SchemaDir: frameworkSchemaDir, + }, promptkit.WithProfiles(promptkit.Profile{ + ID: "weather-light", + BaseProfileID: "deepseek-4-flash", + ReasoningEffort: "high", + TimeoutSeconds: 120, + })) + if err != nil { + t.Fatalf("construct alias engine: %v", err) + } + + base, err := engine.InspectProfile(context.Background(), "deepseek-4-flash") + if err != nil { + t.Fatalf("inspect base: %v", err) + } + child, err := engine.InspectProfile(context.Background(), "weather-light") + if err != nil { + t.Fatalf("inspect alias: %v", err) + } + if child.ProfileID != "weather-light" || + child.EffectiveModelParams.BackendID != base.EffectiveModelParams.BackendID || + child.EffectiveModelParams.Model != base.EffectiveModelParams.Model || + child.EffectiveModelParams.ReasoningEffort != "high" || + child.EffectiveModelParams.TimeoutSeconds != 120 { + t.Fatalf("alias inspection = %+v, base = %+v", child, base) + } + + prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + ProfileID: "weather-light", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + "glossary": promptkit.Inline("gate: A guarded passage."), + }, + }) + if err != nil { + t.Fatalf("prepare alias: %v", err) + } + if prepared.SelectedProfileID != "weather-light" { + t.Fatalf("SelectedProfileID = %q", prepared.SelectedProfileID) + } + + timeout := 15 + reasoning := "low" + overridden, err := engine.Prepare(context.Background(), promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + ProfileID: "weather-light", + Execution: &promptkit.ExecutionTargetOverride{ + TimeoutSeconds: &timeout, + ReasoningEffort: &reasoning, + }, + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + "glossary": promptkit.Inline("gate: A guarded passage."), + }, + }) + if err != nil { + t.Fatalf("prepare override: %v", err) + } + if overridden.EffectiveModelParams.TimeoutSeconds != timeout || + overridden.EffectiveModelParams.ReasoningEffort != reasoning { + t.Fatalf("runtime override target = %+v", overridden.EffectiveModelParams) + } +} + +func TestProfileInheritanceYAMLAliasOfBuiltIn(t *testing.T) { + engine, err := promptkit.NewEngine(promptkit.Config{}, + promptkit.WithPromptFS(fstest.MapFS{}, "."), + promptkit.WithProfileFS(fstest.MapFS{ + "alias.yaml": &fstest.MapFile{Data: []byte("id: yaml-alias\nbase_profile: deepseek-4-flash\n")}, + }, "."), + ) + if err != nil { + t.Fatalf("construct YAML alias engine: %v", err) + } + inspection, err := engine.InspectProfile(context.Background(), "yaml-alias") + if err != nil { + t.Fatalf("inspect YAML alias: %v", err) + } + if inspection.ProfileID != "yaml-alias" || inspection.EffectiveModelParams.Model == "" { + t.Fatalf("YAML alias inspection = %+v", inspection) + } +} + +func TestProfileInheritanceRetainsRequiredCredentialBehavior(t *testing.T) { + engine, err := promptkit.NewEngine(promptkit.Config{}, + promptkit.WithPromptFS(fstest.MapFS{}, "."), + promptkit.WithBackend(promptkit.Backend{ + ID: "credential-backend", + Endpoint: "https://credential.example/v1", + APIKeyEnv: "OPTIONAL_BACKEND_KEY", + }), + promptkit.WithProfiles( + promptkit.Profile{ + ID: "credential-base", + BackendID: "credential-backend", + Model: "model", + APIKeyRequired: true, + }, + promptkit.Profile{ID: "credential-child", BaseProfileID: "credential-base"}, + ), + ) + if err != nil { + t.Fatalf("construct credential inheritance engine: %v", err) + } + inspection, err := engine.InspectProfile(context.Background(), "credential-child") + if err != nil { + t.Fatalf("inspect credential child: %v", err) + } + if !inspection.APIKeyRequired || inspection.EffectiveModelParams.APIKeyEnv != "" { + t.Fatalf("credential inspection = %+v", inspection) + } +} + +func TestProfileInheritancePreservesPublicErrorIdentities(t *testing.T) { + newEngine := func(t *testing.T, profiles fs.FS) *promptkit.Engine { + t.Helper() + options := []promptkit.Option{promptkit.WithPromptFS(fstest.MapFS{}, ".")} + if profiles != nil { + options = append(options, promptkit.WithProfileFS(profiles, ".")) + } + engine, err := promptkit.NewEngine(promptkit.Config{}, options...) + if err != nil { + t.Fatalf("construct engine: %v", err) + } + return engine + } + + tests := []struct { + name string + profiles fs.FS + profile string + contains []string + want error + wantNot error + }{ + { + name: "missing selected profile", + profile: "missing", + want: promptkit.ErrProfileNotFound, + wantNot: promptkit.ErrProfileLoad, + }, + { + name: "missing base", + profiles: fstest.MapFS{ + "child.yaml": &fstest.MapFile{Data: []byte("id: child\nbase_profile: missing\n")}, + }, + profile: "child", + contains: []string{"child", "missing"}, + want: promptkit.ErrProfileLoad, + wantNot: promptkit.ErrProfileNotFound, + }, + { + name: "cycle", + profiles: fstest.MapFS{ + "a.yaml": &fstest.MapFile{Data: []byte("id: a\nbase_profile: b\n")}, + "b.yaml": &fstest.MapFile{Data: []byte("id: b\nbase_profile: a\n")}, + }, + profile: "a", + contains: []string{"a", "b"}, + want: promptkit.ErrProfileLoad, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result, err := newEngine(t, tc.profiles).InspectProfile(context.Background(), tc.profile) + if result != nil || !errors.Is(err, tc.want) || (tc.wantNot != nil && errors.Is(err, tc.wantNot)) { + t.Fatalf("inspection=(%+v, %v), want %v without %v", result, err, tc.want, tc.wantNot) + } + for _, fragment := range tc.contains { + if !strings.Contains(err.Error(), fragment) { + t.Fatalf("error = %v, want %q", err, fragment) + } + } + }) + } +} + +func TestProfileInheritanceFreezesPreparedExecution(t *testing.T) { + profiles := &mutableInheritanceProfileFS{files: fstest.MapFS{ + "child.yaml": &fstest.MapFile{Data: []byte("id: child\nbase_profile: base\n")}, + "base.yaml": &fstest.MapFile{Data: []byte("id: base\nendpoint: https://base.example/v1\nmodel: first-model\n")}, + }} + client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}} + engine, err := promptkit.NewEngine(promptkit.Config{}, + promptkit.WithPromptFS(contractPromptFS("prepared", "child", "content"), "."), + promptkit.WithProfileFS(profiles, "."), + promptkit.WithLLMClient(client), + ) + if err != nil { + t.Fatalf("construct engine: %v", err) + } + + prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"}) + if err != nil { + t.Fatalf("prepare execution: %v", err) + } + profiles.set("base.yaml", "id: base\nendpoint: https://base.example/v1\nmodel: second-model\n") + + result, err := engine.RunPrepared(context.Background(), prepared) + if err != nil || result == nil || len(client.requests) != 1 || client.requests[0].Target.Model != "first-model" { + t.Fatalf("prepared execution=(%+v, %v), requests=%+v", result, err, client.requests) + } + inspection, err := engine.InspectProfile(context.Background(), "child") + if err != nil || inspection.EffectiveModelParams.Model != "second-model" { + t.Fatalf("fresh inspection=(%+v, %v)", inspection, err) + } +} + +type mutableInheritanceProfileFS struct { + mu sync.RWMutex + files fstest.MapFS +} + +func (f *mutableInheritanceProfileFS) Open(name string) (fs.File, error) { + f.mu.RLock() + defer f.mu.RUnlock() + return f.files.Open(name) +} + +func (f *mutableInheritanceProfileFS) set(name, content string) { + f.mu.Lock() + defer f.mu.Unlock() + f.files[name] = &fstest.MapFile{Data: []byte(content)} +}