promptkit/profile_inheritance_contract_test.go

package promptkit_test

import (
	"context"
	"errors"
	"io/fs"
	"sync"
	"testing"
	"testing/fstest"

	"gitea.maximumdirect.net/eric/ConfigKit"
	"gitea.maximumdirect.net/eric/promptkit"
)

func aliasPrompt() map[string]any {
	return map[string]any{"prompts": map[string]any{"prepared": promptDefinitionWithVersion("1", map[string]any{
		"default_profile": "child",
		"messages":        []any{map[string]any{"role": "user", "content": "hello"}},
		"output":          map[string]any{"format": "text", "validation_mode": "none"},
	})}}
}

func TestProfileInheritanceConfiguredBaseAliasWorkflow(t *testing.T) {
	baseID, baseEndpoint, baseModel, reasoning, timeout := "base", "https://profile.example/v1", "synthetic-model", "high", 120
	engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
		Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("application alias", map[string]promptkit.ProfileConfig{
			"base":  {Endpoint: &baseEndpoint, Model: &baseModel},
			"child": {Extends: &baseID, ReasoningEffort: &reasoning, TimeoutSeconds: &timeout},
		})}}}},
		Main: &configkit.Input{Source: configkit.Mapping("prompt", aliasPrompt())},
	}})
	if err != nil {
		t.Fatalf("construct alias engine: %v", err)
	}
	base, err := engine.InspectProfile(context.Background(), baseID)
	if err != nil {
		t.Fatalf("inspect base: %v", err)
	}
	child, err := engine.InspectProfile(context.Background(), "child")
	if err != nil {
		t.Fatalf("inspect alias: %v", err)
	}
	if child.ProfileID != "child" || child.EffectiveModelParams.BackendID != base.EffectiveModelParams.BackendID ||
		child.EffectiveModelParams.Model != base.EffectiveModelParams.Model ||
		child.EffectiveModelParams.ReasoningEffort != reasoning || child.EffectiveModelParams.TimeoutSeconds != timeout {
		t.Fatalf("alias inspection = %+v, base = %+v", child, base)
	}
	prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prepared"})
	if err != nil || prepared.SelectedProfileID != "child" {
		t.Fatalf("prepare alias = (%+v, %v)", prepared, err)
	}
	shorter, lower := 15, "low"
	overridden, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prepared", Execution: &promptkit.ExecutionTargetOverride{
		TimeoutSeconds: &shorter, ReasoningEffort: &lower,
	}})
	if err != nil || overridden.EffectiveModelParams.TimeoutSeconds != shorter || overridden.EffectiveModelParams.ReasoningEffort != lower {
		t.Fatalf("runtime override = (%+v, %v)", overridden, err)
	}
}

func TestProfileInheritanceYAMLAliasOfConfiguredProfile(t *testing.T) {
	inputs, err := promptkit.FilesystemInputs("profile alias", fstest.MapFS{
		"config/alias.yaml": &fstest.MapFile{Data: []byte("profiles:\n  base:\n    model: synthetic-model\n  yaml-alias:\n    extends: base\n")},
	}, "config")
	if err != nil {
		t.Fatal(err)
	}
	engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: inputs}}}})
	if err != nil {
		t.Fatal(err)
	}
	inspection, err := engine.InspectProfile(context.Background(), "yaml-alias")
	if err != nil || inspection.ProfileID != "yaml-alias" || inspection.EffectiveModelParams.Model == "" {
		t.Fatalf("YAML alias inspection = (%+v, %v)", inspection, err)
	}
}

func TestProfileInheritanceRetainsRequiredCredentialBehavior(t *testing.T) {
	engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
		Source: configkit.Mapping("credentials", map[string]any{
			"backends": map[string]any{"credential-backend": map[string]any{
				"endpoint": "https://credential.example/v1", "api_key_env": "OPTIONAL_BACKEND_KEY",
			}},
			"profiles": map[string]any{
				"credential-base":  map[string]any{"backend": "credential-backend", "model": "model", "api_key_required": true},
				"credential-child": map[string]any{"extends": "credential-base"},
			},
		}),
	}}})
	if err != nil {
		t.Fatal(err)
	}
	inspection, err := engine.InspectProfile(context.Background(), "credential-child")
	if err != nil || !inspection.APIKeyRequired || inspection.EffectiveModelParams.APIKeyEnv != "" {
		t.Fatalf("credential inspection = (%+v, %v)", inspection, err)
	}
}

func TestProfileInheritancePreservesPublicErrorIdentities(t *testing.T) {
	engine, err := promptkit.NewEngine(promptkit.Config{})
	if err != nil {
		t.Fatal(err)
	}
	if inspection, err := engine.InspectProfile(context.Background(), "missing"); inspection != nil || !errors.Is(err, promptkit.ErrProfileNotFound) {
		t.Fatalf("missing profile = (%+v, %v)", inspection, err)
	}
	for _, document := range []map[string]any{
		{"profiles": map[string]any{"child": map[string]any{"extends": "missing"}}},
		{"profiles": map[string]any{"a": map[string]any{"extends": "b"}, "b": map[string]any{"extends": "a"}}},
	} {
		invalid, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
			Source: configkit.Mapping("invalid inheritance", document),
		}}})
		if invalid != nil || !errors.Is(err, promptkit.ErrInvalidConfig) {
			t.Fatalf("invalid inheritance = (%+v, %v)", invalid, err)
		}
	}
}

func TestProfileInheritanceFreezesPreparedExecution(t *testing.T) {
	profiles := &mutableInheritanceProfileFS{files: fstest.MapFS{
		"config/profiles.yaml": &fstest.MapFile{Data: []byte("profiles:\n  base:\n    endpoint: https://base.example/v1\n    model: first-model\n  child:\n    extends: base\n")},
	}}
	inputs, err := promptkit.FilesystemInputs("profiles", profiles, "config")
	if err != nil {
		t.Fatal(err)
	}
	client := &recordingProfileClient{}
	engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
		Underlays: []configkit.Underlay{{Inputs: inputs}},
		Main:      &configkit.Input{Source: configkit.Mapping("prompt", aliasPrompt())},
	}}, promptkit.WithLLMClient(client))
	if err != nil {
		t.Fatal(err)
	}
	prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"})
	if err != nil {
		t.Fatal(err)
	}
	profiles.set("config/profiles.yaml", "profiles:\n  base:\n    endpoint: https://base.example/v1\n    model: second-model\n  child:\n    extends: base\n")
	result, err := engine.RunPrepared(context.Background(), prepared)
	if err != nil || result == nil || client.model != "first-model" {
		t.Fatalf("prepared execution = (%+v, %v), model %q", result, err, client.model)
	}
	inspection, err := engine.InspectProfile(context.Background(), "child")
	if err != nil || inspection.EffectiveModelParams.Model != "first-model" {
		t.Fatalf("existing engine inspection = (%+v, %v)", inspection, err)
	}
}

type recordingProfileClient struct{ model string }

func (client *recordingProfileClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
	client.model = request.Target.Model
	return &promptkit.GenerateResponse{Content: "ok"}, ctx.Err()
}

type mutableInheritanceProfileFS struct {
	mu    sync.RWMutex
	files fstest.MapFS
}

func (source *mutableInheritanceProfileFS) Open(name string) (fs.File, error) {
	source.mu.RLock()
	defer source.mu.RUnlock()
	return source.files.Open(name)
}

func (source *mutableInheritanceProfileFS) set(name, content string) {
	source.mu.Lock()
	defer source.mu.Unlock()
	source.files[name] = &fstest.MapFile{Data: []byte(content)}
}