promptkit/profile_inheritance_contract_test.go

package promptkit_test

import (
	"context"
	"errors"
	"io/fs"
	"strings"
	"sync"
	"testing"
	"testing/fstest"

	"gitea.maximumdirect.net/eric/promptkit"
)

func TestProfileInheritanceBuiltInAliasWorkflow(t *testing.T) {
	engine, err := promptkit.NewEngine(promptkit.Config{
		PromptDir: frameworkPromptDir,
		SchemaDir: frameworkSchemaDir,
	}, promptkit.WithProfiles(promptkit.Profile{
		ID:              "weather-light",
		BaseProfileID:   "deepseek-4-flash",
		ReasoningEffort: "high",
		TimeoutSeconds:  120,
	}))
	if err != nil {
		t.Fatalf("construct alias engine: %v", err)
	}

	base, err := engine.InspectProfile(context.Background(), "deepseek-4-flash")
	if err != nil {
		t.Fatalf("inspect base: %v", err)
	}
	child, err := engine.InspectProfile(context.Background(), "weather-light")
	if err != nil {
		t.Fatalf("inspect alias: %v", err)
	}
	if child.ProfileID != "weather-light" ||
		child.EffectiveModelParams.BackendID != base.EffectiveModelParams.BackendID ||
		child.EffectiveModelParams.Model != base.EffectiveModelParams.Model ||
		child.EffectiveModelParams.ReasoningEffort != "high" ||
		child.EffectiveModelParams.TimeoutSeconds != 120 {
		t.Fatalf("alias inspection = %+v, base = %+v", child, base)
	}

	prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
		PromptID:  frameworkMarkdownSummaryPromptID,
		ProfileID: "weather-light",
		Inputs: map[string]promptkit.ArtifactRef{
			"transcript": promptkit.Inline("Rin opens the gate."),
			"glossary":   promptkit.Inline("gate: A guarded passage."),
		},
	})
	if err != nil {
		t.Fatalf("prepare alias: %v", err)
	}
	if prepared.SelectedProfileID != "weather-light" {
		t.Fatalf("SelectedProfileID = %q", prepared.SelectedProfileID)
	}

	timeout := 15
	reasoning := "low"
	overridden, err := engine.Prepare(context.Background(), promptkit.RunRequest{
		PromptID:  frameworkMarkdownSummaryPromptID,
		ProfileID: "weather-light",
		Execution: &promptkit.ExecutionTargetOverride{
			TimeoutSeconds:  &timeout,
			ReasoningEffort: &reasoning,
		},
		Inputs: map[string]promptkit.ArtifactRef{
			"transcript": promptkit.Inline("Rin opens the gate."),
			"glossary":   promptkit.Inline("gate: A guarded passage."),
		},
	})
	if err != nil {
		t.Fatalf("prepare override: %v", err)
	}
	if overridden.EffectiveModelParams.TimeoutSeconds != timeout ||
		overridden.EffectiveModelParams.ReasoningEffort != reasoning {
		t.Fatalf("runtime override target = %+v", overridden.EffectiveModelParams)
	}
}

func TestProfileInheritanceYAMLAliasOfBuiltIn(t *testing.T) {
	engine, err := promptkit.NewEngine(promptkit.Config{},
		promptkit.WithPromptFS(fstest.MapFS{}, "."),
		promptkit.WithProfileFS(fstest.MapFS{
			"alias.yaml": &fstest.MapFile{Data: []byte("id: yaml-alias\nbase_profile: deepseek-4-flash\n")},
		}, "."),
	)
	if err != nil {
		t.Fatalf("construct YAML alias engine: %v", err)
	}
	inspection, err := engine.InspectProfile(context.Background(), "yaml-alias")
	if err != nil {
		t.Fatalf("inspect YAML alias: %v", err)
	}
	if inspection.ProfileID != "yaml-alias" || inspection.EffectiveModelParams.Model == "" {
		t.Fatalf("YAML alias inspection = %+v", inspection)
	}
}

func TestProfileInheritanceRetainsRequiredCredentialBehavior(t *testing.T) {
	engine, err := promptkit.NewEngine(promptkit.Config{},
		promptkit.WithPromptFS(fstest.MapFS{}, "."),
		promptkit.WithBackend(promptkit.Backend{
			ID:        "credential-backend",
			Endpoint:  "https://credential.example/v1",
			APIKeyEnv: "OPTIONAL_BACKEND_KEY",
		}),
		promptkit.WithProfiles(
			promptkit.Profile{
				ID:             "credential-base",
				BackendID:      "credential-backend",
				Model:          "model",
				APIKeyRequired: true,
			},
			promptkit.Profile{ID: "credential-child", BaseProfileID: "credential-base"},
		),
	)
	if err != nil {
		t.Fatalf("construct credential inheritance engine: %v", err)
	}
	inspection, err := engine.InspectProfile(context.Background(), "credential-child")
	if err != nil {
		t.Fatalf("inspect credential child: %v", err)
	}
	if !inspection.APIKeyRequired || inspection.EffectiveModelParams.APIKeyEnv != "" {
		t.Fatalf("credential inspection = %+v", inspection)
	}
}

func TestProfileInheritancePreservesPublicErrorIdentities(t *testing.T) {
	newEngine := func(t *testing.T, profiles fs.FS) *promptkit.Engine {
		t.Helper()
		options := []promptkit.Option{promptkit.WithPromptFS(fstest.MapFS{}, ".")}
		if profiles != nil {
			options = append(options, promptkit.WithProfileFS(profiles, "."))
		}
		engine, err := promptkit.NewEngine(promptkit.Config{}, options...)
		if err != nil {
			t.Fatalf("construct engine: %v", err)
		}
		return engine
	}

	tests := []struct {
		name     string
		profiles fs.FS
		profile  string
		contains []string
		want     error
		wantNot  error
	}{
		{
			name:    "missing selected profile",
			profile: "missing",
			want:    promptkit.ErrProfileNotFound,
			wantNot: promptkit.ErrProfileLoad,
		},
		{
			name: "missing base",
			profiles: fstest.MapFS{
				"child.yaml": &fstest.MapFile{Data: []byte("id: child\nbase_profile: missing\n")},
			},
			profile:  "child",
			contains: []string{"child", "missing"},
			want:     promptkit.ErrProfileLoad,
			wantNot:  promptkit.ErrProfileNotFound,
		},
		{
			name: "cycle",
			profiles: fstest.MapFS{
				"a.yaml": &fstest.MapFile{Data: []byte("id: a\nbase_profile: b\n")},
				"b.yaml": &fstest.MapFile{Data: []byte("id: b\nbase_profile: a\n")},
			},
			profile:  "a",
			contains: []string{"a", "b"},
			want:     promptkit.ErrProfileLoad,
		},
	}

	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			result, err := newEngine(t, tc.profiles).InspectProfile(context.Background(), tc.profile)
			if result != nil || !errors.Is(err, tc.want) || (tc.wantNot != nil && errors.Is(err, tc.wantNot)) {
				t.Fatalf("inspection=(%+v, %v), want %v without %v", result, err, tc.want, tc.wantNot)
			}
			for _, fragment := range tc.contains {
				if !strings.Contains(err.Error(), fragment) {
					t.Fatalf("error = %v, want %q", err, fragment)
				}
			}
		})
	}
}

func TestProfileInheritanceFreezesPreparedExecution(t *testing.T) {
	profiles := &mutableInheritanceProfileFS{files: fstest.MapFS{
		"child.yaml": &fstest.MapFile{Data: []byte("id: child\nbase_profile: base\n")},
		"base.yaml":  &fstest.MapFile{Data: []byte("id: base\nendpoint: https://base.example/v1\nmodel: first-model\n")},
	}}
	client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
	engine, err := promptkit.NewEngine(promptkit.Config{},
		promptkit.WithPromptFS(contractPromptFS("prepared", "child", "content"), "."),
		promptkit.WithProfileFS(profiles, "."),
		promptkit.WithLLMClient(client),
	)
	if err != nil {
		t.Fatalf("construct engine: %v", err)
	}

	prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"})
	if err != nil {
		t.Fatalf("prepare execution: %v", err)
	}
	profiles.set("base.yaml", "id: base\nendpoint: https://base.example/v1\nmodel: second-model\n")

	result, err := engine.RunPrepared(context.Background(), prepared)
	if err != nil || result == nil || len(client.requests) != 1 || client.requests[0].Target.Model != "first-model" {
		t.Fatalf("prepared execution=(%+v, %v), requests=%+v", result, err, client.requests)
	}
	inspection, err := engine.InspectProfile(context.Background(), "child")
	if err != nil || inspection.EffectiveModelParams.Model != "second-model" {
		t.Fatalf("fresh inspection=(%+v, %v)", inspection, err)
	}
}

type mutableInheritanceProfileFS struct {
	mu    sync.RWMutex
	files fstest.MapFS
}

func (f *mutableInheritanceProfileFS) Open(name string) (fs.File, error) {
	f.mu.RLock()
	defer f.mu.RUnlock()
	return f.files.Open(name)
}

func (f *mutableInheritanceProfileFS) set(name, content string) {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.files[name] = &fstest.MapFile{Data: []byte(content)}
}