weatherreporter/internal/app/prompt_inspection_test.go

package app

import (
	"context"
	"errors"
	"strings"
	"testing"
	"time"

	"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
	"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
	"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
)

func TestInspectPromptExecutionSelectsDefaultAndOverrideProfiles(t *testing.T) {
	resolved := inspectionResolved(t)
	executor := &inspectionExecutor{
		prompt: validPromptInspection(resolved.Definition),
		profiles: map[string]promptexec.ProfileInspection{
			"default-profile":  {ProfileID: "default-profile", BackendID: "local", ModelName: "default-model"},
			"override-profile": {ProfileID: "override-profile", BackendID: "cloud", ModelName: "override-model"},
		},
	}
	defaultResult, err := InspectPromptExecution(context.Background(), PromptInspectionRequest{Resolved: resolved, Executor: executor})
	if err != nil {
		t.Fatalf("InspectPromptExecution(default) error = %v", err)
	}
	if defaultResult.ProfileID != "default-profile" || defaultResult.ModelName != "default-model" {
		t.Fatalf("default result = %#v", defaultResult)
	}
	overrideResult, err := InspectPromptExecution(context.Background(), PromptInspectionRequest{
		Resolved: resolved, Executor: executor, Promptkit: config.PromptkitConfig{Profile: "override-profile"},
	})
	if err != nil {
		t.Fatalf("InspectPromptExecution(override) error = %v", err)
	}
	if overrideResult.ProfileID != "override-profile" || overrideResult.ModelName != "override-model" {
		t.Fatalf("override result = %#v", overrideResult)
	}
	if len(executor.promptRequests) != 2 || executor.promptRequests[0].version != resolved.Definition.PromptVersion || executor.profileRequests[0] != "default-profile" || executor.profileRequests[1] != "override-profile" {
		t.Fatalf("inspection requests = prompts %#v profiles %#v", executor.promptRequests, executor.profileRequests)
	}
}

func TestInspectPromptExecutionRejectsInvalidContractsAndCredentials(t *testing.T) {
	resolved := inspectionResolved(t)
	basePrompt := validPromptInspection(resolved.Definition)
	tests := []struct {
		name         string
		prompt       promptexec.PromptInspection
		profile      promptexec.ProfileInspection
		lookupEnv    func(string) (string, bool)
		wantCategory promptexec.ErrorCategory
	}{
		{
			name: "extra input",
			prompt: func() promptexec.PromptInspection {
				value := basePrompt
				value.Inputs = append(value.Inputs, promptexec.InputDefinition{Name: "unexpected"})
				return value
			}(),
			wantCategory: promptexec.InvalidConfiguration,
		},
		{
			name: "wrong schema",
			prompt: func() promptexec.PromptInspection {
				value := basePrompt
				value.Output.SchemaPath = "unexpected.schema.json"
				return value
			}(),
			wantCategory: promptexec.InvalidConfiguration,
		},
		{
			name:         "direct key",
			prompt:       basePrompt,
			profile:      promptexec.ProfileInspection{ProfileID: "default-profile", CredentialRequired: true},
			wantCategory: promptexec.MissingCredential,
		},
		{
			name:         "missing environment credential",
			prompt:       basePrompt,
			profile:      promptexec.ProfileInspection{ProfileID: "default-profile", APIKeyEnv: "PROMPT_API_KEY"},
			lookupEnv:    func(string) (string, bool) { return "", false },
			wantCategory: promptexec.MissingCredential,
		},
	}
	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			executor := &inspectionExecutor{prompt: test.prompt, profiles: map[string]promptexec.ProfileInspection{"default-profile": test.profile}}
			_, err := InspectPromptExecution(context.Background(), PromptInspectionRequest{Resolved: resolved, Executor: executor, LookupEnv: test.lookupEnv})
			if err == nil || promptexec.CategoryOf(err) != test.wantCategory {
				t.Fatalf("error/category = %v/%q, want %q", err, promptexec.CategoryOf(err), test.wantCategory)
			}
		})
	}
}

func TestInspectPromptExecutionReturnsSafeInspectionError(t *testing.T) {
	resolved := inspectionResolved(t)
	executor := &inspectionExecutor{promptErr: errors.New("provider response contains resolved-secret-value")}
	_, err := InspectPromptExecution(context.Background(), PromptInspectionRequest{Resolved: resolved, Executor: executor})
	if err == nil || promptexec.CategoryOf(err) != promptexec.InvalidConfiguration {
		t.Fatalf("error/category = %v/%q", err, promptexec.CategoryOf(err))
	}
	if strings.Contains(err.Error(), "resolved-secret-value") {
		t.Fatalf("inspection error leaks provider value: %v", err)
	}
}

func TestInspectPromptExecutionsReusesEffectiveProfile(t *testing.T) {
	first := inspectionResolved(t)
	second := first
	second.Definition.ID = report.Today
	second.Definition.PromptID = "weather.today"
	executor := &inspectionExecutor{
		prompt: validPromptInspection(first.Definition),
		profiles: map[string]promptexec.ProfileInspection{
			"default-profile": {ProfileID: "default-profile", BackendID: "local", ModelName: "model"},
		},
	}
	executor.prompts = map[string]promptexec.PromptInspection{
		first.Definition.PromptID:  validPromptInspection(first.Definition),
		second.Definition.PromptID: validPromptInspection(second.Definition),
	}
	results, err := InspectPromptExecutions(context.Background(), PromptExecutionsInspectionRequest{Resolved: []report.Resolved{first, second}, Executor: executor})
	if err != nil {
		t.Fatalf("InspectPromptExecutions() error = %v", err)
	}
	if len(results) != 2 || len(executor.profileRequests) != 1 {
		t.Fatalf("results/profile requests = %#v/%#v, want two results and one profile inspection", results, executor.profileRequests)
	}
}

type inspectionPromptRequest struct {
	id      string
	version string
}

type inspectionExecutor struct {
	prompt          promptexec.PromptInspection
	prompts         map[string]promptexec.PromptInspection
	profiles        map[string]promptexec.ProfileInspection
	promptErr       error
	promptRequests  []inspectionPromptRequest
	profileRequests []string
}

func (e *inspectionExecutor) InspectPrompt(_ context.Context, id string, version string) (promptexec.PromptInspection, error) {
	e.promptRequests = append(e.promptRequests, inspectionPromptRequest{id: id, version: version})
	if e.promptErr != nil {
		return promptexec.PromptInspection{}, e.promptErr
	}
	if prompt, ok := e.prompts[id]; ok {
		return prompt, nil
	}
	return e.prompt, nil
}

func (e *inspectionExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) {
	e.profileRequests = append(e.profileRequests, id)
	value, ok := e.profiles[id]
	if !ok {
		return promptexec.ProfileInspection{}, errors.New("profile missing")
	}
	return value, nil
}

func (e *inspectionExecutor) Execute(context.Context, promptexec.ExecuteRequest, promptexec.PreparationCallback) (*promptexec.Execution, error) {
	return nil, errors.New("unexpected execution")
}

func inspectionResolved(t *testing.T) report.Resolved {
	t.Helper()
	resolved, err := report.DefaultRegistry().Resolve(report.Daily, report.ResolveRequest{
		Now:      time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC),
		Date:     time.Date(2026, 5, 29, 0, 0, 0, 0, time.UTC),
		Location: time.UTC,
	})
	if err != nil {
		t.Fatalf("Resolve() error = %v", err)
	}
	return resolved
}

func validPromptInspection(definition report.Definition) promptexec.PromptInspection {
	return promptexec.PromptInspection{
		PromptID: definition.PromptID, PromptVersion: definition.PromptVersion, PromptHash: "prompt-hash", DefaultProfileID: "default-profile",
		Inputs: []promptexec.InputDefinition{{Name: "data_package", Required: true, ContentType: "application/yaml"}},
		Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: definition.GeneratedTextSchemaID + ".generated_text.schema.json"},
	}
}