scriptorium/engine_test.go

package scriptorium_test

import (
	"context"
	"encoding/json"
	"errors"
	"os"
	"strings"
	"testing"

	"gitea.maximumdirect.net/eric/scriptorium"
)

func TestNewEngineRejectsMissingPromptDir(t *testing.T) {
	_, err := scriptorium.NewEngine(scriptorium.Config{ProfileDir: "./examples/profiles"})
	if !errors.Is(err, scriptorium.ErrInvalidConfig) {
		t.Fatalf("expected ErrInvalidConfig, got %v", err)
	}
}

func TestNewEngineRejectsMissingProfileDir(t *testing.T) {
	_, err := scriptorium.NewEngine(scriptorium.Config{PromptDir: "./examples/prompts"})
	if !errors.Is(err, scriptorium.ErrInvalidConfig) {
		t.Fatalf("expected ErrInvalidConfig, got %v", err)
	}
}

func TestPrepareWorksWithExampleDirectoriesAndFileInputs(t *testing.T) {
	engine := newExampleEngine(t)

	prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
		PromptID: "generic.markdown_summary",
		Inputs: map[string]scriptorium.ArtifactRef{
			"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
			"glossary":   scriptorium.File("./examples/fixtures/glossary.yml"),
		},
	})
	if err != nil {
		t.Fatalf("expected prepare to succeed, got %v", err)
	}
	if prepared.PromptID != "generic.markdown_summary" {
		t.Fatalf("unexpected prompt id: %q", prepared.PromptID)
	}
	if prepared.SelectedProfileID != "local-fast" {
		t.Fatalf("unexpected selected profile: %q", prepared.SelectedProfileID)
	}
	if prepared.EffectiveModelParams.Model != "gpt-4o-mini" {
		t.Fatalf("unexpected effective model: %q", prepared.EffectiveModelParams.Model)
	}
	if len(prepared.Messages) != 2 {
		t.Fatalf("expected rendered messages, got %d", len(prepared.Messages))
	}
	if prepared.InputHashes["transcript"] == "" || prepared.InputHashes["glossary"] == "" {
		t.Fatalf("expected input hashes, got %#v", prepared.InputHashes)
	}
}

func TestPrepareWorksWithInlineInputs(t *testing.T) {
	engine := newExampleEngine(t)

	prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
		PromptID: "generic.markdown_summary",
		Inputs: map[string]scriptorium.ArtifactRef{
			"transcript": scriptorium.Inline("Rin scouts the tower.\nKara lights a lantern."),
			"glossary":   scriptorium.InlineWithURI("memory://glossary.yml", "party:\n  - Rin\n  - Kara\n"),
		},
	})
	if err != nil {
		t.Fatalf("expected prepare to succeed, got %v", err)
	}
	if len(prepared.Messages) != 2 {
		t.Fatalf("expected rendered messages, got %d", len(prepared.Messages))
	}
	rendered := prepared.Messages[1].Content
	if !strings.Contains(rendered, "Rin scouts the tower.") || !strings.Contains(rendered, "party:") {
		t.Fatalf("expected inline inputs in rendered prompt, got %q", rendered)
	}
}

func TestPreparedRunJSONDoesNotExposeSecretOrTargetPresence(t *testing.T) {
	const envName = "SCRIPTORIUM_API_KEY"
	const secret = "public-api-test-secret"
	t.Setenv(envName, secret)

	engine := newExampleEngine(t)
	prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
		PromptID: "generic.structured_events",
		Inputs: map[string]scriptorium.ArtifactRef{
			"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
			"glossary":   scriptorium.File("./examples/fixtures/glossary.yml"),
		},
	})
	if err != nil {
		t.Fatalf("expected prepare to succeed, got %v", err)
	}

	payload, err := json.Marshal(prepared)
	if err != nil {
		t.Fatalf("expected prepared run to marshal, got %v", err)
	}
	out := string(payload)
	if strings.Contains(out, secret) {
		t.Fatalf("prepared run JSON leaked raw API key value: %s", out)
	}
	if !strings.Contains(out, envName) {
		t.Fatalf("prepared run JSON should retain api_key_env name, got %s", out)
	}
	for _, forbidden := range []string{"TargetPresence", "target_presence"} {
		if strings.Contains(out, forbidden) {
			t.Fatalf("prepared run JSON exposed internal target presence metadata %q: %s", forbidden, out)
		}
	}
}

func TestPreparePreservesExplicitZeroExecutionOverrides(t *testing.T) {
	engine := newExampleEngine(t)
	zeroFloat := 0.0
	zeroInt := 0

	prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
		PromptID: "generic.markdown_summary",
		Inputs: map[string]scriptorium.ArtifactRef{
			"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
			"glossary":   scriptorium.File("./examples/fixtures/glossary.yml"),
		},
		Execution: &scriptorium.ExecutionTargetOverride{
			Temperature:    &zeroFloat,
			MaxTokens:      &zeroInt,
			TopP:           &zeroFloat,
			TimeoutSeconds: &zeroInt,
		},
	})
	if err != nil {
		t.Fatalf("expected prepare to succeed, got %v", err)
	}
	target := prepared.EffectiveModelParams
	if target.Temperature != 0 || target.MaxTokens != 0 || target.TopP != 0 || target.TimeoutSeconds != 0 {
		t.Fatalf("expected explicit zero overrides in effective target, got %+v", target)
	}
}

func newExampleEngine(t *testing.T) *scriptorium.Engine {
	t.Helper()

	for _, path := range []string{
		"./examples/prompts",
		"./examples/profiles",
		"./examples/schemas",
	} {
		if _, err := os.Stat(path); err != nil {
			t.Fatalf("expected example path %s to exist: %v", path, err)
		}
	}

	engine, err := scriptorium.NewEngine(scriptorium.Config{
		PromptDir:  "./examples/prompts",
		ProfileDir: "./examples/profiles",
		SchemaDir:  "./examples/schemas",
	})
	if err != nil {
		t.Fatalf("expected engine construction to succeed, got %v", err)
	}
	return engine
}