promptkit/frozen_rendering_contract_test.go

package promptkit_test

import (
	"context"
	"errors"
	"fmt"
	"io/fs"
	"os"
	"path/filepath"
	"sync"
	"sync/atomic"
	"testing"
	"testing/fstest"

	"gitea.maximumdirect.net/eric/ConfigKit"
	"gitea.maximumdirect.net/eric/promptkit"
)

const capturedPromptDocument = `
prompts:
  captured:
    version: "1"
    default_profile: offline
    session_id: '{{.session}}'
    messages:
      - role: user
        content_file: body.tmpl
    output:
      format: text
      validation_mode: none
`

func capturedPromptEngine(t *testing.T, source fs.FS) *promptkit.Engine {
	t.Helper()
	inputs, err := promptkit.FilesystemInputs("application prompts", source, "config")
	if err != nil {
		t.Fatalf("select prompt inputs: %v", err)
	}
	endpoint, model := "https://example.invalid/v1", "offline-model"
	engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
		Underlays: []configkit.Underlay{
			{Inputs: inputs},
			{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("offline profile", map[string]promptkit.ProfileConfig{
				"offline": {Endpoint: &endpoint, Model: &model},
			})}}},
		},
	}})
	if err != nil {
		t.Fatalf("construct engine: %v", err)
	}
	return engine
}

func TestEngineCapturesPromptContentUntilReconstruction(t *testing.T) {
	files := fstest.MapFS{
		"config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)},
		"config/body.tmpl":   &fstest.MapFile{Data: []byte("before {{.name}}")},
	}
	first := capturedPromptEngine(t, files)
	files["config/body.tmpl"] = &fstest.MapFile{Data: []byte("after {{.name}}")}
	request := promptkit.RunRequest{PromptID: "captured", Vars: map[string]string{"name": "Ada", "session": "session-1"}}
	oldPrepared, err := first.Prepare(context.Background(), request)
	if err != nil {
		t.Fatalf("prepare with existing engine: %v", err)
	}
	second := capturedPromptEngine(t, files)
	newPrepared, err := second.Prepare(context.Background(), request)
	if err != nil {
		t.Fatalf("prepare with rebuilt engine: %v", err)
	}
	if oldPrepared.Messages[0].Content != "before Ada" || newPrepared.Messages[0].Content != "after Ada" || oldPrepared.PromptHash == newPrepared.PromptHash {
		t.Fatalf("captured content or hash did not follow engine lifetime: old=%+v new=%+v", oldPrepared, newPrepared)
	}
}

type unreadableAfterConstructionFS struct {
	fs.FS
	closed atomic.Bool
	reads  atomic.Int64
}

func (source *unreadableAfterConstructionFS) Open(name string) (fs.File, error) {
	if source.closed.Load() {
		source.reads.Add(1)
		return nil, fs.ErrPermission
	}
	return source.FS.Open(name)
}

func TestCapturedTemplatesBindInputsIndependentlyAcrossConcurrentPreparations(t *testing.T) {
	source := &unreadableAfterConstructionFS{FS: fstest.MapFS{
		"config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)},
		"config/body.tmpl":   &fstest.MapFile{Data: []byte(`{{.name}}: {{input "note"}}`)},
	}}
	engine := capturedPromptEngine(t, source)
	source.closed.Store(true)

	const calls = 24
	var wait sync.WaitGroup
	errs := make(chan error, calls)
	for index := 0; index < calls; index++ {
		wait.Add(1)
		go func(index int) {
			defer wait.Done()
			name := fmt.Sprintf("caller-%d", index)
			body := fmt.Sprintf("body-%d", index)
			prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
				PromptID: "captured", SessionID: "direct-session",
				Vars:   map[string]string{"name": name},
				Inputs: map[string]promptkit.ArtifactRef{"note": promptkit.Inline(body)},
			})
			if err == nil && (prepared.Messages[0].Content != name+": "+body || prepared.SessionID != "direct-session") {
				err = fmt.Errorf("unexpected prepared content: %+v", prepared)
			}
			errs <- err
		}(index)
	}
	wait.Wait()
	close(errs)
	for err := range errs {
		if err != nil {
			t.Fatal(err)
		}
	}
	if source.reads.Load() != 0 {
		t.Fatalf("preparation reopened captured source %d times", source.reads.Load())
	}

	_, err := engine.Prepare(context.Background(), promptkit.RunRequest{
		PromptID: "captured", Vars: map[string]string{"name": "Ada"},
		Inputs: map[string]promptkit.ArtifactRef{"note": promptkit.Inline("body")},
	})
	if !errors.Is(err, promptkit.ErrPromptRender) {
		t.Fatalf("missing session variable = %v, want runtime render error", err)
	}
	_, err = engine.Prepare(context.Background(), promptkit.RunRequest{
		PromptID: "captured", SessionID: "direct-session", Vars: map[string]string{"name": "Ada"},
	})
	if !errors.Is(err, promptkit.ErrPromptRender) {
		t.Fatalf("missing input helper artifact = %v, want runtime render error", err)
	}
}

func TestPreparationStillReadsRequestArtifactsAndSchemasAfterConstruction(t *testing.T) {
	endpoint, model := "https://example.invalid/v1", "offline-model"
	schemas := fstest.MapFS{}
	engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
		Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("profile", map[string]promptkit.ProfileConfig{
			"offline": {Endpoint: &endpoint, Model: &model},
		})}}}},
		Main: &configkit.Input{Source: configkit.Mapping("structured prompt", map[string]any{
			"prompts": map[string]any{"structured": map[string]any{
				"version":         "1",
				"default_profile": "offline",
				"inputs":          []any{map[string]any{"name": "note", "required": true}},
				"messages":        []any{map[string]any{"role": "user", "content": `{{input "note"}}`}},
				"output":          map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"},
			}},
		})},
	}}, promptkit.WithSchemaFS(schemas, "."))
	if err != nil {
		t.Fatalf("construct engine before request resources exist: %v", err)
	}
	schemas["schema.json"] = &fstest.MapFile{Data: []byte(`{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object"}`)}
	artifactPath := filepath.Join(t.TempDir(), "note.txt")
	if err := os.WriteFile(artifactPath, []byte("request artifact"), 0o600); err != nil {
		t.Fatalf("write request artifact: %v", err)
	}
	prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "structured", Inputs: map[string]promptkit.ArtifactRef{
		"note": promptkit.File(artifactPath),
	}})
	if err != nil {
		t.Fatalf("prepare with resources supplied after construction: %v", err)
	}
	if len(prepared.Messages) != 1 || prepared.Messages[0].Content != "request artifact" || prepared.StructuredOutput == nil || prepared.StructuredOutput.JSONSchema == nil {
		t.Fatalf("runtime artifact or schema was not prepared: %+v", prepared)
	}
}