promptkit/internal/prompt/compiled_renderer_benchmark_test.go

package prompt

import (
	"bytes"
	"context"
	"runtime"
	"strconv"
	"testing"
	"text/template"

	"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)

const rendererBenchmarkArtifactSize = 256 << 10

var (
	renderedPromptBenchmarkSink  *domain.RenderedPrompt
	renderedPromptsBenchmarkSink []*domain.RenderedPrompt
)

type rendererBenchmarkIdentity struct {
	id      string
	version string
}

type rendererBenchmarkLookup struct {
	identity  rendererBenchmarkIdentity
	templates CompiledTemplates
}

func (lookup rendererBenchmarkLookup) GetCompiledTemplates(id, version string) (CompiledTemplates, bool) {
	if id != lookup.identity.id || version != lookup.identity.version {
		return CompiledTemplates{}, false
	}
	return lookup.templates, true
}

type compiledRenderingBenchmarkCase struct {
	name        string
	renderer    Renderer
	definition  *domain.PromptDefinition
	inputs      map[string]*domain.Artifact
	vars        map[string]string
	ctx         context.Context
	outputBytes int
	check       func(*testing.B, *domain.RenderedPrompt)
}

func BenchmarkCompiledRendering(b *testing.B) {
	body := bytes.Repeat([]byte("x"), rendererBenchmarkArtifactSize)
	bodyText := string(body)

	small := newCompiledRenderingBenchmarkCase(b,
		"small_message", "", []string{"Summary for {{.audience}}."},
		&domain.PromptDefinition{
			ID: "compiled-render-benchmark", Version: "v1",
			Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "Summary for {{.audience}}."}},
		}, nil, map[string]string{"audience": "project leads"},
		len("Summary for project leads."),
		func(b *testing.B, rendered *domain.RenderedPrompt) {
			if rendered.SessionID != "" || len(rendered.Messages) != 1 || rendered.Messages[0].Content != "Summary for project leads." {
				b.Fatalf("unexpected small compiled render: %+v", rendered)
			}
		},
	)

	artifactInputs := map[string]*domain.Artifact{
		"document": {Name: "document", Body: body},
	}
	largeContent := `{{input "document"}}`
	largeDefinition := &domain.PromptDefinition{
		ID: "compiled-render-benchmark", Version: "v1",
		Inputs:    []domain.PromptInput{{Name: "document"}},
		Templates: []domain.PromptMessageTemplate{{Role: "user", Content: largeContent}},
	}
	large := newCompiledRenderingBenchmarkCase(b,
		"artifact_backed_256KiB", "", []string{largeContent},
		largeDefinition, artifactInputs, nil, len(bodyText),
		func(b *testing.B, rendered *domain.RenderedPrompt) {
			if len(rendered.Messages) != 1 {
				b.Fatalf("artifact render message count=%d, want 1", len(rendered.Messages))
			}
			if rendered.Messages[0].Content != bodyText {
				b.Fatalf("artifact render length=%d, want %d", len(rendered.Messages[0].Content), len(bodyText))
			}
		},
	)

	repeatedSession := `document-size={{len (input "document")}}`
	repeatedMessages := []string{
		`{{input "document"}}`,
		`{{input "document"}}|{{input "document"}}`,
	}
	repeatedDefinition := &domain.PromptDefinition{
		ID: "compiled-render-benchmark", Version: "v1", SessionID: repeatedSession,
		Inputs: []domain.PromptInput{{Name: "document"}},
		Templates: []domain.PromptMessageTemplate{
			{Role: "system", Content: repeatedMessages[0]},
			{Role: "user", Content: repeatedMessages[1]},
		},
	}
	repeatedOutputBytes := len(bodyText)*3 + 1
	repeated := newCompiledRenderingBenchmarkCase(b,
		"repeated_across_session_and_messages", repeatedSession, repeatedMessages,
		repeatedDefinition, artifactInputs, nil, repeatedOutputBytes,
		func(b *testing.B, rendered *domain.RenderedPrompt) {
			if rendered.SessionID != "document-size="+strconv.Itoa(len(bodyText)) || len(rendered.Messages) != 2 ||
				rendered.Messages[0].Content != bodyText || rendered.Messages[1].Content != bodyText+"|"+bodyText {
				b.Fatalf("unexpected repeated compiled render: session=%q message lengths=%v", rendered.SessionID, renderedMessageLengths(rendered))
			}
		},
	)

	for _, benchmark := range []compiledRenderingBenchmarkCase{small, large, repeated} {
		b.Run(benchmark.name, func(b *testing.B) {
			rendered, err := benchmark.renderer.Render(benchmark.ctx, benchmark.definition, benchmark.inputs, benchmark.vars)
			if err != nil {
				b.Fatalf("check compiled renderer output: %v", err)
			}
			benchmark.check(b, rendered)

			b.ReportAllocs()
			b.SetBytes(int64(benchmark.outputBytes))
			b.ResetTimer()
			for range b.N {
				rendered, err := benchmark.renderer.Render(benchmark.ctx, benchmark.definition, benchmark.inputs, benchmark.vars)
				if err != nil {
					b.Fatalf("render compiled template: %v", err)
				}
				renderedPromptBenchmarkSink = rendered
			}
			b.StopTimer()
			runtime.KeepAlive(renderedPromptBenchmarkSink)
		})
	}
}

// BenchmarkRetainedRenderedOutput reports post-GC live heap for a fixed batch
// of rendered outputs above the same fixture's setup-only baseline.
func BenchmarkRetainedRenderedOutput(b *testing.B) {
	body := bytes.Repeat([]byte("x"), rendererBenchmarkArtifactSize)
	bodyText := string(body)
	content := `{{input "document"}}`
	definition := &domain.PromptDefinition{
		ID: "retained-render-benchmark", Version: "v1",
		Inputs:    []domain.PromptInput{{Name: "document"}},
		Templates: []domain.PromptMessageTemplate{{Role: "user", Content: content}},
	}
	compiled := CompiledTemplates{Messages: []*template.Template{mustCompileRendererBenchmarkTemplate(b, content)}}
	renderer := NewCompiledRenderer(rendererBenchmarkLookup{
		identity:  rendererBenchmarkIdentity{id: definition.ID, version: definition.Version},
		templates: compiled,
	})
	inputs := map[string]*domain.Artifact{"document": {Name: "document", Body: body}}
	ctx := context.Background()
	const retainedBatchSize = 4
	retained := make([]*domain.RenderedPrompt, retainedBatchSize)

	check, err := renderer.Render(ctx, definition, inputs, nil)
	if err != nil {
		b.Fatalf("check retained-output fixture: %v", err)
	}
	if len(check.Messages) != 1 || check.Messages[0].Content != bodyText {
		b.Fatalf("retained-output fixture returned unexpected content")
	}
	check = nil
	// Repeated -count runs must start from the same setup-only heap baseline.
	renderedPromptBenchmarkSink = nil
	renderedPromptsBenchmarkSink = nil

	runtime.GC()
	var setupMemory runtime.MemStats
	runtime.ReadMemStats(&setupMemory)
	runtime.KeepAlive(renderer)
	runtime.KeepAlive(definition)
	runtime.KeepAlive(inputs)

	b.ReportAllocs()
	b.SetBytes(int64(len(bodyText)))
	b.ResetTimer()
	for iteration := 0; iteration < b.N; iteration++ {
		rendered, err := renderer.Render(ctx, definition, inputs, nil)
		if err != nil {
			b.Fatalf("render retained-output fixture: %v", err)
		}
		retained[iteration%len(retained)] = rendered
	}
	b.StopTimer()
	for i, rendered := range retained {
		if rendered != nil {
			continue
		}
		rendered, err := renderer.Render(ctx, definition, inputs, nil)
		if err != nil {
			b.Fatalf("fill retained-output fixture: %v", err)
		}
		retained[i] = rendered
	}
	renderedPromptsBenchmarkSink = retained
	runtime.GC()
	var retainedMemory runtime.MemStats
	runtime.ReadMemStats(&retainedMemory)
	b.ReportMetric(float64(int64(retainedMemory.HeapAlloc)-int64(setupMemory.HeapAlloc)), "retained-B")
	runtime.KeepAlive(retained)
	runtime.KeepAlive(renderer)
	runtime.KeepAlive(definition)
	runtime.KeepAlive(inputs)
}

func newCompiledRenderingBenchmarkCase(
	b *testing.B,
	name string,
	session string,
	messages []string,
	definition *domain.PromptDefinition,
	inputs map[string]*domain.Artifact,
	vars map[string]string,
	outputBytes int,
	check func(*testing.B, *domain.RenderedPrompt),
) compiledRenderingBenchmarkCase {
	compiled := CompiledTemplates{}
	if session != "" {
		compiled.Session = mustCompileRendererBenchmarkTemplate(b, session)
	}
	for _, content := range messages {
		compiled.Messages = append(compiled.Messages, mustCompileRendererBenchmarkTemplate(b, content))
	}
	return compiledRenderingBenchmarkCase{
		name: name,
		renderer: NewCompiledRenderer(rendererBenchmarkLookup{
			identity:  rendererBenchmarkIdentity{id: definition.ID, version: definition.Version},
			templates: compiled,
		}),
		definition: definition, inputs: inputs, vars: vars, ctx: context.Background(),
		outputBytes: outputBytes, check: check,
	}
}

func mustCompileRendererBenchmarkTemplate(b *testing.B, source string) *template.Template {
	b.Helper()
	compiled, err := CompileTemplate("benchmark", source)
	if err != nil {
		b.Fatalf("compile benchmark template: %v", err)
	}
	return compiled
}

func renderedMessageLengths(rendered *domain.RenderedPrompt) []int {
	lengths := make([]int, len(rendered.Messages))
	for i, message := range rendered.Messages {
		lengths[i] = len(message.Content)
	}
	return lengths
}