Add validation and rendering benchmarks

2026-10-09 00:09:15 +00:00
parent 3fde1f6cbb
commit 0eb009ad5a
3 changed files with 299 additions and 0 deletions
--- a/internal/prompt/compiled_renderer_benchmark_test.go
+++ b/internal/prompt/compiled_renderer_benchmark_test.go
@@ -0,0 +1,254 @@
+package prompt
+
+import (
+	"bytes"
+	"context"
+	"runtime"
+	"strconv"
+	"testing"
+	"text/template"
+
+	"gitea.maximumdirect.net/eric/promptkit/internal/domain"
+)
+
+const rendererBenchmarkArtifactSize = 256 << 10
+
+var (
+	renderedPromptBenchmarkSink  *domain.RenderedPrompt
+	renderedPromptsBenchmarkSink []*domain.RenderedPrompt
+)
+
+type rendererBenchmarkIdentity struct {
+	id      string
+	version string
+}
+
+type rendererBenchmarkLookup struct {
+	identity  rendererBenchmarkIdentity
+	templates CompiledTemplates
+}
+
+func (lookup rendererBenchmarkLookup) GetCompiledTemplates(id, version string) (CompiledTemplates, bool) {
+	if id != lookup.identity.id || version != lookup.identity.version {
+		return CompiledTemplates{}, false
+	}
+	return lookup.templates, true
+}
+
+type compiledRenderingBenchmarkCase struct {
+	name        string
+	renderer    Renderer
+	definition  *domain.PromptDefinition
+	inputs      map[string]*domain.Artifact
+	vars        map[string]string
+	ctx         context.Context
+	outputBytes int
+	check       func(*testing.B, *domain.RenderedPrompt)
+}
+
+func BenchmarkCompiledRendering(b *testing.B) {
+	body := bytes.Repeat([]byte("x"), rendererBenchmarkArtifactSize)
+	bodyText := string(body)
+
+	small := newCompiledRenderingBenchmarkCase(b,
+		"small_message", "", []string{"Summary for {{.audience}}."},
+		&domain.PromptDefinition{
+			ID: "compiled-render-benchmark", Version: "v1",
+			Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "Summary for {{.audience}}."}},
+		}, nil, map[string]string{"audience": "project leads"},
+		len("Summary for project leads."),
+		func(b *testing.B, rendered *domain.RenderedPrompt) {
+			if rendered.SessionID != "" || len(rendered.Messages) != 1 || rendered.Messages[0].Content != "Summary for project leads." {
+				b.Fatalf("unexpected small compiled render: %+v", rendered)
+			}
+		},
+	)
+
+	artifactInputs := map[string]*domain.Artifact{
+		"document": {Name: "document", Body: body},
+	}
+	largeContent := `{{input "document"}}`
+	largeDefinition := &domain.PromptDefinition{
+		ID: "compiled-render-benchmark", Version: "v1",
+		Inputs:    []domain.PromptInput{{Name: "document"}},
+		Templates: []domain.PromptMessageTemplate{{Role: "user", Content: largeContent}},
+	}
+	large := newCompiledRenderingBenchmarkCase(b,
+		"artifact_backed_256KiB", "", []string{largeContent},
+		largeDefinition, artifactInputs, nil, len(bodyText),
+		func(b *testing.B, rendered *domain.RenderedPrompt) {
+			if len(rendered.Messages) != 1 {
+				b.Fatalf("artifact render message count=%d, want 1", len(rendered.Messages))
+			}
+			if rendered.Messages[0].Content != bodyText {
+				b.Fatalf("artifact render length=%d, want %d", len(rendered.Messages[0].Content), len(bodyText))
+			}
+		},
+	)
+
+	repeatedSession := `document-size={{len (input "document")}}`
+	repeatedMessages := []string{
+		`{{input "document"}}`,
+		`{{input "document"}}|{{input "document"}}`,
+	}
+	repeatedDefinition := &domain.PromptDefinition{
+		ID: "compiled-render-benchmark", Version: "v1", SessionID: repeatedSession,
+		Inputs: []domain.PromptInput{{Name: "document"}},
+		Templates: []domain.PromptMessageTemplate{
+			{Role: "system", Content: repeatedMessages[0]},
+			{Role: "user", Content: repeatedMessages[1]},
+		},
+	}
+	repeatedOutputBytes := len(bodyText)*3 + 1
+	repeated := newCompiledRenderingBenchmarkCase(b,
+		"repeated_across_session_and_messages", repeatedSession, repeatedMessages,
+		repeatedDefinition, artifactInputs, nil, repeatedOutputBytes,
+		func(b *testing.B, rendered *domain.RenderedPrompt) {
+			if rendered.SessionID != "document-size="+strconv.Itoa(len(bodyText)) || len(rendered.Messages) != 2 ||
+				rendered.Messages[0].Content != bodyText || rendered.Messages[1].Content != bodyText+"|"+bodyText {
+				b.Fatalf("unexpected repeated compiled render: session=%q message lengths=%v", rendered.SessionID, renderedMessageLengths(rendered))
+			}
+		},
+	)
+
+	for _, benchmark := range []compiledRenderingBenchmarkCase{small, large, repeated} {
+		b.Run(benchmark.name, func(b *testing.B) {
+			rendered, err := benchmark.renderer.Render(benchmark.ctx, benchmark.definition, benchmark.inputs, benchmark.vars)
+			if err != nil {
+				b.Fatalf("check compiled renderer output: %v", err)
+			}
+			benchmark.check(b, rendered)
+
+			b.ReportAllocs()
+			b.SetBytes(int64(benchmark.outputBytes))
+			b.ResetTimer()
+			for range b.N {
+				rendered, err := benchmark.renderer.Render(benchmark.ctx, benchmark.definition, benchmark.inputs, benchmark.vars)
+				if err != nil {
+					b.Fatalf("render compiled template: %v", err)
+				}
+				renderedPromptBenchmarkSink = rendered
+			}
+			b.StopTimer()
+			runtime.KeepAlive(renderedPromptBenchmarkSink)
+		})
+	}
+}
+
+// BenchmarkRetainedRenderedOutput reports post-GC live heap for a fixed batch
+// of rendered outputs above the same fixture's setup-only baseline.
+func BenchmarkRetainedRenderedOutput(b *testing.B) {
+	body := bytes.Repeat([]byte("x"), rendererBenchmarkArtifactSize)
+	bodyText := string(body)
+	content := `{{input "document"}}`
+	definition := &domain.PromptDefinition{
+		ID: "retained-render-benchmark", Version: "v1",
+		Inputs:    []domain.PromptInput{{Name: "document"}},
+		Templates: []domain.PromptMessageTemplate{{Role: "user", Content: content}},
+	}
+	compiled := CompiledTemplates{Messages: []*template.Template{mustCompileRendererBenchmarkTemplate(b, content)}}
+	renderer := NewCompiledRenderer(rendererBenchmarkLookup{
+		identity:  rendererBenchmarkIdentity{id: definition.ID, version: definition.Version},
+		templates: compiled,
+	})
+	inputs := map[string]*domain.Artifact{"document": {Name: "document", Body: body}}
+	ctx := context.Background()
+	const retainedBatchSize = 4
+	retained := make([]*domain.RenderedPrompt, retainedBatchSize)
+
+	check, err := renderer.Render(ctx, definition, inputs, nil)
+	if err != nil {
+		b.Fatalf("check retained-output fixture: %v", err)
+	}
+	if len(check.Messages) != 1 || check.Messages[0].Content != bodyText {
+		b.Fatalf("retained-output fixture returned unexpected content")
+	}
+	check = nil
+	// Repeated -count runs must start from the same setup-only heap baseline.
+	renderedPromptBenchmarkSink = nil
+	renderedPromptsBenchmarkSink = nil
+
+	runtime.GC()
+	var setupMemory runtime.MemStats
+	runtime.ReadMemStats(&setupMemory)
+	runtime.KeepAlive(renderer)
+	runtime.KeepAlive(definition)
+	runtime.KeepAlive(inputs)
+
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bodyText)))
+	b.ResetTimer()
+	for iteration := 0; iteration < b.N; iteration++ {
+		rendered, err := renderer.Render(ctx, definition, inputs, nil)
+		if err != nil {
+			b.Fatalf("render retained-output fixture: %v", err)
+		}
+		retained[iteration%len(retained)] = rendered
+	}
+	b.StopTimer()
+	for i, rendered := range retained {
+		if rendered != nil {
+			continue
+		}
+		rendered, err := renderer.Render(ctx, definition, inputs, nil)
+		if err != nil {
+			b.Fatalf("fill retained-output fixture: %v", err)
+		}
+		retained[i] = rendered
+	}
+	renderedPromptsBenchmarkSink = retained
+	runtime.GC()
+	var retainedMemory runtime.MemStats
+	runtime.ReadMemStats(&retainedMemory)
+	b.ReportMetric(float64(int64(retainedMemory.HeapAlloc)-int64(setupMemory.HeapAlloc)), "retained-B")
+	runtime.KeepAlive(retained)
+	runtime.KeepAlive(renderer)
+	runtime.KeepAlive(definition)
+	runtime.KeepAlive(inputs)
+}
+
+func newCompiledRenderingBenchmarkCase(
+	b *testing.B,
+	name string,
+	session string,
+	messages []string,
+	definition *domain.PromptDefinition,
+	inputs map[string]*domain.Artifact,
+	vars map[string]string,
+	outputBytes int,
+	check func(*testing.B, *domain.RenderedPrompt),
+) compiledRenderingBenchmarkCase {
+	compiled := CompiledTemplates{}
+	if session != "" {
+		compiled.Session = mustCompileRendererBenchmarkTemplate(b, session)
+	}
+	for _, content := range messages {
+		compiled.Messages = append(compiled.Messages, mustCompileRendererBenchmarkTemplate(b, content))
+	}
+	return compiledRenderingBenchmarkCase{
+		name: name,
+		renderer: NewCompiledRenderer(rendererBenchmarkLookup{
+			identity:  rendererBenchmarkIdentity{id: definition.ID, version: definition.Version},
+			templates: compiled,
+		}),
+		definition: definition, inputs: inputs, vars: vars, ctx: context.Background(),
+		outputBytes: outputBytes, check: check,
+	}
+}
+
+func mustCompileRendererBenchmarkTemplate(b *testing.B, source string) *template.Template {
+	b.Helper()
+	compiled, err := CompileTemplate("benchmark", source)
+	if err != nil {
+		b.Fatalf("compile benchmark template: %v", err)
+	}
+	return compiled
+}
+
+func renderedMessageLengths(rendered *domain.RenderedPrompt) []int {
+	lengths := make([]int, len(rendered.Messages))
+	for i, message := range rendered.Messages {
+		lengths[i] = len(message.Content)
+	}
+	return lengths
+}
--- a/internal/prompt/renderer_test.go
+++ b/internal/prompt/renderer_test.go
@@ -501,6 +501,8 @@ func TestGoRendererArtifactTextLifecycle(t *testing.T) {
 	}
 }

+// BenchmarkGoRendererArtifactReferences measures NewGoRenderer's per-call
+// template-parsing fallback rather than the captured compiled path.
 func BenchmarkGoRendererArtifactReferences(b *testing.B) {
 	body := bytes.Repeat([]byte("document content "), (artifactTextChunkSize*4)/len("document content "))
 	inputs := map[string]*domain.Artifact{
--- a/internal/validate/standard_validator_test.go
+++ b/internal/validate/standard_validator_test.go
@@ -1,6 +1,7 @@
 package validate

 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"net/url"
@@ -917,6 +918,48 @@ func BenchmarkJSONValidation(b *testing.B) {
 	}
 }

+var basicValidationBenchmarkResult domain.ValidationResult
+
+func BenchmarkBasicValidation(b *testing.B) {
+	largeText := bytes.Repeat([]byte("x"), 1<<20)
+	largeWhitespace := bytes.Repeat([]byte(" \t\n"), (1<<20)/3)
+	unicodeWhitespace := []byte("\u2003\u00a0\u3000")
+	benchmarks := []struct {
+		name      string
+		body      []byte
+		wantValid bool
+	}{
+		{name: "short_text", body: []byte("A concise response."), wantValid: true},
+		{name: "1MiB_nonempty_text", body: largeText, wantValid: true},
+		{name: "whitespace_only_text", body: largeWhitespace},
+		{name: "unicode_whitespace", body: unicodeWhitespace},
+	}
+	validator := NewStandardValidator("")
+	contract := domain.OutputContract{ValidationMode: domain.ValidationBasic}
+	ctx := context.Background()
+
+	for _, benchmark := range benchmarks {
+		b.Run(benchmark.name, func(b *testing.B) {
+			artifact := &domain.Artifact{Body: benchmark.body}
+			result, err := validator.Validate(ctx, artifact, contract)
+			if err != nil || result.IsValid != benchmark.wantValid {
+				b.Fatalf("validate basic output: result=%+v error=%v", result, err)
+			}
+
+			b.ReportAllocs()
+			b.SetBytes(int64(len(benchmark.body)))
+			b.ResetTimer()
+			for range b.N {
+				result, err := validator.Validate(ctx, artifact, contract)
+				if err != nil || result.IsValid != benchmark.wantValid {
+					b.Fatalf("validate basic output: result=%+v error=%v", result, err)
+				}
+				basicValidationBenchmarkResult = result
+			}
+		})
+	}
+}
+
 func TestJSONSchemaDialectIsDraft2020(t *testing.T) {
 	tests := []struct {
 		name      string