From 0eb009ad5a3248e2a1e12cad9b3a05906d6b0850 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 9 Oct 2026 00:09:15 +0000 Subject: [PATCH] Add validation and rendering benchmarks --- .../compiled_renderer_benchmark_test.go | 254 ++++++++++++++++++ internal/prompt/renderer_test.go | 2 + internal/validate/standard_validator_test.go | 43 +++ 3 files changed, 299 insertions(+) create mode 100644 internal/prompt/compiled_renderer_benchmark_test.go diff --git a/internal/prompt/compiled_renderer_benchmark_test.go b/internal/prompt/compiled_renderer_benchmark_test.go new file mode 100644 index 0000000..85df1b1 --- /dev/null +++ b/internal/prompt/compiled_renderer_benchmark_test.go @@ -0,0 +1,254 @@ +package prompt + +import ( + "bytes" + "context" + "runtime" + "strconv" + "testing" + "text/template" + + "gitea.maximumdirect.net/eric/promptkit/internal/domain" +) + +const rendererBenchmarkArtifactSize = 256 << 10 + +var ( + renderedPromptBenchmarkSink *domain.RenderedPrompt + renderedPromptsBenchmarkSink []*domain.RenderedPrompt +) + +type rendererBenchmarkIdentity struct { + id string + version string +} + +type rendererBenchmarkLookup struct { + identity rendererBenchmarkIdentity + templates CompiledTemplates +} + +func (lookup rendererBenchmarkLookup) GetCompiledTemplates(id, version string) (CompiledTemplates, bool) { + if id != lookup.identity.id || version != lookup.identity.version { + return CompiledTemplates{}, false + } + return lookup.templates, true +} + +type compiledRenderingBenchmarkCase struct { + name string + renderer Renderer + definition *domain.PromptDefinition + inputs map[string]*domain.Artifact + vars map[string]string + ctx context.Context + outputBytes int + check func(*testing.B, *domain.RenderedPrompt) +} + +func BenchmarkCompiledRendering(b *testing.B) { + body := bytes.Repeat([]byte("x"), rendererBenchmarkArtifactSize) + bodyText := string(body) + + small := newCompiledRenderingBenchmarkCase(b, + "small_message", "", []string{"Summary for {{.audience}}."}, + &domain.PromptDefinition{ + ID: "compiled-render-benchmark", Version: "v1", + Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "Summary for {{.audience}}."}}, + }, nil, map[string]string{"audience": "project leads"}, + len("Summary for project leads."), + func(b *testing.B, rendered *domain.RenderedPrompt) { + if rendered.SessionID != "" || len(rendered.Messages) != 1 || rendered.Messages[0].Content != "Summary for project leads." { + b.Fatalf("unexpected small compiled render: %+v", rendered) + } + }, + ) + + artifactInputs := map[string]*domain.Artifact{ + "document": {Name: "document", Body: body}, + } + largeContent := `{{input "document"}}` + largeDefinition := &domain.PromptDefinition{ + ID: "compiled-render-benchmark", Version: "v1", + Inputs: []domain.PromptInput{{Name: "document"}}, + Templates: []domain.PromptMessageTemplate{{Role: "user", Content: largeContent}}, + } + large := newCompiledRenderingBenchmarkCase(b, + "artifact_backed_256KiB", "", []string{largeContent}, + largeDefinition, artifactInputs, nil, len(bodyText), + func(b *testing.B, rendered *domain.RenderedPrompt) { + if len(rendered.Messages) != 1 { + b.Fatalf("artifact render message count=%d, want 1", len(rendered.Messages)) + } + if rendered.Messages[0].Content != bodyText { + b.Fatalf("artifact render length=%d, want %d", len(rendered.Messages[0].Content), len(bodyText)) + } + }, + ) + + repeatedSession := `document-size={{len (input "document")}}` + repeatedMessages := []string{ + `{{input "document"}}`, + `{{input "document"}}|{{input "document"}}`, + } + repeatedDefinition := &domain.PromptDefinition{ + ID: "compiled-render-benchmark", Version: "v1", SessionID: repeatedSession, + Inputs: []domain.PromptInput{{Name: "document"}}, + Templates: []domain.PromptMessageTemplate{ + {Role: "system", Content: repeatedMessages[0]}, + {Role: "user", Content: repeatedMessages[1]}, + }, + } + repeatedOutputBytes := len(bodyText)*3 + 1 + repeated := newCompiledRenderingBenchmarkCase(b, + "repeated_across_session_and_messages", repeatedSession, repeatedMessages, + repeatedDefinition, artifactInputs, nil, repeatedOutputBytes, + func(b *testing.B, rendered *domain.RenderedPrompt) { + if rendered.SessionID != "document-size="+strconv.Itoa(len(bodyText)) || len(rendered.Messages) != 2 || + rendered.Messages[0].Content != bodyText || rendered.Messages[1].Content != bodyText+"|"+bodyText { + b.Fatalf("unexpected repeated compiled render: session=%q message lengths=%v", rendered.SessionID, renderedMessageLengths(rendered)) + } + }, + ) + + for _, benchmark := range []compiledRenderingBenchmarkCase{small, large, repeated} { + b.Run(benchmark.name, func(b *testing.B) { + rendered, err := benchmark.renderer.Render(benchmark.ctx, benchmark.definition, benchmark.inputs, benchmark.vars) + if err != nil { + b.Fatalf("check compiled renderer output: %v", err) + } + benchmark.check(b, rendered) + + b.ReportAllocs() + b.SetBytes(int64(benchmark.outputBytes)) + b.ResetTimer() + for range b.N { + rendered, err := benchmark.renderer.Render(benchmark.ctx, benchmark.definition, benchmark.inputs, benchmark.vars) + if err != nil { + b.Fatalf("render compiled template: %v", err) + } + renderedPromptBenchmarkSink = rendered + } + b.StopTimer() + runtime.KeepAlive(renderedPromptBenchmarkSink) + }) + } +} + +// BenchmarkRetainedRenderedOutput reports post-GC live heap for a fixed batch +// of rendered outputs above the same fixture's setup-only baseline. +func BenchmarkRetainedRenderedOutput(b *testing.B) { + body := bytes.Repeat([]byte("x"), rendererBenchmarkArtifactSize) + bodyText := string(body) + content := `{{input "document"}}` + definition := &domain.PromptDefinition{ + ID: "retained-render-benchmark", Version: "v1", + Inputs: []domain.PromptInput{{Name: "document"}}, + Templates: []domain.PromptMessageTemplate{{Role: "user", Content: content}}, + } + compiled := CompiledTemplates{Messages: []*template.Template{mustCompileRendererBenchmarkTemplate(b, content)}} + renderer := NewCompiledRenderer(rendererBenchmarkLookup{ + identity: rendererBenchmarkIdentity{id: definition.ID, version: definition.Version}, + templates: compiled, + }) + inputs := map[string]*domain.Artifact{"document": {Name: "document", Body: body}} + ctx := context.Background() + const retainedBatchSize = 4 + retained := make([]*domain.RenderedPrompt, retainedBatchSize) + + check, err := renderer.Render(ctx, definition, inputs, nil) + if err != nil { + b.Fatalf("check retained-output fixture: %v", err) + } + if len(check.Messages) != 1 || check.Messages[0].Content != bodyText { + b.Fatalf("retained-output fixture returned unexpected content") + } + check = nil + // Repeated -count runs must start from the same setup-only heap baseline. + renderedPromptBenchmarkSink = nil + renderedPromptsBenchmarkSink = nil + + runtime.GC() + var setupMemory runtime.MemStats + runtime.ReadMemStats(&setupMemory) + runtime.KeepAlive(renderer) + runtime.KeepAlive(definition) + runtime.KeepAlive(inputs) + + b.ReportAllocs() + b.SetBytes(int64(len(bodyText))) + b.ResetTimer() + for iteration := 0; iteration < b.N; iteration++ { + rendered, err := renderer.Render(ctx, definition, inputs, nil) + if err != nil { + b.Fatalf("render retained-output fixture: %v", err) + } + retained[iteration%len(retained)] = rendered + } + b.StopTimer() + for i, rendered := range retained { + if rendered != nil { + continue + } + rendered, err := renderer.Render(ctx, definition, inputs, nil) + if err != nil { + b.Fatalf("fill retained-output fixture: %v", err) + } + retained[i] = rendered + } + renderedPromptsBenchmarkSink = retained + runtime.GC() + var retainedMemory runtime.MemStats + runtime.ReadMemStats(&retainedMemory) + b.ReportMetric(float64(int64(retainedMemory.HeapAlloc)-int64(setupMemory.HeapAlloc)), "retained-B") + runtime.KeepAlive(retained) + runtime.KeepAlive(renderer) + runtime.KeepAlive(definition) + runtime.KeepAlive(inputs) +} + +func newCompiledRenderingBenchmarkCase( + b *testing.B, + name string, + session string, + messages []string, + definition *domain.PromptDefinition, + inputs map[string]*domain.Artifact, + vars map[string]string, + outputBytes int, + check func(*testing.B, *domain.RenderedPrompt), +) compiledRenderingBenchmarkCase { + compiled := CompiledTemplates{} + if session != "" { + compiled.Session = mustCompileRendererBenchmarkTemplate(b, session) + } + for _, content := range messages { + compiled.Messages = append(compiled.Messages, mustCompileRendererBenchmarkTemplate(b, content)) + } + return compiledRenderingBenchmarkCase{ + name: name, + renderer: NewCompiledRenderer(rendererBenchmarkLookup{ + identity: rendererBenchmarkIdentity{id: definition.ID, version: definition.Version}, + templates: compiled, + }), + definition: definition, inputs: inputs, vars: vars, ctx: context.Background(), + outputBytes: outputBytes, check: check, + } +} + +func mustCompileRendererBenchmarkTemplate(b *testing.B, source string) *template.Template { + b.Helper() + compiled, err := CompileTemplate("benchmark", source) + if err != nil { + b.Fatalf("compile benchmark template: %v", err) + } + return compiled +} + +func renderedMessageLengths(rendered *domain.RenderedPrompt) []int { + lengths := make([]int, len(rendered.Messages)) + for i, message := range rendered.Messages { + lengths[i] = len(message.Content) + } + return lengths +} diff --git a/internal/prompt/renderer_test.go b/internal/prompt/renderer_test.go index 08937ab..f1d9426 100644 --- a/internal/prompt/renderer_test.go +++ b/internal/prompt/renderer_test.go @@ -501,6 +501,8 @@ func TestGoRendererArtifactTextLifecycle(t *testing.T) { } } +// BenchmarkGoRendererArtifactReferences measures NewGoRenderer's per-call +// template-parsing fallback rather than the captured compiled path. func BenchmarkGoRendererArtifactReferences(b *testing.B) { body := bytes.Repeat([]byte("document content "), (artifactTextChunkSize*4)/len("document content ")) inputs := map[string]*domain.Artifact{ diff --git a/internal/validate/standard_validator_test.go b/internal/validate/standard_validator_test.go index 72209cb..2368d0c 100644 --- a/internal/validate/standard_validator_test.go +++ b/internal/validate/standard_validator_test.go @@ -1,6 +1,7 @@ package validate import ( + "bytes" "context" "encoding/json" "net/url" @@ -917,6 +918,48 @@ func BenchmarkJSONValidation(b *testing.B) { } } +var basicValidationBenchmarkResult domain.ValidationResult + +func BenchmarkBasicValidation(b *testing.B) { + largeText := bytes.Repeat([]byte("x"), 1<<20) + largeWhitespace := bytes.Repeat([]byte(" \t\n"), (1<<20)/3) + unicodeWhitespace := []byte("\u2003\u00a0\u3000") + benchmarks := []struct { + name string + body []byte + wantValid bool + }{ + {name: "short_text", body: []byte("A concise response."), wantValid: true}, + {name: "1MiB_nonempty_text", body: largeText, wantValid: true}, + {name: "whitespace_only_text", body: largeWhitespace}, + {name: "unicode_whitespace", body: unicodeWhitespace}, + } + validator := NewStandardValidator("") + contract := domain.OutputContract{ValidationMode: domain.ValidationBasic} + ctx := context.Background() + + for _, benchmark := range benchmarks { + b.Run(benchmark.name, func(b *testing.B) { + artifact := &domain.Artifact{Body: benchmark.body} + result, err := validator.Validate(ctx, artifact, contract) + if err != nil || result.IsValid != benchmark.wantValid { + b.Fatalf("validate basic output: result=%+v error=%v", result, err) + } + + b.ReportAllocs() + b.SetBytes(int64(len(benchmark.body))) + b.ResetTimer() + for range b.N { + result, err := validator.Validate(ctx, artifact, contract) + if err != nil || result.IsValid != benchmark.wantValid { + b.Fatalf("validate basic output: result=%+v error=%v", result, err) + } + basicValidationBenchmarkResult = result + } + }) + } +} + func TestJSONSchemaDialectIsDraft2020(t *testing.T) { tests := []struct { name string