eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Publish effective LLM backend provenance

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-30 02:18:14 +00:00
parent f8333f2c15
commit 71a004bfc8
11 changed files with 194 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
internal/framework/llm/promptkit_client.go
View File

@@ -155,9 +155,11 @@ func (c *PromptKitClient) responseFromResult(result *promptkit.RunResult, prepar
content = []byte(result.RawOutput)
}
profile := artifacts.LLMProfileManifest{
ID: strings.TrimSpace(result.SelectedProfileID),
Provider: promptKitProviderName,
Model: firstNonEmpty(result.ModelName, result.EffectiveModelParams.Model),
ID: strings.TrimSpace(result.SelectedProfileID),
Provider: promptKitProviderName,
Model: firstNonEmpty(result.ModelName, result.EffectiveModelParams.Model),
BackendID: strings.TrimSpace(result.SelectedBackendID),
ReasoningEffort: strings.TrimSpace(result.EffectiveModelParams.ReasoningEffort),
}
if c.recorder != nil {
c.recorder.Record(profile)
@@ -205,6 +207,7 @@ func promptKitDebugPrompt(prepared *promptkit.PreparedRun) *contracts.LLMDebugPr
PromptVersion: prepared.PromptVersion,
PromptHash: prepared.PromptHash,
SelectedProfileID: prepared.SelectedProfileID,
SelectedBackendID: prepared.SelectedBackendID,
SessionID: prepared.SessionID,
RenderedPromptHash: prepared.RenderedPromptHash,
Messages: messages,
@@ -304,7 +307,9 @@ func (r *LLMProfileRecorder) Record(profile artifacts.LLMProfileManifest) {
profile.ID = strings.TrimSpace(profile.ID)
profile.Provider = strings.TrimSpace(profile.Provider)
profile.Model = strings.TrimSpace(profile.Model)
key := profile.ID + "\x00" + profile.Provider + "\x00" + profile.Model
profile.BackendID = strings.TrimSpace(profile.BackendID)
profile.ReasoningEffort = strings.TrimSpace(profile.ReasoningEffort)
key := profile.ID + "\x00" + profile.Provider + "\x00" + profile.Model + "\x00" + profile.BackendID + "\x00" + profile.ReasoningEffort
r.mu.Lock()
defer r.mu.Unlock()
if r.profiles == nil {

50
internal/framework/llm/promptkit_client_test.go
View File

@@ -8,6 +8,7 @@ import (
"net/http"
"os"
"path/filepath"
"reflect"
"strings"
"sync"
"sync/atomic"
@@ -15,6 +16,7 @@ import (
"testing/fstest"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/promptkit"
)
@@ -54,8 +56,13 @@ func TestPromptKitClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
}
if resp.Debug.Prompt.PromptID != "adapter.direct-session" ||
resp.Debug.Prompt.SelectedProfileID != "explicit-profile" ||
resp.Debug.Prompt.SelectedBackendID != "test-backend" ||
resp.Debug.Prompt.SessionID != "session-123" {
t.Fatalf("debug prompt metadata = %#v, want prompt/profile/session", resp.Debug.Prompt)
t.Fatalf("debug prompt metadata = %#v, want prompt/profile/backend/session", resp.Debug.Prompt)
}
if resp.Debug.Prompt.EffectiveModelParams["backend_id"] != "test-backend" ||
resp.Debug.Prompt.EffectiveModelParams["reasoning_effort"] != "profile-reasoning" {
t.Fatalf("debug effective model params = %#v, want backend and reasoning", resp.Debug.Prompt.EffectiveModelParams)
}
if len(resp.Debug.Prompt.Messages) != 1 || !strings.Contains(resp.Debug.Prompt.Messages[0].Content, `{"source":true}`) {
t.Fatalf("debug prompt messages = %#v, want rendered input content", resp.Debug.Prompt.Messages)
@@ -66,6 +73,10 @@ func TestPromptKitClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
if resp.Debug.Response.Usage.CachedTokens != 5 || resp.Debug.Response.Usage.CacheWriteTokens != 3 {
t.Fatalf("debug usage = %#v, want cached token counts", resp.Debug.Response.Usage)
}
if resp.Debug.Response.EffectiveModelParams["backend_id"] != "test-backend" ||
resp.Debug.Response.EffectiveModelParams["reasoning_effort"] != "profile-reasoning" {
t.Fatalf("debug response effective model params = %#v, want backend and reasoning", resp.Debug.Response.EffectiveModelParams)
}
debugJSON, err := json.Marshal(resp.Debug)
if err != nil {
t.Fatalf("marshal debug material: %v", err)
@@ -80,6 +91,9 @@ func TestPromptKitClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
if gotReq.Target.Model != "explicit-model" {
t.Fatalf("model = %q, want explicit-model", gotReq.Target.Model)
}
if gotReq.Target.BackendID != "test-backend" {
t.Fatalf("backend id = %q, want test-backend", gotReq.Target.BackendID)
}
if len(gotReq.Prompt.Messages) != 1 ||
!strings.Contains(gotReq.Prompt.Messages[0].Content, `{"source":true}`) ||
!strings.Contains(gotReq.Prompt.Messages[0].Content, "value") {
@@ -92,7 +106,9 @@ func TestPromptKitClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
if len(manifests) != 1 ||
manifests[0].ID != "explicit-profile" ||
manifests[0].Provider != "promptkit" ||
manifests[0].Model != "explicit-model" {
manifests[0].Model != "explicit-model" ||
manifests[0].BackendID != "test-backend" ||
manifests[0].ReasoningEffort != "profile-reasoning" {
t.Fatalf("profile manifests = %#v", manifests)
}
}
@@ -312,6 +328,30 @@ func TestPromptKitClientUsesPromptDefaultProfileWhenRequestProfileEmpty(t *testi
if got := fake.lastRequest().Target.Model; got != "default-model" {
t.Fatalf("model = %q, want prompt default profile model", got)
}
manifests := client.LLMProfileManifests()
if len(manifests) != 1 || manifests[0].BackendID != "" || manifests[0].ReasoningEffort != "profile-reasoning" {
t.Fatalf("endpoint-only profile manifests = %#v, want omitted backend and effective reasoning", manifests)
}
}
func TestLLMProfileRecorderDistinguishesEffectiveTargets(t *testing.T) {
recorder := NewLLMProfileRecorder()
for _, profile := range []artifacts.LLMProfileManifest{
{ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-b", ReasoningEffort: "low"},
{ID: " profile ", Provider: " promptkit ", Model: " model ", BackendID: " backend-a ", ReasoningEffort: " low "},
{ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "high"},
{ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "low"},
} {
recorder.Record(profile)
}
want := []artifacts.LLMProfileManifest{
{ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "high"},
{ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "low"},
{ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-b", ReasoningEffort: "low"},
}
if got := recorder.Manifests(); !reflect.DeepEqual(got, want) {
t.Fatalf("profile manifests = %#v, want %#v", got, want)
}
}
func TestPromptKitClientValidationFailureReturnsError(t *testing.T) {
@@ -532,6 +572,10 @@ func newTestPromptKitClientWithReasoning(t *testing.T, fake *fakePromptKitLLM, r
Assets: registry,
ReasoningEffort: reasoningEffort,
EngineOptions: []promptkit.Option{
promptkit.WithBackend(promptkit.Backend{
ID: "test-backend",
Endpoint: "http://127.0.0.1:1/v1",
}),
promptkit.WithProfiles(
promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "default-profile",
@@ -541,7 +585,7 @@ func newTestPromptKitClientWithReasoning(t *testing.T, fake *fakePromptKitLLM, r
}),
promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "explicit-profile",
Endpoint: "http://127.0.0.1:1/v1",
BackendID: "test-backend",
Model: "explicit-model",
ReasoningEffort: "profile-reasoning",
}),