eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Publish effective LLM backend provenance

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-30 02:18:14 +00:00
parent f8333f2c15
commit 71a004bfc8
11 changed files with 194 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
internal/core/artifacts/artifacts.go
View File

@@ -26,9 +26,11 @@ type ValidatorManifest struct {
}
type LLMProfileManifest struct {
ID string `json:"id"`
Provider string `json:"provider,omitempty"`
Model string `json:"model,omitempty"`
ID string `json:"id"`
Provider string `json:"provider,omitempty"`
Model string `json:"model,omitempty"`
BackendID string `json:"backend_id,omitempty"`
ReasoningEffort string `json:"reasoning_effort,omitempty"`
}
type ReferenceProvenance struct {

13
internal/core/artifacts/artifacts_test.go
View File

@@ -53,7 +53,13 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
PipelineID: "pipeline-1",
PipelineDigest: "sha256:abc123",
LLMProfiles: []LLMProfileManifest{
{ID: "default", Provider: "promptkit", Model: "model-a"},
{
ID: "default",
Provider: "promptkit",
Model: "model-a",
BackendID: "openrouter",
ReasoningEffort: "high",
},
},
ArtifactLanes: []ArtifactLaneManifest{
{
@@ -101,10 +107,13 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
if !ok {
t.Fatalf("llm_profiles[0] = %#v, want object", profiles[0])
}
assertHasKeys(t, profile, "id", "provider", "model")
assertHasKeys(t, profile, "id", "provider", "model", "backend_id", "reasoning_effort")
if profile["provider"] != "promptkit" {
t.Fatalf("llm_profiles[0].provider = %#v, want promptkit", profile["provider"])
}
if profile["backend_id"] != "openrouter" || profile["reasoning_effort"] != "high" {
t.Fatalf("llm_profiles[0] = %#v, want backend and reasoning provenance", profile)
}
lanes, ok := got["artifact_lanes"].([]any)
if !ok {

1
internal/framework/contracts/contracts.go
View File

@@ -43,6 +43,7 @@ type LLMDebugPrompt struct {
PromptVersion string `json:"prompt_version,omitempty"`
PromptHash string `json:"prompt_hash,omitempty"`
SelectedProfileID string `json:"selected_profile_id,omitempty"`
SelectedBackendID string `json:"selected_backend_id,omitempty"`
SessionID string `json:"session_id,omitempty"`
RenderedPromptHash string `json:"rendered_prompt_hash,omitempty"`
Messages []LLMDebugMessage `json:"messages,omitempty"`

13
internal/framework/llm/promptkit_client.go
View File

@@ -155,9 +155,11 @@ func (c *PromptKitClient) responseFromResult(result *promptkit.RunResult, prepar
content = []byte(result.RawOutput)
}
profile := artifacts.LLMProfileManifest{
ID: strings.TrimSpace(result.SelectedProfileID),
Provider: promptKitProviderName,
Model: firstNonEmpty(result.ModelName, result.EffectiveModelParams.Model),
ID: strings.TrimSpace(result.SelectedProfileID),
Provider: promptKitProviderName,
Model: firstNonEmpty(result.ModelName, result.EffectiveModelParams.Model),
BackendID: strings.TrimSpace(result.SelectedBackendID),
ReasoningEffort: strings.TrimSpace(result.EffectiveModelParams.ReasoningEffort),
}
if c.recorder != nil {
c.recorder.Record(profile)
@@ -205,6 +207,7 @@ func promptKitDebugPrompt(prepared *promptkit.PreparedRun) *contracts.LLMDebugPr
PromptVersion: prepared.PromptVersion,
PromptHash: prepared.PromptHash,
SelectedProfileID: prepared.SelectedProfileID,
SelectedBackendID: prepared.SelectedBackendID,
SessionID: prepared.SessionID,
RenderedPromptHash: prepared.RenderedPromptHash,
Messages: messages,
@@ -304,7 +307,9 @@ func (r *LLMProfileRecorder) Record(profile artifacts.LLMProfileManifest) {
profile.ID = strings.TrimSpace(profile.ID)
profile.Provider = strings.TrimSpace(profile.Provider)
profile.Model = strings.TrimSpace(profile.Model)
key := profile.ID + "\x00" + profile.Provider + "\x00" + profile.Model
profile.BackendID = strings.TrimSpace(profile.BackendID)
profile.ReasoningEffort = strings.TrimSpace(profile.ReasoningEffort)
key := profile.ID + "\x00" + profile.Provider + "\x00" + profile.Model + "\x00" + profile.BackendID + "\x00" + profile.ReasoningEffort
r.mu.Lock()
defer r.mu.Unlock()
if r.profiles == nil {

50
internal/framework/llm/promptkit_client_test.go
View File

@@ -8,6 +8,7 @@ import (
"net/http"
"os"
"path/filepath"
"reflect"
"strings"
"sync"
"sync/atomic"
@@ -15,6 +16,7 @@ import (
"testing/fstest"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/promptkit"
)
@@ -54,8 +56,13 @@ func TestPromptKitClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
}
if resp.Debug.Prompt.PromptID != "adapter.direct-session" ||
resp.Debug.Prompt.SelectedProfileID != "explicit-profile" ||
resp.Debug.Prompt.SelectedBackendID != "test-backend" ||
resp.Debug.Prompt.SessionID != "session-123" {
t.Fatalf("debug prompt metadata = %#v, want prompt/profile/session", resp.Debug.Prompt)
t.Fatalf("debug prompt metadata = %#v, want prompt/profile/backend/session", resp.Debug.Prompt)
}
if resp.Debug.Prompt.EffectiveModelParams["backend_id"] != "test-backend" ||
resp.Debug.Prompt.EffectiveModelParams["reasoning_effort"] != "profile-reasoning" {
t.Fatalf("debug effective model params = %#v, want backend and reasoning", resp.Debug.Prompt.EffectiveModelParams)
}
if len(resp.Debug.Prompt.Messages) != 1 || !strings.Contains(resp.Debug.Prompt.Messages[0].Content, `{"source":true}`) {
t.Fatalf("debug prompt messages = %#v, want rendered input content", resp.Debug.Prompt.Messages)
@@ -66,6 +73,10 @@ func TestPromptKitClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
if resp.Debug.Response.Usage.CachedTokens != 5 || resp.Debug.Response.Usage.CacheWriteTokens != 3 {
t.Fatalf("debug usage = %#v, want cached token counts", resp.Debug.Response.Usage)
}
if resp.Debug.Response.EffectiveModelParams["backend_id"] != "test-backend" ||
resp.Debug.Response.EffectiveModelParams["reasoning_effort"] != "profile-reasoning" {
t.Fatalf("debug response effective model params = %#v, want backend and reasoning", resp.Debug.Response.EffectiveModelParams)
}
debugJSON, err := json.Marshal(resp.Debug)
if err != nil {
t.Fatalf("marshal debug material: %v", err)
@@ -80,6 +91,9 @@ func TestPromptKitClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
if gotReq.Target.Model != "explicit-model" {
t.Fatalf("model = %q, want explicit-model", gotReq.Target.Model)
}
if gotReq.Target.BackendID != "test-backend" {
t.Fatalf("backend id = %q, want test-backend", gotReq.Target.BackendID)
}
if len(gotReq.Prompt.Messages) != 1 ||
!strings.Contains(gotReq.Prompt.Messages[0].Content, `{"source":true}`) ||
!strings.Contains(gotReq.Prompt.Messages[0].Content, "value") {
@@ -92,7 +106,9 @@ func TestPromptKitClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
if len(manifests) != 1 ||
manifests[0].ID != "explicit-profile" ||
manifests[0].Provider != "promptkit" ||
manifests[0].Model != "explicit-model" {
manifests[0].Model != "explicit-model" ||
manifests[0].BackendID != "test-backend" ||
manifests[0].ReasoningEffort != "profile-reasoning" {
t.Fatalf("profile manifests = %#v", manifests)
}
}
@@ -312,6 +328,30 @@ func TestPromptKitClientUsesPromptDefaultProfileWhenRequestProfileEmpty(t *testi
if got := fake.lastRequest().Target.Model; got != "default-model" {
t.Fatalf("model = %q, want prompt default profile model", got)
}
manifests := client.LLMProfileManifests()
if len(manifests) != 1 || manifests[0].BackendID != "" || manifests[0].ReasoningEffort != "profile-reasoning" {
t.Fatalf("endpoint-only profile manifests = %#v, want omitted backend and effective reasoning", manifests)
}
}
func TestLLMProfileRecorderDistinguishesEffectiveTargets(t *testing.T) {
recorder := NewLLMProfileRecorder()
for _, profile := range []artifacts.LLMProfileManifest{
{ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-b", ReasoningEffort: "low"},
{ID: " profile ", Provider: " promptkit ", Model: " model ", BackendID: " backend-a ", ReasoningEffort: " low "},
{ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "high"},
{ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "low"},
} {
recorder.Record(profile)
}
want := []artifacts.LLMProfileManifest{
{ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "high"},
{ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "low"},
{ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-b", ReasoningEffort: "low"},
}
if got := recorder.Manifests(); !reflect.DeepEqual(got, want) {
t.Fatalf("profile manifests = %#v, want %#v", got, want)
}
}
func TestPromptKitClientValidationFailureReturnsError(t *testing.T) {
@@ -532,6 +572,10 @@ func newTestPromptKitClientWithReasoning(t *testing.T, fake *fakePromptKitLLM, r
Assets: registry,
ReasoningEffort: reasoningEffort,
EngineOptions: []promptkit.Option{
promptkit.WithBackend(promptkit.Backend{
ID: "test-backend",
Endpoint: "http://127.0.0.1:1/v1",
}),
promptkit.WithProfiles(
promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "default-profile",
@@ -541,7 +585,7 @@ func newTestPromptKitClientWithReasoning(t *testing.T, fake *fakePromptKitLLM, r
}),
promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "explicit-profile",
Endpoint: "http://127.0.0.1:1/v1",
BackendID: "test-backend",
Model: "explicit-model",
ReasoningEffort: "profile-reasoning",
}),

12
internal/framework/pipeline/runner.go
View File

@@ -832,14 +832,18 @@ func mergeLLMProfileManifests(sources ...[]artifacts.LLMProfileManifest) []artif
id := strings.TrimSpace(profile.ID)
provider := strings.TrimSpace(profile.Provider)
model := strings.TrimSpace(profile.Model)
key := id + "\x00" + provider + "\x00" + model
backendID := strings.TrimSpace(profile.BackendID)
reasoningEffort := strings.TrimSpace(profile.ReasoningEffort)
key := id + "\x00" + provider + "\x00" + model + "\x00" + backendID + "\x00" + reasoningEffort
if _, exists := merged[key]; exists {
continue
}
merged[key] = artifacts.LLMProfileManifest{
ID: id,
Provider: provider,
Model: model,
ID: id,
Provider: provider,
Model: model,
BackendID: backendID,
ReasoningEffort: reasoningEffort,
}
}
}

29
internal/framework/pipeline/runner_manifest_test.go Normal file
View File

@@ -0,0 +1,29 @@
package pipeline
import (
"reflect"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
)
func TestMergeLLMProfileManifestsDistinguishesEffectiveTargets(t *testing.T) {
got := mergeLLMProfileManifests(
[]artifacts.LLMProfileManifest{
{ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-b", ReasoningEffort: "low"},
{ID: " profile ", Provider: " promptkit ", Model: " model ", BackendID: " backend-a ", ReasoningEffort: " low "},
},
[]artifacts.LLMProfileManifest{
{ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "high"},
{ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "low"},
},
)
want := []artifacts.LLMProfileManifest{
{ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "high"},
{ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "low"},
{ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-b", ReasoningEffort: "low"},
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("merged profiles = %#v, want %#v", got, want)
}
}

40
internal/modules/generic/output/json/encoder_test.go
View File

@@ -449,6 +449,46 @@ func TestEncodePrettyPrintsJSON(t *testing.T) {
}
}
func TestEncodePublishesLLMProfileProvenance(t *testing.T) {
result, err := New().Encode(context.Background(), contracts.OutputRequest{
Manifest: artifacts.RunManifest{
RunID: "run-1",
LLMProfiles: []artifacts.LLMProfileManifest{
{
ID: "endpoint-profile",
Provider: "promptkit",
Model: "endpoint-model",
ReasoningEffort: "low",
},
{
ID: "profile",
Provider: "promptkit",
Model: "model",
BackendID: "openrouter",
ReasoningEffort: "high",
},
},
},
})
if err != nil {
t.Fatalf("Encode() error = %v, want nil", err)
}
manifest := decodeObject(t, fileBytes(t, result.Files, "manifest.json"))
profiles := manifest["llm_profiles"].([]any)
if len(profiles) != 2 {
t.Fatalf("llm_profiles = %#v, want two entries", profiles)
}
endpointProfile := profiles[0].(map[string]any)
if _, exists := endpointProfile["backend_id"]; exists || endpointProfile["reasoning_effort"] != "low" {
t.Fatalf("endpoint-only LLM profile = %#v, want omitted backend and published reasoning", endpointProfile)
}
backendProfile := profiles[1].(map[string]any)
if backendProfile["backend_id"] != "openrouter" || backendProfile["reasoning_effort"] != "high" {
t.Fatalf("backend LLM profile = %#v, want published backend and reasoning fields", backendProfile)
}
}
func TestEncodeIncludesManifestReferences(t *testing.T) {
result, err := New().Encode(context.Background(), contracts.OutputRequest{
Manifest: artifacts.RunManifest{