From 71a004bfc83f7a4aace602c77afe45a8bc6c1f23 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Thu, 30 Jul 2026 02:18:14 +0000 Subject: [PATCH] Publish effective LLM backend provenance --- docs/integrations/json-output.md | 15 ++++++ docs/integrations/pkg-promptkit.md | 9 +++- docs/internal/llm.md | 36 +++++++------ internal/core/artifacts/artifacts.go | 8 +-- internal/core/artifacts/artifacts_test.go | 13 ++++- internal/framework/contracts/contracts.go | 1 + internal/framework/llm/promptkit_client.go | 13 +++-- .../framework/llm/promptkit_client_test.go | 50 +++++++++++++++++-- internal/framework/pipeline/runner.go | 12 +++-- .../pipeline/runner_manifest_test.go | 29 +++++++++++ .../generic/output/json/encoder_test.go | 40 +++++++++++++++ 11 files changed, 194 insertions(+), 32 deletions(-) create mode 100644 internal/framework/pipeline/runner_manifest_test.go diff --git a/docs/integrations/json-output.md b/docs/integrations/json-output.md index b5eea11..1885f34 100644 --- a/docs/integrations/json-output.md +++ b/docs/integrations/json-output.md @@ -98,6 +98,21 @@ summarize results without embedding lane payload bytes. A chunk-plan summary is provenance for the plan used by this run; cache records, debug artifacts, and other operational state are not published as bundle files. +Each `llm_profiles` entry identifies effective, non-secret LLM execution +provenance: + +| Field | Required | Meaning | +| --- | --- | --- | +| `id` | Yes | Selected PromptKit profile identifier. | +| `provider` | No | Notarius adapter provider identifier. | +| `model` | No | Effective provider model identifier. | +| `backend_id` | No | Effective PromptKit backend registration identifier. Endpoint-only profiles omit it. | +| `reasoning_effort` | No | Effective opaque provider reasoning setting. An empty or explicitly cleared setting is omitted. | + +These values describe observed execution; they are not a backend-registration +interface. Entries that differ by backend or effective reasoning remain +distinct even when their profile, provider, and model are otherwise equal. + ## Rejections And Warnings `rejected.json` is always an object with a `rejected` array. Each entry has diff --git a/docs/integrations/pkg-promptkit.md b/docs/integrations/pkg-promptkit.md index a025d7b..ff34e8a 100644 --- a/docs/integrations/pkg-promptkit.md +++ b/docs/integrations/pkg-promptkit.md @@ -17,7 +17,7 @@ Notarius relies on the root `promptkit` package to: - prepare and run a `RunRequest` with named inline artifacts, variables, a direct session ID, prompt identity, and profile selection; - return rendered debug material, validated structured output, selected - profile and model metadata, and token usage; + profile, backend, effective model metadata, and token usage; - distinguish structured-output validation failure from execution failure; and - identify a missing explicit profile through `ErrProfileNotFound`. @@ -33,6 +33,13 @@ the same value as the `session_id` prompt variable for maintained prompt compatibility. Session IDs are stable, non-secret correlation identifiers and may be exposed to providers and provider observability. +Notarius records PromptKit's selected backend ID and effective reasoning +setting as optional run-manifest provenance. Endpoint-only profiles have no +backend ID. Debug prompt material also retains the selected backend ID and +PromptKit's stable lower-case `effective_model_params` JSON, which may include +`backend_id`. Notarius production configuration does not expose user-defined +PromptKit backend registration. + ## Notarius Ownership [LLM Runtime Internals](../internal/llm.md) describes how Notarius mounts diff --git a/docs/internal/llm.md b/docs/internal/llm.md index 287337a..d181c14 100644 --- a/docs/internal/llm.md +++ b/docs/internal/llm.md @@ -49,10 +49,12 @@ to module calls, retries, and LLM-backed validators for the whole run. An empty request profile lets the prompt select its configured default. The CLI prepares every explicitly selected binding profile before a run begins, so a missing explicit profile fails before stage execution. Calls record the profile -actually selected by PromptKit; the recorder deduplicates non-secret profile -identity, provider, and model values for manifest use. Successful completion -responses and recorded profile manifests identify the adapter provider as -`promptkit`. +actually selected by PromptKit. The recorder trims and deduplicates non-secret +profile identity, provider, model, selected backend ID, and effective reasoning +values for manifest use. Entries that differ in backend or reasoning remain +distinct and deterministically ordered. Endpoint-only profiles retain an empty +backend ID, which the published JSON omits. Successful completion responses and +recorded profile manifests identify the adapter provider as `promptkit`. Before execution, the adapter also contributes a non-secret checkpoint fingerprint for the effective PromptKit profile source. It combines the @@ -159,18 +161,22 @@ contract is identified in When debug recording is enabled, the pipeline decorates the shared client. The wrapper records prepared prompt and response material, timing, selected profile -and model, and call identifiers in the run’s debug bundle, including material -available from a failed structured completion. For a successful completion, a -debug-write failure is surfaced; when the completion already failed, its call -error remains the result. Debug-bundle location, retention, and handling are -operational concerns documented in [Operations](../operations.md#debug-bundles). +and backend, effective model parameters, and call identifiers in the run’s +debug bundle, including material available from a failed structured completion. +Effective parameters use PromptKit's stable lower-case JSON field names and may +include `backend_id`. For a successful completion, a debug-write failure is +surfaced; when the completion already failed, its call error remains the +result. Debug-bundle location, retention, and handling are operational concerns +documented in [Operations](../operations.md#debug-bundles). -Run manifests receive selected profile summaries and component identities, not -prompt, schema, source, reference, or response content. Provider error text is -wrapped with prompt context and bearer credentials are redacted before it -crosses the runtime boundary. Known-secret redaction is available to other -runtime collaborators; it does not make prompt or response contents safe for -general logging. +Run manifests receive selected profile summaries, including optional effective +backend and reasoning provenance, and component identities—not prompt, schema, +source, reference, or response content. The published field semantics belong +to the [JSON output contract](../integrations/json-output.md#manifestjson). +Provider error text is wrapped with prompt context and bearer credentials are +redacted before it crosses the runtime boundary. Known-secret redaction is +available to other runtime collaborators; it does not make prompt or response +contents safe for general logging. ## Failure Boundaries diff --git a/internal/core/artifacts/artifacts.go b/internal/core/artifacts/artifacts.go index afffd93..8abe914 100644 --- a/internal/core/artifacts/artifacts.go +++ b/internal/core/artifacts/artifacts.go @@ -26,9 +26,11 @@ type ValidatorManifest struct { } type LLMProfileManifest struct { - ID string `json:"id"` - Provider string `json:"provider,omitempty"` - Model string `json:"model,omitempty"` + ID string `json:"id"` + Provider string `json:"provider,omitempty"` + Model string `json:"model,omitempty"` + BackendID string `json:"backend_id,omitempty"` + ReasoningEffort string `json:"reasoning_effort,omitempty"` } type ReferenceProvenance struct { diff --git a/internal/core/artifacts/artifacts_test.go b/internal/core/artifacts/artifacts_test.go index b743786..c69f79d 100644 --- a/internal/core/artifacts/artifacts_test.go +++ b/internal/core/artifacts/artifacts_test.go @@ -53,7 +53,13 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) { PipelineID: "pipeline-1", PipelineDigest: "sha256:abc123", LLMProfiles: []LLMProfileManifest{ - {ID: "default", Provider: "promptkit", Model: "model-a"}, + { + ID: "default", + Provider: "promptkit", + Model: "model-a", + BackendID: "openrouter", + ReasoningEffort: "high", + }, }, ArtifactLanes: []ArtifactLaneManifest{ { @@ -101,10 +107,13 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) { if !ok { t.Fatalf("llm_profiles[0] = %#v, want object", profiles[0]) } - assertHasKeys(t, profile, "id", "provider", "model") + assertHasKeys(t, profile, "id", "provider", "model", "backend_id", "reasoning_effort") if profile["provider"] != "promptkit" { t.Fatalf("llm_profiles[0].provider = %#v, want promptkit", profile["provider"]) } + if profile["backend_id"] != "openrouter" || profile["reasoning_effort"] != "high" { + t.Fatalf("llm_profiles[0] = %#v, want backend and reasoning provenance", profile) + } lanes, ok := got["artifact_lanes"].([]any) if !ok { diff --git a/internal/framework/contracts/contracts.go b/internal/framework/contracts/contracts.go index 4cd3688..98682c3 100644 --- a/internal/framework/contracts/contracts.go +++ b/internal/framework/contracts/contracts.go @@ -43,6 +43,7 @@ type LLMDebugPrompt struct { PromptVersion string `json:"prompt_version,omitempty"` PromptHash string `json:"prompt_hash,omitempty"` SelectedProfileID string `json:"selected_profile_id,omitempty"` + SelectedBackendID string `json:"selected_backend_id,omitempty"` SessionID string `json:"session_id,omitempty"` RenderedPromptHash string `json:"rendered_prompt_hash,omitempty"` Messages []LLMDebugMessage `json:"messages,omitempty"` diff --git a/internal/framework/llm/promptkit_client.go b/internal/framework/llm/promptkit_client.go index d036080..c712ee9 100644 --- a/internal/framework/llm/promptkit_client.go +++ b/internal/framework/llm/promptkit_client.go @@ -155,9 +155,11 @@ func (c *PromptKitClient) responseFromResult(result *promptkit.RunResult, prepar content = []byte(result.RawOutput) } profile := artifacts.LLMProfileManifest{ - ID: strings.TrimSpace(result.SelectedProfileID), - Provider: promptKitProviderName, - Model: firstNonEmpty(result.ModelName, result.EffectiveModelParams.Model), + ID: strings.TrimSpace(result.SelectedProfileID), + Provider: promptKitProviderName, + Model: firstNonEmpty(result.ModelName, result.EffectiveModelParams.Model), + BackendID: strings.TrimSpace(result.SelectedBackendID), + ReasoningEffort: strings.TrimSpace(result.EffectiveModelParams.ReasoningEffort), } if c.recorder != nil { c.recorder.Record(profile) @@ -205,6 +207,7 @@ func promptKitDebugPrompt(prepared *promptkit.PreparedRun) *contracts.LLMDebugPr PromptVersion: prepared.PromptVersion, PromptHash: prepared.PromptHash, SelectedProfileID: prepared.SelectedProfileID, + SelectedBackendID: prepared.SelectedBackendID, SessionID: prepared.SessionID, RenderedPromptHash: prepared.RenderedPromptHash, Messages: messages, @@ -304,7 +307,9 @@ func (r *LLMProfileRecorder) Record(profile artifacts.LLMProfileManifest) { profile.ID = strings.TrimSpace(profile.ID) profile.Provider = strings.TrimSpace(profile.Provider) profile.Model = strings.TrimSpace(profile.Model) - key := profile.ID + "\x00" + profile.Provider + "\x00" + profile.Model + profile.BackendID = strings.TrimSpace(profile.BackendID) + profile.ReasoningEffort = strings.TrimSpace(profile.ReasoningEffort) + key := profile.ID + "\x00" + profile.Provider + "\x00" + profile.Model + "\x00" + profile.BackendID + "\x00" + profile.ReasoningEffort r.mu.Lock() defer r.mu.Unlock() if r.profiles == nil { diff --git a/internal/framework/llm/promptkit_client_test.go b/internal/framework/llm/promptkit_client_test.go index c1ac236..f2dc017 100644 --- a/internal/framework/llm/promptkit_client_test.go +++ b/internal/framework/llm/promptkit_client_test.go @@ -8,6 +8,7 @@ import ( "net/http" "os" "path/filepath" + "reflect" "strings" "sync" "sync/atomic" @@ -15,6 +16,7 @@ import ( "testing/fstest" "time" + "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/promptkit" ) @@ -54,8 +56,13 @@ func TestPromptKitClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) { } if resp.Debug.Prompt.PromptID != "adapter.direct-session" || resp.Debug.Prompt.SelectedProfileID != "explicit-profile" || + resp.Debug.Prompt.SelectedBackendID != "test-backend" || resp.Debug.Prompt.SessionID != "session-123" { - t.Fatalf("debug prompt metadata = %#v, want prompt/profile/session", resp.Debug.Prompt) + t.Fatalf("debug prompt metadata = %#v, want prompt/profile/backend/session", resp.Debug.Prompt) + } + if resp.Debug.Prompt.EffectiveModelParams["backend_id"] != "test-backend" || + resp.Debug.Prompt.EffectiveModelParams["reasoning_effort"] != "profile-reasoning" { + t.Fatalf("debug effective model params = %#v, want backend and reasoning", resp.Debug.Prompt.EffectiveModelParams) } if len(resp.Debug.Prompt.Messages) != 1 || !strings.Contains(resp.Debug.Prompt.Messages[0].Content, `{"source":true}`) { t.Fatalf("debug prompt messages = %#v, want rendered input content", resp.Debug.Prompt.Messages) @@ -66,6 +73,10 @@ func TestPromptKitClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) { if resp.Debug.Response.Usage.CachedTokens != 5 || resp.Debug.Response.Usage.CacheWriteTokens != 3 { t.Fatalf("debug usage = %#v, want cached token counts", resp.Debug.Response.Usage) } + if resp.Debug.Response.EffectiveModelParams["backend_id"] != "test-backend" || + resp.Debug.Response.EffectiveModelParams["reasoning_effort"] != "profile-reasoning" { + t.Fatalf("debug response effective model params = %#v, want backend and reasoning", resp.Debug.Response.EffectiveModelParams) + } debugJSON, err := json.Marshal(resp.Debug) if err != nil { t.Fatalf("marshal debug material: %v", err) @@ -80,6 +91,9 @@ func TestPromptKitClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) { if gotReq.Target.Model != "explicit-model" { t.Fatalf("model = %q, want explicit-model", gotReq.Target.Model) } + if gotReq.Target.BackendID != "test-backend" { + t.Fatalf("backend id = %q, want test-backend", gotReq.Target.BackendID) + } if len(gotReq.Prompt.Messages) != 1 || !strings.Contains(gotReq.Prompt.Messages[0].Content, `{"source":true}`) || !strings.Contains(gotReq.Prompt.Messages[0].Content, "value") { @@ -92,7 +106,9 @@ func TestPromptKitClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) { if len(manifests) != 1 || manifests[0].ID != "explicit-profile" || manifests[0].Provider != "promptkit" || - manifests[0].Model != "explicit-model" { + manifests[0].Model != "explicit-model" || + manifests[0].BackendID != "test-backend" || + manifests[0].ReasoningEffort != "profile-reasoning" { t.Fatalf("profile manifests = %#v", manifests) } } @@ -312,6 +328,30 @@ func TestPromptKitClientUsesPromptDefaultProfileWhenRequestProfileEmpty(t *testi if got := fake.lastRequest().Target.Model; got != "default-model" { t.Fatalf("model = %q, want prompt default profile model", got) } + manifests := client.LLMProfileManifests() + if len(manifests) != 1 || manifests[0].BackendID != "" || manifests[0].ReasoningEffort != "profile-reasoning" { + t.Fatalf("endpoint-only profile manifests = %#v, want omitted backend and effective reasoning", manifests) + } +} + +func TestLLMProfileRecorderDistinguishesEffectiveTargets(t *testing.T) { + recorder := NewLLMProfileRecorder() + for _, profile := range []artifacts.LLMProfileManifest{ + {ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-b", ReasoningEffort: "low"}, + {ID: " profile ", Provider: " promptkit ", Model: " model ", BackendID: " backend-a ", ReasoningEffort: " low "}, + {ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "high"}, + {ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "low"}, + } { + recorder.Record(profile) + } + want := []artifacts.LLMProfileManifest{ + {ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "high"}, + {ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "low"}, + {ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-b", ReasoningEffort: "low"}, + } + if got := recorder.Manifests(); !reflect.DeepEqual(got, want) { + t.Fatalf("profile manifests = %#v, want %#v", got, want) + } } func TestPromptKitClientValidationFailureReturnsError(t *testing.T) { @@ -532,6 +572,10 @@ func newTestPromptKitClientWithReasoning(t *testing.T, fake *fakePromptKitLLM, r Assets: registry, ReasoningEffort: reasoningEffort, EngineOptions: []promptkit.Option{ + promptkit.WithBackend(promptkit.Backend{ + ID: "test-backend", + Endpoint: "http://127.0.0.1:1/v1", + }), promptkit.WithProfiles( promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ ID: "default-profile", @@ -541,7 +585,7 @@ func newTestPromptKitClientWithReasoning(t *testing.T, fake *fakePromptKitLLM, r }), promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ ID: "explicit-profile", - Endpoint: "http://127.0.0.1:1/v1", + BackendID: "test-backend", Model: "explicit-model", ReasoningEffort: "profile-reasoning", }), diff --git a/internal/framework/pipeline/runner.go b/internal/framework/pipeline/runner.go index bc3f8ab..3e14be2 100644 --- a/internal/framework/pipeline/runner.go +++ b/internal/framework/pipeline/runner.go @@ -832,14 +832,18 @@ func mergeLLMProfileManifests(sources ...[]artifacts.LLMProfileManifest) []artif id := strings.TrimSpace(profile.ID) provider := strings.TrimSpace(profile.Provider) model := strings.TrimSpace(profile.Model) - key := id + "\x00" + provider + "\x00" + model + backendID := strings.TrimSpace(profile.BackendID) + reasoningEffort := strings.TrimSpace(profile.ReasoningEffort) + key := id + "\x00" + provider + "\x00" + model + "\x00" + backendID + "\x00" + reasoningEffort if _, exists := merged[key]; exists { continue } merged[key] = artifacts.LLMProfileManifest{ - ID: id, - Provider: provider, - Model: model, + ID: id, + Provider: provider, + Model: model, + BackendID: backendID, + ReasoningEffort: reasoningEffort, } } } diff --git a/internal/framework/pipeline/runner_manifest_test.go b/internal/framework/pipeline/runner_manifest_test.go new file mode 100644 index 0000000..7c8e95c --- /dev/null +++ b/internal/framework/pipeline/runner_manifest_test.go @@ -0,0 +1,29 @@ +package pipeline + +import ( + "reflect" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" +) + +func TestMergeLLMProfileManifestsDistinguishesEffectiveTargets(t *testing.T) { + got := mergeLLMProfileManifests( + []artifacts.LLMProfileManifest{ + {ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-b", ReasoningEffort: "low"}, + {ID: " profile ", Provider: " promptkit ", Model: " model ", BackendID: " backend-a ", ReasoningEffort: " low "}, + }, + []artifacts.LLMProfileManifest{ + {ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "high"}, + {ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "low"}, + }, + ) + want := []artifacts.LLMProfileManifest{ + {ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "high"}, + {ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-a", ReasoningEffort: "low"}, + {ID: "profile", Provider: "promptkit", Model: "model", BackendID: "backend-b", ReasoningEffort: "low"}, + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("merged profiles = %#v, want %#v", got, want) + } +} diff --git a/internal/modules/generic/output/json/encoder_test.go b/internal/modules/generic/output/json/encoder_test.go index 2bbc4f2..ad8ee55 100644 --- a/internal/modules/generic/output/json/encoder_test.go +++ b/internal/modules/generic/output/json/encoder_test.go @@ -449,6 +449,46 @@ func TestEncodePrettyPrintsJSON(t *testing.T) { } } +func TestEncodePublishesLLMProfileProvenance(t *testing.T) { + result, err := New().Encode(context.Background(), contracts.OutputRequest{ + Manifest: artifacts.RunManifest{ + RunID: "run-1", + LLMProfiles: []artifacts.LLMProfileManifest{ + { + ID: "endpoint-profile", + Provider: "promptkit", + Model: "endpoint-model", + ReasoningEffort: "low", + }, + { + ID: "profile", + Provider: "promptkit", + Model: "model", + BackendID: "openrouter", + ReasoningEffort: "high", + }, + }, + }, + }) + if err != nil { + t.Fatalf("Encode() error = %v, want nil", err) + } + + manifest := decodeObject(t, fileBytes(t, result.Files, "manifest.json")) + profiles := manifest["llm_profiles"].([]any) + if len(profiles) != 2 { + t.Fatalf("llm_profiles = %#v, want two entries", profiles) + } + endpointProfile := profiles[0].(map[string]any) + if _, exists := endpointProfile["backend_id"]; exists || endpointProfile["reasoning_effort"] != "low" { + t.Fatalf("endpoint-only LLM profile = %#v, want omitted backend and published reasoning", endpointProfile) + } + backendProfile := profiles[1].(map[string]any) + if backendProfile["backend_id"] != "openrouter" || backendProfile["reasoning_effort"] != "high" { + t.Fatalf("backend LLM profile = %#v, want published backend and reasoning fields", backendProfile) + } +} + func TestEncodeIncludesManifestReferences(t *testing.T) { result, err := New().Encode(context.Background(), contracts.OutputRequest{ Manifest: artifacts.RunManifest{