Align run provenance with PromptKit

This commit is contained in:
2026-07-28 16:47:07 +00:00
parent 7c569a3d8c
commit 4bca6d3103
4 changed files with 50 additions and 10 deletions

View File

@@ -3,8 +3,9 @@
## Status
In progress. The dependency, framework adapter, version 4 PromptKit
configuration migration, and provider-neutral module prompt-asset support are
implemented; remaining provenance alignment is still planned.
configuration migration, provider-neutral module prompt-asset support, and
provenance alignment are implemented; canonical documentation and final
repository verification are still planned.
## Objective

View File

@@ -6,8 +6,10 @@ import (
"fmt"
"reflect"
"strings"
"sync"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
@@ -28,7 +30,7 @@ func TestProductionSceneDescriptionWorkflow(t *testing.T) {
Output: pipeline.Binding("json"),
Artifacts: map[string]pipeline.ArtifactLaneProfile{
"scene-descriptions": {
Extract: pipeline.Binding(sceneextract.Key),
Extract: pipeline.ModuleBinding{Module: sceneextract.Key, LLMProfile: "scene-description-profile"},
Normalize: pipeline.Binding(scenenormalize.Key),
},
},
@@ -45,7 +47,8 @@ func TestProductionSceneDescriptionWorkflow(t *testing.T) {
t.Fatalf("resolved references = %#v / %#v, want no generated or required references", lane.ExtractReferences, lane.NormalizeReferences)
}
prepared, err := pipeline.Prepare(effective.ResolvedPipeline, components.registries, pipeline.ModuleDependencies{LLM: sceneDescriptionLLM{}})
llmClient := &sceneDescriptionLLM{}
prepared, err := pipeline.Prepare(effective.ResolvedPipeline, components.registries, pipeline.ModuleDependencies{LLM: llmClient})
if err != nil {
t.Fatalf("Prepare() error = %v", err)
}
@@ -60,6 +63,14 @@ func TestProductionSceneDescriptionWorkflow(t *testing.T) {
if output.Manifest.ValidationStatus != "approved" || len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 {
t.Fatalf("run output = %#v, want one approved normalized artifact", output)
}
wantProfiles := []artifacts.LLMProfileManifest{{
ID: "scene-description-profile",
Provider: "promptkit",
Model: "deterministic",
}}
if !reflect.DeepEqual(output.Manifest.LLMProfiles, wantProfiles) {
t.Fatalf("manifest LLM profiles = %#v, want %#v", output.Manifest.LLMProfiles, wantProfiles)
}
normalizedOutput := output.NormalizeOutputs[0]
if normalizedOutput.NormalizerKey != scenenormalize.Key || normalizedOutput.Artifact.Kind != dnd.SceneDescriptionListKind || normalizedOutput.Artifact.Schema.ID != scenecodec.SchemaID || normalizedOutput.Artifact.Schema.Name != scenecodec.SchemaName || normalizedOutput.Artifact.Schema.Version != scenecodec.SchemaVersion {
t.Fatalf("normalized output = %#v, want registered durable scene-description schema", normalizedOutput)
@@ -85,9 +96,12 @@ func TestProductionSceneDescriptionWorkflow(t *testing.T) {
}
}
type sceneDescriptionLLM struct{}
type sceneDescriptionLLM struct {
mu sync.Mutex
profile *artifacts.LLMProfileManifest
}
func (sceneDescriptionLLM) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
func (client *sceneDescriptionLLM) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
if err := ctx.Err(); err != nil {
return contracts.StructuredCompletionResponse{}, err
}
@@ -107,5 +121,27 @@ func (sceneDescriptionLLM) CompleteStructured(ctx context.Context, req contracts
if err := json.Unmarshal([]byte(content), out); err != nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate structured response: %w", err)
}
return contracts.StructuredCompletionResponse{Content: []byte(content), Provider: "test", Model: "deterministic", ProfileID: req.ProfileID}, nil
profile := artifacts.LLMProfileManifest{
ID: req.ProfileID,
Provider: "promptkit",
Model: "deterministic",
}
client.mu.Lock()
client.profile = &profile
client.mu.Unlock()
return contracts.StructuredCompletionResponse{
Content: []byte(content),
Provider: profile.Provider,
Model: profile.Model,
ProfileID: profile.ID,
}, nil
}
func (client *sceneDescriptionLLM) LLMProfileManifests() []artifacts.LLMProfileManifest {
client.mu.Lock()
defer client.mu.Unlock()
if client.profile == nil {
return nil
}
return []artifacts.LLMProfileManifest{*client.profile}
}

View File

@@ -53,7 +53,7 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
PipelineID: "pipeline-1",
PipelineDigest: "sha256:abc123",
LLMProfiles: []LLMProfileManifest{
{ID: "default", Provider: "scriptorium", Model: "model-a"},
{ID: "default", Provider: "promptkit", Model: "model-a"},
},
ArtifactLanes: []ArtifactLaneManifest{
{
@@ -102,6 +102,9 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
t.Fatalf("llm_profiles[0] = %#v, want object", profiles[0])
}
assertHasKeys(t, profile, "id", "provider", "model")
if profile["provider"] != "promptkit" {
t.Fatalf("llm_profiles[0].provider = %#v, want promptkit", profile["provider"])
}
lanes, ok := got["artifact_lanes"].([]any)
if !ok {

View File

@@ -41,7 +41,7 @@ func TestPromptKitClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
if !out.OK {
t.Fatalf("decoded output OK = false, want true")
}
if resp.Provider != promptKitProviderName || resp.Model != "explicit-model" || resp.ProfileID != "explicit-profile" {
if resp.Provider != "promptkit" || resp.Model != "explicit-model" || resp.ProfileID != "explicit-profile" {
t.Fatalf("response metadata = %#v", resp)
}
if resp.PromptTokens != 11 || resp.CompletionTokens != 7 || resp.TotalTokens != 18 {
@@ -87,7 +87,7 @@ func TestPromptKitClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
manifests := client.LLMProfileManifests()
if len(manifests) != 1 ||
manifests[0].ID != "explicit-profile" ||
manifests[0].Provider != promptKitProviderName ||
manifests[0].Provider != "promptkit" ||
manifests[0].Model != "explicit-model" {
t.Fatalf("profile manifests = %#v", manifests)
}