|
|
|
|
@@ -6,8 +6,10 @@ import (
|
|
|
|
|
"fmt"
|
|
|
|
|
"reflect"
|
|
|
|
|
"strings"
|
|
|
|
|
"sync"
|
|
|
|
|
"testing"
|
|
|
|
|
|
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
|
|
|
@@ -28,7 +30,7 @@ func TestProductionSceneDescriptionWorkflow(t *testing.T) {
|
|
|
|
|
Output: pipeline.Binding("json"),
|
|
|
|
|
Artifacts: map[string]pipeline.ArtifactLaneProfile{
|
|
|
|
|
"scene-descriptions": {
|
|
|
|
|
Extract: pipeline.Binding(sceneextract.Key),
|
|
|
|
|
Extract: pipeline.ModuleBinding{Module: sceneextract.Key, LLMProfile: "scene-description-profile"},
|
|
|
|
|
Normalize: pipeline.Binding(scenenormalize.Key),
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
@@ -45,7 +47,8 @@ func TestProductionSceneDescriptionWorkflow(t *testing.T) {
|
|
|
|
|
t.Fatalf("resolved references = %#v / %#v, want no generated or required references", lane.ExtractReferences, lane.NormalizeReferences)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
prepared, err := pipeline.Prepare(effective.ResolvedPipeline, components.registries, pipeline.ModuleDependencies{LLM: sceneDescriptionLLM{}})
|
|
|
|
|
llmClient := &sceneDescriptionLLM{}
|
|
|
|
|
prepared, err := pipeline.Prepare(effective.ResolvedPipeline, components.registries, pipeline.ModuleDependencies{LLM: llmClient})
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatalf("Prepare() error = %v", err)
|
|
|
|
|
}
|
|
|
|
|
@@ -60,6 +63,14 @@ func TestProductionSceneDescriptionWorkflow(t *testing.T) {
|
|
|
|
|
if output.Manifest.ValidationStatus != "approved" || len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 {
|
|
|
|
|
t.Fatalf("run output = %#v, want one approved normalized artifact", output)
|
|
|
|
|
}
|
|
|
|
|
wantProfiles := []artifacts.LLMProfileManifest{{
|
|
|
|
|
ID: "scene-description-profile",
|
|
|
|
|
Provider: "promptkit",
|
|
|
|
|
Model: "deterministic",
|
|
|
|
|
}}
|
|
|
|
|
if !reflect.DeepEqual(output.Manifest.LLMProfiles, wantProfiles) {
|
|
|
|
|
t.Fatalf("manifest LLM profiles = %#v, want %#v", output.Manifest.LLMProfiles, wantProfiles)
|
|
|
|
|
}
|
|
|
|
|
normalizedOutput := output.NormalizeOutputs[0]
|
|
|
|
|
if normalizedOutput.NormalizerKey != scenenormalize.Key || normalizedOutput.Artifact.Kind != dnd.SceneDescriptionListKind || normalizedOutput.Artifact.Schema.ID != scenecodec.SchemaID || normalizedOutput.Artifact.Schema.Name != scenecodec.SchemaName || normalizedOutput.Artifact.Schema.Version != scenecodec.SchemaVersion {
|
|
|
|
|
t.Fatalf("normalized output = %#v, want registered durable scene-description schema", normalizedOutput)
|
|
|
|
|
@@ -85,9 +96,12 @@ func TestProductionSceneDescriptionWorkflow(t *testing.T) {
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type sceneDescriptionLLM struct{}
|
|
|
|
|
type sceneDescriptionLLM struct {
|
|
|
|
|
mu sync.Mutex
|
|
|
|
|
profile *artifacts.LLMProfileManifest
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (sceneDescriptionLLM) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
|
|
|
|
func (client *sceneDescriptionLLM) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
|
|
|
|
if err := ctx.Err(); err != nil {
|
|
|
|
|
return contracts.StructuredCompletionResponse{}, err
|
|
|
|
|
}
|
|
|
|
|
@@ -107,5 +121,27 @@ func (sceneDescriptionLLM) CompleteStructured(ctx context.Context, req contracts
|
|
|
|
|
if err := json.Unmarshal([]byte(content), out); err != nil {
|
|
|
|
|
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate structured response: %w", err)
|
|
|
|
|
}
|
|
|
|
|
return contracts.StructuredCompletionResponse{Content: []byte(content), Provider: "test", Model: "deterministic", ProfileID: req.ProfileID}, nil
|
|
|
|
|
profile := artifacts.LLMProfileManifest{
|
|
|
|
|
ID: req.ProfileID,
|
|
|
|
|
Provider: "promptkit",
|
|
|
|
|
Model: "deterministic",
|
|
|
|
|
}
|
|
|
|
|
client.mu.Lock()
|
|
|
|
|
client.profile = &profile
|
|
|
|
|
client.mu.Unlock()
|
|
|
|
|
return contracts.StructuredCompletionResponse{
|
|
|
|
|
Content: []byte(content),
|
|
|
|
|
Provider: profile.Provider,
|
|
|
|
|
Model: profile.Model,
|
|
|
|
|
ProfileID: profile.ID,
|
|
|
|
|
}, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (client *sceneDescriptionLLM) LLMProfileManifests() []artifacts.LLMProfileManifest {
|
|
|
|
|
client.mu.Lock()
|
|
|
|
|
defer client.mu.Unlock()
|
|
|
|
|
if client.profile == nil {
|
|
|
|
|
return nil
|
|
|
|
|
}
|
|
|
|
|
return []artifacts.LLMProfileManifest{*client.profile}
|
|
|
|
|
}
|
|
|
|
|
|