79 lines
2.7 KiB
Go
79 lines
2.7 KiB
Go
package itemoccurrences
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
|
"gitea.maximumdirect.net/eric/promptkit"
|
|
)
|
|
|
|
func TestPromptAssetsPrepareItemOccurrencePrompt(t *testing.T) {
|
|
registry := llm.NewAssetRegistry()
|
|
if err := RegisterPromptAssets(registry); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
options, err := registry.PromptKitOptions()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
|
ID: "item-occurrences-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "item-occurrences-test-model",
|
|
})))
|
|
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
|
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "item-occurrences-test-profile",
|
|
Inputs: map[string]promptkit.ArtifactRef{
|
|
"transcript": promptkit.InlineWithURI("file:///session.json", `{"units":[{"sentinel":"item-occurrence-transcript"}]}`),
|
|
"players": promptkit.Inline("item-occurrence-player"),
|
|
"party": promptkit.Inline(" "),
|
|
"glossary": promptkit.Inline(" "),
|
|
"item_registry": promptkit.Inline(`{"items":[{"name":"Torch"}]}`),
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_item_occurrences_llm.v1.json" {
|
|
t.Fatalf("prepared prompt = %#v", prepared)
|
|
}
|
|
rendered := make([]string, len(prepared.Messages))
|
|
for index, message := range prepared.Messages {
|
|
rendered[index] = message.Content
|
|
}
|
|
content := strings.Join(rendered, "\n")
|
|
for _, field := range []string{"start_unit_id", "end_unit_id"} {
|
|
if !strings.Contains(content, field) {
|
|
t.Fatalf("prepared prompt does not include shared evidence field %q", field)
|
|
}
|
|
}
|
|
for _, obsolete := range []string{"source_id", "start_segment", "end_segment"} {
|
|
if strings.Contains(content, obsolete) {
|
|
t.Fatalf("prepared prompt contains obsolete evidence field %q: %s", obsolete, content)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestPromptAssetsDoNotLeakIntoMetadata(t *testing.T) {
|
|
hash, err := promptAssetMetadata()
|
|
if err != nil || !strings.HasPrefix(hash, "sha256:") {
|
|
t.Fatalf("promptAssetMetadata() = %q, %v", hash, err)
|
|
}
|
|
metadata := newExtractor(t, &fakeItemOccurrencesLLMClient{}).ManifestMetadata()
|
|
payload, err := json.Marshal(metadata)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, forbidden := range []string{"common-dnd-system", "dnd_item_occurrences_llm.v1.json"} {
|
|
if strings.Contains(string(payload), forbidden) {
|
|
t.Fatalf("metadata leaked raw asset content %q: %s", forbidden, payload)
|
|
}
|
|
}
|
|
}
|