116 lines
3.7 KiB
Go
116 lines
3.7 KiB
Go
package usecase
|
|
|
|
import (
|
|
"context"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/scriptorium/internal/artifact"
|
|
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
|
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
|
|
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
|
"gitea.maximumdirect.net/eric/scriptorium/internal/validate"
|
|
)
|
|
|
|
type integrationLLM struct{}
|
|
|
|
func (f *integrationLLM) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
|
return &domain.GenerateResponse{
|
|
Content: `{"summary":"Party discovered a captive scout beneath the tower.","events":[{"title":"Scout found in cellar","type":"discovery","notes":"Scout requested rescue from goblin raiders."}]}`,
|
|
Usage: domain.TokenUsage{
|
|
PromptTokens: 42,
|
|
CompletionTokens: 36,
|
|
TotalTokens: 78,
|
|
},
|
|
}, nil
|
|
}
|
|
|
|
func TestRunnerIntegrationWithPromptAndProfileFixturesAndValidation(t *testing.T) {
|
|
root, err := filepath.Abs(filepath.Join("..", ".."))
|
|
if err != nil {
|
|
t.Fatalf("failed to resolve repo root: %v", err)
|
|
}
|
|
|
|
promptsDir := filepath.Join(root, "prompts")
|
|
profilesDir := filepath.Join(root, "profiles")
|
|
schemasDir := filepath.Join(root, "schemas")
|
|
fixturesDir := filepath.Join(root, "examples", "fixtures")
|
|
t.Setenv("SCRIPTORIUM_API_KEY", "test-key")
|
|
|
|
runner := NewRunner(
|
|
promptdef.NewFilesystemRepository(promptsDir),
|
|
profile.NewFilesystemRepository(profilesDir),
|
|
artifact.NewCompositeReader(),
|
|
prompt.NewGoRenderer(),
|
|
&integrationLLM{},
|
|
validate.NewStandardValidator(schemasDir),
|
|
)
|
|
|
|
res, err := runner.Run(context.Background(), domain.RunRequest{
|
|
PromptID: "generic.structured_events",
|
|
Inputs: map[string]domain.ArtifactRef{
|
|
"transcript": {
|
|
Type: domain.ArtifactRefFile,
|
|
URI: filepath.Join(fixturesDir, "transcript.md"),
|
|
},
|
|
"glossary": {
|
|
Type: domain.ArtifactRefFile,
|
|
URI: filepath.Join(fixturesDir, "glossary.yml"),
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("expected no error, got %v", err)
|
|
}
|
|
|
|
if res.PromptID != "generic.structured_events" {
|
|
t.Fatalf("unexpected prompt id: %q", res.PromptID)
|
|
}
|
|
if res.SelectedProfileID != "local-quality" {
|
|
t.Fatalf("expected selected profile local-quality from prompt default, got %q", res.SelectedProfileID)
|
|
}
|
|
if res.RunID == "" {
|
|
t.Fatal("expected run id")
|
|
}
|
|
if res.PromptHash == "" {
|
|
t.Fatal("expected prompt hash")
|
|
}
|
|
if res.PromptVersion != "1.0.0" {
|
|
t.Fatalf("unexpected prompt version: %q", res.PromptVersion)
|
|
}
|
|
if res.Validation.Status != domain.ValidationPassed {
|
|
t.Fatalf("expected passed validation, got %q", res.Validation.Status)
|
|
}
|
|
if res.Validation.Mode != domain.ValidationJSONSchema {
|
|
t.Fatalf("expected json_schema mode, got %q", res.Validation.Mode)
|
|
}
|
|
if res.Artifact.ContentType != "application/json" {
|
|
t.Fatalf("expected application/json output, got %q", res.Artifact.ContentType)
|
|
}
|
|
if len(res.RawOutput) == 0 {
|
|
t.Fatal("expected raw output to be preserved")
|
|
}
|
|
if res.PromptHash == "" {
|
|
t.Fatal("expected non-empty prompt hash")
|
|
}
|
|
if len(res.InputHashes) != 2 {
|
|
t.Fatalf("expected two input hashes, got %d", len(res.InputHashes))
|
|
}
|
|
if res.InputHashes["transcript"] == "" || res.InputHashes["glossary"] == "" {
|
|
t.Fatalf("expected both input hashes to be set, got %#v", res.InputHashes)
|
|
}
|
|
if res.Usage.TotalTokens != 78 {
|
|
t.Fatalf("expected usage from fake llm, got %+v", res.Usage)
|
|
}
|
|
if res.StartTime.IsZero() || res.EndTime.IsZero() {
|
|
t.Fatal("expected start/end timestamps")
|
|
}
|
|
if res.EndTime.Before(res.StartTime) {
|
|
t.Fatalf("expected end >= start, got start=%v end=%v", res.StartTime, res.EndTime)
|
|
}
|
|
if res.Duration < 0 {
|
|
t.Fatalf("expected non-negative duration, got %s", res.Duration)
|
|
}
|
|
}
|