234 lines
11 KiB
Go
234 lines
11 KiB
Go
package promptkit_test
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
"testing/fstest"
|
|
|
|
"gitea.maximumdirect.net/eric/promptkit"
|
|
)
|
|
|
|
func observationEngine(t *testing.T, client promptkit.LLMClient) *promptkit.Engine {
|
|
t.Helper()
|
|
engine, err := promptkit.NewEngine(promptkit.Config{},
|
|
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "hello"), "."),
|
|
promptkit.WithProfiles(promptkit.Profile{ID: "profile", Endpoint: "https://example.invalid/v1", Model: "model"}),
|
|
promptkit.WithLLMClient(client))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return engine
|
|
}
|
|
|
|
func TestRunObservationCoversFacadeErrors(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
engine *promptkit.Engine
|
|
request promptkit.RunRequest
|
|
failure promptkit.ExecutionFailureKind
|
|
}{
|
|
{name: "nil engine", failure: promptkit.ExecutionFailureInvalidConfig},
|
|
{name: "conversion", request: promptkit.RunRequest{AppendedMessages: []promptkit.RenderedMessage{{Role: "invalid"}}}, failure: promptkit.ExecutionFailureInvalidRequest},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
var events []promptkit.ExecutionEvent
|
|
request := tc.request
|
|
request.Observer = func(event promptkit.ExecutionEvent) { events = append(events, event) }
|
|
engine := tc.engine
|
|
if tc.name == "conversion" {
|
|
engine = observationEngine(t, &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}})
|
|
}
|
|
result, err := engine.Run(context.Background(), request)
|
|
if result != nil || err == nil || len(events) != 2 || events[0].Kind != promptkit.ExecutionStarted || events[1].Kind != promptkit.ExecutionFinished || events[1].Failure != tc.failure || events[1].Attempts != 0 {
|
|
t.Fatalf("result=%+v error=%v events=%+v", result, err, events)
|
|
}
|
|
if events[0].PromptID != "" || events[1].PromptID != "" || events[0].RunID == "" || events[0].RunID != events[1].RunID {
|
|
t.Fatalf("identities=%+v", events)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRunObservationHasResolvedIdentitiesAndAccounting(t *testing.T) {
|
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok", Usage: promptkit.TokenUsage{PromptTokens: 3}}}
|
|
engine := observationEngine(t, client)
|
|
var events []promptkit.ExecutionEvent
|
|
request := promptkit.RunRequest{PromptID: "prompt", Observer: func(event promptkit.ExecutionEvent) { events = append(events, event) }}
|
|
result, err := engine.Run(context.Background(), request)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(events) != 3 || events[0].Kind != promptkit.ExecutionStarted || events[1].Kind != promptkit.ExecutionAttempt || events[2].Kind != promptkit.ExecutionFinished {
|
|
t.Fatalf("events=%+v", events)
|
|
}
|
|
if events[0].PromptID != "" || events[1].PromptID != "prompt" || events[1].PromptVersion != "1" || events[1].ProfileID != "profile" || events[1].Model != "model" {
|
|
t.Fatalf("selection=%+v", events)
|
|
}
|
|
if result.RunID == "" || events[0].RunID != result.RunID || events[2].RunID != result.RunID || events[2].Attempts != 1 || events[2].TransientRetries != 0 || events[2].RepairCalls != 0 || events[2].Outcome != promptkit.ExecutionOutcomeSuccess {
|
|
t.Fatalf("result=%+v terminal=%+v", result, events[2])
|
|
}
|
|
if got := events[2].Usage.PromptTokens; got.Value != 3 || !got.Known || !got.Complete {
|
|
t.Fatalf("usage=%+v", got)
|
|
}
|
|
if got := events[2].Usage.CompletionTokens; got.Known || got.Complete {
|
|
t.Fatalf("unknown usage=%+v", got)
|
|
}
|
|
encoded, err := json.Marshal(request)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if string(encoded) == "" || json.Valid(encoded) == false {
|
|
t.Fatalf("request JSON=%q", encoded)
|
|
}
|
|
var fields map[string]json.RawMessage
|
|
if err := json.Unmarshal(encoded, &fields); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, ok := fields["Observer"]; ok {
|
|
t.Fatalf("observer leaked into JSON: %s", encoded)
|
|
}
|
|
}
|
|
|
|
func TestRunObservationClassifiesInvalidOutputAndGenerationFailure(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name, content string
|
|
err error
|
|
validation promptkit.ValidationMode
|
|
outcome promptkit.ExecutionOutcome
|
|
failure promptkit.ExecutionFailureKind
|
|
}{
|
|
{name: "invalid", content: "not json", validation: promptkit.ValidationJSON, outcome: promptkit.ExecutionOutcomeInvalidOutput},
|
|
{name: "skipped", content: "anything", validation: promptkit.ValidationNone, outcome: promptkit.ExecutionOutcomeSuccess},
|
|
{name: "generation error", err: errors.New("private model detail"), outcome: promptkit.ExecutionOutcomeError, failure: promptkit.ExecutionFailureGeneration},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: tc.content}, err: tc.err}
|
|
engine := observationEngine(t, client)
|
|
var events []promptkit.ExecutionEvent
|
|
request := promptkit.RunRequest{PromptID: "prompt", Observer: func(event promptkit.ExecutionEvent) { events = append(events, event) }}
|
|
if tc.validation != "" {
|
|
request.Validation = &promptkit.OutputContract{Format: promptkit.FormatText, ValidationMode: tc.validation}
|
|
}
|
|
result, err := engine.Run(context.Background(), request)
|
|
if tc.err != nil && (result != nil || !errors.Is(err, promptkit.ErrLLMGenerate)) {
|
|
t.Fatalf("result=%+v error=%v", result, err)
|
|
}
|
|
if tc.err == nil && (err != nil || result == nil) {
|
|
t.Fatalf("result=%+v error=%v", result, err)
|
|
}
|
|
terminal := events[len(events)-1]
|
|
if terminal.Outcome != tc.outcome || terminal.Failure != tc.failure || terminal.Attempts != 1 {
|
|
t.Fatalf("terminal=%+v", terminal)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRunObservationPromptFailureAndPanicCleanup(t *testing.T) {
|
|
engine, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithPromptFS(fstest.MapFS{}, "."))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var events []promptkit.ExecutionEvent
|
|
_, err = engine.Run(context.Background(), promptkit.RunRequest{PromptID: "missing", Observer: func(event promptkit.ExecutionEvent) { events = append(events, event) }})
|
|
if !errors.Is(err, promptkit.ErrPromptNotFound) || len(events) != 2 || events[1].Failure != promptkit.ExecutionFailurePromptNotFound || events[1].PromptID != "" {
|
|
t.Fatalf("error=%v events=%+v", err, events)
|
|
}
|
|
|
|
engine = observationEngine(t, &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}})
|
|
func() {
|
|
defer func() {
|
|
if recover() == nil {
|
|
t.Error("expected observer panic")
|
|
}
|
|
}()
|
|
_, _ = engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Observer: func(event promptkit.ExecutionEvent) {
|
|
if event.Kind == promptkit.ExecutionAttempt {
|
|
panic("callback")
|
|
}
|
|
}})
|
|
}()
|
|
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
|
if err != nil || result == nil {
|
|
t.Fatalf("run after callback panic: result=%+v error=%v", result, err)
|
|
}
|
|
}
|
|
|
|
func TestRunObservationIncludesCompletedUsageBeforeRepairFailure(t *testing.T) {
|
|
client := &fakeLLMClient{responses: []*promptkit.GenerateResponse{
|
|
{Content: "invalid json", Usage: promptkit.TokenUsage{PromptTokens: 7}}, nil,
|
|
}}
|
|
engine := observationEngine(t, client)
|
|
var events []promptkit.ExecutionEvent
|
|
result, err := engine.Run(context.Background(), promptkit.RunRequest{
|
|
PromptID: "prompt", Validation: &promptkit.OutputContract{Format: promptkit.FormatJSON,
|
|
ValidationMode: promptkit.ValidationJSON, RepairAttempts: 1},
|
|
Observer: func(event promptkit.ExecutionEvent) { events = append(events, event) },
|
|
})
|
|
if result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) {
|
|
t.Fatalf("result=%+v error=%v", result, err)
|
|
}
|
|
if len(events) != 4 || events[1].Outcome != promptkit.ExecutionOutcomeSuccess || events[2].Generation != 2 || events[2].Outcome != promptkit.ExecutionOutcomeError {
|
|
t.Fatalf("events=%+v", events)
|
|
}
|
|
terminal := events[3]
|
|
if terminal.Attempts != 2 || terminal.RepairCalls != 1 || terminal.Failure != promptkit.ExecutionFailureMalformedResponse || terminal.Usage.PromptTokens.Value != 7 || terminal.Usage.PromptTokens.Complete {
|
|
t.Fatalf("terminal=%+v", terminal)
|
|
}
|
|
}
|
|
|
|
func TestRunObservationValidationPreparationError(t *testing.T) {
|
|
engine := observationEngine(t, &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}})
|
|
var events []promptkit.ExecutionEvent
|
|
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, SchemaPath: "missing-schema.json"}, Observer: func(event promptkit.ExecutionEvent) { events = append(events, event) }})
|
|
if result != nil || !errors.Is(err, promptkit.ErrValidation) || len(events) != 2 || events[1].Failure != promptkit.ExecutionFailureValidation || events[1].Attempts != 0 {
|
|
t.Fatalf("result=%+v error=%v events=%+v", result, err, events)
|
|
}
|
|
}
|
|
|
|
func TestRunObservationAdmissionRejection(t *testing.T) {
|
|
artifactRelease := make(chan struct{})
|
|
reader := &capacityArtifactReader{entered: make(chan struct{}, 2), release: artifactRelease}
|
|
client := newCapacityGateClient(closedCapacityChannel(), 2)
|
|
engine := newBackendCapacityEngine(t, client, 1, capacityInt(0), reader)
|
|
first := make(chan capacityRunResult, 1)
|
|
go runCapacityRequest(engine, context.Background(), capacityInputRequest("http://first.example/v1"), first)
|
|
awaitCapacitySignal(t, reader.entered, "first artifact read")
|
|
var events []promptkit.ExecutionEvent
|
|
request := capacityInputRequest("http://second.example/v1")
|
|
request.Observer = func(event promptkit.ExecutionEvent) { events = append(events, event) }
|
|
result, err := engine.Run(context.Background(), request)
|
|
close(artifactRelease)
|
|
if firstResult := awaitCapacityRun(t, first); firstResult.err != nil {
|
|
t.Fatalf("first run: %v", firstResult.err)
|
|
}
|
|
if result != nil || !errors.Is(err, promptkit.ErrCapacityExceeded) || len(events) != 2 || events[1].Failure != promptkit.ExecutionFailureCapacity || events[1].BackendID != "limited" || events[1].Attempts != 0 {
|
|
t.Fatalf("result=%+v error=%v events=%+v", result, err, events)
|
|
}
|
|
}
|
|
|
|
func TestRunObservationValuesCannotChangeExecutionAndHideDiagnostics(t *testing.T) {
|
|
const privateDetail = "provider-private-marker"
|
|
client := &fakeLLMClient{err: errors.New(privateDetail)}
|
|
engine := observationEngine(t, client)
|
|
var events []promptkit.ExecutionEvent
|
|
_, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Observer: func(event promptkit.ExecutionEvent) {
|
|
events = append(events, event)
|
|
event.Model = "changed by callback"
|
|
event.RunID = "changed by callback"
|
|
}})
|
|
if !errors.Is(err, promptkit.ErrLLMGenerate) || len(events) != 3 || events[1].RunID != events[2].RunID || events[2].Model != "model" {
|
|
t.Fatalf("error=%v events=%+v", err, events)
|
|
}
|
|
encoded, err := json.Marshal(events)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.Contains(string(encoded), privateDetail) || strings.Contains(string(encoded), "https://example.invalid") {
|
|
t.Fatalf("sensitive data in events: %s", encoded)
|
|
}
|
|
}
|