package promptkit_test import ( "context" "encoding/json" "errors" "strings" "testing" "testing/fstest" "gitea.maximumdirect.net/eric/promptkit" ) func observationEngine(t *testing.T, client promptkit.LLMClient) *promptkit.Engine { t.Helper() engine, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "hello"), "."), promptkit.WithProfiles(promptkit.Profile{ID: "profile", Endpoint: "https://example.invalid/v1", Model: "model"}), promptkit.WithLLMClient(client)) if err != nil { t.Fatal(err) } return engine } func TestRunObservationCoversFacadeErrors(t *testing.T) { for _, tc := range []struct { name string engine *promptkit.Engine request promptkit.RunRequest failure promptkit.ExecutionFailureKind }{ {name: "nil engine", failure: promptkit.ExecutionFailureInvalidConfig}, {name: "conversion", request: promptkit.RunRequest{AppendedMessages: []promptkit.RenderedMessage{{Role: "invalid"}}}, failure: promptkit.ExecutionFailureInvalidRequest}, } { t.Run(tc.name, func(t *testing.T) { var events []promptkit.ExecutionEvent request := tc.request request.Observer = func(event promptkit.ExecutionEvent) { events = append(events, event) } engine := tc.engine if tc.name == "conversion" { engine = observationEngine(t, &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}) } result, err := engine.Run(context.Background(), request) if result != nil || err == nil || len(events) != 2 || events[0].Kind != promptkit.ExecutionStarted || events[1].Kind != promptkit.ExecutionFinished || events[1].Failure != tc.failure || events[1].Attempts != 0 { t.Fatalf("result=%+v error=%v events=%+v", result, err, events) } if events[0].PromptID != "" || events[1].PromptID != "" || events[0].RunID == "" || events[0].RunID != events[1].RunID { t.Fatalf("identities=%+v", events) } }) } } func TestRunObservationHasResolvedIdentitiesAndAccounting(t *testing.T) { client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok", Usage: promptkit.TokenUsage{PromptTokens: 3}}} engine := observationEngine(t, client) var events []promptkit.ExecutionEvent request := promptkit.RunRequest{PromptID: "prompt", Observer: func(event promptkit.ExecutionEvent) { events = append(events, event) }} result, err := engine.Run(context.Background(), request) if err != nil { t.Fatal(err) } if len(events) != 3 || events[0].Kind != promptkit.ExecutionStarted || events[1].Kind != promptkit.ExecutionAttempt || events[2].Kind != promptkit.ExecutionFinished { t.Fatalf("events=%+v", events) } if events[0].PromptID != "" || events[1].PromptID != "prompt" || events[1].PromptVersion != "1" || events[1].ProfileID != "profile" || events[1].Model != "model" { t.Fatalf("selection=%+v", events) } if result.RunID == "" || events[0].RunID != result.RunID || events[2].RunID != result.RunID || events[2].Attempts != 1 || events[2].TransientRetries != 0 || events[2].RepairCalls != 0 || events[2].Outcome != promptkit.ExecutionOutcomeSuccess { t.Fatalf("result=%+v terminal=%+v", result, events[2]) } if got := events[2].Usage.PromptTokens; got.Value != 3 || !got.Known || !got.Complete { t.Fatalf("usage=%+v", got) } if got := events[2].Usage.CompletionTokens; got.Known || got.Complete { t.Fatalf("unknown usage=%+v", got) } encoded, err := json.Marshal(request) if err != nil { t.Fatal(err) } if string(encoded) == "" || json.Valid(encoded) == false { t.Fatalf("request JSON=%q", encoded) } var fields map[string]json.RawMessage if err := json.Unmarshal(encoded, &fields); err != nil { t.Fatal(err) } if _, ok := fields["Observer"]; ok { t.Fatalf("observer leaked into JSON: %s", encoded) } } func TestRunObservationClassifiesInvalidOutputAndGenerationFailure(t *testing.T) { for _, tc := range []struct { name, content string err error validation promptkit.ValidationMode outcome promptkit.ExecutionOutcome failure promptkit.ExecutionFailureKind }{ {name: "invalid", content: "not json", validation: promptkit.ValidationJSON, outcome: promptkit.ExecutionOutcomeInvalidOutput}, {name: "skipped", content: "anything", validation: promptkit.ValidationNone, outcome: promptkit.ExecutionOutcomeSuccess}, {name: "generation error", err: errors.New("private model detail"), outcome: promptkit.ExecutionOutcomeError, failure: promptkit.ExecutionFailureGeneration}, } { t.Run(tc.name, func(t *testing.T) { client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: tc.content}, err: tc.err} engine := observationEngine(t, client) var events []promptkit.ExecutionEvent request := promptkit.RunRequest{PromptID: "prompt", Observer: func(event promptkit.ExecutionEvent) { events = append(events, event) }} if tc.validation != "" { request.Validation = &promptkit.OutputContract{Format: promptkit.FormatText, ValidationMode: tc.validation} } result, err := engine.Run(context.Background(), request) if tc.err != nil && (result != nil || !errors.Is(err, promptkit.ErrLLMGenerate)) { t.Fatalf("result=%+v error=%v", result, err) } if tc.err == nil && (err != nil || result == nil) { t.Fatalf("result=%+v error=%v", result, err) } terminal := events[len(events)-1] if terminal.Outcome != tc.outcome || terminal.Failure != tc.failure || terminal.Attempts != 1 { t.Fatalf("terminal=%+v", terminal) } }) } } func TestRunObservationPromptFailureAndPanicCleanup(t *testing.T) { engine, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithPromptFS(fstest.MapFS{}, ".")) if err != nil { t.Fatal(err) } var events []promptkit.ExecutionEvent _, err = engine.Run(context.Background(), promptkit.RunRequest{PromptID: "missing", Observer: func(event promptkit.ExecutionEvent) { events = append(events, event) }}) if !errors.Is(err, promptkit.ErrPromptNotFound) || len(events) != 2 || events[1].Failure != promptkit.ExecutionFailurePromptNotFound || events[1].PromptID != "" { t.Fatalf("error=%v events=%+v", err, events) } engine = observationEngine(t, &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}) func() { defer func() { if recover() == nil { t.Error("expected observer panic") } }() _, _ = engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Observer: func(event promptkit.ExecutionEvent) { if event.Kind == promptkit.ExecutionAttempt { panic("callback") } }}) }() result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt"}) if err != nil || result == nil { t.Fatalf("run after callback panic: result=%+v error=%v", result, err) } } func TestRunObservationIncludesCompletedUsageBeforeRepairFailure(t *testing.T) { client := &fakeLLMClient{responses: []*promptkit.GenerateResponse{ {Content: "invalid json", Usage: promptkit.TokenUsage{PromptTokens: 7}}, nil, }} engine := observationEngine(t, client) var events []promptkit.ExecutionEvent result, err := engine.Run(context.Background(), promptkit.RunRequest{ PromptID: "prompt", Validation: &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: 1}, Observer: func(event promptkit.ExecutionEvent) { events = append(events, event) }, }) if result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) { t.Fatalf("result=%+v error=%v", result, err) } if len(events) != 4 || events[1].Outcome != promptkit.ExecutionOutcomeSuccess || events[2].Generation != 2 || events[2].Outcome != promptkit.ExecutionOutcomeError { t.Fatalf("events=%+v", events) } terminal := events[3] if terminal.Attempts != 2 || terminal.RepairCalls != 1 || terminal.Failure != promptkit.ExecutionFailureMalformedResponse || terminal.Usage.PromptTokens.Value != 7 || terminal.Usage.PromptTokens.Complete { t.Fatalf("terminal=%+v", terminal) } } func TestRunObservationValidationPreparationError(t *testing.T) { engine := observationEngine(t, &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}) var events []promptkit.ExecutionEvent result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSONSchema, SchemaPath: "missing-schema.json"}, Observer: func(event promptkit.ExecutionEvent) { events = append(events, event) }}) if result != nil || !errors.Is(err, promptkit.ErrValidation) || len(events) != 2 || events[1].Failure != promptkit.ExecutionFailureValidation || events[1].Attempts != 0 { t.Fatalf("result=%+v error=%v events=%+v", result, err, events) } } func TestRunObservationAdmissionRejection(t *testing.T) { artifactRelease := make(chan struct{}) reader := &capacityArtifactReader{entered: make(chan struct{}, 2), release: artifactRelease} client := newCapacityGateClient(closedCapacityChannel(), 2) engine := newBackendCapacityEngine(t, client, 1, capacityInt(0), reader) first := make(chan capacityRunResult, 1) go runCapacityRequest(engine, context.Background(), capacityInputRequest("http://first.example/v1"), first) awaitCapacitySignal(t, reader.entered, "first artifact read") var events []promptkit.ExecutionEvent request := capacityInputRequest("http://second.example/v1") request.Observer = func(event promptkit.ExecutionEvent) { events = append(events, event) } result, err := engine.Run(context.Background(), request) close(artifactRelease) if firstResult := awaitCapacityRun(t, first); firstResult.err != nil { t.Fatalf("first run: %v", firstResult.err) } if result != nil || !errors.Is(err, promptkit.ErrCapacityExceeded) || len(events) != 2 || events[1].Failure != promptkit.ExecutionFailureCapacity || events[1].BackendID != "limited" || events[1].Attempts != 0 { t.Fatalf("result=%+v error=%v events=%+v", result, err, events) } } func TestRunObservationValuesCannotChangeExecutionAndHideDiagnostics(t *testing.T) { const privateDetail = "provider-private-marker" client := &fakeLLMClient{err: errors.New(privateDetail)} engine := observationEngine(t, client) var events []promptkit.ExecutionEvent _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Observer: func(event promptkit.ExecutionEvent) { events = append(events, event) event.Model = "changed by callback" event.RunID = "changed by callback" }}) if !errors.Is(err, promptkit.ErrLLMGenerate) || len(events) != 3 || events[1].RunID != events[2].RunID || events[2].Model != "model" { t.Fatalf("error=%v events=%+v", err, events) } encoded, err := json.Marshal(events) if err != nil { t.Fatal(err) } if strings.Contains(string(encoded), privateDetail) || strings.Contains(string(encoded), "https://example.invalid") { t.Fatalf("sensitive data in events: %s", encoded) } }