diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 918bfff..e8ac8b2 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -328,6 +328,8 @@ propagation and classification, not private helper call sequences. ## Stage 6: Verify Notification Ownership And Failure Lifecycles +Status: complete. + Outcome: applications can safely retain diagnostic details without altering execution or acquiring new lifecycle guarantees. diff --git a/internal/usecase/prepared_notification_test.go b/internal/usecase/prepared_notification_test.go index 0e83710..eceef74 100644 --- a/internal/usecase/prepared_notification_test.go +++ b/internal/usecase/prepared_notification_test.go @@ -18,3 +18,25 @@ func TestPreparedNotificationChecksContextBeforeInvocation(t *testing.T) { t.Fatalf("called=%v error=%v", called, err) } } + +func TestPreparedNotificationPanicClearsClaimedPayload(t *testing.T) { + runner := NewRunner(&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, + &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, + defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil, nil) + handle, err := runner.PrepareExecution(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(), OnPrepared: func(context.Context, *domain.PreparedRun) { panic("notification") }}) + if err != nil { + t.Fatal(err) + } + payload := handle.payload + func() { + defer func() { + if recover() == nil { + t.Error("expected panic") + } + }() + _, _ = runner.RunPrepared(context.Background(), handle) + }() + if payload.prepared != nil || payload.onPrepared != nil || payload.directKey != "" || handle.payload != nil { + t.Fatal("claimed payload retained private state") + } +} diff --git a/prepared_execution_contract_test.go b/prepared_execution_contract_test.go index adb8249..c953ce6 100644 --- a/prepared_execution_contract_test.go +++ b/prepared_execution_contract_test.go @@ -50,8 +50,20 @@ func TestPreparedExecutionFreezesSourcesAndReturnsIndependentDetails(t *testing. extraParams := map[string]any{ "nested": map[string]any{"source": "original"}, } + var notified *promptkit.PreparedRun request := promptkit.RunRequest{ PromptID: "prepared", + APIKey: "synthetic-direct-key", + OnPrepared: func(_ context.Context, details promptkit.PreparedRun) { + notified = &details + if strings.Contains(fmt.Sprintf("%#v", details), "synthetic-direct-key") { + t.Error("notification exposed credential") + } + details.Messages[0].Content = "mutated notification" + details.InputHashes["input"] = "mutated notification" + details.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["source"] = "mutated notification" + details.StructuredOutput.JSONSchema.Schema.(map[string]any)["title"] = "mutated notification" + }, Inputs: map[string]promptkit.ArtifactRef{ "input": promptkit.Inline("original request input"), }, @@ -119,12 +131,18 @@ func TestPreparedExecutionFreezesSourcesAndReturnsIndependentDetails(t *testing. if reader.callCount() != 1 { t.Fatalf("execution reopened artifact source: calls=%d", reader.callCount()) } + if notified == nil || prepared.Details().Messages[0].Content != second.Messages[0].Content || prepared.Details().StructuredOutput.JSONSchema.Schema.(map[string]any)["title"] != "original root" { + t.Fatal("notification changed prepared state") + } requests := client.snapshot() if len(requests) != 1 { t.Fatalf("generation calls=%d, want 1", len(requests)) } generated := requests[0] + if generated.APIKey != "synthetic-direct-key" { + t.Fatal("direct credential did not reach client") + } if generated.Prompt.Messages[0].Content != second.Messages[0].Content || generated.Target.Model != second.EffectiveModelParams.Model || !reflect.DeepEqual(generated.Target.ExtraParams, second.EffectiveModelParams.ExtraParams) || diff --git a/prepared_notification_contract_test.go b/prepared_notification_contract_test.go index e7ff527..210b751 100644 --- a/prepared_notification_contract_test.go +++ b/prepared_notification_contract_test.go @@ -3,6 +3,10 @@ package promptkit_test import ( "context" "errors" + "fmt" + "net/http" + "reflect" + "strings" "testing" "go.maximumdirect.net/promptkit" @@ -47,6 +51,160 @@ func TestPreparedNotificationSelectionAndOrdering(t *testing.T) { } } +func TestNotificationRedactsEnvironmentCredential(t *testing.T) { + t.Setenv("PROMPTKIT_NOTIFICATION_KEY", "synthetic-environment-key") + config := simpleConfig("message") + config.HTTPClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + if req.Header.Get("Authorization") != "Bearer synthetic-environment-key" { + t.Error("environment credential did not reach provider") + } + return retryHTTPResponse(200, "ok"), nil + })} + engine, err := promptkit.NewEngine(config) + if err != nil { + t.Fatal(err) + } + req := simpleGenerationRequest() + req.Execution = &promptkit.ExecutionTargetOverride{APIKeyEnv: "PROMPTKIT_NOTIFICATION_KEY"} + req.OnPrepared = func(_ context.Context, details promptkit.PreparedRun) { + if strings.Contains(fmt.Sprintf("%#v", details), "synthetic-environment-key") { + t.Error("notification exposed credential") + } + } + if _, err := engine.Run(context.Background(), req); err != nil { + t.Fatal(err) + } +} + +func TestContinuationNotificationIsDetachedAndSurvivesFailure(t *testing.T) { + failure := errors.New("provider failure") + calls := 0 + engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(observationClientFunc(func(_ context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + calls++ + if calls == 1 { + return &promptkit.GenerateResponse{Content: "answer"}, nil + } + if req.Prompt.Messages[0].Content != "question" || req.Prompt.Messages[3].Content != "follow-up" || req.APIKey != "synthetic-key" { + t.Fatalf("generation request=%+v", req) + } + return nil, failure + }))) + initial, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true}) + if err != nil { + t.Fatal(err) + } + before := initial.Exchange.Details() + var retained promptkit.PreparedRun + req := promptkit.RunRequest{PromptID: "prompt", ContinueFrom: initial.Exchange, APIKey: "synthetic-key", SessionID: "session", AppendedMessages: []promptkit.RenderedMessage{{Role: promptkit.RoleUser, Content: "follow-up", CacheControl: &promptkit.CacheControl{Type: promptkit.CacheControlEphemeral}}}, OnPrepared: func(_ context.Context, details promptkit.PreparedRun) { + retained = details + details.Messages[0].Content = "changed" + details.Messages[3].Content = "changed" + details.Messages[3].CacheControl.Type = "changed" + }} + result, err := engine.Run(context.Background(), req) + if result != nil || !errors.Is(err, failure) || retained.ParentRunID != initial.RunID || retained.SessionID != "session" || !reflect.DeepEqual(before, initial.Exchange.Details()) { + t.Fatalf("result=%v error=%v retained=%+v", result, err, retained) + } +} + +func TestPreparedNotificationInvalidHandlesAndPreparationOnlyCalls(t *testing.T) { + calls := 0 + engine := newPreparedContractEngine(t, &preparedRecordingClient{response: &promptkit.GenerateResponse{Content: "ok"}}, "content") + foreign := newPreparedContractEngine(t, &preparedRecordingClient{response: &promptkit.GenerateResponse{Content: "ok"}}, "other") + req := promptkit.RunRequest{PromptID: "prepared", OnPrepared: func(context.Context, promptkit.PreparedRun) { calls++ }} + if _, err := engine.Prepare(context.Background(), req); err != nil { + t.Fatal(err) + } + if err := engine.ValidateRequest(context.Background(), req); err != nil { + t.Fatal(err) + } + handle, err := engine.PrepareExecution(context.Background(), req) + if err != nil { + t.Fatal(err) + } + _ = handle.Details() + if _, err := foreign.RunPrepared(context.Background(), handle); !errors.Is(err, promptkit.ErrInvalidRequest) { + t.Fatal(err) + } + handle.Discard() + if _, err := engine.RunPrepared(context.Background(), handle); !errors.Is(err, promptkit.ErrInvalidRequest) { + t.Fatal(err) + } + if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "missing", OnPrepared: req.OnPrepared}); err == nil { + t.Fatal("expected preparation failure") + } + if calls != 0 { + t.Fatalf("unexpected notifications=%d", calls) + } +} + +func TestNotificationPanicReleasesCapacityAndConsumesHandle(t *testing.T) { + engine := newBackendCapacityEngine(t, observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + return &promptkit.GenerateResponse{Content: "ok"}, nil + }), 1, capacityInt(0), nil) + for _, split := range []bool{false, true} { + req := promptkit.RunRequest{PromptID: "prompt", OnPrepared: func(context.Context, promptkit.PreparedRun) { panic("notification") }} + var handle *promptkit.PreparedExecution + if split { + var err error + handle, err = engine.PrepareExecution(context.Background(), req) + if err != nil { + t.Fatal(err) + } + } + func() { + defer func() { + if recover() == nil { + t.Error("expected panic") + } + }() + if split { + _, _ = engine.RunPrepared(context.Background(), handle) + } else { + _, _ = engine.Run(context.Background(), req) + } + }() + if split { + if _, err := engine.RunPrepared(context.Background(), handle); !errors.Is(err, promptkit.ErrInvalidRequest) { + t.Fatal("handle reusable after panic") + } + } + if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt"}); err != nil { + t.Fatalf("capacity not released: %v", err) + } + } +} + +func TestNotificationSkippedOnAdmissionAndPreExecutionCancellation(t *testing.T) { + release := make(chan struct{}) + client := newCapacityGateClient(release, 1) + engine := newBackendCapacityEngine(t, client, 1, capacityInt(0), nil) + results := make(chan capacityRunResult, 1) + go runCapacityRequest(engine, context.Background(), promptkit.RunRequest{PromptID: "prompt"}, results) + <-client.started + calls := 0 + req := promptkit.RunRequest{PromptID: "prompt", OnPrepared: func(context.Context, promptkit.PreparedRun) { calls++ }} + if _, err := engine.Run(context.Background(), req); !errors.Is(err, promptkit.ErrCapacityExceeded) { + t.Fatalf("admission error=%v", err) + } + close(release) + if result := <-results; result.err != nil { + t.Fatal(result.err) + } + handle, err := engine.PrepareExecution(context.Background(), req) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err = engine.RunPrepared(ctx, handle); !errors.Is(err, context.Canceled) { + t.Fatalf("cancellation error=%v", err) + } + if calls != 0 { + t.Fatalf("notifications=%d", calls) + } +} + func TestCancellationInPreparedNotificationPreventsGeneration(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/reliable_execution_contract_test.go b/reliable_execution_contract_test.go index 6f3aa73..f6ccb35 100644 --- a/reliable_execution_contract_test.go +++ b/reliable_execution_contract_test.go @@ -46,10 +46,12 @@ func TestPublicRetryRepairSequenceKeepsUsageAndEventsConsistent(t *testing.T) { } var events []promptkit.ExecutionEvent req := simpleGenerationRequest() + notifications := 0 + req.OnPrepared = func(context.Context, promptkit.PreparedRun) { notifications++ } req.Validation = &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: 1} req.Observer = func(event promptkit.ExecutionEvent) { events = append(events, event) } result, err := engine.Run(context.Background(), req) - if err != nil || result == nil || !result.Validation.IsValid || calls != 4 { + if err != nil || result == nil || !result.Validation.IsValid || calls != 4 || notifications != 1 { t.Fatalf("result=%+v error=%v calls=%d", result, err, calls) } if len(events) != 8 {