Harden public ownership and diagnostic contracts
This commit is contained in:
@@ -203,26 +203,44 @@ func TestPreparedRunJSONDoesNotExposeSecretOrTargetPresence(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunRequestFormattingRedactsDirectAPIKey(t *testing.T) {
|
func TestRunRequestFormattingRedactsDirectAPIKey(t *testing.T) {
|
||||||
const secret = "run-request-secret"
|
const (
|
||||||
|
secret = "run-request-api-key-sentinel"
|
||||||
|
inputURI = "memory://run-request-uri-sentinel"
|
||||||
|
inputBody = "run-request-body-sentinel"
|
||||||
|
variableValue = "run-request-variable-sentinel"
|
||||||
|
)
|
||||||
req := promptkit.RunRequest{
|
req := promptkit.RunRequest{
|
||||||
PromptID: frameworkMarkdownSummaryPromptID,
|
PromptID: frameworkMarkdownSummaryPromptID,
|
||||||
ProfileID: frameworkFastProfileID,
|
ProfileID: frameworkFastProfileID,
|
||||||
APIKey: secret,
|
APIKey: secret,
|
||||||
Inputs: map[string]promptkit.ArtifactRef{
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
"transcript": promptkit.Inline("Rin opens the gate."),
|
"transcript": promptkit.InlineWithURI(inputURI, inputBody),
|
||||||
},
|
},
|
||||||
|
Vars: map[string]string{"audience": variableValue},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, formatted := range []string{
|
for _, formatted := range []string{
|
||||||
fmt.Sprint(req),
|
req.String(),
|
||||||
|
req.GoString(),
|
||||||
|
fmt.Sprintf("%v", req),
|
||||||
fmt.Sprintf("%+v", req),
|
fmt.Sprintf("%+v", req),
|
||||||
fmt.Sprintf("%#v", req),
|
fmt.Sprintf("%#v", req),
|
||||||
} {
|
} {
|
||||||
if strings.Contains(formatted, secret) {
|
for _, privateValue := range []string{secret, inputURI, inputBody, variableValue} {
|
||||||
t.Fatalf("formatted RunRequest leaked API key: %s", formatted)
|
if strings.Contains(formatted, privateValue) {
|
||||||
|
t.Fatalf("formatted RunRequest leaked private value %q: %s", privateValue, formatted)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, summary := range []string{
|
||||||
|
`PromptID:"` + frameworkMarkdownSummaryPromptID + `"`,
|
||||||
|
`ProfileID:"` + frameworkFastProfileID + `"`,
|
||||||
|
"APIKeySet:true",
|
||||||
|
"Inputs:1",
|
||||||
|
"Vars:1",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(formatted, summary) {
|
||||||
|
t.Fatalf("formatted RunRequest omitted structural summary %q: %s", summary, formatted)
|
||||||
}
|
}
|
||||||
if !strings.Contains(formatted, "APIKeySet:true") {
|
|
||||||
t.Fatalf("formatted RunRequest should indicate an API key is set, got %s", formatted)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1868,6 +1886,46 @@ func TestOpenAICompatibleProfileRunsThroughNormalProfilePath(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleProfileMapsEveryField(t *testing.T) {
|
||||||
|
extraParams := map[string]any{"provider_option": "distinct-extra-params"}
|
||||||
|
got := promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||||
|
ID: "distinct-id",
|
||||||
|
BackendID: "distinct-backend",
|
||||||
|
Endpoint: "https://distinct.example/v1",
|
||||||
|
Model: "distinct-model",
|
||||||
|
APIKeyRequired: true,
|
||||||
|
Temperature: 0.25,
|
||||||
|
MaxTokens: 321,
|
||||||
|
TopP: 0.75,
|
||||||
|
TimeoutSeconds: 43,
|
||||||
|
ServiceTier: "distinct-service-tier",
|
||||||
|
ReasoningEffort: "distinct-reasoning-effort",
|
||||||
|
ExtraParams: extraParams,
|
||||||
|
})
|
||||||
|
want := promptkit.Profile{
|
||||||
|
ID: "distinct-id",
|
||||||
|
BackendID: "distinct-backend",
|
||||||
|
Endpoint: "https://distinct.example/v1",
|
||||||
|
Model: "distinct-model",
|
||||||
|
Temperature: 0.25,
|
||||||
|
MaxTokens: 321,
|
||||||
|
TopP: 0.75,
|
||||||
|
TimeoutSeconds: 43,
|
||||||
|
ServiceTier: "distinct-service-tier",
|
||||||
|
ReasoningEffort: "distinct-reasoning-effort",
|
||||||
|
APIKeyRequired: true,
|
||||||
|
ExtraParams: map[string]any{"provider_option": "distinct-extra-params"},
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("profile mapping:\ngot %#v\nwant %#v", got, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
extraParams["added_after_construction"] = true
|
||||||
|
if _, ok := got.ExtraParams["added_after_construction"]; ok {
|
||||||
|
t.Fatal("profile retained the configuration map")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestEngineRunLayersTransportAndGenerationTimeouts(t *testing.T) {
|
func TestEngineRunLayersTransportAndGenerationTimeouts(t *testing.T) {
|
||||||
intPointer := func(value int) *int {
|
intPointer := func(value int) *int {
|
||||||
return &value
|
return &value
|
||||||
|
|||||||
96
llm_adapter_internal_test.go
Normal file
96
llm_adapter_internal_test.go
Normal file
@@ -0,0 +1,96 @@
|
|||||||
|
package promptkit
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"reflect"
|
||||||
|
"strconv"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPublicLLMClientAdapterGivesClientOwnedNestedValues(t *testing.T) {
|
||||||
|
source := adapterOwnershipRequest()
|
||||||
|
want := adapterOwnershipRequest()
|
||||||
|
client := &retainingMutatingLLMClient{}
|
||||||
|
adapter := publicLLMClientAdapter{client: client}
|
||||||
|
|
||||||
|
if _, err := adapter.Generate(context.Background(), source); err != nil {
|
||||||
|
t.Fatalf("generate: %v", err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(source, want) {
|
||||||
|
t.Fatalf("client mutation changed prepared source:\ngot %#v\nwant %#v", source, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
var mutations sync.WaitGroup
|
||||||
|
mutations.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer mutations.Done()
|
||||||
|
for i := 0; i < 10_000; i++ {
|
||||||
|
mutateGenerateRequest(&client.retained, strconv.Itoa(i))
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
for i := 0; i < 10_000; i++ {
|
||||||
|
laterRequest := fromDomainGenerateRequest(source)
|
||||||
|
if laterRequest.Prompt.Messages[0].Content != "source-message" ||
|
||||||
|
laterRequest.Prompt.Messages[0].CacheControl.TTL != "source-ttl" ||
|
||||||
|
laterRequest.Target.ExtraParams["nested"].([]any)[0] != "source-extra" ||
|
||||||
|
laterRequest.StructuredOutput.JSONSchema.Schema.(map[string]any)["enum"].([]any)[0] != "source-schema" {
|
||||||
|
t.Fatal("retained client mutation reached a later execution request")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
mutations.Wait()
|
||||||
|
|
||||||
|
if !reflect.DeepEqual(source, want) {
|
||||||
|
t.Fatalf("retained client mutation changed prepared source:\ngot %#v\nwant %#v", source, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type retainingMutatingLLMClient struct {
|
||||||
|
retained GenerateRequest
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *retainingMutatingLLMClient) Generate(_ context.Context, request GenerateRequest) (*GenerateResponse, error) {
|
||||||
|
c.retained = request
|
||||||
|
mutateGenerateRequest(&c.retained, "client-mutation")
|
||||||
|
return &GenerateResponse{Content: "generated"}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func adapterOwnershipRequest() domain.GenerateRequest {
|
||||||
|
return domain.GenerateRequest{
|
||||||
|
Prompt: domain.RenderedPrompt{
|
||||||
|
SessionID: "source-session",
|
||||||
|
Messages: []domain.RenderedMessage{{
|
||||||
|
Role: "user",
|
||||||
|
Content: "source-message",
|
||||||
|
CacheControl: &domain.CacheControl{
|
||||||
|
Type: domain.CacheControlEphemeral,
|
||||||
|
TTL: "source-ttl",
|
||||||
|
},
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
Target: domain.ExecutionTarget{
|
||||||
|
Model: "source-model",
|
||||||
|
APIKey: "source-api-key",
|
||||||
|
ExtraParams: map[string]any{
|
||||||
|
"nested": []any{"source-extra"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
StructuredOutput: &domain.StructuredOutputSpec{
|
||||||
|
Type: domain.StructuredOutputJSONSchema,
|
||||||
|
JSONSchema: &domain.StructuredOutputJSONSpec{
|
||||||
|
Name: "source-schema-name",
|
||||||
|
Strict: true,
|
||||||
|
Schema: map[string]any{"enum": []any{"source-schema"}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func mutateGenerateRequest(request *GenerateRequest, value string) {
|
||||||
|
request.Prompt.Messages[0].Content = value
|
||||||
|
request.Prompt.Messages[0].CacheControl.TTL = value
|
||||||
|
request.Target.ExtraParams["nested"].([]any)[0] = value
|
||||||
|
request.StructuredOutput.JSONSchema.Schema.(map[string]any)["enum"].([]any)[0] = value
|
||||||
|
}
|
||||||
@@ -44,12 +44,12 @@ func (p *PreparedExecution) Discard() {
|
|||||||
|
|
||||||
// String returns a constant representation that exposes no retained request,
|
// String returns a constant representation that exposes no retained request,
|
||||||
// rendered content, or credential data.
|
// rendered content, or credential data.
|
||||||
func (p *PreparedExecution) String() string {
|
func (p PreparedExecution) String() string {
|
||||||
return preparedExecutionString
|
return preparedExecutionString
|
||||||
}
|
}
|
||||||
|
|
||||||
// GoString returns a constant Go-syntax representation that exposes no
|
// GoString returns a constant Go-syntax representation that exposes no
|
||||||
// retained request, rendered content, or credential data.
|
// retained request, rendered content, or credential data.
|
||||||
func (p *PreparedExecution) GoString() string {
|
func (p PreparedExecution) GoString() string {
|
||||||
return preparedExecutionString
|
return preparedExecutionString
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -409,14 +409,35 @@ func TestPreparedExecutionDiscardAndFormattingDoNotExposePrivateState(t *testing
|
|||||||
t.Fatalf("prepare execution: %v", err)
|
t.Fatalf("prepare execution: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
formattedValues := []string{
|
copied := *prepared
|
||||||
fmt.Sprint(prepared),
|
zeroValue := promptkit.PreparedExecution{}
|
||||||
fmt.Sprintf("%+v", prepared),
|
var nilHandle *promptkit.PreparedExecution
|
||||||
fmt.Sprintf("%#v", prepared),
|
for name, value := range map[string]any{
|
||||||
}
|
"original pointer": prepared,
|
||||||
for _, formatted := range formattedValues {
|
"copied value": copied,
|
||||||
|
"zero value": zeroValue,
|
||||||
|
"zero pointer": &zeroValue,
|
||||||
|
} {
|
||||||
|
for format, formatted := range map[string]string{
|
||||||
|
"String": fmt.Sprintf("%s", value),
|
||||||
|
"GoString": fmt.Sprintf("%#v", value),
|
||||||
|
"v": fmt.Sprintf("%v", value),
|
||||||
|
"+v": fmt.Sprintf("%+v", value),
|
||||||
|
} {
|
||||||
if formatted != "promptkit.PreparedExecution{opaque}" {
|
if formatted != "promptkit.PreparedExecution{opaque}" {
|
||||||
t.Fatalf("unexpected opaque formatting: %q", formatted)
|
t.Fatalf("%s %s formatting = %q, want opaque representation", name, format, formatted)
|
||||||
|
}
|
||||||
|
assertPreparedPrivateValuesAbsent(t, formatted, directCredential, renderedContent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for format, formatted := range map[string]string{
|
||||||
|
"String": fmt.Sprintf("%s", nilHandle),
|
||||||
|
"GoString": fmt.Sprintf("%#v", nilHandle),
|
||||||
|
"v": fmt.Sprintf("%v", nilHandle),
|
||||||
|
"+v": fmt.Sprintf("%+v", nilHandle),
|
||||||
|
} {
|
||||||
|
if formatted != "<nil>" {
|
||||||
|
t.Fatalf("nil pointer %s formatting = %q, want <nil>", format, formatted)
|
||||||
}
|
}
|
||||||
assertPreparedPrivateValuesAbsent(t, formatted, directCredential, renderedContent)
|
assertPreparedPrivateValuesAbsent(t, formatted, directCredential, renderedContent)
|
||||||
}
|
}
|
||||||
@@ -433,27 +454,9 @@ func TestPreparedExecutionDiscardAndFormattingDoNotExposePrivateState(t *testing
|
|||||||
}
|
}
|
||||||
assertPreparedPrivateValuesAbsent(t, string(detailsJSON), directCredential)
|
assertPreparedPrivateValuesAbsent(t, string(detailsJSON), directCredential)
|
||||||
|
|
||||||
prepared.Discard()
|
executionResult, err := engine.RunPrepared(context.Background(), &copied)
|
||||||
prepared.Discard()
|
|
||||||
result, lifecycleErr := engine.RunPrepared(context.Background(), prepared)
|
|
||||||
if result != nil || !errors.Is(lifecycleErr, promptkit.ErrInvalidRequest) {
|
|
||||||
t.Fatalf("discarded execution result=(%+v, %v), want ErrInvalidRequest", result, lifecycleErr)
|
|
||||||
}
|
|
||||||
assertPreparedPrivateValuesAbsent(t, lifecycleErr.Error(), directCredential, renderedContent)
|
|
||||||
if !reflect.DeepEqual(prepared.Details(), detailsBefore) {
|
|
||||||
t.Fatal("details changed after discard")
|
|
||||||
}
|
|
||||||
|
|
||||||
executed, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
|
|
||||||
PromptID: "prepared",
|
|
||||||
APIKey: directCredential,
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("prepare execution for request inspection: %v", err)
|
t.Fatalf("run copied execution after formatting: %v", err)
|
||||||
}
|
|
||||||
executionResult, err := engine.RunPrepared(context.Background(), executed)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("run execution for request inspection: %v", err)
|
|
||||||
}
|
}
|
||||||
requests := client.snapshot()
|
requests := client.snapshot()
|
||||||
if len(requests) != 1 || requests[0].APIKey != directCredential {
|
if len(requests) != 1 || requests[0].APIKey != directCredential {
|
||||||
@@ -478,16 +481,34 @@ func TestPreparedExecutionDiscardAndFormattingDoNotExposePrivateState(t *testing
|
|||||||
}
|
}
|
||||||
assertPreparedPrivateValuesAbsent(t, string(resultJSON), directCredential)
|
assertPreparedPrivateValuesAbsent(t, string(resultJSON), directCredential)
|
||||||
|
|
||||||
var nilHandle *promptkit.PreparedExecution
|
|
||||||
nilHandle.Discard()
|
nilHandle.Discard()
|
||||||
if !reflect.DeepEqual(nilHandle.Details(), promptkit.PreparedRun{}) {
|
if !reflect.DeepEqual(nilHandle.Details(), promptkit.PreparedRun{}) {
|
||||||
t.Fatalf("nil handle details=%+v, want zero value", nilHandle.Details())
|
t.Fatalf("nil handle details=%+v, want zero value", nilHandle.Details())
|
||||||
}
|
}
|
||||||
zeroHandle := &promptkit.PreparedExecution{}
|
zeroHandle := &zeroValue
|
||||||
zeroHandle.Discard()
|
zeroHandle.Discard()
|
||||||
if !reflect.DeepEqual(zeroHandle.Details(), promptkit.PreparedRun{}) {
|
if !reflect.DeepEqual(zeroHandle.Details(), promptkit.PreparedRun{}) {
|
||||||
t.Fatalf("zero handle details=%+v, want zero value", zeroHandle.Details())
|
t.Fatalf("zero handle details=%+v, want zero value", zeroHandle.Details())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
discarded, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
|
||||||
|
PromptID: "prepared",
|
||||||
|
APIKey: directCredential,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare execution for discard: %v", err)
|
||||||
|
}
|
||||||
|
discardedDetails := discarded.Details()
|
||||||
|
discarded.Discard()
|
||||||
|
discarded.Discard()
|
||||||
|
result, lifecycleErr := engine.RunPrepared(context.Background(), discarded)
|
||||||
|
if result != nil || !errors.Is(lifecycleErr, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("discarded execution result=(%+v, %v), want ErrInvalidRequest", result, lifecycleErr)
|
||||||
|
}
|
||||||
|
assertPreparedPrivateValuesAbsent(t, lifecycleErr.Error(), directCredential, renderedContent)
|
||||||
|
if !reflect.DeepEqual(discarded.Details(), discardedDetails) {
|
||||||
|
t.Fatal("details changed after discard")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPreparedExecutionCredentialCapacityAndTimingBoundaries(t *testing.T) {
|
func TestPreparedExecutionCredentialCapacityAndTimingBoundaries(t *testing.T) {
|
||||||
|
|||||||
Reference in New Issue
Block a user