Extend prompt execution contract
This commit is contained in:
@@ -188,6 +188,8 @@ GOWORK=off go test -race -count=1 ./internal/adapters/promptkit ./internal/app
|
|||||||
|
|
||||||
### Stage 3: Extend The Project-Owned Prompt Execution Contract
|
### Stage 3: Extend The Project-Owned Prompt Execution Contract
|
||||||
|
|
||||||
|
Status: Complete.
|
||||||
|
|
||||||
Purpose: establish dependency-neutral repair and structured-generation-error
|
Purpose: establish dependency-neutral repair and structured-generation-error
|
||||||
values before the adapter or application relies on them.
|
values before the adapter or application relies on them.
|
||||||
|
|
||||||
|
|||||||
@@ -193,6 +193,7 @@ func executionValue(value *promptkit.RunResult, captureDebug bool) *promptexec.E
|
|||||||
promptexec.ValidationStatus(value.Validation.Status),
|
promptexec.ValidationStatus(value.Validation.Status),
|
||||||
string(value.Validation.Mode),
|
string(value.Validation.Mode),
|
||||||
value.Validation.SchemaPath,
|
value.Validation.SchemaPath,
|
||||||
|
0,
|
||||||
value.Validation.Errors,
|
value.Validation.Errors,
|
||||||
)
|
)
|
||||||
rawOutput := []byte(nil)
|
rawOutput := []byte(nil)
|
||||||
@@ -203,6 +204,7 @@ func executionValue(value *promptkit.RunResult, captureDebug bool) *promptexec.E
|
|||||||
promptexec.ValidationFailed,
|
promptexec.ValidationFailed,
|
||||||
string(value.Validation.Mode),
|
string(value.Validation.Mode),
|
||||||
value.Validation.SchemaPath,
|
value.Validation.SchemaPath,
|
||||||
|
0,
|
||||||
[]string{"generated output exceeds the configured size limit"},
|
[]string{"generated output exceeds the configured size limit"},
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -202,7 +202,7 @@ func (e *barrierExecutor) Execute(ctx context.Context, req promptexec.ExecuteReq
|
|||||||
PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: generationPromptHash,
|
PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: generationPromptHash,
|
||||||
ProfileID: req.ProfileID, BackendID: profile.BackendID, ModelName: profile.ModelName,
|
ProfileID: req.ProfileID, BackendID: profile.BackendID, ModelName: profile.ModelName,
|
||||||
StartedAt: stamp, EndedAt: stamp, RawOutput: comparisonRawOutput(),
|
StartedAt: stamp, EndedAt: stamp, RawOutput: comparisonRawOutput(),
|
||||||
Validation: promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", generationDefinitionForPrompt(req.PromptID).GeneratedTextSchemaID+".generated_text.schema.json", nil),
|
Validation: promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", generationDefinitionForPrompt(req.PromptID).GeneratedTextSchemaID+".generated_text.schema.json", 0, nil),
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -162,7 +162,7 @@ func (e *generationExecutor) Execute(ctx context.Context, req promptexec.Execute
|
|||||||
if cancelBeforeReturn != nil {
|
if cancelBeforeReturn != nil {
|
||||||
cancelBeforeReturn()
|
cancelBeforeReturn()
|
||||||
}
|
}
|
||||||
execution := &promptexec.Execution{RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: generationPromptHash, RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp, RawOutput: rawOutput, Validation: promptexec.NewValidation(status, "json_schema", generationDefinitionForPrompt(req.PromptID).GeneratedTextSchemaID+".generated_text.schema.json", nil)}
|
execution := &promptexec.Execution{RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: generationPromptHash, RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp, RawOutput: rawOutput, Validation: promptexec.NewValidation(status, "json_schema", generationDefinitionForPrompt(req.PromptID).GeneratedTextSchemaID+".generated_text.schema.json", 0, nil)}
|
||||||
if complete != nil {
|
if complete != nil {
|
||||||
complete(execution)
|
complete(execution)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -349,7 +349,7 @@ func promptDebugExecutionFixture() promptexec.Execution {
|
|||||||
InputHashes: map[string]string{"data_package": "input-hash"}, ProfileID: "local", BackendID: "local", ModelName: "weather-model",
|
InputHashes: map[string]string{"data_package": "input-hash"}, ProfileID: "local", BackendID: "local", ModelName: "weather-model",
|
||||||
GeneratedHash: "generated-hash", Usage: promptexec.TokenUsage{PromptTokens: 10, CompletionTokens: 5, TotalTokens: 15},
|
GeneratedHash: "generated-hash", Usage: promptexec.TokenUsage{PromptTokens: 10, CompletionTokens: 5, TotalTokens: 15},
|
||||||
StartedAt: startedAt, EndedAt: startedAt.Add(time.Second), Duration: time.Second,
|
StartedAt: startedAt, EndedAt: startedAt.Add(time.Second), Duration: time.Second,
|
||||||
Validation: promptexec.NewValidation(promptexec.ValidationPassed, "strict", "schemas/daily.json", []string{"validation details"}),
|
Validation: promptexec.NewValidation(promptexec.ValidationPassed, "strict", "schemas/daily.json", 0, []string{"validation details"}),
|
||||||
RawOutput: []byte("Generated forecast prose."),
|
RawOutput: []byte("Generated forecast prose."),
|
||||||
Debug: &promptexec.ExecutionDebug{ValidationDiagnostics: []string{"validation details"}},
|
Debug: &promptexec.ExecutionDebug{ValidationDiagnostics: []string{"validation details"}},
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -78,3 +78,15 @@ func boundText(value string, limit int) string {
|
|||||||
}
|
}
|
||||||
return value
|
return value
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func boundCodePoints(value string, limit int) string {
|
||||||
|
if limit <= 0 || value == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
value = strings.ToValidUTF8(value, "<22>")
|
||||||
|
runes := []rune(value)
|
||||||
|
if len(runes) <= limit {
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
return string(runes[:limit])
|
||||||
|
}
|
||||||
|
|||||||
@@ -51,6 +51,7 @@ type OutputContract struct {
|
|||||||
Format string
|
Format string
|
||||||
ValidationMode string
|
ValidationMode string
|
||||||
SchemaPath string
|
SchemaPath string
|
||||||
|
RepairAttempts int
|
||||||
}
|
}
|
||||||
|
|
||||||
// ProfileInspection describes the safe, selected execution identity for one profile.
|
// ProfileInspection describes the safe, selected execution identity for one profile.
|
||||||
@@ -144,15 +145,17 @@ type Validation struct {
|
|||||||
Status ValidationStatus
|
Status ValidationStatus
|
||||||
Mode string
|
Mode string
|
||||||
SchemaPath string
|
SchemaPath string
|
||||||
|
RepairAttempts int
|
||||||
Diagnostics []string
|
Diagnostics []string
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewValidation returns a completed validation value with bounded diagnostics.
|
// NewValidation returns a completed validation value with bounded diagnostics.
|
||||||
func NewValidation(status ValidationStatus, mode string, schemaPath string, diagnostics []string) Validation {
|
func NewValidation(status ValidationStatus, mode string, schemaPath string, repairAttempts int, diagnostics []string) Validation {
|
||||||
return Validation{
|
return Validation{
|
||||||
Status: status,
|
Status: status,
|
||||||
Mode: mode,
|
Mode: mode,
|
||||||
SchemaPath: schemaPath,
|
SchemaPath: schemaPath,
|
||||||
|
RepairAttempts: repairAttempts,
|
||||||
Diagnostics: boundDiagnostics(diagnostics),
|
Diagnostics: boundDiagnostics(diagnostics),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -235,6 +238,86 @@ func (e *Error) Category() ErrorCategory {
|
|||||||
return e.category
|
return e.category
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GenerationError retains provider failure details for programmatic handling
|
||||||
|
// without exposing them through routine formatting or serialization.
|
||||||
|
type GenerationError struct {
|
||||||
|
statusCode int
|
||||||
|
providerCode string
|
||||||
|
providerType string
|
||||||
|
providerMessage string
|
||||||
|
err *Error
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewGenerationError returns a classified generation failure with bounded
|
||||||
|
// provider details. The dependency cause remains reachable through err only.
|
||||||
|
func NewGenerationError(statusCode int, providerCode string, providerType string, providerMessage string, cause error) *GenerationError {
|
||||||
|
return &GenerationError{
|
||||||
|
statusCode: statusCode,
|
||||||
|
providerCode: boundCodePoints(providerCode, 256),
|
||||||
|
providerType: boundCodePoints(providerType, 256),
|
||||||
|
providerMessage: boundCodePoints(providerMessage, 4096),
|
||||||
|
err: NewError(Generation, "provider generation failed", cause),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// StatusCode returns the provider HTTP status when one was available.
|
||||||
|
func (e *GenerationError) StatusCode() int {
|
||||||
|
if e == nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return e.statusCode
|
||||||
|
}
|
||||||
|
|
||||||
|
// ProviderCode returns the bounded provider error code.
|
||||||
|
func (e *GenerationError) ProviderCode() string {
|
||||||
|
if e == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return e.providerCode
|
||||||
|
}
|
||||||
|
|
||||||
|
// ProviderType returns the bounded provider error type.
|
||||||
|
func (e *GenerationError) ProviderType() string {
|
||||||
|
if e == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return e.providerType
|
||||||
|
}
|
||||||
|
|
||||||
|
// ProviderMessage returns the bounded provider error message.
|
||||||
|
func (e *GenerationError) ProviderMessage() string {
|
||||||
|
if e == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return e.providerMessage
|
||||||
|
}
|
||||||
|
|
||||||
|
// Category returns Generation for every generation failure.
|
||||||
|
func (e *GenerationError) Category() ErrorCategory { return Generation }
|
||||||
|
|
||||||
|
// Error intentionally excludes provider details from ordinary error text.
|
||||||
|
func (e *GenerationError) Error() string {
|
||||||
|
if e == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
message := NewError(Generation, "provider generation failed", nil).Error()
|
||||||
|
if e.statusCode == 0 {
|
||||||
|
return message
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s (HTTP %d)", message, e.statusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GoString keeps %#v formatting as safe as ordinary error formatting.
|
||||||
|
func (e *GenerationError) GoString() string { return e.Error() }
|
||||||
|
|
||||||
|
// Unwrap preserves the project-owned classified error and its hidden cause.
|
||||||
|
func (e *GenerationError) Unwrap() error {
|
||||||
|
if e == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return e.err
|
||||||
|
}
|
||||||
|
|
||||||
// CapacityError adds the safe backend identity to a capacity failure.
|
// CapacityError adds the safe backend identity to a capacity failure.
|
||||||
type CapacityError struct {
|
type CapacityError struct {
|
||||||
BackendID string
|
BackendID string
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package promptexec
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
@@ -172,8 +173,8 @@ func TestBoundDiagnosticAndErrorText(t *testing.T) {
|
|||||||
if bounded[0] != "a<>b" {
|
if bounded[0] != "a<>b" {
|
||||||
t.Fatalf("invalid UTF-8 diagnostic = %q, want replacement", bounded[0])
|
t.Fatalf("invalid UTF-8 diagnostic = %q, want replacement", bounded[0])
|
||||||
}
|
}
|
||||||
validation := NewValidation(ValidationFailed, "json_schema", "daily.schema.json", values)
|
validation := NewValidation(ValidationFailed, "json_schema", "daily.schema.json", 2, values)
|
||||||
if len(validation.Diagnostics) != maxValidationDiagnostics || validation.Diagnostics[0] != "a<>b" {
|
if validation.RepairAttempts != 2 || len(validation.Diagnostics) != maxValidationDiagnostics || validation.Diagnostics[0] != "a<>b" {
|
||||||
t.Fatalf("validation = %#v, want bounded diagnostics", validation)
|
t.Fatalf("validation = %#v, want bounded diagnostics", validation)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -187,8 +188,53 @@ func TestBoundDiagnosticAndErrorText(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestGenerationErrorKeepsProviderDetailsOutOfRoutineFormatting(t *testing.T) {
|
||||||
|
cause := errors.New("provider response must stay hidden")
|
||||||
|
error := NewGenerationError(
|
||||||
|
429,
|
||||||
|
string([]byte{'c', 0xff})+strings.Repeat("界", 300),
|
||||||
|
strings.Repeat("type", 100),
|
||||||
|
string([]byte{'m', 0xff})+strings.Repeat("界", 4_200),
|
||||||
|
cause,
|
||||||
|
)
|
||||||
|
if error.StatusCode() != 429 || error.Category() != Generation || CategoryOf(error) != Generation {
|
||||||
|
t.Fatalf("generation error identity = %#v", error)
|
||||||
|
}
|
||||||
|
if utf8.RuneCountInString(error.ProviderCode()) != 256 || utf8.RuneCountInString(error.ProviderType()) != 256 || utf8.RuneCountInString(error.ProviderMessage()) != 4096 {
|
||||||
|
t.Fatalf("provider detail bounds = %d/%d/%d", utf8.RuneCountInString(error.ProviderCode()), utf8.RuneCountInString(error.ProviderType()), utf8.RuneCountInString(error.ProviderMessage()))
|
||||||
|
}
|
||||||
|
if !utf8.ValidString(error.ProviderCode()) || !utf8.ValidString(error.ProviderType()) || !utf8.ValidString(error.ProviderMessage()) {
|
||||||
|
t.Fatalf("provider details are not valid UTF-8: %#v", error)
|
||||||
|
}
|
||||||
|
if !errors.Is(error, cause) {
|
||||||
|
t.Fatal("errors.Is() = false, want preserved dependency cause")
|
||||||
|
}
|
||||||
|
var owned *Error
|
||||||
|
if !errors.As(error, &owned) || owned.Category() != Generation {
|
||||||
|
t.Fatalf("errors.As(*Error) = %#v, want project-owned generation error", owned)
|
||||||
|
}
|
||||||
|
want := "generation: provider generation failed (HTTP 429)"
|
||||||
|
if error.Error() != want || fmt.Sprintf("%#v", error) != want {
|
||||||
|
t.Fatalf("ordinary formatting = %q / %#v, want %q", error.Error(), error, want)
|
||||||
|
}
|
||||||
|
for _, private := range []string{cause.Error(), error.ProviderCode(), error.ProviderType(), error.ProviderMessage()} {
|
||||||
|
if strings.Contains(error.Error(), private) || strings.Contains(fmt.Sprintf("%#v", error), private) {
|
||||||
|
t.Fatalf("generation error leaks provider detail %q", private)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
statusOnly := NewGenerationError(503, "", "", "", nil)
|
||||||
|
if statusOnly.Error() != "generation: provider generation failed (HTTP 503)" {
|
||||||
|
t.Fatalf("status-only error = %q", statusOnly)
|
||||||
|
}
|
||||||
|
var nilError *GenerationError
|
||||||
|
if nilError.StatusCode() != 0 || nilError.ProviderCode() != "" || nilError.ProviderType() != "" || nilError.ProviderMessage() != "" || nilError.Category() != Generation || nilError.Error() != "" || nilError.GoString() != "" || nilError.Unwrap() != nil {
|
||||||
|
t.Fatalf("nil generation error = %#v", nilError)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestContractCopiesMutableValues(t *testing.T) {
|
func TestContractCopiesMutableValues(t *testing.T) {
|
||||||
preparation := Preparation{InputHashes: map[string]string{"data_package": "input-hash"}}
|
preparation := Preparation{InputHashes: map[string]string{"data_package": "input-hash"}, Output: OutputContract{RepairAttempts: 3}}
|
||||||
preparationDebug := &PreparationDebug{
|
preparationDebug := &PreparationDebug{
|
||||||
RenderedMessages: []RenderedMessage{{Role: "user", Content: "rendered input"}},
|
RenderedMessages: []RenderedMessage{{Role: "user", Content: "rendered input"}},
|
||||||
StructuredSchema: []byte("schema body"),
|
StructuredSchema: []byte("schema body"),
|
||||||
@@ -196,7 +242,7 @@ func TestContractCopiesMutableValues(t *testing.T) {
|
|||||||
}
|
}
|
||||||
execution := Execution{
|
execution := Execution{
|
||||||
InputHashes: map[string]string{"data_package": "input-hash"},
|
InputHashes: map[string]string{"data_package": "input-hash"},
|
||||||
Validation: Validation{Diagnostics: []string{"validation detail"}},
|
Validation: Validation{RepairAttempts: 2, Diagnostics: []string{"validation detail"}},
|
||||||
RawOutput: []byte("generated output"),
|
RawOutput: []byte("generated output"),
|
||||||
Debug: &ExecutionDebug{
|
Debug: &ExecutionDebug{
|
||||||
RawOutput: []byte("provider output"),
|
RawOutput: []byte("provider output"),
|
||||||
@@ -217,13 +263,13 @@ func TestContractCopiesMutableValues(t *testing.T) {
|
|||||||
execution.Debug.RawOutput[0] = 'x'
|
execution.Debug.RawOutput[0] = 'x'
|
||||||
execution.Debug.ValidationDiagnostics[0] = "changed"
|
execution.Debug.ValidationDiagnostics[0] = "changed"
|
||||||
|
|
||||||
if preparationCopy.InputHashes["data_package"] != "input-hash" {
|
if preparationCopy.InputHashes["data_package"] != "input-hash" || preparationCopy.Output.RepairAttempts != 3 {
|
||||||
t.Fatalf("preparation copy = %#v", preparationCopy)
|
t.Fatalf("preparation copy = %#v", preparationCopy)
|
||||||
}
|
}
|
||||||
if debugCopy.RenderedMessages[0].Content != "rendered input" || string(debugCopy.StructuredSchema) != "schema body" || string(debugCopy.ParametersJSON) != `{"temperature":0.2}` {
|
if debugCopy.RenderedMessages[0].Content != "rendered input" || string(debugCopy.StructuredSchema) != "schema body" || string(debugCopy.ParametersJSON) != `{"temperature":0.2}` {
|
||||||
t.Fatalf("preparation debug copy = %#v", debugCopy)
|
t.Fatalf("preparation debug copy = %#v", debugCopy)
|
||||||
}
|
}
|
||||||
if executionCopy.InputHashes["data_package"] != "input-hash" || executionCopy.Validation.Diagnostics[0] != "validation detail" || string(executionCopy.RawOutput) != "generated output" || string(executionCopy.Debug.RawOutput) != "provider output" || executionCopy.Debug.ValidationDiagnostics[0] != "detailed validation" {
|
if executionCopy.InputHashes["data_package"] != "input-hash" || executionCopy.Validation.RepairAttempts != 2 || executionCopy.Validation.Diagnostics[0] != "validation detail" || string(executionCopy.RawOutput) != "generated output" || string(executionCopy.Debug.RawOutput) != "provider output" || executionCopy.Debug.ValidationDiagnostics[0] != "detailed validation" {
|
||||||
t.Fatalf("execution copy = %#v", executionCopy)
|
t.Fatalf("execution copy = %#v", executionCopy)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user