eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add retryable normalization fallbacks

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-26 01:20:28 +00:00
parent 26bd59a5a2
commit 8a12c56971
9 changed files with 306 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
internal/framework/llm/scriptorium_client.go
View File

@@ -113,17 +113,17 @@ func (c *ScriptoriumClient) CompleteStructured(ctx context.Context, req contract
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run Scriptorium prompt %q: %w", promptID, redactScriptoriumError(err))
}
if result == nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run Scriptorium prompt %q: empty result", promptID)
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run Scriptorium prompt %q: %w: empty result", promptID, contracts.ErrInvalidStructuredOutput)
}
response := c.responseFromResult(result, prepared)
if result.Validation.Status == scriptorium.ValidationFailed || !result.Validation.IsValid {
return response, fmt.Errorf("run Scriptorium prompt %q: validation failed: %s", promptID, strings.Join(result.Validation.Errors, "; "))
return response, fmt.Errorf("run Scriptorium prompt %q: %w: validation failed: %s", promptID, contracts.ErrInvalidStructuredOutput, strings.Join(result.Validation.Errors, "; "))
}
if len(strings.TrimSpace(string(response.Content))) == 0 {
return response, fmt.Errorf("run Scriptorium prompt %q: empty structured output", promptID)
return response, fmt.Errorf("run Scriptorium prompt %q: %w: empty structured output", promptID, contracts.ErrInvalidStructuredOutput)
}
if err := json.Unmarshal(response.Content, out); err != nil {
return response, fmt.Errorf("decode Scriptorium structured output for prompt %q: %w", promptID, err)
return response, fmt.Errorf("decode Scriptorium structured output for prompt %q: %w: %w", promptID, contracts.ErrInvalidStructuredOutput, err)
}
return response, nil
}

30
internal/framework/llm/scriptorium_client_test.go
View File

@@ -113,7 +113,7 @@ func TestScriptoriumClientValidationFailureReturnsError(t *testing.T) {
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
},
}, &out)
if err == nil || !strings.Contains(err.Error(), "validation failed") {
if err == nil || !errors.Is(err, contracts.ErrInvalidStructuredOutput) || !strings.Contains(err.Error(), "validation failed") {
t.Fatalf("CompleteStructured() error = %v, want validation failure", err)
}
if got := string(resp.Content); got != `{"bad":true}` {
@@ -138,7 +138,7 @@ func TestScriptoriumClientDecodeFailureReturnsRawResponse(t *testing.T) {
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
},
}, &out)
if err == nil || !strings.Contains(err.Error(), "decode Scriptorium structured output") {
if err == nil || !errors.Is(err, contracts.ErrInvalidStructuredOutput) || !strings.Contains(err.Error(), "decode Scriptorium structured output") {
t.Fatalf("CompleteStructured() error = %v, want decode failure", err)
}
if got := string(resp.Content); got != `{"ok":true}` {
@@ -163,6 +163,9 @@ func TestScriptoriumClientProviderFailureIncludesContextAndRedactsBearerToken(t
if err == nil {
t.Fatalf("CompleteStructured() error = nil, want provider error")
}
if errors.Is(err, contracts.ErrInvalidStructuredOutput) {
t.Fatalf("provider error = %v, must not be classified as invalid structured output", err)
}
if !strings.Contains(err.Error(), `run Scriptorium prompt "adapter.test"`) {
t.Fatalf("error = %q, want operation context", err.Error())
}
@@ -186,11 +189,27 @@ func TestScriptoriumClientContextCancellationIsRespected(t *testing.T) {
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
},
}, &out)
if !errors.Is(err, context.Canceled) {
if !errors.Is(err, context.Canceled) || errors.Is(err, contracts.ErrInvalidStructuredOutput) {
t.Fatalf("CompleteStructured() error = %v, want context canceled", err)
}
}
func TestScriptoriumClientClassifiesEmptyStructuredCompletion(t *testing.T) {
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{allowEmpty: true})
var out map[string]any
_, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
PromptID: "adapter.test",
SessionID: "session-123",
Inputs: contracts.LLMInputSet{
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
},
}, &out)
if !errors.Is(err, contracts.ErrInvalidStructuredOutput) {
t.Fatalf("CompleteStructured() error = %v, want invalid structured output", err)
}
}
func TestScheduledScriptoriumClientBoundsConcurrentCalls(t *testing.T) {
fake := &fakeScriptoriumLLM{
content: `{"ok":true}`,
@@ -296,6 +315,7 @@ output:
type fakeScriptoriumLLM struct {
content string
allowEmpty bool
err error
block chan struct{}
mu sync.Mutex
@@ -329,10 +349,10 @@ func (f *fakeScriptoriumLLM) Generate(ctx context.Context, req scriptorium.Gener
return nil, f.err
}
content := f.content
if content == "" {
if content == "" && !f.allowEmpty {
content = `{"ok":true}`
}
if !json.Valid([]byte(content)) {
if !f.allowEmpty && !json.Valid([]byte(content)) {
return nil, errors.New("test fake must return JSON content")
}
return &scriptorium.GenerateResponse{