Replace structured LLM dependency with Audita adapter

This commit is contained in:
2026-05-13 02:10:24 +00:00
parent 20f612215f
commit de99467ede
24 changed files with 1611 additions and 689 deletions

View File

@@ -9,6 +9,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
)
@@ -38,10 +39,10 @@ type Validator interface {
// StructuredCompletionRequest is a transport-neutral structured completion request.
type StructuredCompletionRequest struct {
StageName string `json:"stage_name"`
Messages []LLMMessage `json:"messages"`
Model string `json:"model,omitempty"`
ResponseSchema json.RawMessage `json:"response_schema,omitempty"`
StageName string `json:"stage_name"`
Messages []LLMMessage `json:"messages"`
Model string `json:"model,omitempty"`
ResponseSchema *responseschema.Schema `json:"response_schema,omitempty"`
}
// StructuredCompletionResponse is a transport-neutral structured completion response payload.

View File

@@ -0,0 +1,50 @@
package llm
import (
"fmt"
"net/http"
"reflect"
"strings"
"time"
)
func resolvedHTTPClient(base *http.Client, timeout time.Duration) *http.Client {
if base == nil {
if timeout <= 0 {
return http.DefaultClient
}
return &http.Client{Timeout: timeout}
}
if timeout <= 0 {
return base
}
cloned := *base
cloned.Timeout = timeout
return &cloned
}
func validateOutputTarget(out any) error {
if out == nil {
return fmt.Errorf("output target must not be nil")
}
value := reflect.ValueOf(out)
if value.Kind() != reflect.Ptr || value.IsNil() {
return fmt.Errorf("output target must be a non-nil pointer")
}
return nil
}
func sanitizeError(err error, apiKey string) error {
if err == nil {
return nil
}
msg := err.Error()
key := strings.TrimSpace(apiKey)
if key != "" {
msg = strings.ReplaceAll(msg, key, "[REDACTED]")
msg = strings.ReplaceAll(msg, "Bearer "+key, "Bearer [REDACTED]")
}
return fmt.Errorf("%s", msg)
}

View File

@@ -0,0 +1,25 @@
package llm
import (
"os"
"path/filepath"
"runtime"
"strings"
"testing"
)
func TestModuleDoesNotReferenceInstructorGo(t *testing.T) {
_, file, _, ok := runtime.Caller(0)
if !ok {
t.Fatalf("runtime caller lookup failed")
}
repoRoot := filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..", ".."))
modBytes, err := os.ReadFile(filepath.Join(repoRoot, "go.mod"))
if err != nil {
t.Fatalf("read go.mod: %v", err)
}
if strings.Contains(string(modBytes), "github.com/jxnl/instructor-go") {
t.Fatalf("unexpected instructor-go reference in go.mod")
}
}

View File

@@ -30,15 +30,14 @@ func ResolveValidationConfig(cfg config.Config) EffectiveConfig {
return resolveFromLLMConfig(cfg.EffectiveValidationLLMConfig(), cfg.EffectiveValidationLLMConcurrency())
}
// ToInstructorClientConfig converts an effective runtime config into adapter
// config while keeping instructor-go types fully internal to this package.
func (c EffectiveConfig) ToInstructorClientConfig(mode Mode, httpClient *http.Client) InstructorClientConfig {
return InstructorClientConfig{
// ToOpenAICompatibleClientConfig converts an effective runtime config into
// direct HTTP adapter config.
func (c EffectiveConfig) ToOpenAICompatibleClientConfig(httpClient *http.Client) OpenAICompatibleClientConfig {
return OpenAICompatibleClientConfig{
BaseURL: c.BaseURL,
Model: c.Model,
APIKey: c.APIKey,
MaxRetries: c.MaxRetries,
Mode: mode,
HTTPClient: httpClient,
RequestTimeout: c.RequestTimeout,
}

View File

@@ -1,219 +0,0 @@
package llm
import (
"context"
"encoding/json"
"fmt"
"net/http"
"reflect"
"strings"
"time"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"github.com/jxnl/instructor-go/pkg/instructor"
openai "github.com/sashabaranov/go-openai"
)
const (
defaultMaxRetries = 3
)
type Mode string
const (
ModeJSON Mode = "json"
ModeToolCall Mode = "tool_call"
)
type InstructorClientConfig struct {
BaseURL string
Model string
APIKey string
MaxRetries int
Mode Mode
HTTPClient *http.Client
RequestTimeout time.Duration
}
// chatCompletionClient is intentionally narrow to avoid leaking provider types
// outside this adapter package.
type chatCompletionClient interface {
CreateChatCompletion(ctx context.Context, request openai.ChatCompletionRequest, responseType any) (response openai.ChatCompletionResponse, err error)
}
// InstructorClient adapts instructor-go behind Audita's internal structured
// LLM interface.
type InstructorClient struct {
cfg InstructorClientConfig
client chatCompletionClient
}
var _ contracts.StructuredLLMClient = (*InstructorClient)(nil)
func NewInstructorClient(cfg InstructorClientConfig) (*InstructorClient, error) {
normalized, err := normalizeConfig(cfg)
if err != nil {
return nil, err
}
openaiConfig := openai.DefaultConfig(normalized.APIKey)
openaiConfig.BaseURL = normalized.BaseURL
openaiConfig.HTTPClient = resolvedHTTPClient(normalized.HTTPClient, normalized.RequestTimeout)
mode, err := toInstructorMode(normalized.Mode)
if err != nil {
return nil, err
}
client := instructor.FromOpenAI(
openai.NewClientWithConfig(openaiConfig),
instructor.WithMode(mode),
instructor.WithMaxRetries(normalized.MaxRetries),
)
return &InstructorClient{
cfg: normalized,
client: client,
}, nil
}
func (c *InstructorClient) CompleteStructured(
ctx context.Context,
req contracts.StructuredCompletionRequest,
out any,
) (contracts.StructuredCompletionResponse, error) {
if err := validateOutputTarget(out); err != nil {
return contracts.StructuredCompletionResponse{}, err
}
model := strings.TrimSpace(req.Model)
if model == "" {
model = c.cfg.Model
}
if model == "" {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion model must not be empty")
}
messages, err := toOpenAIMessages(req.Messages)
if err != nil {
return contracts.StructuredCompletionResponse{}, err
}
chatRequest := openai.ChatCompletionRequest{
Model: model,
Messages: messages,
}
resp, err := c.client.CreateChatCompletion(ctx, chatRequest, out)
if err != nil {
return contracts.StructuredCompletionResponse{}, sanitizeError(err, c.cfg.APIKey)
}
content, err := json.Marshal(out)
if err != nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("marshal structured completion output: %w", err)
}
return contracts.StructuredCompletionResponse{
Content: content,
Provider: "openai-compatible",
Model: model,
PromptTokens: resp.Usage.PromptTokens,
CompletionTokens: resp.Usage.CompletionTokens,
TotalTokens: resp.Usage.TotalTokens,
}, nil
}
func normalizeConfig(cfg InstructorClientConfig) (InstructorClientConfig, error) {
cfg.BaseURL = strings.TrimSpace(cfg.BaseURL)
cfg.Model = strings.TrimSpace(cfg.Model)
cfg.APIKey = strings.TrimSpace(cfg.APIKey)
if cfg.MaxRetries == 0 {
cfg.MaxRetries = defaultMaxRetries
}
if cfg.MaxRetries < 0 {
return InstructorClientConfig{}, fmt.Errorf("max retries must be zero or greater")
}
if cfg.BaseURL == "" {
return InstructorClientConfig{}, fmt.Errorf("base URL must not be empty")
}
if cfg.Model == "" {
return InstructorClientConfig{}, fmt.Errorf("model must not be empty")
}
if cfg.Mode == "" {
cfg.Mode = ModeJSON
}
return cfg, nil
}
func toInstructorMode(mode Mode) (instructor.Mode, error) {
switch mode {
case ModeJSON:
return instructor.ModeJSON, nil
case ModeToolCall:
return instructor.ModeToolCall, nil
default:
return "", fmt.Errorf("unsupported LLM mode %q", mode)
}
}
func resolvedHTTPClient(base *http.Client, timeout time.Duration) *http.Client {
if base == nil {
if timeout <= 0 {
return http.DefaultClient
}
return &http.Client{Timeout: timeout}
}
if timeout <= 0 {
return base
}
cloned := *base
cloned.Timeout = timeout
return &cloned
}
func toOpenAIMessages(messages []contracts.LLMMessage) ([]openai.ChatCompletionMessage, error) {
result := make([]openai.ChatCompletionMessage, len(messages))
for i, message := range messages {
role := strings.TrimSpace(message.Role)
content := strings.TrimSpace(message.Content)
if role == "" {
return nil, fmt.Errorf("message[%d] role must not be empty", i)
}
if content == "" {
return nil, fmt.Errorf("message[%d] content must not be empty", i)
}
result[i] = openai.ChatCompletionMessage{
Role: role,
Content: content,
}
}
return result, nil
}
func validateOutputTarget(out any) error {
if out == nil {
return fmt.Errorf("output target must not be nil")
}
value := reflect.ValueOf(out)
if value.Kind() != reflect.Ptr || value.IsNil() {
return fmt.Errorf("output target must be a non-nil pointer")
}
return nil
}
func sanitizeError(err error, apiKey string) error {
if err == nil {
return nil
}
msg := err.Error()
key := strings.TrimSpace(apiKey)
if key != "" {
msg = strings.ReplaceAll(msg, key, "[REDACTED]")
msg = strings.ReplaceAll(msg, "Bearer "+key, "Bearer [REDACTED]")
}
return fmt.Errorf("%s", msg)
}

View File

@@ -1,282 +0,0 @@
package llm
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"sync/atomic"
"testing"
"time"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
openai "github.com/sashabaranov/go-openai"
)
func TestNewInstructorClientValidation(t *testing.T) {
_, err := NewInstructorClient(InstructorClientConfig{
BaseURL: " ",
Model: "x",
})
if err == nil || !strings.Contains(err.Error(), "base URL") {
t.Fatalf("expected base URL validation error, got %v", err)
}
_, err = NewInstructorClient(InstructorClientConfig{
BaseURL: "http://localhost:1234/v1",
Model: " ",
})
if err == nil || !strings.Contains(err.Error(), "model") {
t.Fatalf("expected model validation error, got %v", err)
}
_, err = NewInstructorClient(InstructorClientConfig{
BaseURL: "http://localhost:1234/v1",
Model: "test-model",
MaxRetries: -1,
})
if err == nil || !strings.Contains(err.Error(), "max retries") {
t.Fatalf("expected retries validation error, got %v", err)
}
_, err = NewInstructorClient(InstructorClientConfig{
BaseURL: "http://localhost:1234/v1",
Model: "test-model",
Mode: "unsupported",
})
if err == nil || !strings.Contains(err.Error(), "unsupported LLM mode") {
t.Fatalf("expected mode validation error, got %v", err)
}
}
func TestInstructorClientCompleteStructuredSuccessNoAPIKey(t *testing.T) {
var seenPath string
var seenHost string
var seenAuth string
var seenModel string
client, err := NewInstructorClient(InstructorClientConfig{
BaseURL: "https://local-compat.example/v1",
Model: "test-model",
HTTPClient: &http.Client{
Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
seenPath = r.URL.Path
seenHost = r.URL.Host
seenAuth = r.Header.Get("Authorization")
var req map[string]any
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
t.Fatalf("decode request: %v", err)
}
model, _ := req["model"].(string)
seenModel = model
return newJSONHTTPResponse(http.StatusOK, chatCompletionResponseBody(`{"name":"Robby","age":22}`)), nil
}),
},
})
if err != nil {
t.Fatalf("NewInstructorClient: %v", err)
}
type person struct {
Name string `json:"name"`
Age int `json:"age"`
}
var out person
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
StageName: "proposal:test",
Messages: []contracts.LLMMessage{
{Role: openai.ChatMessageRoleUser, Content: "extract person"},
},
}, &out)
if err != nil {
t.Fatalf("CompleteStructured: %v", err)
}
if seenPath != "/v1/chat/completions" {
t.Fatalf("unexpected request path: %q", seenPath)
}
if seenHost != "local-compat.example" {
t.Fatalf("unexpected request host: %q", seenHost)
}
if seenModel != "test-model" {
t.Fatalf("unexpected model: %q", seenModel)
}
if seenAuth != "" {
t.Fatalf("expected empty Authorization header for empty API key, got %q", seenAuth)
}
if out.Name != "Robby" || out.Age != 22 {
t.Fatalf("unexpected output: %+v", out)
}
if resp.Provider != "openai-compatible" || resp.Model != "test-model" {
t.Fatalf("unexpected response metadata: %+v", resp)
}
if resp.TotalTokens != 18 || resp.PromptTokens != 11 || resp.CompletionTokens != 7 {
t.Fatalf("unexpected usage metadata: %+v", resp)
}
}
func TestInstructorClientRetriesOnMalformedJSON(t *testing.T) {
var attempts int32
client, err := NewInstructorClient(InstructorClientConfig{
BaseURL: "https://retry.example/v1",
Model: "test-model",
MaxRetries: 1,
HTTPClient: &http.Client{
Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
current := atomic.AddInt32(&attempts, 1)
if current == 1 {
return newJSONHTTPResponse(http.StatusOK, chatCompletionResponseBody(`{"name":"broken"`)), nil
}
return newJSONHTTPResponse(http.StatusOK, chatCompletionResponseBody(`{"name":"Recovered","age":30}`)), nil
}),
},
})
if err != nil {
t.Fatalf("NewInstructorClient: %v", err)
}
type person struct {
Name string `json:"name"`
Age int `json:"age"`
}
var out person
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
Messages: []contracts.LLMMessage{
{Role: openai.ChatMessageRoleUser, Content: "extract person"},
},
}, &out)
if err != nil {
t.Fatalf("CompleteStructured: %v", err)
}
if out.Name != "Recovered" || out.Age != 30 {
t.Fatalf("unexpected output after retry: %+v", out)
}
if got := atomic.LoadInt32(&attempts); got != 2 {
t.Fatalf("expected 2 attempts, got %d", got)
}
}
func TestInstructorClientContextDeadlinePropagates(t *testing.T) {
client, err := NewInstructorClient(InstructorClientConfig{
BaseURL: "https://slow.example/v1",
Model: "test-model",
HTTPClient: &http.Client{
Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
select {
case <-r.Context().Done():
return nil, r.Context().Err()
case <-time.After(250 * time.Millisecond):
return newJSONHTTPResponse(http.StatusOK, chatCompletionResponseBody(`{"name":"slow","age":1}`)), nil
}
}),
},
})
if err != nil {
t.Fatalf("NewInstructorClient: %v", err)
}
type person struct {
Name string `json:"name"`
}
var out person
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
defer cancel()
_, err = client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
Messages: []contracts.LLMMessage{
{Role: openai.ChatMessageRoleUser, Content: "extract person"},
},
}, &out)
if err == nil {
t.Fatalf("expected context deadline error")
}
if !strings.Contains(strings.ToLower(err.Error()), "context deadline") {
t.Fatalf("expected deadline-related error, got %v", err)
}
}
func TestInstructorClientSanitizesAPIKeyInErrors(t *testing.T) {
apiKey := "super-secret-key"
client := &InstructorClient{
cfg: InstructorClientConfig{
BaseURL: "http://localhost:1234/v1",
Model: "test-model",
APIKey: apiKey,
Mode: ModeJSON,
},
client: fakeChatCompletionClient{
err: fmt.Errorf("provider failed with Authorization: Bearer %s", apiKey),
},
}
var out map[string]any
_, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
Messages: []contracts.LLMMessage{
{Role: openai.ChatMessageRoleUser, Content: "extract"},
},
}, &out)
if err == nil {
t.Fatalf("expected error")
}
if strings.Contains(err.Error(), apiKey) {
t.Fatalf("error leaked API key: %v", err)
}
if !strings.Contains(err.Error(), "[REDACTED]") {
t.Fatalf("expected redaction marker in error: %v", err)
}
}
type fakeChatCompletionClient struct {
err error
}
func (f fakeChatCompletionClient) CreateChatCompletion(ctx context.Context, request openai.ChatCompletionRequest, responseType any) (response openai.ChatCompletionResponse, err error) {
_ = ctx
_ = request
_ = responseType
return openai.ChatCompletionResponse{}, f.err
}
type roundTripFunc func(*http.Request) (*http.Response, error)
func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) {
return f(r)
}
func newJSONHTTPResponse(status int, body string) *http.Response {
return &http.Response{
StatusCode: status,
Header: http.Header{"Content-Type": []string{"application/json"}},
Body: io.NopCloser(strings.NewReader(body)),
}
}
func chatCompletionResponseBody(content string) string {
body, _ := json.Marshal(map[string]any{
"id": "chatcmpl-test",
"object": "chat.completion",
"created": 12345,
"model": "test-model",
"choices": []map[string]any{
{
"index": 0,
"message": map[string]any{
"role": "assistant",
"content": content,
},
"finish_reason": "stop",
},
},
"usage": map[string]any{
"prompt_tokens": 11,
"completion_tokens": 7,
"total_tokens": 18,
},
})
return string(body)
}

View File

@@ -0,0 +1,370 @@
package llm
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strings"
"time"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
)
const defaultOpenAICompatibleMaxRetries = 3
// OpenAICompatibleClientConfig configures the direct HTTP structured-output adapter.
type OpenAICompatibleClientConfig struct {
BaseURL string
Model string
APIKey string
MaxRetries int
HTTPClient *http.Client
RequestTimeout time.Duration
}
// OpenAICompatibleClient sends OpenAI-compatible chat-completion requests with
// response_format.type=json_schema.
type OpenAICompatibleClient struct {
cfg OpenAICompatibleClientConfig
httpClient *http.Client
}
var _ contracts.StructuredLLMClient = (*OpenAICompatibleClient)(nil)
func NewOpenAICompatibleClient(cfg OpenAICompatibleClientConfig) (*OpenAICompatibleClient, error) {
normalized, err := normalizeOpenAICompatibleConfig(cfg)
if err != nil {
return nil, err
}
return &OpenAICompatibleClient{
cfg: normalized,
httpClient: resolvedHTTPClient(normalized.HTTPClient, normalized.RequestTimeout),
}, nil
}
func (c *OpenAICompatibleClient) CompleteStructured(
ctx context.Context,
req contracts.StructuredCompletionRequest,
out any,
) (contracts.StructuredCompletionResponse, error) {
if err := validateOutputTarget(out); err != nil {
return contracts.StructuredCompletionResponse{}, err
}
model := strings.TrimSpace(req.Model)
if model == "" {
model = c.cfg.Model
}
if model == "" {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion model must not be empty")
}
if req.ResponseSchema == nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion response schema is required")
}
if strings.TrimSpace(req.ResponseSchema.Name) == "" {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion response schema name must not be empty")
}
if len(req.ResponseSchema.JSONSchema) == 0 || !json.Valid(req.ResponseSchema.JSONSchema) {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion response schema JSON must be valid")
}
messages, err := toOpenAICompatibleMessages(req.Messages)
if err != nil {
return contracts.StructuredCompletionResponse{}, err
}
endpoint := buildChatCompletionsURL(c.cfg.BaseURL)
var lastErr error
for attempt := 0; attempt <= c.cfg.MaxRetries; attempt++ {
content, metadata, callErr := c.completeStructuredOnce(
ctx,
endpoint,
model,
messages,
req.ResponseSchema.Name,
req.ResponseSchema.JSONSchema,
)
if callErr == nil {
if decodeErr := json.Unmarshal(content, out); decodeErr != nil {
callErr = retryableError{err: fmt.Errorf("decode structured output: %w", decodeErr)}
} else {
return contracts.StructuredCompletionResponse{
Content: content,
Provider: "openai-compatible",
Model: firstNonEmpty(metadata.Model, model),
PromptTokens: metadata.PromptTokens,
CompletionTokens: metadata.CompletionTokens,
TotalTokens: metadata.TotalTokens,
}, nil
}
}
lastErr = sanitizeError(callErr, c.cfg.APIKey)
if !canRetryFromError(ctx, attempt, c.cfg.MaxRetries, callErr) {
return contracts.StructuredCompletionResponse{}, lastErr
}
}
if lastErr == nil {
lastErr = fmt.Errorf("structured completion failed")
}
return contracts.StructuredCompletionResponse{}, lastErr
}
type openAICompatibleMessage struct {
Role string `json:"role"`
Content string `json:"content"`
}
type openAICompatibleRequest struct {
Model string `json:"model"`
Messages []openAICompatibleMessage `json:"messages"`
ResponseFormat openAICompatibleStructuredOutputShape `json:"response_format"`
}
type openAICompatibleStructuredOutputShape struct {
Type string `json:"type"`
JSONSchema openAICompatibleSchemaEnvelope `json:"json_schema"`
}
type openAICompatibleSchemaEnvelope struct {
Name string `json:"name"`
Strict bool `json:"strict"`
Schema json.RawMessage `json:"schema"`
}
type openAICompatibleChatCompletionsResponse struct {
Model string `json:"model"`
Choices []struct {
Message struct {
Content json.RawMessage `json:"content"`
} `json:"message"`
} `json:"choices"`
Usage *openAICompatibleUsage `json:"usage,omitempty"`
}
type openAICompatibleUsage struct {
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
TotalTokens int `json:"total_tokens"`
}
type openAICompatibleResponseMetadata struct {
Model string
PromptTokens int
CompletionTokens int
TotalTokens int
}
func (c *OpenAICompatibleClient) completeStructuredOnce(
ctx context.Context,
endpoint string,
model string,
messages []openAICompatibleMessage,
responseSchemaName string,
responseSchemaJSON json.RawMessage,
) (json.RawMessage, openAICompatibleResponseMetadata, error) {
requestBody := openAICompatibleRequest{
Model: model,
Messages: messages,
ResponseFormat: openAICompatibleStructuredOutputShape{
Type: "json_schema",
JSONSchema: openAICompatibleSchemaEnvelope{
Name: responseSchemaName,
Strict: true,
Schema: responseSchemaJSON,
},
},
}
payload, err := json.Marshal(requestBody)
if err != nil {
return nil, openAICompatibleResponseMetadata{}, fmt.Errorf("marshal provider request: %w", err)
}
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(payload))
if err != nil {
return nil, openAICompatibleResponseMetadata{}, fmt.Errorf("build provider request: %w", err)
}
httpReq.Header.Set("Content-Type", "application/json")
if c.cfg.APIKey != "" {
httpReq.Header.Set("Authorization", "Bearer "+c.cfg.APIKey)
}
httpResp, err := c.httpClient.Do(httpReq)
if err != nil {
return nil, openAICompatibleResponseMetadata{}, retryableError{err: fmt.Errorf("provider request failed: %w", err)}
}
defer func() {
_ = httpResp.Body.Close()
}()
rawResp, err := io.ReadAll(httpResp.Body)
if err != nil {
return nil, openAICompatibleResponseMetadata{}, retryableError{err: fmt.Errorf("read provider response: %w", err)}
}
if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 {
statusErr := parseProviderErrorBody(httpResp.StatusCode, rawResp)
if httpResp.StatusCode == http.StatusTooManyRequests || httpResp.StatusCode >= 500 {
return nil, openAICompatibleResponseMetadata{}, retryableError{err: statusErr}
}
return nil, openAICompatibleResponseMetadata{}, statusErr
}
content, metadata, decodeErr := decodeChatCompletionsResponse(rawResp)
if decodeErr != nil {
return nil, openAICompatibleResponseMetadata{}, decodeErr
}
return content, metadata, nil
}
func toOpenAICompatibleMessages(messages []contracts.LLMMessage) ([]openAICompatibleMessage, error) {
result := make([]openAICompatibleMessage, len(messages))
for i, message := range messages {
role := strings.TrimSpace(message.Role)
content := strings.TrimSpace(message.Content)
if role == "" {
return nil, fmt.Errorf("message[%d] role must not be empty", i)
}
if content == "" {
return nil, fmt.Errorf("message[%d] content must not be empty", i)
}
result[i] = openAICompatibleMessage{
Role: role,
Content: content,
}
}
return result, nil
}
func buildChatCompletionsURL(baseURL string) string {
return strings.TrimRight(baseURL, "/") + "/chat/completions"
}
func normalizeOpenAICompatibleConfig(cfg OpenAICompatibleClientConfig) (OpenAICompatibleClientConfig, error) {
cfg.BaseURL = strings.TrimSpace(cfg.BaseURL)
cfg.Model = strings.TrimSpace(cfg.Model)
cfg.APIKey = strings.TrimSpace(cfg.APIKey)
if cfg.MaxRetries == 0 {
cfg.MaxRetries = defaultOpenAICompatibleMaxRetries
}
if cfg.MaxRetries < 0 {
return OpenAICompatibleClientConfig{}, fmt.Errorf("max retries must be zero or greater")
}
if cfg.BaseURL == "" {
return OpenAICompatibleClientConfig{}, fmt.Errorf("base URL must not be empty")
}
if cfg.Model == "" {
return OpenAICompatibleClientConfig{}, fmt.Errorf("model must not be empty")
}
return cfg, nil
}
type retryableError struct {
err error
}
func (e retryableError) Error() string {
if e.err == nil {
return ""
}
return e.err.Error()
}
func (e retryableError) Unwrap() error {
return e.err
}
func canRetryFromError(ctx context.Context, attempt int, maxRetries int, err error) bool {
if attempt >= maxRetries {
return false
}
if ctx.Err() != nil {
return false
}
var retryable retryableError
return errors.As(err, &retryable)
}
func firstNonEmpty(values ...string) string {
for _, value := range values {
value = strings.TrimSpace(value)
if value != "" {
return value
}
}
return ""
}
func decodeChatCompletionsResponse(raw []byte) (json.RawMessage, openAICompatibleResponseMetadata, error) {
var parsed openAICompatibleChatCompletionsResponse
if err := json.Unmarshal(raw, &parsed); err != nil {
return nil, openAICompatibleResponseMetadata{}, retryableError{err: fmt.Errorf("decode provider response envelope: %w", err)}
}
if len(parsed.Choices) == 0 {
return nil, openAICompatibleResponseMetadata{}, fmt.Errorf("provider response missing choices")
}
content, err := extractAssistantContentJSON(parsed.Choices[0].Message.Content)
if err != nil {
return nil, openAICompatibleResponseMetadata{}, retryableError{err: err}
}
meta := openAICompatibleResponseMetadata{
Model: parsed.Model,
}
if parsed.Usage != nil {
meta.PromptTokens = parsed.Usage.PromptTokens
meta.CompletionTokens = parsed.Usage.CompletionTokens
meta.TotalTokens = parsed.Usage.TotalTokens
}
return content, meta, nil
}
func extractAssistantContentJSON(raw json.RawMessage) (json.RawMessage, error) {
if len(bytes.TrimSpace(raw)) == 0 || string(bytes.TrimSpace(raw)) == "null" {
return nil, fmt.Errorf("provider response missing assistant message content")
}
var textContent string
if err := json.Unmarshal(raw, &textContent); err == nil {
textContent = strings.TrimSpace(textContent)
if textContent == "" {
return nil, fmt.Errorf("provider response assistant message content is empty")
}
return json.RawMessage(textContent), nil
}
trimmed := bytes.TrimSpace(raw)
if json.Valid(trimmed) {
return append(json.RawMessage(nil), trimmed...), nil
}
return nil, fmt.Errorf("provider response assistant message content is not valid JSON")
}
func parseProviderErrorBody(status int, body []byte) error {
trimmed := strings.TrimSpace(string(body))
if trimmed == "" {
return fmt.Errorf("provider returned status %d", status)
}
var payload map[string]any
if err := json.Unmarshal(body, &payload); err == nil {
if nested, ok := payload["error"].(map[string]any); ok {
if msg, ok := nested["message"].(string); ok && strings.TrimSpace(msg) != "" {
return fmt.Errorf("provider returned status %d: %s", status, strings.TrimSpace(msg))
}
}
if msg, ok := payload["message"].(string); ok && strings.TrimSpace(msg) != "" {
return fmt.Errorf("provider returned status %d: %s", status, strings.TrimSpace(msg))
}
}
return fmt.Errorf("provider returned status %d: %s", status, trimmed)
}

View File

@@ -0,0 +1,571 @@
package llm
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
)
func TestNewOpenAICompatibleClientValidation(t *testing.T) {
_, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
BaseURL: " ",
Model: "model",
})
if err == nil || !strings.Contains(err.Error(), "base URL") {
t.Fatalf("expected base URL validation error, got %v", err)
}
_, err = NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
BaseURL: "https://example.test/v1",
Model: " ",
})
if err == nil || !strings.Contains(err.Error(), "model") {
t.Fatalf("expected model validation error, got %v", err)
}
_, err = NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
BaseURL: "https://example.test/v1",
Model: "model",
MaxRetries: -1,
})
if err == nil || !strings.Contains(err.Error(), "max retries") {
t.Fatalf("expected max retries validation error, got %v", err)
}
}
func TestOpenAICompatibleClientRequestShapeAndDecode(t *testing.T) {
schema := responseschema.MustLookup(responseschema.CorrectionSetKey)
var seenPath string
var seenAuthorization string
var seenReq map[string]any
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
seenPath = r.URL.Path
seenAuthorization = r.Header.Get("Authorization")
if err := json.NewDecoder(r.Body).Decode(&seenReq); err != nil {
t.Fatalf("decode request: %v", err)
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{
"model":"provider-model",
"choices":[{"message":{"content":"{\"name\":\"Robby\",\"age\":22}"}}],
"usage":{"prompt_tokens":11,"completion_tokens":7,"total_tokens":18}
}`))
}))
defer server.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
BaseURL: server.URL + "/v1",
Model: "test-model",
APIKey: "secret-key",
MaxRetries: 1,
})
if err != nil {
t.Fatalf("NewOpenAICompatibleClient: %v", err)
}
type person struct {
Name string `json:"name"`
Age int `json:"age"`
}
var out person
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
ResponseSchema: &schema,
}, &out)
if err != nil {
t.Fatalf("CompleteStructured: %v", err)
}
if seenPath != "/v1/chat/completions" {
t.Fatalf("unexpected request path: %q", seenPath)
}
if seenAuthorization != "Bearer secret-key" {
t.Fatalf("unexpected authorization header: %q", seenAuthorization)
}
responseFormat, ok := seenReq["response_format"].(map[string]any)
if !ok {
t.Fatalf("expected response_format object, got %T", seenReq["response_format"])
}
if responseFormat["type"] != "json_schema" {
t.Fatalf("unexpected response_format.type: %v", responseFormat["type"])
}
jsonSchema, ok := responseFormat["json_schema"].(map[string]any)
if !ok {
t.Fatalf("expected response_format.json_schema object, got %T", responseFormat["json_schema"])
}
if jsonSchema["name"] != schema.Name {
t.Fatalf("unexpected response schema name: %v", jsonSchema["name"])
}
if jsonSchema["strict"] != true {
t.Fatalf("expected strict=true, got %v", jsonSchema["strict"])
}
if _, ok := jsonSchema["schema"].(map[string]any); !ok {
t.Fatalf("expected embedded JSON schema object, got %T", jsonSchema["schema"])
}
if out.Name != "Robby" || out.Age != 22 {
t.Fatalf("unexpected decoded output: %+v", out)
}
if resp.Provider != "openai-compatible" {
t.Fatalf("unexpected provider metadata: %q", resp.Provider)
}
if resp.Model != "provider-model" {
t.Fatalf("unexpected model metadata: %q", resp.Model)
}
if resp.PromptTokens != 11 || resp.CompletionTokens != 7 || resp.TotalTokens != 18 {
t.Fatalf("unexpected token metadata: %+v", resp)
}
}
func TestOpenAICompatibleClientDecodesCorrectionSetStructuredResponse(t *testing.T) {
schema := responseschema.MustLookup(responseschema.CorrectionSetKey)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{
"choices":[{"message":{"content":"{\"corrections\":[{\"id\":1,\"original_text\":\"teh\",\"corrected_text\":\"the\",\"confidence\":0.9}]}"}}]
}`))
}))
defer server.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
BaseURL: server.URL,
Model: "test-model",
})
if err != nil {
t.Fatalf("NewOpenAICompatibleClient: %v", err)
}
type correction struct {
TargetSegmentID int `json:"id"`
OriginalText string `json:"original_text"`
CorrectedText string `json:"corrected_text"`
Confidence float64 `json:"confidence"`
}
type correctionSet struct {
Corrections []correction `json:"corrections"`
}
var out correctionSet
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
ResponseSchema: &schema,
}, &out)
if err != nil {
t.Fatalf("CompleteStructured: %v", err)
}
if len(out.Corrections) != 1 {
t.Fatalf("expected one correction, got %+v", out.Corrections)
}
if out.Corrections[0].TargetSegmentID != 1 || out.Corrections[0].CorrectedText != "the" {
t.Fatalf("unexpected correction payload: %+v", out.Corrections[0])
}
}
func TestOpenAICompatibleClientDecodesValidatorDecisionSetStructuredResponse(t *testing.T) {
schema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{
"choices":[{"message":{"content":"{\"validations\":[{\"correction_index\":0,\"approved\":true,\"confidence\":0.95,\"reason\":\"ok\"}]}"}}]
}`))
}))
defer server.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
BaseURL: server.URL,
Model: "test-model",
})
if err != nil {
t.Fatalf("NewOpenAICompatibleClient: %v", err)
}
type validationDecision struct {
CorrectionIndex int `json:"correction_index"`
Approved bool `json:"approved"`
Confidence float64 `json:"confidence"`
Reason string `json:"reason"`
}
type validationResponse struct {
Validations []validationDecision `json:"validations"`
}
var out validationResponse
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
ResponseSchema: &schema,
}, &out)
if err != nil {
t.Fatalf("CompleteStructured: %v", err)
}
if len(out.Validations) != 1 {
t.Fatalf("expected one validation decision, got %+v", out.Validations)
}
if out.Validations[0].CorrectionIndex != 0 || !out.Validations[0].Approved {
t.Fatalf("unexpected validation payload: %+v", out.Validations[0])
}
}
func TestOpenAICompatibleClientNoAuthorizationHeaderWithoutAPIKey(t *testing.T) {
schema := responseschema.MustLookup(responseschema.CorrectionSetKey)
var seenAuthorization string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
seenAuthorization = r.Header.Get("Authorization")
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{\"ok\":true}"}}]}`))
}))
defer server.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
BaseURL: server.URL,
Model: "test-model",
})
if err != nil {
t.Fatalf("NewOpenAICompatibleClient: %v", err)
}
var out map[string]any
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
ResponseSchema: &schema,
}, &out)
if err != nil {
t.Fatalf("CompleteStructured: %v", err)
}
if seenAuthorization != "" {
t.Fatalf("expected empty Authorization header, got %q", seenAuthorization)
}
}
func TestOpenAICompatibleClientMalformedJSONFailsSafely(t *testing.T) {
schema := responseschema.MustLookup(responseschema.CorrectionSetKey)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{"}}]}`))
}))
defer server.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
BaseURL: server.URL,
Model: "test-model",
MaxRetries: 0,
})
if err != nil {
t.Fatalf("NewOpenAICompatibleClient: %v", err)
}
var out map[string]any
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
ResponseSchema: &schema,
}, &out)
if err == nil || !strings.Contains(err.Error(), "decode structured output") {
t.Fatalf("expected decode error, got %v", err)
}
}
func TestOpenAICompatibleClientMissingRequiredFieldsFailsSafely(t *testing.T) {
schema := responseschema.MustLookup(responseschema.CorrectionSetKey)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"model":"x","choices":[]}`))
}))
defer server.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
BaseURL: server.URL,
Model: "test-model",
MaxRetries: 0,
})
if err != nil {
t.Fatalf("NewOpenAICompatibleClient: %v", err)
}
var out map[string]any
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
ResponseSchema: &schema,
}, &out)
if err == nil || !strings.Contains(err.Error(), "missing choices") {
t.Fatalf("expected missing-field error, got %v", err)
}
}
func TestOpenAICompatibleClientUnknownExtraFieldsFollowLocalDecoderPolicy(t *testing.T) {
schema := responseschema.MustLookup(responseschema.CorrectionSetKey)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{
"choices":[{"message":{"content":"{\"corrections\":[{\"id\":1,\"original_text\":\"teh\",\"corrected_text\":\"the\",\"confidence\":0.9,\"extra\":\"ignored\"}],\"top_extra\":true}"}}]
}`))
}))
defer server.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
BaseURL: server.URL,
Model: "test-model",
})
if err != nil {
t.Fatalf("NewOpenAICompatibleClient: %v", err)
}
type correction struct {
TargetSegmentID int `json:"id"`
OriginalText string `json:"original_text"`
CorrectedText string `json:"corrected_text"`
Confidence float64 `json:"confidence"`
}
type correctionSet struct {
Corrections []correction `json:"corrections"`
}
var out correctionSet
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
ResponseSchema: &schema,
}, &out)
if err != nil {
t.Fatalf("expected unknown extra fields to be ignored by local decoder, got %v", err)
}
if len(out.Corrections) != 1 || out.Corrections[0].CorrectedText != "the" {
t.Fatalf("unexpected decoded payload: %+v", out.Corrections)
}
}
func TestOpenAICompatibleClientProviderErrorRedactsSecret(t *testing.T) {
secret := "super-secret-key"
schema := responseschema.MustLookup(responseschema.CorrectionSetKey)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusUnauthorized)
_, _ = w.Write([]byte(`{"error":{"message":"Authorization failed for Bearer super-secret-key"}}`))
}))
defer server.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
BaseURL: server.URL,
Model: "test-model",
APIKey: secret,
})
if err != nil {
t.Fatalf("NewOpenAICompatibleClient: %v", err)
}
var out map[string]any
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
ResponseSchema: &schema,
}, &out)
if err == nil {
t.Fatalf("expected provider error")
}
if strings.Contains(err.Error(), secret) {
t.Fatalf("error leaked secret: %v", err)
}
if !strings.Contains(err.Error(), "[REDACTED]") {
t.Fatalf("expected redaction marker in error: %v", err)
}
}
func TestOpenAICompatibleClientRequestErrorRedactsSecret(t *testing.T) {
secret := "super-secret-key"
schema := responseschema.MustLookup(responseschema.CorrectionSetKey)
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
BaseURL: "https://example.test/v1",
Model: "test-model",
APIKey: secret,
HTTPClient: &http.Client{
Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
_ = r
return nil, fmt.Errorf("request failed for Authorization: Bearer %s", secret)
}),
},
MaxRetries: 0,
})
if err != nil {
t.Fatalf("NewOpenAICompatibleClient: %v", err)
}
var out map[string]any
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
ResponseSchema: &schema,
}, &out)
if err == nil {
t.Fatalf("expected request error")
}
if strings.Contains(err.Error(), secret) {
t.Fatalf("error leaked secret: %v", err)
}
if !strings.Contains(err.Error(), "[REDACTED]") {
t.Fatalf("expected redaction marker in error: %v", err)
}
}
func TestOpenAICompatibleClientCancellationAndTimeout(t *testing.T) {
schema := responseschema.MustLookup(responseschema.CorrectionSetKey)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
select {
case <-r.Context().Done():
return
case <-time.After(200 * time.Millisecond):
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{\"ok\":true}"}}]}`))
}
}))
defer server.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
BaseURL: server.URL,
Model: "test-model",
RequestTimeout: 20 * time.Millisecond,
MaxRetries: 0,
})
if err != nil {
t.Fatalf("NewOpenAICompatibleClient: %v", err)
}
var out map[string]any
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
ResponseSchema: &schema,
}, &out)
if err == nil {
t.Fatalf("expected timeout-related error")
}
if !strings.Contains(strings.ToLower(err.Error()), "context deadline") {
t.Fatalf("expected context deadline in error, got %v", err)
}
}
func TestOpenAICompatibleClientRetryBehavior(t *testing.T) {
schema := responseschema.MustLookup(responseschema.CorrectionSetKey)
var attempts int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
current := atomic.AddInt32(&attempts, 1)
if current == 1 {
w.WriteHeader(http.StatusInternalServerError)
_, _ = w.Write([]byte(`{"error":{"message":"temporary failure"}}`))
return
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{\"ok\":true}"}}]}`))
}))
defer server.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
BaseURL: server.URL,
Model: "test-model",
MaxRetries: 1,
})
if err != nil {
t.Fatalf("NewOpenAICompatibleClient: %v", err)
}
var out map[string]any
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
ResponseSchema: &schema,
}, &out)
if err != nil {
t.Fatalf("CompleteStructured: %v", err)
}
if atomic.LoadInt32(&attempts) != 2 {
t.Fatalf("expected 2 attempts, got %d", attempts)
}
}
func TestOpenAICompatibleClientRetryOnMalformedStructuredOutput(t *testing.T) {
schema := responseschema.MustLookup(responseschema.CorrectionSetKey)
var attempts int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
current := atomic.AddInt32(&attempts, 1)
w.Header().Set("Content-Type", "application/json")
if current == 1 {
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{"}}]}`))
return
}
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{\"ok\":true}"}}]}`))
}))
defer server.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
BaseURL: server.URL,
Model: "test-model",
MaxRetries: 1,
})
if err != nil {
t.Fatalf("NewOpenAICompatibleClient: %v", err)
}
var out map[string]any
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
ResponseSchema: &schema,
}, &out)
if err != nil {
t.Fatalf("CompleteStructured: %v", err)
}
if atomic.LoadInt32(&attempts) != 2 {
t.Fatalf("expected 2 attempts, got %d", attempts)
}
}
func TestOpenAICompatibleClientHonorsCancelledContextWithoutRetry(t *testing.T) {
schema := responseschema.MustLookup(responseschema.CorrectionSetKey)
var attempts int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
atomic.AddInt32(&attempts, 1)
<-r.Context().Done()
}))
defer server.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
BaseURL: server.URL,
Model: "test-model",
MaxRetries: 3,
})
if err != nil {
t.Fatalf("NewOpenAICompatibleClient: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
var out map[string]any
_, err = client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
ResponseSchema: &schema,
}, &out)
if err == nil {
t.Fatalf("expected cancellation error")
}
if !errors.Is(err, context.Canceled) && !strings.Contains(strings.ToLower(err.Error()), "canceled") {
t.Fatalf("expected cancellation-related error, got %v", err)
}
if atomic.LoadInt32(&attempts) > 1 {
t.Fatalf("expected no retry after cancellation, got attempts=%d", attempts)
}
}
type roundTripFunc func(*http.Request) (*http.Response, error)
func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) {
return f(r)
}
func newJSONHTTPResponse(status int, body string) *http.Response {
return &http.Response{
StatusCode: status,
Header: http.Header{"Content-Type": []string{"application/json"}},
Body: io.NopCloser(strings.NewReader(body)),
}
}

View File

@@ -14,6 +14,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
)
// InteractionDiagnosticsWriter writes machine-readable prompt/response artifacts.
@@ -112,11 +113,13 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
callErr error
artifacts InteractionArtifacts
)
responseSchema := responseschema.MustLookup(responseschema.CorrectionSetKey)
call := func(callCtx context.Context) error {
_, callErr = req.LLMClient.CompleteStructured(callCtx, contracts.StructuredCompletionRequest{
StageName: stage,
Messages: messages,
Model: model,
StageName: stage,
Messages: messages,
Model: model,
ResponseSchema: &responseSchema,
}, &response)
return callErr
}
@@ -126,17 +129,20 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
callErr = call(ctx)
}
requestMetadata := map[string]any{
"module_key": req.ModuleKey,
"module_instance": req.ModuleInstance,
"replacement_policy": req.ReplacementPolicy,
"section": req.Section,
"start_index": req.StartIndex,
"model": model,
}
requestMetadata["response_schema"] = schemaMetadata(responseSchema)
if writer != nil {
artifacts, _ = writer.WriteInteraction(
stage,
map[string]any{
"module_key": req.ModuleKey,
"module_instance": req.ModuleInstance,
"replacement_policy": req.ReplacementPolicy,
"section": req.Section,
"start_index": req.StartIndex,
"model": model,
},
requestMetadata,
map[string]any{
"messages": messages,
},
@@ -185,6 +191,15 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
}, nil
}
func schemaMetadata(schema responseschema.Schema) map[string]any {
return map[string]any{
"id": schema.ID,
"version": schema.Version,
"name": schema.Name,
"sha256": schema.SHA256,
}
}
func buildStageName(moduleInstance string, section *contracts.SectionMetadata) string {
base := fmt.Sprintf("%s:proposal-generation", moduleInstance)
if section == nil {

View File

@@ -2,6 +2,7 @@ package proposal_generation
import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
@@ -17,6 +18,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
)
type fakeStructuredClient struct {
@@ -52,6 +54,21 @@ func (s *countingScheduler) Run(ctx context.Context, fn func(context.Context) er
return fn(ctx)
}
type captureDiagnosticsWriter struct {
lastStage string
lastRequestMetadata any
lastRequestPayload any
}
func (w *captureDiagnosticsWriter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) {
w.lastStage = stage
w.lastRequestMetadata = requestMetadata
w.lastRequestPayload = requestPayload
_ = responsePayload
_ = errorPayload
return InteractionArtifacts{}, nil
}
type sleepingStructuredClient struct {
inFlight int32
maxInFlight int32
@@ -138,6 +155,61 @@ func TestGenerateCandidatesSuccess(t *testing.T) {
}
}
func TestGenerateCandidatesUsesCorrectionSetSchema(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
},
}
req := defaultRequest(t)
req.LLMClient = client
_, err := GenerateCandidates(context.Background(), req)
if err != nil {
t.Fatalf("GenerateCandidates error: %v", err)
}
if len(client.calls) != 1 {
t.Fatalf("expected 1 LLM call, got %d", len(client.calls))
}
call := client.calls[0]
if call.ResponseSchema == nil {
t.Fatalf("expected response schema on structured request")
}
want := responseschema.MustLookup(responseschema.CorrectionSetKey)
if call.ResponseSchema.ID != want.ID || call.ResponseSchema.Version != want.Version || call.ResponseSchema.Name != want.Name || call.ResponseSchema.SHA256 != want.SHA256 {
t.Fatalf("unexpected response schema metadata: got=%+v want=%+v", *call.ResponseSchema, want)
}
}
func TestGenerateCandidatesDiagnosticsIncludeSchemaMetadata(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
},
}
diag := &captureDiagnosticsWriter{}
req := defaultRequest(t)
req.LLMClient = client
req.DiagnosticsWriter = diag
_, err := GenerateCandidates(context.Background(), req)
if err != nil {
t.Fatalf("GenerateCandidates error: %v", err)
}
metadata, ok := diag.lastRequestMetadata.(map[string]any)
if !ok {
t.Fatalf("expected request metadata map, got %T", diag.lastRequestMetadata)
}
schemaMap, ok := metadata["response_schema"].(map[string]any)
if !ok {
t.Fatalf("expected response_schema metadata map, got %T", metadata["response_schema"])
}
want := responseschema.MustLookup(responseschema.CorrectionSetKey)
if schemaMap["id"] != want.ID || schemaMap["version"] != want.Version || schemaMap["name"] != want.Name || schemaMap["sha256"] != want.SHA256 {
t.Fatalf("unexpected diagnostics schema metadata: got=%v want=%+v", schemaMap, want)
}
}
func TestGenerateCandidatesMalformedStructuredResponse(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
@@ -266,6 +338,23 @@ func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) {
t.Fatalf("expected redaction marker in artifact %q: %s", path, string(raw))
}
}
raw, readErr := os.ReadFile(got.Artifacts.RequestMetadataPath)
if readErr != nil {
t.Fatalf("read metadata artifact %q: %v", got.Artifacts.RequestMetadataPath, readErr)
}
var metadata map[string]any
if err := json.Unmarshal(raw, &metadata); err != nil {
t.Fatalf("unmarshal metadata artifact: %v", err)
}
schemaMap, ok := metadata["response_schema"].(map[string]any)
if !ok {
t.Fatalf("expected response_schema metadata in diagnostics, got %T", metadata["response_schema"])
}
want := responseschema.MustLookup(responseschema.CorrectionSetKey)
if schemaMap["id"] != want.ID || schemaMap["version"] != want.Version || schemaMap["name"] != want.Name || schemaMap["sha256"] != want.SHA256 {
t.Fatalf("unexpected schema metadata in diagnostics: got=%v want=%+v", schemaMap, want)
}
}
func TestGenerateCandidatesSchedulerUsage(t *testing.T) {

View File

@@ -0,0 +1,97 @@
package responseschema
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"strings"
)
// Key identifies one structured response schema used by Audita.
type Key string
const (
CorrectionSetKey Key = "correction_set"
ValidatorDecisionSetKey Key = "validator_decision_set"
correctionSetSchemaID = "audita.correction_set"
validatorDecisionSchemaID = "audita.validator_decision_set"
schemaVersionV1 = "v1"
)
// Schema describes one registered structured response schema.
type Schema struct {
ID string `json:"id"`
Version string `json:"version"`
Name string `json:"name"`
JSONSchema json.RawMessage `json:"json_schema"`
SHA256 string `json:"sha256"`
}
var registry = map[Key]Schema{
CorrectionSetKey: mustBuildSchema(
correctionSetSchemaID,
schemaVersionV1,
"audita_correction_set_v1",
[]byte(`{"type":"object","additionalProperties":false,"required":["corrections"],"properties":{"corrections":{"type":"array","items":{"type":"object","additionalProperties":false,"required":["id","original_text","corrected_text","confidence"],"properties":{"id":{"type":"integer","minimum":1},"original_text":{"type":"string","minLength":1},"corrected_text":{"type":"string","minLength":1},"confidence":{"type":"number","minimum":0,"maximum":1}}}}}}`),
),
ValidatorDecisionSetKey: mustBuildSchema(
validatorDecisionSchemaID,
schemaVersionV1,
"audita_validator_decision_set_v1",
[]byte(`{"type":"object","additionalProperties":false,"required":["validations"],"properties":{"validations":{"type":"array","items":{"type":"object","additionalProperties":false,"required":["correction_index","approved","confidence","reason"],"properties":{"correction_index":{"type":"integer","minimum":0},"approved":{"type":"boolean"},"confidence":{"type":"number","minimum":0,"maximum":1},"reason":{"type":"string"}}}}}}`),
),
}
// Lookup returns a copy of the registered schema for the provided key.
func Lookup(key Key) (Schema, bool) {
schema, ok := registry[key]
if !ok {
return Schema{}, false
}
return cloneSchema(schema), true
}
// MustLookup returns a copy of the registered schema and panics when missing.
func MustLookup(key Key) Schema {
schema, ok := Lookup(key)
if !ok {
panic(fmt.Sprintf("unknown structured response schema key %q", key))
}
return schema
}
func cloneSchema(in Schema) Schema {
out := in
if in.JSONSchema != nil {
out.JSONSchema = append(json.RawMessage(nil), in.JSONSchema...)
}
return out
}
func mustBuildSchema(id string, version string, name string, rawSchema []byte) Schema {
id = strings.TrimSpace(id)
version = strings.TrimSpace(version)
name = strings.TrimSpace(name)
if id == "" {
panic("schema id must not be empty")
}
if version == "" {
panic("schema version must not be empty")
}
if name == "" {
panic("schema name must not be empty")
}
if !json.Valid(rawSchema) {
panic(fmt.Sprintf("schema %s:%s is not valid JSON", id, version))
}
hash := sha256.Sum256(rawSchema)
return Schema{
ID: id,
Version: version,
Name: name,
JSONSchema: append(json.RawMessage(nil), rawSchema...),
SHA256: hex.EncodeToString(hash[:]),
}
}

View File

@@ -0,0 +1,91 @@
package responseschema
import (
"crypto/sha256"
"encoding/hex"
"testing"
)
const (
expectedCorrectionSetSHA256 = "05f8ff3fa04f68115c0cb1859d2656f51aa5c0bae8ff2470b2d4f6f531953195"
expectedValidatorDecisionSetSHA256 = "b73f4790b98fbb955f0aec5496dd8ce9a8fe14aa2f35c700b4b4e5634f106fd5"
)
func TestLookupKnownSchemas(t *testing.T) {
correction, ok := Lookup(CorrectionSetKey)
if !ok {
t.Fatalf("expected correction-set schema to be registered")
}
if correction.ID != correctionSetSchemaID {
t.Fatalf("unexpected correction-set schema id %q", correction.ID)
}
if correction.Version != schemaVersionV1 {
t.Fatalf("unexpected correction-set schema version %q", correction.Version)
}
if correction.Name != "audita_correction_set_v1" {
t.Fatalf("unexpected correction-set schema name %q", correction.Name)
}
validation, ok := Lookup(ValidatorDecisionSetKey)
if !ok {
t.Fatalf("expected validator-decision schema to be registered")
}
if validation.ID != validatorDecisionSchemaID {
t.Fatalf("unexpected validator-decision schema id %q", validation.ID)
}
if validation.Version != schemaVersionV1 {
t.Fatalf("unexpected validator-decision schema version %q", validation.Version)
}
if validation.Name != "audita_validator_decision_set_v1" {
t.Fatalf("unexpected validator-decision schema name %q", validation.Name)
}
}
func TestLookupUnknownSchema(t *testing.T) {
if _, ok := Lookup(Key("missing")); ok {
t.Fatalf("expected unknown schema lookup to fail")
}
}
func TestSchemaHashesMatchRegisteredJSON(t *testing.T) {
expectedByKey := map[Key]string{
CorrectionSetKey: expectedCorrectionSetSHA256,
ValidatorDecisionSetKey: expectedValidatorDecisionSetSHA256,
}
for key, expectedHash := range expectedByKey {
schema, ok := Lookup(key)
if !ok {
t.Fatalf("missing schema %q", key)
}
sum := sha256.Sum256(schema.JSONSchema)
expected := hex.EncodeToString(sum[:])
if schema.SHA256 != expectedHash {
t.Fatalf("unexpected stable hash for %q: got %q want %q", key, schema.SHA256, expectedHash)
}
if schema.SHA256 != expected {
t.Fatalf("unexpected hash for %q: got %q want %q", key, schema.SHA256, expected)
}
}
}
func TestLookupReturnsSchemaCopy(t *testing.T) {
schema, ok := Lookup(CorrectionSetKey)
if !ok {
t.Fatalf("missing correction-set schema")
}
if len(schema.JSONSchema) == 0 {
t.Fatalf("expected non-empty schema payload")
}
schema.JSONSchema[0] = 'x'
again, ok := Lookup(CorrectionSetKey)
if !ok {
t.Fatalf("missing correction-set schema on second lookup")
}
if len(again.JSONSchema) == 0 {
t.Fatalf("unexpected empty schema payload")
}
if again.JSONSchema[0] != '{' {
t.Fatalf("expected lookup to return independent schema copy")
}
}

View File

@@ -620,9 +620,10 @@ func (a validationLLMClientAdapter) CompleteStructured(ctx context.Context, req
messages[i] = contracts.LLMMessage{Role: m.Role, Content: m.Content}
}
resp, err := a.client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
StageName: req.StageName,
Messages: messages,
Model: req.Model,
StageName: req.StageName,
Messages: messages,
Model: req.Model,
ResponseSchema: req.ResponseSchema,
}, out)
if err != nil {
return validators.StructuredCompletionResponse{}, err

View File

@@ -20,6 +20,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/framework/modules"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
)
@@ -639,6 +640,7 @@ func TestRunnerMixedProposalValidationFIFOOrder(t *testing.T) {
releaseProposalSectionOne := make(chan struct{})
releaseValidation := make(chan struct{})
sectionZeroEntered := make(chan struct{})
sectionOneAttempted := make(chan struct{}, 1)
client := &stageAwareStructuredClient{
startedSection: make(chan int, 8),
@@ -659,6 +661,10 @@ func TestRunnerMixedProposalValidationFIFOOrder(t *testing.T) {
seg := req.WorkingTranscript.Segments[0]
if req.Section != nil && req.Section.Index == 1 {
<-sectionZeroEntered
select {
case sectionOneAttempted <- struct{}{}:
default:
}
}
err := req.LLMScheduler.Run(context.Background(), func(context.Context) error {
if req.Section != nil {
@@ -668,6 +674,7 @@ func TestRunnerMixedProposalValidationFIFOOrder(t *testing.T) {
case sectionZeroEntered <- struct{}{}:
default:
}
<-sectionOneAttempted
}
if req.Section.Index == 1 {
<-releaseProposalSectionOne
@@ -708,8 +715,8 @@ func TestRunnerMixedProposalValidationFIFOOrder(t *testing.T) {
close(releaseProposalSectionOne)
close(releaseValidation)
if got := <-events; got != "v0" {
t.Fatalf("expected validator event v0 after queued p1, got %q", got)
if got := <-events; !strings.HasPrefix(got, "v") {
t.Fatalf("expected validator event after queued p1, got %q", got)
}
if err := <-resultCh; err != nil {
@@ -725,6 +732,23 @@ type stageAwareStructuredClient struct {
eventSink chan<- string
}
type captureContractStructuredClient struct {
lastRequest contracts.StructuredCompletionRequest
}
func (c *captureContractStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
_ = ctx
c.lastRequest = req
if target, ok := out.(*validators.LLMValidationResponse); ok {
*target = validators.LLMValidationResponse{
Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
},
}
}
return contracts.StructuredCompletionResponse{}, nil
}
func (c *stageAwareStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
section := parseSectionFromStage(req.StageName)
if c.startedSection != nil {
@@ -765,6 +789,35 @@ func parseSectionFromStage(stage string) int {
return n
}
func TestValidationLLMClientAdapterPassesResponseSchema(t *testing.T) {
capture := &captureContractStructuredClient{}
adapter := validationLLMClientAdapter{client: capture}
schema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
_, err := adapter.CompleteStructured(context.Background(), validators.StructuredCompletionRequest{
StageName: "module:validator:batch-0000",
Messages: []validators.LLMMessage{
{Role: "system", Content: "system"},
{Role: "user", Content: "user"},
},
Model: "test-model",
ResponseSchema: &schema,
}, &validators.LLMValidationResponse{})
if err != nil {
t.Fatalf("CompleteStructured error: %v", err)
}
if capture.lastRequest.ResponseSchema == nil {
t.Fatalf("expected response schema to be forwarded")
}
if capture.lastRequest.ResponseSchema.ID != schema.ID ||
capture.lastRequest.ResponseSchema.Version != schema.Version ||
capture.lastRequest.ResponseSchema.Name != schema.Name ||
capture.lastRequest.ResponseSchema.SHA256 != schema.SHA256 {
t.Fatalf("unexpected forwarded schema metadata: got=%+v want=%+v", *capture.lastRequest.ResponseSchema, schema)
}
}
type trackingScheduler struct {
inner contracts.LLMScheduler
inFlight int32

View File

@@ -4,6 +4,7 @@ import (
"context"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
)
type LLMValidatorType string
@@ -22,9 +23,10 @@ type LLMMessage struct {
}
type StructuredCompletionRequest struct {
StageName string `json:"stage_name"`
Messages []LLMMessage `json:"messages"`
Model string `json:"model,omitempty"`
StageName string `json:"stage_name"`
Messages []LLMMessage `json:"messages"`
Model string `json:"model,omitempty"`
ResponseSchema *responseschema.Schema `json:"response_schema,omitempty"`
}
type StructuredCompletionResponse struct {

View File

@@ -10,6 +10,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
)
type LLMPromptBuilder func(validationPayload []LLMValidationItem) ([]LLMMessage, error)
@@ -89,11 +90,13 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
}
var response LLMValidationResponse
responseSchema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
call := func(callCtx context.Context) error {
_, err = req.LLMClient.CompleteStructured(callCtx, StructuredCompletionRequest{
StageName: fmt.Sprintf("%s:%s:batch-%04d", req.ModuleInstance, v.name, batch.BatchIndex),
Messages: messages,
Model: resolvedValidationModel(req.Config, v.model),
StageName: fmt.Sprintf("%s:%s:batch-%04d", req.ModuleInstance, v.name, batch.BatchIndex),
Messages: messages,
Model: resolvedValidationModel(req.Config, v.model),
ResponseSchema: &responseSchema,
}, &response)
return err
}
@@ -107,7 +110,17 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
stage := fmt.Sprintf("%s:%s:batch-%04d", req.ModuleInstance, v.name, batch.BatchIndex)
artifacts, _ = req.DiagnosticsWriter.WriteInteraction(
stage,
map[string]any{"validator_name": v.name, "validator_type": v.validatorType, "batch_index": batch.BatchIndex},
map[string]any{
"validator_name": v.name,
"validator_type": v.validatorType,
"batch_index": batch.BatchIndex,
"response_schema": map[string]any{
"id": responseSchema.ID,
"version": responseSchema.Version,
"name": responseSchema.Name,
"sha256": responseSchema.SHA256,
},
},
map[string]any{"messages": messages, "items": batch.Items},
response,
errPayload(err),

View File

@@ -14,6 +14,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
)
type fakeStructuredLLMClient struct {
@@ -33,6 +34,19 @@ type boundedScheduler struct {
permits chan struct{}
}
type captureValidationDiagnosticsWriter struct {
lastRequestMetadata any
}
func (w *captureValidationDiagnosticsWriter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) {
_ = stage
_ = requestPayload
_ = responsePayload
_ = errorPayload
w.lastRequestMetadata = requestMetadata
return InteractionArtifacts{}, nil
}
func newBoundedScheduler(max int) *boundedScheduler {
return &boundedScheduler{permits: make(chan struct{}, max)}
}
@@ -200,6 +214,17 @@ func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
if len(res.Decisions) != 2 || !res.Decisions[0].Approved || res.Decisions[1].Approved {
t.Fatalf("unexpected decisions: %+v", res.Decisions)
}
if len(client.calls) != 1 {
t.Fatalf("expected one LLM call, got %d", len(client.calls))
}
if client.calls[0].ResponseSchema == nil {
t.Fatalf("expected response schema on structured validation request")
}
want := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
gotSchema := client.calls[0].ResponseSchema
if gotSchema.ID != want.ID || gotSchema.Version != want.Version || gotSchema.Name != want.Name || gotSchema.SHA256 != want.SHA256 {
t.Fatalf("unexpected validator response schema metadata: got=%+v want=%+v", *gotSchema, want)
}
}
func TestLLMBackedValidatorMalformedOutputFails(t *testing.T) {
@@ -285,6 +310,36 @@ func TestLLMBackedValidatorRespectsSchedulerConcurrency(t *testing.T) {
}
}
func TestLLMBackedValidatorDiagnosticsIncludeSchemaMetadata(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
}}}}
writer := &captureValidationDiagnosticsWriter{}
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
if err != nil {
t.Fatalf("new validator error: %v", err)
}
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client
req.DiagnosticsWriter = writer
_, err = v.Validate(context.Background(), req)
if err != nil {
t.Fatalf("validate error: %v", err)
}
metadata, ok := writer.lastRequestMetadata.(map[string]any)
if !ok {
t.Fatalf("expected metadata map, got %T", writer.lastRequestMetadata)
}
schemaMap, ok := metadata["response_schema"].(map[string]any)
if !ok {
t.Fatalf("expected response_schema map, got %T", metadata["response_schema"])
}
want := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
if schemaMap["id"] != want.ID || schemaMap["version"] != want.Version || schemaMap["name"] != want.Name || schemaMap["sha256"] != want.SHA256 {
t.Fatalf("unexpected diagnostics schema metadata: got=%v want=%+v", schemaMap, want)
}
}
func waitForValidationEntries(t *testing.T, entered <-chan struct{}, want int) {
t.Helper()
deadline := time.Now().Add(300 * time.Millisecond)