Replace structured LLM dependency with Audita adapter
This commit is contained in:
@@ -9,6 +9,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||
)
|
||||
|
||||
@@ -38,10 +39,10 @@ type Validator interface {
|
||||
|
||||
// StructuredCompletionRequest is a transport-neutral structured completion request.
|
||||
type StructuredCompletionRequest struct {
|
||||
StageName string `json:"stage_name"`
|
||||
Messages []LLMMessage `json:"messages"`
|
||||
Model string `json:"model,omitempty"`
|
||||
ResponseSchema json.RawMessage `json:"response_schema,omitempty"`
|
||||
StageName string `json:"stage_name"`
|
||||
Messages []LLMMessage `json:"messages"`
|
||||
Model string `json:"model,omitempty"`
|
||||
ResponseSchema *responseschema.Schema `json:"response_schema,omitempty"`
|
||||
}
|
||||
|
||||
// StructuredCompletionResponse is a transport-neutral structured completion response payload.
|
||||
|
||||
50
internal/framework/llm/client_common.go
Normal file
50
internal/framework/llm/client_common.go
Normal file
@@ -0,0 +1,50 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func resolvedHTTPClient(base *http.Client, timeout time.Duration) *http.Client {
|
||||
if base == nil {
|
||||
if timeout <= 0 {
|
||||
return http.DefaultClient
|
||||
}
|
||||
return &http.Client{Timeout: timeout}
|
||||
}
|
||||
|
||||
if timeout <= 0 {
|
||||
return base
|
||||
}
|
||||
|
||||
cloned := *base
|
||||
cloned.Timeout = timeout
|
||||
return &cloned
|
||||
}
|
||||
|
||||
func validateOutputTarget(out any) error {
|
||||
if out == nil {
|
||||
return fmt.Errorf("output target must not be nil")
|
||||
}
|
||||
value := reflect.ValueOf(out)
|
||||
if value.Kind() != reflect.Ptr || value.IsNil() {
|
||||
return fmt.Errorf("output target must be a non-nil pointer")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sanitizeError(err error, apiKey string) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
msg := err.Error()
|
||||
key := strings.TrimSpace(apiKey)
|
||||
if key != "" {
|
||||
msg = strings.ReplaceAll(msg, key, "[REDACTED]")
|
||||
msg = strings.ReplaceAll(msg, "Bearer "+key, "Bearer [REDACTED]")
|
||||
}
|
||||
return fmt.Errorf("%s", msg)
|
||||
}
|
||||
25
internal/framework/llm/dependency_guard_test.go
Normal file
25
internal/framework/llm/dependency_guard_test.go
Normal file
@@ -0,0 +1,25 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestModuleDoesNotReferenceInstructorGo(t *testing.T) {
|
||||
_, file, _, ok := runtime.Caller(0)
|
||||
if !ok {
|
||||
t.Fatalf("runtime caller lookup failed")
|
||||
}
|
||||
repoRoot := filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..", ".."))
|
||||
|
||||
modBytes, err := os.ReadFile(filepath.Join(repoRoot, "go.mod"))
|
||||
if err != nil {
|
||||
t.Fatalf("read go.mod: %v", err)
|
||||
}
|
||||
if strings.Contains(string(modBytes), "github.com/jxnl/instructor-go") {
|
||||
t.Fatalf("unexpected instructor-go reference in go.mod")
|
||||
}
|
||||
}
|
||||
@@ -30,15 +30,14 @@ func ResolveValidationConfig(cfg config.Config) EffectiveConfig {
|
||||
return resolveFromLLMConfig(cfg.EffectiveValidationLLMConfig(), cfg.EffectiveValidationLLMConcurrency())
|
||||
}
|
||||
|
||||
// ToInstructorClientConfig converts an effective runtime config into adapter
|
||||
// config while keeping instructor-go types fully internal to this package.
|
||||
func (c EffectiveConfig) ToInstructorClientConfig(mode Mode, httpClient *http.Client) InstructorClientConfig {
|
||||
return InstructorClientConfig{
|
||||
// ToOpenAICompatibleClientConfig converts an effective runtime config into
|
||||
// direct HTTP adapter config.
|
||||
func (c EffectiveConfig) ToOpenAICompatibleClientConfig(httpClient *http.Client) OpenAICompatibleClientConfig {
|
||||
return OpenAICompatibleClientConfig{
|
||||
BaseURL: c.BaseURL,
|
||||
Model: c.Model,
|
||||
APIKey: c.APIKey,
|
||||
MaxRetries: c.MaxRetries,
|
||||
Mode: mode,
|
||||
HTTPClient: httpClient,
|
||||
RequestTimeout: c.RequestTimeout,
|
||||
}
|
||||
|
||||
@@ -1,219 +0,0 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
|
||||
"github.com/jxnl/instructor-go/pkg/instructor"
|
||||
openai "github.com/sashabaranov/go-openai"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultMaxRetries = 3
|
||||
)
|
||||
|
||||
type Mode string
|
||||
|
||||
const (
|
||||
ModeJSON Mode = "json"
|
||||
ModeToolCall Mode = "tool_call"
|
||||
)
|
||||
|
||||
type InstructorClientConfig struct {
|
||||
BaseURL string
|
||||
Model string
|
||||
APIKey string
|
||||
MaxRetries int
|
||||
Mode Mode
|
||||
HTTPClient *http.Client
|
||||
RequestTimeout time.Duration
|
||||
}
|
||||
|
||||
// chatCompletionClient is intentionally narrow to avoid leaking provider types
|
||||
// outside this adapter package.
|
||||
type chatCompletionClient interface {
|
||||
CreateChatCompletion(ctx context.Context, request openai.ChatCompletionRequest, responseType any) (response openai.ChatCompletionResponse, err error)
|
||||
}
|
||||
|
||||
// InstructorClient adapts instructor-go behind Audita's internal structured
|
||||
// LLM interface.
|
||||
type InstructorClient struct {
|
||||
cfg InstructorClientConfig
|
||||
client chatCompletionClient
|
||||
}
|
||||
|
||||
var _ contracts.StructuredLLMClient = (*InstructorClient)(nil)
|
||||
|
||||
func NewInstructorClient(cfg InstructorClientConfig) (*InstructorClient, error) {
|
||||
normalized, err := normalizeConfig(cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
openaiConfig := openai.DefaultConfig(normalized.APIKey)
|
||||
openaiConfig.BaseURL = normalized.BaseURL
|
||||
openaiConfig.HTTPClient = resolvedHTTPClient(normalized.HTTPClient, normalized.RequestTimeout)
|
||||
|
||||
mode, err := toInstructorMode(normalized.Mode)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
client := instructor.FromOpenAI(
|
||||
openai.NewClientWithConfig(openaiConfig),
|
||||
instructor.WithMode(mode),
|
||||
instructor.WithMaxRetries(normalized.MaxRetries),
|
||||
)
|
||||
|
||||
return &InstructorClient{
|
||||
cfg: normalized,
|
||||
client: client,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *InstructorClient) CompleteStructured(
|
||||
ctx context.Context,
|
||||
req contracts.StructuredCompletionRequest,
|
||||
out any,
|
||||
) (contracts.StructuredCompletionResponse, error) {
|
||||
if err := validateOutputTarget(out); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
|
||||
model := strings.TrimSpace(req.Model)
|
||||
if model == "" {
|
||||
model = c.cfg.Model
|
||||
}
|
||||
if model == "" {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion model must not be empty")
|
||||
}
|
||||
|
||||
messages, err := toOpenAIMessages(req.Messages)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
|
||||
chatRequest := openai.ChatCompletionRequest{
|
||||
Model: model,
|
||||
Messages: messages,
|
||||
}
|
||||
|
||||
resp, err := c.client.CreateChatCompletion(ctx, chatRequest, out)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, sanitizeError(err, c.cfg.APIKey)
|
||||
}
|
||||
|
||||
content, err := json.Marshal(out)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("marshal structured completion output: %w", err)
|
||||
}
|
||||
|
||||
return contracts.StructuredCompletionResponse{
|
||||
Content: content,
|
||||
Provider: "openai-compatible",
|
||||
Model: model,
|
||||
PromptTokens: resp.Usage.PromptTokens,
|
||||
CompletionTokens: resp.Usage.CompletionTokens,
|
||||
TotalTokens: resp.Usage.TotalTokens,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func normalizeConfig(cfg InstructorClientConfig) (InstructorClientConfig, error) {
|
||||
cfg.BaseURL = strings.TrimSpace(cfg.BaseURL)
|
||||
cfg.Model = strings.TrimSpace(cfg.Model)
|
||||
cfg.APIKey = strings.TrimSpace(cfg.APIKey)
|
||||
if cfg.MaxRetries == 0 {
|
||||
cfg.MaxRetries = defaultMaxRetries
|
||||
}
|
||||
if cfg.MaxRetries < 0 {
|
||||
return InstructorClientConfig{}, fmt.Errorf("max retries must be zero or greater")
|
||||
}
|
||||
if cfg.BaseURL == "" {
|
||||
return InstructorClientConfig{}, fmt.Errorf("base URL must not be empty")
|
||||
}
|
||||
if cfg.Model == "" {
|
||||
return InstructorClientConfig{}, fmt.Errorf("model must not be empty")
|
||||
}
|
||||
if cfg.Mode == "" {
|
||||
cfg.Mode = ModeJSON
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func toInstructorMode(mode Mode) (instructor.Mode, error) {
|
||||
switch mode {
|
||||
case ModeJSON:
|
||||
return instructor.ModeJSON, nil
|
||||
case ModeToolCall:
|
||||
return instructor.ModeToolCall, nil
|
||||
default:
|
||||
return "", fmt.Errorf("unsupported LLM mode %q", mode)
|
||||
}
|
||||
}
|
||||
|
||||
func resolvedHTTPClient(base *http.Client, timeout time.Duration) *http.Client {
|
||||
if base == nil {
|
||||
if timeout <= 0 {
|
||||
return http.DefaultClient
|
||||
}
|
||||
return &http.Client{Timeout: timeout}
|
||||
}
|
||||
|
||||
if timeout <= 0 {
|
||||
return base
|
||||
}
|
||||
|
||||
cloned := *base
|
||||
cloned.Timeout = timeout
|
||||
return &cloned
|
||||
}
|
||||
|
||||
func toOpenAIMessages(messages []contracts.LLMMessage) ([]openai.ChatCompletionMessage, error) {
|
||||
result := make([]openai.ChatCompletionMessage, len(messages))
|
||||
for i, message := range messages {
|
||||
role := strings.TrimSpace(message.Role)
|
||||
content := strings.TrimSpace(message.Content)
|
||||
if role == "" {
|
||||
return nil, fmt.Errorf("message[%d] role must not be empty", i)
|
||||
}
|
||||
if content == "" {
|
||||
return nil, fmt.Errorf("message[%d] content must not be empty", i)
|
||||
}
|
||||
result[i] = openai.ChatCompletionMessage{
|
||||
Role: role,
|
||||
Content: content,
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func validateOutputTarget(out any) error {
|
||||
if out == nil {
|
||||
return fmt.Errorf("output target must not be nil")
|
||||
}
|
||||
value := reflect.ValueOf(out)
|
||||
if value.Kind() != reflect.Ptr || value.IsNil() {
|
||||
return fmt.Errorf("output target must be a non-nil pointer")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sanitizeError(err error, apiKey string) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
msg := err.Error()
|
||||
key := strings.TrimSpace(apiKey)
|
||||
if key != "" {
|
||||
msg = strings.ReplaceAll(msg, key, "[REDACTED]")
|
||||
msg = strings.ReplaceAll(msg, "Bearer "+key, "Bearer [REDACTED]")
|
||||
}
|
||||
return fmt.Errorf("%s", msg)
|
||||
}
|
||||
@@ -1,282 +0,0 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
|
||||
openai "github.com/sashabaranov/go-openai"
|
||||
)
|
||||
|
||||
func TestNewInstructorClientValidation(t *testing.T) {
|
||||
_, err := NewInstructorClient(InstructorClientConfig{
|
||||
BaseURL: " ",
|
||||
Model: "x",
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "base URL") {
|
||||
t.Fatalf("expected base URL validation error, got %v", err)
|
||||
}
|
||||
|
||||
_, err = NewInstructorClient(InstructorClientConfig{
|
||||
BaseURL: "http://localhost:1234/v1",
|
||||
Model: " ",
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "model") {
|
||||
t.Fatalf("expected model validation error, got %v", err)
|
||||
}
|
||||
|
||||
_, err = NewInstructorClient(InstructorClientConfig{
|
||||
BaseURL: "http://localhost:1234/v1",
|
||||
Model: "test-model",
|
||||
MaxRetries: -1,
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "max retries") {
|
||||
t.Fatalf("expected retries validation error, got %v", err)
|
||||
}
|
||||
|
||||
_, err = NewInstructorClient(InstructorClientConfig{
|
||||
BaseURL: "http://localhost:1234/v1",
|
||||
Model: "test-model",
|
||||
Mode: "unsupported",
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "unsupported LLM mode") {
|
||||
t.Fatalf("expected mode validation error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInstructorClientCompleteStructuredSuccessNoAPIKey(t *testing.T) {
|
||||
var seenPath string
|
||||
var seenHost string
|
||||
var seenAuth string
|
||||
var seenModel string
|
||||
client, err := NewInstructorClient(InstructorClientConfig{
|
||||
BaseURL: "https://local-compat.example/v1",
|
||||
Model: "test-model",
|
||||
HTTPClient: &http.Client{
|
||||
Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
|
||||
seenPath = r.URL.Path
|
||||
seenHost = r.URL.Host
|
||||
seenAuth = r.Header.Get("Authorization")
|
||||
|
||||
var req map[string]any
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
t.Fatalf("decode request: %v", err)
|
||||
}
|
||||
model, _ := req["model"].(string)
|
||||
seenModel = model
|
||||
|
||||
return newJSONHTTPResponse(http.StatusOK, chatCompletionResponseBody(`{"name":"Robby","age":22}`)), nil
|
||||
}),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewInstructorClient: %v", err)
|
||||
}
|
||||
|
||||
type person struct {
|
||||
Name string `json:"name"`
|
||||
Age int `json:"age"`
|
||||
}
|
||||
var out person
|
||||
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
StageName: "proposal:test",
|
||||
Messages: []contracts.LLMMessage{
|
||||
{Role: openai.ChatMessageRoleUser, Content: "extract person"},
|
||||
},
|
||||
}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("CompleteStructured: %v", err)
|
||||
}
|
||||
|
||||
if seenPath != "/v1/chat/completions" {
|
||||
t.Fatalf("unexpected request path: %q", seenPath)
|
||||
}
|
||||
if seenHost != "local-compat.example" {
|
||||
t.Fatalf("unexpected request host: %q", seenHost)
|
||||
}
|
||||
if seenModel != "test-model" {
|
||||
t.Fatalf("unexpected model: %q", seenModel)
|
||||
}
|
||||
if seenAuth != "" {
|
||||
t.Fatalf("expected empty Authorization header for empty API key, got %q", seenAuth)
|
||||
}
|
||||
if out.Name != "Robby" || out.Age != 22 {
|
||||
t.Fatalf("unexpected output: %+v", out)
|
||||
}
|
||||
if resp.Provider != "openai-compatible" || resp.Model != "test-model" {
|
||||
t.Fatalf("unexpected response metadata: %+v", resp)
|
||||
}
|
||||
if resp.TotalTokens != 18 || resp.PromptTokens != 11 || resp.CompletionTokens != 7 {
|
||||
t.Fatalf("unexpected usage metadata: %+v", resp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInstructorClientRetriesOnMalformedJSON(t *testing.T) {
|
||||
var attempts int32
|
||||
client, err := NewInstructorClient(InstructorClientConfig{
|
||||
BaseURL: "https://retry.example/v1",
|
||||
Model: "test-model",
|
||||
MaxRetries: 1,
|
||||
HTTPClient: &http.Client{
|
||||
Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
|
||||
current := atomic.AddInt32(&attempts, 1)
|
||||
if current == 1 {
|
||||
return newJSONHTTPResponse(http.StatusOK, chatCompletionResponseBody(`{"name":"broken"`)), nil
|
||||
}
|
||||
return newJSONHTTPResponse(http.StatusOK, chatCompletionResponseBody(`{"name":"Recovered","age":30}`)), nil
|
||||
}),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewInstructorClient: %v", err)
|
||||
}
|
||||
|
||||
type person struct {
|
||||
Name string `json:"name"`
|
||||
Age int `json:"age"`
|
||||
}
|
||||
var out person
|
||||
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
Messages: []contracts.LLMMessage{
|
||||
{Role: openai.ChatMessageRoleUser, Content: "extract person"},
|
||||
},
|
||||
}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("CompleteStructured: %v", err)
|
||||
}
|
||||
if out.Name != "Recovered" || out.Age != 30 {
|
||||
t.Fatalf("unexpected output after retry: %+v", out)
|
||||
}
|
||||
if got := atomic.LoadInt32(&attempts); got != 2 {
|
||||
t.Fatalf("expected 2 attempts, got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInstructorClientContextDeadlinePropagates(t *testing.T) {
|
||||
client, err := NewInstructorClient(InstructorClientConfig{
|
||||
BaseURL: "https://slow.example/v1",
|
||||
Model: "test-model",
|
||||
HTTPClient: &http.Client{
|
||||
Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
|
||||
select {
|
||||
case <-r.Context().Done():
|
||||
return nil, r.Context().Err()
|
||||
case <-time.After(250 * time.Millisecond):
|
||||
return newJSONHTTPResponse(http.StatusOK, chatCompletionResponseBody(`{"name":"slow","age":1}`)), nil
|
||||
}
|
||||
}),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewInstructorClient: %v", err)
|
||||
}
|
||||
|
||||
type person struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
var out person
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
_, err = client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
Messages: []contracts.LLMMessage{
|
||||
{Role: openai.ChatMessageRoleUser, Content: "extract person"},
|
||||
},
|
||||
}, &out)
|
||||
if err == nil {
|
||||
t.Fatalf("expected context deadline error")
|
||||
}
|
||||
if !strings.Contains(strings.ToLower(err.Error()), "context deadline") {
|
||||
t.Fatalf("expected deadline-related error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInstructorClientSanitizesAPIKeyInErrors(t *testing.T) {
|
||||
apiKey := "super-secret-key"
|
||||
client := &InstructorClient{
|
||||
cfg: InstructorClientConfig{
|
||||
BaseURL: "http://localhost:1234/v1",
|
||||
Model: "test-model",
|
||||
APIKey: apiKey,
|
||||
Mode: ModeJSON,
|
||||
},
|
||||
client: fakeChatCompletionClient{
|
||||
err: fmt.Errorf("provider failed with Authorization: Bearer %s", apiKey),
|
||||
},
|
||||
}
|
||||
|
||||
var out map[string]any
|
||||
_, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
Messages: []contracts.LLMMessage{
|
||||
{Role: openai.ChatMessageRoleUser, Content: "extract"},
|
||||
},
|
||||
}, &out)
|
||||
if err == nil {
|
||||
t.Fatalf("expected error")
|
||||
}
|
||||
if strings.Contains(err.Error(), apiKey) {
|
||||
t.Fatalf("error leaked API key: %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "[REDACTED]") {
|
||||
t.Fatalf("expected redaction marker in error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeChatCompletionClient struct {
|
||||
err error
|
||||
}
|
||||
|
||||
func (f fakeChatCompletionClient) CreateChatCompletion(ctx context.Context, request openai.ChatCompletionRequest, responseType any) (response openai.ChatCompletionResponse, err error) {
|
||||
_ = ctx
|
||||
_ = request
|
||||
_ = responseType
|
||||
return openai.ChatCompletionResponse{}, f.err
|
||||
}
|
||||
|
||||
type roundTripFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) {
|
||||
return f(r)
|
||||
}
|
||||
|
||||
func newJSONHTTPResponse(status int, body string) *http.Response {
|
||||
return &http.Response{
|
||||
StatusCode: status,
|
||||
Header: http.Header{"Content-Type": []string{"application/json"}},
|
||||
Body: io.NopCloser(strings.NewReader(body)),
|
||||
}
|
||||
}
|
||||
|
||||
func chatCompletionResponseBody(content string) string {
|
||||
body, _ := json.Marshal(map[string]any{
|
||||
"id": "chatcmpl-test",
|
||||
"object": "chat.completion",
|
||||
"created": 12345,
|
||||
"model": "test-model",
|
||||
"choices": []map[string]any{
|
||||
{
|
||||
"index": 0,
|
||||
"message": map[string]any{
|
||||
"role": "assistant",
|
||||
"content": content,
|
||||
},
|
||||
"finish_reason": "stop",
|
||||
},
|
||||
},
|
||||
"usage": map[string]any{
|
||||
"prompt_tokens": 11,
|
||||
"completion_tokens": 7,
|
||||
"total_tokens": 18,
|
||||
},
|
||||
})
|
||||
return string(body)
|
||||
}
|
||||
370
internal/framework/llm/openai_compatible_client.go
Normal file
370
internal/framework/llm/openai_compatible_client.go
Normal file
@@ -0,0 +1,370 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
)
|
||||
|
||||
const defaultOpenAICompatibleMaxRetries = 3
|
||||
|
||||
// OpenAICompatibleClientConfig configures the direct HTTP structured-output adapter.
|
||||
type OpenAICompatibleClientConfig struct {
|
||||
BaseURL string
|
||||
Model string
|
||||
APIKey string
|
||||
MaxRetries int
|
||||
HTTPClient *http.Client
|
||||
RequestTimeout time.Duration
|
||||
}
|
||||
|
||||
// OpenAICompatibleClient sends OpenAI-compatible chat-completion requests with
|
||||
// response_format.type=json_schema.
|
||||
type OpenAICompatibleClient struct {
|
||||
cfg OpenAICompatibleClientConfig
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
var _ contracts.StructuredLLMClient = (*OpenAICompatibleClient)(nil)
|
||||
|
||||
func NewOpenAICompatibleClient(cfg OpenAICompatibleClientConfig) (*OpenAICompatibleClient, error) {
|
||||
normalized, err := normalizeOpenAICompatibleConfig(cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &OpenAICompatibleClient{
|
||||
cfg: normalized,
|
||||
httpClient: resolvedHTTPClient(normalized.HTTPClient, normalized.RequestTimeout),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *OpenAICompatibleClient) CompleteStructured(
|
||||
ctx context.Context,
|
||||
req contracts.StructuredCompletionRequest,
|
||||
out any,
|
||||
) (contracts.StructuredCompletionResponse, error) {
|
||||
if err := validateOutputTarget(out); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
|
||||
model := strings.TrimSpace(req.Model)
|
||||
if model == "" {
|
||||
model = c.cfg.Model
|
||||
}
|
||||
if model == "" {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion model must not be empty")
|
||||
}
|
||||
if req.ResponseSchema == nil {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion response schema is required")
|
||||
}
|
||||
if strings.TrimSpace(req.ResponseSchema.Name) == "" {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion response schema name must not be empty")
|
||||
}
|
||||
if len(req.ResponseSchema.JSONSchema) == 0 || !json.Valid(req.ResponseSchema.JSONSchema) {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion response schema JSON must be valid")
|
||||
}
|
||||
|
||||
messages, err := toOpenAICompatibleMessages(req.Messages)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
|
||||
endpoint := buildChatCompletionsURL(c.cfg.BaseURL)
|
||||
|
||||
var lastErr error
|
||||
for attempt := 0; attempt <= c.cfg.MaxRetries; attempt++ {
|
||||
content, metadata, callErr := c.completeStructuredOnce(
|
||||
ctx,
|
||||
endpoint,
|
||||
model,
|
||||
messages,
|
||||
req.ResponseSchema.Name,
|
||||
req.ResponseSchema.JSONSchema,
|
||||
)
|
||||
if callErr == nil {
|
||||
if decodeErr := json.Unmarshal(content, out); decodeErr != nil {
|
||||
callErr = retryableError{err: fmt.Errorf("decode structured output: %w", decodeErr)}
|
||||
} else {
|
||||
return contracts.StructuredCompletionResponse{
|
||||
Content: content,
|
||||
Provider: "openai-compatible",
|
||||
Model: firstNonEmpty(metadata.Model, model),
|
||||
PromptTokens: metadata.PromptTokens,
|
||||
CompletionTokens: metadata.CompletionTokens,
|
||||
TotalTokens: metadata.TotalTokens,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
lastErr = sanitizeError(callErr, c.cfg.APIKey)
|
||||
if !canRetryFromError(ctx, attempt, c.cfg.MaxRetries, callErr) {
|
||||
return contracts.StructuredCompletionResponse{}, lastErr
|
||||
}
|
||||
}
|
||||
|
||||
if lastErr == nil {
|
||||
lastErr = fmt.Errorf("structured completion failed")
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{}, lastErr
|
||||
}
|
||||
|
||||
type openAICompatibleMessage struct {
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
}
|
||||
|
||||
type openAICompatibleRequest struct {
|
||||
Model string `json:"model"`
|
||||
Messages []openAICompatibleMessage `json:"messages"`
|
||||
ResponseFormat openAICompatibleStructuredOutputShape `json:"response_format"`
|
||||
}
|
||||
|
||||
type openAICompatibleStructuredOutputShape struct {
|
||||
Type string `json:"type"`
|
||||
JSONSchema openAICompatibleSchemaEnvelope `json:"json_schema"`
|
||||
}
|
||||
|
||||
type openAICompatibleSchemaEnvelope struct {
|
||||
Name string `json:"name"`
|
||||
Strict bool `json:"strict"`
|
||||
Schema json.RawMessage `json:"schema"`
|
||||
}
|
||||
|
||||
type openAICompatibleChatCompletionsResponse struct {
|
||||
Model string `json:"model"`
|
||||
Choices []struct {
|
||||
Message struct {
|
||||
Content json.RawMessage `json:"content"`
|
||||
} `json:"message"`
|
||||
} `json:"choices"`
|
||||
Usage *openAICompatibleUsage `json:"usage,omitempty"`
|
||||
}
|
||||
|
||||
type openAICompatibleUsage struct {
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
}
|
||||
|
||||
type openAICompatibleResponseMetadata struct {
|
||||
Model string
|
||||
PromptTokens int
|
||||
CompletionTokens int
|
||||
TotalTokens int
|
||||
}
|
||||
|
||||
func (c *OpenAICompatibleClient) completeStructuredOnce(
|
||||
ctx context.Context,
|
||||
endpoint string,
|
||||
model string,
|
||||
messages []openAICompatibleMessage,
|
||||
responseSchemaName string,
|
||||
responseSchemaJSON json.RawMessage,
|
||||
) (json.RawMessage, openAICompatibleResponseMetadata, error) {
|
||||
requestBody := openAICompatibleRequest{
|
||||
Model: model,
|
||||
Messages: messages,
|
||||
ResponseFormat: openAICompatibleStructuredOutputShape{
|
||||
Type: "json_schema",
|
||||
JSONSchema: openAICompatibleSchemaEnvelope{
|
||||
Name: responseSchemaName,
|
||||
Strict: true,
|
||||
Schema: responseSchemaJSON,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
payload, err := json.Marshal(requestBody)
|
||||
if err != nil {
|
||||
return nil, openAICompatibleResponseMetadata{}, fmt.Errorf("marshal provider request: %w", err)
|
||||
}
|
||||
|
||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return nil, openAICompatibleResponseMetadata{}, fmt.Errorf("build provider request: %w", err)
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
if c.cfg.APIKey != "" {
|
||||
httpReq.Header.Set("Authorization", "Bearer "+c.cfg.APIKey)
|
||||
}
|
||||
|
||||
httpResp, err := c.httpClient.Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, openAICompatibleResponseMetadata{}, retryableError{err: fmt.Errorf("provider request failed: %w", err)}
|
||||
}
|
||||
defer func() {
|
||||
_ = httpResp.Body.Close()
|
||||
}()
|
||||
|
||||
rawResp, err := io.ReadAll(httpResp.Body)
|
||||
if err != nil {
|
||||
return nil, openAICompatibleResponseMetadata{}, retryableError{err: fmt.Errorf("read provider response: %w", err)}
|
||||
}
|
||||
|
||||
if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 {
|
||||
statusErr := parseProviderErrorBody(httpResp.StatusCode, rawResp)
|
||||
if httpResp.StatusCode == http.StatusTooManyRequests || httpResp.StatusCode >= 500 {
|
||||
return nil, openAICompatibleResponseMetadata{}, retryableError{err: statusErr}
|
||||
}
|
||||
return nil, openAICompatibleResponseMetadata{}, statusErr
|
||||
}
|
||||
|
||||
content, metadata, decodeErr := decodeChatCompletionsResponse(rawResp)
|
||||
if decodeErr != nil {
|
||||
return nil, openAICompatibleResponseMetadata{}, decodeErr
|
||||
}
|
||||
return content, metadata, nil
|
||||
}
|
||||
|
||||
func toOpenAICompatibleMessages(messages []contracts.LLMMessage) ([]openAICompatibleMessage, error) {
|
||||
result := make([]openAICompatibleMessage, len(messages))
|
||||
for i, message := range messages {
|
||||
role := strings.TrimSpace(message.Role)
|
||||
content := strings.TrimSpace(message.Content)
|
||||
if role == "" {
|
||||
return nil, fmt.Errorf("message[%d] role must not be empty", i)
|
||||
}
|
||||
if content == "" {
|
||||
return nil, fmt.Errorf("message[%d] content must not be empty", i)
|
||||
}
|
||||
result[i] = openAICompatibleMessage{
|
||||
Role: role,
|
||||
Content: content,
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func buildChatCompletionsURL(baseURL string) string {
|
||||
return strings.TrimRight(baseURL, "/") + "/chat/completions"
|
||||
}
|
||||
|
||||
func normalizeOpenAICompatibleConfig(cfg OpenAICompatibleClientConfig) (OpenAICompatibleClientConfig, error) {
|
||||
cfg.BaseURL = strings.TrimSpace(cfg.BaseURL)
|
||||
cfg.Model = strings.TrimSpace(cfg.Model)
|
||||
cfg.APIKey = strings.TrimSpace(cfg.APIKey)
|
||||
if cfg.MaxRetries == 0 {
|
||||
cfg.MaxRetries = defaultOpenAICompatibleMaxRetries
|
||||
}
|
||||
if cfg.MaxRetries < 0 {
|
||||
return OpenAICompatibleClientConfig{}, fmt.Errorf("max retries must be zero or greater")
|
||||
}
|
||||
if cfg.BaseURL == "" {
|
||||
return OpenAICompatibleClientConfig{}, fmt.Errorf("base URL must not be empty")
|
||||
}
|
||||
if cfg.Model == "" {
|
||||
return OpenAICompatibleClientConfig{}, fmt.Errorf("model must not be empty")
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
type retryableError struct {
|
||||
err error
|
||||
}
|
||||
|
||||
func (e retryableError) Error() string {
|
||||
if e.err == nil {
|
||||
return ""
|
||||
}
|
||||
return e.err.Error()
|
||||
}
|
||||
|
||||
func (e retryableError) Unwrap() error {
|
||||
return e.err
|
||||
}
|
||||
|
||||
func canRetryFromError(ctx context.Context, attempt int, maxRetries int, err error) bool {
|
||||
if attempt >= maxRetries {
|
||||
return false
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return false
|
||||
}
|
||||
var retryable retryableError
|
||||
return errors.As(err, &retryable)
|
||||
}
|
||||
|
||||
func firstNonEmpty(values ...string) string {
|
||||
for _, value := range values {
|
||||
value = strings.TrimSpace(value)
|
||||
if value != "" {
|
||||
return value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func decodeChatCompletionsResponse(raw []byte) (json.RawMessage, openAICompatibleResponseMetadata, error) {
|
||||
var parsed openAICompatibleChatCompletionsResponse
|
||||
if err := json.Unmarshal(raw, &parsed); err != nil {
|
||||
return nil, openAICompatibleResponseMetadata{}, retryableError{err: fmt.Errorf("decode provider response envelope: %w", err)}
|
||||
}
|
||||
if len(parsed.Choices) == 0 {
|
||||
return nil, openAICompatibleResponseMetadata{}, fmt.Errorf("provider response missing choices")
|
||||
}
|
||||
|
||||
content, err := extractAssistantContentJSON(parsed.Choices[0].Message.Content)
|
||||
if err != nil {
|
||||
return nil, openAICompatibleResponseMetadata{}, retryableError{err: err}
|
||||
}
|
||||
|
||||
meta := openAICompatibleResponseMetadata{
|
||||
Model: parsed.Model,
|
||||
}
|
||||
if parsed.Usage != nil {
|
||||
meta.PromptTokens = parsed.Usage.PromptTokens
|
||||
meta.CompletionTokens = parsed.Usage.CompletionTokens
|
||||
meta.TotalTokens = parsed.Usage.TotalTokens
|
||||
}
|
||||
return content, meta, nil
|
||||
}
|
||||
|
||||
func extractAssistantContentJSON(raw json.RawMessage) (json.RawMessage, error) {
|
||||
if len(bytes.TrimSpace(raw)) == 0 || string(bytes.TrimSpace(raw)) == "null" {
|
||||
return nil, fmt.Errorf("provider response missing assistant message content")
|
||||
}
|
||||
|
||||
var textContent string
|
||||
if err := json.Unmarshal(raw, &textContent); err == nil {
|
||||
textContent = strings.TrimSpace(textContent)
|
||||
if textContent == "" {
|
||||
return nil, fmt.Errorf("provider response assistant message content is empty")
|
||||
}
|
||||
return json.RawMessage(textContent), nil
|
||||
}
|
||||
|
||||
trimmed := bytes.TrimSpace(raw)
|
||||
if json.Valid(trimmed) {
|
||||
return append(json.RawMessage(nil), trimmed...), nil
|
||||
}
|
||||
return nil, fmt.Errorf("provider response assistant message content is not valid JSON")
|
||||
}
|
||||
|
||||
func parseProviderErrorBody(status int, body []byte) error {
|
||||
trimmed := strings.TrimSpace(string(body))
|
||||
if trimmed == "" {
|
||||
return fmt.Errorf("provider returned status %d", status)
|
||||
}
|
||||
|
||||
var payload map[string]any
|
||||
if err := json.Unmarshal(body, &payload); err == nil {
|
||||
if nested, ok := payload["error"].(map[string]any); ok {
|
||||
if msg, ok := nested["message"].(string); ok && strings.TrimSpace(msg) != "" {
|
||||
return fmt.Errorf("provider returned status %d: %s", status, strings.TrimSpace(msg))
|
||||
}
|
||||
}
|
||||
if msg, ok := payload["message"].(string); ok && strings.TrimSpace(msg) != "" {
|
||||
return fmt.Errorf("provider returned status %d: %s", status, strings.TrimSpace(msg))
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Errorf("provider returned status %d: %s", status, trimmed)
|
||||
}
|
||||
571
internal/framework/llm/openai_compatible_client_test.go
Normal file
571
internal/framework/llm/openai_compatible_client_test.go
Normal file
@@ -0,0 +1,571 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
|
||||
)
|
||||
|
||||
func TestNewOpenAICompatibleClientValidation(t *testing.T) {
|
||||
_, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
|
||||
BaseURL: " ",
|
||||
Model: "model",
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "base URL") {
|
||||
t.Fatalf("expected base URL validation error, got %v", err)
|
||||
}
|
||||
|
||||
_, err = NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
|
||||
BaseURL: "https://example.test/v1",
|
||||
Model: " ",
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "model") {
|
||||
t.Fatalf("expected model validation error, got %v", err)
|
||||
}
|
||||
|
||||
_, err = NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
|
||||
BaseURL: "https://example.test/v1",
|
||||
Model: "model",
|
||||
MaxRetries: -1,
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "max retries") {
|
||||
t.Fatalf("expected max retries validation error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientRequestShapeAndDecode(t *testing.T) {
|
||||
schema := responseschema.MustLookup(responseschema.CorrectionSetKey)
|
||||
var seenPath string
|
||||
var seenAuthorization string
|
||||
var seenReq map[string]any
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
seenPath = r.URL.Path
|
||||
seenAuthorization = r.Header.Get("Authorization")
|
||||
if err := json.NewDecoder(r.Body).Decode(&seenReq); err != nil {
|
||||
t.Fatalf("decode request: %v", err)
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"model":"provider-model",
|
||||
"choices":[{"message":{"content":"{\"name\":\"Robby\",\"age\":22}"}}],
|
||||
"usage":{"prompt_tokens":11,"completion_tokens":7,"total_tokens":18}
|
||||
}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
|
||||
BaseURL: server.URL + "/v1",
|
||||
Model: "test-model",
|
||||
APIKey: "secret-key",
|
||||
MaxRetries: 1,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewOpenAICompatibleClient: %v", err)
|
||||
}
|
||||
|
||||
type person struct {
|
||||
Name string `json:"name"`
|
||||
Age int `json:"age"`
|
||||
}
|
||||
var out person
|
||||
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
|
||||
ResponseSchema: &schema,
|
||||
}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("CompleteStructured: %v", err)
|
||||
}
|
||||
|
||||
if seenPath != "/v1/chat/completions" {
|
||||
t.Fatalf("unexpected request path: %q", seenPath)
|
||||
}
|
||||
if seenAuthorization != "Bearer secret-key" {
|
||||
t.Fatalf("unexpected authorization header: %q", seenAuthorization)
|
||||
}
|
||||
|
||||
responseFormat, ok := seenReq["response_format"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected response_format object, got %T", seenReq["response_format"])
|
||||
}
|
||||
if responseFormat["type"] != "json_schema" {
|
||||
t.Fatalf("unexpected response_format.type: %v", responseFormat["type"])
|
||||
}
|
||||
jsonSchema, ok := responseFormat["json_schema"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected response_format.json_schema object, got %T", responseFormat["json_schema"])
|
||||
}
|
||||
if jsonSchema["name"] != schema.Name {
|
||||
t.Fatalf("unexpected response schema name: %v", jsonSchema["name"])
|
||||
}
|
||||
if jsonSchema["strict"] != true {
|
||||
t.Fatalf("expected strict=true, got %v", jsonSchema["strict"])
|
||||
}
|
||||
if _, ok := jsonSchema["schema"].(map[string]any); !ok {
|
||||
t.Fatalf("expected embedded JSON schema object, got %T", jsonSchema["schema"])
|
||||
}
|
||||
|
||||
if out.Name != "Robby" || out.Age != 22 {
|
||||
t.Fatalf("unexpected decoded output: %+v", out)
|
||||
}
|
||||
if resp.Provider != "openai-compatible" {
|
||||
t.Fatalf("unexpected provider metadata: %q", resp.Provider)
|
||||
}
|
||||
if resp.Model != "provider-model" {
|
||||
t.Fatalf("unexpected model metadata: %q", resp.Model)
|
||||
}
|
||||
if resp.PromptTokens != 11 || resp.CompletionTokens != 7 || resp.TotalTokens != 18 {
|
||||
t.Fatalf("unexpected token metadata: %+v", resp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientDecodesCorrectionSetStructuredResponse(t *testing.T) {
|
||||
schema := responseschema.MustLookup(responseschema.CorrectionSetKey)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"choices":[{"message":{"content":"{\"corrections\":[{\"id\":1,\"original_text\":\"teh\",\"corrected_text\":\"the\",\"confidence\":0.9}]}"}}]
|
||||
}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
|
||||
BaseURL: server.URL,
|
||||
Model: "test-model",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewOpenAICompatibleClient: %v", err)
|
||||
}
|
||||
|
||||
type correction struct {
|
||||
TargetSegmentID int `json:"id"`
|
||||
OriginalText string `json:"original_text"`
|
||||
CorrectedText string `json:"corrected_text"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
}
|
||||
type correctionSet struct {
|
||||
Corrections []correction `json:"corrections"`
|
||||
}
|
||||
var out correctionSet
|
||||
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
|
||||
ResponseSchema: &schema,
|
||||
}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("CompleteStructured: %v", err)
|
||||
}
|
||||
if len(out.Corrections) != 1 {
|
||||
t.Fatalf("expected one correction, got %+v", out.Corrections)
|
||||
}
|
||||
if out.Corrections[0].TargetSegmentID != 1 || out.Corrections[0].CorrectedText != "the" {
|
||||
t.Fatalf("unexpected correction payload: %+v", out.Corrections[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientDecodesValidatorDecisionSetStructuredResponse(t *testing.T) {
|
||||
schema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"choices":[{"message":{"content":"{\"validations\":[{\"correction_index\":0,\"approved\":true,\"confidence\":0.95,\"reason\":\"ok\"}]}"}}]
|
||||
}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
|
||||
BaseURL: server.URL,
|
||||
Model: "test-model",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewOpenAICompatibleClient: %v", err)
|
||||
}
|
||||
|
||||
type validationDecision struct {
|
||||
CorrectionIndex int `json:"correction_index"`
|
||||
Approved bool `json:"approved"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
type validationResponse struct {
|
||||
Validations []validationDecision `json:"validations"`
|
||||
}
|
||||
var out validationResponse
|
||||
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
|
||||
ResponseSchema: &schema,
|
||||
}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("CompleteStructured: %v", err)
|
||||
}
|
||||
if len(out.Validations) != 1 {
|
||||
t.Fatalf("expected one validation decision, got %+v", out.Validations)
|
||||
}
|
||||
if out.Validations[0].CorrectionIndex != 0 || !out.Validations[0].Approved {
|
||||
t.Fatalf("unexpected validation payload: %+v", out.Validations[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientNoAuthorizationHeaderWithoutAPIKey(t *testing.T) {
|
||||
schema := responseschema.MustLookup(responseschema.CorrectionSetKey)
|
||||
var seenAuthorization string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
seenAuthorization = r.Header.Get("Authorization")
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{\"ok\":true}"}}]}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
|
||||
BaseURL: server.URL,
|
||||
Model: "test-model",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewOpenAICompatibleClient: %v", err)
|
||||
}
|
||||
|
||||
var out map[string]any
|
||||
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
|
||||
ResponseSchema: &schema,
|
||||
}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("CompleteStructured: %v", err)
|
||||
}
|
||||
if seenAuthorization != "" {
|
||||
t.Fatalf("expected empty Authorization header, got %q", seenAuthorization)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientMalformedJSONFailsSafely(t *testing.T) {
|
||||
schema := responseschema.MustLookup(responseschema.CorrectionSetKey)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{"}}]}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
|
||||
BaseURL: server.URL,
|
||||
Model: "test-model",
|
||||
MaxRetries: 0,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewOpenAICompatibleClient: %v", err)
|
||||
}
|
||||
|
||||
var out map[string]any
|
||||
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
|
||||
ResponseSchema: &schema,
|
||||
}, &out)
|
||||
if err == nil || !strings.Contains(err.Error(), "decode structured output") {
|
||||
t.Fatalf("expected decode error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientMissingRequiredFieldsFailsSafely(t *testing.T) {
|
||||
schema := responseschema.MustLookup(responseschema.CorrectionSetKey)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"model":"x","choices":[]}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
|
||||
BaseURL: server.URL,
|
||||
Model: "test-model",
|
||||
MaxRetries: 0,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewOpenAICompatibleClient: %v", err)
|
||||
}
|
||||
|
||||
var out map[string]any
|
||||
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
|
||||
ResponseSchema: &schema,
|
||||
}, &out)
|
||||
if err == nil || !strings.Contains(err.Error(), "missing choices") {
|
||||
t.Fatalf("expected missing-field error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientUnknownExtraFieldsFollowLocalDecoderPolicy(t *testing.T) {
|
||||
schema := responseschema.MustLookup(responseschema.CorrectionSetKey)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"choices":[{"message":{"content":"{\"corrections\":[{\"id\":1,\"original_text\":\"teh\",\"corrected_text\":\"the\",\"confidence\":0.9,\"extra\":\"ignored\"}],\"top_extra\":true}"}}]
|
||||
}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
|
||||
BaseURL: server.URL,
|
||||
Model: "test-model",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewOpenAICompatibleClient: %v", err)
|
||||
}
|
||||
|
||||
type correction struct {
|
||||
TargetSegmentID int `json:"id"`
|
||||
OriginalText string `json:"original_text"`
|
||||
CorrectedText string `json:"corrected_text"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
}
|
||||
type correctionSet struct {
|
||||
Corrections []correction `json:"corrections"`
|
||||
}
|
||||
var out correctionSet
|
||||
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
|
||||
ResponseSchema: &schema,
|
||||
}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("expected unknown extra fields to be ignored by local decoder, got %v", err)
|
||||
}
|
||||
if len(out.Corrections) != 1 || out.Corrections[0].CorrectedText != "the" {
|
||||
t.Fatalf("unexpected decoded payload: %+v", out.Corrections)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientProviderErrorRedactsSecret(t *testing.T) {
|
||||
secret := "super-secret-key"
|
||||
schema := responseschema.MustLookup(responseschema.CorrectionSetKey)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
_, _ = w.Write([]byte(`{"error":{"message":"Authorization failed for Bearer super-secret-key"}}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
|
||||
BaseURL: server.URL,
|
||||
Model: "test-model",
|
||||
APIKey: secret,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewOpenAICompatibleClient: %v", err)
|
||||
}
|
||||
|
||||
var out map[string]any
|
||||
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
|
||||
ResponseSchema: &schema,
|
||||
}, &out)
|
||||
if err == nil {
|
||||
t.Fatalf("expected provider error")
|
||||
}
|
||||
if strings.Contains(err.Error(), secret) {
|
||||
t.Fatalf("error leaked secret: %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "[REDACTED]") {
|
||||
t.Fatalf("expected redaction marker in error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientRequestErrorRedactsSecret(t *testing.T) {
|
||||
secret := "super-secret-key"
|
||||
schema := responseschema.MustLookup(responseschema.CorrectionSetKey)
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
|
||||
BaseURL: "https://example.test/v1",
|
||||
Model: "test-model",
|
||||
APIKey: secret,
|
||||
HTTPClient: &http.Client{
|
||||
Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
|
||||
_ = r
|
||||
return nil, fmt.Errorf("request failed for Authorization: Bearer %s", secret)
|
||||
}),
|
||||
},
|
||||
MaxRetries: 0,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewOpenAICompatibleClient: %v", err)
|
||||
}
|
||||
|
||||
var out map[string]any
|
||||
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
|
||||
ResponseSchema: &schema,
|
||||
}, &out)
|
||||
if err == nil {
|
||||
t.Fatalf("expected request error")
|
||||
}
|
||||
if strings.Contains(err.Error(), secret) {
|
||||
t.Fatalf("error leaked secret: %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "[REDACTED]") {
|
||||
t.Fatalf("expected redaction marker in error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientCancellationAndTimeout(t *testing.T) {
|
||||
schema := responseschema.MustLookup(responseschema.CorrectionSetKey)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
select {
|
||||
case <-r.Context().Done():
|
||||
return
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{\"ok\":true}"}}]}`))
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
|
||||
BaseURL: server.URL,
|
||||
Model: "test-model",
|
||||
RequestTimeout: 20 * time.Millisecond,
|
||||
MaxRetries: 0,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewOpenAICompatibleClient: %v", err)
|
||||
}
|
||||
|
||||
var out map[string]any
|
||||
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
|
||||
ResponseSchema: &schema,
|
||||
}, &out)
|
||||
if err == nil {
|
||||
t.Fatalf("expected timeout-related error")
|
||||
}
|
||||
if !strings.Contains(strings.ToLower(err.Error()), "context deadline") {
|
||||
t.Fatalf("expected context deadline in error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientRetryBehavior(t *testing.T) {
|
||||
schema := responseschema.MustLookup(responseschema.CorrectionSetKey)
|
||||
var attempts int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
current := atomic.AddInt32(&attempts, 1)
|
||||
if current == 1 {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
_, _ = w.Write([]byte(`{"error":{"message":"temporary failure"}}`))
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{\"ok\":true}"}}]}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
|
||||
BaseURL: server.URL,
|
||||
Model: "test-model",
|
||||
MaxRetries: 1,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewOpenAICompatibleClient: %v", err)
|
||||
}
|
||||
|
||||
var out map[string]any
|
||||
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
|
||||
ResponseSchema: &schema,
|
||||
}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("CompleteStructured: %v", err)
|
||||
}
|
||||
if atomic.LoadInt32(&attempts) != 2 {
|
||||
t.Fatalf("expected 2 attempts, got %d", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientRetryOnMalformedStructuredOutput(t *testing.T) {
|
||||
schema := responseschema.MustLookup(responseschema.CorrectionSetKey)
|
||||
var attempts int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
current := atomic.AddInt32(&attempts, 1)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if current == 1 {
|
||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{"}}]}`))
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{\"ok\":true}"}}]}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
|
||||
BaseURL: server.URL,
|
||||
Model: "test-model",
|
||||
MaxRetries: 1,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewOpenAICompatibleClient: %v", err)
|
||||
}
|
||||
|
||||
var out map[string]any
|
||||
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
|
||||
ResponseSchema: &schema,
|
||||
}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("CompleteStructured: %v", err)
|
||||
}
|
||||
if atomic.LoadInt32(&attempts) != 2 {
|
||||
t.Fatalf("expected 2 attempts, got %d", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientHonorsCancelledContextWithoutRetry(t *testing.T) {
|
||||
schema := responseschema.MustLookup(responseschema.CorrectionSetKey)
|
||||
var attempts int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
atomic.AddInt32(&attempts, 1)
|
||||
<-r.Context().Done()
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
|
||||
BaseURL: server.URL,
|
||||
Model: "test-model",
|
||||
MaxRetries: 3,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewOpenAICompatibleClient: %v", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
var out map[string]any
|
||||
_, err = client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
|
||||
ResponseSchema: &schema,
|
||||
}, &out)
|
||||
if err == nil {
|
||||
t.Fatalf("expected cancellation error")
|
||||
}
|
||||
if !errors.Is(err, context.Canceled) && !strings.Contains(strings.ToLower(err.Error()), "canceled") {
|
||||
t.Fatalf("expected cancellation-related error, got %v", err)
|
||||
}
|
||||
if atomic.LoadInt32(&attempts) > 1 {
|
||||
t.Fatalf("expected no retry after cancellation, got attempts=%d", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
type roundTripFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) {
|
||||
return f(r)
|
||||
}
|
||||
|
||||
func newJSONHTTPResponse(status int, body string) *http.Response {
|
||||
return &http.Response{
|
||||
StatusCode: status,
|
||||
Header: http.Header{"Content-Type": []string{"application/json"}},
|
||||
Body: io.NopCloser(strings.NewReader(body)),
|
||||
}
|
||||
}
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
|
||||
)
|
||||
|
||||
// InteractionDiagnosticsWriter writes machine-readable prompt/response artifacts.
|
||||
@@ -112,11 +113,13 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
callErr error
|
||||
artifacts InteractionArtifacts
|
||||
)
|
||||
responseSchema := responseschema.MustLookup(responseschema.CorrectionSetKey)
|
||||
call := func(callCtx context.Context) error {
|
||||
_, callErr = req.LLMClient.CompleteStructured(callCtx, contracts.StructuredCompletionRequest{
|
||||
StageName: stage,
|
||||
Messages: messages,
|
||||
Model: model,
|
||||
StageName: stage,
|
||||
Messages: messages,
|
||||
Model: model,
|
||||
ResponseSchema: &responseSchema,
|
||||
}, &response)
|
||||
return callErr
|
||||
}
|
||||
@@ -126,17 +129,20 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
callErr = call(ctx)
|
||||
}
|
||||
|
||||
requestMetadata := map[string]any{
|
||||
"module_key": req.ModuleKey,
|
||||
"module_instance": req.ModuleInstance,
|
||||
"replacement_policy": req.ReplacementPolicy,
|
||||
"section": req.Section,
|
||||
"start_index": req.StartIndex,
|
||||
"model": model,
|
||||
}
|
||||
requestMetadata["response_schema"] = schemaMetadata(responseSchema)
|
||||
|
||||
if writer != nil {
|
||||
artifacts, _ = writer.WriteInteraction(
|
||||
stage,
|
||||
map[string]any{
|
||||
"module_key": req.ModuleKey,
|
||||
"module_instance": req.ModuleInstance,
|
||||
"replacement_policy": req.ReplacementPolicy,
|
||||
"section": req.Section,
|
||||
"start_index": req.StartIndex,
|
||||
"model": model,
|
||||
},
|
||||
requestMetadata,
|
||||
map[string]any{
|
||||
"messages": messages,
|
||||
},
|
||||
@@ -185,6 +191,15 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
func schemaMetadata(schema responseschema.Schema) map[string]any {
|
||||
return map[string]any{
|
||||
"id": schema.ID,
|
||||
"version": schema.Version,
|
||||
"name": schema.Name,
|
||||
"sha256": schema.SHA256,
|
||||
}
|
||||
}
|
||||
|
||||
func buildStageName(moduleInstance string, section *contracts.SectionMetadata) string {
|
||||
base := fmt.Sprintf("%s:proposal-generation", moduleInstance)
|
||||
if section == nil {
|
||||
|
||||
@@ -2,6 +2,7 @@ package proposal_generation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -17,6 +18,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
|
||||
)
|
||||
|
||||
type fakeStructuredClient struct {
|
||||
@@ -52,6 +54,21 @@ func (s *countingScheduler) Run(ctx context.Context, fn func(context.Context) er
|
||||
return fn(ctx)
|
||||
}
|
||||
|
||||
type captureDiagnosticsWriter struct {
|
||||
lastStage string
|
||||
lastRequestMetadata any
|
||||
lastRequestPayload any
|
||||
}
|
||||
|
||||
func (w *captureDiagnosticsWriter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) {
|
||||
w.lastStage = stage
|
||||
w.lastRequestMetadata = requestMetadata
|
||||
w.lastRequestPayload = requestPayload
|
||||
_ = responsePayload
|
||||
_ = errorPayload
|
||||
return InteractionArtifacts{}, nil
|
||||
}
|
||||
|
||||
type sleepingStructuredClient struct {
|
||||
inFlight int32
|
||||
maxInFlight int32
|
||||
@@ -138,6 +155,61 @@ func TestGenerateCandidatesSuccess(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesUsesCorrectionSetSchema(t *testing.T) {
|
||||
client := &fakeStructuredClient{
|
||||
responses: []StructuredCorrectionSet{
|
||||
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
|
||||
},
|
||||
}
|
||||
req := defaultRequest(t)
|
||||
req.LLMClient = client
|
||||
|
||||
_, err := GenerateCandidates(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateCandidates error: %v", err)
|
||||
}
|
||||
if len(client.calls) != 1 {
|
||||
t.Fatalf("expected 1 LLM call, got %d", len(client.calls))
|
||||
}
|
||||
call := client.calls[0]
|
||||
if call.ResponseSchema == nil {
|
||||
t.Fatalf("expected response schema on structured request")
|
||||
}
|
||||
want := responseschema.MustLookup(responseschema.CorrectionSetKey)
|
||||
if call.ResponseSchema.ID != want.ID || call.ResponseSchema.Version != want.Version || call.ResponseSchema.Name != want.Name || call.ResponseSchema.SHA256 != want.SHA256 {
|
||||
t.Fatalf("unexpected response schema metadata: got=%+v want=%+v", *call.ResponseSchema, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesDiagnosticsIncludeSchemaMetadata(t *testing.T) {
|
||||
client := &fakeStructuredClient{
|
||||
responses: []StructuredCorrectionSet{
|
||||
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
|
||||
},
|
||||
}
|
||||
diag := &captureDiagnosticsWriter{}
|
||||
req := defaultRequest(t)
|
||||
req.LLMClient = client
|
||||
req.DiagnosticsWriter = diag
|
||||
|
||||
_, err := GenerateCandidates(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateCandidates error: %v", err)
|
||||
}
|
||||
metadata, ok := diag.lastRequestMetadata.(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected request metadata map, got %T", diag.lastRequestMetadata)
|
||||
}
|
||||
schemaMap, ok := metadata["response_schema"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected response_schema metadata map, got %T", metadata["response_schema"])
|
||||
}
|
||||
want := responseschema.MustLookup(responseschema.CorrectionSetKey)
|
||||
if schemaMap["id"] != want.ID || schemaMap["version"] != want.Version || schemaMap["name"] != want.Name || schemaMap["sha256"] != want.SHA256 {
|
||||
t.Fatalf("unexpected diagnostics schema metadata: got=%v want=%+v", schemaMap, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesMalformedStructuredResponse(t *testing.T) {
|
||||
client := &fakeStructuredClient{
|
||||
responses: []StructuredCorrectionSet{
|
||||
@@ -266,6 +338,23 @@ func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) {
|
||||
t.Fatalf("expected redaction marker in artifact %q: %s", path, string(raw))
|
||||
}
|
||||
}
|
||||
|
||||
raw, readErr := os.ReadFile(got.Artifacts.RequestMetadataPath)
|
||||
if readErr != nil {
|
||||
t.Fatalf("read metadata artifact %q: %v", got.Artifacts.RequestMetadataPath, readErr)
|
||||
}
|
||||
var metadata map[string]any
|
||||
if err := json.Unmarshal(raw, &metadata); err != nil {
|
||||
t.Fatalf("unmarshal metadata artifact: %v", err)
|
||||
}
|
||||
schemaMap, ok := metadata["response_schema"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected response_schema metadata in diagnostics, got %T", metadata["response_schema"])
|
||||
}
|
||||
want := responseschema.MustLookup(responseschema.CorrectionSetKey)
|
||||
if schemaMap["id"] != want.ID || schemaMap["version"] != want.Version || schemaMap["name"] != want.Name || schemaMap["sha256"] != want.SHA256 {
|
||||
t.Fatalf("unexpected schema metadata in diagnostics: got=%v want=%+v", schemaMap, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesSchedulerUsage(t *testing.T) {
|
||||
|
||||
97
internal/framework/responseschema/registry.go
Normal file
97
internal/framework/responseschema/registry.go
Normal file
@@ -0,0 +1,97 @@
|
||||
package responseschema
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Key identifies one structured response schema used by Audita.
|
||||
type Key string
|
||||
|
||||
const (
|
||||
CorrectionSetKey Key = "correction_set"
|
||||
ValidatorDecisionSetKey Key = "validator_decision_set"
|
||||
correctionSetSchemaID = "audita.correction_set"
|
||||
validatorDecisionSchemaID = "audita.validator_decision_set"
|
||||
schemaVersionV1 = "v1"
|
||||
)
|
||||
|
||||
// Schema describes one registered structured response schema.
|
||||
type Schema struct {
|
||||
ID string `json:"id"`
|
||||
Version string `json:"version"`
|
||||
Name string `json:"name"`
|
||||
JSONSchema json.RawMessage `json:"json_schema"`
|
||||
SHA256 string `json:"sha256"`
|
||||
}
|
||||
|
||||
var registry = map[Key]Schema{
|
||||
CorrectionSetKey: mustBuildSchema(
|
||||
correctionSetSchemaID,
|
||||
schemaVersionV1,
|
||||
"audita_correction_set_v1",
|
||||
[]byte(`{"type":"object","additionalProperties":false,"required":["corrections"],"properties":{"corrections":{"type":"array","items":{"type":"object","additionalProperties":false,"required":["id","original_text","corrected_text","confidence"],"properties":{"id":{"type":"integer","minimum":1},"original_text":{"type":"string","minLength":1},"corrected_text":{"type":"string","minLength":1},"confidence":{"type":"number","minimum":0,"maximum":1}}}}}}`),
|
||||
),
|
||||
ValidatorDecisionSetKey: mustBuildSchema(
|
||||
validatorDecisionSchemaID,
|
||||
schemaVersionV1,
|
||||
"audita_validator_decision_set_v1",
|
||||
[]byte(`{"type":"object","additionalProperties":false,"required":["validations"],"properties":{"validations":{"type":"array","items":{"type":"object","additionalProperties":false,"required":["correction_index","approved","confidence","reason"],"properties":{"correction_index":{"type":"integer","minimum":0},"approved":{"type":"boolean"},"confidence":{"type":"number","minimum":0,"maximum":1},"reason":{"type":"string"}}}}}}`),
|
||||
),
|
||||
}
|
||||
|
||||
// Lookup returns a copy of the registered schema for the provided key.
|
||||
func Lookup(key Key) (Schema, bool) {
|
||||
schema, ok := registry[key]
|
||||
if !ok {
|
||||
return Schema{}, false
|
||||
}
|
||||
return cloneSchema(schema), true
|
||||
}
|
||||
|
||||
// MustLookup returns a copy of the registered schema and panics when missing.
|
||||
func MustLookup(key Key) Schema {
|
||||
schema, ok := Lookup(key)
|
||||
if !ok {
|
||||
panic(fmt.Sprintf("unknown structured response schema key %q", key))
|
||||
}
|
||||
return schema
|
||||
}
|
||||
|
||||
func cloneSchema(in Schema) Schema {
|
||||
out := in
|
||||
if in.JSONSchema != nil {
|
||||
out.JSONSchema = append(json.RawMessage(nil), in.JSONSchema...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func mustBuildSchema(id string, version string, name string, rawSchema []byte) Schema {
|
||||
id = strings.TrimSpace(id)
|
||||
version = strings.TrimSpace(version)
|
||||
name = strings.TrimSpace(name)
|
||||
if id == "" {
|
||||
panic("schema id must not be empty")
|
||||
}
|
||||
if version == "" {
|
||||
panic("schema version must not be empty")
|
||||
}
|
||||
if name == "" {
|
||||
panic("schema name must not be empty")
|
||||
}
|
||||
if !json.Valid(rawSchema) {
|
||||
panic(fmt.Sprintf("schema %s:%s is not valid JSON", id, version))
|
||||
}
|
||||
|
||||
hash := sha256.Sum256(rawSchema)
|
||||
return Schema{
|
||||
ID: id,
|
||||
Version: version,
|
||||
Name: name,
|
||||
JSONSchema: append(json.RawMessage(nil), rawSchema...),
|
||||
SHA256: hex.EncodeToString(hash[:]),
|
||||
}
|
||||
}
|
||||
91
internal/framework/responseschema/registry_test.go
Normal file
91
internal/framework/responseschema/registry_test.go
Normal file
@@ -0,0 +1,91 @@
|
||||
package responseschema
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const (
|
||||
expectedCorrectionSetSHA256 = "05f8ff3fa04f68115c0cb1859d2656f51aa5c0bae8ff2470b2d4f6f531953195"
|
||||
expectedValidatorDecisionSetSHA256 = "b73f4790b98fbb955f0aec5496dd8ce9a8fe14aa2f35c700b4b4e5634f106fd5"
|
||||
)
|
||||
|
||||
func TestLookupKnownSchemas(t *testing.T) {
|
||||
correction, ok := Lookup(CorrectionSetKey)
|
||||
if !ok {
|
||||
t.Fatalf("expected correction-set schema to be registered")
|
||||
}
|
||||
if correction.ID != correctionSetSchemaID {
|
||||
t.Fatalf("unexpected correction-set schema id %q", correction.ID)
|
||||
}
|
||||
if correction.Version != schemaVersionV1 {
|
||||
t.Fatalf("unexpected correction-set schema version %q", correction.Version)
|
||||
}
|
||||
if correction.Name != "audita_correction_set_v1" {
|
||||
t.Fatalf("unexpected correction-set schema name %q", correction.Name)
|
||||
}
|
||||
|
||||
validation, ok := Lookup(ValidatorDecisionSetKey)
|
||||
if !ok {
|
||||
t.Fatalf("expected validator-decision schema to be registered")
|
||||
}
|
||||
if validation.ID != validatorDecisionSchemaID {
|
||||
t.Fatalf("unexpected validator-decision schema id %q", validation.ID)
|
||||
}
|
||||
if validation.Version != schemaVersionV1 {
|
||||
t.Fatalf("unexpected validator-decision schema version %q", validation.Version)
|
||||
}
|
||||
if validation.Name != "audita_validator_decision_set_v1" {
|
||||
t.Fatalf("unexpected validator-decision schema name %q", validation.Name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLookupUnknownSchema(t *testing.T) {
|
||||
if _, ok := Lookup(Key("missing")); ok {
|
||||
t.Fatalf("expected unknown schema lookup to fail")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSchemaHashesMatchRegisteredJSON(t *testing.T) {
|
||||
expectedByKey := map[Key]string{
|
||||
CorrectionSetKey: expectedCorrectionSetSHA256,
|
||||
ValidatorDecisionSetKey: expectedValidatorDecisionSetSHA256,
|
||||
}
|
||||
for key, expectedHash := range expectedByKey {
|
||||
schema, ok := Lookup(key)
|
||||
if !ok {
|
||||
t.Fatalf("missing schema %q", key)
|
||||
}
|
||||
sum := sha256.Sum256(schema.JSONSchema)
|
||||
expected := hex.EncodeToString(sum[:])
|
||||
if schema.SHA256 != expectedHash {
|
||||
t.Fatalf("unexpected stable hash for %q: got %q want %q", key, schema.SHA256, expectedHash)
|
||||
}
|
||||
if schema.SHA256 != expected {
|
||||
t.Fatalf("unexpected hash for %q: got %q want %q", key, schema.SHA256, expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLookupReturnsSchemaCopy(t *testing.T) {
|
||||
schema, ok := Lookup(CorrectionSetKey)
|
||||
if !ok {
|
||||
t.Fatalf("missing correction-set schema")
|
||||
}
|
||||
if len(schema.JSONSchema) == 0 {
|
||||
t.Fatalf("expected non-empty schema payload")
|
||||
}
|
||||
schema.JSONSchema[0] = 'x'
|
||||
|
||||
again, ok := Lookup(CorrectionSetKey)
|
||||
if !ok {
|
||||
t.Fatalf("missing correction-set schema on second lookup")
|
||||
}
|
||||
if len(again.JSONSchema) == 0 {
|
||||
t.Fatalf("unexpected empty schema payload")
|
||||
}
|
||||
if again.JSONSchema[0] != '{' {
|
||||
t.Fatalf("expected lookup to return independent schema copy")
|
||||
}
|
||||
}
|
||||
@@ -620,9 +620,10 @@ func (a validationLLMClientAdapter) CompleteStructured(ctx context.Context, req
|
||||
messages[i] = contracts.LLMMessage{Role: m.Role, Content: m.Content}
|
||||
}
|
||||
resp, err := a.client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
StageName: req.StageName,
|
||||
Messages: messages,
|
||||
Model: req.Model,
|
||||
StageName: req.StageName,
|
||||
Messages: messages,
|
||||
Model: req.Model,
|
||||
ResponseSchema: req.ResponseSchema,
|
||||
}, out)
|
||||
if err != nil {
|
||||
return validators.StructuredCompletionResponse{}, err
|
||||
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/modules"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||
)
|
||||
|
||||
@@ -639,6 +640,7 @@ func TestRunnerMixedProposalValidationFIFOOrder(t *testing.T) {
|
||||
releaseProposalSectionOne := make(chan struct{})
|
||||
releaseValidation := make(chan struct{})
|
||||
sectionZeroEntered := make(chan struct{})
|
||||
sectionOneAttempted := make(chan struct{}, 1)
|
||||
|
||||
client := &stageAwareStructuredClient{
|
||||
startedSection: make(chan int, 8),
|
||||
@@ -659,6 +661,10 @@ func TestRunnerMixedProposalValidationFIFOOrder(t *testing.T) {
|
||||
seg := req.WorkingTranscript.Segments[0]
|
||||
if req.Section != nil && req.Section.Index == 1 {
|
||||
<-sectionZeroEntered
|
||||
select {
|
||||
case sectionOneAttempted <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
err := req.LLMScheduler.Run(context.Background(), func(context.Context) error {
|
||||
if req.Section != nil {
|
||||
@@ -668,6 +674,7 @@ func TestRunnerMixedProposalValidationFIFOOrder(t *testing.T) {
|
||||
case sectionZeroEntered <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
<-sectionOneAttempted
|
||||
}
|
||||
if req.Section.Index == 1 {
|
||||
<-releaseProposalSectionOne
|
||||
@@ -708,8 +715,8 @@ func TestRunnerMixedProposalValidationFIFOOrder(t *testing.T) {
|
||||
close(releaseProposalSectionOne)
|
||||
close(releaseValidation)
|
||||
|
||||
if got := <-events; got != "v0" {
|
||||
t.Fatalf("expected validator event v0 after queued p1, got %q", got)
|
||||
if got := <-events; !strings.HasPrefix(got, "v") {
|
||||
t.Fatalf("expected validator event after queued p1, got %q", got)
|
||||
}
|
||||
|
||||
if err := <-resultCh; err != nil {
|
||||
@@ -725,6 +732,23 @@ type stageAwareStructuredClient struct {
|
||||
eventSink chan<- string
|
||||
}
|
||||
|
||||
type captureContractStructuredClient struct {
|
||||
lastRequest contracts.StructuredCompletionRequest
|
||||
}
|
||||
|
||||
func (c *captureContractStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
_ = ctx
|
||||
c.lastRequest = req
|
||||
if target, ok := out.(*validators.LLMValidationResponse); ok {
|
||||
*target = validators.LLMValidationResponse{
|
||||
Validations: []validators.LLMValidationDecision{
|
||||
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
|
||||
},
|
||||
}
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{}, nil
|
||||
}
|
||||
|
||||
func (c *stageAwareStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
section := parseSectionFromStage(req.StageName)
|
||||
if c.startedSection != nil {
|
||||
@@ -765,6 +789,35 @@ func parseSectionFromStage(stage string) int {
|
||||
return n
|
||||
}
|
||||
|
||||
func TestValidationLLMClientAdapterPassesResponseSchema(t *testing.T) {
|
||||
capture := &captureContractStructuredClient{}
|
||||
adapter := validationLLMClientAdapter{client: capture}
|
||||
schema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
|
||||
|
||||
_, err := adapter.CompleteStructured(context.Background(), validators.StructuredCompletionRequest{
|
||||
StageName: "module:validator:batch-0000",
|
||||
Messages: []validators.LLMMessage{
|
||||
{Role: "system", Content: "system"},
|
||||
{Role: "user", Content: "user"},
|
||||
},
|
||||
Model: "test-model",
|
||||
ResponseSchema: &schema,
|
||||
}, &validators.LLMValidationResponse{})
|
||||
if err != nil {
|
||||
t.Fatalf("CompleteStructured error: %v", err)
|
||||
}
|
||||
|
||||
if capture.lastRequest.ResponseSchema == nil {
|
||||
t.Fatalf("expected response schema to be forwarded")
|
||||
}
|
||||
if capture.lastRequest.ResponseSchema.ID != schema.ID ||
|
||||
capture.lastRequest.ResponseSchema.Version != schema.Version ||
|
||||
capture.lastRequest.ResponseSchema.Name != schema.Name ||
|
||||
capture.lastRequest.ResponseSchema.SHA256 != schema.SHA256 {
|
||||
t.Fatalf("unexpected forwarded schema metadata: got=%+v want=%+v", *capture.lastRequest.ResponseSchema, schema)
|
||||
}
|
||||
}
|
||||
|
||||
type trackingScheduler struct {
|
||||
inner contracts.LLMScheduler
|
||||
inFlight int32
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
|
||||
)
|
||||
|
||||
type LLMValidatorType string
|
||||
@@ -22,9 +23,10 @@ type LLMMessage struct {
|
||||
}
|
||||
|
||||
type StructuredCompletionRequest struct {
|
||||
StageName string `json:"stage_name"`
|
||||
Messages []LLMMessage `json:"messages"`
|
||||
Model string `json:"model,omitempty"`
|
||||
StageName string `json:"stage_name"`
|
||||
Messages []LLMMessage `json:"messages"`
|
||||
Model string `json:"model,omitempty"`
|
||||
ResponseSchema *responseschema.Schema `json:"response_schema,omitempty"`
|
||||
}
|
||||
|
||||
type StructuredCompletionResponse struct {
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
|
||||
)
|
||||
|
||||
type LLMPromptBuilder func(validationPayload []LLMValidationItem) ([]LLMMessage, error)
|
||||
@@ -89,11 +90,13 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
}
|
||||
|
||||
var response LLMValidationResponse
|
||||
responseSchema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
|
||||
call := func(callCtx context.Context) error {
|
||||
_, err = req.LLMClient.CompleteStructured(callCtx, StructuredCompletionRequest{
|
||||
StageName: fmt.Sprintf("%s:%s:batch-%04d", req.ModuleInstance, v.name, batch.BatchIndex),
|
||||
Messages: messages,
|
||||
Model: resolvedValidationModel(req.Config, v.model),
|
||||
StageName: fmt.Sprintf("%s:%s:batch-%04d", req.ModuleInstance, v.name, batch.BatchIndex),
|
||||
Messages: messages,
|
||||
Model: resolvedValidationModel(req.Config, v.model),
|
||||
ResponseSchema: &responseSchema,
|
||||
}, &response)
|
||||
return err
|
||||
}
|
||||
@@ -107,7 +110,17 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
stage := fmt.Sprintf("%s:%s:batch-%04d", req.ModuleInstance, v.name, batch.BatchIndex)
|
||||
artifacts, _ = req.DiagnosticsWriter.WriteInteraction(
|
||||
stage,
|
||||
map[string]any{"validator_name": v.name, "validator_type": v.validatorType, "batch_index": batch.BatchIndex},
|
||||
map[string]any{
|
||||
"validator_name": v.name,
|
||||
"validator_type": v.validatorType,
|
||||
"batch_index": batch.BatchIndex,
|
||||
"response_schema": map[string]any{
|
||||
"id": responseSchema.ID,
|
||||
"version": responseSchema.Version,
|
||||
"name": responseSchema.Name,
|
||||
"sha256": responseSchema.SHA256,
|
||||
},
|
||||
},
|
||||
map[string]any{"messages": messages, "items": batch.Items},
|
||||
response,
|
||||
errPayload(err),
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
|
||||
)
|
||||
|
||||
type fakeStructuredLLMClient struct {
|
||||
@@ -33,6 +34,19 @@ type boundedScheduler struct {
|
||||
permits chan struct{}
|
||||
}
|
||||
|
||||
type captureValidationDiagnosticsWriter struct {
|
||||
lastRequestMetadata any
|
||||
}
|
||||
|
||||
func (w *captureValidationDiagnosticsWriter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) {
|
||||
_ = stage
|
||||
_ = requestPayload
|
||||
_ = responsePayload
|
||||
_ = errorPayload
|
||||
w.lastRequestMetadata = requestMetadata
|
||||
return InteractionArtifacts{}, nil
|
||||
}
|
||||
|
||||
func newBoundedScheduler(max int) *boundedScheduler {
|
||||
return &boundedScheduler{permits: make(chan struct{}, max)}
|
||||
}
|
||||
@@ -200,6 +214,17 @@ func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
|
||||
if len(res.Decisions) != 2 || !res.Decisions[0].Approved || res.Decisions[1].Approved {
|
||||
t.Fatalf("unexpected decisions: %+v", res.Decisions)
|
||||
}
|
||||
if len(client.calls) != 1 {
|
||||
t.Fatalf("expected one LLM call, got %d", len(client.calls))
|
||||
}
|
||||
if client.calls[0].ResponseSchema == nil {
|
||||
t.Fatalf("expected response schema on structured validation request")
|
||||
}
|
||||
want := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
|
||||
gotSchema := client.calls[0].ResponseSchema
|
||||
if gotSchema.ID != want.ID || gotSchema.Version != want.Version || gotSchema.Name != want.Name || gotSchema.SHA256 != want.SHA256 {
|
||||
t.Fatalf("unexpected validator response schema metadata: got=%+v want=%+v", *gotSchema, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMBackedValidatorMalformedOutputFails(t *testing.T) {
|
||||
@@ -285,6 +310,36 @@ func TestLLMBackedValidatorRespectsSchedulerConcurrency(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMBackedValidatorDiagnosticsIncludeSchemaMetadata(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
|
||||
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
|
||||
}}}}
|
||||
writer := &captureValidationDiagnosticsWriter{}
|
||||
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
|
||||
if err != nil {
|
||||
t.Fatalf("new validator error: %v", err)
|
||||
}
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
||||
req.LLMClient = client
|
||||
req.DiagnosticsWriter = writer
|
||||
_, err = v.Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("validate error: %v", err)
|
||||
}
|
||||
metadata, ok := writer.lastRequestMetadata.(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected metadata map, got %T", writer.lastRequestMetadata)
|
||||
}
|
||||
schemaMap, ok := metadata["response_schema"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected response_schema map, got %T", metadata["response_schema"])
|
||||
}
|
||||
want := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
|
||||
if schemaMap["id"] != want.ID || schemaMap["version"] != want.Version || schemaMap["name"] != want.Name || schemaMap["sha256"] != want.SHA256 {
|
||||
t.Fatalf("unexpected diagnostics schema metadata: got=%v want=%+v", schemaMap, want)
|
||||
}
|
||||
}
|
||||
|
||||
func waitForValidationEntries(t *testing.T, entered <-chan struct{}, want int) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(300 * time.Millisecond)
|
||||
|
||||
Reference in New Issue
Block a user