Use PromptKit prepared execution for structured model calls
This commit is contained in:
138
internal/framework/llm/adapter.go
Normal file
138
internal/framework/llm/adapter.go
Normal file
@@ -0,0 +1,138 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||
pk "gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
// CallHooks receive one logical start and finish record and the execution
|
||||
// events for this call. Call records can contain transcript content.
|
||||
type CallHooks struct {
|
||||
OnCall func(structuredoutput.CallRecord)
|
||||
OnEvent pk.ExecutionObserver
|
||||
}
|
||||
|
||||
type RoleClient struct {
|
||||
runtime *Runtime
|
||||
role string
|
||||
hooks CallHooks
|
||||
}
|
||||
|
||||
var _ structuredoutput.Client = (*RoleClient)(nil)
|
||||
|
||||
func (r *Runtime) NewRoleClient(role string, hooks CallHooks) (*RoleClient, error) {
|
||||
if _, ok := r.Profiles[role]; !ok {
|
||||
return nil, fmt.Errorf("unknown LLM role %q", role)
|
||||
}
|
||||
return &RoleClient{runtime: r, role: role, hooks: hooks}, nil
|
||||
}
|
||||
|
||||
func (c *RoleClient) CompleteStructured(parent context.Context, req structuredoutput.Request, out any) (response structuredoutput.Response, err error) {
|
||||
ctx, cancel := context.WithTimeout(parent, c.runtime.Config.LLM.OperationTimeout)
|
||||
defer cancel()
|
||||
record := structuredoutput.CallRecord{Context: req.Context, Started: time.Now().UTC()}
|
||||
record.Context.Role = c.role
|
||||
if c.hooks.OnCall != nil {
|
||||
c.hooks.OnCall(record)
|
||||
}
|
||||
defer func() {
|
||||
record.Finished = time.Now().UTC()
|
||||
if err != nil {
|
||||
if structuredoutput.IsMalformedError(err) {
|
||||
record.Failure = structuredoutput.FailureMalformedOutput
|
||||
} else {
|
||||
record.Failure = structuredoutput.FailureRuntime
|
||||
}
|
||||
}
|
||||
if c.hooks.OnCall != nil {
|
||||
c.hooks.OnCall(record)
|
||||
}
|
||||
}()
|
||||
if err = validateOutputTarget(out); err != nil {
|
||||
return response, &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "invalid structured output target", Cause: err}
|
||||
}
|
||||
schemaPath, ok := responseschema.AssetPath(req.Schema)
|
||||
if !ok {
|
||||
return response, &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "unknown structured response schema", Cause: fmt.Errorf("schema %q", req.Schema)}
|
||||
}
|
||||
profile := c.runtime.Profiles[c.role]
|
||||
backend := c.runtime.Config.Backends[profile.EffectiveModelParams.BackendID]
|
||||
vars := make(map[string]string, len(req.Variables)+1)
|
||||
for key, value := range req.Variables {
|
||||
vars[key] = value
|
||||
}
|
||||
vars["HardeningPolicy"] = prompts.HardeningText()
|
||||
apiKey := ""
|
||||
if backend.Credential != nil {
|
||||
apiKey = backend.Credential.Value()
|
||||
}
|
||||
handle, prepErr := c.runtime.Engine.PrepareExecution(ctx, pk.RunRequest{
|
||||
PromptID: req.PromptID, PromptVersion: req.PromptVersion,
|
||||
ProfileID: profile.ProfileID, APIKey: apiKey, Vars: vars,
|
||||
Validation: &pk.OutputContract{Format: pk.FormatJSON, ValidationMode: pk.ValidationJSONSchema, SchemaPath: schemaPath, RepairAttempts: c.runtime.Config.LLM.RepairAttempts},
|
||||
Observer: c.hooks.OnEvent,
|
||||
})
|
||||
if prepErr != nil {
|
||||
return response, &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "could not prepare structured model call", Cause: prepErr}
|
||||
}
|
||||
defer handle.Discard()
|
||||
details := handle.Details()
|
||||
response.Prompt = promptMetadata(details)
|
||||
response.Model = details.EffectiveModelParams.Model
|
||||
response.Prepared = preparedSnapshot(details)
|
||||
record.Prepared = response.Prepared
|
||||
selected, exists := c.runtime.Config.Backends[details.SelectedBackendID]
|
||||
endpoint := func(s string) string { return strings.TrimRight(strings.TrimSpace(s), "/") }
|
||||
if !exists || details.SelectedProfileID != profile.ProfileID || details.SelectedBackendID != profile.EffectiveModelParams.BackendID || endpoint(details.EffectiveModelParams.Endpoint) != endpoint(selected.Endpoint) || details.EffectiveModelParams.APIKeyEnv != "" {
|
||||
return response, &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "structured model target changed after configuration inspection", Cause: fmt.Errorf("prepared target does not match registered role backend")}
|
||||
}
|
||||
result, runErr := c.runtime.Engine.RunPrepared(ctx, handle)
|
||||
if result != nil {
|
||||
response.FinalContent = result.RawOutput
|
||||
record.FinalContent = result.RawOutput
|
||||
response.Model = result.ModelName
|
||||
response.PromptTokens = result.Usage.PromptTokens
|
||||
response.CompletionTokens = result.Usage.CompletionTokens
|
||||
response.TotalTokens = result.Usage.TotalTokens
|
||||
}
|
||||
if runErr != nil {
|
||||
return response, &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "structured model call failed", Cause: runErr}
|
||||
}
|
||||
if result == nil {
|
||||
return response, &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "structured model call returned no result"}
|
||||
}
|
||||
if !result.Validation.IsValid {
|
||||
return response, &structuredoutput.Failure{Kind: structuredoutput.FailureMalformedOutput, Message: "malformed structured output"}
|
||||
}
|
||||
if decodeErr := json.Unmarshal([]byte(result.RawOutput), out); decodeErr != nil {
|
||||
return response, &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "validated structured output could not be decoded", Cause: decodeErr}
|
||||
}
|
||||
response.Content = json.RawMessage(result.RawOutput)
|
||||
response.Provider = "openai-compatible"
|
||||
return response, nil
|
||||
}
|
||||
|
||||
func promptMetadata(details pk.PreparedRun) structuredoutput.PromptMetadata {
|
||||
return structuredoutput.PromptMetadata{
|
||||
PromptID: details.PromptID, PromptVersion: details.PromptVersion,
|
||||
DefinitionHash: details.PromptHash, RenderedHash: details.RenderedPromptHash,
|
||||
ProfileID: details.SelectedProfileID, BackendID: details.SelectedBackendID,
|
||||
Model: details.EffectiveModelParams.Model,
|
||||
}
|
||||
}
|
||||
|
||||
func preparedSnapshot(details pk.PreparedRun) *structuredoutput.PreparedSnapshot {
|
||||
messages := make([]structuredoutput.Message, len(details.Messages))
|
||||
for i, message := range details.Messages {
|
||||
messages[i] = structuredoutput.Message{Role: string(message.Role), Content: message.Content}
|
||||
}
|
||||
return &structuredoutput.PreparedSnapshot{Messages: messages, Prompt: promptMetadata(details)}
|
||||
}
|
||||
257
internal/framework/llm/adapter_test.go
Normal file
257
internal/framework/llm/adapter_test.go
Normal file
@@ -0,0 +1,257 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||
pk "gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
type reply struct {
|
||||
status int
|
||||
content string
|
||||
}
|
||||
|
||||
func adapterFixture(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request)) (*Runtime, *int, string) {
|
||||
t.Helper()
|
||||
count := 0
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
count++
|
||||
if hook != nil {
|
||||
hook(r)
|
||||
}
|
||||
step := replies[len(replies)-1]
|
||||
if count <= len(replies) {
|
||||
step = replies[count-1]
|
||||
}
|
||||
if step.status == 0 {
|
||||
connection, _, hijackErr := w.(http.Hijacker).Hijack()
|
||||
if hijackErr != nil {
|
||||
t.Error(hijackErr)
|
||||
return
|
||||
}
|
||||
_ = connection.Close()
|
||||
return
|
||||
}
|
||||
if step.status != http.StatusOK {
|
||||
w.WriteHeader(step.status)
|
||||
_, _ = fmt.Fprint(w, `{"error":{"message":"provider error"}}`)
|
||||
return
|
||||
}
|
||||
if step.content == "MALFORMED_ENVELOPE" {
|
||||
_, _ = fmt.Fprint(w, `{"choices":[]}`)
|
||||
return
|
||||
}
|
||||
encoded, _ := json.Marshal(step.content)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = fmt.Fprintf(w, `{"model":"returned-model","choices":[{"message":{"content":%s}}],"usage":{"prompt_tokens":2,"completion_tokens":3,"total_tokens":5}}`, encoded)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
t.Setenv("ADAPTER_KEY", "adapter-secret")
|
||||
base := fmt.Sprintf("version: 2\nbackends:\n offline:\n endpoint: %s/v1\n credential: ADAPTER_KEY\nllm:\n proposal_profile: base\n transient_retries: %d\n repair_attempts: %d\n", server.URL, retries, repairs)
|
||||
dir := profileLayout(t, base, "")
|
||||
main := filepath.Join(dir, "config.yml")
|
||||
profile := "id: base\nbackend: offline\nmodel: full-model\n"
|
||||
if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte(profile), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err := config.Load(ck.File(main), "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runtime, err := NewRuntime(context.Background(), loaded.Config, main, server.Client())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return runtime, &count, dir
|
||||
}
|
||||
|
||||
func adapterRequest(schema responseschema.Key) structuredoutput.Request {
|
||||
id := prompts.PromptIDModuleGlossaryProposal
|
||||
vars := map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"}
|
||||
if schema == responseschema.ValidatorDecisionSetKey {
|
||||
id = prompts.PromptIDValidatorEditorialReview
|
||||
vars = map[string]string{"TranscriptDescriptionBlock": "", "PayloadJSON": "[]"}
|
||||
}
|
||||
return structuredoutput.Request{PromptID: id, PromptVersion: prompts.VersionV1, Variables: vars, Schema: schema, Context: structuredoutput.DiagnosticContext{ModuleKey: "glossary"}}
|
||||
}
|
||||
|
||||
func TestPreparedAdapterRepairsAndClassifies(t *testing.T) {
|
||||
for _, schema := range []responseschema.Key{responseschema.CorrectionSetKey, responseschema.ValidatorDecisionSetKey} {
|
||||
valid := `{"corrections":[]}`
|
||||
if schema == responseschema.ValidatorDecisionSetKey {
|
||||
valid = `{"validations":[]}`
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
replies []reply
|
||||
retries, repairs, attempts, repairCalls, transientRetries int
|
||||
malformed bool
|
||||
cause error
|
||||
}{
|
||||
{"success", []reply{{200, valid}}, 0, 0, 1, 0, 0, false, nil},
|
||||
{"repair succeeds", []reply{{200, `{"wrong":true}`}, {200, valid}}, 0, 1, 2, 1, 0, false, nil},
|
||||
{"repair exhausted", []reply{{200, `{"wrong":true}`}, {200, `{"wrong":true}`}}, 0, 1, 2, 1, 0, true, nil},
|
||||
{"429 recovery", []reply{{429, ""}, {200, valid}}, 1, 0, 2, 0, 1, false, nil},
|
||||
{"5xx recovery", []reply{{503, ""}, {200, valid}}, 1, 0, 2, 0, 1, false, nil},
|
||||
{"zero retry", []reply{{503, ""}}, 0, 0, 1, 0, 0, false, pk.ErrLLMGenerate},
|
||||
{"positive retry exhausted", []reply{{503, ""}, {503, ""}}, 1, 0, 2, 0, 1, false, pk.ErrRetriesExhausted},
|
||||
{"transport failure", []reply{{0, ""}}, 0, 0, 1, 0, 0, false, pk.ErrTransport},
|
||||
{"malformed envelope", []reply{{200, "MALFORMED_ENVELOPE"}}, 0, 0, 1, 0, 0, false, pk.ErrMalformedResponse},
|
||||
{"retry and repair", []reply{{503, ""}, {200, `{"wrong":true}`}, {503, ""}, {200, valid}}, 1, 1, 4, 1, 2, false, nil},
|
||||
} {
|
||||
t.Run(string(schema)+"/"+tc.name, func(t *testing.T) {
|
||||
runtime, count, _ := adapterFixture(t, tc.replies, tc.retries, tc.repairs, nil)
|
||||
role := "proposal"
|
||||
if schema == responseschema.ValidatorDecisionSetKey {
|
||||
role = "validation"
|
||||
}
|
||||
var events []pk.ExecutionEvent
|
||||
var records []structuredoutput.CallRecord
|
||||
client, err := runtime.NewRoleClient(role, CallHooks{OnEvent: func(e pk.ExecutionEvent) { events = append(events, e) }, OnCall: func(r structuredoutput.CallRecord) { records = append(records, r) }})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out map[string]any
|
||||
response, err := client.CompleteStructured(context.Background(), adapterRequest(schema), &out)
|
||||
if *count != tc.attempts {
|
||||
t.Fatalf("HTTP attempts=%d, want %d", *count, tc.attempts)
|
||||
}
|
||||
if tc.malformed != structuredoutput.IsMalformedError(err) {
|
||||
t.Fatalf("malformed classification: %v", err)
|
||||
}
|
||||
if tc.cause != nil && !errors.Is(err, tc.cause) {
|
||||
t.Fatalf("cause %v missing from %v", tc.cause, err)
|
||||
}
|
||||
if tc.cause == nil && !tc.malformed && err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tc.name == "zero retry" && errors.Is(err, pk.ErrRetriesExhausted) {
|
||||
t.Fatal("zero retry reported exhaustion")
|
||||
}
|
||||
if response.Prepared == nil || response.Prompt.DefinitionHash == "" || response.Prompt.RenderedHash == "" || response.Prompt.ProfileID != "base" || response.Prompt.BackendID != "offline" {
|
||||
t.Fatalf("prepared metadata missing: %+v", response)
|
||||
}
|
||||
if len(records) != 2 || records[0].Finished.IsZero() == false || records[1].Finished.IsZero() || records[1].Context.Role != role {
|
||||
t.Fatalf("logical records: %+v", records)
|
||||
}
|
||||
if tc.malformed && records[1].FinalContent == "" {
|
||||
t.Fatal("invalid final content lost")
|
||||
}
|
||||
if len(events) == 0 || events[len(events)-1].Kind != pk.ExecutionFinished || int(events[len(events)-1].RepairCalls) != tc.repairCalls || int(events[len(events)-1].TransientRetries) != tc.transientRetries {
|
||||
t.Fatalf("execution counts: %+v", events)
|
||||
}
|
||||
if err == nil && (out == nil || string(response.Content) != valid || response.TotalTokens != 5*(tc.repairCalls+1) || response.Model != "full-model") {
|
||||
t.Fatalf("decoded response: %+v, %v", response, out)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedAdapterMetadataAndTargetGuard(t *testing.T) {
|
||||
requestSeen := false
|
||||
var posted struct {
|
||||
Messages []structuredoutput.Message `json:"messages"`
|
||||
}
|
||||
runtime, count, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, func(r *http.Request) {
|
||||
requestSeen = true
|
||||
if r.Header.Get("Authorization") != "Bearer adapter-secret" {
|
||||
t.Error("backend credential not used")
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&posted); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
})
|
||||
client, err := runtime.NewRoleClient("proposal", CallHooks{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req := adapterRequest(responseschema.CorrectionSetKey)
|
||||
req.Variables["HardeningPolicy"] = "attacker-controlled"
|
||||
var out map[string]any
|
||||
response, err := client.CompleteStructured(context.Background(), req, &out)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !requestSeen || *count != 1 || response.Prepared == nil || len(response.Prepared.Messages) == 0 {
|
||||
t.Fatal("prepared call missing")
|
||||
}
|
||||
if len(posted.Messages) != len(response.Prepared.Messages) {
|
||||
t.Fatal("prepared and dispatched message counts differ")
|
||||
}
|
||||
for i := range posted.Messages {
|
||||
if posted.Messages[i] != response.Prepared.Messages[i] {
|
||||
t.Fatal("prepared messages differ from dispatched request")
|
||||
}
|
||||
}
|
||||
found := false
|
||||
for _, msg := range response.Prepared.Messages {
|
||||
if strings.Contains(msg.Content, prompts.HardeningText()) {
|
||||
found = true
|
||||
}
|
||||
if strings.Contains(msg.Content, "attacker-controlled") {
|
||||
t.Fatal("reserved hardening variable was overridden")
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("hardening policy missing")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedAdapterRejectsTargetDriftAndPreparationErrors(t *testing.T) {
|
||||
runtime, count, dir := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, nil)
|
||||
client, err := runtime.NewRoleClient("proposal", CallHooks{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req := adapterRequest(responseschema.CorrectionSetKey)
|
||||
req.PromptID = "missing.prompt"
|
||||
var out map[string]any
|
||||
if _, err := client.CompleteStructured(context.Background(), req, &out); err == nil || *count != 0 {
|
||||
t.Fatalf("preparation failure dispatched: %v, %d", err, *count)
|
||||
}
|
||||
// PrepareExecution reloads the profile, so a changed target must be blocked.
|
||||
if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte("id: base\nbackend: offline\nendpoint: http://127.0.0.1:1/v1\nmodel: full-model\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req = adapterRequest(responseschema.CorrectionSetKey)
|
||||
_, err = client.CompleteStructured(context.Background(), req, &out)
|
||||
if err == nil || !strings.Contains(err.Error(), "target changed") || *count != 0 {
|
||||
t.Fatalf("target drift dispatched: %v, %d", err, *count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedAdapterCancellationStopsRetryWait(t *testing.T) {
|
||||
runtime, count, _ := adapterFixture(t, []reply{{503, ""}}, 2, 0, nil)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
var once sync.Once
|
||||
client, err := runtime.NewRoleClient("proposal", CallHooks{OnEvent: func(e pk.ExecutionEvent) {
|
||||
if e.Kind == pk.ExecutionAttempt && e.Dispatched {
|
||||
once.Do(cancel)
|
||||
}
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out map[string]any
|
||||
_, err = client.CompleteStructured(ctx, adapterRequest(responseschema.CorrectionSetKey), &out)
|
||||
if !errors.Is(err, context.Canceled) || *count != 1 {
|
||||
t.Fatalf("cancellation: %v, HTTP calls=%d", err, *count)
|
||||
}
|
||||
}
|
||||
@@ -2,29 +2,9 @@ package llm
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func resolvedHTTPClient(base *http.Client, timeout time.Duration) *http.Client {
|
||||
if base == nil {
|
||||
if timeout <= 0 {
|
||||
return http.DefaultClient
|
||||
}
|
||||
return &http.Client{Timeout: timeout}
|
||||
}
|
||||
|
||||
if timeout <= 0 {
|
||||
return base
|
||||
}
|
||||
|
||||
cloned := *base
|
||||
cloned.Timeout = timeout
|
||||
return &cloned
|
||||
}
|
||||
|
||||
func validateOutputTarget(out any) error {
|
||||
if out == nil {
|
||||
return fmt.Errorf("output target must not be nil")
|
||||
@@ -35,16 +15,3 @@ func validateOutputTarget(out any) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sanitizeError(err error, apiKey string) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
msg := err.Error()
|
||||
key := strings.TrimSpace(apiKey)
|
||||
if key != "" {
|
||||
msg = strings.ReplaceAll(msg, key, "[REDACTED]")
|
||||
msg = strings.ReplaceAll(msg, "Bearer "+key, "Bearer [REDACTED]")
|
||||
}
|
||||
return fmt.Errorf("%s", msg)
|
||||
}
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
)
|
||||
|
||||
// EffectiveConfig is the runtime LLM configuration consumed by adapters and
|
||||
// schedulers after primary/validation inheritance is resolved.
|
||||
type EffectiveConfig struct {
|
||||
APIKey string
|
||||
Model string
|
||||
BaseURL string
|
||||
RequestTimeout time.Duration
|
||||
MaxRetries int
|
||||
Concurrency int
|
||||
}
|
||||
|
||||
// ResolvePrimaryConfig resolves the primary LLM settings into runtime form.
|
||||
func ResolvePrimaryConfig(cfg config.Config) EffectiveConfig {
|
||||
return resolveFromLLMConfig(cfg.PrimaryLLM, cfg.TotalLLMConcurrency)
|
||||
}
|
||||
|
||||
// ResolveValidationConfig resolves validation LLM settings with inheritance
|
||||
// from primary values when validation overrides are unset.
|
||||
func ResolveValidationConfig(cfg config.Config) EffectiveConfig {
|
||||
return resolveFromLLMConfig(cfg.EffectiveValidationLLMConfig(), cfg.EffectiveValidationLLMConcurrency())
|
||||
}
|
||||
|
||||
// ToOpenAICompatibleClientConfig converts an effective runtime config into
|
||||
// direct HTTP adapter config.
|
||||
func (c EffectiveConfig) ToOpenAICompatibleClientConfig(httpClient *http.Client) OpenAICompatibleClientConfig {
|
||||
return OpenAICompatibleClientConfig{
|
||||
BaseURL: c.BaseURL,
|
||||
Model: c.Model,
|
||||
APIKey: c.APIKey,
|
||||
MaxRetries: c.MaxRetries,
|
||||
HTTPClient: httpClient,
|
||||
RequestTimeout: c.RequestTimeout,
|
||||
}
|
||||
}
|
||||
|
||||
func resolveFromLLMConfig(raw config.LLMConfig, concurrency int) EffectiveConfig {
|
||||
return EffectiveConfig{
|
||||
APIKey: strings.TrimSpace(raw.APIKey),
|
||||
Model: strings.TrimSpace(raw.Model),
|
||||
BaseURL: strings.TrimSpace(raw.BaseURL),
|
||||
RequestTimeout: time.Duration(raw.TimeoutSeconds) * time.Second,
|
||||
MaxRetries: raw.MaxRetries,
|
||||
Concurrency: concurrency,
|
||||
}
|
||||
}
|
||||
@@ -1,106 +0,0 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
)
|
||||
|
||||
func TestResolvePrimaryConfig(t *testing.T) {
|
||||
cfg := config.Default()
|
||||
cfg.PrimaryLLM.APIKey = " primary-key "
|
||||
cfg.PrimaryLLM.Model = " model-x "
|
||||
cfg.PrimaryLLM.BaseURL = " https://example.test/v1 "
|
||||
cfg.PrimaryLLM.TimeoutSeconds = 42
|
||||
cfg.PrimaryLLM.MaxRetries = 7
|
||||
cfg.TotalLLMConcurrency = 3
|
||||
cfg.ProposalLLMConcurrency = 3
|
||||
|
||||
effective := ResolvePrimaryConfig(cfg)
|
||||
if effective.APIKey != "primary-key" {
|
||||
t.Fatalf("unexpected api key: %q", effective.APIKey)
|
||||
}
|
||||
if effective.Model != "model-x" {
|
||||
t.Fatalf("unexpected model: %q", effective.Model)
|
||||
}
|
||||
if effective.BaseURL != "https://example.test/v1" {
|
||||
t.Fatalf("unexpected base url: %q", effective.BaseURL)
|
||||
}
|
||||
if effective.RequestTimeout != 42*time.Second {
|
||||
t.Fatalf("unexpected timeout: %s", effective.RequestTimeout)
|
||||
}
|
||||
if effective.MaxRetries != 7 {
|
||||
t.Fatalf("unexpected max retries: %d", effective.MaxRetries)
|
||||
}
|
||||
if effective.Concurrency != 3 {
|
||||
t.Fatalf("unexpected concurrency: %d", effective.Concurrency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveValidationConfigInheritsPrimaryWhenUnset(t *testing.T) {
|
||||
cfg := config.Default()
|
||||
cfg.PrimaryLLM.APIKey = "primary-key"
|
||||
cfg.PrimaryLLM.Model = "primary-model"
|
||||
cfg.PrimaryLLM.BaseURL = "https://primary.example/v1"
|
||||
cfg.PrimaryLLM.TimeoutSeconds = 90
|
||||
cfg.PrimaryLLM.MaxRetries = 4
|
||||
cfg.TotalLLMConcurrency = 2
|
||||
cfg.ProposalLLMConcurrency = 2
|
||||
|
||||
effective := ResolveValidationConfig(cfg)
|
||||
if effective.APIKey != "primary-key" ||
|
||||
effective.Model != "primary-model" ||
|
||||
effective.BaseURL != "https://primary.example/v1" ||
|
||||
effective.RequestTimeout != 90*time.Second ||
|
||||
effective.MaxRetries != 4 ||
|
||||
effective.Concurrency != 2 {
|
||||
t.Fatalf("unexpected inherited config: %+v", effective)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveValidationConfigOverridesPrimaryWhenSet(t *testing.T) {
|
||||
cfg := config.Default()
|
||||
cfg.PrimaryLLM.APIKey = "primary-key"
|
||||
cfg.PrimaryLLM.Model = "primary-model"
|
||||
cfg.PrimaryLLM.BaseURL = "https://primary.example/v1"
|
||||
cfg.PrimaryLLM.TimeoutSeconds = 90
|
||||
cfg.PrimaryLLM.MaxRetries = 4
|
||||
cfg.TotalLLMConcurrency = 2
|
||||
cfg.ProposalLLMConcurrency = 2
|
||||
|
||||
timeout := 12
|
||||
retries := 9
|
||||
concurrency := 6
|
||||
cfg.ValidationLLM.APIKey = "validation-key"
|
||||
cfg.ValidationLLM.Model = "validation-model"
|
||||
cfg.ValidationLLM.BaseURL = "https://validation.example/v1"
|
||||
cfg.ValidationLLM.TimeoutSeconds = &timeout
|
||||
cfg.ValidationLLM.MaxRetries = &retries
|
||||
cfg.ValidationLLMConcurrency = &concurrency
|
||||
|
||||
effective := ResolveValidationConfig(cfg)
|
||||
if effective.APIKey != "validation-key" ||
|
||||
effective.Model != "validation-model" ||
|
||||
effective.BaseURL != "https://validation.example/v1" ||
|
||||
effective.RequestTimeout != 12*time.Second ||
|
||||
effective.MaxRetries != 9 ||
|
||||
effective.Concurrency != 6 {
|
||||
t.Fatalf("unexpected override config: %+v", effective)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveConfigOptionalAPIKey(t *testing.T) {
|
||||
cfg := config.Default()
|
||||
cfg.PrimaryLLM.APIKey = ""
|
||||
cfg.ValidationLLM.APIKey = ""
|
||||
|
||||
primary := ResolvePrimaryConfig(cfg)
|
||||
validation := ResolveValidationConfig(cfg)
|
||||
if primary.APIKey != "" {
|
||||
t.Fatalf("expected empty primary api key, got %q", primary.APIKey)
|
||||
}
|
||||
if validation.APIKey != "" {
|
||||
t.Fatalf("expected empty validation api key, got %q", validation.APIKey)
|
||||
}
|
||||
}
|
||||
@@ -1,370 +0,0 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
)
|
||||
|
||||
const defaultOpenAICompatibleMaxRetries = 3
|
||||
|
||||
// OpenAICompatibleClientConfig configures the direct HTTP structured-output adapter.
|
||||
type OpenAICompatibleClientConfig struct {
|
||||
BaseURL string
|
||||
Model string
|
||||
APIKey string
|
||||
MaxRetries int
|
||||
HTTPClient *http.Client
|
||||
RequestTimeout time.Duration
|
||||
}
|
||||
|
||||
// OpenAICompatibleClient sends OpenAI-compatible chat-completion requests with
|
||||
// response_format.type=json_schema.
|
||||
type OpenAICompatibleClient struct {
|
||||
cfg OpenAICompatibleClientConfig
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
var _ contracts.StructuredLLMClient = (*OpenAICompatibleClient)(nil)
|
||||
|
||||
func NewOpenAICompatibleClient(cfg OpenAICompatibleClientConfig) (*OpenAICompatibleClient, error) {
|
||||
normalized, err := normalizeOpenAICompatibleConfig(cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &OpenAICompatibleClient{
|
||||
cfg: normalized,
|
||||
httpClient: resolvedHTTPClient(normalized.HTTPClient, normalized.RequestTimeout),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *OpenAICompatibleClient) CompleteStructured(
|
||||
ctx context.Context,
|
||||
req contracts.StructuredCompletionRequest,
|
||||
out any,
|
||||
) (contracts.StructuredCompletionResponse, error) {
|
||||
if err := validateOutputTarget(out); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
|
||||
model := strings.TrimSpace(req.Model)
|
||||
if model == "" {
|
||||
model = c.cfg.Model
|
||||
}
|
||||
if model == "" {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion model must not be empty")
|
||||
}
|
||||
if req.ResponseSchema == nil {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion response schema is required")
|
||||
}
|
||||
if strings.TrimSpace(req.ResponseSchema.Name) == "" {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion response schema name must not be empty")
|
||||
}
|
||||
if len(req.ResponseSchema.JSONSchema) == 0 || !json.Valid(req.ResponseSchema.JSONSchema) {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion response schema JSON must be valid")
|
||||
}
|
||||
|
||||
messages, err := toOpenAICompatibleMessages(req.Messages)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
|
||||
endpoint := buildChatCompletionsURL(c.cfg.BaseURL)
|
||||
|
||||
var lastErr error
|
||||
for attempt := 0; attempt <= c.cfg.MaxRetries; attempt++ {
|
||||
content, metadata, callErr := c.completeStructuredOnce(
|
||||
ctx,
|
||||
endpoint,
|
||||
model,
|
||||
messages,
|
||||
req.ResponseSchema.Name,
|
||||
req.ResponseSchema.JSONSchema,
|
||||
)
|
||||
if callErr == nil {
|
||||
if decodeErr := json.Unmarshal(content, out); decodeErr != nil {
|
||||
callErr = retryableError{err: fmt.Errorf("decode structured output: %w", decodeErr)}
|
||||
} else {
|
||||
return contracts.StructuredCompletionResponse{
|
||||
Content: content,
|
||||
Provider: "openai-compatible",
|
||||
Model: firstNonEmpty(metadata.Model, model),
|
||||
PromptTokens: metadata.PromptTokens,
|
||||
CompletionTokens: metadata.CompletionTokens,
|
||||
TotalTokens: metadata.TotalTokens,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
lastErr = sanitizeError(callErr, c.cfg.APIKey)
|
||||
if !canRetryFromError(ctx, attempt, c.cfg.MaxRetries, callErr) {
|
||||
return contracts.StructuredCompletionResponse{}, lastErr
|
||||
}
|
||||
}
|
||||
|
||||
if lastErr == nil {
|
||||
lastErr = fmt.Errorf("structured completion failed")
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{}, lastErr
|
||||
}
|
||||
|
||||
type openAICompatibleMessage struct {
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
}
|
||||
|
||||
type openAICompatibleRequest struct {
|
||||
Model string `json:"model"`
|
||||
Messages []openAICompatibleMessage `json:"messages"`
|
||||
ResponseFormat openAICompatibleStructuredOutputShape `json:"response_format"`
|
||||
}
|
||||
|
||||
type openAICompatibleStructuredOutputShape struct {
|
||||
Type string `json:"type"`
|
||||
JSONSchema openAICompatibleSchemaEnvelope `json:"json_schema"`
|
||||
}
|
||||
|
||||
type openAICompatibleSchemaEnvelope struct {
|
||||
Name string `json:"name"`
|
||||
Strict bool `json:"strict"`
|
||||
Schema json.RawMessage `json:"schema"`
|
||||
}
|
||||
|
||||
type openAICompatibleChatCompletionsResponse struct {
|
||||
Model string `json:"model"`
|
||||
Choices []struct {
|
||||
Message struct {
|
||||
Content json.RawMessage `json:"content"`
|
||||
} `json:"message"`
|
||||
} `json:"choices"`
|
||||
Usage *openAICompatibleUsage `json:"usage,omitempty"`
|
||||
}
|
||||
|
||||
type openAICompatibleUsage struct {
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
}
|
||||
|
||||
type openAICompatibleResponseMetadata struct {
|
||||
Model string
|
||||
PromptTokens int
|
||||
CompletionTokens int
|
||||
TotalTokens int
|
||||
}
|
||||
|
||||
func (c *OpenAICompatibleClient) completeStructuredOnce(
|
||||
ctx context.Context,
|
||||
endpoint string,
|
||||
model string,
|
||||
messages []openAICompatibleMessage,
|
||||
responseSchemaName string,
|
||||
responseSchemaJSON json.RawMessage,
|
||||
) (json.RawMessage, openAICompatibleResponseMetadata, error) {
|
||||
requestBody := openAICompatibleRequest{
|
||||
Model: model,
|
||||
Messages: messages,
|
||||
ResponseFormat: openAICompatibleStructuredOutputShape{
|
||||
Type: "json_schema",
|
||||
JSONSchema: openAICompatibleSchemaEnvelope{
|
||||
Name: responseSchemaName,
|
||||
Strict: true,
|
||||
Schema: responseSchemaJSON,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
payload, err := json.Marshal(requestBody)
|
||||
if err != nil {
|
||||
return nil, openAICompatibleResponseMetadata{}, fmt.Errorf("marshal provider request: %w", err)
|
||||
}
|
||||
|
||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return nil, openAICompatibleResponseMetadata{}, fmt.Errorf("build provider request: %w", err)
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
if c.cfg.APIKey != "" {
|
||||
httpReq.Header.Set("Authorization", "Bearer "+c.cfg.APIKey)
|
||||
}
|
||||
|
||||
httpResp, err := c.httpClient.Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, openAICompatibleResponseMetadata{}, retryableError{err: fmt.Errorf("provider request failed: %w", err)}
|
||||
}
|
||||
defer func() {
|
||||
_ = httpResp.Body.Close()
|
||||
}()
|
||||
|
||||
rawResp, err := io.ReadAll(httpResp.Body)
|
||||
if err != nil {
|
||||
return nil, openAICompatibleResponseMetadata{}, retryableError{err: fmt.Errorf("read provider response: %w", err)}
|
||||
}
|
||||
|
||||
if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 {
|
||||
statusErr := parseProviderErrorBody(httpResp.StatusCode, rawResp)
|
||||
if httpResp.StatusCode == http.StatusTooManyRequests || httpResp.StatusCode >= 500 {
|
||||
return nil, openAICompatibleResponseMetadata{}, retryableError{err: statusErr}
|
||||
}
|
||||
return nil, openAICompatibleResponseMetadata{}, statusErr
|
||||
}
|
||||
|
||||
content, metadata, decodeErr := decodeChatCompletionsResponse(rawResp)
|
||||
if decodeErr != nil {
|
||||
return nil, openAICompatibleResponseMetadata{}, decodeErr
|
||||
}
|
||||
return content, metadata, nil
|
||||
}
|
||||
|
||||
func toOpenAICompatibleMessages(messages []contracts.LLMMessage) ([]openAICompatibleMessage, error) {
|
||||
result := make([]openAICompatibleMessage, len(messages))
|
||||
for i, message := range messages {
|
||||
role := strings.TrimSpace(message.Role)
|
||||
content := strings.TrimSpace(message.Content)
|
||||
if role == "" {
|
||||
return nil, fmt.Errorf("message[%d] role must not be empty", i)
|
||||
}
|
||||
if content == "" {
|
||||
return nil, fmt.Errorf("message[%d] content must not be empty", i)
|
||||
}
|
||||
result[i] = openAICompatibleMessage{
|
||||
Role: role,
|
||||
Content: content,
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func buildChatCompletionsURL(baseURL string) string {
|
||||
return strings.TrimRight(baseURL, "/") + "/chat/completions"
|
||||
}
|
||||
|
||||
func normalizeOpenAICompatibleConfig(cfg OpenAICompatibleClientConfig) (OpenAICompatibleClientConfig, error) {
|
||||
cfg.BaseURL = strings.TrimSpace(cfg.BaseURL)
|
||||
cfg.Model = strings.TrimSpace(cfg.Model)
|
||||
cfg.APIKey = strings.TrimSpace(cfg.APIKey)
|
||||
if cfg.MaxRetries == 0 {
|
||||
cfg.MaxRetries = defaultOpenAICompatibleMaxRetries
|
||||
}
|
||||
if cfg.MaxRetries < 0 {
|
||||
return OpenAICompatibleClientConfig{}, fmt.Errorf("max retries must be zero or greater")
|
||||
}
|
||||
if cfg.BaseURL == "" {
|
||||
return OpenAICompatibleClientConfig{}, fmt.Errorf("base URL must not be empty")
|
||||
}
|
||||
if cfg.Model == "" {
|
||||
return OpenAICompatibleClientConfig{}, fmt.Errorf("model must not be empty")
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
type retryableError struct {
|
||||
err error
|
||||
}
|
||||
|
||||
func (e retryableError) Error() string {
|
||||
if e.err == nil {
|
||||
return ""
|
||||
}
|
||||
return e.err.Error()
|
||||
}
|
||||
|
||||
func (e retryableError) Unwrap() error {
|
||||
return e.err
|
||||
}
|
||||
|
||||
func canRetryFromError(ctx context.Context, attempt int, maxRetries int, err error) bool {
|
||||
if attempt >= maxRetries {
|
||||
return false
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return false
|
||||
}
|
||||
var retryable retryableError
|
||||
return errors.As(err, &retryable)
|
||||
}
|
||||
|
||||
func firstNonEmpty(values ...string) string {
|
||||
for _, value := range values {
|
||||
value = strings.TrimSpace(value)
|
||||
if value != "" {
|
||||
return value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func decodeChatCompletionsResponse(raw []byte) (json.RawMessage, openAICompatibleResponseMetadata, error) {
|
||||
var parsed openAICompatibleChatCompletionsResponse
|
||||
if err := json.Unmarshal(raw, &parsed); err != nil {
|
||||
return nil, openAICompatibleResponseMetadata{}, retryableError{err: fmt.Errorf("decode provider response envelope: %w", err)}
|
||||
}
|
||||
if len(parsed.Choices) == 0 {
|
||||
return nil, openAICompatibleResponseMetadata{}, fmt.Errorf("provider response missing choices")
|
||||
}
|
||||
|
||||
content, err := extractAssistantContentJSON(parsed.Choices[0].Message.Content)
|
||||
if err != nil {
|
||||
return nil, openAICompatibleResponseMetadata{}, retryableError{err: err}
|
||||
}
|
||||
|
||||
meta := openAICompatibleResponseMetadata{
|
||||
Model: parsed.Model,
|
||||
}
|
||||
if parsed.Usage != nil {
|
||||
meta.PromptTokens = parsed.Usage.PromptTokens
|
||||
meta.CompletionTokens = parsed.Usage.CompletionTokens
|
||||
meta.TotalTokens = parsed.Usage.TotalTokens
|
||||
}
|
||||
return content, meta, nil
|
||||
}
|
||||
|
||||
func extractAssistantContentJSON(raw json.RawMessage) (json.RawMessage, error) {
|
||||
if len(bytes.TrimSpace(raw)) == 0 || string(bytes.TrimSpace(raw)) == "null" {
|
||||
return nil, fmt.Errorf("provider response missing assistant message content")
|
||||
}
|
||||
|
||||
var textContent string
|
||||
if err := json.Unmarshal(raw, &textContent); err == nil {
|
||||
textContent = strings.TrimSpace(textContent)
|
||||
if textContent == "" {
|
||||
return nil, fmt.Errorf("provider response assistant message content is empty")
|
||||
}
|
||||
return json.RawMessage(textContent), nil
|
||||
}
|
||||
|
||||
trimmed := bytes.TrimSpace(raw)
|
||||
if json.Valid(trimmed) {
|
||||
return append(json.RawMessage(nil), trimmed...), nil
|
||||
}
|
||||
return nil, fmt.Errorf("provider response assistant message content is not valid JSON")
|
||||
}
|
||||
|
||||
func parseProviderErrorBody(status int, body []byte) error {
|
||||
trimmed := strings.TrimSpace(string(body))
|
||||
if trimmed == "" {
|
||||
return fmt.Errorf("provider returned status %d", status)
|
||||
}
|
||||
|
||||
var payload map[string]any
|
||||
if err := json.Unmarshal(body, &payload); err == nil {
|
||||
if nested, ok := payload["error"].(map[string]any); ok {
|
||||
if msg, ok := nested["message"].(string); ok && strings.TrimSpace(msg) != "" {
|
||||
return fmt.Errorf("provider returned status %d: %s", status, strings.TrimSpace(msg))
|
||||
}
|
||||
}
|
||||
if msg, ok := payload["message"].(string); ok && strings.TrimSpace(msg) != "" {
|
||||
return fmt.Errorf("provider returned status %d: %s", status, strings.TrimSpace(msg))
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Errorf("provider returned status %d: %s", status, trimmed)
|
||||
}
|
||||
@@ -1,571 +0,0 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
|
||||
)
|
||||
|
||||
func TestNewOpenAICompatibleClientValidation(t *testing.T) {
|
||||
_, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
|
||||
BaseURL: " ",
|
||||
Model: "model",
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "base URL") {
|
||||
t.Fatalf("expected base URL validation error, got %v", err)
|
||||
}
|
||||
|
||||
_, err = NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
|
||||
BaseURL: "https://example.test/v1",
|
||||
Model: " ",
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "model") {
|
||||
t.Fatalf("expected model validation error, got %v", err)
|
||||
}
|
||||
|
||||
_, err = NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
|
||||
BaseURL: "https://example.test/v1",
|
||||
Model: "model",
|
||||
MaxRetries: -1,
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "max retries") {
|
||||
t.Fatalf("expected max retries validation error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientRequestShapeAndDecode(t *testing.T) {
|
||||
schema := responseschema.MustLookup(responseschema.CorrectionSetKey)
|
||||
var seenPath string
|
||||
var seenAuthorization string
|
||||
var seenReq map[string]any
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
seenPath = r.URL.Path
|
||||
seenAuthorization = r.Header.Get("Authorization")
|
||||
if err := json.NewDecoder(r.Body).Decode(&seenReq); err != nil {
|
||||
t.Fatalf("decode request: %v", err)
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"model":"provider-model",
|
||||
"choices":[{"message":{"content":"{\"name\":\"Robby\",\"age\":22}"}}],
|
||||
"usage":{"prompt_tokens":11,"completion_tokens":7,"total_tokens":18}
|
||||
}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
|
||||
BaseURL: server.URL + "/v1",
|
||||
Model: "test-model",
|
||||
APIKey: "secret-key",
|
||||
MaxRetries: 1,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewOpenAICompatibleClient: %v", err)
|
||||
}
|
||||
|
||||
type person struct {
|
||||
Name string `json:"name"`
|
||||
Age int `json:"age"`
|
||||
}
|
||||
var out person
|
||||
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
|
||||
ResponseSchema: &schema,
|
||||
}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("CompleteStructured: %v", err)
|
||||
}
|
||||
|
||||
if seenPath != "/v1/chat/completions" {
|
||||
t.Fatalf("unexpected request path: %q", seenPath)
|
||||
}
|
||||
if seenAuthorization != "Bearer secret-key" {
|
||||
t.Fatalf("unexpected authorization header: %q", seenAuthorization)
|
||||
}
|
||||
|
||||
responseFormat, ok := seenReq["response_format"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected response_format object, got %T", seenReq["response_format"])
|
||||
}
|
||||
if responseFormat["type"] != "json_schema" {
|
||||
t.Fatalf("unexpected response_format.type: %v", responseFormat["type"])
|
||||
}
|
||||
jsonSchema, ok := responseFormat["json_schema"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected response_format.json_schema object, got %T", responseFormat["json_schema"])
|
||||
}
|
||||
if jsonSchema["name"] != schema.Name {
|
||||
t.Fatalf("unexpected response schema name: %v", jsonSchema["name"])
|
||||
}
|
||||
if jsonSchema["strict"] != true {
|
||||
t.Fatalf("expected strict=true, got %v", jsonSchema["strict"])
|
||||
}
|
||||
if _, ok := jsonSchema["schema"].(map[string]any); !ok {
|
||||
t.Fatalf("expected embedded JSON schema object, got %T", jsonSchema["schema"])
|
||||
}
|
||||
|
||||
if out.Name != "Robby" || out.Age != 22 {
|
||||
t.Fatalf("unexpected decoded output: %+v", out)
|
||||
}
|
||||
if resp.Provider != "openai-compatible" {
|
||||
t.Fatalf("unexpected provider metadata: %q", resp.Provider)
|
||||
}
|
||||
if resp.Model != "provider-model" {
|
||||
t.Fatalf("unexpected model metadata: %q", resp.Model)
|
||||
}
|
||||
if resp.PromptTokens != 11 || resp.CompletionTokens != 7 || resp.TotalTokens != 18 {
|
||||
t.Fatalf("unexpected token metadata: %+v", resp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientDecodesCorrectionSetStructuredResponse(t *testing.T) {
|
||||
schema := responseschema.MustLookup(responseschema.CorrectionSetKey)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"choices":[{"message":{"content":"{\"corrections\":[{\"id\":1,\"original_text\":\"teh\",\"corrected_text\":\"the\",\"confidence\":0.9}]}"}}]
|
||||
}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
|
||||
BaseURL: server.URL,
|
||||
Model: "test-model",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewOpenAICompatibleClient: %v", err)
|
||||
}
|
||||
|
||||
type correction struct {
|
||||
TargetSegmentID int `json:"id"`
|
||||
OriginalText string `json:"original_text"`
|
||||
CorrectedText string `json:"corrected_text"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
}
|
||||
type correctionSet struct {
|
||||
Corrections []correction `json:"corrections"`
|
||||
}
|
||||
var out correctionSet
|
||||
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
|
||||
ResponseSchema: &schema,
|
||||
}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("CompleteStructured: %v", err)
|
||||
}
|
||||
if len(out.Corrections) != 1 {
|
||||
t.Fatalf("expected one correction, got %+v", out.Corrections)
|
||||
}
|
||||
if out.Corrections[0].TargetSegmentID != 1 || out.Corrections[0].CorrectedText != "the" {
|
||||
t.Fatalf("unexpected correction payload: %+v", out.Corrections[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientDecodesValidatorDecisionSetStructuredResponse(t *testing.T) {
|
||||
schema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"choices":[{"message":{"content":"{\"validations\":[{\"correction_index\":0,\"approved\":true,\"confidence\":0.95,\"reason\":\"ok\"}]}"}}]
|
||||
}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
|
||||
BaseURL: server.URL,
|
||||
Model: "test-model",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewOpenAICompatibleClient: %v", err)
|
||||
}
|
||||
|
||||
type validationDecision struct {
|
||||
CorrectionIndex int `json:"correction_index"`
|
||||
Approved bool `json:"approved"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
type validationResponse struct {
|
||||
Validations []validationDecision `json:"validations"`
|
||||
}
|
||||
var out validationResponse
|
||||
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
|
||||
ResponseSchema: &schema,
|
||||
}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("CompleteStructured: %v", err)
|
||||
}
|
||||
if len(out.Validations) != 1 {
|
||||
t.Fatalf("expected one validation decision, got %+v", out.Validations)
|
||||
}
|
||||
if out.Validations[0].CorrectionIndex != 0 || !out.Validations[0].Approved {
|
||||
t.Fatalf("unexpected validation payload: %+v", out.Validations[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientNoAuthorizationHeaderWithoutAPIKey(t *testing.T) {
|
||||
schema := responseschema.MustLookup(responseschema.CorrectionSetKey)
|
||||
var seenAuthorization string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
seenAuthorization = r.Header.Get("Authorization")
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{\"ok\":true}"}}]}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
|
||||
BaseURL: server.URL,
|
||||
Model: "test-model",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewOpenAICompatibleClient: %v", err)
|
||||
}
|
||||
|
||||
var out map[string]any
|
||||
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
|
||||
ResponseSchema: &schema,
|
||||
}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("CompleteStructured: %v", err)
|
||||
}
|
||||
if seenAuthorization != "" {
|
||||
t.Fatalf("expected empty Authorization header, got %q", seenAuthorization)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientMalformedJSONFailsSafely(t *testing.T) {
|
||||
schema := responseschema.MustLookup(responseschema.CorrectionSetKey)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{"}}]}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
|
||||
BaseURL: server.URL,
|
||||
Model: "test-model",
|
||||
MaxRetries: 0,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewOpenAICompatibleClient: %v", err)
|
||||
}
|
||||
|
||||
var out map[string]any
|
||||
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
|
||||
ResponseSchema: &schema,
|
||||
}, &out)
|
||||
if err == nil || !strings.Contains(err.Error(), "decode structured output") {
|
||||
t.Fatalf("expected decode error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientMissingRequiredFieldsFailsSafely(t *testing.T) {
|
||||
schema := responseschema.MustLookup(responseschema.CorrectionSetKey)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"model":"x","choices":[]}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
|
||||
BaseURL: server.URL,
|
||||
Model: "test-model",
|
||||
MaxRetries: 0,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewOpenAICompatibleClient: %v", err)
|
||||
}
|
||||
|
||||
var out map[string]any
|
||||
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
|
||||
ResponseSchema: &schema,
|
||||
}, &out)
|
||||
if err == nil || !strings.Contains(err.Error(), "missing choices") {
|
||||
t.Fatalf("expected missing-field error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientUnknownExtraFieldsFollowLocalDecoderPolicy(t *testing.T) {
|
||||
schema := responseschema.MustLookup(responseschema.CorrectionSetKey)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"choices":[{"message":{"content":"{\"corrections\":[{\"id\":1,\"original_text\":\"teh\",\"corrected_text\":\"the\",\"confidence\":0.9,\"extra\":\"ignored\"}],\"top_extra\":true}"}}]
|
||||
}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
|
||||
BaseURL: server.URL,
|
||||
Model: "test-model",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewOpenAICompatibleClient: %v", err)
|
||||
}
|
||||
|
||||
type correction struct {
|
||||
TargetSegmentID int `json:"id"`
|
||||
OriginalText string `json:"original_text"`
|
||||
CorrectedText string `json:"corrected_text"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
}
|
||||
type correctionSet struct {
|
||||
Corrections []correction `json:"corrections"`
|
||||
}
|
||||
var out correctionSet
|
||||
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
|
||||
ResponseSchema: &schema,
|
||||
}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("expected unknown extra fields to be ignored by local decoder, got %v", err)
|
||||
}
|
||||
if len(out.Corrections) != 1 || out.Corrections[0].CorrectedText != "the" {
|
||||
t.Fatalf("unexpected decoded payload: %+v", out.Corrections)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientProviderErrorRedactsSecret(t *testing.T) {
|
||||
secret := "super-secret-key"
|
||||
schema := responseschema.MustLookup(responseschema.CorrectionSetKey)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
_, _ = w.Write([]byte(`{"error":{"message":"Authorization failed for Bearer super-secret-key"}}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
|
||||
BaseURL: server.URL,
|
||||
Model: "test-model",
|
||||
APIKey: secret,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewOpenAICompatibleClient: %v", err)
|
||||
}
|
||||
|
||||
var out map[string]any
|
||||
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
|
||||
ResponseSchema: &schema,
|
||||
}, &out)
|
||||
if err == nil {
|
||||
t.Fatalf("expected provider error")
|
||||
}
|
||||
if strings.Contains(err.Error(), secret) {
|
||||
t.Fatalf("error leaked secret: %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "[REDACTED]") {
|
||||
t.Fatalf("expected redaction marker in error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientRequestErrorRedactsSecret(t *testing.T) {
|
||||
secret := "super-secret-key"
|
||||
schema := responseschema.MustLookup(responseschema.CorrectionSetKey)
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
|
||||
BaseURL: "https://example.test/v1",
|
||||
Model: "test-model",
|
||||
APIKey: secret,
|
||||
HTTPClient: &http.Client{
|
||||
Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
|
||||
_ = r
|
||||
return nil, fmt.Errorf("request failed for Authorization: Bearer %s", secret)
|
||||
}),
|
||||
},
|
||||
MaxRetries: 0,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewOpenAICompatibleClient: %v", err)
|
||||
}
|
||||
|
||||
var out map[string]any
|
||||
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
|
||||
ResponseSchema: &schema,
|
||||
}, &out)
|
||||
if err == nil {
|
||||
t.Fatalf("expected request error")
|
||||
}
|
||||
if strings.Contains(err.Error(), secret) {
|
||||
t.Fatalf("error leaked secret: %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "[REDACTED]") {
|
||||
t.Fatalf("expected redaction marker in error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientCancellationAndTimeout(t *testing.T) {
|
||||
schema := responseschema.MustLookup(responseschema.CorrectionSetKey)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
select {
|
||||
case <-r.Context().Done():
|
||||
return
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{\"ok\":true}"}}]}`))
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
|
||||
BaseURL: server.URL,
|
||||
Model: "test-model",
|
||||
RequestTimeout: 20 * time.Millisecond,
|
||||
MaxRetries: 0,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewOpenAICompatibleClient: %v", err)
|
||||
}
|
||||
|
||||
var out map[string]any
|
||||
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
|
||||
ResponseSchema: &schema,
|
||||
}, &out)
|
||||
if err == nil {
|
||||
t.Fatalf("expected timeout-related error")
|
||||
}
|
||||
if !strings.Contains(strings.ToLower(err.Error()), "context deadline") {
|
||||
t.Fatalf("expected context deadline in error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientRetryBehavior(t *testing.T) {
|
||||
schema := responseschema.MustLookup(responseschema.CorrectionSetKey)
|
||||
var attempts int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
current := atomic.AddInt32(&attempts, 1)
|
||||
if current == 1 {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
_, _ = w.Write([]byte(`{"error":{"message":"temporary failure"}}`))
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{\"ok\":true}"}}]}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
|
||||
BaseURL: server.URL,
|
||||
Model: "test-model",
|
||||
MaxRetries: 1,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewOpenAICompatibleClient: %v", err)
|
||||
}
|
||||
|
||||
var out map[string]any
|
||||
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
|
||||
ResponseSchema: &schema,
|
||||
}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("CompleteStructured: %v", err)
|
||||
}
|
||||
if atomic.LoadInt32(&attempts) != 2 {
|
||||
t.Fatalf("expected 2 attempts, got %d", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientRetryOnMalformedStructuredOutput(t *testing.T) {
|
||||
schema := responseschema.MustLookup(responseschema.CorrectionSetKey)
|
||||
var attempts int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
current := atomic.AddInt32(&attempts, 1)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if current == 1 {
|
||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{"}}]}`))
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{\"ok\":true}"}}]}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
|
||||
BaseURL: server.URL,
|
||||
Model: "test-model",
|
||||
MaxRetries: 1,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewOpenAICompatibleClient: %v", err)
|
||||
}
|
||||
|
||||
var out map[string]any
|
||||
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
|
||||
ResponseSchema: &schema,
|
||||
}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("CompleteStructured: %v", err)
|
||||
}
|
||||
if atomic.LoadInt32(&attempts) != 2 {
|
||||
t.Fatalf("expected 2 attempts, got %d", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientHonorsCancelledContextWithoutRetry(t *testing.T) {
|
||||
schema := responseschema.MustLookup(responseschema.CorrectionSetKey)
|
||||
var attempts int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
atomic.AddInt32(&attempts, 1)
|
||||
<-r.Context().Done()
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
|
||||
BaseURL: server.URL,
|
||||
Model: "test-model",
|
||||
MaxRetries: 3,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewOpenAICompatibleClient: %v", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
var out map[string]any
|
||||
_, err = client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
Messages: []contracts.LLMMessage{{Role: "user", Content: "extract"}},
|
||||
ResponseSchema: &schema,
|
||||
}, &out)
|
||||
if err == nil {
|
||||
t.Fatalf("expected cancellation error")
|
||||
}
|
||||
if !errors.Is(err, context.Canceled) && !strings.Contains(strings.ToLower(err.Error()), "canceled") {
|
||||
t.Fatalf("expected cancellation-related error, got %v", err)
|
||||
}
|
||||
if atomic.LoadInt32(&attempts) > 1 {
|
||||
t.Fatalf("expected no retry after cancellation, got attempts=%d", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
type roundTripFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) {
|
||||
return f(r)
|
||||
}
|
||||
|
||||
func newJSONHTTPResponse(status int, body string) *http.Response {
|
||||
return &http.Response{
|
||||
StatusCode: status,
|
||||
Header: http.Header{"Content-Type": []string{"application/json"}},
|
||||
Body: io.NopCloser(strings.NewReader(body)),
|
||||
}
|
||||
}
|
||||
@@ -8,10 +8,11 @@ func ConfiguredSecrets(cfg *config.Config) []string {
|
||||
if cfg == nil {
|
||||
return nil
|
||||
}
|
||||
effectiveValidation := cfg.EffectiveValidationLLMConfig()
|
||||
return []string{
|
||||
cfg.PrimaryLLM.APIKey,
|
||||
cfg.ValidationLLM.APIKey,
|
||||
effectiveValidation.APIKey,
|
||||
secrets := make([]string, 0, len(cfg.Backends))
|
||||
for _, backend := range cfg.Backends {
|
||||
if backend.Credential != nil && backend.Credential.Value() != "" {
|
||||
secrets = append(secrets, backend.Credential.Value())
|
||||
}
|
||||
}
|
||||
return secrets
|
||||
}
|
||||
|
||||
@@ -1,38 +1,31 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
)
|
||||
|
||||
func TestConfiguredSecretsNilConfig(t *testing.T) {
|
||||
if got := ConfiguredSecrets(nil); got != nil {
|
||||
t.Fatalf("expected nil secrets for nil config, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfiguredSecretsWithValidationOverride(t *testing.T) {
|
||||
cfg := config.Default()
|
||||
cfg.PrimaryLLM.APIKey = "primary-secret"
|
||||
cfg.ValidationLLM.APIKey = "validation-secret"
|
||||
|
||||
got := ConfiguredSecrets(&cfg)
|
||||
want := []string{"primary-secret", "validation-secret", "validation-secret"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("unexpected secrets: got=%v want=%v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfiguredSecretsWithInheritedValidationKey(t *testing.T) {
|
||||
cfg := config.Default()
|
||||
cfg.PrimaryLLM.APIKey = "primary-secret"
|
||||
cfg.ValidationLLM.APIKey = ""
|
||||
|
||||
got := ConfiguredSecrets(&cfg)
|
||||
want := []string{"primary-secret", "", "primary-secret"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("unexpected inherited secrets: got=%v want=%v", got, want)
|
||||
func TestConfiguredSecretsIncludesUnselectedBackends(t *testing.T) {
|
||||
if ConfiguredSecrets(nil) != nil {
|
||||
t.Fatal("nil configuration has secrets")
|
||||
}
|
||||
t.Setenv("SELECTED_KEY", "selected-secret")
|
||||
t.Setenv("OTHER_KEY", "unselected-secret")
|
||||
loaded, err := config.Load(ck.Bytes("config.yml", []byte("version: 2\nbackends:\n selected:\n endpoint: https://selected.invalid/v1\n credential: SELECTED_KEY\n other:\n endpoint: https://other.invalid/v1\n credential: OTHER_KEY\nllm:\n proposal_profile: base\n")), "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := ConfiguredSecrets(&loaded.Config)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("secrets count = %d", len(got))
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, value := range got {
|
||||
seen[value] = true
|
||||
}
|
||||
if !seen["selected-secret"] || !seen["unselected-secret"] {
|
||||
t.Fatal("missing resolved backend secret")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package structuredoutput
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
|
||||
)
|
||||
@@ -39,13 +40,38 @@ type PromptMetadata struct {
|
||||
}
|
||||
|
||||
type Response struct {
|
||||
Content json.RawMessage `json:"content,omitempty"`
|
||||
Provider string `json:"provider,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
PromptTokens int `json:"prompt_tokens,omitempty"`
|
||||
CompletionTokens int `json:"completion_tokens,omitempty"`
|
||||
TotalTokens int `json:"total_tokens,omitempty"`
|
||||
Prompt PromptMetadata `json:"prompt"`
|
||||
Content json.RawMessage `json:"content,omitempty"`
|
||||
Provider string `json:"provider,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
PromptTokens int `json:"prompt_tokens,omitempty"`
|
||||
CompletionTokens int `json:"completion_tokens,omitempty"`
|
||||
TotalTokens int `json:"total_tokens,omitempty"`
|
||||
Prompt PromptMetadata `json:"prompt"`
|
||||
Prepared *PreparedSnapshot `json:"prepared,omitempty"`
|
||||
FinalContent string `json:"final_content,omitempty"`
|
||||
}
|
||||
|
||||
// PreparedSnapshot is the exact rendered request snapshot. Its messages may
|
||||
// contain transcript and glossary content and belong only in protected diagnostics.
|
||||
type PreparedSnapshot struct {
|
||||
Messages []Message `json:"messages"`
|
||||
Prompt PromptMetadata `json:"prompt"`
|
||||
}
|
||||
|
||||
type Message struct {
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
}
|
||||
|
||||
// CallRecord brackets one logical operation, including preparation. Content
|
||||
// is present only when a model returned it; consumers must protect it.
|
||||
type CallRecord struct {
|
||||
Context DiagnosticContext `json:"context"`
|
||||
Started time.Time `json:"started"`
|
||||
Finished time.Time `json:"finished,omitempty"`
|
||||
Prepared *PreparedSnapshot `json:"prepared,omitempty"`
|
||||
FinalContent string `json:"final_content,omitempty"`
|
||||
Failure FailureKind `json:"failure,omitempty"`
|
||||
}
|
||||
|
||||
type Client interface {
|
||||
|
||||
27
internal/framework/structuredoutput/failure.go
Normal file
27
internal/framework/structuredoutput/failure.go
Normal file
@@ -0,0 +1,27 @@
|
||||
package structuredoutput
|
||||
|
||||
import "errors"
|
||||
|
||||
// FailureKind separates invalid generated content from operational failures.
|
||||
type FailureKind string
|
||||
|
||||
const (
|
||||
FailureMalformedOutput FailureKind = "malformed_output"
|
||||
FailureRuntime FailureKind = "runtime"
|
||||
)
|
||||
|
||||
// Failure exposes a safe message while retaining the typed cause for callers.
|
||||
// The cause must not be serialized or formatted directly in user artifacts.
|
||||
type Failure struct {
|
||||
Kind FailureKind
|
||||
Message string
|
||||
Cause error
|
||||
}
|
||||
|
||||
func (f *Failure) Error() string { return f.Message }
|
||||
func (f *Failure) Unwrap() error { return f.Cause }
|
||||
|
||||
func IsMalformedError(err error) bool {
|
||||
var failure *Failure
|
||||
return errors.As(err, &failure) && failure.Kind == FailureMalformedOutput
|
||||
}
|
||||
18
internal/framework/structuredoutput/failure_test.go
Normal file
18
internal/framework/structuredoutput/failure_test.go
Normal file
@@ -0,0 +1,18 @@
|
||||
package structuredoutput
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFailureClassificationKeepsCauseAndSafeMessage(t *testing.T) {
|
||||
cause := errors.New("provider token secret-value")
|
||||
malformed := &Failure{Kind: FailureMalformedOutput, Message: "malformed structured output", Cause: cause}
|
||||
if !IsMalformedError(malformed) || !errors.Is(malformed, cause) || malformed.Error() != "malformed structured output" {
|
||||
t.Fatalf("malformed classification: %v", malformed)
|
||||
}
|
||||
runtime := &Failure{Kind: FailureRuntime, Message: "structured model call failed", Cause: cause}
|
||||
if IsMalformedError(runtime) || IsMalformedError(cause) || IsMalformedError(nil) || !errors.Is(runtime, cause) || runtime.Error() != "structured model call failed" {
|
||||
t.Fatalf("runtime classification: %v", runtime)
|
||||
}
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
package structuredoutput
|
||||
|
||||
import "strings"
|
||||
|
||||
var malformedMarkers = []string{
|
||||
"malformed structured output",
|
||||
"decode structured output:",
|
||||
"decode provider response envelope:",
|
||||
"provider response missing choices",
|
||||
"provider response missing assistant message content",
|
||||
"provider response assistant message content is empty",
|
||||
"provider response assistant message content is not valid JSON",
|
||||
}
|
||||
|
||||
// IsMalformedError reports whether err matches provider malformed
|
||||
// structured-output failure markers that should be downgraded.
|
||||
func IsMalformedError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
msg := err.Error()
|
||||
for _, marker := range malformedMarkers {
|
||||
if strings.Contains(msg, marker) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
package structuredoutput
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestIsMalformedError(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
{name: "nil", err: nil, want: false},
|
||||
{name: "generic", err: errors.New("network timeout"), want: false},
|
||||
{name: "malformed", err: errors.New("malformed structured output"), want: true},
|
||||
{name: "decode structured", err: errors.New("decode structured output: unexpected end of JSON input"), want: true},
|
||||
{name: "missing choices", err: errors.New("provider response missing choices"), want: true},
|
||||
{name: "missing content", err: errors.New("provider response missing assistant message content"), want: true},
|
||||
{name: "empty content", err: errors.New("provider response assistant message content is empty"), want: true},
|
||||
{name: "invalid content json", err: errors.New("provider response assistant message content is not valid JSON"), want: true},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := IsMalformedError(tc.err)
|
||||
if got != tc.want {
|
||||
t.Fatalf("IsMalformedError(%v): got=%v want=%v", tc.err, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user