86 lines
3.3 KiB
Go
86 lines
3.3 KiB
Go
package proposal_generation
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
|
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
|
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
|
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
|
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
|
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
|
)
|
|
|
|
func TestRepairedProposalStillFacesDomainChecks(t *testing.T) {
|
|
calls := 0
|
|
outputs := []string{`{"wrong":true}`, `{"corrections":[{"id":1,"original_text":"absent","corrected_text":"present","confidence":0.4}]}`}
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
output := outputs[len(outputs)-1]
|
|
if calls < len(outputs) {
|
|
output = outputs[calls]
|
|
}
|
|
calls++
|
|
content, _ := json.Marshal(output)
|
|
_, _ = fmt.Fprintf(w, `{"choices":[{"message":{"content":%s}}]}`, content)
|
|
}))
|
|
defer server.Close()
|
|
dir := t.TempDir()
|
|
if err := os.Mkdir(filepath.Join(dir, "profiles"), 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
main := filepath.Join(dir, "config.yml")
|
|
content := fmt.Sprintf("version: 2\nbackends:\n offline:\n endpoint: %s/v1\nllm:\n proposal_profile: base\n transient_retries: 0\n repair_attempts: 1\n", server.URL)
|
|
if err := os.WriteFile(main, []byte(content), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte("id: base\nbackend: offline\nmodel: local-model\n"), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
loaded, err := config.Load(ck.File(main), "")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
runtime, err := llm.NewRuntime(context.Background(), loaded.Config, main, server.Client())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
client, err := runtime.NewRoleClient("proposal", llm.CallHooks{})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "gestures"}}}
|
|
vars, err := BuildProposalVariables(transcript, &schema.Glossary{}, 0, "")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
generated, err := GenerateCandidates(context.Background(), Request{ModuleKey: "grammar", ModuleInstance: "grammar", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, Config: &loaded.Config, PromptID: prompts.PromptIDModuleGrammarProposal, PromptVersion: prompts.VersionV1, Variables: vars, LLMClient: client})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if calls != 2 || len(generated.Enriched) != 1 {
|
|
t.Fatalf("repair path: calls=%d, result=%+v", calls, generated)
|
|
}
|
|
cfg := loaded.Config
|
|
cfg.Thresholds.Grammar = .8
|
|
input := validators.Request{ModuleKey: "grammar", Config: &cfg, CandidateProposal: generated.Enriched, WorkingTranscript: transcript}
|
|
low, err := (validators.ConfidenceThresholdValidator{}).Validate(context.Background(), input)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
missing, err := (validators.OriginalTextPresenceValidator{}).Validate(context.Background(), input)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if low.Decisions[0].ReasonCode != validators.ReasonLowConfidence || missing.Decisions[0].ReasonCode != validators.ReasonMissingOriginalText || calls != 2 {
|
|
t.Fatalf("domain checks or extra semantic retry: low=%+v missing=%+v calls=%d", low, missing, calls)
|
|
}
|
|
}
|