Move LLM validators to PromptKit prompt variables

2026-09-24 02:07:17 +00:00
parent f577847ca2
commit a4a72168d5
17 changed files with 310 additions and 257 deletions
--- a/internal/framework/proposal_generation/adapter_integration_test.go
+++ b/internal/framework/proposal_generation/adapter_integration_test.go
@@ -0,0 +1,85 @@
+package proposal_generation
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+
+	ck "gitea.maximumdirect.net/eric/ConfigKit"
+	"gitea.maximumdirect.net/eric/audita/internal/core/config"
+	"gitea.maximumdirect.net/eric/audita/internal/core/schema"
+	"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
+	"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
+	"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
+	"gitea.maximumdirect.net/eric/audita/internal/prompts"
+)
+
+func TestRepairedProposalStillFacesDomainChecks(t *testing.T) {
+	calls := 0
+	outputs := []string{`{"wrong":true}`, `{"corrections":[{"id":1,"original_text":"absent","corrected_text":"present","confidence":0.4}]}`}
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		output := outputs[len(outputs)-1]
+		if calls < len(outputs) {
+			output = outputs[calls]
+		}
+		calls++
+		content, _ := json.Marshal(output)
+		_, _ = fmt.Fprintf(w, `{"choices":[{"message":{"content":%s}}]}`, content)
+	}))
+	defer server.Close()
+	dir := t.TempDir()
+	if err := os.Mkdir(filepath.Join(dir, "profiles"), 0700); err != nil {
+		t.Fatal(err)
+	}
+	main := filepath.Join(dir, "config.yml")
+	content := fmt.Sprintf("version: 2\nbackends:\n  offline:\n    endpoint: %s/v1\nllm:\n  proposal_profile: base\n  transient_retries: 0\n  repair_attempts: 1\n", server.URL)
+	if err := os.WriteFile(main, []byte(content), 0600); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte("id: base\nbackend: offline\nmodel: local-model\n"), 0600); err != nil {
+		t.Fatal(err)
+	}
+	loaded, err := config.Load(ck.File(main), "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	runtime, err := llm.NewRuntime(context.Background(), loaded.Config, main, server.Client())
+	if err != nil {
+		t.Fatal(err)
+	}
+	client, err := runtime.NewRoleClient("proposal", llm.CallHooks{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "gestures"}}}
+	vars, err := BuildProposalVariables(transcript, &schema.Glossary{}, 0, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	generated, err := GenerateCandidates(context.Background(), Request{ModuleKey: "grammar", ModuleInstance: "grammar", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, Config: &loaded.Config, PromptID: prompts.PromptIDModuleGrammarProposal, PromptVersion: prompts.VersionV1, Variables: vars, LLMClient: client})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if calls != 2 || len(generated.Enriched) != 1 {
+		t.Fatalf("repair path: calls=%d, result=%+v", calls, generated)
+	}
+	cfg := loaded.Config
+	cfg.Thresholds.Grammar = .8
+	input := validators.Request{ModuleKey: "grammar", Config: &cfg, CandidateProposal: generated.Enriched, WorkingTranscript: transcript}
+	low, err := (validators.ConfidenceThresholdValidator{}).Validate(context.Background(), input)
+	if err != nil {
+		t.Fatal(err)
+	}
+	missing, err := (validators.OriginalTextPresenceValidator{}).Validate(context.Background(), input)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if low.Decisions[0].ReasonCode != validators.ReasonLowConfidence || missing.Decisions[0].ReasonCode != validators.ReasonMissingOriginalText || calls != 2 {
+		t.Fatalf("domain checks or extra semantic retry: low=%+v missing=%+v calls=%d", low, missing, calls)
+	}
+}