Complete Phase 11 proposal generation framework

2026-05-12 02:25:33 +00:00
parent 12202508bf
commit b360493cdc
12 changed files with 1032 additions and 56 deletions
--- a/internal/framework/proposal_generation/generate_test.go
+++ b/internal/framework/proposal_generation/generate_test.go
@@ -0,0 +1,267 @@
+package proposal_generation
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/audita/internal/core/config"
+	"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
+	"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
+)
+
+type fakeStructuredClient struct {
+	responses []StructuredCorrectionSet
+	err       error
+	calls     []contracts.StructuredCompletionRequest
+}
+
+func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
+	_ = ctx
+	f.calls = append(f.calls, req)
+	if f.err != nil {
+		return contracts.StructuredCompletionResponse{}, f.err
+	}
+	target, ok := out.(*StructuredCorrectionSet)
+	if !ok {
+		return contracts.StructuredCompletionResponse{}, errors.New("unexpected output type")
+	}
+	if len(f.responses) == 0 {
+		return contracts.StructuredCompletionResponse{}, errors.New("unexpected call")
+	}
+	*target = f.responses[0]
+	f.responses = f.responses[1:]
+	return contracts.StructuredCompletionResponse{}, nil
+}
+
+type countingScheduler struct {
+	runs int
+}
+
+func (s *countingScheduler) Run(ctx context.Context, fn func(context.Context) error) error {
+	s.runs++
+	return fn(ctx)
+}
+
+func defaultRequest(t *testing.T) Request {
+	t.Helper()
+	cfg := config.Default()
+	return Request{
+		ModuleKey:         "test_module",
+		ModuleInstance:    "test_module",
+		ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
+		Config:            &cfg,
+		Messages: []contracts.LLMMessage{
+			{Role: "system", Content: "system prompt"},
+			{Role: "user", Content: "user prompt"},
+		},
+		StartIndex: 0,
+	}
+}
+
+func TestGenerateCandidatesSuccess(t *testing.T) {
+	client := &fakeStructuredClient{
+		responses: []StructuredCorrectionSet{
+			{
+				Corrections: []StructuredCorrectionProposal{
+					{TargetSegmentID: 7, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9},
+				},
+			},
+		},
+	}
+	section := contracts.SectionMetadata{Index: 2}
+	req := defaultRequest(t)
+	req.LLMClient = client
+	req.StartIndex = 10
+	req.Section = &section
+
+	got, err := GenerateCandidates(context.Background(), req)
+	if err != nil {
+		t.Fatalf("GenerateCandidates error: %v", err)
+	}
+	if len(got.Corrections) != 1 || len(got.Enriched) != 1 {
+		t.Fatalf("unexpected proposal lengths: %+v", got)
+	}
+	if got.Enriched[0].ProposalIndex != 10 {
+		t.Fatalf("expected proposal index 10, got %d", got.Enriched[0].ProposalIndex)
+	}
+	if got.Enriched[0].SectionIndex == nil || *got.Enriched[0].SectionIndex != 2 {
+		t.Fatalf("expected section index 2, got %v", got.Enriched[0].SectionIndex)
+	}
+}
+
+func TestGenerateCandidatesMalformedStructuredResponse(t *testing.T) {
+	client := &fakeStructuredClient{
+		responses: []StructuredCorrectionSet{
+			{
+				Corrections: []StructuredCorrectionProposal{
+					{TargetSegmentID: 1, OriginalText: "x", CorrectedText: "", Confidence: 0.9},
+				},
+			},
+		},
+	}
+	req := defaultRequest(t)
+	req.LLMClient = client
+
+	_, err := GenerateCandidates(context.Background(), req)
+	if err == nil || !strings.Contains(err.Error(), "invalid structured correction") {
+		t.Fatalf("expected structured response validation failure, got %v", err)
+	}
+}
+
+func TestGenerateCandidatesDeterministicIndexAssignment(t *testing.T) {
+	baseResponse := StructuredCorrectionSet{
+		Corrections: []StructuredCorrectionProposal{
+			{TargetSegmentID: 1, OriginalText: "a", CorrectedText: "A", Confidence: 0.9},
+			{TargetSegmentID: 2, OriginalText: "b", CorrectedText: "B", Confidence: 0.9},
+		},
+	}
+	clientA := &fakeStructuredClient{responses: []StructuredCorrectionSet{baseResponse}}
+	clientB := &fakeStructuredClient{responses: []StructuredCorrectionSet{baseResponse}}
+
+	reqA := defaultRequest(t)
+	reqA.LLMClient = clientA
+	reqA.StartIndex = 3
+	first, err := GenerateCandidates(context.Background(), reqA)
+	if err != nil {
+		t.Fatalf("first generation failed: %v", err)
+	}
+
+	reqB := defaultRequest(t)
+	reqB.LLMClient = clientB
+	reqB.StartIndex = 3
+	second, err := GenerateCandidates(context.Background(), reqB)
+	if err != nil {
+		t.Fatalf("second generation failed: %v", err)
+	}
+
+	if !reflect.DeepEqual(first.Enriched, second.Enriched) {
+		t.Fatalf("expected stable enriched proposals\nfirst=%+v\nsecond=%+v", first.Enriched, second.Enriched)
+	}
+}
+
+func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) {
+	client := &fakeStructuredClient{
+		responses: []StructuredCorrectionSet{
+			{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "alpha", CorrectedText: "ALPHA", Confidence: 0.9}}},
+			{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 3, OriginalText: "charlie", CorrectedText: "CHARLIE", Confidence: 0.9}}},
+		},
+	}
+	section0 := contracts.SectionMetadata{Index: 0}
+	section1 := contracts.SectionMetadata{Index: 1}
+
+	req0 := defaultRequest(t)
+	req0.LLMClient = client
+	req0.Section = &section0
+	req0.StartIndex = 0
+	part0, err := GenerateCandidates(context.Background(), req0)
+	if err != nil {
+		t.Fatalf("section 0 generation failed: %v", err)
+	}
+
+	req1 := defaultRequest(t)
+	req1.LLMClient = client
+	req1.Section = &section1
+	req1.StartIndex = len(part0.Enriched)
+	part1, err := GenerateCandidates(context.Background(), req1)
+	if err != nil {
+		t.Fatalf("section 1 generation failed: %v", err)
+	}
+
+	all := append(append([]proposals.EnrichedCorrectionProposal(nil), part0.Enriched...), part1.Enriched...)
+	if len(all) != 2 {
+		t.Fatalf("expected 2 proposals, got %d", len(all))
+	}
+	if all[0].ProposalIndex != 0 || all[1].ProposalIndex != 1 {
+		t.Fatalf("unexpected proposal indexes: %d, %d", all[0].ProposalIndex, all[1].ProposalIndex)
+	}
+	if all[0].SectionIndex == nil || *all[0].SectionIndex != 0 || all[1].SectionIndex == nil || *all[1].SectionIndex != 1 {
+		t.Fatalf("unexpected section metadata: %+v", all)
+	}
+}
+
+func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) {
+	secret := "phase11-secret"
+	client := &fakeStructuredClient{
+		responses: []StructuredCorrectionSet{
+			{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: secret, CorrectedText: "safe", Confidence: 0.9}}},
+		},
+	}
+	cfg := config.Default()
+	cfg.PrimaryLLM.APIKey = secret
+	req := defaultRequest(t)
+	req.Config = &cfg
+	req.LLMClient = client
+	req.DiagnosticsDir = t.TempDir()
+	req.Messages = []contracts.LLMMessage{
+		{Role: "system", Content: "include secret " + secret},
+		{Role: "user", Content: "fix it"},
+	}
+
+	got, err := GenerateCandidates(context.Background(), req)
+	if err != nil {
+		t.Fatalf("GenerateCandidates error: %v", err)
+	}
+	if got.Artifacts.ResponsePayloadPath == "" || got.Artifacts.RequestPayloadPath == "" {
+		t.Fatalf("expected diagnostics artifact paths, got %+v", got.Artifacts)
+	}
+
+	for _, path := range []string{got.Artifacts.RequestPayloadPath, got.Artifacts.ResponsePayloadPath} {
+		raw, readErr := os.ReadFile(path)
+		if readErr != nil {
+			t.Fatalf("read artifact %q: %v", path, readErr)
+		}
+		if strings.Contains(string(raw), secret) {
+			t.Fatalf("artifact leaked secret %q: %s", path, string(raw))
+		}
+		if !strings.Contains(string(raw), "[REDACTED]") {
+			t.Fatalf("expected redaction marker in artifact %q: %s", path, string(raw))
+		}
+	}
+}
+
+func TestGenerateCandidatesSchedulerUsage(t *testing.T) {
+	client := &fakeStructuredClient{
+		responses: []StructuredCorrectionSet{
+			{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "x", CorrectedText: "y", Confidence: 0.9}}},
+		},
+	}
+	scheduler := &countingScheduler{}
+	req := defaultRequest(t)
+	req.LLMClient = client
+	req.Scheduler = scheduler
+
+	_, err := GenerateCandidates(context.Background(), req)
+	if err != nil {
+		t.Fatalf("GenerateCandidates error: %v", err)
+	}
+	if scheduler.runs != 1 {
+		t.Fatalf("expected scheduler to run once, got %d", scheduler.runs)
+	}
+}
+
+func TestGenerateCandidatesClientError(t *testing.T) {
+	client := &fakeStructuredClient{err: errors.New("boom")}
+	req := defaultRequest(t)
+	req.LLMClient = client
+	req.DiagnosticsDir = t.TempDir()
+
+	got, err := GenerateCandidates(context.Background(), req)
+	if err == nil || !strings.Contains(err.Error(), "completion failed") {
+		t.Fatalf("expected completion failure, got %v", err)
+	}
+	if got.Artifacts.ResponsePayloadPath != "" {
+		t.Fatalf("expected zero result on error, got %+v", got)
+	}
+	matches, globErr := filepath.Glob(filepath.Join(req.DiagnosticsDir, req.ModuleInstance, "*error-payload.json"))
+	if globErr != nil {
+		t.Fatalf("glob error: %v", globErr)
+	}
+	if len(matches) == 0 {
+		t.Fatalf("expected error diagnostics artifact under %s", req.DiagnosticsDir)
+	}
+}