From b360493cdc62dd3c6d08f658bd70125a83fd7935 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 12 May 2026 02:25:33 +0000 Subject: [PATCH] Complete Phase 11 proposal generation framework --- docs/architecture.md | 57 +++- docs/rewrite-notes.md | 72 +++-- internal/cli/run.go | 7 +- internal/cli/run_test.go | 50 +++- internal/core/reporting/report_test.go | 4 +- internal/framework/contracts/contracts.go | 10 +- internal/framework/modules/registry.go | 153 ++++++++++ internal/framework/modules/registry_test.go | 125 ++++++++ .../framework/proposal_generation/generate.go | 235 +++++++++++++++ .../proposal_generation/generate_test.go | 267 ++++++++++++++++++ internal/framework/runner/runner.go | 10 +- internal/framework/runner/runner_test.go | 98 +++++++ 12 files changed, 1032 insertions(+), 56 deletions(-) create mode 100644 internal/framework/modules/registry.go create mode 100644 internal/framework/modules/registry_test.go create mode 100644 internal/framework/proposal_generation/generate.go create mode 100644 internal/framework/proposal_generation/generate_test.go diff --git a/docs/architecture.md b/docs/architecture.md index c25e9be..d5f39de 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -29,17 +29,21 @@ Implemented today: - LLM-backed validator models, prompt builders, batching, and runtime execution. - Runner wiring for LLM validators via the internal structured LLM abstraction and scheduler hooks. - LLM validator diagnostics artifacts and report-level decision metadata paths. +- Shared LLM proposal-generation helper with structured correction-set parsing. +- Deterministic proposal-index assignment and enriched proposal mapping for shared generation. +- Proposal-generation diagnostics artifacts with secret redaction. +- Production module registry scaffolding with known-key recognition and explicit unsupported/unimplemented errors. Not implemented in CLI runtime path today: - Real module execution pipeline (`glossary`, `homophones`, `spoken_word`, `grammar`). -- Structured LLM proposal generation. -- Shared module proposal-generation framework and module registry for real modules. +- Real domain proposal prompts for production modules. - End-to-end transcript polishing with real module behavior. Phase sequencing note: - Phase 9 LLM infrastructure is complete (structured client, scheduler, effective config resolution, diagnostics primitives); - Phase 10 LLM-backed validator runtime integration is complete; -- shared module proposal-generation and module registry work remain Phase 11. +- Phase 11 shared proposal-generation framework and module-registry scaffolding are complete; +- next recommended phase is Phase 12 (grammar module). ## Actual Go package layout @@ -92,6 +96,12 @@ internal/framework/proposals/ internal/framework/runner/ runner.go +internal/framework/proposal_generation/ + generate.go + +internal/framework/modules/ + registry.go + internal/framework/validators/ models.go deterministic.go @@ -126,9 +136,9 @@ Current runtime flow (`internal/cli/run.go`): 9. Write normalized transcript and normalization summary artifacts. 10. Chunk normalized transcript and compute chunk summaries. 11. Write chunking summary artifact. -12. Optionally execute runner modules sequentially when an injected module registry/factory is available (used by deterministic tests today). +12. Optionally execute runner modules sequentially when a module factory is injected (tests currently use this path; production defaults still avoid real module execution). 13. Output working transcript to `--output` file or stdout. -14. Build process report (`phase` currently set to `phase10-llm-validators`). +14. Build process report (`phase` currently set to `phase11-proposal-generation-framework`). 15. Optionally write `--report-json`; always write run-dir `report.json`. 16. Apply work-dir retention. @@ -183,7 +193,7 @@ Implemented config surfaces include: - work-dir and retention mode Current caveat: -- LLM/module-related settings are mostly infrastructure-only today; runtime path does not execute LLM or modules. +- LLM/module-related settings are mostly infrastructure-only today; default runtime path does not execute real modules. ## Implemented structured LLM infrastructure `internal/framework/contracts` now defines a typed structured-completion contract: @@ -280,6 +290,35 @@ Validator rejections are reported distinctly from proposal-application skips. - bounded scheduler hooks for validator call execution; - diagnostics writer hooks for machine-readable prompt/response artifacts with secret redaction. +## Implemented shared proposal-generation infrastructure +`internal/framework/proposal_generation` provides a reusable, prompt-agnostic helper for future real modules: +- structured request model including module key/instance, replacement policy, working transcript context, optional section metadata, glossary, config, and diagnostics context; +- structured correction-set response model (`corrections`) mapped into existing `proposals.CorrectionProposal` and `proposals.EnrichedCorrectionProposal` models; +- deterministic proposal-index assignment through a caller-provided `start_index`; +- structured LLM calls through `contracts.StructuredLLMClient` only (no direct provider calls); +- optional bounded execution through scheduler hooks (`contracts.LLMScheduler`); +- prompt/response diagnostics artifact writing via the generic `internal/framework/llm` diagnostics primitives with redaction of API keys/secrets. + +This helper only produces candidate proposals; validator-chain execution and proposal application remain runner responsibilities. + +## Implemented production module-registry scaffolding +`internal/framework/modules` now provides a production registry scaffold: +- recognizes intended module keys: + - `glossary` + - `homophones` + - `spoken_word` + - `grammar` +- supports explicit constructor registration with dependency injection for: + - run spec + - config + - glossary + - proposal/validation structured LLM clients + - proposal/validation schedulers + - diagnostics directory context +- returns explicit errors for unknown keys (`unsupported_module`) and recognized-but-unimplemented keys (`unimplemented_module`). + +No real production correction modules are registered yet. + ## Reports and diagnostics (implemented) Current per-run artifacts include: - `source-transcript.json` @@ -321,7 +360,7 @@ Current runtime note: - real module execution is not implemented yet, so normal successful runs generally have no skipped corrections and `auto` typically removes clean successful run directories. Intentionally deferred to module/LLM phases: -- module proposal-generation prompt/response diagnostics remain tied to later real-module phases. +- real domain proposal prompts and production module implementations remain tied to later module phases. ## Current tests and quality posture Implemented tests currently cover: @@ -337,8 +376,10 @@ Implemented tests currently cover: - CLI runner integration through injected fake module factories (`internal/cli/run_test.go`) - validator models, cardinality enforcement, and deterministic validators (`internal/framework/validators/*_test.go`) - LLM-backed validator batching, prompt builders, structured-response safety, scheduler hooks, and diagnostics redaction (`internal/framework/validators/*_test.go`, `internal/framework/runner/*_test.go`) +- shared proposal-generation request/response parsing, deterministic indexing, scheduler hooks, and diagnostics redaction (`internal/framework/proposal_generation/*_test.go`, `internal/framework/runner/*_test.go`) +- production module-registry known-key recognition and unsupported/unimplemented error behavior (`internal/framework/modules/*_test.go`, `internal/cli/run_test.go`) -Not covered yet (because not implemented): shared module proposal generation, real module implementations, and full transcript-polishing runtime behavior. +Not covered yet (because not implemented): real production module implementations and full transcript-polishing runtime behavior. ## Intended final architecture (not yet implemented) The intended end-state still matches the rewrite plan: diff --git a/docs/rewrite-notes.md b/docs/rewrite-notes.md index b115847..b8f09cf 100644 --- a/docs/rewrite-notes.md +++ b/docs/rewrite-notes.md @@ -55,6 +55,10 @@ Implemented: - LLM validator batching by validation prompt-token budget. - LLM validator runtime integration through structured LLM client abstraction and scheduler hooks. - LLM validator prompt/response diagnostics artifact wiring with secret redaction. +- Shared LLM proposal-generation helper with structured correction-set parsing. +- Deterministic proposal-index assignment for shared proposal generation. +- Proposal-generation prompt/response diagnostics artifact wiring with secret redaction. +- Production module-registry scaffolding with known key recognition and explicit unsupported/unimplemented errors. - Broad deterministic and CLI/subprocess test coverage for implemented phases through `go test ./...`. - Internal typed structured LLM contract (`StructuredLLMClient.CompleteStructured(ctx, req, out)`). - `internal/framework/llm` instructor-go-backed adapter with: @@ -71,7 +75,7 @@ Implemented: Not yet implemented in runtime pipeline: - Real correction modules. -- Shared module proposal generation and module registry wiring. +- Domain proposal prompts for real modules. - End-to-end transcript polishing behavior. ## Completed phases @@ -210,7 +214,7 @@ Not implemented in Phase 8 (by design): ## Remaining work plan -Next recommended phase: **Phase 11 (shared LLM proposal generation framework and module registry)**. +Next recommended phase: **Phase 12 (grammar module)**. ## Phase 9: Structured LLM client and scheduler infrastructure @@ -261,7 +265,7 @@ Do not implement: ### Expected behavior at end of phase -The codebase has tested Phase 9 LLM infrastructure, but default CLI runtime behavior remains deterministic preprocessing/reporting because real modules and LLM-backed validators are not implemented. +At the end of Phase 9, the codebase had tested LLM infrastructure primitives, while default CLI runtime behavior remained deterministic preprocessing/reporting because real modules were not implemented yet. ### Definition of done status @@ -298,48 +302,40 @@ Implemented: Not implemented in Phase 10 (by design): - Real correction modules (`glossary`, `homophones`, `spoken_word`, `grammar`). -- Shared module proposal generation and module registry work (Phase 11). +- Real module implementation and full runtime wiring (Phase 12+). - Domain proposal prompts. - Default CLI end-to-end transcript polishing behavior. ## Phase 11: Shared LLM proposal generation framework and module registry -### Purpose +Completed. -Create the reusable proposal-generation layer used by all real modules, and establish the real module registry without yet requiring all modules to be fully implemented. +Implemented: +- Shared proposal-generation package `internal/framework/proposal_generation`. +- Reusable request model for proposal generation including: + - module key/instance + - replacement policy + - working transcript context + - optional section metadata + - glossary/config context + - diagnostics context + - injected structured LLM client/scheduler dependencies. +- Structured correction-set response model and parsing into existing proposal models: + - `proposals.CorrectionProposal` + - `proposals.EnrichedCorrectionProposal`. +- Deterministic proposal-index assignment via caller-provided start index. +- Proposal-generation diagnostics artifact writing using generic LLM diagnostics primitives with secret redaction. +- Scheduler-aware proposal generation through the internal LLM scheduler interface. +- Production module-registry scaffolding in `internal/framework/modules` with: + - known module-key recognition for `glossary`, `homophones`, `spoken_word`, `grammar` + - constructor registration and dependency-injection path + - explicit unsupported and recognized-but-unimplemented module errors. +- Runner/CLI injection-path tests showing shared proposal generation can flow through runner validation/application semantics using fake modules/clients. -### Scope - -Implement: -- Shared LLM proposal generation helper. -- Proposal prompt request/response models. -- Structured correction set response parsing. -- Proposal index assignment. -- Module prompt/response diagnostics. -- Module registry package for real module keys. -- Module construction from run specs. -- Clean unsupported-module behavior. -- Shared module test harness using fake LLM responses. -- One minimal real module may be implemented as a proof of the proposal-generation path if that keeps the phase coherent, but only if it does not blur scope. - -Do not implement: -- All real modules. -- Full default pipeline parity. -- Prompt improvements beyond faithful porting of Python behavior. - -### Expected behavior at end of phase - -The framework can support real LLM proposal generation, and modules can be registered and instantiated consistently. At least the infrastructure for real modules exists, even if most modules are implemented in later phases. - -### Definition of done - -- Shared proposal-generation helper exists. -- Proposal prompt/response diagnostics are written for module proposal calls. -- Module registry resolves known module keys deterministically. -- Unsupported modules fail cleanly. -- Fake module tests exercise shared proposal-generation behavior. -- Module reports include proposal-generation failures where applicable. -- `go test ./...` passes. +Not implemented in Phase 11 (by design): +- Real production `glossary`, `homophones`, `spoken_word`, and `grammar` modules. +- Domain proposal prompts for production modules. +- Default CLI end-to-end transcript polishing behavior. ## Phase 12: Grammar module diff --git a/internal/cli/run.go b/internal/cli/run.go index 912e0bc..1bea029 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -30,6 +30,8 @@ type processInvocation struct { } var processModuleFactory runner.ModuleFactory +var processProposalLLMClient contracts.StructuredLLMClient +var processProposalLLMScheduler runner.ValidationScheduler var processValidationLLMClient contracts.StructuredLLMClient var processValidationLLMScheduler runner.ValidationScheduler @@ -136,6 +138,9 @@ var processRunner = func(inv processInvocation, stdout io.Writer) (*normalizatio Transcript: normalizedTranscript, Glossary: glossary, ModuleSpecs: moduleSpecs, + ProposalLLMClient: processProposalLLMClient, + ProposalLLMScheduler: processProposalLLMScheduler, + ProposalDiagnosticsDir: runDir.Path(), ValidationLLMClient: processValidationLLMClient, ValidationLLMScheduler: processValidationLLMScheduler, ValidationDiagnosticsDir: runDir.Path(), @@ -400,7 +405,7 @@ func extractErrorPhase(err error) (phase string, message string) { func buildProcessReport(status string, inv processInvocation, runDir *diagnostics.RunDirectory, startedAt, completedAt time.Time, errorMessage string, errorPhase string, normalizationSummary *normalization.NormalizationSummary, chunkingSummary *chunking.Summary, runOutput *runner.RunOutput) reporting.ProcessReport { report := reporting.ProcessReport{ - Phase: "phase10-llm-validators", + Phase: "phase11-proposal-generation-framework", Status: status, Operation: "process", TranscriptPath: inv.TranscriptPath, diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index 1e1f744..7ca607e 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -14,6 +14,7 @@ import ( "gitea.maximumdirect.net/eric/audita/internal/core/reporting" "gitea.maximumdirect.net/eric/audita/internal/core/schema" "gitea.maximumdirect.net/eric/audita/internal/framework/contracts" + "gitea.maximumdirect.net/eric/audita/internal/framework/modules" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/internal/framework/validators" ) @@ -614,8 +615,8 @@ func TestRunProcessReportJSONIncludesChunkingSummary(t *testing.T) { if report.Chunking.MaxSectionTokens == 0 { t.Errorf("expected max_section_tokens in report") } - if report.Phase != "phase10-llm-validators" { - t.Errorf("expected phase 'phase10-llm-validators', got %q", report.Phase) + if report.Phase != "phase11-proposal-generation-framework" { + t.Errorf("expected phase 'phase11-proposal-generation-framework', got %q", report.Phase) } } @@ -894,6 +895,51 @@ func TestRunProcessInjectedFactoryFailureWritesFailedReport(t *testing.T) { } } +func TestRunProcessProductionRegistryUnimplementedModuleFailsCleanly(t *testing.T) { + cfg := modules.Dependencies{} + processModuleFactory = modules.NewFactory(cfg) + t.Cleanup(func() { processModuleFactory = nil }) + + var stdout, stderr bytes.Buffer + workDir := t.TempDir() + reportPath := filepath.Join(t.TempDir(), "report.json") + transcriptPath := writeFile(t, "transcript.json", `[ + {"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"} +]`) + + exitCode := Run([]string{ + "process", transcriptPath, + "--glossary", fixturePath("tiny_glossary.yaml"), + "--modules", "glossary", + "--work-dir", workDir, + "--work-dir-retention", "always", + "--report-json", reportPath, + }, &stdout, &stderr) + if exitCode == 0 { + t.Fatal("expected failure exit code") + } + if stdout.Len() != 0 { + t.Fatalf("expected empty stdout on failure, got %q", stdout.String()) + } + if !strings.Contains(stderr.String(), "runner_execution") { + t.Fatalf("expected runner_execution failure on stderr, got %q", stderr.String()) + } + if !strings.Contains(stderr.String(), "recognized but not implemented") { + t.Fatalf("expected explicit unimplemented module message, got %q", stderr.String()) + } + + report := readProcessReport(t, reportPath) + if report.Status != "failed" { + t.Fatalf("expected failed report status, got %q", report.Status) + } + if report.ErrorPhase != "runner_execution" { + t.Fatalf("expected runner_execution phase, got %q", report.ErrorPhase) + } + if !strings.Contains(report.ErrorMessage, "recognized but not implemented") { + t.Fatalf("expected report error message to mention unimplemented module, got %q", report.ErrorMessage) + } +} + func TestRunProcessChunkingSummaryArtifactWritten(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer diff --git a/internal/core/reporting/report_test.go b/internal/core/reporting/report_test.go index 5c0f760..83afcde 100644 --- a/internal/core/reporting/report_test.go +++ b/internal/core/reporting/report_test.go @@ -11,7 +11,7 @@ import ( func TestProcessReportModuleResultsJSONSuccessAndSkipped(t *testing.T) { now := time.Now().UTC() report := ProcessReport{ - Phase: "phase10-llm-validators", + Phase: "phase11-proposal-generation-framework", Status: "success", ModuleResults: []ModuleReport{ { @@ -80,7 +80,7 @@ func TestProcessReportModuleResultsJSONSuccessAndSkipped(t *testing.T) { func TestProcessReportModuleResultsJSONFailedModule(t *testing.T) { now := time.Now().UTC() report := ProcessReport{ - Phase: "phase10-llm-validators", + Phase: "phase11-proposal-generation-framework", Status: "failed", ModuleResults: []ModuleReport{ { diff --git a/internal/framework/contracts/contracts.go b/internal/framework/contracts/contracts.go index 3fe1472..23ac31a 100644 --- a/internal/framework/contracts/contracts.go +++ b/internal/framework/contracts/contracts.go @@ -17,6 +17,11 @@ type StructuredLLMClient interface { CompleteStructured(ctx context.Context, req StructuredCompletionRequest, out any) (StructuredCompletionResponse, error) } +// LLMScheduler provides bounded execution for LLM call sites. +type LLMScheduler interface { + Run(ctx context.Context, fn func(context.Context) error) error +} + // TranscriptModule is the minimal contract for framework-integrated modules. type TranscriptModule interface { Key() string @@ -92,8 +97,9 @@ type ExecutionContext struct { // ProposalRequest is the input to module proposal generation. type ProposalRequest struct { ExecutionContext - RunSpec ModuleRunSpec `json:"run_spec"` - LLMClient StructuredLLMClient `json:"-"` + RunSpec ModuleRunSpec `json:"run_spec"` + LLMClient StructuredLLMClient `json:"-"` + LLMScheduler LLMScheduler `json:"-"` } // ValidationRequest is the input to validator execution. diff --git a/internal/framework/modules/registry.go b/internal/framework/modules/registry.go new file mode 100644 index 0000000..e57424f --- /dev/null +++ b/internal/framework/modules/registry.go @@ -0,0 +1,153 @@ +package modules + +import ( + "context" + "fmt" + "strings" + + "gitea.maximumdirect.net/eric/audita/internal/core/config" + "gitea.maximumdirect.net/eric/audita/internal/core/schema" + "gitea.maximumdirect.net/eric/audita/internal/framework/contracts" +) + +const ( + ModuleKeyGlossary = "glossary" + ModuleKeyHomophones = "homophones" + ModuleKeySpokenWord = "spoken_word" + ModuleKeyGrammar = "grammar" +) + +const ( + ReasonUnsupportedModule = "unsupported_module" + ReasonUnimplementedModule = "unimplemented_module" +) + +var knownModuleKeys = map[string]struct{}{ + ModuleKeyGlossary: {}, + ModuleKeyHomophones: {}, + ModuleKeySpokenWord: {}, + ModuleKeyGrammar: {}, +} + +// IsKnownModuleKey reports whether a module key is recognized by the production +// registry scaffold. +func IsKnownModuleKey(key string) bool { + _, ok := knownModuleKeys[strings.TrimSpace(key)] + return ok +} + +// Dependencies holds explicit constructor dependencies for module creation. +type Dependencies struct { + Config *config.Config + Glossary *schema.Glossary + ProposalLLMClient contracts.StructuredLLMClient + ProposalLLMScheduler contracts.LLMScheduler + ValidationLLMClient contracts.StructuredLLMClient + ValidationLLMScheduler contracts.LLMScheduler + DiagnosticsDir string +} + +// ConstructRequest is one module-construction request. +type ConstructRequest struct { + RunSpec contracts.ModuleRunSpec + Dependencies +} + +// Constructor builds one module instance from a run spec and explicit deps. +type Constructor func(ctx context.Context, req ConstructRequest) (contracts.TranscriptModule, error) + +// Factory resolves configured module specs into module instances. +type Factory struct { + deps Dependencies + constructors map[string]Constructor +} + +// NewFactory creates a production registry scaffold with known module keys but +// no real module constructors registered yet. +func NewFactory(deps Dependencies) *Factory { + return &Factory{ + deps: deps, + constructors: make(map[string]Constructor, len(knownModuleKeys)), + } +} + +// RegisterConstructor registers a constructor for a known module key. +func (f *Factory) RegisterConstructor(moduleKey string, constructor Constructor) error { + if f == nil { + return fmt.Errorf("module factory is nil") + } + key := strings.TrimSpace(moduleKey) + if !IsKnownModuleKey(key) { + return &UnsupportedModuleError{ModuleKey: key} + } + if constructor == nil { + return fmt.Errorf("constructor for module %q must not be nil", key) + } + f.constructors[key] = constructor + return nil +} + +// ModuleForSpec resolves one configured run spec into a module instance. +func (f *Factory) ModuleForSpec(spec contracts.ModuleRunSpec) (contracts.TranscriptModule, error) { + if f == nil { + return nil, fmt.Errorf("module factory is nil") + } + + key := strings.TrimSpace(spec.ModuleKey) + if !IsKnownModuleKey(key) { + return nil, &UnsupportedModuleError{ModuleKey: key} + } + + constructor, ok := f.constructors[key] + if !ok || constructor == nil { + return nil, &UnimplementedModuleError{ModuleKey: key} + } + + module, err := constructor(context.Background(), ConstructRequest{ + RunSpec: spec, + Dependencies: Dependencies{ + Config: f.deps.Config, + Glossary: f.deps.Glossary, + ProposalLLMClient: f.deps.ProposalLLMClient, + ProposalLLMScheduler: f.deps.ProposalLLMScheduler, + ValidationLLMClient: f.deps.ValidationLLMClient, + ValidationLLMScheduler: f.deps.ValidationLLMScheduler, + DiagnosticsDir: f.deps.DiagnosticsDir, + }, + }) + if err != nil { + return nil, fmt.Errorf("construct module %q: %w", spec.InstanceName, err) + } + if module == nil { + return nil, fmt.Errorf("constructor for module %q returned nil module", key) + } + return module, nil +} + +// UnsupportedModuleError indicates a configured module key is unknown. +type UnsupportedModuleError struct { + ModuleKey string +} + +func (e *UnsupportedModuleError) Error() string { + return fmt.Sprintf("unsupported module key %q", strings.TrimSpace(e.ModuleKey)) +} + +// ReasonCode returns a stable reason code suitable for reporting. +func (e *UnsupportedModuleError) ReasonCode() string { + return ReasonUnsupportedModule +} + +// UnimplementedModuleError indicates a known module key without constructor. +type UnimplementedModuleError struct { + ModuleKey string +} + +func (e *UnimplementedModuleError) Error() string { + return fmt.Sprintf("module %q is recognized but not implemented", strings.TrimSpace(e.ModuleKey)) +} + +// ReasonCode returns a stable reason code suitable for reporting. +func (e *UnimplementedModuleError) ReasonCode() string { + return ReasonUnimplementedModule +} diff --git a/internal/framework/modules/registry_test.go b/internal/framework/modules/registry_test.go new file mode 100644 index 0000000..35392af --- /dev/null +++ b/internal/framework/modules/registry_test.go @@ -0,0 +1,125 @@ +package modules + +import ( + "context" + "errors" + "testing" + + "gitea.maximumdirect.net/eric/audita/internal/framework/contracts" + "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" +) + +type noopModule struct { + key string +} + +func (m noopModule) Key() string { return m.key } +func (m noopModule) ReplacementPolicy() proposals.ReplacementPolicy { + return proposals.ReplacementPolicyRequireUnique +} +func (m noopModule) Validators() []contracts.Validator { return nil } +func (m noopModule) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { + _ = ctx + _ = req + return nil, nil +} + +func TestKnownModuleKeyRecognition(t *testing.T) { + for _, key := range []string{ModuleKeyGlossary, ModuleKeyHomophones, ModuleKeySpokenWord, ModuleKeyGrammar} { + if !IsKnownModuleKey(key) { + t.Fatalf("expected key %q to be recognized", key) + } + } +} + +func TestUnknownModuleKeyNotRecognized(t *testing.T) { + if IsKnownModuleKey("made_up") { + t.Fatal("expected unknown key to be unrecognized") + } +} + +func TestRepeatedRunSpecNamingRemainsDeterministic(t *testing.T) { + specs, err := contracts.ResolveModuleRunSpecs([]string{"glossary", "glossary", "grammar"}) + if err != nil { + t.Fatalf("ResolveModuleRunSpecs error: %v", err) + } + if specs[0].InstanceName != "glossary_1" || specs[1].InstanceName != "glossary_2" || specs[2].InstanceName != "grammar" { + t.Fatalf("unexpected instance names: %+v", specs) + } +} + +func TestUnsupportedUnknownModuleKeyFailsCleanly(t *testing.T) { + factory := NewFactory(Dependencies{}) + _, err := factory.ModuleForSpec(contracts.ModuleRunSpec{ModuleKey: "unknown", InstanceName: "unknown"}) + if err == nil { + t.Fatal("expected unsupported-module error") + } + + var unsupported *UnsupportedModuleError + if !errors.As(err, &unsupported) { + t.Fatalf("expected UnsupportedModuleError, got %T (%v)", err, err) + } + if unsupported.ReasonCode() != ReasonUnsupportedModule { + t.Fatalf("unexpected reason code: %q", unsupported.ReasonCode()) + } +} + +func TestRecognizedButUnimplementedModuleKeyFailsCleanly(t *testing.T) { + factory := NewFactory(Dependencies{}) + for _, key := range []string{ModuleKeyGlossary, ModuleKeyHomophones, ModuleKeySpokenWord, ModuleKeyGrammar} { + t.Run(key, func(t *testing.T) { + _, err := factory.ModuleForSpec(contracts.ModuleRunSpec{ModuleKey: key, InstanceName: key}) + if err == nil { + t.Fatal("expected unimplemented-module error") + } + + var unimplemented *UnimplementedModuleError + if !errors.As(err, &unimplemented) { + t.Fatalf("expected UnimplementedModuleError, got %T (%v)", err, err) + } + if unimplemented.ReasonCode() != ReasonUnimplementedModule { + t.Fatalf("unexpected reason code: %q", unimplemented.ReasonCode()) + } + }) + } +} + +func TestRegisterConstructorAndConstruct(t *testing.T) { + factory := NewFactory(Dependencies{}) + if err := factory.RegisterConstructor(ModuleKeyGlossary, func(ctx context.Context, req ConstructRequest) (contracts.TranscriptModule, error) { + _ = ctx + if req.RunSpec.InstanceName != "glossary_1" { + t.Fatalf("expected run spec instance name, got %q", req.RunSpec.InstanceName) + } + return noopModule{key: req.RunSpec.ModuleKey}, nil + }); err != nil { + t.Fatalf("RegisterConstructor error: %v", err) + } + + module, err := factory.ModuleForSpec(contracts.ModuleRunSpec{ + ModuleKey: ModuleKeyGlossary, + InstanceName: "glossary_1", + }) + if err != nil { + t.Fatalf("ModuleForSpec error: %v", err) + } + if module.Key() != ModuleKeyGlossary { + t.Fatalf("unexpected module key %q", module.Key()) + } +} + +func TestRegisterConstructorRejectsUnknownModuleKey(t *testing.T) { + factory := NewFactory(Dependencies{}) + err := factory.RegisterConstructor("unknown", func(ctx context.Context, req ConstructRequest) (contracts.TranscriptModule, error) { + _ = ctx + _ = req + return noopModule{key: "unknown"}, nil + }) + if err == nil { + t.Fatal("expected register failure for unknown key") + } + var unsupported *UnsupportedModuleError + if !errors.As(err, &unsupported) { + t.Fatalf("expected UnsupportedModuleError, got %T (%v)", err, err) + } +} diff --git a/internal/framework/proposal_generation/generate.go b/internal/framework/proposal_generation/generate.go new file mode 100644 index 0000000..320eab0 --- /dev/null +++ b/internal/framework/proposal_generation/generate.go @@ -0,0 +1,235 @@ +// Package proposal_generation provides shared, deterministic LLM-backed +// proposal-generation helpers. It only produces candidate proposals; validation +// and application remain runner responsibilities. +package proposal_generation + +import ( + "context" + "fmt" + "path/filepath" + "strings" + + "gitea.maximumdirect.net/eric/audita/internal/core/config" + "gitea.maximumdirect.net/eric/audita/internal/core/schema" + "gitea.maximumdirect.net/eric/audita/internal/framework/contracts" + "gitea.maximumdirect.net/eric/audita/internal/framework/llm" + "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" +) + +// InteractionDiagnosticsWriter writes machine-readable prompt/response artifacts. +type InteractionDiagnosticsWriter interface { + WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) +} + +// InteractionArtifacts contains written diagnostics paths. +type InteractionArtifacts struct { + RequestMetadataPath string `json:"request_metadata_path,omitempty"` + RequestPayloadPath string `json:"request_payload_path,omitempty"` + ResponsePayloadPath string `json:"response_payload_path,omitempty"` + ErrorPayloadPath string `json:"error_payload_path,omitempty"` +} + +// StructuredCorrectionProposal is one LLM response correction payload. +type StructuredCorrectionProposal struct { + TargetSegmentID int `json:"id"` + OriginalText string `json:"original_text"` + CorrectedText string `json:"corrected_text"` + Confidence float64 `json:"confidence"` +} + +// StructuredCorrectionSet is the reusable structured LLM response model for +// candidate correction proposals. +type StructuredCorrectionSet struct { + Corrections []StructuredCorrectionProposal `json:"corrections"` +} + +// Request captures reusable proposal-generation inputs for future modules. +type Request struct { + ModuleKey string `json:"module_key"` + ModuleInstance string `json:"module_instance"` + ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"` + WorkingTranscript *schema.Transcript `json:"-"` + Section *contracts.SectionMetadata `json:"section,omitempty"` + Glossary *schema.Glossary `json:"-"` + Config *config.Config `json:"-"` + Messages []contracts.LLMMessage `json:"messages"` + Model string `json:"model,omitempty"` + StartIndex int `json:"start_index"` + LLMClient contracts.StructuredLLMClient + Scheduler contracts.LLMScheduler + DiagnosticsDir string + DiagnosticsWriter InteractionDiagnosticsWriter +} + +// Result contains generated candidate proposals and optional diagnostics paths. +type Result struct { + Corrections []proposals.CorrectionProposal `json:"corrections"` + Enriched []proposals.EnrichedCorrectionProposal `json:"enriched"` + Artifacts InteractionArtifacts `json:"artifacts,omitempty"` +} + +// GenerateCandidates executes one structured LLM call and deterministically maps +// its correction-set response into framework proposal types. +func GenerateCandidates(ctx context.Context, req Request) (Result, error) { + if strings.TrimSpace(req.ModuleKey) == "" { + return Result{}, fmt.Errorf("module key must not be empty") + } + if strings.TrimSpace(req.ModuleInstance) == "" { + return Result{}, fmt.Errorf("module instance must not be empty") + } + if req.StartIndex < 0 { + return Result{}, fmt.Errorf("start index must be non-negative") + } + if req.LLMClient == nil { + return Result{}, fmt.Errorf("structured LLM client is required") + } + if len(req.Messages) == 0 { + return Result{}, fmt.Errorf("messages must not be empty") + } + + stage := buildStageName(req.ModuleInstance, req.Section) + model := resolveModel(req.Config, req.Model) + messages := append([]contracts.LLMMessage(nil), req.Messages...) + + var writer InteractionDiagnosticsWriter + if req.DiagnosticsWriter != nil { + writer = req.DiagnosticsWriter + } else if strings.TrimSpace(req.DiagnosticsDir) != "" { + writer = diagnosticsWriterAdapter{ + writer: llm.NewDiagnosticsWriter( + filepath.Join(req.DiagnosticsDir, req.ModuleInstance), + proposalGenerationSecrets(req.Config), + ), + } + } + + var ( + response StructuredCorrectionSet + callErr error + artifacts InteractionArtifacts + ) + call := func(callCtx context.Context) error { + _, callErr = req.LLMClient.CompleteStructured(callCtx, contracts.StructuredCompletionRequest{ + StageName: stage, + Messages: messages, + Model: model, + }, &response) + return callErr + } + if req.Scheduler != nil { + callErr = req.Scheduler.Run(ctx, call) + } else { + callErr = call(ctx) + } + + if writer != nil { + artifacts, _ = writer.WriteInteraction( + stage, + map[string]any{ + "module_key": req.ModuleKey, + "module_instance": req.ModuleInstance, + "replacement_policy": req.ReplacementPolicy, + "section": req.Section, + "start_index": req.StartIndex, + "model": model, + }, + map[string]any{ + "messages": messages, + }, + response, + errPayload(callErr), + ) + } + + if callErr != nil { + return Result{}, fmt.Errorf("proposal generation completion failed: %w", callErr) + } + + corrections := make([]proposals.CorrectionProposal, 0, len(response.Corrections)) + enriched := make([]proposals.EnrichedCorrectionProposal, 0, len(response.Corrections)) + for i, raw := range response.Corrections { + candidate := proposals.CorrectionProposal{ + TargetSegmentID: raw.TargetSegmentID, + OriginalText: raw.OriginalText, + CorrectedText: raw.CorrectedText, + Confidence: raw.Confidence, + } + if err := candidate.Validate(); err != nil { + return Result{}, fmt.Errorf("invalid structured correction at index %d: %w", i, err) + } + + corrections = append(corrections, candidate) + enrichedCandidate := proposals.EnrichedCorrectionProposal{ + CorrectionProposal: candidate, + ProposalMetadata: proposals.ProposalMetadata{ + ProposalIndex: req.StartIndex + i, + ModuleKey: req.ModuleKey, + ModuleInstance: req.ModuleInstance, + }, + } + if req.Section != nil { + sectionIndex := req.Section.Index + enrichedCandidate.SectionIndex = §ionIndex + } + enriched = append(enriched, enrichedCandidate) + } + + return Result{ + Corrections: corrections, + Enriched: enriched, + Artifacts: artifacts, + }, nil +} + +func buildStageName(moduleInstance string, section *contracts.SectionMetadata) string { + base := fmt.Sprintf("%s:proposal-generation", moduleInstance) + if section == nil { + return base + } + return fmt.Sprintf("%s:section-%04d", base, section.Index) +} + +func resolveModel(cfg *config.Config, override string) string { + if strings.TrimSpace(override) != "" { + return strings.TrimSpace(override) + } + if cfg == nil { + return "" + } + return llm.ResolvePrimaryConfig(*cfg).Model +} + +func proposalGenerationSecrets(cfg *config.Config) []string { + if cfg == nil { + return nil + } + return []string{ + cfg.PrimaryLLM.APIKey, + cfg.ValidationLLM.APIKey, + cfg.EffectiveValidationLLMConfig().APIKey, + } +} + +func errPayload(err error) any { + if err == nil { + return nil + } + return map[string]any{"error": err.Error()} +} + +type diagnosticsWriterAdapter struct { + writer *llm.DiagnosticsWriter +} + +func (a diagnosticsWriterAdapter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) { + artifacts, err := a.writer.WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload) + if err != nil { + return InteractionArtifacts{}, err + } + return InteractionArtifacts{ + RequestMetadataPath: artifacts.RequestMetadataPath, + RequestPayloadPath: artifacts.RequestPayloadPath, + ResponsePayloadPath: artifacts.ResponsePayloadPath, + ErrorPayloadPath: artifacts.ErrorPayloadPath, + }, nil +} diff --git a/internal/framework/proposal_generation/generate_test.go b/internal/framework/proposal_generation/generate_test.go new file mode 100644 index 0000000..5186c95 --- /dev/null +++ b/internal/framework/proposal_generation/generate_test.go @@ -0,0 +1,267 @@ +package proposal_generation + +import ( + "context" + "errors" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/audita/internal/core/config" + "gitea.maximumdirect.net/eric/audita/internal/framework/contracts" + "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" +) + +type fakeStructuredClient struct { + responses []StructuredCorrectionSet + err error + calls []contracts.StructuredCompletionRequest +} + +func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { + _ = ctx + f.calls = append(f.calls, req) + if f.err != nil { + return contracts.StructuredCompletionResponse{}, f.err + } + target, ok := out.(*StructuredCorrectionSet) + if !ok { + return contracts.StructuredCompletionResponse{}, errors.New("unexpected output type") + } + if len(f.responses) == 0 { + return contracts.StructuredCompletionResponse{}, errors.New("unexpected call") + } + *target = f.responses[0] + f.responses = f.responses[1:] + return contracts.StructuredCompletionResponse{}, nil +} + +type countingScheduler struct { + runs int +} + +func (s *countingScheduler) Run(ctx context.Context, fn func(context.Context) error) error { + s.runs++ + return fn(ctx) +} + +func defaultRequest(t *testing.T) Request { + t.Helper() + cfg := config.Default() + return Request{ + ModuleKey: "test_module", + ModuleInstance: "test_module", + ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, + Config: &cfg, + Messages: []contracts.LLMMessage{ + {Role: "system", Content: "system prompt"}, + {Role: "user", Content: "user prompt"}, + }, + StartIndex: 0, + } +} + +func TestGenerateCandidatesSuccess(t *testing.T) { + client := &fakeStructuredClient{ + responses: []StructuredCorrectionSet{ + { + Corrections: []StructuredCorrectionProposal{ + {TargetSegmentID: 7, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}, + }, + }, + }, + } + section := contracts.SectionMetadata{Index: 2} + req := defaultRequest(t) + req.LLMClient = client + req.StartIndex = 10 + req.Section = §ion + + got, err := GenerateCandidates(context.Background(), req) + if err != nil { + t.Fatalf("GenerateCandidates error: %v", err) + } + if len(got.Corrections) != 1 || len(got.Enriched) != 1 { + t.Fatalf("unexpected proposal lengths: %+v", got) + } + if got.Enriched[0].ProposalIndex != 10 { + t.Fatalf("expected proposal index 10, got %d", got.Enriched[0].ProposalIndex) + } + if got.Enriched[0].SectionIndex == nil || *got.Enriched[0].SectionIndex != 2 { + t.Fatalf("expected section index 2, got %v", got.Enriched[0].SectionIndex) + } +} + +func TestGenerateCandidatesMalformedStructuredResponse(t *testing.T) { + client := &fakeStructuredClient{ + responses: []StructuredCorrectionSet{ + { + Corrections: []StructuredCorrectionProposal{ + {TargetSegmentID: 1, OriginalText: "x", CorrectedText: "", Confidence: 0.9}, + }, + }, + }, + } + req := defaultRequest(t) + req.LLMClient = client + + _, err := GenerateCandidates(context.Background(), req) + if err == nil || !strings.Contains(err.Error(), "invalid structured correction") { + t.Fatalf("expected structured response validation failure, got %v", err) + } +} + +func TestGenerateCandidatesDeterministicIndexAssignment(t *testing.T) { + baseResponse := StructuredCorrectionSet{ + Corrections: []StructuredCorrectionProposal{ + {TargetSegmentID: 1, OriginalText: "a", CorrectedText: "A", Confidence: 0.9}, + {TargetSegmentID: 2, OriginalText: "b", CorrectedText: "B", Confidence: 0.9}, + }, + } + clientA := &fakeStructuredClient{responses: []StructuredCorrectionSet{baseResponse}} + clientB := &fakeStructuredClient{responses: []StructuredCorrectionSet{baseResponse}} + + reqA := defaultRequest(t) + reqA.LLMClient = clientA + reqA.StartIndex = 3 + first, err := GenerateCandidates(context.Background(), reqA) + if err != nil { + t.Fatalf("first generation failed: %v", err) + } + + reqB := defaultRequest(t) + reqB.LLMClient = clientB + reqB.StartIndex = 3 + second, err := GenerateCandidates(context.Background(), reqB) + if err != nil { + t.Fatalf("second generation failed: %v", err) + } + + if !reflect.DeepEqual(first.Enriched, second.Enriched) { + t.Fatalf("expected stable enriched proposals\nfirst=%+v\nsecond=%+v", first.Enriched, second.Enriched) + } +} + +func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) { + client := &fakeStructuredClient{ + responses: []StructuredCorrectionSet{ + {Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "alpha", CorrectedText: "ALPHA", Confidence: 0.9}}}, + {Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 3, OriginalText: "charlie", CorrectedText: "CHARLIE", Confidence: 0.9}}}, + }, + } + section0 := contracts.SectionMetadata{Index: 0} + section1 := contracts.SectionMetadata{Index: 1} + + req0 := defaultRequest(t) + req0.LLMClient = client + req0.Section = §ion0 + req0.StartIndex = 0 + part0, err := GenerateCandidates(context.Background(), req0) + if err != nil { + t.Fatalf("section 0 generation failed: %v", err) + } + + req1 := defaultRequest(t) + req1.LLMClient = client + req1.Section = §ion1 + req1.StartIndex = len(part0.Enriched) + part1, err := GenerateCandidates(context.Background(), req1) + if err != nil { + t.Fatalf("section 1 generation failed: %v", err) + } + + all := append(append([]proposals.EnrichedCorrectionProposal(nil), part0.Enriched...), part1.Enriched...) + if len(all) != 2 { + t.Fatalf("expected 2 proposals, got %d", len(all)) + } + if all[0].ProposalIndex != 0 || all[1].ProposalIndex != 1 { + t.Fatalf("unexpected proposal indexes: %d, %d", all[0].ProposalIndex, all[1].ProposalIndex) + } + if all[0].SectionIndex == nil || *all[0].SectionIndex != 0 || all[1].SectionIndex == nil || *all[1].SectionIndex != 1 { + t.Fatalf("unexpected section metadata: %+v", all) + } +} + +func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) { + secret := "phase11-secret" + client := &fakeStructuredClient{ + responses: []StructuredCorrectionSet{ + {Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: secret, CorrectedText: "safe", Confidence: 0.9}}}, + }, + } + cfg := config.Default() + cfg.PrimaryLLM.APIKey = secret + req := defaultRequest(t) + req.Config = &cfg + req.LLMClient = client + req.DiagnosticsDir = t.TempDir() + req.Messages = []contracts.LLMMessage{ + {Role: "system", Content: "include secret " + secret}, + {Role: "user", Content: "fix it"}, + } + + got, err := GenerateCandidates(context.Background(), req) + if err != nil { + t.Fatalf("GenerateCandidates error: %v", err) + } + if got.Artifacts.ResponsePayloadPath == "" || got.Artifacts.RequestPayloadPath == "" { + t.Fatalf("expected diagnostics artifact paths, got %+v", got.Artifacts) + } + + for _, path := range []string{got.Artifacts.RequestPayloadPath, got.Artifacts.ResponsePayloadPath} { + raw, readErr := os.ReadFile(path) + if readErr != nil { + t.Fatalf("read artifact %q: %v", path, readErr) + } + if strings.Contains(string(raw), secret) { + t.Fatalf("artifact leaked secret %q: %s", path, string(raw)) + } + if !strings.Contains(string(raw), "[REDACTED]") { + t.Fatalf("expected redaction marker in artifact %q: %s", path, string(raw)) + } + } +} + +func TestGenerateCandidatesSchedulerUsage(t *testing.T) { + client := &fakeStructuredClient{ + responses: []StructuredCorrectionSet{ + {Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "x", CorrectedText: "y", Confidence: 0.9}}}, + }, + } + scheduler := &countingScheduler{} + req := defaultRequest(t) + req.LLMClient = client + req.Scheduler = scheduler + + _, err := GenerateCandidates(context.Background(), req) + if err != nil { + t.Fatalf("GenerateCandidates error: %v", err) + } + if scheduler.runs != 1 { + t.Fatalf("expected scheduler to run once, got %d", scheduler.runs) + } +} + +func TestGenerateCandidatesClientError(t *testing.T) { + client := &fakeStructuredClient{err: errors.New("boom")} + req := defaultRequest(t) + req.LLMClient = client + req.DiagnosticsDir = t.TempDir() + + got, err := GenerateCandidates(context.Background(), req) + if err == nil || !strings.Contains(err.Error(), "completion failed") { + t.Fatalf("expected completion failure, got %v", err) + } + if got.Artifacts.ResponsePayloadPath != "" { + t.Fatalf("expected zero result on error, got %+v", got) + } + matches, globErr := filepath.Glob(filepath.Join(req.DiagnosticsDir, req.ModuleInstance, "*error-payload.json")) + if globErr != nil { + t.Fatalf("glob error: %v", globErr) + } + if len(matches) == 0 { + t.Fatalf("expected error diagnostics artifact under %s", req.DiagnosticsDir) + } +} diff --git a/internal/framework/runner/runner.go b/internal/framework/runner/runner.go index 2e44d98..b294690 100644 --- a/internal/framework/runner/runner.go +++ b/internal/framework/runner/runner.go @@ -29,9 +29,7 @@ type Runner struct { factory ModuleFactory } -type ValidationScheduler interface { - Run(ctx context.Context, fn func(context.Context) error) error -} +type ValidationScheduler = contracts.LLMScheduler // ModuleResult captures deterministic per-module execution output. type ModuleResult struct { @@ -76,6 +74,9 @@ type RunInput struct { Transcript *schema.Transcript Glossary *schema.Glossary ModuleSpecs []contracts.ModuleRunSpec + ProposalLLMClient contracts.StructuredLLMClient + ProposalLLMScheduler contracts.LLMScheduler + ProposalDiagnosticsDir string ValidationLLMClient contracts.StructuredLLMClient ValidationLLMScheduler ValidationScheduler ValidationDiagnosticsDir string @@ -121,12 +122,15 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) { Config: input.Config, WorkingTranscript: working, Glossary: input.Glossary, + DiagnosticsDir: input.ProposalDiagnosticsDir, }, RunSpec: contracts.ModuleRunSpec{ ModuleKey: spec.ModuleKey, InstanceName: spec.InstanceName, ReplacementPolicy: policy, }, + LLMClient: input.ProposalLLMClient, + LLMScheduler: input.ProposalLLMScheduler, }) if err != nil { failed := ModuleResult{ diff --git a/internal/framework/runner/runner_test.go b/internal/framework/runner/runner_test.go index 7f289a3..432f982 100644 --- a/internal/framework/runner/runner_test.go +++ b/internal/framework/runner/runner_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "os" + "path/filepath" "strings" "testing" @@ -11,6 +12,7 @@ import ( "gitea.maximumdirect.net/eric/audita/internal/core/schema" "gitea.maximumdirect.net/eric/audita/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/internal/framework/llm" + "gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/internal/framework/validators" ) @@ -553,3 +555,99 @@ func TestRunnerAcceptsLLMSchedulerType(t *testing.T) { t.Fatal("expected scheduler instance") } } + +type proposalGenerationModule struct { + key string + policy proposals.ReplacementPolicy +} + +func (m proposalGenerationModule) Key() string { return m.key } +func (m proposalGenerationModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy } +func (m proposalGenerationModule) Validators() []contracts.Validator { return nil } +func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { + result, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{ + ModuleKey: req.RunSpec.ModuleKey, + ModuleInstance: req.RunSpec.InstanceName, + ReplacementPolicy: req.RunSpec.ReplacementPolicy, + WorkingTranscript: req.WorkingTranscript, + Config: req.Config, + Glossary: req.Glossary, + Messages: []contracts.LLMMessage{ + {Role: "system", Content: "return transcript corrections"}, + {Role: "user", Content: "produce one safe correction"}, + }, + LLMClient: req.LLMClient, + Scheduler: req.LLMScheduler, + DiagnosticsDir: req.DiagnosticsDir, + }) + if err != nil { + return nil, err + } + return result.Corrections, nil +} + +type fakeProposalStructuredClient struct { + responses []proposal_generation.StructuredCorrectionSet +} + +func (f *fakeProposalStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { + _ = ctx + _ = req + if len(f.responses) == 0 { + return contracts.StructuredCompletionResponse{}, errors.New("unexpected call") + } + target, ok := out.(*proposal_generation.StructuredCorrectionSet) + if !ok { + return contracts.StructuredCompletionResponse{}, errors.New("unexpected output type") + } + *target = f.responses[0] + f.responses = f.responses[1:] + return contracts.StructuredCompletionResponse{}, nil +} + +func TestRunnerProposalGenerationHelperFlowsThroughPipeline(t *testing.T) { + client := &fakeProposalStructuredClient{ + responses: []proposal_generation.StructuredCorrectionSet{ + { + Corrections: []proposal_generation.StructuredCorrectionProposal{ + {TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}, + }, + }, + }, + } + scheduler := &countingScheduler{} + cfg := config.Default() + diagDir := t.TempDir() + + r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ + "m": proposalGenerationModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique}, + }}) + + out, err := r.Run(context.Background(), RunInput{ + Config: &cfg, + Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}, + ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, + ProposalLLMClient: client, + ProposalLLMScheduler: scheduler, + ProposalDiagnosticsDir: diagDir, + }) + if err != nil { + t.Fatalf("unexpected run error: %v", err) + } + if out.FinalTranscript.Segments[0].Text != "the cat" { + t.Fatalf("expected proposal-generated correction to apply, got %q", out.FinalTranscript.Segments[0].Text) + } + if scheduler.runs != 1 { + t.Fatalf("expected proposal scheduler use, got %d runs", scheduler.runs) + } + if len(out.ModuleResults) != 1 || len(out.ModuleResults[0].AppliedChanges) != 1 { + t.Fatalf("expected one applied change, got %+v", out.ModuleResults) + } + matches, globErr := filepath.Glob(filepath.Join(diagDir, "m", "*proposal-generation*response-payload.json")) + if globErr != nil { + t.Fatalf("glob diagnostics: %v", globErr) + } + if len(matches) == 0 { + t.Fatalf("expected proposal-generation diagnostics artifacts in %s", filepath.Join(diagDir, "m")) + } +}