Complete Phase 11 proposal generation framework
This commit is contained in:
@@ -17,6 +17,11 @@ type StructuredLLMClient interface {
|
||||
CompleteStructured(ctx context.Context, req StructuredCompletionRequest, out any) (StructuredCompletionResponse, error)
|
||||
}
|
||||
|
||||
// LLMScheduler provides bounded execution for LLM call sites.
|
||||
type LLMScheduler interface {
|
||||
Run(ctx context.Context, fn func(context.Context) error) error
|
||||
}
|
||||
|
||||
// TranscriptModule is the minimal contract for framework-integrated modules.
|
||||
type TranscriptModule interface {
|
||||
Key() string
|
||||
@@ -92,8 +97,9 @@ type ExecutionContext struct {
|
||||
// ProposalRequest is the input to module proposal generation.
|
||||
type ProposalRequest struct {
|
||||
ExecutionContext
|
||||
RunSpec ModuleRunSpec `json:"run_spec"`
|
||||
LLMClient StructuredLLMClient `json:"-"`
|
||||
RunSpec ModuleRunSpec `json:"run_spec"`
|
||||
LLMClient StructuredLLMClient `json:"-"`
|
||||
LLMScheduler LLMScheduler `json:"-"`
|
||||
}
|
||||
|
||||
// ValidationRequest is the input to validator execution.
|
||||
|
||||
153
internal/framework/modules/registry.go
Normal file
153
internal/framework/modules/registry.go
Normal file
@@ -0,0 +1,153 @@
|
||||
package modules
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
)
|
||||
|
||||
const (
|
||||
ModuleKeyGlossary = "glossary"
|
||||
ModuleKeyHomophones = "homophones"
|
||||
ModuleKeySpokenWord = "spoken_word"
|
||||
ModuleKeyGrammar = "grammar"
|
||||
)
|
||||
|
||||
const (
|
||||
ReasonUnsupportedModule = "unsupported_module"
|
||||
ReasonUnimplementedModule = "unimplemented_module"
|
||||
)
|
||||
|
||||
var knownModuleKeys = map[string]struct{}{
|
||||
ModuleKeyGlossary: {},
|
||||
ModuleKeyHomophones: {},
|
||||
ModuleKeySpokenWord: {},
|
||||
ModuleKeyGrammar: {},
|
||||
}
|
||||
|
||||
// IsKnownModuleKey reports whether a module key is recognized by the production
|
||||
// registry scaffold.
|
||||
func IsKnownModuleKey(key string) bool {
|
||||
_, ok := knownModuleKeys[strings.TrimSpace(key)]
|
||||
return ok
|
||||
}
|
||||
|
||||
// Dependencies holds explicit constructor dependencies for module creation.
|
||||
type Dependencies struct {
|
||||
Config *config.Config
|
||||
Glossary *schema.Glossary
|
||||
ProposalLLMClient contracts.StructuredLLMClient
|
||||
ProposalLLMScheduler contracts.LLMScheduler
|
||||
ValidationLLMClient contracts.StructuredLLMClient
|
||||
ValidationLLMScheduler contracts.LLMScheduler
|
||||
DiagnosticsDir string
|
||||
}
|
||||
|
||||
// ConstructRequest is one module-construction request.
|
||||
type ConstructRequest struct {
|
||||
RunSpec contracts.ModuleRunSpec
|
||||
Dependencies
|
||||
}
|
||||
|
||||
// Constructor builds one module instance from a run spec and explicit deps.
|
||||
type Constructor func(ctx context.Context, req ConstructRequest) (contracts.TranscriptModule, error)
|
||||
|
||||
// Factory resolves configured module specs into module instances.
|
||||
type Factory struct {
|
||||
deps Dependencies
|
||||
constructors map[string]Constructor
|
||||
}
|
||||
|
||||
// NewFactory creates a production registry scaffold with known module keys but
|
||||
// no real module constructors registered yet.
|
||||
func NewFactory(deps Dependencies) *Factory {
|
||||
return &Factory{
|
||||
deps: deps,
|
||||
constructors: make(map[string]Constructor, len(knownModuleKeys)),
|
||||
}
|
||||
}
|
||||
|
||||
// RegisterConstructor registers a constructor for a known module key.
|
||||
func (f *Factory) RegisterConstructor(moduleKey string, constructor Constructor) error {
|
||||
if f == nil {
|
||||
return fmt.Errorf("module factory is nil")
|
||||
}
|
||||
key := strings.TrimSpace(moduleKey)
|
||||
if !IsKnownModuleKey(key) {
|
||||
return &UnsupportedModuleError{ModuleKey: key}
|
||||
}
|
||||
if constructor == nil {
|
||||
return fmt.Errorf("constructor for module %q must not be nil", key)
|
||||
}
|
||||
f.constructors[key] = constructor
|
||||
return nil
|
||||
}
|
||||
|
||||
// ModuleForSpec resolves one configured run spec into a module instance.
|
||||
func (f *Factory) ModuleForSpec(spec contracts.ModuleRunSpec) (contracts.TranscriptModule, error) {
|
||||
if f == nil {
|
||||
return nil, fmt.Errorf("module factory is nil")
|
||||
}
|
||||
|
||||
key := strings.TrimSpace(spec.ModuleKey)
|
||||
if !IsKnownModuleKey(key) {
|
||||
return nil, &UnsupportedModuleError{ModuleKey: key}
|
||||
}
|
||||
|
||||
constructor, ok := f.constructors[key]
|
||||
if !ok || constructor == nil {
|
||||
return nil, &UnimplementedModuleError{ModuleKey: key}
|
||||
}
|
||||
|
||||
module, err := constructor(context.Background(), ConstructRequest{
|
||||
RunSpec: spec,
|
||||
Dependencies: Dependencies{
|
||||
Config: f.deps.Config,
|
||||
Glossary: f.deps.Glossary,
|
||||
ProposalLLMClient: f.deps.ProposalLLMClient,
|
||||
ProposalLLMScheduler: f.deps.ProposalLLMScheduler,
|
||||
ValidationLLMClient: f.deps.ValidationLLMClient,
|
||||
ValidationLLMScheduler: f.deps.ValidationLLMScheduler,
|
||||
DiagnosticsDir: f.deps.DiagnosticsDir,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("construct module %q: %w", spec.InstanceName, err)
|
||||
}
|
||||
if module == nil {
|
||||
return nil, fmt.Errorf("constructor for module %q returned nil module", key)
|
||||
}
|
||||
return module, nil
|
||||
}
|
||||
|
||||
// UnsupportedModuleError indicates a configured module key is unknown.
|
||||
type UnsupportedModuleError struct {
|
||||
ModuleKey string
|
||||
}
|
||||
|
||||
func (e *UnsupportedModuleError) Error() string {
|
||||
return fmt.Sprintf("unsupported module key %q", strings.TrimSpace(e.ModuleKey))
|
||||
}
|
||||
|
||||
// ReasonCode returns a stable reason code suitable for reporting.
|
||||
func (e *UnsupportedModuleError) ReasonCode() string {
|
||||
return ReasonUnsupportedModule
|
||||
}
|
||||
|
||||
// UnimplementedModuleError indicates a known module key without constructor.
|
||||
type UnimplementedModuleError struct {
|
||||
ModuleKey string
|
||||
}
|
||||
|
||||
func (e *UnimplementedModuleError) Error() string {
|
||||
return fmt.Sprintf("module %q is recognized but not implemented", strings.TrimSpace(e.ModuleKey))
|
||||
}
|
||||
|
||||
// ReasonCode returns a stable reason code suitable for reporting.
|
||||
func (e *UnimplementedModuleError) ReasonCode() string {
|
||||
return ReasonUnimplementedModule
|
||||
}
|
||||
125
internal/framework/modules/registry_test.go
Normal file
125
internal/framework/modules/registry_test.go
Normal file
@@ -0,0 +1,125 @@
|
||||
package modules
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
)
|
||||
|
||||
type noopModule struct {
|
||||
key string
|
||||
}
|
||||
|
||||
func (m noopModule) Key() string { return m.key }
|
||||
func (m noopModule) ReplacementPolicy() proposals.ReplacementPolicy {
|
||||
return proposals.ReplacementPolicyRequireUnique
|
||||
}
|
||||
func (m noopModule) Validators() []contracts.Validator { return nil }
|
||||
func (m noopModule) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
_ = ctx
|
||||
_ = req
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func TestKnownModuleKeyRecognition(t *testing.T) {
|
||||
for _, key := range []string{ModuleKeyGlossary, ModuleKeyHomophones, ModuleKeySpokenWord, ModuleKeyGrammar} {
|
||||
if !IsKnownModuleKey(key) {
|
||||
t.Fatalf("expected key %q to be recognized", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnknownModuleKeyNotRecognized(t *testing.T) {
|
||||
if IsKnownModuleKey("made_up") {
|
||||
t.Fatal("expected unknown key to be unrecognized")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepeatedRunSpecNamingRemainsDeterministic(t *testing.T) {
|
||||
specs, err := contracts.ResolveModuleRunSpecs([]string{"glossary", "glossary", "grammar"})
|
||||
if err != nil {
|
||||
t.Fatalf("ResolveModuleRunSpecs error: %v", err)
|
||||
}
|
||||
if specs[0].InstanceName != "glossary_1" || specs[1].InstanceName != "glossary_2" || specs[2].InstanceName != "grammar" {
|
||||
t.Fatalf("unexpected instance names: %+v", specs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnsupportedUnknownModuleKeyFailsCleanly(t *testing.T) {
|
||||
factory := NewFactory(Dependencies{})
|
||||
_, err := factory.ModuleForSpec(contracts.ModuleRunSpec{ModuleKey: "unknown", InstanceName: "unknown"})
|
||||
if err == nil {
|
||||
t.Fatal("expected unsupported-module error")
|
||||
}
|
||||
|
||||
var unsupported *UnsupportedModuleError
|
||||
if !errors.As(err, &unsupported) {
|
||||
t.Fatalf("expected UnsupportedModuleError, got %T (%v)", err, err)
|
||||
}
|
||||
if unsupported.ReasonCode() != ReasonUnsupportedModule {
|
||||
t.Fatalf("unexpected reason code: %q", unsupported.ReasonCode())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecognizedButUnimplementedModuleKeyFailsCleanly(t *testing.T) {
|
||||
factory := NewFactory(Dependencies{})
|
||||
for _, key := range []string{ModuleKeyGlossary, ModuleKeyHomophones, ModuleKeySpokenWord, ModuleKeyGrammar} {
|
||||
t.Run(key, func(t *testing.T) {
|
||||
_, err := factory.ModuleForSpec(contracts.ModuleRunSpec{ModuleKey: key, InstanceName: key})
|
||||
if err == nil {
|
||||
t.Fatal("expected unimplemented-module error")
|
||||
}
|
||||
|
||||
var unimplemented *UnimplementedModuleError
|
||||
if !errors.As(err, &unimplemented) {
|
||||
t.Fatalf("expected UnimplementedModuleError, got %T (%v)", err, err)
|
||||
}
|
||||
if unimplemented.ReasonCode() != ReasonUnimplementedModule {
|
||||
t.Fatalf("unexpected reason code: %q", unimplemented.ReasonCode())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegisterConstructorAndConstruct(t *testing.T) {
|
||||
factory := NewFactory(Dependencies{})
|
||||
if err := factory.RegisterConstructor(ModuleKeyGlossary, func(ctx context.Context, req ConstructRequest) (contracts.TranscriptModule, error) {
|
||||
_ = ctx
|
||||
if req.RunSpec.InstanceName != "glossary_1" {
|
||||
t.Fatalf("expected run spec instance name, got %q", req.RunSpec.InstanceName)
|
||||
}
|
||||
return noopModule{key: req.RunSpec.ModuleKey}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("RegisterConstructor error: %v", err)
|
||||
}
|
||||
|
||||
module, err := factory.ModuleForSpec(contracts.ModuleRunSpec{
|
||||
ModuleKey: ModuleKeyGlossary,
|
||||
InstanceName: "glossary_1",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ModuleForSpec error: %v", err)
|
||||
}
|
||||
if module.Key() != ModuleKeyGlossary {
|
||||
t.Fatalf("unexpected module key %q", module.Key())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegisterConstructorRejectsUnknownModuleKey(t *testing.T) {
|
||||
factory := NewFactory(Dependencies{})
|
||||
err := factory.RegisterConstructor("unknown", func(ctx context.Context, req ConstructRequest) (contracts.TranscriptModule, error) {
|
||||
_ = ctx
|
||||
_ = req
|
||||
return noopModule{key: "unknown"}, nil
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected register failure for unknown key")
|
||||
}
|
||||
var unsupported *UnsupportedModuleError
|
||||
if !errors.As(err, &unsupported) {
|
||||
t.Fatalf("expected UnsupportedModuleError, got %T (%v)", err, err)
|
||||
}
|
||||
}
|
||||
235
internal/framework/proposal_generation/generate.go
Normal file
235
internal/framework/proposal_generation/generate.go
Normal file
@@ -0,0 +1,235 @@
|
||||
// Package proposal_generation provides shared, deterministic LLM-backed
|
||||
// proposal-generation helpers. It only produces candidate proposals; validation
|
||||
// and application remain runner responsibilities.
|
||||
package proposal_generation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
)
|
||||
|
||||
// InteractionDiagnosticsWriter writes machine-readable prompt/response artifacts.
|
||||
type InteractionDiagnosticsWriter interface {
|
||||
WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error)
|
||||
}
|
||||
|
||||
// InteractionArtifacts contains written diagnostics paths.
|
||||
type InteractionArtifacts struct {
|
||||
RequestMetadataPath string `json:"request_metadata_path,omitempty"`
|
||||
RequestPayloadPath string `json:"request_payload_path,omitempty"`
|
||||
ResponsePayloadPath string `json:"response_payload_path,omitempty"`
|
||||
ErrorPayloadPath string `json:"error_payload_path,omitempty"`
|
||||
}
|
||||
|
||||
// StructuredCorrectionProposal is one LLM response correction payload.
|
||||
type StructuredCorrectionProposal struct {
|
||||
TargetSegmentID int `json:"id"`
|
||||
OriginalText string `json:"original_text"`
|
||||
CorrectedText string `json:"corrected_text"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
}
|
||||
|
||||
// StructuredCorrectionSet is the reusable structured LLM response model for
|
||||
// candidate correction proposals.
|
||||
type StructuredCorrectionSet struct {
|
||||
Corrections []StructuredCorrectionProposal `json:"corrections"`
|
||||
}
|
||||
|
||||
// Request captures reusable proposal-generation inputs for future modules.
|
||||
type Request struct {
|
||||
ModuleKey string `json:"module_key"`
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
|
||||
WorkingTranscript *schema.Transcript `json:"-"`
|
||||
Section *contracts.SectionMetadata `json:"section,omitempty"`
|
||||
Glossary *schema.Glossary `json:"-"`
|
||||
Config *config.Config `json:"-"`
|
||||
Messages []contracts.LLMMessage `json:"messages"`
|
||||
Model string `json:"model,omitempty"`
|
||||
StartIndex int `json:"start_index"`
|
||||
LLMClient contracts.StructuredLLMClient
|
||||
Scheduler contracts.LLMScheduler
|
||||
DiagnosticsDir string
|
||||
DiagnosticsWriter InteractionDiagnosticsWriter
|
||||
}
|
||||
|
||||
// Result contains generated candidate proposals and optional diagnostics paths.
|
||||
type Result struct {
|
||||
Corrections []proposals.CorrectionProposal `json:"corrections"`
|
||||
Enriched []proposals.EnrichedCorrectionProposal `json:"enriched"`
|
||||
Artifacts InteractionArtifacts `json:"artifacts,omitempty"`
|
||||
}
|
||||
|
||||
// GenerateCandidates executes one structured LLM call and deterministically maps
|
||||
// its correction-set response into framework proposal types.
|
||||
func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
if strings.TrimSpace(req.ModuleKey) == "" {
|
||||
return Result{}, fmt.Errorf("module key must not be empty")
|
||||
}
|
||||
if strings.TrimSpace(req.ModuleInstance) == "" {
|
||||
return Result{}, fmt.Errorf("module instance must not be empty")
|
||||
}
|
||||
if req.StartIndex < 0 {
|
||||
return Result{}, fmt.Errorf("start index must be non-negative")
|
||||
}
|
||||
if req.LLMClient == nil {
|
||||
return Result{}, fmt.Errorf("structured LLM client is required")
|
||||
}
|
||||
if len(req.Messages) == 0 {
|
||||
return Result{}, fmt.Errorf("messages must not be empty")
|
||||
}
|
||||
|
||||
stage := buildStageName(req.ModuleInstance, req.Section)
|
||||
model := resolveModel(req.Config, req.Model)
|
||||
messages := append([]contracts.LLMMessage(nil), req.Messages...)
|
||||
|
||||
var writer InteractionDiagnosticsWriter
|
||||
if req.DiagnosticsWriter != nil {
|
||||
writer = req.DiagnosticsWriter
|
||||
} else if strings.TrimSpace(req.DiagnosticsDir) != "" {
|
||||
writer = diagnosticsWriterAdapter{
|
||||
writer: llm.NewDiagnosticsWriter(
|
||||
filepath.Join(req.DiagnosticsDir, req.ModuleInstance),
|
||||
proposalGenerationSecrets(req.Config),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
response StructuredCorrectionSet
|
||||
callErr error
|
||||
artifacts InteractionArtifacts
|
||||
)
|
||||
call := func(callCtx context.Context) error {
|
||||
_, callErr = req.LLMClient.CompleteStructured(callCtx, contracts.StructuredCompletionRequest{
|
||||
StageName: stage,
|
||||
Messages: messages,
|
||||
Model: model,
|
||||
}, &response)
|
||||
return callErr
|
||||
}
|
||||
if req.Scheduler != nil {
|
||||
callErr = req.Scheduler.Run(ctx, call)
|
||||
} else {
|
||||
callErr = call(ctx)
|
||||
}
|
||||
|
||||
if writer != nil {
|
||||
artifacts, _ = writer.WriteInteraction(
|
||||
stage,
|
||||
map[string]any{
|
||||
"module_key": req.ModuleKey,
|
||||
"module_instance": req.ModuleInstance,
|
||||
"replacement_policy": req.ReplacementPolicy,
|
||||
"section": req.Section,
|
||||
"start_index": req.StartIndex,
|
||||
"model": model,
|
||||
},
|
||||
map[string]any{
|
||||
"messages": messages,
|
||||
},
|
||||
response,
|
||||
errPayload(callErr),
|
||||
)
|
||||
}
|
||||
|
||||
if callErr != nil {
|
||||
return Result{}, fmt.Errorf("proposal generation completion failed: %w", callErr)
|
||||
}
|
||||
|
||||
corrections := make([]proposals.CorrectionProposal, 0, len(response.Corrections))
|
||||
enriched := make([]proposals.EnrichedCorrectionProposal, 0, len(response.Corrections))
|
||||
for i, raw := range response.Corrections {
|
||||
candidate := proposals.CorrectionProposal{
|
||||
TargetSegmentID: raw.TargetSegmentID,
|
||||
OriginalText: raw.OriginalText,
|
||||
CorrectedText: raw.CorrectedText,
|
||||
Confidence: raw.Confidence,
|
||||
}
|
||||
if err := candidate.Validate(); err != nil {
|
||||
return Result{}, fmt.Errorf("invalid structured correction at index %d: %w", i, err)
|
||||
}
|
||||
|
||||
corrections = append(corrections, candidate)
|
||||
enrichedCandidate := proposals.EnrichedCorrectionProposal{
|
||||
CorrectionProposal: candidate,
|
||||
ProposalMetadata: proposals.ProposalMetadata{
|
||||
ProposalIndex: req.StartIndex + i,
|
||||
ModuleKey: req.ModuleKey,
|
||||
ModuleInstance: req.ModuleInstance,
|
||||
},
|
||||
}
|
||||
if req.Section != nil {
|
||||
sectionIndex := req.Section.Index
|
||||
enrichedCandidate.SectionIndex = §ionIndex
|
||||
}
|
||||
enriched = append(enriched, enrichedCandidate)
|
||||
}
|
||||
|
||||
return Result{
|
||||
Corrections: corrections,
|
||||
Enriched: enriched,
|
||||
Artifacts: artifacts,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func buildStageName(moduleInstance string, section *contracts.SectionMetadata) string {
|
||||
base := fmt.Sprintf("%s:proposal-generation", moduleInstance)
|
||||
if section == nil {
|
||||
return base
|
||||
}
|
||||
return fmt.Sprintf("%s:section-%04d", base, section.Index)
|
||||
}
|
||||
|
||||
func resolveModel(cfg *config.Config, override string) string {
|
||||
if strings.TrimSpace(override) != "" {
|
||||
return strings.TrimSpace(override)
|
||||
}
|
||||
if cfg == nil {
|
||||
return ""
|
||||
}
|
||||
return llm.ResolvePrimaryConfig(*cfg).Model
|
||||
}
|
||||
|
||||
func proposalGenerationSecrets(cfg *config.Config) []string {
|
||||
if cfg == nil {
|
||||
return nil
|
||||
}
|
||||
return []string{
|
||||
cfg.PrimaryLLM.APIKey,
|
||||
cfg.ValidationLLM.APIKey,
|
||||
cfg.EffectiveValidationLLMConfig().APIKey,
|
||||
}
|
||||
}
|
||||
|
||||
func errPayload(err error) any {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
return map[string]any{"error": err.Error()}
|
||||
}
|
||||
|
||||
type diagnosticsWriterAdapter struct {
|
||||
writer *llm.DiagnosticsWriter
|
||||
}
|
||||
|
||||
func (a diagnosticsWriterAdapter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) {
|
||||
artifacts, err := a.writer.WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload)
|
||||
if err != nil {
|
||||
return InteractionArtifacts{}, err
|
||||
}
|
||||
return InteractionArtifacts{
|
||||
RequestMetadataPath: artifacts.RequestMetadataPath,
|
||||
RequestPayloadPath: artifacts.RequestPayloadPath,
|
||||
ResponsePayloadPath: artifacts.ResponsePayloadPath,
|
||||
ErrorPayloadPath: artifacts.ErrorPayloadPath,
|
||||
}, nil
|
||||
}
|
||||
267
internal/framework/proposal_generation/generate_test.go
Normal file
267
internal/framework/proposal_generation/generate_test.go
Normal file
@@ -0,0 +1,267 @@
|
||||
package proposal_generation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
)
|
||||
|
||||
type fakeStructuredClient struct {
|
||||
responses []StructuredCorrectionSet
|
||||
err error
|
||||
calls []contracts.StructuredCompletionRequest
|
||||
}
|
||||
|
||||
func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
_ = ctx
|
||||
f.calls = append(f.calls, req)
|
||||
if f.err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, f.err
|
||||
}
|
||||
target, ok := out.(*StructuredCorrectionSet)
|
||||
if !ok {
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output type")
|
||||
}
|
||||
if len(f.responses) == 0 {
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("unexpected call")
|
||||
}
|
||||
*target = f.responses[0]
|
||||
f.responses = f.responses[1:]
|
||||
return contracts.StructuredCompletionResponse{}, nil
|
||||
}
|
||||
|
||||
type countingScheduler struct {
|
||||
runs int
|
||||
}
|
||||
|
||||
func (s *countingScheduler) Run(ctx context.Context, fn func(context.Context) error) error {
|
||||
s.runs++
|
||||
return fn(ctx)
|
||||
}
|
||||
|
||||
func defaultRequest(t *testing.T) Request {
|
||||
t.Helper()
|
||||
cfg := config.Default()
|
||||
return Request{
|
||||
ModuleKey: "test_module",
|
||||
ModuleInstance: "test_module",
|
||||
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||
Config: &cfg,
|
||||
Messages: []contracts.LLMMessage{
|
||||
{Role: "system", Content: "system prompt"},
|
||||
{Role: "user", Content: "user prompt"},
|
||||
},
|
||||
StartIndex: 0,
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesSuccess(t *testing.T) {
|
||||
client := &fakeStructuredClient{
|
||||
responses: []StructuredCorrectionSet{
|
||||
{
|
||||
Corrections: []StructuredCorrectionProposal{
|
||||
{TargetSegmentID: 7, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
section := contracts.SectionMetadata{Index: 2}
|
||||
req := defaultRequest(t)
|
||||
req.LLMClient = client
|
||||
req.StartIndex = 10
|
||||
req.Section = §ion
|
||||
|
||||
got, err := GenerateCandidates(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateCandidates error: %v", err)
|
||||
}
|
||||
if len(got.Corrections) != 1 || len(got.Enriched) != 1 {
|
||||
t.Fatalf("unexpected proposal lengths: %+v", got)
|
||||
}
|
||||
if got.Enriched[0].ProposalIndex != 10 {
|
||||
t.Fatalf("expected proposal index 10, got %d", got.Enriched[0].ProposalIndex)
|
||||
}
|
||||
if got.Enriched[0].SectionIndex == nil || *got.Enriched[0].SectionIndex != 2 {
|
||||
t.Fatalf("expected section index 2, got %v", got.Enriched[0].SectionIndex)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesMalformedStructuredResponse(t *testing.T) {
|
||||
client := &fakeStructuredClient{
|
||||
responses: []StructuredCorrectionSet{
|
||||
{
|
||||
Corrections: []StructuredCorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "x", CorrectedText: "", Confidence: 0.9},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
req := defaultRequest(t)
|
||||
req.LLMClient = client
|
||||
|
||||
_, err := GenerateCandidates(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "invalid structured correction") {
|
||||
t.Fatalf("expected structured response validation failure, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesDeterministicIndexAssignment(t *testing.T) {
|
||||
baseResponse := StructuredCorrectionSet{
|
||||
Corrections: []StructuredCorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "a", CorrectedText: "A", Confidence: 0.9},
|
||||
{TargetSegmentID: 2, OriginalText: "b", CorrectedText: "B", Confidence: 0.9},
|
||||
},
|
||||
}
|
||||
clientA := &fakeStructuredClient{responses: []StructuredCorrectionSet{baseResponse}}
|
||||
clientB := &fakeStructuredClient{responses: []StructuredCorrectionSet{baseResponse}}
|
||||
|
||||
reqA := defaultRequest(t)
|
||||
reqA.LLMClient = clientA
|
||||
reqA.StartIndex = 3
|
||||
first, err := GenerateCandidates(context.Background(), reqA)
|
||||
if err != nil {
|
||||
t.Fatalf("first generation failed: %v", err)
|
||||
}
|
||||
|
||||
reqB := defaultRequest(t)
|
||||
reqB.LLMClient = clientB
|
||||
reqB.StartIndex = 3
|
||||
second, err := GenerateCandidates(context.Background(), reqB)
|
||||
if err != nil {
|
||||
t.Fatalf("second generation failed: %v", err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(first.Enriched, second.Enriched) {
|
||||
t.Fatalf("expected stable enriched proposals\nfirst=%+v\nsecond=%+v", first.Enriched, second.Enriched)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) {
|
||||
client := &fakeStructuredClient{
|
||||
responses: []StructuredCorrectionSet{
|
||||
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "alpha", CorrectedText: "ALPHA", Confidence: 0.9}}},
|
||||
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 3, OriginalText: "charlie", CorrectedText: "CHARLIE", Confidence: 0.9}}},
|
||||
},
|
||||
}
|
||||
section0 := contracts.SectionMetadata{Index: 0}
|
||||
section1 := contracts.SectionMetadata{Index: 1}
|
||||
|
||||
req0 := defaultRequest(t)
|
||||
req0.LLMClient = client
|
||||
req0.Section = §ion0
|
||||
req0.StartIndex = 0
|
||||
part0, err := GenerateCandidates(context.Background(), req0)
|
||||
if err != nil {
|
||||
t.Fatalf("section 0 generation failed: %v", err)
|
||||
}
|
||||
|
||||
req1 := defaultRequest(t)
|
||||
req1.LLMClient = client
|
||||
req1.Section = §ion1
|
||||
req1.StartIndex = len(part0.Enriched)
|
||||
part1, err := GenerateCandidates(context.Background(), req1)
|
||||
if err != nil {
|
||||
t.Fatalf("section 1 generation failed: %v", err)
|
||||
}
|
||||
|
||||
all := append(append([]proposals.EnrichedCorrectionProposal(nil), part0.Enriched...), part1.Enriched...)
|
||||
if len(all) != 2 {
|
||||
t.Fatalf("expected 2 proposals, got %d", len(all))
|
||||
}
|
||||
if all[0].ProposalIndex != 0 || all[1].ProposalIndex != 1 {
|
||||
t.Fatalf("unexpected proposal indexes: %d, %d", all[0].ProposalIndex, all[1].ProposalIndex)
|
||||
}
|
||||
if all[0].SectionIndex == nil || *all[0].SectionIndex != 0 || all[1].SectionIndex == nil || *all[1].SectionIndex != 1 {
|
||||
t.Fatalf("unexpected section metadata: %+v", all)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) {
|
||||
secret := "phase11-secret"
|
||||
client := &fakeStructuredClient{
|
||||
responses: []StructuredCorrectionSet{
|
||||
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: secret, CorrectedText: "safe", Confidence: 0.9}}},
|
||||
},
|
||||
}
|
||||
cfg := config.Default()
|
||||
cfg.PrimaryLLM.APIKey = secret
|
||||
req := defaultRequest(t)
|
||||
req.Config = &cfg
|
||||
req.LLMClient = client
|
||||
req.DiagnosticsDir = t.TempDir()
|
||||
req.Messages = []contracts.LLMMessage{
|
||||
{Role: "system", Content: "include secret " + secret},
|
||||
{Role: "user", Content: "fix it"},
|
||||
}
|
||||
|
||||
got, err := GenerateCandidates(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateCandidates error: %v", err)
|
||||
}
|
||||
if got.Artifacts.ResponsePayloadPath == "" || got.Artifacts.RequestPayloadPath == "" {
|
||||
t.Fatalf("expected diagnostics artifact paths, got %+v", got.Artifacts)
|
||||
}
|
||||
|
||||
for _, path := range []string{got.Artifacts.RequestPayloadPath, got.Artifacts.ResponsePayloadPath} {
|
||||
raw, readErr := os.ReadFile(path)
|
||||
if readErr != nil {
|
||||
t.Fatalf("read artifact %q: %v", path, readErr)
|
||||
}
|
||||
if strings.Contains(string(raw), secret) {
|
||||
t.Fatalf("artifact leaked secret %q: %s", path, string(raw))
|
||||
}
|
||||
if !strings.Contains(string(raw), "[REDACTED]") {
|
||||
t.Fatalf("expected redaction marker in artifact %q: %s", path, string(raw))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesSchedulerUsage(t *testing.T) {
|
||||
client := &fakeStructuredClient{
|
||||
responses: []StructuredCorrectionSet{
|
||||
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "x", CorrectedText: "y", Confidence: 0.9}}},
|
||||
},
|
||||
}
|
||||
scheduler := &countingScheduler{}
|
||||
req := defaultRequest(t)
|
||||
req.LLMClient = client
|
||||
req.Scheduler = scheduler
|
||||
|
||||
_, err := GenerateCandidates(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateCandidates error: %v", err)
|
||||
}
|
||||
if scheduler.runs != 1 {
|
||||
t.Fatalf("expected scheduler to run once, got %d", scheduler.runs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesClientError(t *testing.T) {
|
||||
client := &fakeStructuredClient{err: errors.New("boom")}
|
||||
req := defaultRequest(t)
|
||||
req.LLMClient = client
|
||||
req.DiagnosticsDir = t.TempDir()
|
||||
|
||||
got, err := GenerateCandidates(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "completion failed") {
|
||||
t.Fatalf("expected completion failure, got %v", err)
|
||||
}
|
||||
if got.Artifacts.ResponsePayloadPath != "" {
|
||||
t.Fatalf("expected zero result on error, got %+v", got)
|
||||
}
|
||||
matches, globErr := filepath.Glob(filepath.Join(req.DiagnosticsDir, req.ModuleInstance, "*error-payload.json"))
|
||||
if globErr != nil {
|
||||
t.Fatalf("glob error: %v", globErr)
|
||||
}
|
||||
if len(matches) == 0 {
|
||||
t.Fatalf("expected error diagnostics artifact under %s", req.DiagnosticsDir)
|
||||
}
|
||||
}
|
||||
@@ -29,9 +29,7 @@ type Runner struct {
|
||||
factory ModuleFactory
|
||||
}
|
||||
|
||||
type ValidationScheduler interface {
|
||||
Run(ctx context.Context, fn func(context.Context) error) error
|
||||
}
|
||||
type ValidationScheduler = contracts.LLMScheduler
|
||||
|
||||
// ModuleResult captures deterministic per-module execution output.
|
||||
type ModuleResult struct {
|
||||
@@ -76,6 +74,9 @@ type RunInput struct {
|
||||
Transcript *schema.Transcript
|
||||
Glossary *schema.Glossary
|
||||
ModuleSpecs []contracts.ModuleRunSpec
|
||||
ProposalLLMClient contracts.StructuredLLMClient
|
||||
ProposalLLMScheduler contracts.LLMScheduler
|
||||
ProposalDiagnosticsDir string
|
||||
ValidationLLMClient contracts.StructuredLLMClient
|
||||
ValidationLLMScheduler ValidationScheduler
|
||||
ValidationDiagnosticsDir string
|
||||
@@ -121,12 +122,15 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
|
||||
Config: input.Config,
|
||||
WorkingTranscript: working,
|
||||
Glossary: input.Glossary,
|
||||
DiagnosticsDir: input.ProposalDiagnosticsDir,
|
||||
},
|
||||
RunSpec: contracts.ModuleRunSpec{
|
||||
ModuleKey: spec.ModuleKey,
|
||||
InstanceName: spec.InstanceName,
|
||||
ReplacementPolicy: policy,
|
||||
},
|
||||
LLMClient: input.ProposalLLMClient,
|
||||
LLMScheduler: input.ProposalLLMScheduler,
|
||||
})
|
||||
if err != nil {
|
||||
failed := ModuleResult{
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -11,6 +12,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||
)
|
||||
@@ -553,3 +555,99 @@ func TestRunnerAcceptsLLMSchedulerType(t *testing.T) {
|
||||
t.Fatal("expected scheduler instance")
|
||||
}
|
||||
}
|
||||
|
||||
type proposalGenerationModule struct {
|
||||
key string
|
||||
policy proposals.ReplacementPolicy
|
||||
}
|
||||
|
||||
func (m proposalGenerationModule) Key() string { return m.key }
|
||||
func (m proposalGenerationModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy }
|
||||
func (m proposalGenerationModule) Validators() []contracts.Validator { return nil }
|
||||
func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
result, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
|
||||
ModuleKey: req.RunSpec.ModuleKey,
|
||||
ModuleInstance: req.RunSpec.InstanceName,
|
||||
ReplacementPolicy: req.RunSpec.ReplacementPolicy,
|
||||
WorkingTranscript: req.WorkingTranscript,
|
||||
Config: req.Config,
|
||||
Glossary: req.Glossary,
|
||||
Messages: []contracts.LLMMessage{
|
||||
{Role: "system", Content: "return transcript corrections"},
|
||||
{Role: "user", Content: "produce one safe correction"},
|
||||
},
|
||||
LLMClient: req.LLMClient,
|
||||
Scheduler: req.LLMScheduler,
|
||||
DiagnosticsDir: req.DiagnosticsDir,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return result.Corrections, nil
|
||||
}
|
||||
|
||||
type fakeProposalStructuredClient struct {
|
||||
responses []proposal_generation.StructuredCorrectionSet
|
||||
}
|
||||
|
||||
func (f *fakeProposalStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
_ = ctx
|
||||
_ = req
|
||||
if len(f.responses) == 0 {
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("unexpected call")
|
||||
}
|
||||
target, ok := out.(*proposal_generation.StructuredCorrectionSet)
|
||||
if !ok {
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output type")
|
||||
}
|
||||
*target = f.responses[0]
|
||||
f.responses = f.responses[1:]
|
||||
return contracts.StructuredCompletionResponse{}, nil
|
||||
}
|
||||
|
||||
func TestRunnerProposalGenerationHelperFlowsThroughPipeline(t *testing.T) {
|
||||
client := &fakeProposalStructuredClient{
|
||||
responses: []proposal_generation.StructuredCorrectionSet{
|
||||
{
|
||||
Corrections: []proposal_generation.StructuredCorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
scheduler := &countingScheduler{}
|
||||
cfg := config.Default()
|
||||
diagDir := t.TempDir()
|
||||
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": proposalGenerationModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique},
|
||||
}})
|
||||
|
||||
out, err := r.Run(context.Background(), RunInput{
|
||||
Config: &cfg,
|
||||
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}},
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
|
||||
ProposalLLMClient: client,
|
||||
ProposalLLMScheduler: scheduler,
|
||||
ProposalDiagnosticsDir: diagDir,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected run error: %v", err)
|
||||
}
|
||||
if out.FinalTranscript.Segments[0].Text != "the cat" {
|
||||
t.Fatalf("expected proposal-generated correction to apply, got %q", out.FinalTranscript.Segments[0].Text)
|
||||
}
|
||||
if scheduler.runs != 1 {
|
||||
t.Fatalf("expected proposal scheduler use, got %d runs", scheduler.runs)
|
||||
}
|
||||
if len(out.ModuleResults) != 1 || len(out.ModuleResults[0].AppliedChanges) != 1 {
|
||||
t.Fatalf("expected one applied change, got %+v", out.ModuleResults)
|
||||
}
|
||||
matches, globErr := filepath.Glob(filepath.Join(diagDir, "m", "*proposal-generation*response-payload.json"))
|
||||
if globErr != nil {
|
||||
t.Fatalf("glob diagnostics: %v", globErr)
|
||||
}
|
||||
if len(matches) == 0 {
|
||||
t.Fatalf("expected proposal-generation diagnostics artifacts in %s", filepath.Join(diagDir, "m"))
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user