eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Fix the five library adoption review findings

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-24 15:32:13 +00:00
parent c42bc01052
commit 665ee85215
19 changed files with 942 additions and 208 deletions
Download Patch File Download Diff File Expand all files Collapse all files

145
internal/framework/runner/cancellation_test.go Normal file
View File

@@ -0,0 +1,145 @@
package runner
import (
"context"
"errors"
"net/http"
"os"
"path/filepath"
"sync/atomic"
"testing"
"time"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
"gitea.maximumdirect.net/eric/audita/internal/prompts"
pk "gitea.maximumdirect.net/eric/promptkit"
)
type contextValidator struct {
validate func(context.Context, validators.Request) (validators.Result, error)
}
func (v contextValidator) Name() string { return "cancel-test" }
func (v contextValidator) Validate(ctx context.Context, req validators.Request) (validators.Result, error) {
return v.validate(ctx, req)
}
type cancelTransport struct {
entered chan struct{}
calls atomic.Int32
}
func (t *cancelTransport) RoundTrip(req *http.Request) (*http.Response, error) {
t.calls.Add(1)
t.entered <- struct{}{}
<-req.Context().Done()
return nil, req.Context().Err()
}
func TestValidatorFailureCancelsActiveAndPendingProviderWork(t *testing.T) {
dir := t.TempDir()
if err := os.Mkdir(filepath.Join(dir, "profiles"), 0700); err != nil {
t.Fatal(err)
}
main := filepath.Join(dir, "config.yml")
body := "version: 2\nbackends:\n shared:\n endpoint: http://provider.invalid/v1\n concurrency: 1\n queue_capacity: 1\nllm:\n proposal_profile: base\n transient_retries: 0\n repair_attempts: 0\n"
if err := os.WriteFile(main, []byte(body), 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "profiles/base.yaml"), []byte("id: base\nbackend: shared\nmodel: local\ntimeout_seconds: 30\n"), 0600); err != nil {
t.Fatal(err)
}
loaded, err := config.Load(ck.File(main), "")
if err != nil {
t.Fatal(err)
}
transport := &cancelTransport{entered: make(chan struct{}, 3)}
runtime, err := llm.NewRuntime(context.Background(), loaded.Config, main, &http.Client{Transport: transport})
if err != nil {
t.Fatal(err)
}
collector := llm.NewObservationCollector()
hooks := collector.Hooks()
started := make(chan struct{}, 2)
hooks.OnEvent = func(event pk.ExecutionEvent) {
if event.Kind == pk.ExecutionStarted {
started <- struct{}{}
}
}
client, err := runtime.NewRoleClient("validation", hooks)
if err != nil {
t.Fatal(err)
}
origin := errors.New("originating validator failure")
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
validator := contextValidator{validate: func(ctx context.Context, req validators.Request) (validators.Result, error) {
if *req.SectionIndex == 0 {
// Hold the failure until one call is active and both peer executions started.
for i := 0; i < 2; i++ {
select {
case <-started:
case <-ctx.Done():
return validators.Result{}, ctx.Err()
}
}
select {
case <-transport.entered:
case <-ctx.Done():
return validators.Result{}, ctx.Err()
}
return validators.Result{}, origin
}
var out map[string]any
_, err := client.CompleteStructured(ctx, structuredoutput.Request{
PromptID: prompts.PromptIDValidatorEditorialReview, PromptVersion: prompts.VersionV1,
Variables: map[string]string{"TranscriptDescriptionBlock": "", "PayloadJSON": "[]"}, Schema: responseschema.ValidatorDecisionSetKey,
Context: structuredoutput.DiagnosticContext{ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: req.SectionIndex},
}, &out)
return validators.Result{}, err
}}
module := fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "hello", CorrectedText: "Hello", Confidence: 1}}, nil
}}
sections := []chunking.Section{}
for i := 0; i < 3; i++ {
sections = append(sections, chunking.Section{Index: i, Segments: []schema.Segment{{ID: 1, Text: "hello"}}})
}
done := make(chan error, 1)
go func() {
_, err := runModulePipeline(ctx, collectSectionProposalsInput{Module: module, Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar"}, Policy: module.policy, Config: &loaded.Config, Sections: sections})
done <- err
}()
select {
case err := <-done:
if !errors.Is(err, origin) {
t.Fatalf("lost originating failure: %v", err)
}
case <-time.After(2 * time.Second):
t.Fatal("validator failure did not cancel and join peers")
}
if transport.calls.Load() != 1 {
t.Fatalf("pending call dispatched: %d", transport.calls.Load())
}
snapshot := collector.Snapshot()
if len(snapshot.Calls) != 2 {
t.Fatalf("partial observations missing: %+v", snapshot)
}
for _, call := range snapshot.Calls {
if call.Finished.IsZero() || call.Failure != structuredoutput.FailureRuntime {
t.Fatalf("unfinished/cause lost: %+v", call)
}
}
if got := llm.Summarize(snapshot); got.ProviderAttempts != 1 {
t.Fatalf("attempt count: %+v", got)
}
}

70
internal/framework/runner/runner.go
View File

@@ -332,7 +332,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
var (
firstErr error
errOnce sync.Once
errMu sync.Mutex
vwg sync.WaitGroup
)
@@ -340,10 +340,17 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
if err == nil {
return
}
errOnce.Do(func() {
errMu.Lock()
defer errMu.Unlock()
if firstErr == nil {
firstErr = err
cancel()
})
}
}
getErr := func() error {
errMu.Lock()
defer errMu.Unlock()
return firstErr
}
for result := range proposalResults {
@@ -351,13 +358,16 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
setErr(result.err)
continue
}
if firstErr != nil {
if getErr() != nil {
continue
}
out.Warnings = append(out.Warnings, result.warnings...)
pending[result.sectionPos] = result
for {
if getErr() != nil {
break
}
sectionResult, ok := pending[nextSectionToProcess]
if !ok {
break
@@ -385,18 +395,19 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
go func(sectionPos int, enriched []proposals.EnrichedCorrectionProposal, sectionMetadata contracts.SectionMetadata) {
defer vwg.Done()
validated, err := validateSectionCandidates(runCtx, validateSectionCandidatesInput{
Spec: input.Spec,
Policy: input.Policy,
Glossary: input.Glossary,
Config: input.Config,
WorkingTranscript: transcriptFromSection(sectionResult.section),
ModuleInstanceForStages: fmt.Sprintf("%s:section-%04d", input.Spec.InstanceName, sectionMetadata.Index),
Validators: validatorsOrdered,
SectionEnriched: enriched,
ValidationLLMClient: input.ValidationClient,
DiagnosticsDir: input.ValidationDiagnosticsDir,
Timings: input.Timings,
Spec: input.Spec,
Policy: input.Policy,
Glossary: input.Glossary,
Config: input.Config,
WorkingTranscript: transcriptFromSection(sectionResult.section),
SectionIndex: &sectionMetadata.Index,
Validators: validatorsOrdered,
SectionEnriched: enriched,
ValidationLLMClient: input.ValidationClient,
DiagnosticsDir: input.ValidationDiagnosticsDir,
Timings: input.Timings,
})
setErr(err)
validationResults <- sectionValidationResult{
sectionPos: sectionPos,
enriched: enriched,
@@ -477,25 +488,25 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
return out.Warnings[i].Message < out.Warnings[j].Message
})
if firstErr != nil {
return out, firstErr
if err := getErr(); err != nil {
return out, err
}
return out, nil
}
type validateSectionCandidatesInput struct {
Spec contracts.ModuleRunSpec
Policy proposals.ReplacementPolicy
Glossary *schema.Glossary
Config *config.Config
WorkingTranscript *schema.Transcript
ModuleInstanceForStages string
Validators []contracts.Validator
SectionEnriched []proposals.EnrichedCorrectionProposal
ValidationLLMClient contracts.StructuredLLMClient
DiagnosticsDir string
Timings *validatorTimingCollector
Spec contracts.ModuleRunSpec
Policy proposals.ReplacementPolicy
Glossary *schema.Glossary
Config *config.Config
WorkingTranscript *schema.Transcript
SectionIndex *int
Validators []contracts.Validator
SectionEnriched []proposals.EnrichedCorrectionProposal
ValidationLLMClient contracts.StructuredLLMClient
DiagnosticsDir string
Timings *validatorTimingCollector
}
type validateSectionCandidatesResult struct {
@@ -527,7 +538,8 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
WorkingTranscript: input.WorkingTranscript,
CandidateProposal: eligible,
ModuleKey: input.Spec.ModuleKey,
ModuleInstance: input.ModuleInstanceForStages,
ModuleInstance: input.Spec.InstanceName,
SectionIndex: input.SectionIndex,
ReplacementPolicy: input.Policy,
Glossary: input.Glossary,
Config: input.Config,

4
internal/framework/runner/runner_test.go
View File

@@ -438,8 +438,8 @@ func (c *captureContractStructuredClient) CompleteStructured(ctx context.Context
func (c *stageAwareStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
section := 0
if req.Context.BatchIndex != nil {
section = *req.Context.BatchIndex
if req.Context.SectionIndex != nil {
section = *req.Context.SectionIndex
}
if c.startedSection != nil {
select {