diff --git a/cmd/audita/correction_repair_integration_test.go b/cmd/audita/correction_repair_integration_test.go index e955026..c55e54f 100644 --- a/cmd/audita/correction_repair_integration_test.go +++ b/cmd/audita/correction_repair_integration_test.go @@ -243,6 +243,25 @@ func TestSubprocessCorrectionRepairConversation(t *testing.T) { if len(ledger) != 3 { t.Fatalf("ledger: %+v", ledger) } + paths, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*", "llm-execution.json")) + if err != nil || len(paths) != 1 { + t.Fatalf("execution path: %v %v", paths, err) + } + data, err := os.ReadFile(paths[0]) + if err != nil { + t.Fatal(err) + } + var execution struct { + Summary struct { + ProviderAttempts int `json:"provider_attempts"` + } `json:"summary"` + } + if err := json.Unmarshal(data, &execution); err != nil { + t.Fatal(err) + } + if got := execution.Summary.ProviderAttempts; got != len(requests) { + t.Fatalf("execution provider attempts = %d, captured requests = %d", got, len(requests)) + } } }) } diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index d1ecf02..74d69a4 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -7,6 +7,8 @@ and [developer guide](../development.md) for the applicable policies. ## Stage 1 — Finish the PromptKit Adoption Follow-ups +**Complete.** Focused decoder, runner, reporting, and subprocess checks passed. + 1. In the debug case of `TestSubprocessCorrectionRepairConversation`, read the run's `llm-execution.json` and compare its top-level `summary.provider_attempts` with the number of requests captured by the diff --git a/internal/framework/processreport/llm.go b/internal/framework/processreport/llm.go index 350db1f..40661bb 100644 --- a/internal/framework/processreport/llm.go +++ b/internal/framework/processreport/llm.go @@ -68,12 +68,13 @@ func BuildLLMExecution(snapshot llm.ObservationSnapshot) any { } for _, call := range snapshot.Calls { group := llm.ObservationSnapshot{Calls: []llm.LogicalCall{call}, Events: eventsByCall[call.CallID]} + summary := llm.Summarize(group) var finished *time.Time if !call.Finished.IsZero() { value := call.Finished finished = &value } - calls = append(calls, executionCall{TerminalCoverageComplete: llm.Summarize(group).TerminalCoverageComplete, RequestedPromptID: call.RequestedPromptID, CallID: call.CallID, Context: call.Context, Prompt: call.Prompt, Started: call.Started, Finished: finished, ElapsedMS: call.WallDuration.Milliseconds(), Failure: call.Failure, Summary: mapLLMSummary(llm.Summarize(group))}) + calls = append(calls, executionCall{TerminalCoverageComplete: summary.TerminalCoverageComplete, RequestedPromptID: call.RequestedPromptID, CallID: call.CallID, Context: call.Context, Prompt: call.Prompt, Started: call.Started, Finished: finished, ElapsedMS: call.WallDuration.Milliseconds(), Failure: call.Failure, Summary: mapLLMSummary(summary)}) } for _, event := range snapshot.Events { var generation, attempt *uint64 diff --git a/internal/framework/proposal_generation/repair.go b/internal/framework/proposal_generation/repair.go index b2da5f8..10578d3 100644 --- a/internal/framework/proposal_generation/repair.go +++ b/internal/framework/proposal_generation/repair.go @@ -15,13 +15,14 @@ type RepairResponse struct { } // DecodeRepairResponse checks correspondence after structural schema validation. +// It returns the decoded response for diagnostics when correspondence fails. func DecodeRepairResponse(content []byte, expectedRef int) (RepairResponse, error) { var response RepairResponse if err := json.Unmarshal(content, &response); err != nil { return RepairResponse{}, invalidRepairResponse(err) } if response.ProposalRef != expectedRef || (response.Action != "revise" || response.Correction == nil) && (response.Action != "withdraw" || response.Correction != nil) { - return RepairResponse{}, invalidRepairResponse(nil) + return response, invalidRepairResponse(nil) } return response, nil } diff --git a/internal/framework/proposal_generation/repair_test.go b/internal/framework/proposal_generation/repair_test.go index 63b0624..8e9d77d 100644 --- a/internal/framework/proposal_generation/repair_test.go +++ b/internal/framework/proposal_generation/repair_test.go @@ -20,11 +20,14 @@ func TestDecodeRepairResponse(t *testing.T) { {"withdrawal with revision", `{"proposal_ref":2,"action":"withdraw","correction":{"id":1}}`, false}, } { t.Run(tc.name, func(t *testing.T) { - _, err := DecodeRepairResponse([]byte(tc.content), 2) + response, err := DecodeRepairResponse([]byte(tc.content), 2) if tc.valid && err != nil { t.Fatal(err) } if !tc.valid { + if response.ProposalRef == 0 || response.Action == "" { + t.Fatalf("missing decoded evidence: %+v", response) + } var failure *structuredoutput.Failure if !errors.As(err, &failure) || failure.Reason != outcomes.ReasonInvalidOutput || failure.Class != structuredoutput.FailureRecoverable { t.Fatalf("failure = %v", err) diff --git a/internal/framework/runner/repair.go b/internal/framework/runner/repair.go index 3553c0c..6d69c39 100644 --- a/internal/framework/runner/repair.go +++ b/internal/framework/runner/repair.go @@ -2,7 +2,6 @@ package runner import ( "context" - "encoding/json" "errors" "fmt" "path/filepath" @@ -151,9 +150,6 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput, completion, callErr := *turn.Response, turn.Err proposalIndex, attemptNumber := original.ProposalIndex, branch.AttemptCount+1 var response proposal_generation.RepairResponse - if callErr == nil { - _ = json.Unmarshal(completion.Content, &response) - } operation := outcomes.Operation{ Reference: outcomes.Reference{ @@ -202,7 +198,8 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput, } return nil, callErr } - decoded, err := proposal_generation.DecodeRepairResponse(completion.Content, position+1) + var err error + response, err = proposal_generation.DecodeRepairResponse(completion.Content, position+1) if err != nil { operation.FailureReason = outcomes.ReasonInvalidOutput finishAttempt(err, nil, validateSectionCandidatesResult{}) @@ -211,12 +208,12 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput, } operation.Succeeded = true operation.RecoveryReasons = outcomes.ExecutionRecoveries(operation.Counts) - if decoded.Action == "withdraw" { + if response.Action == "withdraw" { finishAttempt(nil, nil, validateSectionCandidatesResult{}) branch.Outcome = "withdrawn" return nil, nil } - revision := decoded.Correction + revision := response.Correction localRef = revision.SegmentReference target := 0 if input.WorkingTranscript != nil && localRef > 0 && localRef <= len(input.WorkingTranscript.Segments) {