Finish PromptKit repair and execution accounting follow-ups
This commit is contained in:
@@ -243,6 +243,25 @@ func TestSubprocessCorrectionRepairConversation(t *testing.T) {
|
||||
if len(ledger) != 3 {
|
||||
t.Fatalf("ledger: %+v", ledger)
|
||||
}
|
||||
paths, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*", "llm-execution.json"))
|
||||
if err != nil || len(paths) != 1 {
|
||||
t.Fatalf("execution path: %v %v", paths, err)
|
||||
}
|
||||
data, err := os.ReadFile(paths[0])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var execution struct {
|
||||
Summary struct {
|
||||
ProviderAttempts int `json:"provider_attempts"`
|
||||
} `json:"summary"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &execution); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := execution.Summary.ProviderAttempts; got != len(requests) {
|
||||
t.Fatalf("execution provider attempts = %d, captured requests = %d", got, len(requests))
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -7,6 +7,8 @@ and [developer guide](../development.md) for the applicable policies.
|
||||
|
||||
## Stage 1 — Finish the PromptKit Adoption Follow-ups
|
||||
|
||||
**Complete.** Focused decoder, runner, reporting, and subprocess checks passed.
|
||||
|
||||
1. In the debug case of `TestSubprocessCorrectionRepairConversation`, read the
|
||||
run's `llm-execution.json` and compare its top-level
|
||||
`summary.provider_attempts` with the number of requests captured by the
|
||||
|
||||
@@ -68,12 +68,13 @@ func BuildLLMExecution(snapshot llm.ObservationSnapshot) any {
|
||||
}
|
||||
for _, call := range snapshot.Calls {
|
||||
group := llm.ObservationSnapshot{Calls: []llm.LogicalCall{call}, Events: eventsByCall[call.CallID]}
|
||||
summary := llm.Summarize(group)
|
||||
var finished *time.Time
|
||||
if !call.Finished.IsZero() {
|
||||
value := call.Finished
|
||||
finished = &value
|
||||
}
|
||||
calls = append(calls, executionCall{TerminalCoverageComplete: llm.Summarize(group).TerminalCoverageComplete, RequestedPromptID: call.RequestedPromptID, CallID: call.CallID, Context: call.Context, Prompt: call.Prompt, Started: call.Started, Finished: finished, ElapsedMS: call.WallDuration.Milliseconds(), Failure: call.Failure, Summary: mapLLMSummary(llm.Summarize(group))})
|
||||
calls = append(calls, executionCall{TerminalCoverageComplete: summary.TerminalCoverageComplete, RequestedPromptID: call.RequestedPromptID, CallID: call.CallID, Context: call.Context, Prompt: call.Prompt, Started: call.Started, Finished: finished, ElapsedMS: call.WallDuration.Milliseconds(), Failure: call.Failure, Summary: mapLLMSummary(summary)})
|
||||
}
|
||||
for _, event := range snapshot.Events {
|
||||
var generation, attempt *uint64
|
||||
|
||||
@@ -15,13 +15,14 @@ type RepairResponse struct {
|
||||
}
|
||||
|
||||
// DecodeRepairResponse checks correspondence after structural schema validation.
|
||||
// It returns the decoded response for diagnostics when correspondence fails.
|
||||
func DecodeRepairResponse(content []byte, expectedRef int) (RepairResponse, error) {
|
||||
var response RepairResponse
|
||||
if err := json.Unmarshal(content, &response); err != nil {
|
||||
return RepairResponse{}, invalidRepairResponse(err)
|
||||
}
|
||||
if response.ProposalRef != expectedRef || (response.Action != "revise" || response.Correction == nil) && (response.Action != "withdraw" || response.Correction != nil) {
|
||||
return RepairResponse{}, invalidRepairResponse(nil)
|
||||
return response, invalidRepairResponse(nil)
|
||||
}
|
||||
return response, nil
|
||||
}
|
||||
|
||||
@@ -20,11 +20,14 @@ func TestDecodeRepairResponse(t *testing.T) {
|
||||
{"withdrawal with revision", `{"proposal_ref":2,"action":"withdraw","correction":{"id":1}}`, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := DecodeRepairResponse([]byte(tc.content), 2)
|
||||
response, err := DecodeRepairResponse([]byte(tc.content), 2)
|
||||
if tc.valid && err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !tc.valid {
|
||||
if response.ProposalRef == 0 || response.Action == "" {
|
||||
t.Fatalf("missing decoded evidence: %+v", response)
|
||||
}
|
||||
var failure *structuredoutput.Failure
|
||||
if !errors.As(err, &failure) || failure.Reason != outcomes.ReasonInvalidOutput || failure.Class != structuredoutput.FailureRecoverable {
|
||||
t.Fatalf("failure = %v", err)
|
||||
|
||||
@@ -2,7 +2,6 @@ package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
@@ -151,9 +150,6 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
completion, callErr := *turn.Response, turn.Err
|
||||
proposalIndex, attemptNumber := original.ProposalIndex, branch.AttemptCount+1
|
||||
var response proposal_generation.RepairResponse
|
||||
if callErr == nil {
|
||||
_ = json.Unmarshal(completion.Content, &response)
|
||||
}
|
||||
|
||||
operation := outcomes.Operation{
|
||||
Reference: outcomes.Reference{
|
||||
@@ -202,7 +198,8 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
}
|
||||
return nil, callErr
|
||||
}
|
||||
decoded, err := proposal_generation.DecodeRepairResponse(completion.Content, position+1)
|
||||
var err error
|
||||
response, err = proposal_generation.DecodeRepairResponse(completion.Content, position+1)
|
||||
if err != nil {
|
||||
operation.FailureReason = outcomes.ReasonInvalidOutput
|
||||
finishAttempt(err, nil, validateSectionCandidatesResult{})
|
||||
@@ -211,12 +208,12 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
}
|
||||
operation.Succeeded = true
|
||||
operation.RecoveryReasons = outcomes.ExecutionRecoveries(operation.Counts)
|
||||
if decoded.Action == "withdraw" {
|
||||
if response.Action == "withdraw" {
|
||||
finishAttempt(nil, nil, validateSectionCandidatesResult{})
|
||||
branch.Outcome = "withdrawn"
|
||||
return nil, nil
|
||||
}
|
||||
revision := decoded.Correction
|
||||
revision := response.Correction
|
||||
localRef = revision.SegmentReference
|
||||
target := 0
|
||||
if input.WorkingTranscript != nil && localRef > 0 && localRef <= len(input.WorkingTranscript.Segments) {
|
||||
|
||||
Reference in New Issue
Block a user