Fix the five library adoption review findings
This commit is contained in:
@@ -29,86 +29,74 @@ func BuildUtilization(output *runner.RunOutput, backends map[string]config.Backe
|
||||
return result
|
||||
}
|
||||
result.RunWallTimeMS = output.RunWallTimeMS
|
||||
for id, backend := range backends {
|
||||
group := llm.ObservationSnapshot{}
|
||||
for _, call := range output.Observations.Calls {
|
||||
if call.Prompt.BackendID == id {
|
||||
group.Calls = append(group.Calls, call)
|
||||
addModuleCall(&result, call)
|
||||
groups := make(map[string]*llm.ObservationSnapshot, len(backends))
|
||||
callBackends := make(map[uint64]string, len(output.Observations.Calls))
|
||||
moduleIndices := make(map[string]int)
|
||||
moduleIndex := func(key, instance string) int {
|
||||
if i, ok := moduleIndices[instance]; ok {
|
||||
return i
|
||||
}
|
||||
i := len(result.Modules)
|
||||
moduleIndices[instance] = i
|
||||
result.Modules = append(result.Modules, reporting.ModuleUtilization{ModuleKey: key, ModuleInstance: instance})
|
||||
return i
|
||||
}
|
||||
// Seed configured order, including modules that made no LLM calls.
|
||||
for _, module := range output.ModuleResults {
|
||||
i := moduleIndex(module.ModuleKey, module.ModuleInstance)
|
||||
result.Modules[i].ModuleWallTimeMS = module.CompletedAt.Sub(module.StartedAt).Milliseconds()
|
||||
}
|
||||
for _, call := range output.Observations.Calls {
|
||||
id := call.Prompt.BackendID
|
||||
if groups[id] == nil {
|
||||
groups[id] = &llm.ObservationSnapshot{}
|
||||
}
|
||||
groups[id].Calls = append(groups[id].Calls, call)
|
||||
callBackends[call.CallID] = id
|
||||
if call.Context.ModuleInstance != "" {
|
||||
i := moduleIndex(call.Context.ModuleKey, call.Context.ModuleInstance)
|
||||
switch call.Context.Role {
|
||||
case "proposal":
|
||||
result.Modules[i].ProposalLLMCalls++
|
||||
case "validation":
|
||||
result.Modules[i].ValidationLLMCalls++
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, event := range output.Observations.Events {
|
||||
if id, ok := callBackends[event.CallID]; ok {
|
||||
groups[id].Events = append(groups[id].Events, event)
|
||||
}
|
||||
}
|
||||
for id, backend := range backends {
|
||||
group := groups[id]
|
||||
if group == nil {
|
||||
group = &llm.ObservationSnapshot{}
|
||||
}
|
||||
var wait, generation, retry, wall time.Duration
|
||||
for _, call := range group.Calls {
|
||||
wall += call.WallDuration
|
||||
}
|
||||
for _, event := range output.Observations.Events {
|
||||
if hasCall(group.Calls, event.CallID) {
|
||||
group.Events = append(group.Events, event)
|
||||
if event.Kind == string(pk.ExecutionFinished) {
|
||||
wait += event.CapacityWait
|
||||
generation += event.GenerationDuration
|
||||
retry += event.RetryDelay
|
||||
}
|
||||
for _, event := range group.Events {
|
||||
if event.Kind == string(pk.ExecutionFinished) {
|
||||
wait += event.CapacityWait
|
||||
generation += event.GenerationDuration
|
||||
retry += event.RetryDelay
|
||||
}
|
||||
}
|
||||
result.Backends[id] = reporting.BackendUtilization{
|
||||
Concurrency: backend.Concurrency, QueueCapacity: backend.QueueCapacity,
|
||||
LLMSummary: mapLLMSummary(llm.Summarize(group)),
|
||||
LLMSummary: mapLLMSummary(llm.Summarize(*group)),
|
||||
CapacityWaitMS: wait.Milliseconds(), GenerationMS: generation.Milliseconds(),
|
||||
RetryDelayMS: retry.Milliseconds(), LogicalWallMS: wall.Milliseconds(),
|
||||
}
|
||||
}
|
||||
for _, module := range output.ModuleResults {
|
||||
found := false
|
||||
for i := range result.Modules {
|
||||
if result.Modules[i].ModuleInstance == module.ModuleInstance {
|
||||
result.Modules[i].ModuleWallTimeMS = module.CompletedAt.Sub(module.StartedAt).Milliseconds()
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
result.Modules = append(result.Modules, reporting.ModuleUtilization{ModuleKey: module.ModuleKey, ModuleInstance: module.ModuleInstance, ModuleWallTimeMS: module.CompletedAt.Sub(module.StartedAt).Milliseconds()})
|
||||
}
|
||||
}
|
||||
for _, timing := range output.ValidatorTimings {
|
||||
result.Validators = append(result.Validators, reporting.ValidatorUtilization{ModuleKey: timing.ModuleKey, ModuleInstance: timing.ModuleInstance, ValidatorKey: timing.ValidatorName, LLMBacked: timing.LLMBacked, Calls: timing.Calls, ElapsedMS: timing.ElapsedMS})
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func hasCall(calls []llm.LogicalCall, id uint64) bool {
|
||||
for _, call := range calls {
|
||||
if call.CallID == id {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func addModuleCall(result *reporting.UtilizationDiagnostics, call llm.LogicalCall) {
|
||||
if call.Context.ModuleInstance == "" {
|
||||
return
|
||||
}
|
||||
for i := range result.Modules {
|
||||
if result.Modules[i].ModuleInstance == call.Context.ModuleInstance {
|
||||
if call.Context.Role == "proposal" {
|
||||
result.Modules[i].ProposalLLMCalls++
|
||||
} else if call.Context.Role == "validation" {
|
||||
result.Modules[i].ValidationLLMCalls++
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
module := reporting.ModuleUtilization{ModuleKey: call.Context.ModuleKey, ModuleInstance: call.Context.ModuleInstance}
|
||||
if call.Context.Role == "proposal" {
|
||||
module.ProposalLLMCalls++
|
||||
} else if call.Context.Role == "validation" {
|
||||
module.ValidationLLMCalls++
|
||||
}
|
||||
result.Modules = append(result.Modules, module)
|
||||
}
|
||||
|
||||
type executionCall struct {
|
||||
CallID uint64 `json:"call_id"`
|
||||
Context structuredoutput.DiagnosticContext `json:"context"`
|
||||
@@ -150,13 +138,12 @@ type executionEvent struct {
|
||||
func BuildLLMExecution(snapshot llm.ObservationSnapshot) any {
|
||||
calls := make([]executionCall, 0, len(snapshot.Calls))
|
||||
events := make([]executionEvent, 0, len(snapshot.Events))
|
||||
eventsByCall := make(map[uint64][]llm.ExecutionObservation, len(snapshot.Calls))
|
||||
for _, event := range snapshot.Events {
|
||||
eventsByCall[event.CallID] = append(eventsByCall[event.CallID], event)
|
||||
}
|
||||
for _, call := range snapshot.Calls {
|
||||
group := llm.ObservationSnapshot{Calls: []llm.LogicalCall{call}}
|
||||
for _, event := range snapshot.Events {
|
||||
if event.CallID == call.CallID {
|
||||
group.Events = append(group.Events, event)
|
||||
}
|
||||
}
|
||||
group := llm.ObservationSnapshot{Calls: []llm.LogicalCall{call}, Events: eventsByCall[call.CallID]}
|
||||
var finished *time.Time
|
||||
if !call.Finished.IsZero() {
|
||||
value := call.Finished
|
||||
|
||||
@@ -2,6 +2,7 @@ package processreport
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -54,3 +55,72 @@ func TestReportSummaryAvailability(t *testing.T) {
|
||||
t.Fatalf("initialized empty summary: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkExecutionReports(b *testing.B) {
|
||||
for _, n := range []int{1000, 10000, 20000} {
|
||||
snapshot := llm.ObservationSnapshot{}
|
||||
for i := 0; i < n; i++ {
|
||||
snapshot.Calls = append(snapshot.Calls, llm.LogicalCall{CallID: uint64(i + 1), Prompt: structuredoutput.PromptMetadata{BackendID: "local"}, Context: structuredoutput.DiagnosticContext{ModuleInstance: "grammar", Role: "proposal"}})
|
||||
snapshot.Events = append(snapshot.Events, llm.ExecutionObservation{CallID: uint64(i + 1), Kind: string(pk.ExecutionFinished), Attempts: 1})
|
||||
}
|
||||
output := &runner.RunOutput{Observations: snapshot}
|
||||
b.Run(fmt.Sprintf("execution/%d", n), func(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
BuildLLMExecution(snapshot)
|
||||
}
|
||||
})
|
||||
b.Run(fmt.Sprintf("utilization/%d", n), func(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
BuildUtilization(output, map[string]config.Backend{"local": {Concurrency: 1}})
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIndexedReportsPreserveOrderAndPartialUsage(t *testing.T) {
|
||||
snapshot := llm.ObservationSnapshot{
|
||||
Calls: []llm.LogicalCall{
|
||||
{CallID: 3, Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleInstance: "grammar_2"}, Prompt: structuredoutput.PromptMetadata{BackendID: "b"}},
|
||||
{CallID: 1, Context: structuredoutput.DiagnosticContext{Role: "proposal", ModuleInstance: "grammar_1"}, Prompt: structuredoutput.PromptMetadata{BackendID: "a"}},
|
||||
{CallID: 2, Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleInstance: "grammar_1"}, Prompt: structuredoutput.PromptMetadata{BackendID: "a"}},
|
||||
},
|
||||
Events: []llm.ExecutionObservation{
|
||||
{CallID: 1, Kind: string(pk.ExecutionFinished), Attempts: 1, Usage: llm.Usage{Total: llm.TokenCount{Value: 7, Known: true, Complete: true}}},
|
||||
{CallID: 3, Kind: string(pk.ExecutionFinished), Attempts: 1},
|
||||
{CallID: 2, Kind: string(pk.ExecutionFinished), Attempts: 1},
|
||||
},
|
||||
}
|
||||
raw, err := json.Marshal(BuildLLMExecution(snapshot))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var execution struct {
|
||||
Calls []executionCall `json:"calls"`
|
||||
Events []executionEvent `json:"events"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &execution); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i, call := range snapshot.Calls {
|
||||
if execution.Calls[i].CallID != call.CallID {
|
||||
t.Fatal("call order changed")
|
||||
}
|
||||
}
|
||||
for i, event := range snapshot.Events {
|
||||
if execution.Events[i].CallID != event.CallID {
|
||||
t.Fatal("event order changed")
|
||||
}
|
||||
}
|
||||
if execution.Calls[1].Summary.Usage.Total.Value != 7 || !execution.Calls[1].Summary.Usage.Total.Complete || execution.Calls[0].Summary.Usage.Total.Known {
|
||||
t.Fatalf("call usage: %+v", execution.Calls)
|
||||
}
|
||||
output := &runner.RunOutput{Observations: snapshot}
|
||||
utilization := BuildUtilization(output, map[string]config.Backend{"a": {}, "b": {}})
|
||||
total := utilization.Backends["a"].Usage.Total
|
||||
if total.Value != 7 || !total.Known || total.Complete || utilization.Backends["a"].ProviderAttempts != 2 {
|
||||
t.Fatalf("partial backend usage: %+v", utilization)
|
||||
}
|
||||
if len(utilization.Modules) != 2 || utilization.Modules[1].ProposalLLMCalls != 1 || utilization.Modules[1].ValidationLLMCalls != 1 {
|
||||
t.Fatalf("module grouping: %+v", utilization.Modules)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user