Add version two LLM reports and safe execution diagnostics
This commit is contained in:
183
internal/framework/processreport/llm.go
Normal file
183
internal/framework/processreport/llm.go
Normal file
@@ -0,0 +1,183 @@
|
||||
package processreport
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
|
||||
pk "gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func mapLLMSummary(source llm.ExecutionSummary) reporting.LLMSummary {
|
||||
metric := func(value llm.TokenCount) reporting.TokenMetric {
|
||||
return reporting.TokenMetric{Value: value.Value, Known: value.Known, Complete: value.Complete}
|
||||
}
|
||||
return reporting.LLMSummary{
|
||||
LogicalCalls: source.LogicalCalls, ProviderAttempts: source.ProviderAttempts,
|
||||
TransientRetries: source.TransientRetries, StructuralRepairs: source.StructuralRepairs,
|
||||
Usage: reporting.LLMUsage{Prompt: metric(source.Usage.Prompt), Completion: metric(source.Usage.Completion), Total: metric(source.Usage.Total), Cached: metric(source.Usage.Cached), CacheWrite: metric(source.Usage.CacheWrite)},
|
||||
}
|
||||
}
|
||||
|
||||
// BuildUtilization groups complete logical calls by their selected backend.
|
||||
func BuildUtilization(output *runner.RunOutput, backends map[string]config.Backend) reporting.UtilizationDiagnostics {
|
||||
result := reporting.UtilizationDiagnostics{Backends: make(map[string]reporting.BackendUtilization, len(backends))}
|
||||
if output == nil {
|
||||
return result
|
||||
}
|
||||
result.RunWallTimeMS = output.RunWallTimeMS
|
||||
for id, backend := range backends {
|
||||
group := llm.ObservationSnapshot{}
|
||||
for _, call := range output.Observations.Calls {
|
||||
if call.Prompt.BackendID == id {
|
||||
group.Calls = append(group.Calls, call)
|
||||
addModuleCall(&result, call)
|
||||
}
|
||||
}
|
||||
var wait, generation, retry, wall time.Duration
|
||||
for _, call := range group.Calls {
|
||||
wall += call.WallDuration
|
||||
}
|
||||
for _, event := range output.Observations.Events {
|
||||
if hasCall(group.Calls, event.CallID) {
|
||||
group.Events = append(group.Events, event)
|
||||
if event.Kind == string(pk.ExecutionFinished) {
|
||||
wait += event.CapacityWait
|
||||
generation += event.GenerationDuration
|
||||
retry += event.RetryDelay
|
||||
}
|
||||
}
|
||||
}
|
||||
result.Backends[id] = reporting.BackendUtilization{
|
||||
Concurrency: backend.Concurrency, QueueCapacity: backend.QueueCapacity,
|
||||
LLMSummary: mapLLMSummary(llm.Summarize(group)),
|
||||
CapacityWaitMS: wait.Milliseconds(), GenerationMS: generation.Milliseconds(),
|
||||
RetryDelayMS: retry.Milliseconds(), LogicalWallMS: wall.Milliseconds(),
|
||||
}
|
||||
}
|
||||
for _, module := range output.ModuleResults {
|
||||
found := false
|
||||
for i := range result.Modules {
|
||||
if result.Modules[i].ModuleInstance == module.ModuleInstance {
|
||||
result.Modules[i].ModuleWallTimeMS = module.CompletedAt.Sub(module.StartedAt).Milliseconds()
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
result.Modules = append(result.Modules, reporting.ModuleUtilization{ModuleKey: module.ModuleKey, ModuleInstance: module.ModuleInstance, ModuleWallTimeMS: module.CompletedAt.Sub(module.StartedAt).Milliseconds()})
|
||||
}
|
||||
}
|
||||
for _, timing := range output.ValidatorTimings {
|
||||
result.Validators = append(result.Validators, reporting.ValidatorUtilization{ModuleKey: timing.ModuleKey, ModuleInstance: timing.ModuleInstance, ValidatorKey: timing.ValidatorName, LLMBacked: timing.LLMBacked, Calls: timing.Calls, ElapsedMS: timing.ElapsedMS})
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func hasCall(calls []llm.LogicalCall, id uint64) bool {
|
||||
for _, call := range calls {
|
||||
if call.CallID == id {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func addModuleCall(result *reporting.UtilizationDiagnostics, call llm.LogicalCall) {
|
||||
if call.Context.ModuleInstance == "" {
|
||||
return
|
||||
}
|
||||
for i := range result.Modules {
|
||||
if result.Modules[i].ModuleInstance == call.Context.ModuleInstance {
|
||||
if call.Context.Role == "proposal" {
|
||||
result.Modules[i].ProposalLLMCalls++
|
||||
} else if call.Context.Role == "validation" {
|
||||
result.Modules[i].ValidationLLMCalls++
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
module := reporting.ModuleUtilization{ModuleKey: call.Context.ModuleKey, ModuleInstance: call.Context.ModuleInstance}
|
||||
if call.Context.Role == "proposal" {
|
||||
module.ProposalLLMCalls++
|
||||
} else if call.Context.Role == "validation" {
|
||||
module.ValidationLLMCalls++
|
||||
}
|
||||
result.Modules = append(result.Modules, module)
|
||||
}
|
||||
|
||||
type executionCall struct {
|
||||
CallID uint64 `json:"call_id"`
|
||||
Context structuredoutput.DiagnosticContext `json:"context"`
|
||||
Prompt structuredoutput.PromptMetadata `json:"prompt"`
|
||||
Started time.Time `json:"started"`
|
||||
Finished *time.Time `json:"finished,omitempty"`
|
||||
ElapsedMS int64 `json:"elapsed_ms"`
|
||||
Failure structuredoutput.FailureKind `json:"failure,omitempty"`
|
||||
Summary reporting.LLMSummary `json:"summary"`
|
||||
}
|
||||
|
||||
type executionEvent struct {
|
||||
CallID uint64 `json:"call_id"`
|
||||
Context structuredoutput.DiagnosticContext `json:"context"`
|
||||
Kind string `json:"kind"`
|
||||
RunID string `json:"run_id,omitempty"`
|
||||
Time time.Time `json:"time"`
|
||||
BackendID string `json:"backend_id,omitempty"`
|
||||
ProfileID string `json:"profile_id,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
Generation *uint64 `json:"generation,omitempty"`
|
||||
Attempt *uint64 `json:"attempt,omitempty"`
|
||||
Dispatched bool `json:"dispatched"`
|
||||
CapacityWaitMS int64 `json:"capacity_wait_ms"`
|
||||
GenerationMS int64 `json:"generation_ms"`
|
||||
RetryDelayMS int64 `json:"retry_delay_ms"`
|
||||
DurationMS int64 `json:"duration_ms"`
|
||||
Outcome string `json:"outcome,omitempty"`
|
||||
Failure string `json:"failure,omitempty"`
|
||||
StatusCode int `json:"status_code,omitempty"`
|
||||
RetriesExhausted bool `json:"retries_exhausted,omitempty"`
|
||||
Attempts uint64 `json:"attempts"`
|
||||
TransientRetries uint64 `json:"transient_retries"`
|
||||
StructuralRepairs uint64 `json:"structural_repairs"`
|
||||
Usage reporting.LLMUsage `json:"usage"`
|
||||
}
|
||||
|
||||
// BuildLLMExecution serializes only safe observation fields and millisecond durations.
|
||||
func BuildLLMExecution(snapshot llm.ObservationSnapshot) any {
|
||||
calls := make([]executionCall, 0, len(snapshot.Calls))
|
||||
events := make([]executionEvent, 0, len(snapshot.Events))
|
||||
for _, call := range snapshot.Calls {
|
||||
group := llm.ObservationSnapshot{Calls: []llm.LogicalCall{call}}
|
||||
for _, event := range snapshot.Events {
|
||||
if event.CallID == call.CallID {
|
||||
group.Events = append(group.Events, event)
|
||||
}
|
||||
}
|
||||
var finished *time.Time
|
||||
if !call.Finished.IsZero() {
|
||||
value := call.Finished
|
||||
finished = &value
|
||||
}
|
||||
calls = append(calls, executionCall{CallID: call.CallID, Context: call.Context, Prompt: call.Prompt, Started: call.Started, Finished: finished, ElapsedMS: call.WallDuration.Milliseconds(), Failure: call.Failure, Summary: mapLLMSummary(llm.Summarize(group))})
|
||||
}
|
||||
for _, event := range snapshot.Events {
|
||||
var generation, attempt *uint64
|
||||
if event.Generation != 0 {
|
||||
value := event.Generation
|
||||
generation = &value
|
||||
}
|
||||
if event.Attempt != 0 {
|
||||
value := event.Attempt
|
||||
attempt = &value
|
||||
}
|
||||
events = append(events, executionEvent{CallID: event.CallID, Context: event.Context, Kind: event.Kind, RunID: event.RunID, Time: event.Time, BackendID: event.BackendID, ProfileID: event.ProfileID, Model: event.Model, Generation: generation, Attempt: attempt, Dispatched: event.Dispatched, CapacityWaitMS: event.CapacityWait.Milliseconds(), GenerationMS: event.GenerationDuration.Milliseconds(), RetryDelayMS: event.RetryDelay.Milliseconds(), DurationMS: event.Duration.Milliseconds(), Outcome: event.Outcome, Failure: event.Failure, StatusCode: event.StatusCode, RetriesExhausted: event.RetriesExhausted, Attempts: event.Attempts, TransientRetries: event.TransientRetries, StructuralRepairs: event.StructuralRepairs, Usage: mapLLMSummary(llm.ExecutionSummary{Usage: event.Usage}).Usage})
|
||||
}
|
||||
return struct {
|
||||
Calls []executionCall `json:"calls"`
|
||||
Events []executionEvent `json:"events"`
|
||||
}{calls, events}
|
||||
}
|
||||
Reference in New Issue
Block a user