184 lines
8.5 KiB
Go
184 lines
8.5 KiB
Go
package processreport
|
|
|
|
import (
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
|
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
|
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
|
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
|
"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
|
|
pk "gitea.maximumdirect.net/eric/promptkit"
|
|
)
|
|
|
|
func mapLLMSummary(source llm.ExecutionSummary) reporting.LLMSummary {
|
|
metric := func(value llm.TokenCount) reporting.TokenMetric {
|
|
return reporting.TokenMetric{Value: value.Value, Known: value.Known, Complete: value.Complete}
|
|
}
|
|
return reporting.LLMSummary{
|
|
LogicalCalls: source.LogicalCalls, ProviderAttempts: source.ProviderAttempts,
|
|
TransientRetries: source.TransientRetries, StructuralRepairs: source.StructuralRepairs,
|
|
Usage: reporting.LLMUsage{Prompt: metric(source.Usage.Prompt), Completion: metric(source.Usage.Completion), Total: metric(source.Usage.Total), Cached: metric(source.Usage.Cached), CacheWrite: metric(source.Usage.CacheWrite)},
|
|
}
|
|
}
|
|
|
|
// BuildUtilization groups complete logical calls by their selected backend.
|
|
func BuildUtilization(output *runner.RunOutput, backends map[string]config.Backend) reporting.UtilizationDiagnostics {
|
|
result := reporting.UtilizationDiagnostics{Backends: make(map[string]reporting.BackendUtilization, len(backends))}
|
|
if output == nil {
|
|
return result
|
|
}
|
|
result.RunWallTimeMS = output.RunWallTimeMS
|
|
for id, backend := range backends {
|
|
group := llm.ObservationSnapshot{}
|
|
for _, call := range output.Observations.Calls {
|
|
if call.Prompt.BackendID == id {
|
|
group.Calls = append(group.Calls, call)
|
|
addModuleCall(&result, call)
|
|
}
|
|
}
|
|
var wait, generation, retry, wall time.Duration
|
|
for _, call := range group.Calls {
|
|
wall += call.WallDuration
|
|
}
|
|
for _, event := range output.Observations.Events {
|
|
if hasCall(group.Calls, event.CallID) {
|
|
group.Events = append(group.Events, event)
|
|
if event.Kind == string(pk.ExecutionFinished) {
|
|
wait += event.CapacityWait
|
|
generation += event.GenerationDuration
|
|
retry += event.RetryDelay
|
|
}
|
|
}
|
|
}
|
|
result.Backends[id] = reporting.BackendUtilization{
|
|
Concurrency: backend.Concurrency, QueueCapacity: backend.QueueCapacity,
|
|
LLMSummary: mapLLMSummary(llm.Summarize(group)),
|
|
CapacityWaitMS: wait.Milliseconds(), GenerationMS: generation.Milliseconds(),
|
|
RetryDelayMS: retry.Milliseconds(), LogicalWallMS: wall.Milliseconds(),
|
|
}
|
|
}
|
|
for _, module := range output.ModuleResults {
|
|
found := false
|
|
for i := range result.Modules {
|
|
if result.Modules[i].ModuleInstance == module.ModuleInstance {
|
|
result.Modules[i].ModuleWallTimeMS = module.CompletedAt.Sub(module.StartedAt).Milliseconds()
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
result.Modules = append(result.Modules, reporting.ModuleUtilization{ModuleKey: module.ModuleKey, ModuleInstance: module.ModuleInstance, ModuleWallTimeMS: module.CompletedAt.Sub(module.StartedAt).Milliseconds()})
|
|
}
|
|
}
|
|
for _, timing := range output.ValidatorTimings {
|
|
result.Validators = append(result.Validators, reporting.ValidatorUtilization{ModuleKey: timing.ModuleKey, ModuleInstance: timing.ModuleInstance, ValidatorKey: timing.ValidatorName, LLMBacked: timing.LLMBacked, Calls: timing.Calls, ElapsedMS: timing.ElapsedMS})
|
|
}
|
|
return result
|
|
}
|
|
|
|
func hasCall(calls []llm.LogicalCall, id uint64) bool {
|
|
for _, call := range calls {
|
|
if call.CallID == id {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func addModuleCall(result *reporting.UtilizationDiagnostics, call llm.LogicalCall) {
|
|
if call.Context.ModuleInstance == "" {
|
|
return
|
|
}
|
|
for i := range result.Modules {
|
|
if result.Modules[i].ModuleInstance == call.Context.ModuleInstance {
|
|
if call.Context.Role == "proposal" {
|
|
result.Modules[i].ProposalLLMCalls++
|
|
} else if call.Context.Role == "validation" {
|
|
result.Modules[i].ValidationLLMCalls++
|
|
}
|
|
return
|
|
}
|
|
}
|
|
module := reporting.ModuleUtilization{ModuleKey: call.Context.ModuleKey, ModuleInstance: call.Context.ModuleInstance}
|
|
if call.Context.Role == "proposal" {
|
|
module.ProposalLLMCalls++
|
|
} else if call.Context.Role == "validation" {
|
|
module.ValidationLLMCalls++
|
|
}
|
|
result.Modules = append(result.Modules, module)
|
|
}
|
|
|
|
type executionCall struct {
|
|
CallID uint64 `json:"call_id"`
|
|
Context structuredoutput.DiagnosticContext `json:"context"`
|
|
Prompt structuredoutput.PromptMetadata `json:"prompt"`
|
|
Started time.Time `json:"started"`
|
|
Finished *time.Time `json:"finished,omitempty"`
|
|
ElapsedMS int64 `json:"elapsed_ms"`
|
|
Failure structuredoutput.FailureKind `json:"failure,omitempty"`
|
|
Summary reporting.LLMSummary `json:"summary"`
|
|
}
|
|
|
|
type executionEvent struct {
|
|
CallID uint64 `json:"call_id"`
|
|
Context structuredoutput.DiagnosticContext `json:"context"`
|
|
Kind string `json:"kind"`
|
|
RunID string `json:"run_id,omitempty"`
|
|
Time time.Time `json:"time"`
|
|
BackendID string `json:"backend_id,omitempty"`
|
|
ProfileID string `json:"profile_id,omitempty"`
|
|
Model string `json:"model,omitempty"`
|
|
Generation *uint64 `json:"generation,omitempty"`
|
|
Attempt *uint64 `json:"attempt,omitempty"`
|
|
Dispatched bool `json:"dispatched"`
|
|
CapacityWaitMS int64 `json:"capacity_wait_ms"`
|
|
GenerationMS int64 `json:"generation_ms"`
|
|
RetryDelayMS int64 `json:"retry_delay_ms"`
|
|
DurationMS int64 `json:"duration_ms"`
|
|
Outcome string `json:"outcome,omitempty"`
|
|
Failure string `json:"failure,omitempty"`
|
|
StatusCode int `json:"status_code,omitempty"`
|
|
RetriesExhausted bool `json:"retries_exhausted,omitempty"`
|
|
Attempts uint64 `json:"attempts"`
|
|
TransientRetries uint64 `json:"transient_retries"`
|
|
StructuralRepairs uint64 `json:"structural_repairs"`
|
|
Usage reporting.LLMUsage `json:"usage"`
|
|
}
|
|
|
|
// BuildLLMExecution serializes only safe observation fields and millisecond durations.
|
|
func BuildLLMExecution(snapshot llm.ObservationSnapshot) any {
|
|
calls := make([]executionCall, 0, len(snapshot.Calls))
|
|
events := make([]executionEvent, 0, len(snapshot.Events))
|
|
for _, call := range snapshot.Calls {
|
|
group := llm.ObservationSnapshot{Calls: []llm.LogicalCall{call}}
|
|
for _, event := range snapshot.Events {
|
|
if event.CallID == call.CallID {
|
|
group.Events = append(group.Events, event)
|
|
}
|
|
}
|
|
var finished *time.Time
|
|
if !call.Finished.IsZero() {
|
|
value := call.Finished
|
|
finished = &value
|
|
}
|
|
calls = append(calls, executionCall{CallID: call.CallID, Context: call.Context, Prompt: call.Prompt, Started: call.Started, Finished: finished, ElapsedMS: call.WallDuration.Milliseconds(), Failure: call.Failure, Summary: mapLLMSummary(llm.Summarize(group))})
|
|
}
|
|
for _, event := range snapshot.Events {
|
|
var generation, attempt *uint64
|
|
if event.Generation != 0 {
|
|
value := event.Generation
|
|
generation = &value
|
|
}
|
|
if event.Attempt != 0 {
|
|
value := event.Attempt
|
|
attempt = &value
|
|
}
|
|
events = append(events, executionEvent{CallID: event.CallID, Context: event.Context, Kind: event.Kind, RunID: event.RunID, Time: event.Time, BackendID: event.BackendID, ProfileID: event.ProfileID, Model: event.Model, Generation: generation, Attempt: attempt, Dispatched: event.Dispatched, CapacityWaitMS: event.CapacityWait.Milliseconds(), GenerationMS: event.GenerationDuration.Milliseconds(), RetryDelayMS: event.RetryDelay.Milliseconds(), DurationMS: event.Duration.Milliseconds(), Outcome: event.Outcome, Failure: event.Failure, StatusCode: event.StatusCode, RetriesExhausted: event.RetriesExhausted, Attempts: event.Attempts, TransientRetries: event.TransientRetries, StructuralRepairs: event.StructuralRepairs, Usage: mapLLMSummary(llm.ExecutionSummary{Usage: event.Usage}).Usage})
|
|
}
|
|
return struct {
|
|
Calls []executionCall `json:"calls"`
|
|
Events []executionEvent `json:"events"`
|
|
}{calls, events}
|
|
}
|