Add version two LLM reports and safe execution diagnostics
This commit is contained in:
183
internal/framework/processreport/llm.go
Normal file
183
internal/framework/processreport/llm.go
Normal file
@@ -0,0 +1,183 @@
|
||||
package processreport
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
|
||||
pk "gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func mapLLMSummary(source llm.ExecutionSummary) reporting.LLMSummary {
|
||||
metric := func(value llm.TokenCount) reporting.TokenMetric {
|
||||
return reporting.TokenMetric{Value: value.Value, Known: value.Known, Complete: value.Complete}
|
||||
}
|
||||
return reporting.LLMSummary{
|
||||
LogicalCalls: source.LogicalCalls, ProviderAttempts: source.ProviderAttempts,
|
||||
TransientRetries: source.TransientRetries, StructuralRepairs: source.StructuralRepairs,
|
||||
Usage: reporting.LLMUsage{Prompt: metric(source.Usage.Prompt), Completion: metric(source.Usage.Completion), Total: metric(source.Usage.Total), Cached: metric(source.Usage.Cached), CacheWrite: metric(source.Usage.CacheWrite)},
|
||||
}
|
||||
}
|
||||
|
||||
// BuildUtilization groups complete logical calls by their selected backend.
|
||||
func BuildUtilization(output *runner.RunOutput, backends map[string]config.Backend) reporting.UtilizationDiagnostics {
|
||||
result := reporting.UtilizationDiagnostics{Backends: make(map[string]reporting.BackendUtilization, len(backends))}
|
||||
if output == nil {
|
||||
return result
|
||||
}
|
||||
result.RunWallTimeMS = output.RunWallTimeMS
|
||||
for id, backend := range backends {
|
||||
group := llm.ObservationSnapshot{}
|
||||
for _, call := range output.Observations.Calls {
|
||||
if call.Prompt.BackendID == id {
|
||||
group.Calls = append(group.Calls, call)
|
||||
addModuleCall(&result, call)
|
||||
}
|
||||
}
|
||||
var wait, generation, retry, wall time.Duration
|
||||
for _, call := range group.Calls {
|
||||
wall += call.WallDuration
|
||||
}
|
||||
for _, event := range output.Observations.Events {
|
||||
if hasCall(group.Calls, event.CallID) {
|
||||
group.Events = append(group.Events, event)
|
||||
if event.Kind == string(pk.ExecutionFinished) {
|
||||
wait += event.CapacityWait
|
||||
generation += event.GenerationDuration
|
||||
retry += event.RetryDelay
|
||||
}
|
||||
}
|
||||
}
|
||||
result.Backends[id] = reporting.BackendUtilization{
|
||||
Concurrency: backend.Concurrency, QueueCapacity: backend.QueueCapacity,
|
||||
LLMSummary: mapLLMSummary(llm.Summarize(group)),
|
||||
CapacityWaitMS: wait.Milliseconds(), GenerationMS: generation.Milliseconds(),
|
||||
RetryDelayMS: retry.Milliseconds(), LogicalWallMS: wall.Milliseconds(),
|
||||
}
|
||||
}
|
||||
for _, module := range output.ModuleResults {
|
||||
found := false
|
||||
for i := range result.Modules {
|
||||
if result.Modules[i].ModuleInstance == module.ModuleInstance {
|
||||
result.Modules[i].ModuleWallTimeMS = module.CompletedAt.Sub(module.StartedAt).Milliseconds()
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
result.Modules = append(result.Modules, reporting.ModuleUtilization{ModuleKey: module.ModuleKey, ModuleInstance: module.ModuleInstance, ModuleWallTimeMS: module.CompletedAt.Sub(module.StartedAt).Milliseconds()})
|
||||
}
|
||||
}
|
||||
for _, timing := range output.ValidatorTimings {
|
||||
result.Validators = append(result.Validators, reporting.ValidatorUtilization{ModuleKey: timing.ModuleKey, ModuleInstance: timing.ModuleInstance, ValidatorKey: timing.ValidatorName, LLMBacked: timing.LLMBacked, Calls: timing.Calls, ElapsedMS: timing.ElapsedMS})
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func hasCall(calls []llm.LogicalCall, id uint64) bool {
|
||||
for _, call := range calls {
|
||||
if call.CallID == id {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func addModuleCall(result *reporting.UtilizationDiagnostics, call llm.LogicalCall) {
|
||||
if call.Context.ModuleInstance == "" {
|
||||
return
|
||||
}
|
||||
for i := range result.Modules {
|
||||
if result.Modules[i].ModuleInstance == call.Context.ModuleInstance {
|
||||
if call.Context.Role == "proposal" {
|
||||
result.Modules[i].ProposalLLMCalls++
|
||||
} else if call.Context.Role == "validation" {
|
||||
result.Modules[i].ValidationLLMCalls++
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
module := reporting.ModuleUtilization{ModuleKey: call.Context.ModuleKey, ModuleInstance: call.Context.ModuleInstance}
|
||||
if call.Context.Role == "proposal" {
|
||||
module.ProposalLLMCalls++
|
||||
} else if call.Context.Role == "validation" {
|
||||
module.ValidationLLMCalls++
|
||||
}
|
||||
result.Modules = append(result.Modules, module)
|
||||
}
|
||||
|
||||
type executionCall struct {
|
||||
CallID uint64 `json:"call_id"`
|
||||
Context structuredoutput.DiagnosticContext `json:"context"`
|
||||
Prompt structuredoutput.PromptMetadata `json:"prompt"`
|
||||
Started time.Time `json:"started"`
|
||||
Finished *time.Time `json:"finished,omitempty"`
|
||||
ElapsedMS int64 `json:"elapsed_ms"`
|
||||
Failure structuredoutput.FailureKind `json:"failure,omitempty"`
|
||||
Summary reporting.LLMSummary `json:"summary"`
|
||||
}
|
||||
|
||||
type executionEvent struct {
|
||||
CallID uint64 `json:"call_id"`
|
||||
Context structuredoutput.DiagnosticContext `json:"context"`
|
||||
Kind string `json:"kind"`
|
||||
RunID string `json:"run_id,omitempty"`
|
||||
Time time.Time `json:"time"`
|
||||
BackendID string `json:"backend_id,omitempty"`
|
||||
ProfileID string `json:"profile_id,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
Generation *uint64 `json:"generation,omitempty"`
|
||||
Attempt *uint64 `json:"attempt,omitempty"`
|
||||
Dispatched bool `json:"dispatched"`
|
||||
CapacityWaitMS int64 `json:"capacity_wait_ms"`
|
||||
GenerationMS int64 `json:"generation_ms"`
|
||||
RetryDelayMS int64 `json:"retry_delay_ms"`
|
||||
DurationMS int64 `json:"duration_ms"`
|
||||
Outcome string `json:"outcome,omitempty"`
|
||||
Failure string `json:"failure,omitempty"`
|
||||
StatusCode int `json:"status_code,omitempty"`
|
||||
RetriesExhausted bool `json:"retries_exhausted,omitempty"`
|
||||
Attempts uint64 `json:"attempts"`
|
||||
TransientRetries uint64 `json:"transient_retries"`
|
||||
StructuralRepairs uint64 `json:"structural_repairs"`
|
||||
Usage reporting.LLMUsage `json:"usage"`
|
||||
}
|
||||
|
||||
// BuildLLMExecution serializes only safe observation fields and millisecond durations.
|
||||
func BuildLLMExecution(snapshot llm.ObservationSnapshot) any {
|
||||
calls := make([]executionCall, 0, len(snapshot.Calls))
|
||||
events := make([]executionEvent, 0, len(snapshot.Events))
|
||||
for _, call := range snapshot.Calls {
|
||||
group := llm.ObservationSnapshot{Calls: []llm.LogicalCall{call}}
|
||||
for _, event := range snapshot.Events {
|
||||
if event.CallID == call.CallID {
|
||||
group.Events = append(group.Events, event)
|
||||
}
|
||||
}
|
||||
var finished *time.Time
|
||||
if !call.Finished.IsZero() {
|
||||
value := call.Finished
|
||||
finished = &value
|
||||
}
|
||||
calls = append(calls, executionCall{CallID: call.CallID, Context: call.Context, Prompt: call.Prompt, Started: call.Started, Finished: finished, ElapsedMS: call.WallDuration.Milliseconds(), Failure: call.Failure, Summary: mapLLMSummary(llm.Summarize(group))})
|
||||
}
|
||||
for _, event := range snapshot.Events {
|
||||
var generation, attempt *uint64
|
||||
if event.Generation != 0 {
|
||||
value := event.Generation
|
||||
generation = &value
|
||||
}
|
||||
if event.Attempt != 0 {
|
||||
value := event.Attempt
|
||||
attempt = &value
|
||||
}
|
||||
events = append(events, executionEvent{CallID: event.CallID, Context: event.Context, Kind: event.Kind, RunID: event.RunID, Time: event.Time, BackendID: event.BackendID, ProfileID: event.ProfileID, Model: event.Model, Generation: generation, Attempt: attempt, Dispatched: event.Dispatched, CapacityWaitMS: event.CapacityWait.Milliseconds(), GenerationMS: event.GenerationDuration.Milliseconds(), RetryDelayMS: event.RetryDelay.Milliseconds(), DurationMS: event.Duration.Milliseconds(), Outcome: event.Outcome, Failure: event.Failure, StatusCode: event.StatusCode, RetriesExhausted: event.RetriesExhausted, Attempts: event.Attempts, TransientRetries: event.TransientRetries, StructuralRepairs: event.StructuralRepairs, Usage: mapLLMSummary(llm.ExecutionSummary{Usage: event.Usage}).Usage})
|
||||
}
|
||||
return struct {
|
||||
Calls []executionCall `json:"calls"`
|
||||
Events []executionEvent `json:"events"`
|
||||
}{calls, events}
|
||||
}
|
||||
56
internal/framework/processreport/llm_test.go
Normal file
56
internal/framework/processreport/llm_test.go
Normal file
@@ -0,0 +1,56 @@
|
||||
package processreport
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
|
||||
pk "gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestExecutionReportingGroupsPredispatchFailureAndKeepsSafeMilliseconds(t *testing.T) {
|
||||
start := time.Now().UTC()
|
||||
snapshot := llm.ObservationSnapshot{
|
||||
Calls: []llm.LogicalCall{
|
||||
{CallID: 1, Context: structuredoutput.DiagnosticContext{Role: "proposal", ModuleInstance: "grammar-1"}, Prompt: structuredoutput.PromptMetadata{BackendID: "primary", ProfileID: "draft", Model: "m"}, Started: start, Finished: start.Add(1500 * time.Millisecond), WallDuration: 1500 * time.Millisecond},
|
||||
{CallID: 2, Context: structuredoutput.DiagnosticContext{Role: "validation"}, Prompt: structuredoutput.PromptMetadata{BackendID: "secondary"}, Started: start, Finished: start.Add(5 * time.Millisecond), WallDuration: 5 * time.Millisecond, Failure: structuredoutput.FailureRuntime},
|
||||
},
|
||||
Events: []llm.ExecutionObservation{{CallID: 1, Kind: string(pk.ExecutionFinished), BackendID: "primary", Attempts: 2, TransientRetries: 1, CapacityWait: 20 * time.Millisecond, GenerationDuration: 500 * time.Millisecond, RetryDelay: 100 * time.Millisecond}},
|
||||
}
|
||||
output := &runner.RunOutput{Observations: snapshot, ObservationsAvailable: true, LLMSummary: llm.Summarize(snapshot)}
|
||||
utilization := BuildUtilization(output, map[string]config.Backend{"primary": {Concurrency: 2, QueueCapacity: 3}, "secondary": {Concurrency: 1}})
|
||||
if utilization.Backends["primary"].LogicalCalls != 1 || utilization.Backends["primary"].ProviderAttempts != 2 || utilization.Backends["primary"].CapacityWaitMS != 20 || utilization.Backends["primary"].LogicalWallMS != 1500 {
|
||||
t.Fatalf("primary backend: %+v", utilization.Backends["primary"])
|
||||
}
|
||||
if utilization.Backends["secondary"].LogicalCalls != 1 || utilization.Backends["secondary"].ProviderAttempts != 0 || !utilization.Backends["secondary"].Usage.Total.Known {
|
||||
t.Fatalf("predispatch backend: %+v", utilization.Backends["secondary"])
|
||||
}
|
||||
raw, err := json.Marshal(BuildLLMExecution(snapshot))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
serialized := string(raw)
|
||||
if !strings.Contains(serialized, `"elapsed_ms":1500`) || !strings.Contains(serialized, `"generation_ms":500`) || strings.Contains(serialized, "_ns") || strings.Contains(serialized, `"attempt":0`) || strings.Contains(serialized, `"section_index":0`) {
|
||||
t.Fatalf("execution serialization: %s", serialized)
|
||||
}
|
||||
if !strings.Contains(serialized, `"calls"`) || !strings.Contains(serialized, `"events"`) {
|
||||
t.Fatal(serialized)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReportSummaryAvailability(t *testing.T) {
|
||||
input := BuildInput{Status: "failed", StartedAt: time.Now(), CompletedAt: time.Now()}
|
||||
if got := Build(input).LLMSummary; got != nil {
|
||||
t.Fatalf("unavailable summary: %+v", got)
|
||||
}
|
||||
input.RunOutput = &runner.RunOutput{ObservationsAvailable: true, LLMSummary: llm.Summarize(llm.ObservationSnapshot{})}
|
||||
got := Build(input).LLMSummary
|
||||
if got == nil || got.LogicalCalls != 0 || !got.Usage.Total.Known || !got.Usage.Total.Complete {
|
||||
t.Fatalf("initialized empty summary: %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||
@@ -28,6 +29,7 @@ type BuildInput struct {
|
||||
NormalizationSummary *normalization.NormalizationSummary
|
||||
ChunkingSummary *chunking.Summary
|
||||
RunOutput *runner.RunOutput
|
||||
Backends map[string]config.Backend
|
||||
}
|
||||
|
||||
// Build creates the public process report without owning command parsing or config loading.
|
||||
@@ -80,6 +82,10 @@ func Build(input BuildInput) reporting.ProcessReport {
|
||||
}
|
||||
}
|
||||
report.ModulesSummary, report.ModuleResults = buildModuleReporting(input.RunOutput)
|
||||
if input.RunOutput != nil && input.RunOutput.ObservationsAvailable {
|
||||
summary := mapLLMSummary(input.RunOutput.LLMSummary)
|
||||
report.LLMSummary = &summary
|
||||
}
|
||||
return report
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user