audita/internal/framework/processreport/llm_test.go

package processreport

import (
	"encoding/json"
	"fmt"
	"strings"
	"testing"
	"time"

	"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
	"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
	"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
	"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
	pk "gitea.maximumdirect.net/eric/promptkit"
)

func TestExecutionReportingGroupsPredispatchFailureAndKeepsSafeMilliseconds(t *testing.T) {
	start := time.Now().UTC()
	snapshot := llm.ObservationSnapshot{
		Calls: []llm.LogicalCall{
			{CallID: 1, Context: structuredoutput.DiagnosticContext{Role: "proposal", ModuleInstance: "grammar-1"}, Prompt: structuredoutput.PromptMetadata{BackendID: "primary", ProfileID: "draft", Model: "m"}, Started: start, Finished: start.Add(1500 * time.Millisecond), WallDuration: 1500 * time.Millisecond},
			{CallID: 2, Context: structuredoutput.DiagnosticContext{Role: "validation"}, Prompt: structuredoutput.PromptMetadata{BackendID: "secondary"}, Started: start, Finished: start.Add(5 * time.Millisecond), WallDuration: 5 * time.Millisecond, Failure: structuredoutput.FailureRuntime},
		},
		Events: []llm.ExecutionObservation{{CallID: 1, Kind: string(pk.ExecutionFinished), BackendID: "primary", Attempts: 2, TransientRetries: 1, CapacityWait: 20 * time.Millisecond, GenerationDuration: 500 * time.Millisecond, RetryDelay: 100 * time.Millisecond}},
	}
	output := &runner.RunOutput{Observations: snapshot, ObservationsAvailable: true, LLMSummary: llm.Summarize(snapshot)}
	utilization := BuildUtilization(output, map[string]config.Backend{"primary": {Concurrency: 2, QueueCapacity: 3}, "secondary": {Concurrency: 1}})
	if utilization.Backends["primary"].LogicalCalls != 1 || utilization.Backends["primary"].ProviderAttempts != 2 || utilization.Backends["primary"].CapacityWaitMS != 20 || utilization.Backends["primary"].LogicalWallMS != 1500 {
		t.Fatalf("primary backend: %+v", utilization.Backends["primary"])
	}
	if utilization.Backends["secondary"].LogicalCalls != 1 || utilization.Backends["secondary"].ProviderAttempts != 0 || !utilization.Backends["secondary"].Usage.Total.Known {
		t.Fatalf("predispatch backend: %+v", utilization.Backends["secondary"])
	}
	raw, err := json.Marshal(BuildLLMExecution(snapshot))
	if err != nil {
		t.Fatal(err)
	}
	serialized := string(raw)
	if !strings.Contains(serialized, `"elapsed_ms":1500`) || !strings.Contains(serialized, `"generation_ms":500`) || strings.Contains(serialized, "_ns") || strings.Contains(serialized, `"attempt":0`) || strings.Contains(serialized, `"section_index":0`) {
		t.Fatalf("execution serialization: %s", serialized)
	}
	if !strings.Contains(serialized, `"calls"`) || !strings.Contains(serialized, `"events"`) {
		t.Fatal(serialized)
	}
}

func TestReportSummaryAvailability(t *testing.T) {
	got := mapLLMSummary(llm.Summarize(llm.ObservationSnapshot{}))
	if got.LogicalCalls != 0 || !got.Usage.Total.Known || !got.Usage.Total.Complete {
		t.Fatalf("initialized empty summary: %+v", got)
	}
}

func BenchmarkExecutionReports(b *testing.B) {
	for _, n := range []int{1000, 10000, 20000} {
		snapshot := llm.ObservationSnapshot{}
		for i := 0; i < n; i++ {
			snapshot.Calls = append(snapshot.Calls, llm.LogicalCall{CallID: uint64(i + 1), Prompt: structuredoutput.PromptMetadata{BackendID: "local"}, Context: structuredoutput.DiagnosticContext{ModuleInstance: "grammar", Role: "proposal"}})
			snapshot.Events = append(snapshot.Events, llm.ExecutionObservation{CallID: uint64(i + 1), Kind: string(pk.ExecutionFinished), Attempts: 1})
		}
		output := &runner.RunOutput{Observations: snapshot}
		b.Run(fmt.Sprintf("execution/%d", n), func(b *testing.B) {
			for i := 0; i < b.N; i++ {
				BuildLLMExecution(snapshot)
			}
		})
		b.Run(fmt.Sprintf("utilization/%d", n), func(b *testing.B) {
			for i := 0; i < b.N; i++ {
				BuildUtilization(output, map[string]config.Backend{"local": {Concurrency: 1}})
			}
		})
	}
}

func TestIndexedReportsPreserveOrderAndPartialUsage(t *testing.T) {
	snapshot := llm.ObservationSnapshot{
		Calls: []llm.LogicalCall{
			{CallID: 3, Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleInstance: "grammar_2"}, Prompt: structuredoutput.PromptMetadata{BackendID: "b"}},
			{CallID: 1, Context: structuredoutput.DiagnosticContext{Role: "proposal", ModuleInstance: "grammar_1"}, Prompt: structuredoutput.PromptMetadata{BackendID: "a"}},
			{CallID: 2, Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleInstance: "grammar_1"}, Prompt: structuredoutput.PromptMetadata{BackendID: "a"}},
		},
		Events: []llm.ExecutionObservation{
			{CallID: 1, Kind: string(pk.ExecutionFinished), Attempts: 1, Usage: llm.Usage{Total: llm.TokenCount{Value: 7, Known: true, Complete: true}}},
			{CallID: 3, Kind: string(pk.ExecutionFinished), Attempts: 1},
			{CallID: 2, Kind: string(pk.ExecutionFinished), Attempts: 1},
		},
	}
	raw, err := json.Marshal(BuildLLMExecution(snapshot))
	if err != nil {
		t.Fatal(err)
	}
	var execution struct {
		Calls  []executionCall  `json:"calls"`
		Events []executionEvent `json:"events"`
	}
	if err := json.Unmarshal(raw, &execution); err != nil {
		t.Fatal(err)
	}
	for i, call := range snapshot.Calls {
		if execution.Calls[i].CallID != call.CallID {
			t.Fatal("call order changed")
		}
	}
	for i, event := range snapshot.Events {
		if execution.Events[i].CallID != event.CallID {
			t.Fatal("event order changed")
		}
	}
	if execution.Calls[1].Summary.Usage.Total.Value != 7 || !execution.Calls[1].Summary.Usage.Total.Complete || execution.Calls[0].Summary.Usage.Total.Known {
		t.Fatalf("call usage: %+v", execution.Calls)
	}
	output := &runner.RunOutput{Observations: snapshot}
	utilization := BuildUtilization(output, map[string]config.Backend{"a": {}, "b": {}})
	total := utilization.Backends["a"].Usage.Total
	if total.Value != 7 || !total.Known || total.Complete || utilization.Backends["a"].ProviderAttempts != 2 {
		t.Fatalf("partial backend usage: %+v", utilization)
	}
	if len(utilization.Modules) != 2 || utilization.Modules[1].ProposalLLMCalls != 1 || utilization.Modules[1].ValidationLLMCalls != 1 {
		t.Fatalf("module grouping: %+v", utilization.Modules)
	}
}