Project terminal execution observations into shared reports

docs/internal/adapters.md

@@ -23,6 +23,8 @@ described by its tagged
   and errors to HTTP DTOs.
 - `internal/format` renders `promptkit.PreparedRun` values as deterministic text
   or JSON.
+- `internal/report` maps Promptkit's terminal execution event to a small
+  application-owned value without invoking the engine or writing output.
 
 ## Wiring Flows
 
@@ -71,6 +73,11 @@ Failure mapping uses `errors.Is` against Promptkit's public sentinels and the
 HTTP reader's Scriptorium-owned containment and size errors. Wrapped reader
 errors preserve their identity through Promptkit's artifact-load boundary.
 
+The report collector is per invocation. It ignores nonterminal events and
+copies terminal aggregates once, including usage presence and completeness.
+It retains no history or shared state; adapters decide whether to attach it and
+how to present its snapshot.
+
 ## Package-Local Guarantees
 
 - Adapters contain no copied framework types or orchestration.

docs/internal/overview.md

@@ -12,6 +12,7 @@ public behavior belongs in the linked contracts.
 | `internal/config` | Discovers and strictly decodes application configuration and applies built-in and CLI precedence. | [configuration contract](../config.md), [adapter internals](adapters.md) |
 | `internal/defaults` | Holds Scriptorium-owned application and HTTP defaults. | [configuration contract](../config.md) |
 | `internal/format` | Formats Promptkit prepared-run and prompt/profile inspection values for CLI text or JSON output. | [CLI contract](../cli.md), [adapter internals](adapters.md) |
+| `internal/report` | Projects one terminal Promptkit observation into Scriptorium-owned execution metadata. | [adapter internals](adapters.md) |
 
 Framework implementation packages are provided by
 [Promptkit v0.11.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.0/docs/consumers/pkg-promptkit.md)

docs/roadmap/implementation.md

@@ -247,7 +247,7 @@ extra timer queue or admission manager.
 
 ## Stage 6: Implement A Shared Terminal Report Projection
 
-**Status: Pending.**
+**Status: Complete.**
 
 Create `internal/report` with a small Scriptorium-owned DTO and per-invocation
 collector. It depends only on Promptkit's supported root types and the standard

internal/report/report.go

@@ -0,0 +1,105 @@
+package report
+
+import "gitea.maximumdirect.net/eric/promptkit"
+
+// Execution is the terminal observation exposed by Scriptorium's opt-in adapters.
+type Execution struct {
+	RunID                string `json:"run_id,omitempty"`
+	PromptID             string `json:"prompt_id,omitempty"`
+	PromptVersion        string `json:"prompt_version,omitempty"`
+	ProfileID            string `json:"profile_id,omitempty"`
+	BackendID            string `json:"backend_id,omitempty"`
+	Model                string `json:"model,omitempty"`
+	Outcome              string `json:"outcome"`
+	Failure              string `json:"failure"`
+	StatusCode           int    `json:"status_code,omitempty"`
+	RetriesExhausted     bool   `json:"retries_exhausted"`
+	Attempts             uint64 `json:"attempts"`
+	TransientRetries     uint64 `json:"transient_retries"`
+	RepairCalls          uint64 `json:"repair_calls"`
+	DurationMS           int64  `json:"duration_ms"`
+	CapacityWaitMS       int64  `json:"capacity_wait_ms"`
+	GenerationDurationMS int64  `json:"generation_duration_ms"`
+	RetryDelayMS         int64  `json:"retry_delay_ms"`
+	Usage                Usage  `json:"usage"`
+}
+
+type Usage struct {
+	PromptTokens     TokenCount `json:"prompt_tokens"`
+	CompletionTokens TokenCount `json:"completion_tokens"`
+	TotalTokens      TokenCount `json:"total_tokens"`
+	CachedTokens     TokenCount `json:"cached_tokens"`
+	CacheWriteTokens TokenCount `json:"cache_write_tokens"`
+}
+
+type TokenCount struct {
+	Value    int  `json:"value"`
+	Known    bool `json:"known"`
+	Complete bool `json:"complete"`
+}
+
+// Collector retains at most one terminal observation from a synchronous run.
+type Collector struct {
+	report *Execution
+}
+
+func NewCollector() *Collector { return &Collector{} }
+
+func (collector *Collector) Observe(event promptkit.ExecutionEvent) {
+	if event.Kind != promptkit.ExecutionFinished {
+		return
+	}
+	count := func(value promptkit.ObservedTokenCount) TokenCount {
+		return TokenCount{Value: value.Value, Known: value.Known, Complete: value.Complete}
+	}
+	collector.report = &Execution{
+		RunID: event.RunID, PromptID: event.PromptID, PromptVersion: event.PromptVersion,
+		ProfileID: event.ProfileID, BackendID: event.BackendID, Model: event.Model,
+		Outcome: outcome(event.Outcome), Failure: failure(event.Failure),
+		StatusCode: event.StatusCode, RetriesExhausted: event.RetriesExhausted,
+		Attempts: event.Attempts, TransientRetries: event.TransientRetries, RepairCalls: event.RepairCalls,
+		DurationMS: event.Duration.Milliseconds(), CapacityWaitMS: event.CapacityWait.Milliseconds(),
+		GenerationDurationMS: event.GenerationDuration.Milliseconds(), RetryDelayMS: event.RetryDelay.Milliseconds(),
+		Usage: Usage{
+			PromptTokens: count(event.Usage.PromptTokens), CompletionTokens: count(event.Usage.CompletionTokens),
+			TotalTokens: count(event.Usage.TotalTokens), CachedTokens: count(event.Usage.CachedTokens),
+			CacheWriteTokens: count(event.Usage.CacheWriteTokens),
+		},
+	}
+}
+
+func (collector *Collector) Snapshot() *Execution {
+	if collector.report == nil {
+		return nil
+	}
+	value := *collector.report
+	return &value
+}
+
+func outcome(value promptkit.ExecutionOutcome) string {
+	switch value {
+	case promptkit.ExecutionOutcomeSuccess, promptkit.ExecutionOutcomeInvalidOutput, promptkit.ExecutionOutcomeError:
+		return string(value)
+	default:
+		return "other"
+	}
+}
+
+func failure(value promptkit.ExecutionFailureKind) string {
+	switch value {
+	case promptkit.ExecutionFailureNone:
+		return "none"
+	case promptkit.ExecutionFailureCanceled, promptkit.ExecutionFailureDeadline,
+		promptkit.ExecutionFailureTransport, promptkit.ExecutionFailureProviderStatus,
+		promptkit.ExecutionFailureMalformedResponse, promptkit.ExecutionFailureCapacity,
+		promptkit.ExecutionFailureInvalidConfig, promptkit.ExecutionFailureInvalidRequest,
+		promptkit.ExecutionFailurePromptNotFound, promptkit.ExecutionFailureProfileNotFound,
+		promptkit.ExecutionFailurePromptLoad, promptkit.ExecutionFailureProfileLoad,
+		promptkit.ExecutionFailureArtifactLoad, promptkit.ExecutionFailurePromptRender,
+		promptkit.ExecutionFailureValidation, promptkit.ExecutionFailureGeneration,
+		promptkit.ExecutionFailureOther:
+		return string(value)
+	default:
+		return "other"
+	}
+}

internal/report/report_test.go

@@ -0,0 +1,126 @@
+package report
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.maximumdirect.net/eric/promptkit"
+)
+
+func TestCollectorProjectsOnlyTerminalAggregates(t *testing.T) {
+	collector := NewCollector()
+	if collector.Snapshot() != nil {
+		t.Fatal("empty collector produced a report")
+	}
+	collector.Observe(promptkit.ExecutionEvent{Kind: promptkit.ExecutionAttempt, Attempts: 99})
+	if collector.Snapshot() != nil {
+		t.Fatal("nonterminal event produced a report")
+	}
+	collector.Observe(promptkit.ExecutionEvent{
+		Kind: promptkit.ExecutionFinished, RunID: "run-1", PromptID: "prompt-1",
+		PromptVersion: "1", ProfileID: "profile-1", BackendID: "backend-1", Model: "model-1",
+		Outcome: promptkit.ExecutionOutcomeError, Failure: promptkit.ExecutionFailureProviderStatus,
+		StatusCode: 503, RetriesExhausted: true, Attempts: 3, TransientRetries: 2, RepairCalls: 1,
+		Duration: 1500 * time.Millisecond, CapacityWait: 20 * time.Millisecond,
+		GenerationDuration: 300 * time.Millisecond, RetryDelay: 700 * time.Millisecond,
+		Usage: promptkit.ObservedTokenUsage{
+			PromptTokens:     promptkit.ObservedTokenCount{Value: 0, Known: true, Complete: true},
+			CompletionTokens: promptkit.ObservedTokenCount{Value: 0, Known: false, Complete: false},
+			TotalTokens:      promptkit.ObservedTokenCount{Value: 11, Known: true, Complete: false},
+		},
+	})
+	collector.Observe(promptkit.ExecutionEvent{Kind: promptkit.ExecutionRetryDelay, Attempts: 50})
+	report := collector.Snapshot()
+	if report == nil || report.Attempts != 3 || report.TransientRetries != 2 || report.RepairCalls != 1 || report.StatusCode != 503 || !report.RetriesExhausted {
+		t.Fatalf("terminal aggregates changed: %+v", report)
+	}
+	if report.DurationMS != 1500 || report.CapacityWaitMS != 20 || report.GenerationDurationMS != 300 || report.RetryDelayMS != 700 {
+		t.Fatalf("terminal timings changed: %+v", report)
+	}
+	if report.Usage.PromptTokens != (TokenCount{Known: true, Complete: true}) || report.Usage.CompletionTokens != (TokenCount{}) || report.Usage.TotalTokens != (TokenCount{Value: 11, Known: true}) {
+		t.Fatalf("presence-aware usage changed: %+v", report.Usage)
+	}
+	encoded, err := json.Marshal(report)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var fields map[string]any
+	if err := json.Unmarshal(encoded, &fields); err != nil {
+		t.Fatal(err)
+	}
+	for _, key := range []string{"run_id", "prompt_id", "prompt_version", "profile_id", "backend_id", "model", "outcome", "failure", "status_code", "retries_exhausted", "attempts", "transient_retries", "repair_calls", "duration_ms", "capacity_wait_ms", "generation_duration_ms", "retry_delay_ms", "usage"} {
+		if _, ok := fields[key]; !ok {
+			t.Fatalf("report missing %q: %s", key, encoded)
+		}
+	}
+	for _, key := range []string{"prompt_tokens", "completion_tokens", "total_tokens", "cached_tokens", "cache_write_tokens"} {
+		item := fields["usage"].(map[string]any)[key].(map[string]any)
+		if len(item) != 3 {
+			t.Fatalf("incomplete token count %q: %#v", key, item)
+		}
+	}
+	for _, forbidden := range []string{"session_id", "headers", "endpoint", "api_key", "error", "output", "input"} {
+		if _, present := fields[forbidden]; present {
+			t.Fatalf("report exposed %q: %s", forbidden, encoded)
+		}
+	}
+}
+
+func TestCollectorHandlesUnknownValuesAndKeepsInvocationsSeparate(t *testing.T) {
+	first := NewCollector()
+	second := NewCollector()
+	first.Observe(promptkit.ExecutionEvent{Kind: promptkit.ExecutionFinished, RunID: "first", Outcome: "future", Failure: "future"})
+	second.Observe(promptkit.ExecutionEvent{Kind: promptkit.ExecutionFinished, RunID: "second", Outcome: promptkit.ExecutionOutcomeSuccess})
+	one := first.Snapshot()
+	two := second.Snapshot()
+	if one.RunID != "first" || one.Outcome != "other" || one.Failure != "other" || two.RunID != "second" || two.Failure != "none" {
+		t.Fatalf("collector isolation or classification failed: first=%+v second=%+v", one, two)
+	}
+	one.RunID = "changed"
+	if first.Snapshot().RunID != "first" {
+		t.Fatal("snapshot mutation changed stored report")
+	}
+	encoded, err := json.Marshal(two)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, absent := range []string{"prompt_id", "prompt_version", "profile_id", "backend_id", "model", "status_code"} {
+		if strings.Contains(string(encoded), `"`+absent+`"`) {
+			t.Fatalf("unavailable optional field %q was included: %s", absent, encoded)
+		}
+	}
+}
+
+type localClient struct{}
+
+func (localClient) Generate(ctx context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
+	return &promptkit.GenerateResponse{Content: "ok"}, nil
+}
+
+func TestCollectorReceivesRealEngineTerminalEvent(t *testing.T) {
+	promptDir := t.TempDir()
+	profileDir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte("id: p\nversion: \"1\"\ndefault_profile: local\nmessages:\n  - role: user\n    content: hi\noutput:\n  format: text\n  validation_mode: none\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte("id: local\nendpoint: http://127.0.0.1:1/v1\nmodel: local-model\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	engine, err := promptkit.NewEngine(promptkit.Config{PromptDir: promptDir, ProfileDir: profileDir}, promptkit.WithLLMClient(localClient{}))
+	if err != nil {
+		t.Fatal(err)
+	}
+	collector := NewCollector()
+	if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "p", Observer: collector.Observe}); err != nil {
+		t.Fatal(err)
+	}
+	value := collector.Snapshot()
+	if value == nil || value.PromptID != "p" || value.ProfileID != "local" || value.Outcome != "success" || value.Attempts != 1 {
+		t.Fatalf("unexpected terminal event projection: %+v", value)
+	}
+}