diff --git a/docs/internal/adapters.md b/docs/internal/adapters.md index bfd4bfe..8d818a4 100644 --- a/docs/internal/adapters.md +++ b/docs/internal/adapters.md @@ -23,6 +23,8 @@ described by its tagged and errors to HTTP DTOs. - `internal/format` renders `promptkit.PreparedRun` values as deterministic text or JSON. +- `internal/report` maps Promptkit's terminal execution event to a small + application-owned value without invoking the engine or writing output. ## Wiring Flows @@ -71,6 +73,11 @@ Failure mapping uses `errors.Is` against Promptkit's public sentinels and the HTTP reader's Scriptorium-owned containment and size errors. Wrapped reader errors preserve their identity through Promptkit's artifact-load boundary. +The report collector is per invocation. It ignores nonterminal events and +copies terminal aggregates once, including usage presence and completeness. +It retains no history or shared state; adapters decide whether to attach it and +how to present its snapshot. + ## Package-Local Guarantees - Adapters contain no copied framework types or orchestration. diff --git a/docs/internal/overview.md b/docs/internal/overview.md index 4248ecd..238c584 100644 --- a/docs/internal/overview.md +++ b/docs/internal/overview.md @@ -12,6 +12,7 @@ public behavior belongs in the linked contracts. | `internal/config` | Discovers and strictly decodes application configuration and applies built-in and CLI precedence. | [configuration contract](../config.md), [adapter internals](adapters.md) | | `internal/defaults` | Holds Scriptorium-owned application and HTTP defaults. | [configuration contract](../config.md) | | `internal/format` | Formats Promptkit prepared-run and prompt/profile inspection values for CLI text or JSON output. | [CLI contract](../cli.md), [adapter internals](adapters.md) | +| `internal/report` | Projects one terminal Promptkit observation into Scriptorium-owned execution metadata. | [adapter internals](adapters.md) | Framework implementation packages are provided by [Promptkit v0.11.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.0/docs/consumers/pkg-promptkit.md) diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 215ded6..f09b51d 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -247,7 +247,7 @@ extra timer queue or admission manager. ## Stage 6: Implement A Shared Terminal Report Projection -**Status: Pending.** +**Status: Complete.** Create `internal/report` with a small Scriptorium-owned DTO and per-invocation collector. It depends only on Promptkit's supported root types and the standard diff --git a/internal/report/report.go b/internal/report/report.go new file mode 100644 index 0000000..21064af --- /dev/null +++ b/internal/report/report.go @@ -0,0 +1,105 @@ +package report + +import "gitea.maximumdirect.net/eric/promptkit" + +// Execution is the terminal observation exposed by Scriptorium's opt-in adapters. +type Execution struct { + RunID string `json:"run_id,omitempty"` + PromptID string `json:"prompt_id,omitempty"` + PromptVersion string `json:"prompt_version,omitempty"` + ProfileID string `json:"profile_id,omitempty"` + BackendID string `json:"backend_id,omitempty"` + Model string `json:"model,omitempty"` + Outcome string `json:"outcome"` + Failure string `json:"failure"` + StatusCode int `json:"status_code,omitempty"` + RetriesExhausted bool `json:"retries_exhausted"` + Attempts uint64 `json:"attempts"` + TransientRetries uint64 `json:"transient_retries"` + RepairCalls uint64 `json:"repair_calls"` + DurationMS int64 `json:"duration_ms"` + CapacityWaitMS int64 `json:"capacity_wait_ms"` + GenerationDurationMS int64 `json:"generation_duration_ms"` + RetryDelayMS int64 `json:"retry_delay_ms"` + Usage Usage `json:"usage"` +} + +type Usage struct { + PromptTokens TokenCount `json:"prompt_tokens"` + CompletionTokens TokenCount `json:"completion_tokens"` + TotalTokens TokenCount `json:"total_tokens"` + CachedTokens TokenCount `json:"cached_tokens"` + CacheWriteTokens TokenCount `json:"cache_write_tokens"` +} + +type TokenCount struct { + Value int `json:"value"` + Known bool `json:"known"` + Complete bool `json:"complete"` +} + +// Collector retains at most one terminal observation from a synchronous run. +type Collector struct { + report *Execution +} + +func NewCollector() *Collector { return &Collector{} } + +func (collector *Collector) Observe(event promptkit.ExecutionEvent) { + if event.Kind != promptkit.ExecutionFinished { + return + } + count := func(value promptkit.ObservedTokenCount) TokenCount { + return TokenCount{Value: value.Value, Known: value.Known, Complete: value.Complete} + } + collector.report = &Execution{ + RunID: event.RunID, PromptID: event.PromptID, PromptVersion: event.PromptVersion, + ProfileID: event.ProfileID, BackendID: event.BackendID, Model: event.Model, + Outcome: outcome(event.Outcome), Failure: failure(event.Failure), + StatusCode: event.StatusCode, RetriesExhausted: event.RetriesExhausted, + Attempts: event.Attempts, TransientRetries: event.TransientRetries, RepairCalls: event.RepairCalls, + DurationMS: event.Duration.Milliseconds(), CapacityWaitMS: event.CapacityWait.Milliseconds(), + GenerationDurationMS: event.GenerationDuration.Milliseconds(), RetryDelayMS: event.RetryDelay.Milliseconds(), + Usage: Usage{ + PromptTokens: count(event.Usage.PromptTokens), CompletionTokens: count(event.Usage.CompletionTokens), + TotalTokens: count(event.Usage.TotalTokens), CachedTokens: count(event.Usage.CachedTokens), + CacheWriteTokens: count(event.Usage.CacheWriteTokens), + }, + } +} + +func (collector *Collector) Snapshot() *Execution { + if collector.report == nil { + return nil + } + value := *collector.report + return &value +} + +func outcome(value promptkit.ExecutionOutcome) string { + switch value { + case promptkit.ExecutionOutcomeSuccess, promptkit.ExecutionOutcomeInvalidOutput, promptkit.ExecutionOutcomeError: + return string(value) + default: + return "other" + } +} + +func failure(value promptkit.ExecutionFailureKind) string { + switch value { + case promptkit.ExecutionFailureNone: + return "none" + case promptkit.ExecutionFailureCanceled, promptkit.ExecutionFailureDeadline, + promptkit.ExecutionFailureTransport, promptkit.ExecutionFailureProviderStatus, + promptkit.ExecutionFailureMalformedResponse, promptkit.ExecutionFailureCapacity, + promptkit.ExecutionFailureInvalidConfig, promptkit.ExecutionFailureInvalidRequest, + promptkit.ExecutionFailurePromptNotFound, promptkit.ExecutionFailureProfileNotFound, + promptkit.ExecutionFailurePromptLoad, promptkit.ExecutionFailureProfileLoad, + promptkit.ExecutionFailureArtifactLoad, promptkit.ExecutionFailurePromptRender, + promptkit.ExecutionFailureValidation, promptkit.ExecutionFailureGeneration, + promptkit.ExecutionFailureOther: + return string(value) + default: + return "other" + } +} diff --git a/internal/report/report_test.go b/internal/report/report_test.go new file mode 100644 index 0000000..c46a555 --- /dev/null +++ b/internal/report/report_test.go @@ -0,0 +1,126 @@ +package report + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "gitea.maximumdirect.net/eric/promptkit" +) + +func TestCollectorProjectsOnlyTerminalAggregates(t *testing.T) { + collector := NewCollector() + if collector.Snapshot() != nil { + t.Fatal("empty collector produced a report") + } + collector.Observe(promptkit.ExecutionEvent{Kind: promptkit.ExecutionAttempt, Attempts: 99}) + if collector.Snapshot() != nil { + t.Fatal("nonterminal event produced a report") + } + collector.Observe(promptkit.ExecutionEvent{ + Kind: promptkit.ExecutionFinished, RunID: "run-1", PromptID: "prompt-1", + PromptVersion: "1", ProfileID: "profile-1", BackendID: "backend-1", Model: "model-1", + Outcome: promptkit.ExecutionOutcomeError, Failure: promptkit.ExecutionFailureProviderStatus, + StatusCode: 503, RetriesExhausted: true, Attempts: 3, TransientRetries: 2, RepairCalls: 1, + Duration: 1500 * time.Millisecond, CapacityWait: 20 * time.Millisecond, + GenerationDuration: 300 * time.Millisecond, RetryDelay: 700 * time.Millisecond, + Usage: promptkit.ObservedTokenUsage{ + PromptTokens: promptkit.ObservedTokenCount{Value: 0, Known: true, Complete: true}, + CompletionTokens: promptkit.ObservedTokenCount{Value: 0, Known: false, Complete: false}, + TotalTokens: promptkit.ObservedTokenCount{Value: 11, Known: true, Complete: false}, + }, + }) + collector.Observe(promptkit.ExecutionEvent{Kind: promptkit.ExecutionRetryDelay, Attempts: 50}) + report := collector.Snapshot() + if report == nil || report.Attempts != 3 || report.TransientRetries != 2 || report.RepairCalls != 1 || report.StatusCode != 503 || !report.RetriesExhausted { + t.Fatalf("terminal aggregates changed: %+v", report) + } + if report.DurationMS != 1500 || report.CapacityWaitMS != 20 || report.GenerationDurationMS != 300 || report.RetryDelayMS != 700 { + t.Fatalf("terminal timings changed: %+v", report) + } + if report.Usage.PromptTokens != (TokenCount{Known: true, Complete: true}) || report.Usage.CompletionTokens != (TokenCount{}) || report.Usage.TotalTokens != (TokenCount{Value: 11, Known: true}) { + t.Fatalf("presence-aware usage changed: %+v", report.Usage) + } + encoded, err := json.Marshal(report) + if err != nil { + t.Fatal(err) + } + var fields map[string]any + if err := json.Unmarshal(encoded, &fields); err != nil { + t.Fatal(err) + } + for _, key := range []string{"run_id", "prompt_id", "prompt_version", "profile_id", "backend_id", "model", "outcome", "failure", "status_code", "retries_exhausted", "attempts", "transient_retries", "repair_calls", "duration_ms", "capacity_wait_ms", "generation_duration_ms", "retry_delay_ms", "usage"} { + if _, ok := fields[key]; !ok { + t.Fatalf("report missing %q: %s", key, encoded) + } + } + for _, key := range []string{"prompt_tokens", "completion_tokens", "total_tokens", "cached_tokens", "cache_write_tokens"} { + item := fields["usage"].(map[string]any)[key].(map[string]any) + if len(item) != 3 { + t.Fatalf("incomplete token count %q: %#v", key, item) + } + } + for _, forbidden := range []string{"session_id", "headers", "endpoint", "api_key", "error", "output", "input"} { + if _, present := fields[forbidden]; present { + t.Fatalf("report exposed %q: %s", forbidden, encoded) + } + } +} + +func TestCollectorHandlesUnknownValuesAndKeepsInvocationsSeparate(t *testing.T) { + first := NewCollector() + second := NewCollector() + first.Observe(promptkit.ExecutionEvent{Kind: promptkit.ExecutionFinished, RunID: "first", Outcome: "future", Failure: "future"}) + second.Observe(promptkit.ExecutionEvent{Kind: promptkit.ExecutionFinished, RunID: "second", Outcome: promptkit.ExecutionOutcomeSuccess}) + one := first.Snapshot() + two := second.Snapshot() + if one.RunID != "first" || one.Outcome != "other" || one.Failure != "other" || two.RunID != "second" || two.Failure != "none" { + t.Fatalf("collector isolation or classification failed: first=%+v second=%+v", one, two) + } + one.RunID = "changed" + if first.Snapshot().RunID != "first" { + t.Fatal("snapshot mutation changed stored report") + } + encoded, err := json.Marshal(two) + if err != nil { + t.Fatal(err) + } + for _, absent := range []string{"prompt_id", "prompt_version", "profile_id", "backend_id", "model", "status_code"} { + if strings.Contains(string(encoded), `"`+absent+`"`) { + t.Fatalf("unavailable optional field %q was included: %s", absent, encoded) + } + } +} + +type localClient struct{} + +func (localClient) Generate(ctx context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + return &promptkit.GenerateResponse{Content: "ok"}, nil +} + +func TestCollectorReceivesRealEngineTerminalEvent(t *testing.T) { + promptDir := t.TempDir() + profileDir := t.TempDir() + if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte("id: p\nversion: \"1\"\ndefault_profile: local\nmessages:\n - role: user\n content: hi\noutput:\n format: text\n validation_mode: none\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte("id: local\nendpoint: http://127.0.0.1:1/v1\nmodel: local-model\n"), 0o644); err != nil { + t.Fatal(err) + } + engine, err := promptkit.NewEngine(promptkit.Config{PromptDir: promptDir, ProfileDir: profileDir}, promptkit.WithLLMClient(localClient{})) + if err != nil { + t.Fatal(err) + } + collector := NewCollector() + if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "p", Observer: collector.Observe}); err != nil { + t.Fatal(err) + } + value := collector.Snapshot() + if value == nil || value.PromptID != "p" || value.ProfileID != "local" || value.Outcome != "success" || value.Attempts != 1 { + t.Fatalf("unexpected terminal event projection: %+v", value) + } +}