From 870b54a4a044f9f9f8ba229f236e2f5873ce9322 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 31 Jul 2026 16:45:26 +0000 Subject: [PATCH] Harden durable prompt state contracts --- docs/internal/state.md | 6 + internal/state/filesystem.go | 28 ++ internal/state/filesystem_test.go | 396 ++++++++++++++++++++++++ internal/state/prompt_artifacts.go | 100 +++++- internal/state/prompt_artifacts_test.go | 207 +++++++++++++ 5 files changed, 732 insertions(+), 5 deletions(-) create mode 100644 internal/state/filesystem_test.go create mode 100644 internal/state/prompt_artifacts_test.go diff --git a/docs/internal/state.md b/docs/internal/state.md index 3e85d05..5abcbaa 100644 --- a/docs/internal/state.md +++ b/docs/internal/state.md @@ -31,6 +31,12 @@ readable; their historic preflight and generated-text-result fields are mapped to the corresponding preparation and execution views during inspection. New runs never write V1 records. +Prompt preparation and execution records are validated on both save and load. +They require exact report/prompt identity, complete timing, internally +consistent provenance, and status-appropriate validation or bounded classified +errors. Completed execution provenance keeps Promptkit's run identity distinct +from the Weatherreporter run identity. + `PromptDebugWriter` is separate from workspace state. An empty root disables it. An enabled absolute root is checked for safe directories and symlinks, then stores `preparation.json` and `execution.json` beneath diff --git a/internal/state/filesystem.go b/internal/state/filesystem.go index ab8bed7..c47dadb 100644 --- a/internal/state/filesystem.go +++ b/internal/state/filesystem.go @@ -247,9 +247,15 @@ func (s *FilesystemStore) PrepareRenderedReport(_ context.Context, resolved repo } func (s *FilesystemStore) SaveMetadata(_ context.Context, metadata Metadata) (string, error) { + if metadata.SchemaVersion != MetadataSchemaVersion { + return "", fmt.Errorf("new metadata must use schema version %q", MetadataSchemaVersion) + } if err := metadata.Validate(); err != nil { return "", err } + if err := s.validateManagedPath("metadata path", metadata.MetadataPath); err != nil { + return "", err + } if err := fileutil.WriteJSONAtomic(metadata.MetadataPath, metadata); err != nil { return "", err } @@ -483,6 +489,28 @@ func (s *FilesystemStore) join(parts ...string) string { return filepath.Join(all...) } +func (s *FilesystemStore) validateManagedPath(name, path string) error { + if s == nil { + return fmt.Errorf("state store is required") + } + root, err := filepath.Abs(s.root) + if err != nil { + return fmt.Errorf("resolve workspace root: %w", err) + } + target, err := filepath.Abs(path) + if err != nil { + return fmt.Errorf("resolve %s: %w", name, err) + } + relative, err := filepath.Rel(root, target) + if err != nil { + return fmt.Errorf("resolve %s relative to workspace root: %w", name, err) + } + if relative == "." || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return fmt.Errorf("%s must stay within workspace root", name) + } + return nil +} + func validateRelativeDir(name string, value string) error { if value == "" { return fmt.Errorf("%s is required", name) diff --git a/internal/state/filesystem_test.go b/internal/state/filesystem_test.go new file mode 100644 index 0000000..f147687 --- /dev/null +++ b/internal/state/filesystem_test.go @@ -0,0 +1,396 @@ +package state + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "gitea.maximumdirect.net/eric/weatherreporter/internal/briefing" + "gitea.maximumdirect.net/eric/weatherreporter/internal/config" + "gitea.maximumdirect.net/eric/weatherreporter/internal/module" + "gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput" + "gitea.maximumdirect.net/eric/weatherreporter/internal/report" + "gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil" + "gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata" +) + +func TestFilesystemPathsUseExactPromptArtifactLayout(t *testing.T) { + store := newFilesystemTestStore(t) + tests := []struct { + id report.ID + now string + date string + group string + validDate string + runID string + }{ + {report.Daily, "2026-05-29T05:00:00-05:00", "2026-05-29T12:00:00-05:00", "daily", "2026-05-29", "20260529T100000.000000000Z_daily_2026-05-29"}, + {report.Today, "2026-05-29T05:00:00-05:00", "", "today", "2026-05-29", "20260529T100000.000000000Z_today"}, + {report.Tomorrow, "2026-05-29T18:00:00-05:00", "", "tomorrow", "2026-05-30", "20260529T230000.000000000Z_tomorrow"}, + {report.Hourly, "2026-05-29T05:00:00-05:00", "", "hourly", "2026-05-29", "20260529T100000.000000000Z_hourly"}, + } + for _, test := range tests { + t.Run(string(test.id), func(t *testing.T) { + resolved := resolveStateReport(t, test.id, test.now, test.date) + paths, err := store.Paths(resolved) + if err != nil { + t.Fatalf("Paths() error = %v", err) + } + want := ArtifactPaths{ + ModuleSnapshot: filepath.Join(store.root, "snapshots", test.group, test.validDate, "modules."+test.runID+".json"), + Metadata: filepath.Join(store.root, "snapshots", test.group, test.validDate, "metadata."+test.runID+".json"), + DataPackage: filepath.Join(store.root, "data-packages", test.group, test.validDate, "data_package."+test.runID+".yaml"), + Preparation: filepath.Join(store.root, "preflight", test.group, test.validDate, "prompt_preparation."+test.runID+".json"), + Execution: filepath.Join(store.root, "snapshots", test.group, test.validDate, "prompt_execution."+test.runID+".json"), + Notification: filepath.Join(store.root, "notifications", test.group, test.validDate, "distributor."+test.runID+".json"), + RenderedReport: filepath.Join(store.root, "reports", test.group, test.validDate, "report."+test.runID+".md"), + GeneratedTextRaw: filepath.Join(store.root, "snapshots", test.group, test.validDate, "generated_text_raw."+test.runID+".json"), + GeneratedText: filepath.Join(store.root, "snapshots", test.group, test.validDate, "generated_text."+test.runID+".json"), + RenderContext: filepath.Join(store.root, "snapshots", test.group, test.validDate, "render_context."+test.runID+".json"), + } + if paths != want { + t.Fatalf("Paths() = %#v, want %#v", paths, want) + } + }) + } +} + +func TestPromptArtifactsAndMetadataRoundTrip(t *testing.T) { + store := newFilesystemTestStore(t) + resolved := resolveStateReport(t, report.Daily, "2026-05-29T05:00:00-05:00", "2026-05-29T12:00:00-05:00") + paths, err := store.Paths(resolved) + if err != nil { + t.Fatalf("Paths() error = %v", err) + } + preparation := preparationArtifactFor(resolved, paths) + preparationPath, err := store.SavePromptPreparation(context.Background(), resolved, preparation) + if err != nil { + t.Fatalf("SavePromptPreparation() error = %v", err) + } + execution := executionArtifactFor(resolved, paths) + executionPath, err := store.SavePromptExecution(context.Background(), resolved, execution) + if err != nil { + t.Fatalf("SavePromptExecution() error = %v", err) + } + loadedPreparation, err := store.LoadPromptPreparation(context.Background(), preparationPath) + if err != nil || loadedPreparation.Preparation == nil || loadedPreparation.Preparation.PromptHash != "prompt-hash" { + t.Fatalf("LoadPromptPreparation() = %#v, %v", loadedPreparation, err) + } + loadedExecution, err := store.LoadPromptExecution(context.Background(), executionPath) + if err != nil || loadedExecution.Provenance == nil || loadedExecution.Provenance.RunID != "provider-run" || loadedExecution.Validation == nil { + t.Fatalf("LoadPromptExecution() = %#v, %v", loadedExecution, err) + } + + metadata := promptMetadataFor(resolved, paths) + metadata.PreparationPath = preparationPath + metadata.ExecutionPath = executionPath + metadata.GeneratedTextRawPath = paths.GeneratedTextRaw + metadataPath, err := store.SaveMetadata(context.Background(), metadata) + if err != nil { + t.Fatalf("SaveMetadata() error = %v", err) + } + data, err := os.ReadFile(metadataPath) + if err != nil { + t.Fatalf("read metadata: %v", err) + } + text := string(data) + if strings.Contains(text, "preflightPath") || strings.Contains(text, "generatedTextResultPath") || strings.Contains(text, "metadataPath") { + t.Fatalf("v2 metadata contains legacy or runtime aliases: %s", text) + } + loadedMetadata, loadedPath, err := store.LoadMetadataByRunID(context.Background(), metadata.RunID) + if err != nil || loadedPath != metadataPath || loadedMetadata.PreparationPath != preparationPath || loadedMetadata.ExecutionPath != executionPath { + t.Fatalf("LoadMetadataByRunID() = %#v, %q, %v", loadedMetadata, loadedPath, err) + } +} + +func TestMetadataLegacyCompatibilityAndV2OnlyWrites(t *testing.T) { + legacy := Metadata{ + SchemaVersion: MetadataSchemaVersionV1, RunID: "legacy-run", ReportID: report.ID("three_day"), + PromptID: "weather.three_day", ModuleSnapshotPath: "/archive/modules.json", DataPackagePath: "/archive/data.yaml", + PreflightPath: "/archive/render.json", GeneratedTextResultPath: "/archive/result.json", + } + data, err := json.Marshal(legacy) + if err != nil { + t.Fatalf("Marshal() error = %v", err) + } + text := string(data) + if !strings.Contains(text, "preflightPath") || !strings.Contains(text, "generatedTextResultPath") || strings.Contains(text, "preparationPath") || strings.Contains(text, "executionPath") { + t.Fatalf("v1 metadata wire fields = %s", text) + } + var decoded Metadata + if err := json.Unmarshal(data, &decoded); err != nil { + t.Fatalf("Unmarshal() error = %v", err) + } + if decoded.PreparationPath != legacy.PreflightPath || decoded.ExecutionPath != legacy.GeneratedTextResultPath { + t.Fatalf("normalized compatibility aliases = %#v", decoded) + } + remarshaled, err := json.Marshal(decoded) + if err != nil || !strings.Contains(string(remarshaled), "preflightPath") || strings.Contains(string(remarshaled), "preparationPath") { + t.Fatalf("remarshaled v1 metadata = %s, %v", remarshaled, err) + } + if err := json.Unmarshal([]byte(`{"schemaVersion":"weatherreporter.metadata.v99"}`), &decoded); err == nil { + t.Fatal("Unmarshal() error = nil, want unknown schema rejection") + } + + store := newFilesystemTestStore(t) + legacy.MetadataPath = filepath.Join(store.root, "snapshots", "legacy", "metadata.legacy-run.json") + if _, err := store.SaveMetadata(context.Background(), legacy); err == nil { + t.Fatal("SaveMetadata(v1) error = nil, want v2-only write rejection") + } +} + +func TestReportDiscoveryAndArtifactInspection(t *testing.T) { + store := newFilesystemTestStore(t) + older := resolveStateReport(t, report.Daily, "2026-05-29T05:00:00-05:00", "2026-05-29T12:00:00-05:00") + newer := resolveStateReport(t, report.Today, "2026-05-29T08:00:00-05:00", "") + olderPaths := saveStateMetadata(t, store, older) + newerPaths := saveStateMetadata(t, store, newer) + + snapshot, err := module.NewSnapshot([]module.Output{{ID: module.Metadata, StanzaName: "metadata", Value: map[string]any{"run_id": older.Metadata().RunID}}}) + if err != nil { + t.Fatalf("NewSnapshot() error = %v", err) + } + modulePath, err := store.SaveModuleSnapshot(context.Background(), older, snapshot) + if err != nil { + t.Fatalf("SaveModuleSnapshot() error = %v", err) + } + pkg, err := promptinput.Build(promptinput.BuildRequest{ + Metadata: promptinput.Metadata{ + RunID: older.Metadata().RunID, ReportID: older.Definition.ID, PromptID: older.Definition.PromptID, + GeneratedAt: older.GeneratedAt, Timezone: older.Timezone, ValidPeriod: older.ValidPeriod, + }, + Modules: snapshot, + }) + if err != nil { + t.Fatalf("Build() error = %v", err) + } + dataPath, err := store.SaveDataPackage(context.Background(), older, pkg) + if err != nil { + t.Fatalf("SaveDataPackage() error = %v", err) + } + loadedSnapshot, err := store.LoadModuleSnapshot(context.Background(), modulePath) + if err != nil || len(loadedSnapshot.Outputs) != 1 { + t.Fatalf("LoadModuleSnapshot() = %#v, %v", loadedSnapshot, err) + } + loadedPackage, err := store.LoadDataPackage(context.Background(), dataPath) + if err != nil || loadedPackage.RunID != older.Metadata().RunID { + t.Fatalf("LoadDataPackage() = %#v, %v", loadedPackage, err) + } + + records, err := store.ListReports(context.Background(), 0) + if err != nil || len(records) != 2 { + t.Fatalf("ListReports() = %#v, %v", records, err) + } + if records[0].RunID != newer.Metadata().RunID || records[0].MetadataPath != newerPaths.Metadata || records[1].MetadataPath != olderPaths.Metadata { + t.Fatalf("ordered report records = %#v", records) + } + loadedMetadata, loadedPath, err := store.LoadMetadataByRunID(context.Background(), older.Metadata().RunID) + if err != nil || loadedPath != olderPaths.Metadata || len(loadedMetadata.Sources) != 1 || loadedMetadata.Sources[0].Name != "weather-api" { + t.Fatalf("LoadMetadataByRunID() = %#v, %q, %v", loadedMetadata, loadedPath, err) + } +} + +func TestListReportsRetainsHistoricalV1ReportIDs(t *testing.T) { + store := newFilesystemTestStore(t) + for i, id := range []report.ID{"three_day", "weekend", "storm"} { + generatedAt := time.Date(2026, 5, 20+i, 12, 0, 0, 0, time.UTC) + metadata := Metadata{ + SchemaVersion: MetadataSchemaVersionV1, RunID: "historical-" + string(id), ReportID: id, + PromptID: "weather." + string(id), GeneratedAt: generatedAt, Timezone: "UTC", + ValidPeriod: timeutil.Period{Start: generatedAt, End: generatedAt.Add(24 * time.Hour)}, + ModuleSnapshotPath: "/archive/modules.json", DataPackagePath: "/archive/data.yaml", + PreflightPath: "/archive/render.json", GeneratedTextResultPath: "/archive/result.json", + } + path := filepath.Join(store.root, "snapshots", string(id), "2026-05-20", "metadata."+metadata.RunID+".json") + writeJSONFixture(t, path, metadata) + } + records, err := store.ListReports(context.Background(), 0) + if err != nil || len(records) != 3 { + t.Fatalf("ListReports() = %#v, %v", records, err) + } + for _, record := range records { + if record.ReportID != "three_day" && record.ReportID != "weekend" && record.ReportID != "storm" { + t.Fatalf("unexpected historical report record: %#v", record) + } + metadata, path, err := store.LoadMetadataByRunID(context.Background(), record.RunID) + if err != nil || path != record.MetadataPath || metadata.PreparationPath != "/archive/render.json" || metadata.ExecutionPath != "/archive/result.json" { + t.Fatalf("LoadMetadataByRunID(%q) = %#v, %q, %v", record.RunID, metadata, path, err) + } + } +} + +func TestFilesystemWritesAreAtomicAndRejectUnsafePaths(t *testing.T) { + store := newFilesystemTestStore(t) + resolved := resolveStateReport(t, report.Hourly, "2026-05-29T05:00:00-05:00", "") + path, err := store.SaveGeneratedTextRaw(context.Background(), resolved, []byte(`{"value":"first"}`)) + if err != nil { + t.Fatalf("first SaveGeneratedTextRaw() error = %v", err) + } + if _, err := store.SaveGeneratedTextRaw(context.Background(), resolved, []byte(`{"value":"second"}`)); err != nil { + t.Fatalf("second SaveGeneratedTextRaw() error = %v", err) + } + data, err := os.ReadFile(path) + if err != nil || string(data) != `{"value":"second"}` { + t.Fatalf("atomic replacement = %q, %v", data, err) + } + entries, err := os.ReadDir(filepath.Dir(path)) + if err != nil || len(entries) != 1 || entries[0].Name() != filepath.Base(path) { + t.Fatalf("artifact directory after atomic write = %#v, %v", entries, err) + } + + metadata := promptMetadataFor(resolved, mustStatePaths(t, store, resolved)) + metadata.PreparationPath = "/saved/preparation.json" + metadata.MetadataPath = filepath.Join(t.TempDir(), "outside.json") + if _, err := store.SaveMetadata(context.Background(), metadata); err == nil { + t.Fatal("SaveMetadata(outside workspace) error = nil") + } + + cfg := config.Defaults().Workspace + cfg.Root = t.TempDir() + cfg.SnapshotsDir = "../snapshots" + if _, err := NewFilesystemStore(cfg); err == nil { + t.Fatal("NewFilesystemStore(unsafe directory) error = nil") + } + unsafeResolved := resolved + unsafeResolved.Definition.ID = report.ID("hourly/bad") + if _, err := store.Paths(unsafeResolved); err == nil { + t.Fatal("Paths(unsafe run id) error = nil") + } +} + +func TestFindPriorSnapshotForSupportedReports(t *testing.T) { + tests := []struct { + id report.ID + firstNow string + secondNow string + date string + wantPrior bool + }{ + {report.Daily, "2026-05-29T05:00:00-05:00", "2026-05-29T08:00:00-05:00", "2026-05-29T12:00:00-05:00", true}, + {report.Today, "2026-05-29T05:00:00-05:00", "2026-05-29T08:00:00-05:00", "", true}, + {report.Tomorrow, "2026-05-29T17:00:00-05:00", "2026-05-29T18:00:00-05:00", "", true}, + {report.Hourly, "2026-05-29T05:00:00-05:00", "2026-05-29T06:00:00-05:00", "", false}, + } + for _, test := range tests { + t.Run(string(test.id), func(t *testing.T) { + store := newFilesystemTestStore(t) + first := resolveStateReport(t, test.id, test.firstNow, test.date) + second := resolveStateReport(t, test.id, test.secondNow, test.date) + paths := saveStateMetadata(t, store, first) + prior, err := store.FindPriorSnapshot(context.Background(), second) + if err != nil { + t.Fatalf("FindPriorSnapshot() error = %v", err) + } + if !test.wantPrior && prior != nil { + t.Fatalf("FindPriorSnapshot() = %#v, want nil", prior) + } + if test.wantPrior && (prior == nil || prior.Metadata.RunID != first.Metadata().RunID || prior.ModuleSnapshotPath != paths.ModuleSnapshot) { + t.Fatalf("FindPriorSnapshot() = %#v, want run %q", prior, first.Metadata().RunID) + } + }) + } +} + +func newFilesystemTestStore(t *testing.T) *FilesystemStore { + t.Helper() + cfg := config.Defaults().Workspace + cfg.Root = t.TempDir() + store, err := NewFilesystemStore(cfg) + if err != nil { + t.Fatalf("NewFilesystemStore() error = %v", err) + } + return store +} + +func resolveStateReport(t *testing.T, id report.ID, nowValue, dateValue string) report.Resolved { + t.Helper() + location, err := time.LoadLocation("America/Chicago") + if err != nil { + t.Fatalf("LoadLocation() error = %v", err) + } + now, err := time.Parse(time.RFC3339, nowValue) + if err != nil { + t.Fatalf("parse now: %v", err) + } + req := report.ResolveRequest{Now: now, Location: location} + if dateValue != "" { + req.Date, err = time.Parse(time.RFC3339, dateValue) + if err != nil { + t.Fatalf("parse date: %v", err) + } + } + resolved, err := report.DefaultRegistry().Resolve(id, req) + if err != nil { + t.Fatalf("Resolve() error = %v", err) + } + return resolved +} + +func mustStatePaths(t *testing.T, store *FilesystemStore, resolved report.Resolved) ArtifactPaths { + t.Helper() + paths, err := store.Paths(resolved) + if err != nil { + t.Fatalf("Paths() error = %v", err) + } + return paths +} + +func promptMetadataFor(resolved report.Resolved, paths ArtifactPaths) Metadata { + metadata := BuildPromptMetadataFromBriefingMetadata(resolved, briefing.Metadata{ + Sources: []briefing.SourceMetadata{{Name: "weather-api", FetchedAt: resolved.GeneratedAt}}, + SourceWarnings: []weatherdata.SourceWarning{}, + }, ArtifactPaths{ModuleSnapshot: paths.ModuleSnapshot, Metadata: paths.Metadata, DataPackage: paths.DataPackage}) + return metadata +} + +func saveStateMetadata(t *testing.T, store *FilesystemStore, resolved report.Resolved) ArtifactPaths { + t.Helper() + paths := mustStatePaths(t, store, resolved) + metadata := promptMetadataFor(resolved, paths) + metadata.PreparationPath = paths.Preparation + if _, err := store.SaveMetadata(context.Background(), metadata); err != nil { + t.Fatalf("SaveMetadata() error = %v", err) + } + return paths +} + +func preparationArtifactFor(resolved report.Resolved, paths ArtifactPaths) PromptPreparationArtifact { + artifact := validPreparationArtifact() + metadata := resolved.Metadata() + artifact.ReportID, artifact.RunID = metadata.ReportID, metadata.RunID + artifact.PromptID, artifact.PromptVersion = resolved.Definition.PromptID, resolved.Definition.PromptVersion + artifact.DataPackagePath = paths.DataPackage + artifact.Preparation.PromptID, artifact.Preparation.PromptVersion = artifact.PromptID, artifact.PromptVersion + artifact.Preparation.PromptHash = "prompt-hash" + artifact.Preparation.DataPackagePath = paths.DataPackage + return artifact +} + +func executionArtifactFor(resolved report.Resolved, paths ArtifactPaths) PromptExecutionArtifact { + artifact := validExecutionArtifact() + metadata := resolved.Metadata() + artifact.ReportID, artifact.RunID = metadata.ReportID, metadata.RunID + artifact.PromptID, artifact.PromptVersion = resolved.Definition.PromptID, resolved.Definition.PromptVersion + artifact.Provenance.PromptID, artifact.Provenance.PromptVersion = artifact.PromptID, artifact.PromptVersion + artifact.Provenance.DataPackagePath = paths.DataPackage + artifact.Paths.RawOutputPath = paths.GeneratedTextRaw + return artifact +} + +func writeJSONFixture(t *testing.T, path string, value any) { + t.Helper() + data, err := json.Marshal(value) + if err != nil { + t.Fatalf("marshal fixture: %v", err) + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("create fixture directory: %v", err) + } + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatalf("write fixture: %v", err) + } +} diff --git a/internal/state/prompt_artifacts.go b/internal/state/prompt_artifacts.go index b54b19a..7bf8928 100644 --- a/internal/state/prompt_artifacts.go +++ b/internal/state/prompt_artifacts.go @@ -67,21 +67,30 @@ func (a PromptPreparationArtifact) Validate() error { if a.SchemaVersion != PromptPreparationSchemaVersion { return fmt.Errorf("unsupported prompt preparation schema version %q", a.SchemaVersion) } - if strings.TrimSpace(a.RunID) == "" || a.ReportID == "" || strings.TrimSpace(a.PromptID) == "" { - return fmt.Errorf("prompt preparation identity is required") + if err := validatePromptArtifactIdentity("prompt preparation", a.ReportID, a.RunID, a.PromptID, a.PromptVersion); err != nil { + return err } if strings.TrimSpace(a.DataPackagePath) == "" { return fmt.Errorf("prompt preparation data package path is required") } + if err := validatePromptArtifactTiming("prompt preparation", a.StartedAt, a.EndedAt, a.Duration); err != nil { + return err + } switch a.Status { case PromptPreparationSucceeded: if a.Preparation == nil || a.Error != nil { return fmt.Errorf("successful prompt preparation requires preparation without an error") } + if a.Preparation.PromptID != a.PromptID || a.Preparation.PromptVersion != a.PromptVersion || a.Preparation.DataPackagePath != a.DataPackagePath { + return fmt.Errorf("successful prompt preparation provenance must match the artifact") + } case PromptPreparationFailed: if !validPromptArtifactError(a.Error) { return fmt.Errorf("failed prompt preparation requires a classified error") } + if a.Preparation != nil { + return fmt.Errorf("failed prompt preparation must not include preparation provenance") + } default: return fmt.Errorf("unsupported prompt preparation status %q", a.Status) } @@ -164,22 +173,34 @@ func (a PromptExecutionArtifact) Validate() error { if a.SchemaVersion != PromptExecutionSchemaVersion { return fmt.Errorf("unsupported prompt execution schema version %q", a.SchemaVersion) } - if strings.TrimSpace(a.RunID) == "" || a.ReportID == "" || strings.TrimSpace(a.PromptID) == "" { - return fmt.Errorf("prompt execution identity is required") + if err := validatePromptArtifactIdentity("prompt execution", a.ReportID, a.RunID, a.PromptID, a.PromptVersion); err != nil { + return err + } + if err := validatePromptArtifactTiming("prompt execution", a.StartedAt, a.EndedAt, a.Duration); err != nil { + return err } switch a.Status { case PromptExecutionSucceeded: if a.Provenance == nil || a.Validation == nil || a.Validation.Status != promptexec.ValidationPassed || a.Error != nil { return fmt.Errorf("successful prompt execution requires passed validation without an error") } + if err := validatePromptExecutionProvenance(a); err != nil { + return err + } case PromptExecutionValidationRejected: if a.Provenance == nil || a.Validation == nil || a.Validation.Status != promptexec.ValidationFailed || a.Error != nil { return fmt.Errorf("validation-rejected prompt execution requires failed validation without an error") } + if err := validatePromptExecutionProvenance(a); err != nil { + return err + } case PromptExecutionFailed: if !validPromptArtifactError(a.Error) { return fmt.Errorf("failed prompt execution requires a classified error") } + if a.Provenance != nil || a.Validation != nil { + return fmt.Errorf("failed prompt execution must not include completed provenance or validation") + } default: return fmt.Errorf("unsupported prompt execution status %q", a.Status) } @@ -187,5 +208,74 @@ func (a PromptExecutionArtifact) Validate() error { } func validPromptArtifactError(value *PromptArtifactError) bool { - return value != nil && value.Category != "" && strings.TrimSpace(value.Message) != "" && len(value.Message) <= promptArtifactErrorLimit && utf8.ValidString(value.Message) + return value != nil && validPromptErrorCategory(value.Category) && strings.TrimSpace(value.Message) != "" && len(value.Message) <= promptArtifactErrorLimit && utf8.ValidString(value.Message) +} + +func validatePromptArtifactIdentity(kind string, reportID report.ID, runID, promptID, promptVersion string) error { + if reportID == "" || strings.TrimSpace(runID) == "" || strings.TrimSpace(promptID) == "" { + return fmt.Errorf("%s identity is required", kind) + } + definition, err := report.DefaultRegistry().Lookup(reportID) + if err != nil { + return fmt.Errorf("%s report id is unsupported: %w", kind, err) + } + if promptID != definition.PromptID { + return fmt.Errorf("%s prompt id must match report %q", kind, reportID) + } + if promptVersion != definition.PromptVersion { + return fmt.Errorf("%s prompt version must be %q", kind, definition.PromptVersion) + } + return nil +} + +func validatePromptArtifactTiming(kind string, startedAt, endedAt time.Time, duration time.Duration) error { + if startedAt.IsZero() || endedAt.IsZero() { + return fmt.Errorf("%s start and end times are required", kind) + } + if duration < 0 { + return fmt.Errorf("%s duration must not be negative", kind) + } + if endedAt.Before(startedAt) { + return fmt.Errorf("%s end time must not be earlier than its start time", kind) + } + return nil +} + +func validatePromptExecutionProvenance(artifact PromptExecutionArtifact) error { + value := artifact.Provenance + if value == nil { + return fmt.Errorf("completed prompt execution provenance is required") + } + if value.PromptID != artifact.PromptID || value.PromptVersion != artifact.PromptVersion { + return fmt.Errorf("completed prompt execution provenance must match the artifact") + } + for _, required := range []struct { + name string + value string + }{ + {"run id", value.RunID}, {"prompt hash", value.PromptHash}, {"rendered prompt hash", value.RenderedPromptHash}, + {"profile id", value.ProfileID}, {"backend id", value.BackendID}, {"model name", value.ModelName}, + {"data package path", value.DataPackagePath}, + } { + if strings.TrimSpace(required.value) == "" { + return fmt.Errorf("completed prompt execution provenance %s is required", required.name) + } + } + if err := validatePromptArtifactTiming("completed prompt execution provenance", value.StartedAt, value.EndedAt, value.Duration); err != nil { + return err + } + return nil +} + +func validPromptErrorCategory(category promptexec.ErrorCategory) bool { + switch category { + case promptexec.InvalidConfiguration, promptexec.InvalidRequest, promptexec.PromptNotFound, + promptexec.PromptLoad, promptexec.ProfileNotFound, promptexec.ProfileLoad, + promptexec.MissingCredential, promptexec.ArtifactLoad, promptexec.PromptRender, + promptexec.Capacity, promptexec.Generation, promptexec.OperationalValidation, + promptexec.ValidationRejected, promptexec.Canceled, promptexec.DeadlineExceeded: + return true + default: + return false + } } diff --git a/internal/state/prompt_artifacts_test.go b/internal/state/prompt_artifacts_test.go new file mode 100644 index 0000000..4ed6895 --- /dev/null +++ b/internal/state/prompt_artifacts_test.go @@ -0,0 +1,207 @@ +package state + +import ( + "strings" + "testing" + "time" + + "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" + "gitea.maximumdirect.net/eric/weatherreporter/internal/report" +) + +func TestPromptPreparationArtifactValidation(t *testing.T) { + valid := validPreparationArtifact() + if err := valid.Validate(); err != nil { + t.Fatalf("valid successful preparation: %v", err) + } + failed := valid + failed.Status = PromptPreparationFailed + failed.Preparation = nil + failed.Error = &PromptArtifactError{Category: promptexec.Generation, Message: "provider unavailable"} + if err := failed.Validate(); err != nil { + t.Fatalf("valid failed preparation: %v", err) + } + + tests := []struct { + name string + mutate func(*PromptPreparationArtifact) + }{ + {"report id", func(a *PromptPreparationArtifact) { a.ReportID = "" }}, + {"run id", func(a *PromptPreparationArtifact) { a.RunID = "" }}, + {"prompt id", func(a *PromptPreparationArtifact) { a.PromptID = "" }}, + {"prompt id for another report", func(a *PromptPreparationArtifact) { a.PromptID = "weather.hourly_generated_text" }}, + {"prompt version", func(a *PromptPreparationArtifact) { a.PromptVersion = "latest" }}, + {"data package", func(a *PromptPreparationArtifact) { a.DataPackagePath = "" }}, + {"start time", func(a *PromptPreparationArtifact) { a.StartedAt = time.Time{} }}, + {"end time", func(a *PromptPreparationArtifact) { a.EndedAt = time.Time{} }}, + {"negative duration", func(a *PromptPreparationArtifact) { a.Duration = -time.Second }}, + {"reversed times", func(a *PromptPreparationArtifact) { a.EndedAt = a.StartedAt.Add(-time.Second) }}, + {"missing provenance", func(a *PromptPreparationArtifact) { a.Preparation = nil }}, + {"success error", func(a *PromptPreparationArtifact) { + a.Error = &PromptArtifactError{Category: promptexec.Generation, Message: "failed"} + }}, + {"provenance prompt id", func(a *PromptPreparationArtifact) { a.Preparation.PromptID = "other" }}, + {"provenance prompt version", func(a *PromptPreparationArtifact) { a.Preparation.PromptVersion = "other" }}, + {"provenance data package", func(a *PromptPreparationArtifact) { a.Preparation.DataPackagePath = "/other/data.yaml" }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + artifact := validPreparationArtifact() + test.mutate(&artifact) + if err := artifact.Validate(); err == nil { + t.Fatalf("Validate() error = nil for %#v", artifact) + } + }) + } +} + +func TestFailedPromptPreparationRejectsContradictoryDetails(t *testing.T) { + tests := []struct { + name string + mutate func(*PromptPreparationArtifact) + }{ + {"missing error", func(a *PromptPreparationArtifact) { a.Error = nil }}, + {"unknown category", func(a *PromptPreparationArtifact) { a.Error.Category = promptexec.ErrorCategory("other") }}, + {"empty message", func(a *PromptPreparationArtifact) { a.Error.Message = " " }}, + {"oversized message", func(a *PromptPreparationArtifact) { a.Error.Message = strings.Repeat("x", promptArtifactErrorLimit+1) }}, + {"invented provenance", func(a *PromptPreparationArtifact) { a.Preparation = validPreparationArtifact().Preparation }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + artifact := validPreparationArtifact() + artifact.Status = PromptPreparationFailed + artifact.Preparation = nil + artifact.Error = &PromptArtifactError{Category: promptexec.Generation, Message: "provider unavailable"} + test.mutate(&artifact) + if err := artifact.Validate(); err == nil { + t.Fatalf("Validate() error = nil for %#v", artifact) + } + }) + } +} + +func TestPromptExecutionArtifactValidation(t *testing.T) { + valid := validExecutionArtifact() + valid.Provenance.RunID = "provider-run-different-from-weatherreporter" + valid.Provenance.GeneratedHash = "" + valid.Provenance.Usage = promptexec.TokenUsage{} + if err := valid.Validate(); err != nil { + t.Fatalf("valid completed execution with provider run identity and omitted counters: %v", err) + } + rejected := validExecutionArtifact() + rejected.Status = PromptExecutionValidationRejected + validation := promptexec.NewValidation(promptexec.ValidationFailed, "json_schema", "daily.generated_text.schema.json", []string{"schema mismatch"}) + rejected.Validation = &validation + if err := rejected.Validate(); err != nil { + t.Fatalf("valid validation rejection: %v", err) + } + failed := validFailedExecutionArtifact() + if err := failed.Validate(); err != nil { + t.Fatalf("valid operational failure: %v", err) + } + + tests := []struct { + name string + mutate func(*PromptExecutionArtifact) + }{ + {"report id", func(a *PromptExecutionArtifact) { a.ReportID = "" }}, + {"run id", func(a *PromptExecutionArtifact) { a.RunID = "" }}, + {"prompt id", func(a *PromptExecutionArtifact) { a.PromptID = "" }}, + {"prompt version", func(a *PromptExecutionArtifact) { a.PromptVersion = "latest" }}, + {"start time", func(a *PromptExecutionArtifact) { a.StartedAt = time.Time{} }}, + {"end time", func(a *PromptExecutionArtifact) { a.EndedAt = time.Time{} }}, + {"negative duration", func(a *PromptExecutionArtifact) { a.Duration = -time.Second }}, + {"reversed times", func(a *PromptExecutionArtifact) { a.EndedAt = a.StartedAt.Add(-time.Second) }}, + {"missing provenance", func(a *PromptExecutionArtifact) { a.Provenance = nil }}, + {"missing validation", func(a *PromptExecutionArtifact) { a.Validation = nil }}, + {"wrong validation", func(a *PromptExecutionArtifact) { + value := promptexec.NewValidation(promptexec.ValidationFailed, "json_schema", "schema.json", nil) + a.Validation = &value + }}, + {"operational error", func(a *PromptExecutionArtifact) { + a.Error = &PromptArtifactError{Category: promptexec.Generation, Message: "failed"} + }}, + {"provenance prompt id", func(a *PromptExecutionArtifact) { a.Provenance.PromptID = "other" }}, + {"provenance prompt version", func(a *PromptExecutionArtifact) { a.Provenance.PromptVersion = "other" }}, + {"provenance run id", func(a *PromptExecutionArtifact) { a.Provenance.RunID = "" }}, + {"prompt hash", func(a *PromptExecutionArtifact) { a.Provenance.PromptHash = "" }}, + {"rendered hash", func(a *PromptExecutionArtifact) { a.Provenance.RenderedPromptHash = "" }}, + {"profile id", func(a *PromptExecutionArtifact) { a.Provenance.ProfileID = "" }}, + {"backend id", func(a *PromptExecutionArtifact) { a.Provenance.BackendID = "" }}, + {"model name", func(a *PromptExecutionArtifact) { a.Provenance.ModelName = "" }}, + {"data package", func(a *PromptExecutionArtifact) { a.Provenance.DataPackagePath = "" }}, + {"provenance start time", func(a *PromptExecutionArtifact) { a.Provenance.StartedAt = time.Time{} }}, + {"provenance end time", func(a *PromptExecutionArtifact) { a.Provenance.EndedAt = time.Time{} }}, + {"provenance negative duration", func(a *PromptExecutionArtifact) { a.Provenance.Duration = -time.Second }}, + {"provenance reversed times", func(a *PromptExecutionArtifact) { a.Provenance.EndedAt = a.Provenance.StartedAt.Add(-time.Second) }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + artifact := validExecutionArtifact() + test.mutate(&artifact) + if err := artifact.Validate(); err == nil { + t.Fatalf("Validate() error = nil for %#v", artifact) + } + }) + } +} + +func TestFailedPromptExecutionRejectsContradictoryDetails(t *testing.T) { + tests := []struct { + name string + mutate func(*PromptExecutionArtifact) + }{ + {"missing error", func(a *PromptExecutionArtifact) { a.Error = nil }}, + {"unknown category", func(a *PromptExecutionArtifact) { a.Error.Category = promptexec.ErrorCategory("other") }}, + {"provenance", func(a *PromptExecutionArtifact) { a.Provenance = validExecutionArtifact().Provenance }}, + {"completed validation", func(a *PromptExecutionArtifact) { a.Validation = validExecutionArtifact().Validation }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + artifact := validFailedExecutionArtifact() + test.mutate(&artifact) + if err := artifact.Validate(); err == nil { + t.Fatalf("Validate() error = nil for %#v", artifact) + } + }) + } +} + +func validPreparationArtifact() PromptPreparationArtifact { + started := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC) + return PromptPreparationArtifact{ + SchemaVersion: PromptPreparationSchemaVersion, Status: PromptPreparationSucceeded, + ReportID: report.Daily, RunID: "weatherreporter-run", PromptID: "weather.daily_generated_text", + PromptVersion: "1.0.0", DataPackagePath: "/workspace/data.yaml", + Preparation: &promptexec.Preparation{ + PromptID: "weather.daily_generated_text", PromptVersion: "1.0.0", + DataPackagePath: "/workspace/data.yaml", + }, + StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second, + } +} + +func validExecutionArtifact() PromptExecutionArtifact { + started := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC) + validation := promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", "daily.generated_text.schema.json", nil) + return PromptExecutionArtifact{ + SchemaVersion: PromptExecutionSchemaVersion, Status: PromptExecutionSucceeded, + ReportID: report.Daily, RunID: "weatherreporter-run", PromptID: "weather.daily_generated_text", PromptVersion: "1.0.0", + Provenance: &PromptExecutionProvenance{ + RunID: "provider-run", PromptID: "weather.daily_generated_text", PromptVersion: "1.0.0", + PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: "profile", + BackendID: "backend", ModelName: "model", DataPackagePath: "/workspace/data.yaml", + StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second, + }, + Validation: &validation, StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second, + } +} + +func validFailedExecutionArtifact() PromptExecutionArtifact { + started := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC) + return PromptExecutionArtifact{ + SchemaVersion: PromptExecutionSchemaVersion, Status: PromptExecutionFailed, + ReportID: report.Daily, RunID: "weatherreporter-run", PromptID: "weather.daily_generated_text", PromptVersion: "1.0.0", + StartedAt: started, EndedAt: started, Error: &PromptArtifactError{Category: promptexec.Generation, Message: "provider unavailable"}, + } +}