Harden durable prompt state contracts
This commit is contained in:
@@ -247,9 +247,15 @@ func (s *FilesystemStore) PrepareRenderedReport(_ context.Context, resolved repo
|
||||
}
|
||||
|
||||
func (s *FilesystemStore) SaveMetadata(_ context.Context, metadata Metadata) (string, error) {
|
||||
if metadata.SchemaVersion != MetadataSchemaVersion {
|
||||
return "", fmt.Errorf("new metadata must use schema version %q", MetadataSchemaVersion)
|
||||
}
|
||||
if err := metadata.Validate(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := s.validateManagedPath("metadata path", metadata.MetadataPath); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := fileutil.WriteJSONAtomic(metadata.MetadataPath, metadata); err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -483,6 +489,28 @@ func (s *FilesystemStore) join(parts ...string) string {
|
||||
return filepath.Join(all...)
|
||||
}
|
||||
|
||||
func (s *FilesystemStore) validateManagedPath(name, path string) error {
|
||||
if s == nil {
|
||||
return fmt.Errorf("state store is required")
|
||||
}
|
||||
root, err := filepath.Abs(s.root)
|
||||
if err != nil {
|
||||
return fmt.Errorf("resolve workspace root: %w", err)
|
||||
}
|
||||
target, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("resolve %s: %w", name, err)
|
||||
}
|
||||
relative, err := filepath.Rel(root, target)
|
||||
if err != nil {
|
||||
return fmt.Errorf("resolve %s relative to workspace root: %w", name, err)
|
||||
}
|
||||
if relative == "." || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
|
||||
return fmt.Errorf("%s must stay within workspace root", name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateRelativeDir(name string, value string) error {
|
||||
if value == "" {
|
||||
return fmt.Errorf("%s is required", name)
|
||||
|
||||
396
internal/state/filesystem_test.go
Normal file
396
internal/state/filesystem_test.go
Normal file
@@ -0,0 +1,396 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/briefing"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/module"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata"
|
||||
)
|
||||
|
||||
func TestFilesystemPathsUseExactPromptArtifactLayout(t *testing.T) {
|
||||
store := newFilesystemTestStore(t)
|
||||
tests := []struct {
|
||||
id report.ID
|
||||
now string
|
||||
date string
|
||||
group string
|
||||
validDate string
|
||||
runID string
|
||||
}{
|
||||
{report.Daily, "2026-05-29T05:00:00-05:00", "2026-05-29T12:00:00-05:00", "daily", "2026-05-29", "20260529T100000.000000000Z_daily_2026-05-29"},
|
||||
{report.Today, "2026-05-29T05:00:00-05:00", "", "today", "2026-05-29", "20260529T100000.000000000Z_today"},
|
||||
{report.Tomorrow, "2026-05-29T18:00:00-05:00", "", "tomorrow", "2026-05-30", "20260529T230000.000000000Z_tomorrow"},
|
||||
{report.Hourly, "2026-05-29T05:00:00-05:00", "", "hourly", "2026-05-29", "20260529T100000.000000000Z_hourly"},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(string(test.id), func(t *testing.T) {
|
||||
resolved := resolveStateReport(t, test.id, test.now, test.date)
|
||||
paths, err := store.Paths(resolved)
|
||||
if err != nil {
|
||||
t.Fatalf("Paths() error = %v", err)
|
||||
}
|
||||
want := ArtifactPaths{
|
||||
ModuleSnapshot: filepath.Join(store.root, "snapshots", test.group, test.validDate, "modules."+test.runID+".json"),
|
||||
Metadata: filepath.Join(store.root, "snapshots", test.group, test.validDate, "metadata."+test.runID+".json"),
|
||||
DataPackage: filepath.Join(store.root, "data-packages", test.group, test.validDate, "data_package."+test.runID+".yaml"),
|
||||
Preparation: filepath.Join(store.root, "preflight", test.group, test.validDate, "prompt_preparation."+test.runID+".json"),
|
||||
Execution: filepath.Join(store.root, "snapshots", test.group, test.validDate, "prompt_execution."+test.runID+".json"),
|
||||
Notification: filepath.Join(store.root, "notifications", test.group, test.validDate, "distributor."+test.runID+".json"),
|
||||
RenderedReport: filepath.Join(store.root, "reports", test.group, test.validDate, "report."+test.runID+".md"),
|
||||
GeneratedTextRaw: filepath.Join(store.root, "snapshots", test.group, test.validDate, "generated_text_raw."+test.runID+".json"),
|
||||
GeneratedText: filepath.Join(store.root, "snapshots", test.group, test.validDate, "generated_text."+test.runID+".json"),
|
||||
RenderContext: filepath.Join(store.root, "snapshots", test.group, test.validDate, "render_context."+test.runID+".json"),
|
||||
}
|
||||
if paths != want {
|
||||
t.Fatalf("Paths() = %#v, want %#v", paths, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptArtifactsAndMetadataRoundTrip(t *testing.T) {
|
||||
store := newFilesystemTestStore(t)
|
||||
resolved := resolveStateReport(t, report.Daily, "2026-05-29T05:00:00-05:00", "2026-05-29T12:00:00-05:00")
|
||||
paths, err := store.Paths(resolved)
|
||||
if err != nil {
|
||||
t.Fatalf("Paths() error = %v", err)
|
||||
}
|
||||
preparation := preparationArtifactFor(resolved, paths)
|
||||
preparationPath, err := store.SavePromptPreparation(context.Background(), resolved, preparation)
|
||||
if err != nil {
|
||||
t.Fatalf("SavePromptPreparation() error = %v", err)
|
||||
}
|
||||
execution := executionArtifactFor(resolved, paths)
|
||||
executionPath, err := store.SavePromptExecution(context.Background(), resolved, execution)
|
||||
if err != nil {
|
||||
t.Fatalf("SavePromptExecution() error = %v", err)
|
||||
}
|
||||
loadedPreparation, err := store.LoadPromptPreparation(context.Background(), preparationPath)
|
||||
if err != nil || loadedPreparation.Preparation == nil || loadedPreparation.Preparation.PromptHash != "prompt-hash" {
|
||||
t.Fatalf("LoadPromptPreparation() = %#v, %v", loadedPreparation, err)
|
||||
}
|
||||
loadedExecution, err := store.LoadPromptExecution(context.Background(), executionPath)
|
||||
if err != nil || loadedExecution.Provenance == nil || loadedExecution.Provenance.RunID != "provider-run" || loadedExecution.Validation == nil {
|
||||
t.Fatalf("LoadPromptExecution() = %#v, %v", loadedExecution, err)
|
||||
}
|
||||
|
||||
metadata := promptMetadataFor(resolved, paths)
|
||||
metadata.PreparationPath = preparationPath
|
||||
metadata.ExecutionPath = executionPath
|
||||
metadata.GeneratedTextRawPath = paths.GeneratedTextRaw
|
||||
metadataPath, err := store.SaveMetadata(context.Background(), metadata)
|
||||
if err != nil {
|
||||
t.Fatalf("SaveMetadata() error = %v", err)
|
||||
}
|
||||
data, err := os.ReadFile(metadataPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read metadata: %v", err)
|
||||
}
|
||||
text := string(data)
|
||||
if strings.Contains(text, "preflightPath") || strings.Contains(text, "generatedTextResultPath") || strings.Contains(text, "metadataPath") {
|
||||
t.Fatalf("v2 metadata contains legacy or runtime aliases: %s", text)
|
||||
}
|
||||
loadedMetadata, loadedPath, err := store.LoadMetadataByRunID(context.Background(), metadata.RunID)
|
||||
if err != nil || loadedPath != metadataPath || loadedMetadata.PreparationPath != preparationPath || loadedMetadata.ExecutionPath != executionPath {
|
||||
t.Fatalf("LoadMetadataByRunID() = %#v, %q, %v", loadedMetadata, loadedPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetadataLegacyCompatibilityAndV2OnlyWrites(t *testing.T) {
|
||||
legacy := Metadata{
|
||||
SchemaVersion: MetadataSchemaVersionV1, RunID: "legacy-run", ReportID: report.ID("three_day"),
|
||||
PromptID: "weather.three_day", ModuleSnapshotPath: "/archive/modules.json", DataPackagePath: "/archive/data.yaml",
|
||||
PreflightPath: "/archive/render.json", GeneratedTextResultPath: "/archive/result.json",
|
||||
}
|
||||
data, err := json.Marshal(legacy)
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal() error = %v", err)
|
||||
}
|
||||
text := string(data)
|
||||
if !strings.Contains(text, "preflightPath") || !strings.Contains(text, "generatedTextResultPath") || strings.Contains(text, "preparationPath") || strings.Contains(text, "executionPath") {
|
||||
t.Fatalf("v1 metadata wire fields = %s", text)
|
||||
}
|
||||
var decoded Metadata
|
||||
if err := json.Unmarshal(data, &decoded); err != nil {
|
||||
t.Fatalf("Unmarshal() error = %v", err)
|
||||
}
|
||||
if decoded.PreparationPath != legacy.PreflightPath || decoded.ExecutionPath != legacy.GeneratedTextResultPath {
|
||||
t.Fatalf("normalized compatibility aliases = %#v", decoded)
|
||||
}
|
||||
remarshaled, err := json.Marshal(decoded)
|
||||
if err != nil || !strings.Contains(string(remarshaled), "preflightPath") || strings.Contains(string(remarshaled), "preparationPath") {
|
||||
t.Fatalf("remarshaled v1 metadata = %s, %v", remarshaled, err)
|
||||
}
|
||||
if err := json.Unmarshal([]byte(`{"schemaVersion":"weatherreporter.metadata.v99"}`), &decoded); err == nil {
|
||||
t.Fatal("Unmarshal() error = nil, want unknown schema rejection")
|
||||
}
|
||||
|
||||
store := newFilesystemTestStore(t)
|
||||
legacy.MetadataPath = filepath.Join(store.root, "snapshots", "legacy", "metadata.legacy-run.json")
|
||||
if _, err := store.SaveMetadata(context.Background(), legacy); err == nil {
|
||||
t.Fatal("SaveMetadata(v1) error = nil, want v2-only write rejection")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReportDiscoveryAndArtifactInspection(t *testing.T) {
|
||||
store := newFilesystemTestStore(t)
|
||||
older := resolveStateReport(t, report.Daily, "2026-05-29T05:00:00-05:00", "2026-05-29T12:00:00-05:00")
|
||||
newer := resolveStateReport(t, report.Today, "2026-05-29T08:00:00-05:00", "")
|
||||
olderPaths := saveStateMetadata(t, store, older)
|
||||
newerPaths := saveStateMetadata(t, store, newer)
|
||||
|
||||
snapshot, err := module.NewSnapshot([]module.Output{{ID: module.Metadata, StanzaName: "metadata", Value: map[string]any{"run_id": older.Metadata().RunID}}})
|
||||
if err != nil {
|
||||
t.Fatalf("NewSnapshot() error = %v", err)
|
||||
}
|
||||
modulePath, err := store.SaveModuleSnapshot(context.Background(), older, snapshot)
|
||||
if err != nil {
|
||||
t.Fatalf("SaveModuleSnapshot() error = %v", err)
|
||||
}
|
||||
pkg, err := promptinput.Build(promptinput.BuildRequest{
|
||||
Metadata: promptinput.Metadata{
|
||||
RunID: older.Metadata().RunID, ReportID: older.Definition.ID, PromptID: older.Definition.PromptID,
|
||||
GeneratedAt: older.GeneratedAt, Timezone: older.Timezone, ValidPeriod: older.ValidPeriod,
|
||||
},
|
||||
Modules: snapshot,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v", err)
|
||||
}
|
||||
dataPath, err := store.SaveDataPackage(context.Background(), older, pkg)
|
||||
if err != nil {
|
||||
t.Fatalf("SaveDataPackage() error = %v", err)
|
||||
}
|
||||
loadedSnapshot, err := store.LoadModuleSnapshot(context.Background(), modulePath)
|
||||
if err != nil || len(loadedSnapshot.Outputs) != 1 {
|
||||
t.Fatalf("LoadModuleSnapshot() = %#v, %v", loadedSnapshot, err)
|
||||
}
|
||||
loadedPackage, err := store.LoadDataPackage(context.Background(), dataPath)
|
||||
if err != nil || loadedPackage.RunID != older.Metadata().RunID {
|
||||
t.Fatalf("LoadDataPackage() = %#v, %v", loadedPackage, err)
|
||||
}
|
||||
|
||||
records, err := store.ListReports(context.Background(), 0)
|
||||
if err != nil || len(records) != 2 {
|
||||
t.Fatalf("ListReports() = %#v, %v", records, err)
|
||||
}
|
||||
if records[0].RunID != newer.Metadata().RunID || records[0].MetadataPath != newerPaths.Metadata || records[1].MetadataPath != olderPaths.Metadata {
|
||||
t.Fatalf("ordered report records = %#v", records)
|
||||
}
|
||||
loadedMetadata, loadedPath, err := store.LoadMetadataByRunID(context.Background(), older.Metadata().RunID)
|
||||
if err != nil || loadedPath != olderPaths.Metadata || len(loadedMetadata.Sources) != 1 || loadedMetadata.Sources[0].Name != "weather-api" {
|
||||
t.Fatalf("LoadMetadataByRunID() = %#v, %q, %v", loadedMetadata, loadedPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListReportsRetainsHistoricalV1ReportIDs(t *testing.T) {
|
||||
store := newFilesystemTestStore(t)
|
||||
for i, id := range []report.ID{"three_day", "weekend", "storm"} {
|
||||
generatedAt := time.Date(2026, 5, 20+i, 12, 0, 0, 0, time.UTC)
|
||||
metadata := Metadata{
|
||||
SchemaVersion: MetadataSchemaVersionV1, RunID: "historical-" + string(id), ReportID: id,
|
||||
PromptID: "weather." + string(id), GeneratedAt: generatedAt, Timezone: "UTC",
|
||||
ValidPeriod: timeutil.Period{Start: generatedAt, End: generatedAt.Add(24 * time.Hour)},
|
||||
ModuleSnapshotPath: "/archive/modules.json", DataPackagePath: "/archive/data.yaml",
|
||||
PreflightPath: "/archive/render.json", GeneratedTextResultPath: "/archive/result.json",
|
||||
}
|
||||
path := filepath.Join(store.root, "snapshots", string(id), "2026-05-20", "metadata."+metadata.RunID+".json")
|
||||
writeJSONFixture(t, path, metadata)
|
||||
}
|
||||
records, err := store.ListReports(context.Background(), 0)
|
||||
if err != nil || len(records) != 3 {
|
||||
t.Fatalf("ListReports() = %#v, %v", records, err)
|
||||
}
|
||||
for _, record := range records {
|
||||
if record.ReportID != "three_day" && record.ReportID != "weekend" && record.ReportID != "storm" {
|
||||
t.Fatalf("unexpected historical report record: %#v", record)
|
||||
}
|
||||
metadata, path, err := store.LoadMetadataByRunID(context.Background(), record.RunID)
|
||||
if err != nil || path != record.MetadataPath || metadata.PreparationPath != "/archive/render.json" || metadata.ExecutionPath != "/archive/result.json" {
|
||||
t.Fatalf("LoadMetadataByRunID(%q) = %#v, %q, %v", record.RunID, metadata, path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilesystemWritesAreAtomicAndRejectUnsafePaths(t *testing.T) {
|
||||
store := newFilesystemTestStore(t)
|
||||
resolved := resolveStateReport(t, report.Hourly, "2026-05-29T05:00:00-05:00", "")
|
||||
path, err := store.SaveGeneratedTextRaw(context.Background(), resolved, []byte(`{"value":"first"}`))
|
||||
if err != nil {
|
||||
t.Fatalf("first SaveGeneratedTextRaw() error = %v", err)
|
||||
}
|
||||
if _, err := store.SaveGeneratedTextRaw(context.Background(), resolved, []byte(`{"value":"second"}`)); err != nil {
|
||||
t.Fatalf("second SaveGeneratedTextRaw() error = %v", err)
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil || string(data) != `{"value":"second"}` {
|
||||
t.Fatalf("atomic replacement = %q, %v", data, err)
|
||||
}
|
||||
entries, err := os.ReadDir(filepath.Dir(path))
|
||||
if err != nil || len(entries) != 1 || entries[0].Name() != filepath.Base(path) {
|
||||
t.Fatalf("artifact directory after atomic write = %#v, %v", entries, err)
|
||||
}
|
||||
|
||||
metadata := promptMetadataFor(resolved, mustStatePaths(t, store, resolved))
|
||||
metadata.PreparationPath = "/saved/preparation.json"
|
||||
metadata.MetadataPath = filepath.Join(t.TempDir(), "outside.json")
|
||||
if _, err := store.SaveMetadata(context.Background(), metadata); err == nil {
|
||||
t.Fatal("SaveMetadata(outside workspace) error = nil")
|
||||
}
|
||||
|
||||
cfg := config.Defaults().Workspace
|
||||
cfg.Root = t.TempDir()
|
||||
cfg.SnapshotsDir = "../snapshots"
|
||||
if _, err := NewFilesystemStore(cfg); err == nil {
|
||||
t.Fatal("NewFilesystemStore(unsafe directory) error = nil")
|
||||
}
|
||||
unsafeResolved := resolved
|
||||
unsafeResolved.Definition.ID = report.ID("hourly/bad")
|
||||
if _, err := store.Paths(unsafeResolved); err == nil {
|
||||
t.Fatal("Paths(unsafe run id) error = nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindPriorSnapshotForSupportedReports(t *testing.T) {
|
||||
tests := []struct {
|
||||
id report.ID
|
||||
firstNow string
|
||||
secondNow string
|
||||
date string
|
||||
wantPrior bool
|
||||
}{
|
||||
{report.Daily, "2026-05-29T05:00:00-05:00", "2026-05-29T08:00:00-05:00", "2026-05-29T12:00:00-05:00", true},
|
||||
{report.Today, "2026-05-29T05:00:00-05:00", "2026-05-29T08:00:00-05:00", "", true},
|
||||
{report.Tomorrow, "2026-05-29T17:00:00-05:00", "2026-05-29T18:00:00-05:00", "", true},
|
||||
{report.Hourly, "2026-05-29T05:00:00-05:00", "2026-05-29T06:00:00-05:00", "", false},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(string(test.id), func(t *testing.T) {
|
||||
store := newFilesystemTestStore(t)
|
||||
first := resolveStateReport(t, test.id, test.firstNow, test.date)
|
||||
second := resolveStateReport(t, test.id, test.secondNow, test.date)
|
||||
paths := saveStateMetadata(t, store, first)
|
||||
prior, err := store.FindPriorSnapshot(context.Background(), second)
|
||||
if err != nil {
|
||||
t.Fatalf("FindPriorSnapshot() error = %v", err)
|
||||
}
|
||||
if !test.wantPrior && prior != nil {
|
||||
t.Fatalf("FindPriorSnapshot() = %#v, want nil", prior)
|
||||
}
|
||||
if test.wantPrior && (prior == nil || prior.Metadata.RunID != first.Metadata().RunID || prior.ModuleSnapshotPath != paths.ModuleSnapshot) {
|
||||
t.Fatalf("FindPriorSnapshot() = %#v, want run %q", prior, first.Metadata().RunID)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func newFilesystemTestStore(t *testing.T) *FilesystemStore {
|
||||
t.Helper()
|
||||
cfg := config.Defaults().Workspace
|
||||
cfg.Root = t.TempDir()
|
||||
store, err := NewFilesystemStore(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||
}
|
||||
return store
|
||||
}
|
||||
|
||||
func resolveStateReport(t *testing.T, id report.ID, nowValue, dateValue string) report.Resolved {
|
||||
t.Helper()
|
||||
location, err := time.LoadLocation("America/Chicago")
|
||||
if err != nil {
|
||||
t.Fatalf("LoadLocation() error = %v", err)
|
||||
}
|
||||
now, err := time.Parse(time.RFC3339, nowValue)
|
||||
if err != nil {
|
||||
t.Fatalf("parse now: %v", err)
|
||||
}
|
||||
req := report.ResolveRequest{Now: now, Location: location}
|
||||
if dateValue != "" {
|
||||
req.Date, err = time.Parse(time.RFC3339, dateValue)
|
||||
if err != nil {
|
||||
t.Fatalf("parse date: %v", err)
|
||||
}
|
||||
}
|
||||
resolved, err := report.DefaultRegistry().Resolve(id, req)
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
return resolved
|
||||
}
|
||||
|
||||
func mustStatePaths(t *testing.T, store *FilesystemStore, resolved report.Resolved) ArtifactPaths {
|
||||
t.Helper()
|
||||
paths, err := store.Paths(resolved)
|
||||
if err != nil {
|
||||
t.Fatalf("Paths() error = %v", err)
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
func promptMetadataFor(resolved report.Resolved, paths ArtifactPaths) Metadata {
|
||||
metadata := BuildPromptMetadataFromBriefingMetadata(resolved, briefing.Metadata{
|
||||
Sources: []briefing.SourceMetadata{{Name: "weather-api", FetchedAt: resolved.GeneratedAt}},
|
||||
SourceWarnings: []weatherdata.SourceWarning{},
|
||||
}, ArtifactPaths{ModuleSnapshot: paths.ModuleSnapshot, Metadata: paths.Metadata, DataPackage: paths.DataPackage})
|
||||
return metadata
|
||||
}
|
||||
|
||||
func saveStateMetadata(t *testing.T, store *FilesystemStore, resolved report.Resolved) ArtifactPaths {
|
||||
t.Helper()
|
||||
paths := mustStatePaths(t, store, resolved)
|
||||
metadata := promptMetadataFor(resolved, paths)
|
||||
metadata.PreparationPath = paths.Preparation
|
||||
if _, err := store.SaveMetadata(context.Background(), metadata); err != nil {
|
||||
t.Fatalf("SaveMetadata() error = %v", err)
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
func preparationArtifactFor(resolved report.Resolved, paths ArtifactPaths) PromptPreparationArtifact {
|
||||
artifact := validPreparationArtifact()
|
||||
metadata := resolved.Metadata()
|
||||
artifact.ReportID, artifact.RunID = metadata.ReportID, metadata.RunID
|
||||
artifact.PromptID, artifact.PromptVersion = resolved.Definition.PromptID, resolved.Definition.PromptVersion
|
||||
artifact.DataPackagePath = paths.DataPackage
|
||||
artifact.Preparation.PromptID, artifact.Preparation.PromptVersion = artifact.PromptID, artifact.PromptVersion
|
||||
artifact.Preparation.PromptHash = "prompt-hash"
|
||||
artifact.Preparation.DataPackagePath = paths.DataPackage
|
||||
return artifact
|
||||
}
|
||||
|
||||
func executionArtifactFor(resolved report.Resolved, paths ArtifactPaths) PromptExecutionArtifact {
|
||||
artifact := validExecutionArtifact()
|
||||
metadata := resolved.Metadata()
|
||||
artifact.ReportID, artifact.RunID = metadata.ReportID, metadata.RunID
|
||||
artifact.PromptID, artifact.PromptVersion = resolved.Definition.PromptID, resolved.Definition.PromptVersion
|
||||
artifact.Provenance.PromptID, artifact.Provenance.PromptVersion = artifact.PromptID, artifact.PromptVersion
|
||||
artifact.Provenance.DataPackagePath = paths.DataPackage
|
||||
artifact.Paths.RawOutputPath = paths.GeneratedTextRaw
|
||||
return artifact
|
||||
}
|
||||
|
||||
func writeJSONFixture(t *testing.T, path string, value any) {
|
||||
t.Helper()
|
||||
data, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal fixture: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatalf("create fixture directory: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(path, data, 0o600); err != nil {
|
||||
t.Fatalf("write fixture: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -67,21 +67,30 @@ func (a PromptPreparationArtifact) Validate() error {
|
||||
if a.SchemaVersion != PromptPreparationSchemaVersion {
|
||||
return fmt.Errorf("unsupported prompt preparation schema version %q", a.SchemaVersion)
|
||||
}
|
||||
if strings.TrimSpace(a.RunID) == "" || a.ReportID == "" || strings.TrimSpace(a.PromptID) == "" {
|
||||
return fmt.Errorf("prompt preparation identity is required")
|
||||
if err := validatePromptArtifactIdentity("prompt preparation", a.ReportID, a.RunID, a.PromptID, a.PromptVersion); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(a.DataPackagePath) == "" {
|
||||
return fmt.Errorf("prompt preparation data package path is required")
|
||||
}
|
||||
if err := validatePromptArtifactTiming("prompt preparation", a.StartedAt, a.EndedAt, a.Duration); err != nil {
|
||||
return err
|
||||
}
|
||||
switch a.Status {
|
||||
case PromptPreparationSucceeded:
|
||||
if a.Preparation == nil || a.Error != nil {
|
||||
return fmt.Errorf("successful prompt preparation requires preparation without an error")
|
||||
}
|
||||
if a.Preparation.PromptID != a.PromptID || a.Preparation.PromptVersion != a.PromptVersion || a.Preparation.DataPackagePath != a.DataPackagePath {
|
||||
return fmt.Errorf("successful prompt preparation provenance must match the artifact")
|
||||
}
|
||||
case PromptPreparationFailed:
|
||||
if !validPromptArtifactError(a.Error) {
|
||||
return fmt.Errorf("failed prompt preparation requires a classified error")
|
||||
}
|
||||
if a.Preparation != nil {
|
||||
return fmt.Errorf("failed prompt preparation must not include preparation provenance")
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("unsupported prompt preparation status %q", a.Status)
|
||||
}
|
||||
@@ -164,22 +173,34 @@ func (a PromptExecutionArtifact) Validate() error {
|
||||
if a.SchemaVersion != PromptExecutionSchemaVersion {
|
||||
return fmt.Errorf("unsupported prompt execution schema version %q", a.SchemaVersion)
|
||||
}
|
||||
if strings.TrimSpace(a.RunID) == "" || a.ReportID == "" || strings.TrimSpace(a.PromptID) == "" {
|
||||
return fmt.Errorf("prompt execution identity is required")
|
||||
if err := validatePromptArtifactIdentity("prompt execution", a.ReportID, a.RunID, a.PromptID, a.PromptVersion); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validatePromptArtifactTiming("prompt execution", a.StartedAt, a.EndedAt, a.Duration); err != nil {
|
||||
return err
|
||||
}
|
||||
switch a.Status {
|
||||
case PromptExecutionSucceeded:
|
||||
if a.Provenance == nil || a.Validation == nil || a.Validation.Status != promptexec.ValidationPassed || a.Error != nil {
|
||||
return fmt.Errorf("successful prompt execution requires passed validation without an error")
|
||||
}
|
||||
if err := validatePromptExecutionProvenance(a); err != nil {
|
||||
return err
|
||||
}
|
||||
case PromptExecutionValidationRejected:
|
||||
if a.Provenance == nil || a.Validation == nil || a.Validation.Status != promptexec.ValidationFailed || a.Error != nil {
|
||||
return fmt.Errorf("validation-rejected prompt execution requires failed validation without an error")
|
||||
}
|
||||
if err := validatePromptExecutionProvenance(a); err != nil {
|
||||
return err
|
||||
}
|
||||
case PromptExecutionFailed:
|
||||
if !validPromptArtifactError(a.Error) {
|
||||
return fmt.Errorf("failed prompt execution requires a classified error")
|
||||
}
|
||||
if a.Provenance != nil || a.Validation != nil {
|
||||
return fmt.Errorf("failed prompt execution must not include completed provenance or validation")
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("unsupported prompt execution status %q", a.Status)
|
||||
}
|
||||
@@ -187,5 +208,74 @@ func (a PromptExecutionArtifact) Validate() error {
|
||||
}
|
||||
|
||||
func validPromptArtifactError(value *PromptArtifactError) bool {
|
||||
return value != nil && value.Category != "" && strings.TrimSpace(value.Message) != "" && len(value.Message) <= promptArtifactErrorLimit && utf8.ValidString(value.Message)
|
||||
return value != nil && validPromptErrorCategory(value.Category) && strings.TrimSpace(value.Message) != "" && len(value.Message) <= promptArtifactErrorLimit && utf8.ValidString(value.Message)
|
||||
}
|
||||
|
||||
func validatePromptArtifactIdentity(kind string, reportID report.ID, runID, promptID, promptVersion string) error {
|
||||
if reportID == "" || strings.TrimSpace(runID) == "" || strings.TrimSpace(promptID) == "" {
|
||||
return fmt.Errorf("%s identity is required", kind)
|
||||
}
|
||||
definition, err := report.DefaultRegistry().Lookup(reportID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s report id is unsupported: %w", kind, err)
|
||||
}
|
||||
if promptID != definition.PromptID {
|
||||
return fmt.Errorf("%s prompt id must match report %q", kind, reportID)
|
||||
}
|
||||
if promptVersion != definition.PromptVersion {
|
||||
return fmt.Errorf("%s prompt version must be %q", kind, definition.PromptVersion)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validatePromptArtifactTiming(kind string, startedAt, endedAt time.Time, duration time.Duration) error {
|
||||
if startedAt.IsZero() || endedAt.IsZero() {
|
||||
return fmt.Errorf("%s start and end times are required", kind)
|
||||
}
|
||||
if duration < 0 {
|
||||
return fmt.Errorf("%s duration must not be negative", kind)
|
||||
}
|
||||
if endedAt.Before(startedAt) {
|
||||
return fmt.Errorf("%s end time must not be earlier than its start time", kind)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validatePromptExecutionProvenance(artifact PromptExecutionArtifact) error {
|
||||
value := artifact.Provenance
|
||||
if value == nil {
|
||||
return fmt.Errorf("completed prompt execution provenance is required")
|
||||
}
|
||||
if value.PromptID != artifact.PromptID || value.PromptVersion != artifact.PromptVersion {
|
||||
return fmt.Errorf("completed prompt execution provenance must match the artifact")
|
||||
}
|
||||
for _, required := range []struct {
|
||||
name string
|
||||
value string
|
||||
}{
|
||||
{"run id", value.RunID}, {"prompt hash", value.PromptHash}, {"rendered prompt hash", value.RenderedPromptHash},
|
||||
{"profile id", value.ProfileID}, {"backend id", value.BackendID}, {"model name", value.ModelName},
|
||||
{"data package path", value.DataPackagePath},
|
||||
} {
|
||||
if strings.TrimSpace(required.value) == "" {
|
||||
return fmt.Errorf("completed prompt execution provenance %s is required", required.name)
|
||||
}
|
||||
}
|
||||
if err := validatePromptArtifactTiming("completed prompt execution provenance", value.StartedAt, value.EndedAt, value.Duration); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validPromptErrorCategory(category promptexec.ErrorCategory) bool {
|
||||
switch category {
|
||||
case promptexec.InvalidConfiguration, promptexec.InvalidRequest, promptexec.PromptNotFound,
|
||||
promptexec.PromptLoad, promptexec.ProfileNotFound, promptexec.ProfileLoad,
|
||||
promptexec.MissingCredential, promptexec.ArtifactLoad, promptexec.PromptRender,
|
||||
promptexec.Capacity, promptexec.Generation, promptexec.OperationalValidation,
|
||||
promptexec.ValidationRejected, promptexec.Canceled, promptexec.DeadlineExceeded:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
207
internal/state/prompt_artifacts_test.go
Normal file
207
internal/state/prompt_artifacts_test.go
Normal file
@@ -0,0 +1,207 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
||||
)
|
||||
|
||||
func TestPromptPreparationArtifactValidation(t *testing.T) {
|
||||
valid := validPreparationArtifact()
|
||||
if err := valid.Validate(); err != nil {
|
||||
t.Fatalf("valid successful preparation: %v", err)
|
||||
}
|
||||
failed := valid
|
||||
failed.Status = PromptPreparationFailed
|
||||
failed.Preparation = nil
|
||||
failed.Error = &PromptArtifactError{Category: promptexec.Generation, Message: "provider unavailable"}
|
||||
if err := failed.Validate(); err != nil {
|
||||
t.Fatalf("valid failed preparation: %v", err)
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(*PromptPreparationArtifact)
|
||||
}{
|
||||
{"report id", func(a *PromptPreparationArtifact) { a.ReportID = "" }},
|
||||
{"run id", func(a *PromptPreparationArtifact) { a.RunID = "" }},
|
||||
{"prompt id", func(a *PromptPreparationArtifact) { a.PromptID = "" }},
|
||||
{"prompt id for another report", func(a *PromptPreparationArtifact) { a.PromptID = "weather.hourly_generated_text" }},
|
||||
{"prompt version", func(a *PromptPreparationArtifact) { a.PromptVersion = "latest" }},
|
||||
{"data package", func(a *PromptPreparationArtifact) { a.DataPackagePath = "" }},
|
||||
{"start time", func(a *PromptPreparationArtifact) { a.StartedAt = time.Time{} }},
|
||||
{"end time", func(a *PromptPreparationArtifact) { a.EndedAt = time.Time{} }},
|
||||
{"negative duration", func(a *PromptPreparationArtifact) { a.Duration = -time.Second }},
|
||||
{"reversed times", func(a *PromptPreparationArtifact) { a.EndedAt = a.StartedAt.Add(-time.Second) }},
|
||||
{"missing provenance", func(a *PromptPreparationArtifact) { a.Preparation = nil }},
|
||||
{"success error", func(a *PromptPreparationArtifact) {
|
||||
a.Error = &PromptArtifactError{Category: promptexec.Generation, Message: "failed"}
|
||||
}},
|
||||
{"provenance prompt id", func(a *PromptPreparationArtifact) { a.Preparation.PromptID = "other" }},
|
||||
{"provenance prompt version", func(a *PromptPreparationArtifact) { a.Preparation.PromptVersion = "other" }},
|
||||
{"provenance data package", func(a *PromptPreparationArtifact) { a.Preparation.DataPackagePath = "/other/data.yaml" }},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
artifact := validPreparationArtifact()
|
||||
test.mutate(&artifact)
|
||||
if err := artifact.Validate(); err == nil {
|
||||
t.Fatalf("Validate() error = nil for %#v", artifact)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailedPromptPreparationRejectsContradictoryDetails(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(*PromptPreparationArtifact)
|
||||
}{
|
||||
{"missing error", func(a *PromptPreparationArtifact) { a.Error = nil }},
|
||||
{"unknown category", func(a *PromptPreparationArtifact) { a.Error.Category = promptexec.ErrorCategory("other") }},
|
||||
{"empty message", func(a *PromptPreparationArtifact) { a.Error.Message = " " }},
|
||||
{"oversized message", func(a *PromptPreparationArtifact) { a.Error.Message = strings.Repeat("x", promptArtifactErrorLimit+1) }},
|
||||
{"invented provenance", func(a *PromptPreparationArtifact) { a.Preparation = validPreparationArtifact().Preparation }},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
artifact := validPreparationArtifact()
|
||||
artifact.Status = PromptPreparationFailed
|
||||
artifact.Preparation = nil
|
||||
artifact.Error = &PromptArtifactError{Category: promptexec.Generation, Message: "provider unavailable"}
|
||||
test.mutate(&artifact)
|
||||
if err := artifact.Validate(); err == nil {
|
||||
t.Fatalf("Validate() error = nil for %#v", artifact)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptExecutionArtifactValidation(t *testing.T) {
|
||||
valid := validExecutionArtifact()
|
||||
valid.Provenance.RunID = "provider-run-different-from-weatherreporter"
|
||||
valid.Provenance.GeneratedHash = ""
|
||||
valid.Provenance.Usage = promptexec.TokenUsage{}
|
||||
if err := valid.Validate(); err != nil {
|
||||
t.Fatalf("valid completed execution with provider run identity and omitted counters: %v", err)
|
||||
}
|
||||
rejected := validExecutionArtifact()
|
||||
rejected.Status = PromptExecutionValidationRejected
|
||||
validation := promptexec.NewValidation(promptexec.ValidationFailed, "json_schema", "daily.generated_text.schema.json", []string{"schema mismatch"})
|
||||
rejected.Validation = &validation
|
||||
if err := rejected.Validate(); err != nil {
|
||||
t.Fatalf("valid validation rejection: %v", err)
|
||||
}
|
||||
failed := validFailedExecutionArtifact()
|
||||
if err := failed.Validate(); err != nil {
|
||||
t.Fatalf("valid operational failure: %v", err)
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(*PromptExecutionArtifact)
|
||||
}{
|
||||
{"report id", func(a *PromptExecutionArtifact) { a.ReportID = "" }},
|
||||
{"run id", func(a *PromptExecutionArtifact) { a.RunID = "" }},
|
||||
{"prompt id", func(a *PromptExecutionArtifact) { a.PromptID = "" }},
|
||||
{"prompt version", func(a *PromptExecutionArtifact) { a.PromptVersion = "latest" }},
|
||||
{"start time", func(a *PromptExecutionArtifact) { a.StartedAt = time.Time{} }},
|
||||
{"end time", func(a *PromptExecutionArtifact) { a.EndedAt = time.Time{} }},
|
||||
{"negative duration", func(a *PromptExecutionArtifact) { a.Duration = -time.Second }},
|
||||
{"reversed times", func(a *PromptExecutionArtifact) { a.EndedAt = a.StartedAt.Add(-time.Second) }},
|
||||
{"missing provenance", func(a *PromptExecutionArtifact) { a.Provenance = nil }},
|
||||
{"missing validation", func(a *PromptExecutionArtifact) { a.Validation = nil }},
|
||||
{"wrong validation", func(a *PromptExecutionArtifact) {
|
||||
value := promptexec.NewValidation(promptexec.ValidationFailed, "json_schema", "schema.json", nil)
|
||||
a.Validation = &value
|
||||
}},
|
||||
{"operational error", func(a *PromptExecutionArtifact) {
|
||||
a.Error = &PromptArtifactError{Category: promptexec.Generation, Message: "failed"}
|
||||
}},
|
||||
{"provenance prompt id", func(a *PromptExecutionArtifact) { a.Provenance.PromptID = "other" }},
|
||||
{"provenance prompt version", func(a *PromptExecutionArtifact) { a.Provenance.PromptVersion = "other" }},
|
||||
{"provenance run id", func(a *PromptExecutionArtifact) { a.Provenance.RunID = "" }},
|
||||
{"prompt hash", func(a *PromptExecutionArtifact) { a.Provenance.PromptHash = "" }},
|
||||
{"rendered hash", func(a *PromptExecutionArtifact) { a.Provenance.RenderedPromptHash = "" }},
|
||||
{"profile id", func(a *PromptExecutionArtifact) { a.Provenance.ProfileID = "" }},
|
||||
{"backend id", func(a *PromptExecutionArtifact) { a.Provenance.BackendID = "" }},
|
||||
{"model name", func(a *PromptExecutionArtifact) { a.Provenance.ModelName = "" }},
|
||||
{"data package", func(a *PromptExecutionArtifact) { a.Provenance.DataPackagePath = "" }},
|
||||
{"provenance start time", func(a *PromptExecutionArtifact) { a.Provenance.StartedAt = time.Time{} }},
|
||||
{"provenance end time", func(a *PromptExecutionArtifact) { a.Provenance.EndedAt = time.Time{} }},
|
||||
{"provenance negative duration", func(a *PromptExecutionArtifact) { a.Provenance.Duration = -time.Second }},
|
||||
{"provenance reversed times", func(a *PromptExecutionArtifact) { a.Provenance.EndedAt = a.Provenance.StartedAt.Add(-time.Second) }},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
artifact := validExecutionArtifact()
|
||||
test.mutate(&artifact)
|
||||
if err := artifact.Validate(); err == nil {
|
||||
t.Fatalf("Validate() error = nil for %#v", artifact)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailedPromptExecutionRejectsContradictoryDetails(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(*PromptExecutionArtifact)
|
||||
}{
|
||||
{"missing error", func(a *PromptExecutionArtifact) { a.Error = nil }},
|
||||
{"unknown category", func(a *PromptExecutionArtifact) { a.Error.Category = promptexec.ErrorCategory("other") }},
|
||||
{"provenance", func(a *PromptExecutionArtifact) { a.Provenance = validExecutionArtifact().Provenance }},
|
||||
{"completed validation", func(a *PromptExecutionArtifact) { a.Validation = validExecutionArtifact().Validation }},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
artifact := validFailedExecutionArtifact()
|
||||
test.mutate(&artifact)
|
||||
if err := artifact.Validate(); err == nil {
|
||||
t.Fatalf("Validate() error = nil for %#v", artifact)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func validPreparationArtifact() PromptPreparationArtifact {
|
||||
started := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
|
||||
return PromptPreparationArtifact{
|
||||
SchemaVersion: PromptPreparationSchemaVersion, Status: PromptPreparationSucceeded,
|
||||
ReportID: report.Daily, RunID: "weatherreporter-run", PromptID: "weather.daily_generated_text",
|
||||
PromptVersion: "1.0.0", DataPackagePath: "/workspace/data.yaml",
|
||||
Preparation: &promptexec.Preparation{
|
||||
PromptID: "weather.daily_generated_text", PromptVersion: "1.0.0",
|
||||
DataPackagePath: "/workspace/data.yaml",
|
||||
},
|
||||
StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
func validExecutionArtifact() PromptExecutionArtifact {
|
||||
started := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
|
||||
validation := promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", "daily.generated_text.schema.json", nil)
|
||||
return PromptExecutionArtifact{
|
||||
SchemaVersion: PromptExecutionSchemaVersion, Status: PromptExecutionSucceeded,
|
||||
ReportID: report.Daily, RunID: "weatherreporter-run", PromptID: "weather.daily_generated_text", PromptVersion: "1.0.0",
|
||||
Provenance: &PromptExecutionProvenance{
|
||||
RunID: "provider-run", PromptID: "weather.daily_generated_text", PromptVersion: "1.0.0",
|
||||
PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: "profile",
|
||||
BackendID: "backend", ModelName: "model", DataPackagePath: "/workspace/data.yaml",
|
||||
StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second,
|
||||
},
|
||||
Validation: &validation, StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
func validFailedExecutionArtifact() PromptExecutionArtifact {
|
||||
started := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
|
||||
return PromptExecutionArtifact{
|
||||
SchemaVersion: PromptExecutionSchemaVersion, Status: PromptExecutionFailed,
|
||||
ReportID: report.Daily, RunID: "weatherreporter-run", PromptID: "weather.daily_generated_text", PromptVersion: "1.0.0",
|
||||
StartedAt: started, EndedAt: started, Error: &PromptArtifactError{Category: promptexec.Generation, Message: "provider unavailable"},
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user