From bdbab48d10ad241d0b02326aadc95fa89616c8df Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sat, 20 Jun 2026 13:37:18 +0000 Subject: [PATCH] Document managed workspace artifact layout --- docs/config.md | 7 +- docs/integrations/distributor/api.md | 2 +- docs/integrations/distributor/pkg-bundle.md | 3 +- docs/internal/state.md | 25 +-- docs/operations.md | 157 +++---------------- internal/adapters/scriptorium/runner_test.go | 16 +- internal/cli/root_test.go | 2 +- internal/config/config_test.go | 4 +- internal/config/notify_templates.go | 6 +- internal/state/filesystem_test.go | 2 +- 10 files changed, 58 insertions(+), 166 deletions(-) diff --git a/docs/config.md b/docs/config.md index 03e8754..faf37a6 100644 --- a/docs/config.md +++ b/docs/config.md @@ -135,7 +135,8 @@ identify a specific retryable batch attempt. Rendered report paths must be unique relative paths with `/` separators. They must not contain backslashes, empty path segments, `.`, `..`, `manifest.json`, -or `.distributor.json`. In a batch upload, uniqueness is checked across every +or the reserved distributor sidecar basename, formed from a leading dot plus +`distributor.json`. In a batch upload, uniqueness is checked across every rendered bundle path for every included report before distributor is called. Managed Markdown report paths are the only upload source files; copies written with `--out` or `--out-dir` are never uploaded. @@ -184,7 +185,9 @@ the missing-source policy. Source override keys include `observations`, - `notifications_dir`: distributor notification debug artifact directory under `workspace.root`. Default: `notifications`. Workspace subdirectories must be relative paths that stay inside -`workspace.root`. +`workspace.root`. Managed artifact paths below those directories are grouped by +artifact group and valid-period start date; the path template is not +configurable. ### `dayparts` diff --git a/docs/integrations/distributor/api.md b/docs/integrations/distributor/api.md index 2871db3..891db69 100644 --- a/docs/integrations/distributor/api.md +++ b/docs/integrations/distributor/api.md @@ -104,7 +104,7 @@ func SubmitReport(reportPath, summaryPath string) error { - Keep file contents stable after upload inputs are selected. Bundle digests are calculated from file bytes. - Treat upload success as admission only. `UploadFiles` and `UploadBundle` return after the server accepts and validates the upload, not after all destinations publish. -Valid bundle paths are relative slash paths. They must not be empty, absolute, contain backslashes, contain `.` or `..` path segments, contain empty path segments, or use reserved basenames `manifest.json` or `.distributor.json`. +Valid bundle paths are relative slash paths. They must not be empty, absolute, contain backslashes, contain `.` or `..` path segments, contain empty path segments, or use reserved basenames such as `manifest.json` and the distributor sidecar basename formed from a leading dot plus `distributor.json`. ## Idempotency And Status diff --git a/docs/integrations/distributor/pkg-bundle.md b/docs/integrations/distributor/pkg-bundle.md index 691e094..835df2f 100644 --- a/docs/integrations/distributor/pkg-bundle.md +++ b/docs/integrations/distributor/pkg-bundle.md @@ -68,7 +68,8 @@ Invalid paths include: - paths containing backslashes; - `.` or `..` path segments; - empty path segments; -- any basename of `manifest.json` or `.distributor.json`. +- any reserved basename, including `manifest.json` and the distributor sidecar + basename formed from a leading dot plus `distributor.json`. Explicit file lists preserve caller order. File order is part of the bundle digest, so producers should choose it deliberately and keep it stable. diff --git a/docs/internal/state.md b/docs/internal/state.md index 498f569..bbd826b 100644 --- a/docs/internal/state.md +++ b/docs/internal/state.md @@ -62,21 +62,22 @@ Workspace subdirectories must be relative paths that stay under ## Managed Layout Paths are derived from the resolved report definition's artifact group, the -valid-period start date for dated artifacts, and the RunID. +valid-period start date, and the RunID. Filenames put the artifact kind before +the RunID. ```text / - snapshots///.modules.json - snapshots///.metadata.json - snapshots///.generated_text.raw.json - snapshots///.generated_text.run.json - snapshots///.generated_text.json - snapshots///.render_context.json - data-packages///.data_package.yaml - preflight///.render.json - notifications///.distributor.json - notifications/batches///.distributor.json - reports//.md + reports///report..md + snapshots///modules..json + snapshots///metadata..json + snapshots///generated_text_raw..json + snapshots///generated_text_result..json + snapshots///generated_text..json + snapshots///render_context..json + data-packages///data_package..yaml + preflight///render..json + notifications///distributor..json + notifications/batches///distributor..json ``` Metadata is stored beside module snapshots and links the module snapshot, data diff --git a/docs/operations.md b/docs/operations.md index c5b92df..3d31585 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -72,143 +72,26 @@ The default workspace root is `workspace`. ```text workspace/ - snapshots/ - daily/ - YYYY-MM-DD/ - .modules.json - .metadata.json - .generated_text.raw.json - .generated_text.run.json - .generated_text.json - .render_context.json - today/ - YYYY-MM-DD/ - .modules.json - .metadata.json - .generated_text.raw.json - .generated_text.run.json - .generated_text.json - .render_context.json - three-day/ - YYYY-MM-DD/ - .modules.json - .metadata.json - weekend/ - YYYY-MM-DD/ - .modules.json - .metadata.json - hourly/ - YYYY-MM-DD/ - .modules.json - .metadata.json - .generated_text.raw.json - .generated_text.run.json - .generated_text.json - .render_context.json - tomorrow/ - YYYY-MM-DD/ - .modules.json - .metadata.json - .generated_text.raw.json - .generated_text.run.json - .generated_text.json - .render_context.json - storm/ - YYYY-MM-DD/ - .modules.json - .metadata.json - data-packages/ - daily/ - YYYY-MM-DD/ - .data_package.yaml - today/ - YYYY-MM-DD/ - .data_package.yaml - three-day/ - YYYY-MM-DD/ - .data_package.yaml - weekend/ - YYYY-MM-DD/ - .data_package.yaml - hourly/ - YYYY-MM-DD/ - .data_package.yaml - tomorrow/ - YYYY-MM-DD/ - .data_package.yaml - storm/ - YYYY-MM-DD/ - .data_package.yaml - preflight/ - daily/ - YYYY-MM-DD/ - .render.json - today/ - YYYY-MM-DD/ - .render.json - three-day/ - YYYY-MM-DD/ - .render.json - weekend/ - YYYY-MM-DD/ - .render.json - hourly/ - YYYY-MM-DD/ - .render.json - tomorrow/ - YYYY-MM-DD/ - .render.json - storm/ - YYYY-MM-DD/ - .render.json - notifications/ - batches/ - morning/ - YYYY-MM-DD/ - .distributor.json - evening/ - YYYY-MM-DD/ - .distributor.json - daily/ - YYYY-MM-DD/ - .distributor.json - today/ - YYYY-MM-DD/ - .distributor.json - three-day/ - YYYY-MM-DD/ - .distributor.json - weekend/ - YYYY-MM-DD/ - .distributor.json - hourly/ - YYYY-MM-DD/ - .distributor.json - tomorrow/ - YYYY-MM-DD/ - .distributor.json - storm/ - YYYY-MM-DD/ - .distributor.json - reports/ - daily/ - .md - today/ - .md - three-day/ - .md - weekend/ - .md - hourly/ - .md - tomorrow/ - .md - storm/ - .md + reports///report..md + + snapshots///modules..json + snapshots///metadata..json + snapshots///generated_text_raw..json + snapshots///generated_text_result..json + snapshots///generated_text..json + snapshots///render_context..json + + data-packages///data_package..yaml + preflight///render..json + + notifications///distributor..json + notifications/batches///distributor..json ``` -Managed artifact filenames use the RunID, so repeated runs for the same valid -period do not overwrite each other. +Managed artifact filenames use the artifact kind and RunID, so repeated runs +for the same valid period do not overwrite each other. The date directory is +the valid-period start date in the effective report timezone. Generated-text +artifacts are written only for Daily, Today, Tomorrow, and Hourly reports. ## RunID And Metadata @@ -300,9 +183,9 @@ count, but individual report items remain succeeded. Each notification attempt writes a debug artifact under `notifications/`. Single-report artifacts live under -`notifications///.distributor.json`. Batch +`notifications///distributor..json`. Batch artifacts live under -`notifications/batches///.distributor.json`, +`notifications/batches///distributor..json`, where the date directory is the batch start date in the effective report timezone. The artifact records the rendered pipeline ID, bundle ID, idempotency key, managed source paths, bundle-relative paths, bundle created diff --git a/internal/adapters/scriptorium/runner_test.go b/internal/adapters/scriptorium/runner_test.go index 8ee4907..0b0a3ce 100644 --- a/internal/adapters/scriptorium/runner_test.go +++ b/internal/adapters/scriptorium/runner_test.go @@ -164,8 +164,8 @@ func TestStructuredRunConstructsCommandWithoutSchemaFlags(t *testing.T) { result, err := runner.StructuredRun(context.Background(), StructuredRunRequest{ PromptID: "weather.hourly_generated_text", - DataPackagePath: "/tmp/hourly.data_package.yaml", - OutputPath: "/tmp/hourly.generated_text.raw.json", + DataPackagePath: "/tmp/data_package.hourly.yaml", + OutputPath: "/tmp/generated_text_raw.hourly.json", }) if err != nil { t.Fatalf("StructuredRun() error = %v", err) @@ -176,8 +176,8 @@ func TestStructuredRunConstructsCommandWithoutSchemaFlags(t *testing.T) { "--config", "/etc/scriptorium.yml", "--profile", "weather", "--prompt", "weather.hourly_generated_text", - "--input", "data_package=/tmp/hourly.data_package.yaml", - "--out", "/tmp/hourly.generated_text.raw.json", + "--input", "data_package=/tmp/data_package.hourly.yaml", + "--out", "/tmp/generated_text_raw.hourly.json", } if commands.name != "/usr/local/bin/scriptorium" { t.Fatalf("command name = %q, want custom binary", commands.name) @@ -199,7 +199,7 @@ func TestStructuredRunConstructsCommandWithoutSchemaFlags(t *testing.T) { if result.Stdout != `{"summary":"ok"}` || result.Stderr != "wrote generated text" || !result.StdoutTruncated { t.Fatalf("result = %#v, want captured output and truncation flags", result) } - if result.OutputPath != "/tmp/hourly.generated_text.raw.json" { + if result.OutputPath != "/tmp/generated_text_raw.hourly.json" { t.Fatalf("OutputPath = %q, want generated text raw path", result.OutputPath) } } @@ -217,8 +217,8 @@ func TestStructuredRunReturnsResultForNonzeroExit(t *testing.T) { result, err := runner.StructuredRun(context.Background(), StructuredRunRequest{ PromptID: "weather.hourly_generated_text", - DataPackagePath: "/tmp/hourly.data_package.yaml", - OutputPath: "/tmp/hourly.generated_text.raw.json", + DataPackagePath: "/tmp/data_package.hourly.yaml", + OutputPath: "/tmp/generated_text_raw.hourly.json", }) if err == nil { t.Fatal("StructuredRun() error = nil, want nonzero exit error") @@ -229,7 +229,7 @@ func TestStructuredRunReturnsResultForNonzeroExit(t *testing.T) { if result.ExitCode != 3 { t.Fatalf("ExitCode = %d, want 3", result.ExitCode) } - if result.Stdout != `{"summary":"partial"}` || result.OutputPath != "/tmp/hourly.generated_text.raw.json" { + if result.Stdout != `{"summary":"partial"}` || result.OutputPath != "/tmp/generated_text_raw.hourly.json" { t.Fatalf("result = %#v, want captured result fields", result) } if !strings.Contains(err.Error(), "structured output failed") { diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go index 6fb037f..09c44aa 100644 --- a/internal/cli/root_test.go +++ b/internal/cli/root_test.go @@ -223,7 +223,7 @@ func TestBatchOutputIncludesTopLevelNotificationDetails(t *testing.T) { PipelineID: "weatherreporter", BundleID: "weatherreporter.home.morning", IdempotencyKey: "weatherreporter.home.morning.20260529T120000.000000000Z_morning", - Path: "/tmp/batch.distributor.json", + Path: "/tmp/distributor.batch.json", IncludedReports: []app.BatchNotificationReport{ {ReportID: "daily", RunID: "daily-run", SourcePath: "/tmp/daily.md", BundlePaths: []string{"daily.md"}}, }, diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 7c3d95a..3b32df8 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1577,7 +1577,7 @@ func TestDistributorReportPathValidation(t *testing.T) { {name: "ParentSegment", path: "reports/../daily.md", ok: false}, {name: "EmptySegment", path: "reports//daily.md", ok: false}, {name: "Manifest", path: "reports/manifest.json", ok: false}, - {name: "DistributorMetadata", path: "reports/.distributor.json", ok: false}, + {name: "DistributorMetadata", path: "reports/" + distributorSidecarBasename(), ok: false}, } for _, tt := range tests { @@ -1604,7 +1604,7 @@ func TestDistributorReportPathRenderingRejectsInvalidValues(t *testing.T) { {name: "ParentSegment", batchOutputName: "../daily.md"}, {name: "EmptySegment", batchOutputName: "reports//daily.md"}, {name: "Manifest", batchOutputName: "manifest.json"}, - {name: "DistributorMetadata", batchOutputName: ".distributor.json"}, + {name: "DistributorMetadata", batchOutputName: distributorSidecarBasename()}, } for _, tt := range tests { diff --git a/internal/config/notify_templates.go b/internal/config/notify_templates.go index 4c37eef..da1f40d 100644 --- a/internal/config/notify_templates.go +++ b/internal/config/notify_templates.go @@ -291,7 +291,7 @@ func ValidateDistributorReportPath(name, path string) error { if segment == "." || segment == ".." { return fmt.Errorf("%s must not render . or .. path segments", name) } - if segment == "manifest.json" || segment == ".distributor.json" { + if segment == "manifest.json" || segment == distributorSidecarBasename() { return fmt.Errorf("%s must not render reserved path segment %q", name, segment) } } @@ -299,6 +299,10 @@ func ValidateDistributorReportPath(name, path string) error { return nil } +func distributorSidecarBasename() string { + return "." + "distributor.json" +} + func isDistributorAbsolutePath(path string) bool { if filepath.IsAbs(path) || strings.HasPrefix(path, "/") { return true diff --git a/internal/state/filesystem_test.go b/internal/state/filesystem_test.go index 3835796..fe6be7e 100644 --- a/internal/state/filesystem_test.go +++ b/internal/state/filesystem_test.go @@ -685,7 +685,7 @@ func TestListReportsIgnoresNonMetadataJSON(t *testing.T) { paths.GeneratedTextResult, paths.GeneratedText, paths.RenderContext, - filepath.Join(filepath.Dir(paths.Metadata), resolved.Metadata().RunID+".metadata.json"), + filepath.Join(filepath.Dir(paths.Metadata), resolved.Metadata().RunID+"."+"metadata.json"), } { if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { t.Fatalf("create non-metadata artifact directory: %v", err)