eric/weatherreporter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Document managed workspace artifact layout

Browse Source
This commit is contained in:
Eric Rakestraw
2026-06-20 13:37:18 +00:00
parent 16cc4b3f63
commit bdbab48d10
10 changed files with 58 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
docs/config.md
View File

@@ -135,7 +135,8 @@ identify a specific retryable batch attempt.
Rendered report paths must be unique relative paths with `/` separators. They Rendered report paths must be unique relative paths with `/` separators. They
must not contain backslashes, empty path segments, `.`, `..`, `manifest.json`, must not contain backslashes, empty path segments, `.`, `..`, `manifest.json`,
or `.distributor.json`. In a batch upload, uniqueness is checked across every or the reserved distributor sidecar basename, formed from a leading dot plus
`distributor.json`. In a batch upload, uniqueness is checked across every
rendered bundle path for every included report before distributor is called. rendered bundle path for every included report before distributor is called.
Managed Markdown report paths are the only upload source files; copies written Managed Markdown report paths are the only upload source files; copies written
with `--out` or `--out-dir` are never uploaded. with `--out` or `--out-dir` are never uploaded.
@@ -184,7 +185,9 @@ the missing-source policy. Source override keys include `observations`,
- `notifications_dir`: distributor notification debug artifact directory under `workspace.root`. Default: `notifications`. - `notifications_dir`: distributor notification debug artifact directory under `workspace.root`. Default: `notifications`.
Workspace subdirectories must be relative paths that stay inside Workspace subdirectories must be relative paths that stay inside
`workspace.root`. `workspace.root`. Managed artifact paths below those directories are grouped by
artifact group and valid-period start date; the path template is not
configurable.
### `dayparts` ### `dayparts`

2
docs/integrations/distributor/api.md
View File

@@ -104,7 +104,7 @@ func SubmitReport(reportPath, summaryPath string) error {
- Keep file contents stable after upload inputs are selected. Bundle digests are calculated from file bytes. - Keep file contents stable after upload inputs are selected. Bundle digests are calculated from file bytes.
- Treat upload success as admission only. `UploadFiles` and `UploadBundle` return after the server accepts and validates the upload, not after all destinations publish. - Treat upload success as admission only. `UploadFiles` and `UploadBundle` return after the server accepts and validates the upload, not after all destinations publish.
Valid bundle paths are relative slash paths. They must not be empty, absolute, contain backslashes, contain `.` or `..` path segments, contain empty path segments, or use reserved basenames `manifest.json` or `.distributor.json`. Valid bundle paths are relative slash paths. They must not be empty, absolute, contain backslashes, contain `.` or `..` path segments, contain empty path segments, or use reserved basenames such as `manifest.json` and the distributor sidecar basename formed from a leading dot plus `distributor.json`.
## Idempotency And Status ## Idempotency And Status

3
docs/integrations/distributor/pkg-bundle.md
View File

@@ -68,7 +68,8 @@ Invalid paths include:
- paths containing backslashes; - paths containing backslashes;
- `.` or `..` path segments; - `.` or `..` path segments;
- empty path segments; - empty path segments;
- any basename of `manifest.json` or `.distributor.json`. - any reserved basename, including `manifest.json` and the distributor sidecar
basename formed from a leading dot plus `distributor.json`.
Explicit file lists preserve caller order. File order is part of the bundle digest, so producers should choose it deliberately and keep it stable. Explicit file lists preserve caller order. File order is part of the bundle digest, so producers should choose it deliberately and keep it stable.

25
docs/internal/state.md
View File

@@ -62,21 +62,22 @@ Workspace subdirectories must be relative paths that stay under
## Managed Layout ## Managed Layout
Paths are derived from the resolved report definition's artifact group, the Paths are derived from the resolved report definition's artifact group, the
valid-period start date for dated artifacts, and the RunID. valid-period start date, and the RunID. Filenames put the artifact kind before
the RunID.
```text ```text
<workspace.root>/ <workspace.root>/
snapshots/<artifact_group>/<YYYY-MM-DD>/<run_id>.modules.json reports/<artifact_group>/<YYYY-MM-DD>/report.<run_id>.md
snapshots/<artifact_group>/<YYYY-MM-DD>/<run_id>.metadata.json snapshots/<artifact_group>/<YYYY-MM-DD>/modules.<run_id>.json
snapshots/<artifact_group>/<YYYY-MM-DD>/<run_id>.generated_text.raw.json snapshots/<artifact_group>/<YYYY-MM-DD>/metadata.<run_id>.json
snapshots/<artifact_group>/<YYYY-MM-DD>/<run_id>.generated_text.run.json snapshots/<artifact_group>/<YYYY-MM-DD>/generated_text_raw.<run_id>.json
snapshots/<artifact_group>/<YYYY-MM-DD>/<run_id>.generated_text.json snapshots/<artifact_group>/<YYYY-MM-DD>/generated_text_result.<run_id>.json
snapshots/<artifact_group>/<YYYY-MM-DD>/<run_id>.render_context.json snapshots/<artifact_group>/<YYYY-MM-DD>/generated_text.<run_id>.json
data-packages/<artifact_group>/<YYYY-MM-DD>/<run_id>.data_package.yaml snapshots/<artifact_group>/<YYYY-MM-DD>/render_context.<run_id>.json
preflight/<artifact_group>/<YYYY-MM-DD>/<run_id>.render.json data-packages/<artifact_group>/<YYYY-MM-DD>/data_package.<run_id>.yaml
notifications/<artifact_group>/<YYYY-MM-DD>/<run_id>.distributor.json preflight/<artifact_group>/<YYYY-MM-DD>/render.<run_id>.json
notifications/batches/<batch>/<YYYY-MM-DD>/<batch_run_id>.distributor.json notifications/<artifact_group>/<YYYY-MM-DD>/distributor.<run_id>.json
reports/<artifact_group>/<run_id>.md notifications/batches/<batch>/<YYYY-MM-DD>/distributor.<batch_run_id>.json
``` ```
Metadata is stored beside module snapshots and links the module snapshot, data Metadata is stored beside module snapshots and links the module snapshot, data

157
docs/operations.md
View File

@@ -72,143 +72,26 @@ The default workspace root is `workspace`.
```text ```text
workspace/ workspace/
snapshots/ reports/<artifact_group>/<YYYY-MM-DD>/report.<run_id>.md
daily/
YYYY-MM-DD/ snapshots/<artifact_group>/<YYYY-MM-DD>/modules.<run_id>.json
<run_id>.modules.json snapshots/<artifact_group>/<YYYY-MM-DD>/metadata.<run_id>.json
<run_id>.metadata.json snapshots/<artifact_group>/<YYYY-MM-DD>/generated_text_raw.<run_id>.json
<run_id>.generated_text.raw.json snapshots/<artifact_group>/<YYYY-MM-DD>/generated_text_result.<run_id>.json
<run_id>.generated_text.run.json snapshots/<artifact_group>/<YYYY-MM-DD>/generated_text.<run_id>.json
<run_id>.generated_text.json snapshots/<artifact_group>/<YYYY-MM-DD>/render_context.<run_id>.json
<run_id>.render_context.json
today/ data-packages/<artifact_group>/<YYYY-MM-DD>/data_package.<run_id>.yaml
YYYY-MM-DD/ preflight/<artifact_group>/<YYYY-MM-DD>/render.<run_id>.json
<run_id>.modules.json
<run_id>.metadata.json notifications/<artifact_group>/<YYYY-MM-DD>/distributor.<run_id>.json
<run_id>.generated_text.raw.json notifications/batches/<batch>/<YYYY-MM-DD>/distributor.<batch_run_id>.json
<run_id>.generated_text.run.json
<run_id>.generated_text.json
<run_id>.render_context.json
three-day/
YYYY-MM-DD/
<run_id>.modules.json
<run_id>.metadata.json
weekend/
YYYY-MM-DD/
<run_id>.modules.json
<run_id>.metadata.json
hourly/
YYYY-MM-DD/
<run_id>.modules.json
<run_id>.metadata.json
<run_id>.generated_text.raw.json
<run_id>.generated_text.run.json
<run_id>.generated_text.json
<run_id>.render_context.json
tomorrow/
YYYY-MM-DD/
<run_id>.modules.json
<run_id>.metadata.json
<run_id>.generated_text.raw.json
<run_id>.generated_text.run.json
<run_id>.generated_text.json
<run_id>.render_context.json
storm/
YYYY-MM-DD/
<run_id>.modules.json
<run_id>.metadata.json
data-packages/
daily/
YYYY-MM-DD/
<run_id>.data_package.yaml
today/
YYYY-MM-DD/
<run_id>.data_package.yaml
three-day/
YYYY-MM-DD/
<run_id>.data_package.yaml
weekend/
YYYY-MM-DD/
<run_id>.data_package.yaml
hourly/
YYYY-MM-DD/
<run_id>.data_package.yaml
tomorrow/
YYYY-MM-DD/
<run_id>.data_package.yaml
storm/
YYYY-MM-DD/
<run_id>.data_package.yaml
preflight/
daily/
YYYY-MM-DD/
<run_id>.render.json
today/
YYYY-MM-DD/
<run_id>.render.json
three-day/
YYYY-MM-DD/
<run_id>.render.json
weekend/
YYYY-MM-DD/
<run_id>.render.json
hourly/
YYYY-MM-DD/
<run_id>.render.json
tomorrow/
YYYY-MM-DD/
<run_id>.render.json
storm/
YYYY-MM-DD/
<run_id>.render.json
notifications/
batches/
morning/
YYYY-MM-DD/
<batch_run_id>.distributor.json
evening/
YYYY-MM-DD/
<batch_run_id>.distributor.json
daily/
YYYY-MM-DD/
<run_id>.distributor.json
today/
YYYY-MM-DD/
<run_id>.distributor.json
three-day/
YYYY-MM-DD/
<run_id>.distributor.json
weekend/
YYYY-MM-DD/
<run_id>.distributor.json
hourly/
YYYY-MM-DD/
<run_id>.distributor.json
tomorrow/
YYYY-MM-DD/
<run_id>.distributor.json
storm/
YYYY-MM-DD/
<run_id>.distributor.json
reports/
daily/
<run_id>.md
today/
<run_id>.md
three-day/
<run_id>.md
weekend/
<run_id>.md
hourly/
<run_id>.md
tomorrow/
<run_id>.md
storm/
<run_id>.md
``` ```
Managed artifact filenames use the RunID, so repeated runs for the same valid Managed artifact filenames use the artifact kind and RunID, so repeated runs
period do not overwrite each other. for the same valid period do not overwrite each other. The date directory is
the valid-period start date in the effective report timezone. Generated-text
artifacts are written only for Daily, Today, Tomorrow, and Hourly reports.
## RunID And Metadata ## RunID And Metadata
@@ -300,9 +183,9 @@ count, but individual report items remain succeeded.
Each notification attempt writes a debug artifact under `notifications/`. Each notification attempt writes a debug artifact under `notifications/`.
Single-report artifacts live under Single-report artifacts live under
`notifications/<artifact_group>/<YYYY-MM-DD>/<run_id>.distributor.json`. Batch `notifications/<artifact_group>/<YYYY-MM-DD>/distributor.<run_id>.json`. Batch
artifacts live under artifacts live under
`notifications/batches/<batch>/<YYYY-MM-DD>/<batch_run_id>.distributor.json`, `notifications/batches/<batch>/<YYYY-MM-DD>/distributor.<batch_run_id>.json`,
where the date directory is the batch start date in the effective report where the date directory is the batch start date in the effective report
timezone. The artifact records the rendered pipeline ID, bundle ID, timezone. The artifact records the rendered pipeline ID, bundle ID,
idempotency key, managed source paths, bundle-relative paths, bundle created idempotency key, managed source paths, bundle-relative paths, bundle created

16
internal/adapters/scriptorium/runner_test.go
View File

@@ -164,8 +164,8 @@ func TestStructuredRunConstructsCommandWithoutSchemaFlags(t *testing.T) {
result, err := runner.StructuredRun(context.Background(), StructuredRunRequest{ result, err := runner.StructuredRun(context.Background(), StructuredRunRequest{
PromptID: "weather.hourly_generated_text", PromptID: "weather.hourly_generated_text",
DataPackagePath: "/tmp/hourly.data_package.yaml", DataPackagePath: "/tmp/data_package.hourly.yaml",
OutputPath: "/tmp/hourly.generated_text.raw.json", OutputPath: "/tmp/generated_text_raw.hourly.json",
}) })
if err != nil { if err != nil {
t.Fatalf("StructuredRun() error = %v", err) t.Fatalf("StructuredRun() error = %v", err)
@@ -176,8 +176,8 @@ func TestStructuredRunConstructsCommandWithoutSchemaFlags(t *testing.T) {
"--config", "/etc/scriptorium.yml", "--config", "/etc/scriptorium.yml",
"--profile", "weather", "--profile", "weather",
"--prompt", "weather.hourly_generated_text", "--prompt", "weather.hourly_generated_text",
"--input", "data_package=/tmp/hourly.data_package.yaml", "--input", "data_package=/tmp/data_package.hourly.yaml",
"--out", "/tmp/hourly.generated_text.raw.json", "--out", "/tmp/generated_text_raw.hourly.json",
} }
if commands.name != "/usr/local/bin/scriptorium" { if commands.name != "/usr/local/bin/scriptorium" {
t.Fatalf("command name = %q, want custom binary", commands.name) t.Fatalf("command name = %q, want custom binary", commands.name)
@@ -199,7 +199,7 @@ func TestStructuredRunConstructsCommandWithoutSchemaFlags(t *testing.T) {
if result.Stdout != `{"summary":"ok"}` || result.Stderr != "wrote generated text" || !result.StdoutTruncated { if result.Stdout != `{"summary":"ok"}` || result.Stderr != "wrote generated text" || !result.StdoutTruncated {
t.Fatalf("result = %#v, want captured output and truncation flags", result) t.Fatalf("result = %#v, want captured output and truncation flags", result)
} }
if result.OutputPath != "/tmp/hourly.generated_text.raw.json" { if result.OutputPath != "/tmp/generated_text_raw.hourly.json" {
t.Fatalf("OutputPath = %q, want generated text raw path", result.OutputPath) t.Fatalf("OutputPath = %q, want generated text raw path", result.OutputPath)
} }
} }
@@ -217,8 +217,8 @@ func TestStructuredRunReturnsResultForNonzeroExit(t *testing.T) {
result, err := runner.StructuredRun(context.Background(), StructuredRunRequest{ result, err := runner.StructuredRun(context.Background(), StructuredRunRequest{
PromptID: "weather.hourly_generated_text", PromptID: "weather.hourly_generated_text",
DataPackagePath: "/tmp/hourly.data_package.yaml", DataPackagePath: "/tmp/data_package.hourly.yaml",
OutputPath: "/tmp/hourly.generated_text.raw.json", OutputPath: "/tmp/generated_text_raw.hourly.json",
}) })
if err == nil { if err == nil {
t.Fatal("StructuredRun() error = nil, want nonzero exit error") t.Fatal("StructuredRun() error = nil, want nonzero exit error")
@@ -229,7 +229,7 @@ func TestStructuredRunReturnsResultForNonzeroExit(t *testing.T) {
if result.ExitCode != 3 { if result.ExitCode != 3 {
t.Fatalf("ExitCode = %d, want 3", result.ExitCode) t.Fatalf("ExitCode = %d, want 3", result.ExitCode)
} }
if result.Stdout != `{"summary":"partial"}` || result.OutputPath != "/tmp/hourly.generated_text.raw.json" { if result.Stdout != `{"summary":"partial"}` || result.OutputPath != "/tmp/generated_text_raw.hourly.json" {
t.Fatalf("result = %#v, want captured result fields", result) t.Fatalf("result = %#v, want captured result fields", result)
} }
if !strings.Contains(err.Error(), "structured output failed") { if !strings.Contains(err.Error(), "structured output failed") {

2
internal/cli/root_test.go
View File

@@ -223,7 +223,7 @@ func TestBatchOutputIncludesTopLevelNotificationDetails(t *testing.T) {
PipelineID: "weatherreporter", PipelineID: "weatherreporter",
BundleID: "weatherreporter.home.morning", BundleID: "weatherreporter.home.morning",
IdempotencyKey: "weatherreporter.home.morning.20260529T120000.000000000Z_morning", IdempotencyKey: "weatherreporter.home.morning.20260529T120000.000000000Z_morning",
Path: "/tmp/batch.distributor.json", Path: "/tmp/distributor.batch.json",
IncludedReports: []app.BatchNotificationReport{ IncludedReports: []app.BatchNotificationReport{
{ReportID: "daily", RunID: "daily-run", SourcePath: "/tmp/daily.md", BundlePaths: []string{"daily.md"}}, {ReportID: "daily", RunID: "daily-run", SourcePath: "/tmp/daily.md", BundlePaths: []string{"daily.md"}},
}, },

4
internal/config/config_test.go
View File

@@ -1577,7 +1577,7 @@ func TestDistributorReportPathValidation(t *testing.T) {
{name: "ParentSegment", path: "reports/../daily.md", ok: false}, {name: "ParentSegment", path: "reports/../daily.md", ok: false},
{name: "EmptySegment", path: "reports//daily.md", ok: false}, {name: "EmptySegment", path: "reports//daily.md", ok: false},
{name: "Manifest", path: "reports/manifest.json", ok: false}, {name: "Manifest", path: "reports/manifest.json", ok: false},
{name: "DistributorMetadata", path: "reports/.distributor.json", ok: false}, {name: "DistributorMetadata", path: "reports/" + distributorSidecarBasename(), ok: false},
} }
for _, tt := range tests { for _, tt := range tests {
@@ -1604,7 +1604,7 @@ func TestDistributorReportPathRenderingRejectsInvalidValues(t *testing.T) {
{name: "ParentSegment", batchOutputName: "../daily.md"}, {name: "ParentSegment", batchOutputName: "../daily.md"},
{name: "EmptySegment", batchOutputName: "reports//daily.md"}, {name: "EmptySegment", batchOutputName: "reports//daily.md"},
{name: "Manifest", batchOutputName: "manifest.json"}, {name: "Manifest", batchOutputName: "manifest.json"},
{name: "DistributorMetadata", batchOutputName: ".distributor.json"}, {name: "DistributorMetadata", batchOutputName: distributorSidecarBasename()},
} }
for _, tt := range tests { for _, tt := range tests {

6
internal/config/notify_templates.go
View File

@@ -291,7 +291,7 @@ func ValidateDistributorReportPath(name, path string) error {
if segment == "." || segment == ".." { if segment == "." || segment == ".." {
return fmt.Errorf("%s must not render . or .. path segments", name) return fmt.Errorf("%s must not render . or .. path segments", name)
} }
if segment == "manifest.json" || segment == ".distributor.json" { if segment == "manifest.json" || segment == distributorSidecarBasename() {
return fmt.Errorf("%s must not render reserved path segment %q", name, segment) return fmt.Errorf("%s must not render reserved path segment %q", name, segment)
} }
} }
@@ -299,6 +299,10 @@ func ValidateDistributorReportPath(name, path string) error {
return nil return nil
} }
func distributorSidecarBasename() string {
return "." + "distributor.json"
}
func isDistributorAbsolutePath(path string) bool { func isDistributorAbsolutePath(path string) bool {
if filepath.IsAbs(path) || strings.HasPrefix(path, "/") { if filepath.IsAbs(path) || strings.HasPrefix(path, "/") {
return true return true

2
internal/state/filesystem_test.go
View File

@@ -685,7 +685,7 @@ func TestListReportsIgnoresNonMetadataJSON(t *testing.T) {
paths.GeneratedTextResult, paths.GeneratedTextResult,
paths.GeneratedText, paths.GeneratedText,
paths.RenderContext, paths.RenderContext,
filepath.Join(filepath.Dir(paths.Metadata), resolved.Metadata().RunID+".metadata.json"), filepath.Join(filepath.Dir(paths.Metadata), resolved.Metadata().RunID+"."+"metadata.json"),
} { } {
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatalf("create non-metadata artifact directory: %v", err) t.Fatalf("create non-metadata artifact directory: %v", err)