Compare commits
120 Commits
v0.7.0
...
5620fc5bcf
| Author | SHA1 | Date | |
|---|---|---|---|
| 5620fc5bcf | |||
| be57e675e0 | |||
| 3971443831 | |||
| a6b0c33e9f | |||
| 96b886e711 | |||
| 7d584ee6cd | |||
| 572a112c31 | |||
| ea87c335d6 | |||
| 7169ff04df | |||
| ef1f650bc0 | |||
| 0d02cb9fa0 | |||
| 0299b128cf | |||
| d723384888 | |||
| 54228055c8 | |||
| 23ed716450 | |||
| ab59bab044 | |||
| 71395bb076 | |||
| 79737edf79 | |||
| df2c765b7f | |||
| f050b9dd54 | |||
| 9c9cb54339 | |||
| 7657ec3ad6 | |||
| cee52aa092 | |||
| e920f3a8d5 | |||
| 591c529a09 | |||
| 7324c5a686 | |||
| d0936fb022 | |||
| 2aa074c5cf | |||
| 782d0cf3b9 | |||
| 083c01cfa0 | |||
| 2937696024 | |||
| b817a5b772 | |||
| 3022f20beb | |||
| ca1ded1821 | |||
| 3752f3ed28 | |||
| 870c2d69d5 | |||
| 135407ba7c | |||
| 228c348e42 | |||
| a813bd5a50 | |||
| d8f58dce31 | |||
| 7111edeca4 | |||
| 3aae4bbb12 | |||
| b29d8eeb50 | |||
| dffb432537 | |||
| 2dd38c7913 | |||
| bc2ade38d9 | |||
| 5be831eb13 | |||
| cae4d99a89 | |||
| e09dc0512d | |||
| ae82bc1ce0 | |||
| 01eb7aa1aa | |||
| 2ca700195c | |||
| 2b08c34539 | |||
| 79f1fc1e09 | |||
| 9c753270bd | |||
| b907cb01aa | |||
| 7824afd4a5 | |||
| 2a4e1e912c | |||
| dd03c09d75 | |||
| 5bc8e8683f | |||
| 648001a8fe | |||
| 6684774f52 | |||
| f3b63bd5e5 | |||
| 23d6470b0f | |||
| 128449040f | |||
| 02ab106ade | |||
| c128970f58 | |||
| d001baa660 | |||
| c5c35cd3b4 | |||
| 574b1cde6c | |||
| ebb21b9201 | |||
| 958f446387 | |||
| 86caf4b222 | |||
| e38ed8ba97 | |||
| 3e79cf4724 | |||
| 859ae1ae10 | |||
| c63ecbab32 | |||
| 8480b74283 | |||
| 087869f7fa | |||
| 2b2a314d65 | |||
| 08b0f4edc5 | |||
| 571a289296 | |||
| 9f80635b42 | |||
| 11a3e174b6 | |||
| 9c5e5d6dc1 | |||
| c4e87f58c7 | |||
| 37daab7857 | |||
| 2356688cb9 | |||
| 1054b64d9f | |||
| 01fb02426c | |||
| 7dc79e052f | |||
| cb525c0f72 | |||
| 622677d038 | |||
| 550288e008 | |||
| e58e545686 | |||
| 6ff54c5a0f | |||
| 924b5d15c6 | |||
| b065663180 | |||
| 3ba564b00f | |||
| a3986cf0d6 | |||
| 539601bd16 | |||
| 6ca1c8d6b0 | |||
| 4b7b50981b | |||
| 33f7ae8f2e | |||
| d5a9ad38f8 | |||
| 6fbefb9867 | |||
| 1665359486 | |||
| 03f2543927 | |||
| fe9c348092 | |||
| f7f8f1a949 | |||
| d40c91acde | |||
| ed4dcf1ef7 | |||
| 24cce49a70 | |||
| 1e6db89dd4 | |||
| 0454296c81 | |||
| 58c6ab2d54 | |||
| 7995c41675 | |||
| 62551d43a0 | |||
| 8395c12dd3 | |||
| dc8e1040f2 |
5
.gitignore
vendored
5
.gitignore
vendored
@@ -2,6 +2,8 @@
|
||||
.codex
|
||||
AGENTS.md
|
||||
|
||||
.DS_Store
|
||||
|
||||
# ---> Go
|
||||
# If you prefer the allow list template instead of the deny list, see community template:
|
||||
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
|
||||
@@ -22,6 +24,9 @@ AGENTS.md
|
||||
# Dependency directories (remove the comment below to include it)
|
||||
# vendor/
|
||||
|
||||
# Go cache
|
||||
.gocache
|
||||
|
||||
# Go workspace file
|
||||
go.work
|
||||
go.work.sum
|
||||
|
||||
283
README.md
283
README.md
@@ -1,279 +1,22 @@
|
||||
# narratio
|
||||
|
||||
`narratio` is a Go orchestration application for processing D&D session audio into transcripts and generated artifacts.
|
||||
Narratio is a stage-driven Go orchestrator for turning D&D session audio into polished transcripts and generated artifacts.
|
||||
|
||||
## Current Implementation
|
||||
|
||||
Implemented now:
|
||||
|
||||
- strict config loading/validation (`pipeline.yml` and `session.yml`)
|
||||
- local workspace/session layout, locking, and manifest persistence
|
||||
- resumable stage control (`run`, `plan`, `resume`, `run-stage`, `status`)
|
||||
- real `prepare`, `transcribe`, `merge`, `polish`, `normalize`, `trim`, and `analyze` stages
|
||||
- real WhisperX, Seriatim, and Audita adapters
|
||||
- real Scriptorium subprocess adapter
|
||||
- optional Scriptorium render diagnostics (`render_debug`)
|
||||
|
||||
Not implemented yet:
|
||||
|
||||
- `archive` stage behavior
|
||||
- `notify` stage behavior
|
||||
- additional analyze artifacts beyond `session_recap`
|
||||
- generic DAG orchestration
|
||||
|
||||
## Config Files
|
||||
|
||||
Narratio expects two YAML files:
|
||||
|
||||
- `pipeline.yml`: pipeline/workspace settings
|
||||
- `session.yml`: per-session settings
|
||||
|
||||
YAML decoding is strict (`KnownFields(true)`), so unknown fields fail fast.
|
||||
|
||||
## Canonical Stage Order
|
||||
|
||||
1. `prepare`
|
||||
2. `transcribe`
|
||||
3. `merge`
|
||||
4. `polish`
|
||||
5. `normalize`
|
||||
6. `trim`
|
||||
7. `analyze`
|
||||
8. `archive`
|
||||
9. `notify`
|
||||
|
||||
## Transcript Tiers
|
||||
|
||||
- `transcripts/merged.json`: canonical deterministic merged transcript from Seriatim merge
|
||||
- `transcripts/processed.json`: full raw Audita-polished transcript output
|
||||
- `transcripts/normalized.json`: Seriatim-normalized transcript from the normalize stage
|
||||
- `transcripts/trimmed.json`: gameplay-only normalized polished transcript from trim stage
|
||||
|
||||
## Normalize Configuration
|
||||
|
||||
`pipeline.normalize` is optional. When omitted, Narratio defaults to:
|
||||
|
||||
- `output_path: transcripts/normalized.json`
|
||||
- `output_schema: seriatim-intermediate`
|
||||
- `report: true`
|
||||
|
||||
Allowed `normalize.output_schema` values:
|
||||
|
||||
- `seriatim-minimal`
|
||||
- `seriatim-intermediate`
|
||||
- `seriatim-full`
|
||||
|
||||
`normalize.output_path` is treated as session-workdir-relative when not absolute.
|
||||
|
||||
Normalize stage behavior summary:
|
||||
|
||||
- normalize runs after `polish` and before `trim`
|
||||
- normalize resolves `transcripts/processed.json`
|
||||
- normalize runs Seriatim `normalize` to produce `transcripts/normalized.json`
|
||||
- normalize diagnostics are written to:
|
||||
- `artifacts/seriatim.normalize.report.json` (when enabled)
|
||||
- `logs/seriatim.normalize.stdout.log`
|
||||
- `logs/seriatim.normalize.stderr.log`
|
||||
- `config/seriatim.normalize.generated.yml`
|
||||
|
||||
## Trim Configuration
|
||||
|
||||
`pipeline.trim` is optional. If omitted, no trim config is loaded. If `trim.enabled` is omitted, it defaults to `false`.
|
||||
|
||||
When `trim.enabled: true`:
|
||||
|
||||
- `trim.output_path` is required
|
||||
- `trim.bounds.prompt_id` is required
|
||||
- `trim.bounds.transcript_input_name` is required
|
||||
- `trim.bounds.output_path` is required
|
||||
- `trim.bounds.timeout` must be a valid Go duration when provided
|
||||
- `trim.bounds.render_debug: true` requires `trim.bounds.render_output_path`
|
||||
- `trim.bounds.profile_id` may be empty to use the prompt default profile
|
||||
|
||||
Trim paths are treated as session-workdir-relative when not absolute.
|
||||
|
||||
Example trim config:
|
||||
|
||||
```yaml
|
||||
trim:
|
||||
enabled: true
|
||||
output_path: "transcripts/trimmed.json"
|
||||
bounds:
|
||||
prompt_id: "dnd_session.bounds"
|
||||
profile_id: ""
|
||||
transcript_input_name: "transcript"
|
||||
output_path: "artifacts/session_bounds.json"
|
||||
timeout: "10m"
|
||||
render_debug: false
|
||||
render_output_path: "artifacts/session_bounds.render.json"
|
||||
seriatim:
|
||||
report: false
|
||||
```
|
||||
|
||||
Trim behavior summary:
|
||||
|
||||
- trim discovers and validates `transcripts/normalized.json`
|
||||
- trim uses Scriptorium bounds (`dnd_session.bounds` by example config) to produce `artifacts/session_bounds.json`
|
||||
- bounds IDs are validated against the same normalized transcript ID space that Seriatim trim will consume
|
||||
- trim converts bounds to Seriatim keep selector (for example `10-868`) and runs Seriatim trim
|
||||
- if trim is disabled, Narratio copies normalized transcript to trimmed transcript and records `trim_action=copy_disabled`
|
||||
|
||||
Trim outputs and diagnostics:
|
||||
|
||||
- `artifacts/session_bounds.json`
|
||||
- `transcripts/trimmed.json`
|
||||
- `logs/scriptorium.bounds.stdout.log`
|
||||
- `logs/scriptorium.bounds.stderr.log`
|
||||
- `config/scriptorium.bounds.generated.yml`
|
||||
- `logs/seriatim.trim.stdout.log`
|
||||
- `logs/seriatim.trim.stderr.log`
|
||||
- `config/seriatim.trim.generated.yml`
|
||||
- optional bounds render-debug outputs:
|
||||
- `artifacts/session_bounds.render.json`
|
||||
- `logs/scriptorium.bounds.render.stdout.log`
|
||||
- `logs/scriptorium.bounds.render.stderr.log`
|
||||
- `config/scriptorium.bounds.render.generated.yml`
|
||||
|
||||
Render-debug files are diagnostics and are not treated as canonical stage output artifact refs.
|
||||
|
||||
## Scriptorium Configuration
|
||||
|
||||
`pipeline.scriptorium` is optional. When present, Narratio validates and uses it for analyze-stage artifact generation.
|
||||
|
||||
Key points:
|
||||
|
||||
- `scriptorium.binary` is required when section is present
|
||||
- `scriptorium.config_path` is optional
|
||||
- `scriptorium.timeout` defaults to `10m` when omitted
|
||||
- `scriptorium.render_debug` enables render diagnostics globally
|
||||
- artifacts are configured under `scriptorium.artifacts` (map shape supports multiple artifacts)
|
||||
- enabled artifacts require `prompt_id` and `output_path`
|
||||
- artifact `render_debug` may override global render setting
|
||||
- `vars` currently support boolean and string values
|
||||
|
||||
Example `session_recap` artifact definition:
|
||||
|
||||
```yaml
|
||||
scriptorium:
|
||||
binary: "scriptorium"
|
||||
config_path: "/etc/scriptorium/config.yml"
|
||||
timeout: "10m"
|
||||
render_debug: false
|
||||
|
||||
artifacts:
|
||||
session_recap:
|
||||
enabled: true
|
||||
prompt_id: "dnd.session_recap"
|
||||
profile_id: "local-quality" # optional
|
||||
output_path: "artifacts/session_recap.md"
|
||||
timeout: "10m"
|
||||
# render_debug: true # optional per-artifact override
|
||||
|
||||
inputs:
|
||||
transcript:
|
||||
source: "trimmed_transcript"
|
||||
required: true
|
||||
|
||||
previous_recap:
|
||||
source: "previous_session_artifact"
|
||||
artifact: "session_recap"
|
||||
path: "" # optional; set when available
|
||||
required: false
|
||||
|
||||
vars:
|
||||
session_id: true
|
||||
session_date: true
|
||||
campaign_name: true
|
||||
previous_session_id: true
|
||||
output_kind: "session_recap"
|
||||
```
|
||||
|
||||
Prompt IDs and profile IDs are configuration values. They are not hardcoded in analyze-stage logic.
|
||||
|
||||
Do not put secrets in `pipeline.yml`. If API-key behavior is configured, use env var names only.
|
||||
|
||||
## Scriptorium Runtime Behavior
|
||||
|
||||
Narratio integrates with Scriptorium through the public CLI subprocess contract:
|
||||
|
||||
- generation: `scriptorium run`
|
||||
- diagnostics/testing: `scriptorium render --format json` when `render_debug` is enabled
|
||||
|
||||
For the initial implementation, only `session_recap` generation is supported.
|
||||
|
||||
Analyze-stage session recap behavior:
|
||||
|
||||
- available transcript input sources for configured artifacts: `processed_transcript`, `normalized_transcript`, `trimmed_transcript`
|
||||
- session recap should use gameplay-only transcript input (`source: trimmed_transcript`)
|
||||
- Narratio resolves `trimmed_transcript` from trim manifest output (`transcript_trimmed`) or fallback `transcripts/trimmed.json`
|
||||
- Narratio resolves `normalized_transcript` from normalize manifest output (`transcript_normalized`) or fallback `transcripts/normalized.json`
|
||||
- missing trimmed transcript fails clearly and advises running trim stage first
|
||||
- `normalized_transcript` is the preferred full-transcript source for future table/meta-analysis artifacts
|
||||
- `processed_transcript` remains supported for advanced/debug use cases
|
||||
- optionally includes `previous_recap` when configured and resolvable
|
||||
- omits optional previous recap when unavailable
|
||||
- fails if required inputs are missing
|
||||
- validates output file exists and is non-empty
|
||||
|
||||
Expected session output paths:
|
||||
|
||||
- `artifacts/session_recap.md`
|
||||
- `logs/scriptorium.session_recap.stdout.log`
|
||||
- `logs/scriptorium.session_recap.stderr.log`
|
||||
- `config/scriptorium.session_recap.generated.yml`
|
||||
- `artifacts/session_recap.render.json` when render diagnostics are enabled
|
||||
|
||||
## Examples
|
||||
|
||||
Starter files:
|
||||
|
||||
- `examples/pipeline.minimal.yml`
|
||||
- `examples/session.minimal.yml`
|
||||
- `examples/speakers.yml`
|
||||
|
||||
## Commands
|
||||
|
||||
Run tests:
|
||||
It runs a deterministic workflow across `prepare`, `transcribe`, `merge`, `polish`, `normalize`, `trim`, `analyze`, and `publish`, with manifest-driven resume and restore support.
|
||||
|
||||
```bash
|
||||
go test ./...
|
||||
narratio run 2026-04-04
|
||||
```
|
||||
|
||||
Plan a run:
|
||||
This requires resolvable `pipeline.yml`, `campaign.yml`, and concrete `session.yml` (or explicit config flags).
|
||||
|
||||
```bash
|
||||
go run ./cmd/narratio plan --config examples/pipeline.minimal.yml --session examples/session.minimal.yml
|
||||
```
|
||||
## Documentation
|
||||
|
||||
Run full pipeline:
|
||||
|
||||
```bash
|
||||
go run ./cmd/narratio run --config examples/pipeline.minimal.yml --session examples/session.minimal.yml
|
||||
```
|
||||
|
||||
Run analyze only:
|
||||
|
||||
```bash
|
||||
go run ./cmd/narratio run-stage --config examples/pipeline.minimal.yml --session examples/session.minimal.yml analyze
|
||||
```
|
||||
|
||||
## Operational Note
|
||||
|
||||
Checksum-based stale detection is not implemented yet.
|
||||
|
||||
If prepared inputs or prompt/runtime config change, rerun the appropriate upstream stages before relying on downstream artifacts.
|
||||
|
||||
Examples:
|
||||
|
||||
- glossary/autocorrect/speaker-context changes: rerun at least `merge`, `polish`, `normalize`, `trim`, and `analyze`
|
||||
- trim bounds prompt/profile/config changes: rerun at least `normalize`, `trim`, and `analyze`
|
||||
- session recap prompt/profile/input-source changes: rerun `analyze`
|
||||
|
||||
## Roadmap
|
||||
|
||||
Near-term roadmap:
|
||||
|
||||
- extend analyze to additional configured artifacts
|
||||
- support workflows where later artifacts consume earlier generated artifacts
|
||||
- keep orchestration explicit without a generic DAG engine
|
||||
- implement archive and notify backends
|
||||
- [CLI Reference](docs/cli.md)
|
||||
- [Configuration](docs/config.md)
|
||||
- [Operations](docs/operations.md)
|
||||
- [Troubleshooting](docs/troubleshooting.md)
|
||||
- [Internal Component Contracts](docs/internal/README.md)
|
||||
- [Development Guide](docs/policy/development.md)
|
||||
- [Architecture Principles](docs/policy/architecture.md)
|
||||
- [Maintained Examples](examples/)
|
||||
|
||||
316
architecture.md
316
architecture.md
@@ -1,316 +0,0 @@
|
||||
# Narratio Architecture
|
||||
|
||||
## 1. Purpose
|
||||
|
||||
`narratio` is a Go orchestrator for D&D session processing. It runs a stage-based local pipeline from audio input through transcript processing and artifact generation, with manifest-based skip/force/resume behavior.
|
||||
|
||||
Narratio integrates with Scriptorium through the **public CLI** (`scriptorium run` and `scriptorium render`) via synchronous subprocess execution.
|
||||
|
||||
## 2. Current Status
|
||||
|
||||
Implemented:
|
||||
|
||||
- strict `pipeline.yml` + `session.yml` loading with strict YAML field checking (`KnownFields(true)`)
|
||||
- local workspace/session layout, lock file handling, artifact path helpers, checksums, and atomic writes
|
||||
- manifest store and stage status transitions for resumable runs
|
||||
- real `prepare`, `transcribe`, `merge`, and `polish` stages
|
||||
- real WhisperX HTTP adapter
|
||||
- real Seriatim subprocess adapter
|
||||
- real Audita subprocess adapter
|
||||
- real Scriptorium subprocess adapter
|
||||
- real `normalize` stage producing `transcripts/normalized.json`
|
||||
- real `trim` stage producing `transcripts/trimmed.json`
|
||||
- real `analyze` stage for initial `session_recap` generation
|
||||
- optional Scriptorium render diagnostics (`render_debug`) before production run
|
||||
|
||||
Still placeholder/future:
|
||||
|
||||
- `archive` stage behavior
|
||||
- `notify` stage behavior
|
||||
- additional Scriptorium artifact types beyond `session_recap`
|
||||
- artifact-to-artifact workflows beyond the initial single-artifact implementation
|
||||
- generic stale detection based on input/config checksums
|
||||
|
||||
## 3. Pipeline and Stage Boundaries
|
||||
|
||||
Canonical stage order:
|
||||
|
||||
1. `prepare`
|
||||
2. `transcribe`
|
||||
3. `merge`
|
||||
4. `polish`
|
||||
5. `normalize`
|
||||
6. `trim`
|
||||
7. `analyze`
|
||||
8. `archive`
|
||||
9. `notify`
|
||||
|
||||
Boundary rules:
|
||||
|
||||
- orchestration logic lives in `internal/app`
|
||||
- stage business logic lives in `internal/stage`
|
||||
- external-tool CLI construction lives in adapter packages
|
||||
- Scriptorium CLI details stay in `internal/adapters/scriptorium`
|
||||
|
||||
## 4. Scriptorium Integration Model
|
||||
|
||||
Integration mode:
|
||||
|
||||
- public CLI subprocesses only (no Scriptorium internal Go packages, no HTTP API)
|
||||
- production generation uses `scriptorium run`
|
||||
- diagnostics/testing render uses `scriptorium render --format json`
|
||||
|
||||
Run invocation shape used by adapter:
|
||||
|
||||
```bash
|
||||
scriptorium run --prompt <prompt_id> --input name=path --out <output_path>
|
||||
```
|
||||
|
||||
Optional flags passed when configured:
|
||||
|
||||
- `--config <path>`
|
||||
- `--profile <profile_id>`
|
||||
- repeated `--var name=value`
|
||||
- repeated `--input name=path`
|
||||
- `--timeout <duration>`
|
||||
- `--api-key-env <ENV_NAME>` when configured
|
||||
|
||||
Render invocation shape used by adapter:
|
||||
|
||||
```bash
|
||||
scriptorium render --prompt <prompt_id> --input name=path --format json --out <render_output_path>
|
||||
```
|
||||
|
||||
Adapter behavior:
|
||||
|
||||
- always passes `--out`
|
||||
- captures stdout/stderr separately
|
||||
- writes generated invocation metadata YAML (redacted, no secrets)
|
||||
- treats exit code `0` as success
|
||||
- treats exit code `1` as failure
|
||||
- treats exit code `2` as failure with `validation_failed=true` and preserves output metadata when available
|
||||
- validates successful output files exist and are non-empty
|
||||
- does not treat non-empty stderr as failure by itself
|
||||
|
||||
## 5. Configuration Contract
|
||||
|
||||
`pipeline.scriptorium` is optional. Existing pipelines without Scriptorium continue to work.
|
||||
|
||||
`pipeline.trim` is optional. Existing pipelines without trim config continue to work.
|
||||
|
||||
`pipeline.normalize` is optional. Existing pipelines without normalize config continue to work.
|
||||
|
||||
When `pipeline.normalize` is omitted, defaults are applied:
|
||||
|
||||
- `output_path: transcripts/normalized.json`
|
||||
- `output_schema: seriatim-intermediate`
|
||||
- `report: true`
|
||||
|
||||
When `pipeline.normalize` is present:
|
||||
|
||||
- `output_path` must be non-empty
|
||||
- `output_schema` must be one of `seriatim-minimal`, `seriatim-intermediate`, or `seriatim-full`
|
||||
- relative `output_path` values are session-workdir-relative paths
|
||||
- Seriatim binary settings still come from `pipeline.seriatim`
|
||||
|
||||
When `pipeline.trim` is present:
|
||||
|
||||
- `enabled` is optional and defaults to `false` when omitted
|
||||
- relative `output_path`, `bounds.output_path`, and `bounds.render_output_path` values are session-workdir-relative paths
|
||||
- do not store secrets in trim config values
|
||||
|
||||
When `pipeline.trim.enabled: true`:
|
||||
|
||||
- `output_path` is required and non-empty
|
||||
- `bounds.prompt_id` is required and non-empty
|
||||
- `bounds.transcript_input_name` is required and non-empty
|
||||
- `bounds.output_path` is required and non-empty
|
||||
- `bounds.timeout` must parse as a Go duration when provided
|
||||
- `bounds.render_debug: true` requires non-empty `bounds.render_output_path`
|
||||
- `bounds.profile_id` may be empty to use the prompt default profile
|
||||
- prompt IDs are config values, not hardcoded stage logic
|
||||
|
||||
When `pipeline.scriptorium` is present:
|
||||
|
||||
- `binary` is required and non-empty
|
||||
- `config_path` is optional; when provided it must be non-empty
|
||||
- `timeout` is optional; when provided it must parse as a Go duration
|
||||
- default `timeout` is `10m`
|
||||
- unknown YAML fields fail strict decode
|
||||
|
||||
Artifacts are configured as a map under `pipeline.scriptorium.artifacts` so multiple artifacts are possible in the config shape.
|
||||
|
||||
For each artifact definition:
|
||||
|
||||
- `enabled: true` requires non-empty `prompt_id`
|
||||
- `enabled: true` requires non-empty `output_path`
|
||||
- `timeout` must parse as Go duration when present
|
||||
- optional per-artifact `render_debug` may override global `scriptorium.render_debug`
|
||||
- `inputs` are named and each input requires non-empty `source`
|
||||
- inputs may be optional (`required: false`)
|
||||
- `vars` values currently support `string` and `bool`
|
||||
|
||||
Prompt IDs and profile IDs are configuration values, not hardcoded stage logic.
|
||||
|
||||
Trim config shape:
|
||||
|
||||
```yaml
|
||||
trim:
|
||||
enabled: true
|
||||
output_path: "transcripts/trimmed.json"
|
||||
bounds:
|
||||
prompt_id: "dnd_session.bounds"
|
||||
profile_id: ""
|
||||
transcript_input_name: "transcript"
|
||||
output_path: "artifacts/session_bounds.json"
|
||||
timeout: "10m"
|
||||
render_debug: false
|
||||
render_output_path: "artifacts/session_bounds.render.json"
|
||||
seriatim:
|
||||
report: false
|
||||
```
|
||||
|
||||
## 6. Transcript Tiers
|
||||
|
||||
Narratio currently produces and uses four transcript tiers:
|
||||
|
||||
- `transcripts/merged.json`: canonical deterministic merged transcript from Seriatim merge
|
||||
- `transcripts/processed.json`: full raw Audita-polished transcript output (includes pre/post-game content)
|
||||
- `transcripts/normalized.json`: normalized transcript generated by Seriatim normalize
|
||||
- `transcripts/trimmed.json`: gameplay-only normalized polished transcript from trim stage
|
||||
|
||||
Trim reads `transcripts/normalized.json`, validates bounds IDs against that same transcript ID space, and writes `transcripts/trimmed.json`.
|
||||
|
||||
## 7. Normalize Stage (Current Implementation)
|
||||
|
||||
Normalize stage behavior:
|
||||
|
||||
- stage order position: after `polish` and before `trim`
|
||||
- discovers processed transcript from manifest polish outputs (`transcript_processed`) when present, else `work/<session_id>/transcripts/processed.json`
|
||||
- validates processed transcript JSON shape (`segments` array required)
|
||||
- runs Seriatim `normalize` to produce normalized transcript
|
||||
- validates normalized transcript JSON shape (`segments` array required)
|
||||
- validates normalize report JSON when enabled
|
||||
|
||||
Expected normalize outputs and diagnostics:
|
||||
|
||||
- `transcripts/normalized.json`
|
||||
- `artifacts/seriatim.normalize.report.json` (when normalize report is enabled)
|
||||
- `logs/seriatim.normalize.stdout.log`
|
||||
- `logs/seriatim.normalize.stderr.log`
|
||||
- `config/seriatim.normalize.generated.yml`
|
||||
|
||||
## 8. Trim Stage (Current Implementation)
|
||||
|
||||
Trim stage behavior:
|
||||
|
||||
- stage order position: after `normalize` and before `analyze`
|
||||
- discovers normalized transcript from manifest normalize outputs (`transcript_normalized`) when present, else `work/<session_id>/transcripts/normalized.json`
|
||||
- validates normalized transcript JSON shape (`segments` array required)
|
||||
- when `trim.enabled: false` (or trim config omitted), deterministically copies normalized transcript to `transcripts/trimmed.json` and records `trim_action=copy_disabled`
|
||||
- when `trim.enabled: true`:
|
||||
- runs Scriptorium bounds prompt using configured `trim.bounds.prompt_id`
|
||||
- writes bounds output to configured path (typically `artifacts/session_bounds.json`)
|
||||
- parses and validates bounds output against the same normalized transcript being trimmed
|
||||
- converts bounds range to Seriatim keep selector (for example `10-868`)
|
||||
- runs Seriatim `trim` to produce `transcripts/trimmed.json`
|
||||
- supports no-trim bounds actions (`none`/`copy`) by copying normalized transcript unchanged
|
||||
- validates trimmed transcript JSON shape (`segments` array required)
|
||||
|
||||
Expected trim outputs and diagnostics:
|
||||
|
||||
- `artifacts/session_bounds.json`
|
||||
- `transcripts/trimmed.json`
|
||||
- `logs/scriptorium.bounds.stdout.log`
|
||||
- `logs/scriptorium.bounds.stderr.log`
|
||||
- `config/scriptorium.bounds.generated.yml`
|
||||
- `logs/seriatim.trim.stdout.log`
|
||||
- `logs/seriatim.trim.stderr.log`
|
||||
- `config/seriatim.trim.generated.yml`
|
||||
- optional bounds render-debug outputs when enabled:
|
||||
- `artifacts/session_bounds.render.json`
|
||||
- `logs/scriptorium.bounds.render.stdout.log`
|
||||
- `logs/scriptorium.bounds.render.stderr.log`
|
||||
- `config/scriptorium.bounds.render.generated.yml`
|
||||
|
||||
Render-debug files are diagnostics. They are recorded in stage metadata/log/config refs and are not treated as canonical stage output artifact refs.
|
||||
|
||||
## 9. Analyze Stage (Current Implementation)
|
||||
|
||||
The current real analyze implementation supports only `scriptorium.artifacts.session_recap`.
|
||||
|
||||
Behavior:
|
||||
|
||||
- if `pipeline.scriptorium` is missing, analyze returns a skipped result with metadata
|
||||
- if no Scriptorium artifacts are enabled, analyze returns a skipped result with metadata
|
||||
- if enabled artifacts exist but `session_recap` is not enabled, analyze fails clearly
|
||||
- available transcript input sources for configured artifacts: `processed_transcript`, `normalized_transcript`, `trimmed_transcript`
|
||||
- `session_recap` should use `trimmed_transcript` input (`transcripts/trimmed.json`) for in-universe recap generation
|
||||
- `trimmed_transcript` input is resolved from manifest (`trim` output kind `transcript_trimmed`) when available, otherwise fallback path `work/<session_id>/transcripts/trimmed.json`
|
||||
- `normalized_transcript` input is resolved from manifest (`normalize` output kind `transcript_normalized`) when available, otherwise fallback path `work/<session_id>/transcripts/normalized.json`
|
||||
- `processed_transcript` input is resolved from manifest (`polish` output kind `transcript_processed`) when available, otherwise fallback path `work/<session_id>/transcripts/processed.json`
|
||||
- `normalized_transcript` is the preferred full-transcript source for future table/meta-analysis artifacts
|
||||
- `processed_transcript` remains available for advanced/debug use cases
|
||||
- transcript inputs are validated as JSON with top-level `segments` array
|
||||
- configured inputs are resolved by source
|
||||
- optional `previous_recap` is omitted when unavailable
|
||||
- required `previous_recap` fails before invocation when unavailable
|
||||
- vars are built from config + session metadata
|
||||
- `render_debug` controls pre-run `scriptorium render` diagnostics
|
||||
- render failure stops stage before production run
|
||||
- render output is validated as JSON
|
||||
- production call uses Scriptorium adapter `RunArtifact`
|
||||
- successful run output must exist and be non-empty
|
||||
- missing `trimmed_transcript` input for configured `trimmed_transcript` source fails clearly with guidance to run trim stage first
|
||||
- manifest records output refs, logs, generated config paths, and non-secret provenance metadata
|
||||
|
||||
## 10. Session Recap Paths
|
||||
|
||||
Current expected paths for `session_recap`:
|
||||
|
||||
- artifact output: `artifacts/session_recap.md`
|
||||
- run stdout log: `logs/scriptorium.session_recap.stdout.log`
|
||||
- run stderr log: `logs/scriptorium.session_recap.stderr.log`
|
||||
- run generated invocation/config: `config/scriptorium.session_recap.generated.yml`
|
||||
- render output (when enabled): `artifacts/session_recap.render.json`
|
||||
- render stdout log: `logs/scriptorium.session_recap.render.stdout.log`
|
||||
- render stderr log: `logs/scriptorium.session_recap.render.stderr.log`
|
||||
- render generated invocation/config: `config/scriptorium.session_recap.render.generated.yml`
|
||||
|
||||
## 11. Security and Privacy
|
||||
|
||||
- do not store secrets in pipeline YAML, generated invocation YAML, logs, or manifest metadata
|
||||
- if API-key integration is configured, pass env var names only (never raw key values)
|
||||
- avoid logging transcript content or rendered prompt content by default
|
||||
- treat generated artifacts and logs as potentially sensitive session material
|
||||
|
||||
## 12. Operational Caveat (Pre-Stale-Detection)
|
||||
|
||||
Checksum-based stale detection is not implemented yet.
|
||||
|
||||
If prepared inputs or prompt/runtime configuration change (for example glossary files, prompt IDs, profile IDs, or relevant pipeline settings), rerun the appropriate prior stages to refresh downstream artifacts.
|
||||
|
||||
Examples:
|
||||
|
||||
- glossary or autocorrect changes usually require rerunning at least `merge`, `polish`, `normalize`, `trim`, and `analyze`
|
||||
- trim prompt/profile changes require rerunning at least `normalize`, `trim`, and `analyze`
|
||||
- session recap prompt/profile/input-source changes require rerunning `analyze`
|
||||
|
||||
## 13. Roadmap
|
||||
|
||||
Planned next steps:
|
||||
|
||||
- extend analyze beyond `session_recap` to additional configured artifacts
|
||||
- support artifact inputs that consume prior generated artifacts
|
||||
- keep this composable without adding a generic DAG engine in the near term
|
||||
- implement real `archive` backend behavior
|
||||
- implement real `notify` backend behavior
|
||||
- add checksum-based stale detection and stale transitions
|
||||
|
||||
Architectural invariants remain:
|
||||
|
||||
- strict config decoding/validation
|
||||
- manifest-driven run control
|
||||
- clear stage/adapter separation
|
||||
- configuration-driven prompt/profile/input/vars/output mapping
|
||||
- Scriptorium integration through public CLI subprocess contract
|
||||
291
docs/cli.md
Normal file
291
docs/cli.md
Normal file
@@ -0,0 +1,291 @@
|
||||
# CLI Reference
|
||||
|
||||
## Shortest Useful Command
|
||||
|
||||
```bash
|
||||
narratio run 2026-04-04
|
||||
```
|
||||
|
||||
This runs the canonical full pipeline for session `2026-04-04`.
|
||||
|
||||
## Command Overview
|
||||
|
||||
Top-level commands:
|
||||
|
||||
- `run <session_id>`: run full stage order.
|
||||
- `resume <session_id>`: continue from first non-succeeded stage.
|
||||
- `run-stage <stage> <session_id>`: run one stage.
|
||||
- `analyze <session_id>`: force-run analyze.
|
||||
- `publish <session_id>`: force-run publish.
|
||||
- `clean <session_id>` or `clean --all`: remove local work/spool state.
|
||||
- `session <subcommand>`: session helper commands.
|
||||
|
||||
Session subcommands:
|
||||
|
||||
- `session init <session_id>`
|
||||
- `session plan <session_id>`
|
||||
- `session validate <session_id>`
|
||||
- `session status <session_id>`
|
||||
- `session restore <session_id>`
|
||||
- `session artifacts <session_id>`
|
||||
- `session locks <session_id>`
|
||||
- `session locks add <session_id> <source>`
|
||||
- `session locks remove <session_id> <source>`
|
||||
|
||||
## Common Config Flags
|
||||
|
||||
Most session-aware commands accept:
|
||||
|
||||
- `--config <pipeline.yml>`
|
||||
- `--campaign <id>`
|
||||
- `--campaign-file <campaign.yml>`
|
||||
- `--session <session.yml>`
|
||||
- `--session-id <session_id>`
|
||||
- `--previous-session-id <session_id>`
|
||||
|
||||
Rules:
|
||||
|
||||
- `--campaign` and `--campaign-file` are mutually exclusive.
|
||||
- `--session` is not used by `session init`.
|
||||
- if both positional `<session_id>` and `--session-id` are provided, values must match.
|
||||
- `clean --all` cannot be combined with campaign/session selectors.
|
||||
|
||||
## Session ID Input Rules
|
||||
|
||||
Session-aware commands accept one of these forms:
|
||||
|
||||
- positional session ID: `... <session_id>`
|
||||
- compatibility flag: `... --session-id <session_id>`
|
||||
|
||||
When both are present, command parsing requires an exact match.
|
||||
|
||||
Commands with additional positionals keep their command-specific order:
|
||||
|
||||
- `run-stage <stage> <session_id>` or `run-stage <stage> --session-id <session_id>`
|
||||
- `session locks add <session_id> <source>` or `session locks add --session-id <session_id> <source>`
|
||||
- `session locks remove <session_id> <source>` or `session locks remove --session-id <session_id> <source>`
|
||||
|
||||
## Command Reference
|
||||
|
||||
### `run`
|
||||
|
||||
```bash
|
||||
narratio run <session_id> [--force] [--artifacts <name[,name...]>] [...common config flags]
|
||||
```
|
||||
|
||||
Behavior:
|
||||
|
||||
- evaluates full stage order;
|
||||
- skips already-succeeded stages unless `--force` is set;
|
||||
- writes session and run manifests.
|
||||
|
||||
### `resume`
|
||||
|
||||
```bash
|
||||
narratio resume <session_id> [--force] [--artifacts <name[,name...]>] [...common config flags]
|
||||
```
|
||||
|
||||
Behavior:
|
||||
|
||||
- when not forced, starts at first non-succeeded stage in manifest order;
|
||||
- with `--force`, reevaluates the selected stage list as runnable.
|
||||
|
||||
### `run-stage`
|
||||
|
||||
```bash
|
||||
narratio run-stage <stage> <session_id> [--force] [--artifacts <name[,name...]>] [...common config flags]
|
||||
```
|
||||
|
||||
Valid stage names:
|
||||
|
||||
- `prepare`
|
||||
- `transcribe`
|
||||
- `merge`
|
||||
- `polish`
|
||||
- `normalize`
|
||||
- `trim`
|
||||
- `analyze`
|
||||
- `publish`
|
||||
- `notify`
|
||||
|
||||
Rules:
|
||||
|
||||
- `--artifacts` is accepted only for `analyze` and `publish` stage targets.
|
||||
|
||||
### `analyze`
|
||||
|
||||
```bash
|
||||
narratio analyze <session_id> [--artifacts <name[,name...]>] [...common config flags]
|
||||
```
|
||||
|
||||
Equivalent to:
|
||||
|
||||
```bash
|
||||
narratio run-stage analyze <session_id> --force [...common config flags]
|
||||
```
|
||||
|
||||
### `publish`
|
||||
|
||||
```bash
|
||||
narratio publish <session_id> [--artifacts <name[,name...]>] [...common config flags]
|
||||
```
|
||||
|
||||
Equivalent to:
|
||||
|
||||
```bash
|
||||
narratio run-stage publish <session_id> --force [...common config flags]
|
||||
```
|
||||
|
||||
### `clean`
|
||||
|
||||
```bash
|
||||
narratio clean <session_id> [--dry-run] [--clear-cache] [...common config flags]
|
||||
narratio clean --all [--dry-run] [--clear-cache] [--config <pipeline.yml>]
|
||||
```
|
||||
|
||||
Behavior:
|
||||
|
||||
- session mode removes:
|
||||
- `{workspace.root}/work/{campaign}/{session_id}`
|
||||
- `{spool.root}/{campaign}/{session_id}`
|
||||
- `--all` removes:
|
||||
- `{workspace.root}/work/*`
|
||||
- direct children under `{spool.root}`
|
||||
- cache remains unless `--clear-cache` is provided.
|
||||
|
||||
### `session plan`
|
||||
|
||||
```bash
|
||||
narratio session plan <session_id> [--force] [...common config flags]
|
||||
```
|
||||
|
||||
Validates config, prepares local workdir layout, and prints run/skip decisions for each stage.
|
||||
|
||||
### `session validate`
|
||||
|
||||
```bash
|
||||
narratio session validate <session_id> [...common config flags]
|
||||
```
|
||||
|
||||
Read-only preflight checks for config validity, required inputs, audio mode, previous-session requirements, publish outputs, and effective locks.
|
||||
|
||||
### `session status`
|
||||
|
||||
```bash
|
||||
narratio session status <session_id> [...common config flags]
|
||||
```
|
||||
|
||||
Prints local manifest state and, when storage is available, remote current-state and published-output status.
|
||||
|
||||
### `session init`
|
||||
|
||||
```bash
|
||||
narratio session init <session_id> --output ./session.yml [options]
|
||||
narratio session init <session_id> --remote [options]
|
||||
```
|
||||
|
||||
Required target selection:
|
||||
|
||||
- exactly one of:
|
||||
- `--output <path>`
|
||||
- `--remote`
|
||||
|
||||
Options:
|
||||
|
||||
- `--config <pipeline.yml>`
|
||||
- `--campaign <id>` or `--campaign-file <campaign.yml>`
|
||||
- `--previous-session-id <id>`
|
||||
- `--date <YYYY-MM-DD>`
|
||||
- `--title <text>`
|
||||
- `--audio-dir <path>`
|
||||
- `--audio-s3-prefix <prefix>`
|
||||
- `--force`
|
||||
|
||||
Rules:
|
||||
|
||||
- `--audio-dir` and `--audio-s3-prefix` are mutually exclusive.
|
||||
- if campaign `session_template_file` is configured, `session init` renders it.
|
||||
- generated session YAML must be concrete (no unresolved `{{ ... }}` placeholders).
|
||||
|
||||
### `session restore`
|
||||
|
||||
```bash
|
||||
narratio session restore <session_id> [--dry-run] [--force] [--include-audio] [...common config flags]
|
||||
```
|
||||
|
||||
Behavior:
|
||||
|
||||
- discovers committed remote current state;
|
||||
- plans local restores;
|
||||
- writes `reports/restore-latest.json` on execution;
|
||||
- blocks conflicting overwrites unless `--force` is set.
|
||||
|
||||
Default restore scope:
|
||||
|
||||
- `manifest.json`
|
||||
- `transcripts/**`
|
||||
- `artifacts/**`
|
||||
- `previous/**` when required by configured previous-session inputs
|
||||
|
||||
`audio/**` is included only with `--include-audio`.
|
||||
|
||||
### `session artifacts`
|
||||
|
||||
```bash
|
||||
narratio session artifacts <session_id> [--remote] [...common config flags]
|
||||
```
|
||||
|
||||
Lists effective built-in and configured artifact sources, publish rules, lock state, and optional remote published-state availability.
|
||||
|
||||
### `session locks`
|
||||
|
||||
```bash
|
||||
narratio session locks <session_id> [...common config flags]
|
||||
narratio session locks add <session_id> <source> [--reason <text>] [--force] [...common config flags]
|
||||
narratio session locks remove <session_id> <source> [...common config flags]
|
||||
```
|
||||
|
||||
Behavior:
|
||||
|
||||
- list mode merges static `pipeline.publish.locks` with remote `{session_prefix}/locks.yml`;
|
||||
- add/remove mutate only remote locks;
|
||||
- static locks from pipeline config cannot be removed by CLI commands.
|
||||
|
||||
## `--artifacts` Selection Rules
|
||||
|
||||
- accepted on `run`, `resume`, `run-stage`, `analyze`, and `publish`;
|
||||
- names must exist in `pipeline.scriptorium.artifacts`;
|
||||
- empty entries are invalid;
|
||||
- repeated names are deduplicated.
|
||||
|
||||
Effects:
|
||||
|
||||
- filters analyze execution to selected configured artifacts;
|
||||
- filters publish rules that source `narratio.artifact.<name>`;
|
||||
- does not filter built-in transcript/bounds publish sources.
|
||||
|
||||
## Common Workflows
|
||||
|
||||
Run full pipeline:
|
||||
|
||||
```bash
|
||||
narratio run 2026-04-04
|
||||
```
|
||||
|
||||
Dry-run restore plan:
|
||||
|
||||
```bash
|
||||
narratio session restore 2026-04-04 --dry-run
|
||||
```
|
||||
|
||||
Generate a concrete session file from template/default structure:
|
||||
|
||||
```bash
|
||||
narratio session init 2026-04-04 --output ./session.yml --date 2026-04-04 --title "Session 12"
|
||||
```
|
||||
|
||||
Force publish only:
|
||||
|
||||
```bash
|
||||
narratio publish 2026-04-04
|
||||
```
|
||||
263
docs/config.md
Normal file
263
docs/config.md
Normal file
@@ -0,0 +1,263 @@
|
||||
# Configuration Reference
|
||||
|
||||
## Purpose
|
||||
|
||||
Narratio resolves three YAML documents:
|
||||
|
||||
- `pipeline.yml`: pipeline/runtime settings
|
||||
- `campaign.yml`: campaign identity and stable input defaults
|
||||
- `session.yml`: session identity, metadata, and audio source selection
|
||||
|
||||
## Discovery and Selection
|
||||
|
||||
### `pipeline.yml`
|
||||
|
||||
When `--config` is omitted, search order is:
|
||||
|
||||
1. `/usr/local/etc/narratio/pipeline.yml`
|
||||
2. `/etc/narratio/pipeline.yml`
|
||||
|
||||
### `campaign.yml`
|
||||
|
||||
Selection rules:
|
||||
|
||||
- if `--campaign-file` is set, use that path;
|
||||
- else if `--campaign <id>` is set, use `{pipeline.campaigns.root}/{id}/campaign.yml`;
|
||||
- else use `{pipeline.campaigns.root}/{pipeline.campaigns.default_campaign_id}/campaign.yml`.
|
||||
|
||||
### `session.yml`
|
||||
|
||||
When `--session` is omitted, local search order is:
|
||||
|
||||
1. `/usr/local/etc/narratio/session.yml`
|
||||
2. `/etc/narratio/session.yml`
|
||||
|
||||
If local session discovery fails and a `session_id` is known, Narratio attempts remote session loading from:
|
||||
|
||||
- `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/session.yml`
|
||||
|
||||
using configured object storage.
|
||||
|
||||
## Validation and Merge Rules
|
||||
|
||||
- YAML decode is strict (`KnownFields(true)`): unknown fields fail load.
|
||||
- Session files must be concrete; unresolved `{{ ... }}` placeholders fail load.
|
||||
- Pipeline defaults are applied before validation.
|
||||
- Campaign and session identities must agree.
|
||||
- Stable files (`speakers_file`, `autocorrect_file`, `glossary_file`) resolve from session overrides when provided, otherwise from campaign defaults.
|
||||
- Exactly one audio mode must be configured in session input:
|
||||
- local (`audio_dir` or `audio_files`), or
|
||||
- S3 (`audio_s3.prefix`).
|
||||
|
||||
## Minimal Working Configuration
|
||||
|
||||
`pipeline.yml`
|
||||
|
||||
```yaml
|
||||
campaigns:
|
||||
root: /usr/local/share/narratio/campaigns
|
||||
default_campaign_id: sample-campaign
|
||||
whisperx:
|
||||
transcribe_url: https://transcription.example.com/transcribe
|
||||
```
|
||||
|
||||
`campaign.yml`
|
||||
|
||||
```yaml
|
||||
campaign_id: sample-campaign
|
||||
inputs:
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
```
|
||||
|
||||
`session.yml` (local audio)
|
||||
|
||||
```yaml
|
||||
session_id: 2026-05-03
|
||||
inputs:
|
||||
audio_dir: ./audio
|
||||
```
|
||||
|
||||
## Secrets Handling
|
||||
|
||||
- Do not place raw secrets in YAML.
|
||||
- Use env var names in config (for example `pipeline.audita.llm_api_key_env`).
|
||||
- Optionally load env files from `pipeline.secrets.env_dir`.
|
||||
- Commands that need storage/auth load filesystem secrets before constructing adapters.
|
||||
|
||||
## Publish Configuration Summary
|
||||
|
||||
Publish rules live under `pipeline.publish`.
|
||||
|
||||
```yaml
|
||||
publish:
|
||||
enabled: true
|
||||
upload_run: true
|
||||
outputs:
|
||||
- source: narratio.transcript.final_trimmed
|
||||
dest: transcripts/final.trimmed.json
|
||||
required: true
|
||||
- source: narratio.artifact.session_recap
|
||||
dest: artifacts/session_recap.md
|
||||
required: true
|
||||
locks:
|
||||
- source: narratio.artifact.session_recap
|
||||
reason: manual post-publish edits
|
||||
```
|
||||
|
||||
Rules:
|
||||
|
||||
- `outputs[].source` is required.
|
||||
- `outputs[].dest` may be omitted when derivable from source.
|
||||
- `outputs[].required` defaults to `true`.
|
||||
- static locks (`pipeline.publish.locks`) merge with remote locks (`{session_prefix}/locks.yml`), with static locks taking precedence on duplicates.
|
||||
|
||||
## Full Schema
|
||||
|
||||
### Pipeline
|
||||
|
||||
| Field | Type | Required | Default / Rule |
|
||||
| --- | --- | --- | --- |
|
||||
| `pipeline.workspace.root` | string | No | `/var/lib/narratio` |
|
||||
| `pipeline.workspace.cleanup_after_publish` | bool | No | `false` |
|
||||
| `pipeline.campaigns.root` | string | No | `/usr/local/share/narratio/campaigns` |
|
||||
| `pipeline.campaigns.default_campaign_id` | string | No | empty |
|
||||
| `pipeline.secrets.env_dir` | string | No | empty |
|
||||
| `pipeline.storage.backend` | string | No | empty |
|
||||
| `pipeline.storage.s3.bucket` | string | Conditional | required for S3 session-audio and for publish upload when backend is `s3` |
|
||||
| `pipeline.storage.s3.root_prefix` | string | No | `dnd` |
|
||||
| `pipeline.storage.s3.region` | string | No | empty |
|
||||
| `pipeline.storage.s3.endpoint` | string | No | empty |
|
||||
| `pipeline.storage.s3.force_path_style` | bool | No | `false` |
|
||||
| `pipeline.storage.s3.access_key_id_env` | string | No | `OBJECT_STORAGE_KEY_ID` |
|
||||
| `pipeline.storage.s3.secret_access_key_env` | string | No | `OBJECT_STORAGE_KEY` |
|
||||
| `pipeline.spool.root` | string | No | `/var/spool/narratio` |
|
||||
| `pipeline.spool.delete_audio_after_publish` | bool | No | `false` |
|
||||
| `pipeline.cache.root` | string | No | `/var/cache/narratio` |
|
||||
| `pipeline.cache.s3_audio` | bool | No | `true` |
|
||||
| `pipeline.publish.enabled` | bool | No | `true` |
|
||||
| `pipeline.publish.upload_run` | bool | No | `true` |
|
||||
| `pipeline.publish.outputs[]` | list | No | defaults to final trimmed transcript output |
|
||||
| `pipeline.publish.outputs[].source` | string | Yes (per rule) | must reference built-in or configured artifact source |
|
||||
| `pipeline.publish.outputs[].dest` | string | Conditional | derived if omitted and source supports derivation |
|
||||
| `pipeline.publish.outputs[].required` | bool | No | `true` |
|
||||
| `pipeline.publish.locks[]` | list | No | empty |
|
||||
| `pipeline.publish.locks[].source` | string | Yes (per lock) | must reference supported publish source |
|
||||
| `pipeline.publish.locks[].reason` | string | No | empty |
|
||||
| `pipeline.whisperx.transcribe_url` | string | Yes | valid URL |
|
||||
| `pipeline.whisperx.language` | string | No | `en` |
|
||||
| `pipeline.whisperx.timeout` | duration | No | `30m` |
|
||||
| `pipeline.whisperx.retries` | int | No | `3` |
|
||||
| `pipeline.whisperx.retry_delay` | duration | No | `2s` |
|
||||
| `pipeline.whisperx.concurrency` | int | No | `2` |
|
||||
| `pipeline.seriatim.binary` | string | No | `seriatim` |
|
||||
| `pipeline.seriatim.timeout` | duration | No | `10m` |
|
||||
| `pipeline.seriatim.output_schema` | string | No | `seriatim-intermediate` |
|
||||
| `pipeline.seriatim.coalesce_gap` | float | No | `3.0` |
|
||||
| `pipeline.seriatim.report` | bool | No | `true` |
|
||||
| `pipeline.seriatim.env.overlap_word_run_gap` | float | No | unset |
|
||||
| `pipeline.seriatim.env.overlap_word_run_reorder_window` | float | No | unset |
|
||||
| `pipeline.seriatim.env.backchannel_max_duration` | float | No | unset |
|
||||
| `pipeline.seriatim.env.filler_max_duration` | float | No | unset |
|
||||
| `pipeline.audita.binary` | string | No | `audita` |
|
||||
| `pipeline.audita.timeout` | duration | No | `3h` |
|
||||
| `pipeline.audita.llm_api_key_env` | string | No | empty |
|
||||
| `pipeline.audita.modules[]` | list[string] | No | empty |
|
||||
| `pipeline.audita.base_url` | string | No | empty |
|
||||
| `pipeline.audita.model` | string | No | empty |
|
||||
| `pipeline.audita.total_llm_concurrency` | int | No | unset |
|
||||
| `pipeline.audita.proposal_llm_concurrency` | int | No | unset |
|
||||
| `pipeline.audita.validation_model` | string | No | empty |
|
||||
| `pipeline.audita.validation_llm_concurrency` | int | No | unset |
|
||||
| `pipeline.audita.transcript_description` | string | No | empty |
|
||||
| `pipeline.audita.config_path` | string | No | empty |
|
||||
| `pipeline.audita.output_schema` | string | No | empty |
|
||||
| `pipeline.audita.work_dir_retention` | string | No | empty |
|
||||
| `pipeline.audita.report` | bool | No | `true` |
|
||||
| `pipeline.normalize.output_path` | string | No | `transcripts/final.json` |
|
||||
| `pipeline.normalize.output_schema` | string | No | `seriatim-intermediate` |
|
||||
| `pipeline.normalize.report` | bool | No | `true` |
|
||||
| `pipeline.trim.enabled` | bool | No | `false` |
|
||||
| `pipeline.trim.output_path` | string | Conditional | required when trim enabled |
|
||||
| `pipeline.trim.bounds.prompt_id` | string | Conditional | required when trim enabled |
|
||||
| `pipeline.trim.bounds.profile_id` | string | No | empty |
|
||||
| `pipeline.trim.bounds.transcript_input_name` | string | Conditional | required when trim enabled |
|
||||
| `pipeline.trim.bounds.output_path` | string | Conditional | required when trim enabled |
|
||||
| `pipeline.trim.bounds.timeout` | duration | No | `10m` |
|
||||
| `pipeline.trim.bounds.render_debug` | bool | No | `false` |
|
||||
| `pipeline.trim.bounds.render_output_path` | string | Conditional | required when `render_debug` is true |
|
||||
| `pipeline.trim.seriatim.report` | bool | No | `false` |
|
||||
| `pipeline.scriptorium.binary` | string | No | `scriptorium` |
|
||||
| `pipeline.scriptorium.config_path` | string | No | empty |
|
||||
| `pipeline.scriptorium.timeout` | duration | No | `10m` |
|
||||
| `pipeline.scriptorium.render_debug` | bool | No | `false` |
|
||||
| `pipeline.scriptorium.artifacts` | map | No | empty |
|
||||
| `pipeline.notification.backend` | string | No | empty |
|
||||
| `pipeline.notification.recipient` | string | No | empty |
|
||||
| `pipeline.notification.timeout` | duration | No | `30s` |
|
||||
|
||||
### Scriptorium Artifact Entries
|
||||
|
||||
For each `pipeline.scriptorium.artifacts.<name>`:
|
||||
|
||||
| Field | Type | Required | Rule |
|
||||
| --- | --- | --- | --- |
|
||||
| `enabled` | bool | No | `false` if omitted |
|
||||
| `depends_on[]` | list[string] | No | must reference configured artifact keys; no self-reference; enabled graph must be acyclic |
|
||||
| `render_debug` | bool | No | per-artifact override |
|
||||
| `prompt_id` | string | Conditional | required when artifact is enabled |
|
||||
| `profile_id` | string | No | empty |
|
||||
| `output_path` | string | Conditional | required when enabled; also required when referenced by publish/output/input rules |
|
||||
| `timeout` | duration | No | artifact override |
|
||||
| `inputs` | map | No | input key names must be non-empty |
|
||||
| `vars` | map | No | values must be string or bool |
|
||||
|
||||
For each artifact input `pipeline.scriptorium.artifacts.<name>.inputs.<input_name>`:
|
||||
|
||||
| Field | Type | Required | Rule |
|
||||
| --- | --- | --- | --- |
|
||||
| `source` | string | Yes | built-in runtime source, `narratio.artifact.<name>`, or `narratio.previous_session.artifact.<name>` |
|
||||
| `artifact` | string | No | optional passthrough adapter field |
|
||||
| `path` | string | No | optional passthrough adapter field |
|
||||
| `required` | bool | No | optional input requirement |
|
||||
|
||||
### Campaign
|
||||
|
||||
| Field | Type | Required | Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| `campaign_id` | string | Yes | canonical campaign identity |
|
||||
| `session_template_file` | string | No | used by `session init` when set |
|
||||
| `inputs.speakers_file` | string | Yes | stable input default |
|
||||
| `inputs.autocorrect_file` | string | Yes | stable input default |
|
||||
| `inputs.glossary_file` | string | Yes | stable input default |
|
||||
|
||||
### Session
|
||||
|
||||
| Field | Type | Required in session file | Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| `session_id` | string | Yes | must match CLI session target when provided |
|
||||
| `previous_session_id` | string | No | must not equal `session_id` |
|
||||
| `campaign` | string | No | filled from `campaign_id` during resolve if omitted |
|
||||
| `date` | string | No | metadata |
|
||||
| `title` | string | No | metadata |
|
||||
| `inputs.speakers_file` | string | No | overrides campaign stable input |
|
||||
| `inputs.autocorrect_file` | string | No | overrides campaign stable input |
|
||||
| `inputs.glossary_file` | string | No | overrides campaign stable input |
|
||||
| `inputs.audio_dir` | string | Conditional | local audio mode |
|
||||
| `inputs.audio_files[]` | list[string] | Conditional | local audio mode |
|
||||
| `inputs.audio_s3.prefix` | string | Conditional | S3 audio mode |
|
||||
|
||||
Audio rules:
|
||||
|
||||
- configure local mode (`audio_dir` or `audio_files`) or S3 mode (`audio_s3.prefix`), not both.
|
||||
|
||||
## Maintained Examples
|
||||
|
||||
- `examples/pipeline.minimal.yml`
|
||||
- `examples/pipeline.production.yml`
|
||||
- `examples/pipeline.full.annotated.yml`
|
||||
- `examples/campaigns/sample-campaign/campaign.yml`
|
||||
- `examples/session.local-audio.yml`
|
||||
- `examples/session.s3-audio.yml`
|
||||
- `examples/session.template.yml`
|
||||
19
docs/integrations/README.md
Normal file
19
docs/integrations/README.md
Normal file
@@ -0,0 +1,19 @@
|
||||
# Integrations Index
|
||||
|
||||
## Audience
|
||||
Developers and coding agents changing Narratio's external integration boundaries.
|
||||
|
||||
## Scope
|
||||
`docs/integrations/` is the implementation-level reference for downstream tool adapter contracts.
|
||||
|
||||
These docs cover what Narratio expects from external tools and what each adapter guarantees back to stage code.
|
||||
|
||||
## Integration Contracts
|
||||
- `audita.md`: transcript polishing adapter (`audita process`).
|
||||
- `seriatim.md`: merge/normalize/trim adapter (`seriatim`).
|
||||
- `scriptorium.md`: artifact run/render adapter (`scriptorium run|render`).
|
||||
|
||||
## Related Canonical Docs
|
||||
- `docs/config.md`: operator-facing configuration reference.
|
||||
- `docs/internal/adapters.md`: shared adapter boundary and runner wiring.
|
||||
- `docs/internal/stage-*.md`: stage-specific integration usage.
|
||||
@@ -1,147 +1,60 @@
|
||||
# Audita
|
||||
# Integration: Audita
|
||||
|
||||
Audita is a framework-first transcript correction application. The public `audita` package provides:
|
||||
## Purpose
|
||||
Define the Audita adapter contract used by the `polish` stage.
|
||||
|
||||
- deterministic transcript normalization
|
||||
- token-batched module orchestration
|
||||
- concrete `glossary`, `homophones`, `spoken_word`, and `grammar` modules built on reusable proposal / validator contracts
|
||||
- structured run reporting and work-dir diagnostics
|
||||
## Adapter Boundary
|
||||
Interface:
|
||||
- `audita.Runner`
|
||||
- method: `Run(ctx, PolishRequest) (PolishResult, error)`
|
||||
|
||||
The previous working implementation has been preserved as `audita_prototype` inside this repository. Its full regression suite lives under `tests/audita_prototype`.
|
||||
Primary implementation:
|
||||
- `internal/adapters/audita/SubprocessRunner`
|
||||
|
||||
## Development
|
||||
Execution mode:
|
||||
- subprocess invocation of `audita process`
|
||||
|
||||
This project is set up for `uv`.
|
||||
## Request Contract
|
||||
`PolishRequest` carries:
|
||||
- required transcript/glossary/output/work-dir paths;
|
||||
- optional report path (required when report mode is enabled);
|
||||
- generated config and stdout/stderr log paths;
|
||||
- optional module/model/base-url/config/output-schema/concurrency settings.
|
||||
|
||||
```sh
|
||||
uv sync --extra dev
|
||||
uv run pytest
|
||||
```
|
||||
## Result Contract
|
||||
`PolishResult` returns:
|
||||
- processed transcript path;
|
||||
- optional report path;
|
||||
- work dir and generated-config/log paths;
|
||||
- exit code, duration, binary provenance;
|
||||
- adapter metadata map.
|
||||
|
||||
## Usage
|
||||
## Validation and Failure Semantics
|
||||
Construction fails for invalid static config values, including:
|
||||
- empty binary;
|
||||
- non-positive timeout;
|
||||
- invalid base URL;
|
||||
- invalid output schema;
|
||||
- invalid work-dir retention value;
|
||||
- invalid concurrency values.
|
||||
|
||||
Process a transcript with the current framework implementation:
|
||||
Run fails for:
|
||||
- missing required request paths;
|
||||
- missing required credential env var when configured (`llm_api_key_env`);
|
||||
- subprocess execution failure;
|
||||
- invalid processed transcript JSON (`segments` array required);
|
||||
- invalid report JSON when reporting is enabled.
|
||||
|
||||
```sh
|
||||
uv run audita process transcript.json --glossary glossary.yaml --output corrected.json
|
||||
```
|
||||
Failure results still include output/log/config/exit metadata for diagnostics.
|
||||
|
||||
The framework currently runs this default module sequence:
|
||||
## Deterministic Behavior
|
||||
- CLI args are built from runner config + request in a fixed order.
|
||||
- Generated invocation YAML (`audita.generated.v1`) is emitted when requested.
|
||||
- Manifest writes are stage-owned; adapter itself is stateless.
|
||||
|
||||
1. `glossary`
|
||||
2. `homophones`
|
||||
3. `glossary`
|
||||
4. `spoken_word`
|
||||
5. `grammar`
|
||||
## Config Mapping
|
||||
Config fields consumed through runner/stage wiring are under `pipeline.audita.*`.
|
||||
|
||||
Resolved run instance names are auto-numbered for repeats, so the default report pipeline is:
|
||||
|
||||
1. `glossary_1`
|
||||
2. `homophones`
|
||||
3. `glossary_2`
|
||||
4. `spoken_word`
|
||||
5. `grammar`
|
||||
|
||||
The default module sequence is fully implemented today:
|
||||
|
||||
- `glossary` proposes glossary-supported acoustic corrections
|
||||
- `homophones` proposes conservative homophone and mistranscription corrections
|
||||
- `spoken_word` proposes conservative dysfluency cleanup
|
||||
- `grammar` proposes punctuation, capitalization, and spacing cleanup only
|
||||
|
||||
To run a custom module sequence, pass `--modules`:
|
||||
|
||||
```sh
|
||||
uv run audita process transcript.json --glossary glossary.yaml --modules grammar --output corrected.json
|
||||
```
|
||||
|
||||
To also write a structured JSON report:
|
||||
|
||||
```sh
|
||||
uv run audita process transcript.json --glossary glossary.yaml --output corrected.json --report-json report.json
|
||||
```
|
||||
|
||||
From a checked-out repository, you can also use the root launcher:
|
||||
|
||||
```sh
|
||||
./audita process transcript.json --glossary glossary.yaml --output corrected.json
|
||||
```
|
||||
|
||||
For a system-wide command, install the source tree under `/usr/local/src/audita`, sync dependencies there, and symlink the root launcher into your `PATH`:
|
||||
|
||||
```sh
|
||||
cd /usr/local/src/audita
|
||||
uv sync --extra dev
|
||||
ln -s /usr/local/src/audita/audita /usr/local/bin/audita
|
||||
audita process transcript.json --glossary glossary.yaml --output corrected.json
|
||||
```
|
||||
|
||||
Without `--output`, Audita writes the corrected transcript JSON to stdout and progress logs to stderr.
|
||||
`--report-json` writes a separate machine-readable run report and never mixes report data into stdout.
|
||||
|
||||
Useful configuration can be supplied by CLI flag or environment variable. CLI flags take precedence over environment variables. Normal runs now require LLM API credentials, because the `glossary`, `homophones`, `spoken_word`, and `grammar` modules make real LLM calls. `AUDITA_LLM_API_KEY` and `--llm-api-key` are the preferred provider-neutral credential surfaces, while `OPENROUTER_API_KEY` remains supported as a backward-compatible fallback.
|
||||
|
||||
| Environment variable | CLI flag | Default | Purpose |
|
||||
| --- | --- | --- | --- |
|
||||
| `AUDITA_MODULES` | `--modules` | `glossary,homophones,glossary,spoken_word,grammar` | Comma-separated logical module keys to run; CLI overrides the environment value |
|
||||
| `AUDITA_LLM_API_KEY` | `--llm-api-key` | unset | Preferred provider-neutral LLM API credential; CLI overrides both environment-key variants |
|
||||
| `AUDITA_VALIDATION_LLM_API_KEY` | `--validation-llm-api-key` | unset | Validation-phase LLM API credential; defaults to the primary LLM API key |
|
||||
| `AUDITA_MODEL` | `--model` | `openrouter/google/gemma-4-31b-it` | LLM model name sent to the configured OpenAI-compatible endpoint |
|
||||
| `AUDITA_VALIDATION_MODEL` | `--validation-model` | unset | Validation-phase LLM model; defaults to `AUDITA_MODEL` |
|
||||
| `AUDITA_BASE_URL` | `--base-url` | `https://openrouter.ai/api/v1` | OpenAI-compatible API base URL |
|
||||
| `AUDITA_VALIDATION_BASE_URL` | `--validation-base-url` | unset | Validation-phase OpenAI-compatible API base URL; defaults to `AUDITA_BASE_URL` |
|
||||
| `AUDITA_LLM_TIMEOUT_SECONDS` | `--llm-timeout-seconds` | `600` | Per-request timeout in seconds for LLM calls to the configured OpenAI-compatible endpoint |
|
||||
| `AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS` | `--validation-llm-timeout-seconds` | unset | Validation-phase per-request timeout in seconds; defaults to `AUDITA_LLM_TIMEOUT_SECONDS` |
|
||||
| `AUDITA_VALIDATION_MAX_PROMPT_TOKENS` | `--validation-max-prompt-tokens` | `2048` | Maximum estimated tokens per validation-phase LLM prompt batch |
|
||||
| `AUDITA_TARGET_SECTIONS` | `--target-sections` | unset | Exact number of contiguous proposal-stage transcript sections; errors if min/max token bounds cannot be satisfied |
|
||||
| `AUDITA_MAX_RETRIES` | `--max-retries` | `3` | Maximum Instructor retries for structured responses |
|
||||
| `AUDITA_VALIDATION_MAX_RETRIES` | `--validation-max-retries` | unset | Validation-phase structured-output retries; defaults to `AUDITA_MAX_RETRIES` |
|
||||
| `AUDITA_VALIDATION_LLM_CONCURRENCY` | `--validation-llm-concurrency` | unset | Validation-phase LLM concurrency; defaults to `AUDITA_LLM_CONCURRENCY` |
|
||||
| `AUDITA_MAX_SECTION_TOKENS` | `--max-section-tokens` | `8192` | Maximum estimated tokens per proposal-stage transcript section |
|
||||
| `AUDITA_MIN_SECTION_TOKENS` | `--min-section-tokens` | `2048` | Minimum estimated tokens per proposal-stage transcript section when balancing for concurrency |
|
||||
| `AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD` | `--glossary-confidence-threshold` | `0.8` | Minimum confidence required for glossary proposals to survive validation |
|
||||
| `AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD` | `--grammar-confidence-threshold` | `0.8` | Minimum confidence required for grammar proposals to survive validation |
|
||||
| `AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD` | `--homophones-confidence-threshold` | `0.8` | Minimum confidence required for homophone proposals to survive validation |
|
||||
| `AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD` | `--spoken-word-confidence-threshold` | `0.8` | Minimum confidence required for spoken-word proposals to survive validation |
|
||||
| `AUDITA_NORMALIZE_MAX_SEGMENT_GAP` | `--normalize-max-segment-gap` | `4.0` | Same-speaker gaps eligible for deterministic merging |
|
||||
| `AUDITA_NORMALIZE_ELLIPSIS_GAP` | `--normalize-ellipsis-gap` | `3.5` | Same-speaker gaps above this value are joined with ` ... ` |
|
||||
| `AUDITA_NORMALIZE_MAX_SEGMENT_DURATION` | `--normalize-max-segment-duration` | `60.0` | Maximum merged segment duration |
|
||||
| `AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS` | `--normalize-max-segment-tokens` | `2048` | Maximum merged segment prompt payload size |
|
||||
| `AUDITA_WORK_DIR` | `--work-dir` | `/tmp/audita` | Per-run scratch diagnostics directory |
|
||||
| `AUDITA_WORK_DIR_RETENTION` | `--work-dir-retention` | `auto` | Whether to retain the per-run work directory: `auto`, `always`, or `never` |
|
||||
|
||||
Set `AUDITA_MODULES=grammar` to run only the grammar module by default, or override it per command with `--modules`.
|
||||
|
||||
Validation-phase LLM settings inherit from the primary `AUDITA_*` LLM settings by default. Set any of the `AUDITA_VALIDATION_*` values only when you want LLM-backed validators to use a different model, endpoint, credential, timeout, retry budget, or concurrency level.
|
||||
|
||||
OpenRouter remains the default out of the box:
|
||||
|
||||
```sh
|
||||
export AUDITA_LLM_API_KEY=your-openrouter-key
|
||||
audita process transcript.json --glossary glossary.yaml --output corrected.json
|
||||
```
|
||||
|
||||
You can point Audita at any OpenAI-compatible endpoint by changing `AUDITA_BASE_URL` and, if needed, `AUDITA_MODEL`. For example, a local vLLM server:
|
||||
|
||||
```sh
|
||||
export AUDITA_LLM_API_KEY=local-dev-key
|
||||
export AUDITA_BASE_URL=http://localhost:8000/v1
|
||||
export AUDITA_MODEL=meta-llama/Llama-3.1-8B-Instruct
|
||||
audita process transcript.json --glossary glossary.yaml --output corrected.json
|
||||
```
|
||||
|
||||
Or the actual OpenAI API:
|
||||
|
||||
```sh
|
||||
export AUDITA_LLM_API_KEY=your-openai-key
|
||||
export AUDITA_BASE_URL=https://api.openai.com/v1
|
||||
export AUDITA_MODEL=gpt-4.1-mini
|
||||
audita process transcript.json --glossary glossary.yaml --output corrected.json
|
||||
```
|
||||
|
||||
`AUDITA_WORK_DIR` stores per-run diagnostics while processing. Under the default `AUDITA_WORK_DIR_RETENTION=auto`, clean successful runs are removed, while failed runs and successful runs with final skipped corrections are preserved. Use `always` to keep every run directory and `never` to remove successful run directories even when skips remain.
|
||||
Failed runs always preserve the run directory and include an authoritative `report.json` alongside normalization and prompt/response diagnostics.
|
||||
|
||||
## Prototype Archive
|
||||
|
||||
The archived prototype remains importable as `audita_prototype` and is still covered by its original regression suite. This is intentional: the new `audita` package is a framework-oriented rewrite, not a thin wrapper around the old code.
|
||||
Maintained example with Audita config:
|
||||
- `examples/pipeline.full.annotated.yml`
|
||||
- `examples/pipeline.production.yml`
|
||||
|
||||
@@ -1,339 +1,66 @@
|
||||
# Narratio -> Scriptorium CLI Integration
|
||||
# Integration: Scriptorium
|
||||
|
||||
## 1. Purpose
|
||||
## Purpose
|
||||
Define the Scriptorium adapter contract used by `analyze` and trim-bounds generation in `trim`.
|
||||
|
||||
This document defines how Narratio should invoke Scriptorium through the **public CLI**.
|
||||
## Adapter Boundary
|
||||
Interface:
|
||||
- `scriptorium.Runner`
|
||||
- methods:
|
||||
- `RunArtifact(ctx, RunArtifactRequest)`
|
||||
- `RenderArtifact(ctx, RenderArtifactRequest)`
|
||||
|
||||
This is a **subprocess integration contract**, not an internal Go API contract.
|
||||
|
||||
## 2. Assumptions
|
||||
|
||||
- `scriptorium` is installed and available on `PATH`.
|
||||
- Scriptorium is configured with `config.yml`.
|
||||
- `config.yml` provides `prompt_dir`, `profile_dir`, and `schema_dir` as needed.
|
||||
- Prompt and profile libraries are already deployed for the environment.
|
||||
- Narratio provides prepared artifact files (for example polished transcript, glossary, previous recap, campaign notes).
|
||||
- Initial integration is synchronous subprocess execution.
|
||||
- Narratio remains the orchestrator.
|
||||
|
||||
In normal operation, Narratio does not need to pass `--prompt-dir` and `--profile-dir` if they are supplied by Scriptorium config.
|
||||
|
||||
Narratio may pass `--config <PATH>` when it must use a non-default Scriptorium config file.
|
||||
|
||||
## 3. Core Commands Narratio May Call
|
||||
|
||||
Primary commands for subprocess integration:
|
||||
Primary implementation:
|
||||
- `internal/adapters/scriptorium/SubprocessRunner`
|
||||
|
||||
Execution modes:
|
||||
- `scriptorium run`
|
||||
- `scriptorium render`
|
||||
|
||||
For production generation, use `scriptorium run`.
|
||||
|
||||
`scriptorium render` is for debugging, dry-runs, test assertions, and validating command construction without LLM execution.
|
||||
|
||||
Note: `scriptorium serve` and HTTP API exist, but they are not the initial integration path.
|
||||
|
||||
## 4. Command Selection Guidance
|
||||
|
||||
- Use `run` to generate an output artifact.
|
||||
- Use `render` to inspect the prepared prompt and effective settings without calling the LLM.
|
||||
- Use `render --format json` when Narratio/tests need structured prepare output.
|
||||
|
||||
## 5. Recommended `run` Invocation Shape
|
||||
|
||||
Production shape:
|
||||
|
||||
```bash
|
||||
scriptorium run \
|
||||
--prompt <prompt_id> \
|
||||
--input transcript=<processed-transcript-path> \
|
||||
--out <output-artifact-path>
|
||||
```
|
||||
|
||||
Common optional additions:
|
||||
|
||||
- `--config <path>`: use a specific Scriptorium config file.
|
||||
- `--profile <profile_id>`: override prompt default profile.
|
||||
- `--var name=value` (repeatable): small metadata values.
|
||||
- `--input name=path` (repeatable): additional named artifacts.
|
||||
- `--timeout <duration>`: per-run timeout override.
|
||||
- Runtime model override flags (`--llm-base-url`, `--model`, etc.) only for exceptional/operator-directed cases.
|
||||
|
||||
## 6. Recommended `render` Invocation Shape
|
||||
|
||||
Human-readable debug shape:
|
||||
|
||||
```bash
|
||||
scriptorium render \
|
||||
--prompt <prompt_id> \
|
||||
--input transcript=<processed-transcript-path> \
|
||||
--format text
|
||||
```
|
||||
|
||||
Structured debug/test shape:
|
||||
|
||||
```bash
|
||||
scriptorium render \
|
||||
--prompt <prompt_id> \
|
||||
--input transcript=<processed-transcript-path> \
|
||||
--format json \
|
||||
--out <render-debug-path>
|
||||
```
|
||||
|
||||
`render` does **not** call the LLM, does **not** validate model output, and does **not** perform repair.
|
||||
|
||||
## 7. Inputs
|
||||
|
||||
- Pass inputs as repeated `--input name=path` flags.
|
||||
- `name` must match the Prompt Definition input name.
|
||||
- Prefer absolute paths, or paths relative to a working directory controlled by Narratio.
|
||||
- Pass Audita output as the primary transcript input.
|
||||
- Additional inputs may include glossary, previous recap, campaign notes, event logs, final state maps, or other prompt-specific artifacts.
|
||||
- Scriptorium reads input files directly; Narratio does not need to inline file content for CLI use.
|
||||
|
||||
## 8. Variables
|
||||
|
||||
Use repeated `--var name=value` for small metadata values.
|
||||
|
||||
Typical examples:
|
||||
|
||||
- `session_date`
|
||||
- `session_id`
|
||||
- `campaign_name`
|
||||
- `previous_session_id`
|
||||
- `output_kind`
|
||||
|
||||
Large content belongs in input files, not `--var` values.
|
||||
|
||||
## 9. Prompt IDs and Output Artifact Types
|
||||
|
||||
Narratio should treat prompt IDs as configuration, not hardcoded business logic.
|
||||
|
||||
Narratio config may map stage/output names to prompt IDs, for example:
|
||||
|
||||
- session recap prompt
|
||||
- structured event extraction prompt
|
||||
- glossary suggestion prompt
|
||||
- player-facing summary prompt
|
||||
|
||||
Prompt IDs used by Narratio should come from the deployed Scriptorium prompt library.
|
||||
|
||||
## 10. Profiles
|
||||
|
||||
- Prompts may declare `default_profile`.
|
||||
- Narratio may omit `--profile` to use prompt default profile.
|
||||
- Narratio may pass `--profile` to force profile selection.
|
||||
- This enables environment/profile selection like `local-fast`, `local-quality`, `frontier`, `batch`, or test profiles.
|
||||
- Profile names should generally be Narratio configuration values.
|
||||
|
||||
## 11. Runtime Overrides
|
||||
|
||||
Supported runtime override flags:
|
||||
|
||||
- `--llm-base-url`
|
||||
- `--model`
|
||||
- `--api-key-env`
|
||||
- `--temperature`
|
||||
- `--max-tokens`
|
||||
- `--top-p`
|
||||
- `--timeout`
|
||||
|
||||
Guidance:
|
||||
|
||||
- Keep normal model/runtime settings in Execution Profiles.
|
||||
- Use runtime overrides only for explicit per-run exceptions, tests, or operator overrides.
|
||||
- Never pass raw API keys on the command line.
|
||||
- `--api-key-env` names an environment variable; Narratio must ensure that variable is set in subprocess environment.
|
||||
|
||||
## 12. Config Behavior
|
||||
|
||||
- Default config path: `/etc/scriptorium/config.yml`.
|
||||
- `--config <PATH>` overrides default path.
|
||||
- Missing default config is allowed by Scriptorium.
|
||||
- If `--config` is provided explicitly, the file must exist and be valid.
|
||||
- CLI flags override `config.yml`.
|
||||
- `config.yml` overrides built-in application defaults.
|
||||
|
||||
Narratio can either:
|
||||
|
||||
- rely on system default config path, or
|
||||
- carry an explicit config path and pass `--config`.
|
||||
|
||||
## 13. Environment Handling
|
||||
|
||||
Subprocess environment recommendations:
|
||||
|
||||
- Pass through required API-key environment variables referenced by `api_key_env`.
|
||||
- Do not pass raw API keys as CLI arguments.
|
||||
- Avoid logging full environment dumps.
|
||||
- Capture stdout and stderr separately.
|
||||
- Use a controlled working directory.
|
||||
- Prefer absolute artifact paths.
|
||||
|
||||
## 14. Output Handling
|
||||
|
||||
For `scriptorium run`:
|
||||
|
||||
- Use `--out` when Narratio needs durable artifact files.
|
||||
- Without `--out`, artifact content is written to stdout.
|
||||
- Preferred orchestration pattern: always use `--out`, then treat the file as stage output artifact.
|
||||
- Capture stderr for diagnostics.
|
||||
|
||||
For `scriptorium render`:
|
||||
|
||||
- Use `--out` to store render diagnostics.
|
||||
- Use `--format json` when tests need to inspect selected profile, effective runtime settings, input hashes, prompt hash, and rendered messages.
|
||||
|
||||
## 15. Exit Status and Errors
|
||||
|
||||
Current CLI behavior (verified from implementation/tests):
|
||||
|
||||
- `0`: success.
|
||||
- `1`: runtime/parse/config/load/render/generation/IO error.
|
||||
- `2`: run completed but output validation failed (`ValidationFailed`).
|
||||
|
||||
Additional details:
|
||||
|
||||
- On `run`, output artifact write happens before exit code selection. If validation fails, artifact may still be written and exit code is `2`.
|
||||
- `stderr` carries both errors and normal run summary output; non-empty stderr alone does not imply failure.
|
||||
- `render` returns `0` on success and `1` on failures.
|
||||
|
||||
Narratio should treat non-zero exit codes as failed stage execution, but may record generated artifact paths if a run exited `2` and output file exists.
|
||||
|
||||
## 16. Recommended Narratio Integration Pattern
|
||||
|
||||
1. Build CLI args from Narratio stage configuration.
|
||||
2. Use subprocess context cancellation/timeout.
|
||||
3. Pass absolute input paths.
|
||||
4. Pass `--out` to a session-scoped artifact path.
|
||||
5. Add `--var` metadata values.
|
||||
6. Optionally add `--config`.
|
||||
7. Optionally add `--profile`.
|
||||
8. Ensure required API-key env vars are present.
|
||||
9. Run subprocess synchronously.
|
||||
10. Capture stdout/stderr separately.
|
||||
11. On success, store output artifact path and invocation metadata in stage artifacts.
|
||||
12. On failure, store exit code and stderr diagnostics in stage status.
|
||||
|
||||
## 17. Suggested Narratio Configuration Shape
|
||||
|
||||
Illustrative `pipeline.yml` shape:
|
||||
|
||||
```yaml
|
||||
scriptorium:
|
||||
binary: scriptorium
|
||||
config_path: /etc/scriptorium/config.yml
|
||||
timeout: 10m
|
||||
render_debug: false
|
||||
artifacts:
|
||||
session_recap:
|
||||
enabled: true
|
||||
prompt_id: dnd.session_recap
|
||||
profile_id: local-quality # optional
|
||||
output_path: artifacts/session_recap.md
|
||||
timeout: 10m
|
||||
render_debug: false # optional artifact override
|
||||
inputs:
|
||||
transcript:
|
||||
source: trimmed_transcript
|
||||
required: true
|
||||
previous_recap:
|
||||
source: previous_session_artifact
|
||||
artifact: session_recap
|
||||
path: "" # optional
|
||||
required: false
|
||||
vars:
|
||||
session_id: true
|
||||
session_date: true
|
||||
campaign_name: true
|
||||
previous_session_id: true
|
||||
output_kind: session_recap
|
||||
```
|
||||
|
||||
The key idea: map Narratio artifact names to prompt ID, optional profile, expected inputs, vars, and output destination.
|
||||
|
||||
## 18. Testing Strategy for Narratio Integration
|
||||
|
||||
- Use `scriptorium render --format json` to verify command construction without LLM calls.
|
||||
- Use dedicated test prompt/profile libraries for integration tests.
|
||||
- Use small fixture transcripts.
|
||||
- Verify missing-input failure behavior.
|
||||
- Verify prompt `default_profile` behavior.
|
||||
- Verify explicit `--profile` override behavior.
|
||||
- Verify `--config` behavior (default and explicit).
|
||||
- Verify output file creation when `--out` is used.
|
||||
- Verify stderr capture on failures.
|
||||
- Avoid real API keys in tests.
|
||||
|
||||
## 19. Security and Privacy Notes
|
||||
|
||||
- Never pass raw API keys on command line.
|
||||
- Do not log full rendered prompts by default; transcripts may contain sensitive content.
|
||||
- Avoid logging prompt content unless explicit debug mode is enabled.
|
||||
- Treat generated artifacts as potentially sensitive.
|
||||
- Use session-scoped, access-controlled output paths.
|
||||
- `api_key_env` names should come from environment management, not embedded secrets.
|
||||
|
||||
## 20. Initial D&D Artifact Generation Examples
|
||||
|
||||
These are examples only. Use prompt IDs from the deployed prompt library.
|
||||
|
||||
Session recap:
|
||||
|
||||
```bash
|
||||
scriptorium run \
|
||||
--prompt dnd.session_recap \
|
||||
--input transcript=/work/session-42/transcript.polished.md \
|
||||
--input glossary=/work/session-42/glossary.yml \
|
||||
--out /work/session-42/artifacts/session_recap.md
|
||||
```
|
||||
|
||||
Structured events:
|
||||
|
||||
```bash
|
||||
scriptorium run \
|
||||
--prompt dnd.structured_events \
|
||||
--input transcript=/work/session-42/transcript.polished.md \
|
||||
--out /work/session-42/artifacts/structured_events.json
|
||||
```
|
||||
|
||||
Glossary suggestions:
|
||||
|
||||
```bash
|
||||
scriptorium run \
|
||||
--prompt dnd.glossary_suggestions \
|
||||
--input transcript=/work/session-42/transcript.polished.md \
|
||||
--input previous_recap=/work/session-41/artifacts/session_recap.md \
|
||||
--out /work/session-42/artifacts/glossary_suggestions.md
|
||||
```
|
||||
|
||||
Player-facing summary:
|
||||
|
||||
```bash
|
||||
scriptorium run \
|
||||
--prompt dnd.player_summary \
|
||||
--input transcript=/work/session-42/transcript.polished.md \
|
||||
--input structured_events=/work/session-42/artifacts/structured_events.json \
|
||||
--out /work/session-42/artifacts/player_summary.md
|
||||
```
|
||||
|
||||
## 21. Non-Goals
|
||||
|
||||
Initial Narratio integration should not:
|
||||
|
||||
- call Scriptorium internal Go packages
|
||||
- use HTTP API as the primary path
|
||||
- expect Scriptorium to read S3 refs directly
|
||||
- make Scriptorium responsible for Narratio stage state
|
||||
- make Scriptorium responsible for notification
|
||||
- require Scriptorium to understand D&D workflow semantics beyond prompt definitions
|
||||
|
||||
## 22. Future Extension Notes
|
||||
|
||||
Possible later extensions:
|
||||
|
||||
- HTTP API integration
|
||||
- S3 artifact references if Scriptorium adds S3 reader support
|
||||
- richer render diagnostics and policy controls
|
||||
- token budgeting/prompt-size checks
|
||||
- batch execution if Scriptorium later adds batch support
|
||||
## Request Contract
|
||||
Both request types carry:
|
||||
- binary/config/prompt/profile IDs;
|
||||
- input map and vars map;
|
||||
- output path;
|
||||
- timeout;
|
||||
- generated config + stdout/stderr log paths;
|
||||
- optional API-key env var name;
|
||||
- optional working directory.
|
||||
|
||||
## Result Contract
|
||||
`ArtifactResult` returns:
|
||||
- output/log/generated-config paths;
|
||||
- exit code and duration;
|
||||
- command mode (`run` or `render`);
|
||||
- prompt/profile provenance;
|
||||
- `ValidationFailed` marker;
|
||||
- metadata map.
|
||||
|
||||
## Validation and Failure Semantics
|
||||
Request validation fails for:
|
||||
- missing binary, prompt id, or output path;
|
||||
- non-positive timeout;
|
||||
- empty input/var names;
|
||||
- empty input path values;
|
||||
- missing required credential env var when `APIKeyEnv` is set.
|
||||
|
||||
Run behavior:
|
||||
- subprocess errors propagate with context;
|
||||
- `run` exit code `2` is mapped to `ValidationFailed=true`;
|
||||
- successful subprocess still fails if output file is missing or empty.
|
||||
|
||||
Render behavior:
|
||||
- subprocess errors propagate;
|
||||
- output file must exist and be non-empty.
|
||||
|
||||
## Deterministic Behavior
|
||||
- input and var maps are sorted into deterministic `--input` and `--var` CLI args.
|
||||
- generated invocation YAML (`scriptorium.generated.v1`) is emitted when requested.
|
||||
- adapter is stateless and does not own artifact-selection policy.
|
||||
|
||||
## Config Mapping
|
||||
Config fields consumed through runner/stage wiring are under `pipeline.scriptorium.*` plus per-artifact settings under `pipeline.scriptorium.artifacts.*`.
|
||||
|
||||
Maintained examples with Scriptorium config:
|
||||
- `examples/pipeline.full.annotated.yml`
|
||||
- `examples/pipeline.production.yml`
|
||||
|
||||
@@ -1,403 +1,56 @@
|
||||
# seriatim
|
||||
|
||||
`seriatim` merges per-speaker WhisperX-style JSON transcripts into a single JSON transcript that preserves speaker identity and chronological order.
|
||||
|
||||
The current implementation supports the `merge` command. It reads one or more input JSON files, optionally maps each input file to a canonical speaker using `speakers.yml`, sorts all segments by timestamp, detects and resolves overlaps when word-level timing is available, assigns consecutive numeric `id` values, and writes a merged JSON artifact.
|
||||
|
||||
## Usage
|
||||
|
||||
Run from source:
|
||||
|
||||
```sh
|
||||
go run ./cmd/seriatim merge \
|
||||
--input-file samples/raw/2026-04-19-Eric_Rakestraw.json \
|
||||
--input-file samples/raw/2026-04-19-Mike_Brown.json \
|
||||
--output-file merged.json
|
||||
```
|
||||
|
||||
Optional report output:
|
||||
|
||||
```sh
|
||||
go run ./cmd/seriatim merge \
|
||||
--input-file eric.json \
|
||||
--input-file mike.json \
|
||||
--output-file merged.json \
|
||||
--report-file report.json
|
||||
```
|
||||
|
||||
## CLI
|
||||
|
||||
```text
|
||||
seriatim merge [flags]
|
||||
```
|
||||
|
||||
Global flags:
|
||||
|
||||
| Flag | Description |
|
||||
| --- | --- |
|
||||
| `--help` | Show command help. |
|
||||
| `--version` | Show application version. Local builds default to `dev`; release builds inject the release version. |
|
||||
|
||||
`merge` flags:
|
||||
|
||||
| Flag | Required | Default | Description |
|
||||
| --- | --- | --- | --- |
|
||||
| `--input-file` | Yes | none | Input transcript JSON file. Repeat once per speaker/input file. |
|
||||
| `--output-file` | Yes | none | Merged transcript JSON output path. |
|
||||
| `--report-file` | No | none | Optional report JSON output path. |
|
||||
| `--speakers` | No | none | Speaker map YAML file. When omitted, input file basenames are used as speaker labels. |
|
||||
| `--autocorrect` | No | none | Autocorrect rules YAML file. When omitted, the default `autocorrect` module leaves text unchanged. |
|
||||
| `--input-reader` | No | `json-files` | Input reader module. |
|
||||
| `--output-modules` | No | `json` | Comma-separated output modules. |
|
||||
| `--output-schema` | No | `seriatim-intermediate` | JSON output contract. Allowed values are `seriatim-minimal`, `seriatim-intermediate`, and `seriatim-full`. If omitted, the runtime default is used; consumers that depend on a specific shape should set this explicitly. |
|
||||
| `--preprocessing-modules` | No | `validate-raw,normalize-speakers,trim-text` | Comma-separated preprocessing modules, evaluated in order. |
|
||||
| `--postprocessing-modules` | No | `detect-overlaps,resolve-overlaps,backchannel,filler,resolve-danglers,coalesce,detect-overlaps,autocorrect,assign-ids,validate-output` | Comma-separated postprocessing modules, evaluated in order. |
|
||||
| `--coalesce-gap` | No | `3.0` | Maximum same-speaker gap in seconds for `coalesce`; also used as the `resolve-overlaps` context window. Must be a non-negative float. |
|
||||
|
||||
Environment variables:
|
||||
|
||||
| Environment Variable | Default | Description |
|
||||
| --- | --- | --- |
|
||||
| `SERIATIM_OUTPUT_SCHEMA` | `seriatim-intermediate` | Output schema used when `--output-schema` is not explicitly provided. Allowed values are `seriatim-minimal`, `seriatim-intermediate`, and `seriatim-full`. The CLI flag takes precedence. |
|
||||
| `SERIATIM_OVERLAP_WORD_RUN_GAP` | `1.0` | Maximum gap in seconds between adjacent timed words when `resolve-overlaps` builds word-run replacement segments. Must be a positive float. |
|
||||
| `SERIATIM_OVERLAP_WORD_RUN_REORDER_WINDOW` | `1.0` | Near-start window in seconds for ordering replacement word runs shortest-first. Must be a positive float. |
|
||||
| `SERIATIM_BACKCHANNEL_MAX_DURATION` | `2.0` | Maximum duration in seconds for `backchannel` classification. Must be a positive float. |
|
||||
| `SERIATIM_FILLER_MAX_DURATION` | `1.25` | Maximum duration in seconds for `filler` classification. Must be a positive float. |
|
||||
|
||||
## Input JSON Format
|
||||
|
||||
Each input file must be valid JSON with a top-level `segments` array. The current parser accepts the WhisperX segment subset needed for merging:
|
||||
|
||||
```json
|
||||
{
|
||||
"segments": [
|
||||
{
|
||||
"start": 1.25,
|
||||
"end": 3.5,
|
||||
"text": "Hello there.",
|
||||
"words": [
|
||||
{"word": "Hello", "start": 1.25, "end": 1.55, "score": 0.98},
|
||||
{"word": "there.", "start": 1.7, "end": 2.0}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Required segment fields:
|
||||
|
||||
- `start`: number, must be `>= 0`.
|
||||
- `end`: number, must be `>= start`.
|
||||
- `text`: string.
|
||||
|
||||
Optional word fields:
|
||||
|
||||
- `words`: array of word timing objects.
|
||||
- `words[].word`: string.
|
||||
- `words[].start`: optional number, must be `>= 0` when present.
|
||||
- `words[].end`: optional number, must be `>= start` when present with `start`.
|
||||
- `words[].score`: optional number.
|
||||
- `words[].speaker`: optional raw speaker label string.
|
||||
|
||||
Word-level timing is preserved internally for overlap resolution. If a word is missing `start` or `end`, seriatim keeps the word text, emits a warning in the optional report, and does not use that word as a timing anchor. Word timing is not emitted in the final JSON artifact.
|
||||
|
||||
## Speaker Map Format
|
||||
|
||||
`speakers.yml` maps input files to canonical speaker names using ordered substring rules:
|
||||
|
||||
This file is optional. If `--speakers` is omitted, `seriatim` uses each input file basename as the segment speaker label.
|
||||
|
||||
```yaml
|
||||
match:
|
||||
- speaker: "Eric Rakestraw"
|
||||
match:
|
||||
- "Eric_Rakestraw"
|
||||
- "Eric"
|
||||
|
||||
- speaker: "Mike Brown"
|
||||
match:
|
||||
- "Mike_Brown"
|
||||
- "mb"
|
||||
```
|
||||
|
||||
For each `--input-file`, `seriatim` takes the file basename and evaluates the rules in order. The first rule with a matching substring wins, and no later rules are evaluated.
|
||||
|
||||
For example, this input:
|
||||
|
||||
```text
|
||||
samples/raw/2026-04-19-Eric_Rakestraw.json
|
||||
```
|
||||
|
||||
matches this rule because the basename contains `Eric_Rakestraw`:
|
||||
|
||||
```yaml
|
||||
- speaker: "Eric Rakestraw"
|
||||
match:
|
||||
- "Eric_Rakestraw"
|
||||
```
|
||||
|
||||
Important details:
|
||||
|
||||
- Matching is against the input file basename, not the full path.
|
||||
- Matching is case-insensitive.
|
||||
- Rules are evaluated from first to last.
|
||||
- Each rule must have a non-empty `speaker`.
|
||||
- Each rule must have at least one non-empty `match` string.
|
||||
- Duplicate speaker names are invalid.
|
||||
- Every input file must match at least one rule or the command fails.
|
||||
|
||||
Deprecated old format:
|
||||
|
||||
```yaml
|
||||
inputs:
|
||||
eric.json:
|
||||
speaker: "Eric Rakestraw"
|
||||
```
|
||||
|
||||
The old `inputs:` direct mapping format is no longer supported.
|
||||
|
||||
## Output JSON Format
|
||||
|
||||
`--output-modules json` controls the writer. `--output-schema` controls the JSON contract that writer serializes.
|
||||
|
||||
The named schemas are stable public contracts. If a consumer depends on a specific shape, it should request that schema explicitly at runtime. The runtime default selection may change in a future release.
|
||||
|
||||
The `seriatim-intermediate` schema is the current default selection when neither `--output-schema` nor `SERIATIM_OUTPUT_SCHEMA` is set. It stays close to the minimal schema, but adds optional `categories` on each segment:
|
||||
|
||||
```json
|
||||
{
|
||||
"metadata": {
|
||||
"application": "seriatim",
|
||||
"version": "dev",
|
||||
"output_schema": "seriatim-intermediate"
|
||||
},
|
||||
"segments": [
|
||||
{
|
||||
"id": 1,
|
||||
"start": 1.25,
|
||||
"end": 3.5,
|
||||
"speaker": "Eric Rakestraw",
|
||||
"text": "Hello there.",
|
||||
"categories": ["backchannel"]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
The `seriatim-full` schema uses the full seriatim envelope:
|
||||
|
||||
```json
|
||||
{
|
||||
"metadata": {
|
||||
"application": "seriatim",
|
||||
"version": "dev",
|
||||
"input_reader": "json-files",
|
||||
"input_files": ["eric.json", "mike.json"],
|
||||
"preprocessing_modules": ["validate-raw", "normalize-speakers", "trim-text"],
|
||||
"postprocessing_modules": ["detect-overlaps", "resolve-overlaps", "backchannel", "filler", "resolve-danglers", "coalesce", "detect-overlaps", "autocorrect", "assign-ids", "validate-output"],
|
||||
"output_modules": ["json"]
|
||||
},
|
||||
"segments": [
|
||||
{
|
||||
"id": 1,
|
||||
"source": "eric.json",
|
||||
"source_segment_index": 0,
|
||||
"speaker": "Eric Rakestraw",
|
||||
"start": 1.25,
|
||||
"end": 3.5,
|
||||
"text": "Hello there.",
|
||||
"overlap_group_id": 1
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"source": "eric.json",
|
||||
"source_ref": "word-run:1:1:1",
|
||||
"derived_from": ["eric.json#0"],
|
||||
"speaker": "Eric Rakestraw",
|
||||
"start": 2.0,
|
||||
"end": 2.5,
|
||||
"text": "Resolved word run",
|
||||
"categories": ["backchannel"]
|
||||
}
|
||||
],
|
||||
"overlap_groups": [
|
||||
{
|
||||
"id": 1,
|
||||
"start": 1.25,
|
||||
"end": 4.0,
|
||||
"segments": ["eric.json#0", "mike.json#0"],
|
||||
"speakers": ["Eric Rakestraw", "Mike Brown"],
|
||||
"class": "unknown",
|
||||
"resolution": "unresolved"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
The `seriatim-minimal` schema emits minimal metadata and compact ordered segments:
|
||||
|
||||
```json
|
||||
{
|
||||
"metadata": {
|
||||
"application": "seriatim",
|
||||
"version": "dev",
|
||||
"output_schema": "seriatim-minimal"
|
||||
},
|
||||
"segments": [
|
||||
{
|
||||
"id": 1,
|
||||
"start": 1.25,
|
||||
"end": 3.5,
|
||||
"speaker": "Eric Rakestraw",
|
||||
"text": "Hello there."
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Minimal output intentionally omits categories, overlap groups, source/provenance fields, and pipeline configuration metadata.
|
||||
|
||||
Intermediate output intentionally omits overlap groups and source/provenance fields, but keeps optional `categories` and minimal metadata.
|
||||
|
||||
Segments are sorted deterministically by:
|
||||
|
||||
```text
|
||||
(start, end, source, source_segment_index/source_ref, speaker)
|
||||
```
|
||||
|
||||
Final segment IDs are assigned after sorting and start at `1`.
|
||||
|
||||
The public Go output contract is available from:
|
||||
|
||||
```go
|
||||
import "gitea.maximumdirect.net/eric/seriatim/schema"
|
||||
```
|
||||
|
||||
The same package embeds machine-readable JSON Schemas in `schema/full-output.schema.json`, `schema/intermediate-output.schema.json`, and `schema/minimal-output.schema.json`. The default `validate-output` postprocessor validates the selected output shape and verifies final segment IDs are present, sequential, and start at `1`.
|
||||
|
||||
## Overlap Detection
|
||||
|
||||
The default postprocessing pipeline detects overlapping segment groups.
|
||||
|
||||
Overlap behavior:
|
||||
|
||||
- A strict timing overlap is required: `next.start < current_group_end`.
|
||||
- Segments that only touch at a boundary are not grouped.
|
||||
- Groups require at least two distinct speakers.
|
||||
- Transitive overlaps are grouped together.
|
||||
- Segments in detected groups receive `overlap_group_id`.
|
||||
- `overlap_groups[].segments` contains stable references in `source#source_segment_index` format.
|
||||
- `class` is currently `unknown`.
|
||||
- `resolution` is `unresolved` until `resolve-overlaps` replaces the group.
|
||||
|
||||
## Overlap Resolution
|
||||
|
||||
The default postprocessing pipeline runs `detect-overlaps`, then `resolve-overlaps`, then `backchannel`, then `filler`, then `resolve-danglers`, then `coalesce`, then a second `detect-overlaps` pass.
|
||||
|
||||
For each detected overlap group, `resolve-overlaps` uses preserved WhisperX word timing to build smaller word-run replacement segments:
|
||||
|
||||
- The resolution window expands the detected overlap group by `--coalesce-gap` seconds on both sides.
|
||||
- Nearby same-speaker context segments are included when they intersect the expanded window and their start or end is within `--coalesce-gap` of the original overlap boundary.
|
||||
- Once a segment is selected for replacement, all timed words from that segment participate in word-run construction; the window controls segment selection, not per-word clipping.
|
||||
- Context segments that are part of another detected overlap group are not pulled into the current group.
|
||||
- Untimed words are included in replacement text in original word order when nearby timed words create a replacement run.
|
||||
- Untimed words do not affect replacement segment start/end times or word-run gap splitting.
|
||||
- Words for the same speaker are merged into one run when the gap between adjacent words is no greater than `SERIATIM_OVERLAP_WORD_RUN_GAP`.
|
||||
- The default word-run gap is `1.0` seconds.
|
||||
- Set `SERIATIM_OVERLAP_WORD_RUN_GAP` to a positive number of seconds to override the default.
|
||||
- Near-start replacement word runs are reordered so shorter segments come first when adjacent starts are within `SERIATIM_OVERLAP_WORD_RUN_REORDER_WINDOW`.
|
||||
- The default word-run reorder window is `1.0` seconds.
|
||||
- Set `SERIATIM_OVERLAP_WORD_RUN_REORDER_WINDOW` to a positive number of seconds to override the default.
|
||||
- Replacement segment text is built by joining word text with single spaces.
|
||||
- Replacement segments include `source_ref` and `derived_from`.
|
||||
- Replacement segments omit `source_segment_index` because they are derived from one or more original segments.
|
||||
- Resolved overlap groups are removed before the second detection pass.
|
||||
- Replacement segments are left without `overlap_group_id` until the second detection pass annotates any remaining overlap.
|
||||
- If a speaker has no usable word timing in a group, that speaker's original segment is kept.
|
||||
- If no speakers in a group have usable word timing, the original group and annotations remain unchanged.
|
||||
|
||||
## Backchannels
|
||||
|
||||
The default pipeline runs `backchannel` before `coalesce`. It tags short acknowledgement segments with:
|
||||
|
||||
```json
|
||||
"categories": ["backchannel"]
|
||||
```
|
||||
|
||||
Backchannel matching is case-insensitive, ignores punctuation for matching and word-count purposes, trims surrounding whitespace, and requires a matching acknowledgement phrase, no more than three whitespace-delimited words, and duration no greater than `SERIATIM_BACKCHANNEL_MAX_DURATION` seconds. The default maximum duration is `2.0` seconds.
|
||||
|
||||
## Fillers
|
||||
|
||||
The default pipeline runs `filler` after `backchannel` and before `coalesce`. It tags short filler utterances with:
|
||||
|
||||
```json
|
||||
"categories": ["filler"]
|
||||
```
|
||||
|
||||
Filler matching is case-insensitive, ignores punctuation for matching and word-count purposes, trims surrounding whitespace, and requires only filler tokens such as `um`, `uh`, `er`, `erm`, `ah`, `eh`, `hmm`, `mm`, or repeated combinations of those tokens. Matching segments must contain no more than three whitespace-delimited words and have duration no greater than `SERIATIM_FILLER_MAX_DURATION` seconds. The default maximum duration is `1.25` seconds.
|
||||
|
||||
## Dangler Resolution
|
||||
|
||||
The default pipeline runs `resolve-danglers` before `coalesce` and before the second overlap detection pass. It repairs short derived fragments when they share provenance with a nearby segment:
|
||||
|
||||
- Dangling-end fragments have no more than two words and end in punctuation.
|
||||
- Dangling-start fragments have no more than two words.
|
||||
- Matching uses same-speaker segments with any shared `derived_from` value.
|
||||
- Merged segments use `source_ref` values such as `resolve-danglers:1`, keep the target segment's transcript position, and union `derived_from`.
|
||||
|
||||
## Coalescing
|
||||
|
||||
The default pipeline runs `coalesce` after `resolve-danglers` and before the second overlap detection pass. It merges adjacent same-speaker segments in the transcript's current order when `next.start - current.end <= --coalesce-gap`.
|
||||
|
||||
Coalesced segments use `source_ref` values such as `coalesce:1`, include `derived_from`, and omit `source_segment_index`.
|
||||
|
||||
Different-speaker backchannel and filler segments do not block coalescing of surrounding same-speaker segments. Same-speaker backchannel and filler segments are merged normally when they are within `--coalesce-gap`. When same-speaker segments are coalesced, any `backchannel` or `filler` category from the merged inputs is dropped from the coalesced segment.
|
||||
|
||||
## Autocorrect
|
||||
|
||||
Autocorrect is included in the default postprocessing pipeline. If `--autocorrect` is omitted, the module leaves transcript text unchanged and records a skip event in the optional report.
|
||||
|
||||
Enable corrections by passing `--autocorrect`:
|
||||
|
||||
```sh
|
||||
go run ./cmd/seriatim merge \
|
||||
--input-file input.json \
|
||||
--autocorrect autocorrect.yml \
|
||||
--output-file merged.json
|
||||
```
|
||||
|
||||
`autocorrect.yml` format:
|
||||
|
||||
```yaml
|
||||
autocorrect:
|
||||
- target: "Hrank"
|
||||
match:
|
||||
- "hrank"
|
||||
- "Frank"
|
||||
|
||||
- target: "Mike Brown"
|
||||
match:
|
||||
- "Mike Pat"
|
||||
```
|
||||
|
||||
Matching behavior:
|
||||
|
||||
- Matching is case-sensitive.
|
||||
- Matches apply only to whole tokens, not substrings inside larger words.
|
||||
- Punctuation and whitespace can surround a match.
|
||||
- Multi-word and hyphenated matches are supported.
|
||||
- Duplicate match strings are invalid, including duplicates across separate rules.
|
||||
|
||||
## Current Limitations
|
||||
|
||||
- Only JSON input is supported.
|
||||
- Overlap resolution depends on WhisperX word timing; groups without usable word timing remain unresolved.
|
||||
- Alternate output formats are not implemented yet.
|
||||
|
||||
## Release Builds
|
||||
|
||||
Local builds record version metadata as `dev`. Release builds should inject the release version with `ldflags`:
|
||||
|
||||
```sh
|
||||
go build -ldflags "-X gitea.maximumdirect.net/eric/seriatim/internal/buildinfo.Version=v1.0.0" ./cmd/seriatim
|
||||
```
|
||||
# Integration: Seriatim
|
||||
|
||||
## Purpose
|
||||
Define the Seriatim adapter contract used by `merge`, `normalize`, and `trim`.
|
||||
|
||||
## Adapter Boundary
|
||||
Interface:
|
||||
- `seriatim.Runner`
|
||||
- methods:
|
||||
- `Run(ctx, MergeRequest)`
|
||||
- `Normalize(ctx, NormalizeRequest)`
|
||||
- `Trim(ctx, TrimRequest)`
|
||||
|
||||
Primary implementation:
|
||||
- `internal/adapters/seriatim/SubprocessRunner`
|
||||
|
||||
Execution modes:
|
||||
- `seriatim merge`
|
||||
- `seriatim normalize`
|
||||
- `seriatim trim`
|
||||
|
||||
## Request/Result Contracts
|
||||
- `MergeRequest`/`MergeResult`: multi-input merge to base transcript, optional report.
|
||||
- `NormalizeRequest`/`NormalizeResult`: transcript normalization with explicit schema.
|
||||
- `TrimRequest`/`TrimResult`: transcript trimming with required keep selector.
|
||||
|
||||
Results include output/log/config paths, timing, exit code, and metadata.
|
||||
|
||||
## Validation and Failure Semantics
|
||||
Runner construction validates:
|
||||
- binary presence;
|
||||
- timeout > 0;
|
||||
- supported output schema (`seriatim-minimal|seriatim-intermediate|seriatim-full`);
|
||||
- non-negative coalesce gap.
|
||||
|
||||
Invocation fails on:
|
||||
- missing required request paths/inputs;
|
||||
- invalid normalize schema override;
|
||||
- subprocess failure;
|
||||
- invalid JSON outputs;
|
||||
- missing `segments` array for normalize/trim transcript outputs.
|
||||
|
||||
When report paths are provided/enabled, report files must parse as JSON.
|
||||
|
||||
## Deterministic Behavior
|
||||
- argument ordering is deterministic per command construction.
|
||||
- merge env overrides are explicit (`SERIATIM_*`) and only emitted when configured.
|
||||
- generated invocation YAML (`seriatim.generated.v1`) is emitted when requested.
|
||||
- adapter does not write manifests or choose stage inputs.
|
||||
|
||||
## Config Mapping
|
||||
Config fields consumed through runner/stage wiring are under `pipeline.seriatim.*`.
|
||||
|
||||
Maintained examples with Seriatim config:
|
||||
- `examples/pipeline.full.annotated.yml`
|
||||
- `examples/pipeline.production.yml`
|
||||
|
||||
43
docs/internal/README.md
Normal file
43
docs/internal/README.md
Normal file
@@ -0,0 +1,43 @@
|
||||
# Internal Documentation Index
|
||||
|
||||
## Audience
|
||||
Developers and coding agents changing Narratio internals.
|
||||
|
||||
## Scope
|
||||
`docs/internal/` documents implemented internal contracts: stage boundaries, manifest/state behavior, artifact resolution, restore behavior, storage boundaries, and workspace invariants.
|
||||
|
||||
User and operator behavior belongs in:
|
||||
- `docs/cli.md`
|
||||
- `docs/config.md`
|
||||
- `docs/operations.md`
|
||||
- `docs/troubleshooting.md`
|
||||
|
||||
## Pipeline Stage Set
|
||||
Canonical stage order from `internal/stage.All()`:
|
||||
1. `prepare`
|
||||
2. `transcribe`
|
||||
3. `merge`
|
||||
4. `polish`
|
||||
5. `normalize`
|
||||
6. `trim`
|
||||
7. `analyze`
|
||||
8. `publish`
|
||||
9. `notify` (placeholder)
|
||||
|
||||
`notify` is currently a placeholder stage with optional notifier call behavior; it has no persisted pipeline outputs.
|
||||
|
||||
## Internal Component Docs
|
||||
- `adapters.md`: external adapter boundaries and default runtime wiring.
|
||||
- `artifacts.md`: canonical source IDs, runtime catalog behavior, and resolution rules.
|
||||
- `manifest.md`: session and run manifest contracts.
|
||||
- `storage.md`: object-store interface and S3 implementation behavior.
|
||||
- `workspace.md`: local session layout, run-local layout, and cleanup guardrails.
|
||||
- `command-restore.md`: restore discovery, planning, execution, and reporting.
|
||||
- `stage-prepare.md`
|
||||
- `stage-transcribe.md`
|
||||
- `stage-merge.md`
|
||||
- `stage-polish.md`
|
||||
- `stage-normalize.md`
|
||||
- `stage-trim.md`
|
||||
- `stage-analyze.md`
|
||||
- `stage-publish.md`
|
||||
52
docs/internal/adapters.md
Normal file
52
docs/internal/adapters.md
Normal file
@@ -0,0 +1,52 @@
|
||||
# Internal: Adapters
|
||||
|
||||
## Purpose
|
||||
Define external integration boundaries and default adapter wiring used by app/stage orchestration.
|
||||
|
||||
## Adapter Boundaries
|
||||
Narratio stage logic depends on adapter interfaces, not transport-specific details.
|
||||
|
||||
Primary adapters:
|
||||
- `whisperx.Client`
|
||||
- `seriatim.Runner`
|
||||
- `audita.Runner`
|
||||
- `scriptorium.Runner`
|
||||
- `storage.ObjectStore`
|
||||
- `notify.Sender`
|
||||
|
||||
Legacy compatibility boundary:
|
||||
- `storage.Backend` remains in the storage adapter package and defaults to `NoopBackend`; current pipeline stages use `storage.ObjectStore`.
|
||||
|
||||
## Ownership
|
||||
Adapters own:
|
||||
- HTTP/subprocess/SDK argument and transport details.
|
||||
- Backend-specific request/response mapping.
|
||||
|
||||
Adapters do not own:
|
||||
- stage ordering/skip/force logic;
|
||||
- manifest transitions;
|
||||
- canonical path policy.
|
||||
|
||||
## Default Wiring
|
||||
`internal/app/runner.go` initializes default adapters when not injected:
|
||||
- WhisperX HTTP client from pipeline config.
|
||||
- Seriatim subprocess runner.
|
||||
- Audita subprocess runner.
|
||||
- Scriptorium subprocess runner.
|
||||
- Noop notifier (`notify.NoopSender`).
|
||||
- Object store only when required by selected stages/config.
|
||||
|
||||
Object-store construction goes through `newCommandObjectStore`, which loads configured filesystem secrets before adapter initialization.
|
||||
|
||||
## Failure Semantics
|
||||
- Constructor errors fail stage execution setup early.
|
||||
- Runtime adapter errors propagate to stage code and then manifest failure handling.
|
||||
- Subprocess adapters persist stage logs/generated configs through stage-managed paths.
|
||||
|
||||
## Test Surfaces
|
||||
- `internal/adapters/whisperx/http_test.go`
|
||||
- `internal/adapters/seriatim/subprocess_test.go`
|
||||
- `internal/adapters/audita/subprocess_test.go`
|
||||
- `internal/adapters/scriptorium/subprocess_test.go`
|
||||
- `internal/adapters/storage/*_test.go`
|
||||
- `internal/app/runner_test.go`
|
||||
112
docs/internal/artifacts.md
Normal file
112
docs/internal/artifacts.md
Normal file
@@ -0,0 +1,112 @@
|
||||
# Internal: Artifacts
|
||||
|
||||
## Purpose
|
||||
Define canonical artifact IDs, runtime catalog behavior, source resolution rules, and shared current-state mechanics used by app and previous-cache code.
|
||||
|
||||
## Built-in Source IDs
|
||||
|
||||
- `narratio.transcript.base` -> `transcripts/base.json` (`merge`)
|
||||
- `narratio.transcript.polished` -> `transcripts/polished.json` (`polish`)
|
||||
- `narratio.transcript.final` -> `transcripts/final.json` (`normalize`)
|
||||
- `narratio.transcript.final_trimmed` -> `transcripts/final.trimmed.json` (`trim`)
|
||||
- `narratio.bounds.session` -> `artifacts/session_bounds.json` (`trim`)
|
||||
|
||||
## Configured and Previous-Session Sources
|
||||
|
||||
- configured source ID format: `narratio.artifact.<artifact_key>`
|
||||
- previous-session source ID format: `narratio.previous_session.artifact.<artifact_key>`
|
||||
|
||||
Both formats are validated by strict source-policy rules.
|
||||
|
||||
## Runtime Catalog
|
||||
|
||||
`ArtifactCatalog` tracks:
|
||||
|
||||
- `planned`: source registered for run context;
|
||||
- `executable`: selected and enabled for analyze execution;
|
||||
- `available`: local file exists and validates;
|
||||
- `provenance`: availability source.
|
||||
|
||||
Current provenance values:
|
||||
|
||||
- `generated.current_analyze_run`
|
||||
- `filesystem.disabled_artifact_output`
|
||||
- `manifest.inputs.previous_cache`
|
||||
- `current_session.previous_cache`
|
||||
|
||||
## Resolution Rules
|
||||
|
||||
Built-ins:
|
||||
|
||||
1. manifest producer outputs (when present)
|
||||
2. canonical session-path fallback
|
||||
|
||||
Configured sources (`narratio.artifact.*`):
|
||||
|
||||
- resolve only through runtime catalog availability.
|
||||
|
||||
Previous-session sources (`narratio.previous_session.artifact.*`):
|
||||
|
||||
- resolve only from local `previous/` cache state;
|
||||
- prefer manifest-backed previous-input paths;
|
||||
- fallback to existing previous-cache filesystem paths.
|
||||
|
||||
Validation by content type:
|
||||
|
||||
- transcript built-ins: JSON with top-level `segments` array;
|
||||
- bounds built-in: valid JSON;
|
||||
- configured/previous-session artifact files: non-empty text file.
|
||||
|
||||
## Previous Requirement Collection
|
||||
|
||||
`CollectPreviousArtifactRequirements`:
|
||||
|
||||
- scans enabled configured artifacts only;
|
||||
- extracts only canonical previous-session sources;
|
||||
- deduplicates by artifact key;
|
||||
- merges required and optional references (required wins);
|
||||
- returns deterministic ordering and source locations.
|
||||
|
||||
## Current-State Helpers
|
||||
|
||||
Artifacts package owns shared remote current-state loading mechanics used by restore, status/validate checks, and previous-cache planning.
|
||||
|
||||
Core helpers:
|
||||
|
||||
- `LoadCurrentRunPointer`
|
||||
- `LoadCurrentManifest`
|
||||
- `LoadCurrentState`
|
||||
- `ValidateCurrentStateIdentity`
|
||||
|
||||
Typed missing-state errors:
|
||||
|
||||
- `CurrentRunPointerMissingError` (`ErrCurrentRunPointerMissing`)
|
||||
- `CurrentManifestMissingError` (`ErrCurrentManifestMissing`)
|
||||
|
||||
Identity validation supports caller-provided expectations:
|
||||
|
||||
- expected campaign;
|
||||
- expected session ID;
|
||||
- expected run ID, or pointer/manifest run-ID consistency check.
|
||||
|
||||
Caller policy is intentionally outside artifacts helpers:
|
||||
|
||||
- some callers fail on missing current state;
|
||||
- some callers downgrade missing state to status/findings;
|
||||
- some callers skip optional behavior when state is missing.
|
||||
|
||||
## Key Path Helpers
|
||||
|
||||
`internal/artifacts/paths.go` and S3-key helpers define canonical helpers for:
|
||||
|
||||
- session/work/run paths;
|
||||
- previous-cache paths;
|
||||
- spool/cache paths;
|
||||
- S3 session/run/current-state key layout.
|
||||
|
||||
## Invariants
|
||||
|
||||
- source ID formats are stable contracts;
|
||||
- artifact resolution is deterministic and manifest-aware;
|
||||
- previous-session source resolution in `analyze` is local-only;
|
||||
- remote current-state key construction remains centralized in artifacts helpers.
|
||||
84
docs/internal/command-restore.md
Normal file
84
docs/internal/command-restore.md
Normal file
@@ -0,0 +1,84 @@
|
||||
# Internal: Command Restore
|
||||
|
||||
## Purpose
|
||||
Define the implemented `narratio session restore` command contract:
|
||||
|
||||
- committed remote current-state discovery;
|
||||
- deterministic restore planning;
|
||||
- safe local install semantics;
|
||||
- durable restore reporting.
|
||||
|
||||
## Discovery Contract
|
||||
|
||||
Restore resolves remote committed state from the session publish current pointers:
|
||||
|
||||
- `current/run_id.txt` (required, non-empty);
|
||||
- `current/manifest.json` (required, decodable).
|
||||
|
||||
Current-state discovery uses shared artifacts-level mechanics and validates identity against the resolved request config:
|
||||
|
||||
- campaign must match;
|
||||
- session ID must match.
|
||||
|
||||
Restore treats any missing or invalid remote current state as a command error.
|
||||
|
||||
## Planning Contract
|
||||
|
||||
Restore planner action kinds:
|
||||
|
||||
- `download`;
|
||||
- `skip_same`;
|
||||
- `conflict`.
|
||||
|
||||
Planner behavior:
|
||||
|
||||
- remote list scope is the resolved session prefix;
|
||||
- remote-to-local mapping is traversal-safe;
|
||||
- actions are sorted deterministically by local relative path.
|
||||
|
||||
Restore scope from current remote state:
|
||||
|
||||
- include `manifest.json`;
|
||||
- include `transcripts/**`;
|
||||
- include `artifacts/**`;
|
||||
- include `audio/**` only with `--include-audio`.
|
||||
|
||||
Explicit exclusions from current remote state mapping:
|
||||
|
||||
- `current/**`;
|
||||
- `runs/**`;
|
||||
- `logs/**`;
|
||||
- `reports/**`;
|
||||
- `config/**`;
|
||||
- `inputs/**`;
|
||||
- `previous/**`.
|
||||
|
||||
Previous-cache files are planned separately through `previouscache.BuildPlan` when configured previous-session requirements exist.
|
||||
|
||||
## Execution Contract
|
||||
|
||||
Execution order and safety:
|
||||
|
||||
- non-manifest downloads happen before manifest install;
|
||||
- `manifest.json` installs last;
|
||||
- downloads use sibling temp files plus atomic rename;
|
||||
- manifest replacement is validated before rename;
|
||||
- failed installs do not roll back files already written in the same execution.
|
||||
|
||||
Audio restore path:
|
||||
|
||||
- uses `audio.MaterializeS3Audio`;
|
||||
- integrates spool and S3 audio cache paths;
|
||||
- supports cache-hit reuse without object redownload.
|
||||
|
||||
## Reporting Contract
|
||||
|
||||
- `--dry-run`: prints summary only; no local writes.
|
||||
- non-dry-run: writes `reports/restore-latest.json`.
|
||||
- report includes plan counts, per-action status, and execution failures.
|
||||
|
||||
## Invariants
|
||||
|
||||
- restore uses committed remote current state as authority;
|
||||
- `current/run_id.txt` is the remote publish commit marker;
|
||||
- restore does not execute pipeline stages.
|
||||
57
docs/internal/manifest.md
Normal file
57
docs/internal/manifest.md
Normal file
@@ -0,0 +1,57 @@
|
||||
# Internal: Manifest
|
||||
|
||||
## Purpose
|
||||
Define durable session state (`manifest.json`) and invocation state (`runs/{run_id}/manifest.json`) contracts.
|
||||
|
||||
## Session Manifest
|
||||
Path:
|
||||
- `{workspace.root}/work/{campaign}/{session_id}/manifest.json`
|
||||
|
||||
Primary model (`manifest.Manifest`):
|
||||
- identity (`session_id`, `campaign`, `run_id`)
|
||||
- local path metadata (`local_workdir`, `local_spool_dir`)
|
||||
- remote identity metadata (`s3_bucket`, `s3_session_prefix`, `s3_run_prefix`)
|
||||
- `inputs` records
|
||||
- durable `artifacts` records
|
||||
- per-stage `stages` map
|
||||
|
||||
Stage status enum:
|
||||
- `pending`
|
||||
- `running`
|
||||
- `succeeded`
|
||||
- `failed`
|
||||
- `skipped`
|
||||
- `stale`
|
||||
- `interrupted`
|
||||
|
||||
## Run Manifest
|
||||
Path:
|
||||
- `{workspace.root}/work/{campaign}/{session_id}/runs/{run_id}/manifest.json`
|
||||
|
||||
Run model (`manifest.RunManifest`):
|
||||
- invocation identity and `force` flag
|
||||
- requested stages
|
||||
- per-stage action (`run` or `skip`)
|
||||
- per-stage status
|
||||
- overall run status (`running`, `succeeded`, `failed`)
|
||||
|
||||
## Persistence Semantics
|
||||
`manifest.LocalStore`:
|
||||
- validates loaded documents;
|
||||
- normalizes missing maps/stage records;
|
||||
- writes atomically via temp file + rename;
|
||||
- updates `updated_at` on save.
|
||||
|
||||
## Execution Semantics
|
||||
Runner updates both manifests per stage transition:
|
||||
- mark running
|
||||
- mark succeeded/failed/skipped
|
||||
- persist logs/generated config refs and metadata
|
||||
|
||||
Session manifest is the authoritative stage-progress ledger across invocations.
|
||||
Run manifest is invocation-scoped audit state.
|
||||
|
||||
## Invariants
|
||||
- stage resume/skip decisions are session-manifest driven.
|
||||
- force reruns stale downstream succeeded stages.
|
||||
- run manifest does not replace session manifest as progress authority.
|
||||
38
docs/internal/stage-analyze.md
Normal file
38
docs/internal/stage-analyze.md
Normal file
@@ -0,0 +1,38 @@
|
||||
# Stage: analyze
|
||||
|
||||
## Purpose
|
||||
Execute selected configured Scriptorium artifacts in dependency order and materialize outputs.
|
||||
|
||||
## Inputs
|
||||
- configured artifacts from `pipeline.scriptorium.artifacts`
|
||||
- optional selected artifact filter (`--artifacts`)
|
||||
- built-in/configured/previous-session source references in artifact inputs
|
||||
|
||||
Supported source families:
|
||||
- built-ins: `narratio.transcript.*`, `narratio.bounds.session`
|
||||
- configured artifacts: `narratio.artifact.<key>`
|
||||
- previous-session cache: `narratio.previous_session.artifact.<key>`
|
||||
|
||||
## Outputs
|
||||
- one materialized output per executed configured artifact (`output_path`)
|
||||
- stage metadata describing selected/generated/reused artifacts
|
||||
|
||||
## Key Behavior
|
||||
- skips with metadata when Scriptorium config is missing or no executable artifacts remain.
|
||||
- builds runtime artifact catalog (built-ins + configured artifacts).
|
||||
- marks non-executable configured artifacts as reusable when output files already exist.
|
||||
- validates selected artifact dependency order (cycle-safe topo ordering).
|
||||
- resolves required/optional inputs per artifact source definition.
|
||||
- resolves previous-session sources from local `previous/` cache only.
|
||||
- runs optional render-debug, then artifact execution.
|
||||
- validates non-empty output files and materializes canonical outputs.
|
||||
|
||||
## Failure Semantics
|
||||
- required missing configured/previous-session inputs fail.
|
||||
- missing required previous-session source includes prepare rerun guidance.
|
||||
- dependency cycles or unavailable required dependencies fail.
|
||||
- adapter validation failures fail stage.
|
||||
|
||||
## Invariants
|
||||
- `analyze` performs no remote storage calls for previous-session source resolution.
|
||||
- output provenance and metadata are deterministic per execution.
|
||||
25
docs/internal/stage-merge.md
Normal file
25
docs/internal/stage-merge.md
Normal file
@@ -0,0 +1,25 @@
|
||||
# Stage: merge
|
||||
|
||||
## Purpose
|
||||
Normalize raw transcript inputs and merge into base transcript via Seriatim.
|
||||
|
||||
## Inputs
|
||||
- `transcripts/raw/*.json`
|
||||
- `inputs/speakers.yml`
|
||||
- `inputs/autocorrect.yml`
|
||||
|
||||
## Outputs
|
||||
- `transcripts/base.json`
|
||||
- optional `artifacts/seriatim.report.json`
|
||||
|
||||
## Key Behavior
|
||||
- discovers and validates raw transcript inputs.
|
||||
- normalizes each raw transcript (`seriatim.Normalize`) into run-local scratch output.
|
||||
- merges normalized inputs (`seriatim.Run`) into base transcript.
|
||||
- validates merged transcript and optional report JSON.
|
||||
- materializes canonical outputs and records stage logs/generated configs.
|
||||
|
||||
## Invariants
|
||||
- merge always consumes normalized forms of raw inputs.
|
||||
- base transcript must validate before stage success.
|
||||
- report output is config-gated.
|
||||
22
docs/internal/stage-normalize.md
Normal file
22
docs/internal/stage-normalize.md
Normal file
@@ -0,0 +1,22 @@
|
||||
# Stage: normalize
|
||||
|
||||
## Purpose
|
||||
Normalize polished transcript into final transcript using Seriatim.
|
||||
|
||||
## Inputs
|
||||
- `transcripts/polished.json`
|
||||
|
||||
## Outputs
|
||||
- `transcripts/final.json` (or configured normalize output path)
|
||||
- optional `artifacts/seriatim.normalize.report.json`
|
||||
|
||||
## Key Behavior
|
||||
- resolves polished transcript from manifest outputs/canonical fallback.
|
||||
- applies `pipeline.normalize` config or default normalize config.
|
||||
- runs Seriatim normalize with configured timeout/binary.
|
||||
- validates normalized transcript and optional report.
|
||||
- materializes canonical outputs and records logs/generated configs.
|
||||
|
||||
## Invariants
|
||||
- final transcript must validate as processed transcript JSON (`segments` array).
|
||||
- normalize defaults are applied when `pipeline.normalize` is unset.
|
||||
23
docs/internal/stage-polish.md
Normal file
23
docs/internal/stage-polish.md
Normal file
@@ -0,0 +1,23 @@
|
||||
# Stage: polish
|
||||
|
||||
## Purpose
|
||||
Run Audita polishing on base transcript and produce polished transcript.
|
||||
|
||||
## Inputs
|
||||
- `transcripts/base.json`
|
||||
- `inputs/glossary.yml`
|
||||
|
||||
## Outputs
|
||||
- `transcripts/polished.json`
|
||||
- optional `artifacts/audita.report.json`
|
||||
|
||||
## Key Behavior
|
||||
- resolves base transcript from merge outputs/canonical fallback.
|
||||
- invokes Audita with configured model/module/runtime options.
|
||||
- validates processed transcript structure (`segments` array required).
|
||||
- validates optional report JSON.
|
||||
- materializes canonical outputs; records logs/generated config and adapter metadata.
|
||||
|
||||
## Invariants
|
||||
- polished transcript schema validation is mandatory.
|
||||
- report output is config-gated.
|
||||
42
docs/internal/stage-prepare.md
Normal file
42
docs/internal/stage-prepare.md
Normal file
@@ -0,0 +1,42 @@
|
||||
# Stage: prepare
|
||||
|
||||
## Purpose
|
||||
Materialize canonical current-session inputs before processing stages.
|
||||
|
||||
## Inputs
|
||||
- resolved `campaign.yml`, `session.yml`, and pipeline config
|
||||
- stable input files (`speakers`, `autocorrect`, `glossary`)
|
||||
- audio source:
|
||||
- local `audio_dir`/`audio_files`, or
|
||||
- S3 `audio_s3.prefix`
|
||||
- enabled configured artifact input requirements for previous-session sources
|
||||
|
||||
## Outputs
|
||||
- `inputs/campaign.yml`
|
||||
- `inputs/session.yml`
|
||||
- `inputs/pipeline.resolved.yml`
|
||||
- `inputs/speakers.yml`
|
||||
- `inputs/autocorrect.yml`
|
||||
- `inputs/glossary.yml`
|
||||
- `audio/*.flac`
|
||||
- optional `previous/manifest.json`
|
||||
- optional `previous/artifacts/**`
|
||||
- deterministic `manifest.inputs` entries (checksums + provenance)
|
||||
|
||||
## Key Behavior
|
||||
- validates required config/store state.
|
||||
- enforces local audio vs S3 audio mutual exclusivity.
|
||||
- materializes S3 audio through spool/cache-aware logic.
|
||||
- scans enabled configured artifact inputs for `narratio.previous_session.artifact.*` requirements.
|
||||
- when previous requirements exist:
|
||||
- clears managed `previous/` state;
|
||||
- builds previous-cache remote plan;
|
||||
- downloads previous manifest/artifacts;
|
||||
- records previous inputs in `manifest.inputs`.
|
||||
|
||||
Required previous-session inputs fail when unavailable; optional missing inputs are skipped.
|
||||
|
||||
## Invariants
|
||||
- only `prepare` hydrates canonical `previous/` cache state.
|
||||
- managed previous artifacts are stored under `previous/artifacts/**` without duplicate `artifacts/artifacts/` nesting.
|
||||
- `manifest.inputs` ordering is deterministic (`kind`, `path`).
|
||||
44
docs/internal/stage-publish.md
Normal file
44
docs/internal/stage-publish.md
Normal file
@@ -0,0 +1,44 @@
|
||||
# Stage: publish
|
||||
|
||||
## Purpose
|
||||
Upload run/session outputs to object storage and atomically advance remote current state.
|
||||
|
||||
## Inputs
|
||||
- successful prerequisite stages: `prepare`, `transcribe`, `merge`, `polish`, `normalize`, `trim`, `analyze`
|
||||
- run root `runs/{run_id}/**`
|
||||
- publish output rules (`pipeline.publish.outputs`)
|
||||
- effective publish locks (static + remote merged lock set)
|
||||
- local `previous/**` files when present
|
||||
|
||||
## Outputs
|
||||
- uploaded run files under remote `runs/{run_id}/...` (excluding `audio/**`)
|
||||
- uploaded selected publish outputs under session prefix
|
||||
- uploaded `previous/**` files under session prefix when present
|
||||
- uploaded `current/manifest.json`
|
||||
- uploaded `current/run_id.txt` written last
|
||||
|
||||
## Key Behavior
|
||||
- stage can self-skip when publish disabled or run upload disabled.
|
||||
- validates prerequisite stage success and object-store availability.
|
||||
- collects deterministic run file list plus run `manifest.json`.
|
||||
- resolves publish output sources through runtime artifact catalog and manifest-aware resolution.
|
||||
- selected artifact filter applies to configured artifact sources only.
|
||||
- locked outputs are skipped intentionally (including required ones).
|
||||
- optional missing outputs are skipped; required missing unlocked outputs fail.
|
||||
- writes remote current manifest before current run pointer.
|
||||
|
||||
## Metadata Signals
|
||||
Includes counts/lists for:
|
||||
- run uploads
|
||||
- published output uploads
|
||||
- previous uploads
|
||||
- skipped optional outputs
|
||||
- skipped unselected outputs
|
||||
- locked outputs
|
||||
- current-state key paths
|
||||
- `current_pointer_written`
|
||||
|
||||
## Invariants
|
||||
- `current/run_id.txt` is the remote commit marker and is written last.
|
||||
- run upload excludes `audio/**`.
|
||||
- publish locks are not overridden by `--force`.
|
||||
22
docs/internal/stage-transcribe.md
Normal file
22
docs/internal/stage-transcribe.md
Normal file
@@ -0,0 +1,22 @@
|
||||
# Stage: transcribe
|
||||
|
||||
## Purpose
|
||||
Generate raw per-speaker transcripts from prepared audio using WhisperX.
|
||||
|
||||
## Inputs
|
||||
- `audio/*.flac` from `prepare`
|
||||
|
||||
## Outputs
|
||||
- `transcripts/raw/<speaker>.json`
|
||||
|
||||
## Key Behavior
|
||||
- discovers prepared audio from manifest inputs or canonical audio directory.
|
||||
- derives speaker ID from `.flac` basename.
|
||||
- runs WhisperX with configured concurrency/retry settings.
|
||||
- validates each output as JSON.
|
||||
- writes run-local outputs then materializes canonical transcript outputs.
|
||||
|
||||
## Invariants
|
||||
- speaker basenames must be unique.
|
||||
- output path returned by adapter must match requested output path.
|
||||
- each successful output is validated before stage success.
|
||||
28
docs/internal/stage-trim.md
Normal file
28
docs/internal/stage-trim.md
Normal file
@@ -0,0 +1,28 @@
|
||||
# Stage: trim
|
||||
|
||||
## Purpose
|
||||
Produce a final-trimmed transcript; optionally generate bounds-driven trim.
|
||||
|
||||
## Inputs
|
||||
- `transcripts/final.json`
|
||||
|
||||
## Outputs
|
||||
- `transcripts/final.trimmed.json` (or configured trim output path)
|
||||
- when trim enabled: `artifacts/session_bounds.json`
|
||||
|
||||
## Key Behavior
|
||||
When `trim.enabled=false`:
|
||||
- copies normalized transcript to trimmed output.
|
||||
|
||||
When `trim.enabled=true`:
|
||||
- runs Scriptorium bounds artifact generation;
|
||||
- optionally runs render-debug output generation;
|
||||
- validates bounds payload against transcript;
|
||||
- derives keep selector;
|
||||
- either copies unchanged transcript or runs Seriatim trim;
|
||||
- validates trimmed transcript and materializes bounds output.
|
||||
|
||||
## Invariants
|
||||
- normalized transcript is required input.
|
||||
- bounds output exists only in enabled trim path.
|
||||
- render-debug output is diagnostic and not a declared stage output.
|
||||
37
docs/internal/storage.md
Normal file
37
docs/internal/storage.md
Normal file
@@ -0,0 +1,37 @@
|
||||
# Internal: Storage
|
||||
|
||||
## Purpose
|
||||
Document remote object-store contracts and S3 implementation behavior.
|
||||
|
||||
## Primary Contract
|
||||
`storage.ObjectStore` interface:
|
||||
- `List(ctx, prefix)`
|
||||
- `Download(ctx, key, localPath)`
|
||||
- `Upload(ctx, localPath, key, opts)`
|
||||
- `Exists(ctx, key)`
|
||||
|
||||
Key invariant:
|
||||
- callers pass full bucket-relative keys;
|
||||
- storage implementations do not infer campaign/session/run prefixes.
|
||||
|
||||
## Configuration
|
||||
`NewObjectStoreFromConfig` currently supports S3-backed stores from `pipeline.storage.*` config.
|
||||
|
||||
S3 constructor behavior:
|
||||
- requires configured bucket;
|
||||
- uses region/endpoint/path-style options when set;
|
||||
- resolves credentials from configured env var names (with defaults).
|
||||
|
||||
## S3 Backend Behavior
|
||||
- normalizes object keys.
|
||||
- `List` paginates and returns normalized `ObjectInfo`.
|
||||
- `Download` writes local files with parent directory creation.
|
||||
- `Upload` streams local file and returns remote metadata.
|
||||
- `Exists` maps not-found responses to `false`.
|
||||
|
||||
## Legacy Compatibility Interface
|
||||
`storage.Backend` (with `ArchiveRequest`) remains as compatibility surface with `NoopBackend`; it is not used by current stage execution.
|
||||
|
||||
## Invariants
|
||||
- storage layer is stateless regarding manifest/stage progression.
|
||||
- publish ordering semantics are owned by stage/app code, not storage adapters.
|
||||
57
docs/internal/workspace.md
Normal file
57
docs/internal/workspace.md
Normal file
@@ -0,0 +1,57 @@
|
||||
# Internal: Workspace
|
||||
|
||||
## Purpose
|
||||
Define local session layout, run-local stage layout, and cleanup guardrails.
|
||||
|
||||
## Canonical Session Layout
|
||||
Session root:
|
||||
- `{workspace.root}/work/{campaign}/{session_id}`
|
||||
|
||||
Core directories/files:
|
||||
- `inputs/`
|
||||
- `audio/`
|
||||
- `transcripts/`
|
||||
- `artifacts/`
|
||||
- `reports/`
|
||||
- `logs/`
|
||||
- `config/`
|
||||
- `current/`
|
||||
- `runs/`
|
||||
- `previous/`
|
||||
- `manifest.json`
|
||||
- `.lock`
|
||||
|
||||
`previous/` reserved files:
|
||||
- `previous/manifest.json`
|
||||
- `previous/artifacts/**`
|
||||
|
||||
## Run-Local Stage Layout
|
||||
When run context is available, stages use:
|
||||
- `runs/{run_id}/{stage}/outputs/`
|
||||
- `runs/{run_id}/{stage}/logs/`
|
||||
- `runs/{run_id}/{stage}/reports/`
|
||||
- `runs/{run_id}/{stage}/config/`
|
||||
- `runs/{run_id}/{stage}/scratch/`
|
||||
|
||||
Run-local outputs are materialized back into canonical session paths before stage success.
|
||||
`previous/**` writes are never redirected to run-local output paths.
|
||||
|
||||
## Locking
|
||||
`artifacts.LocalStore` enforces single-writer session lock via `.lock` file (`ErrLockConflict` on contention).
|
||||
|
||||
## Cleanup Semantics
|
||||
Automatic post-publish cleanup (`runPostArchiveCleanup`):
|
||||
- only runs when publish actually executed and succeeded;
|
||||
- requires `uploaded=true` and `current_pointer_written=true` metadata;
|
||||
- respects `pipeline.spool.delete_audio_after_publish` and `pipeline.workspace.cleanup_after_publish`;
|
||||
- refuses unsafe deletes (root delete, out-of-root delete, symlink paths).
|
||||
|
||||
Manual clean command:
|
||||
- `clean <session_id>` removes session work and spool subtree.
|
||||
- `clean --all` removes all workspace work and spool children.
|
||||
- durable cache is preserved unless `--clear-cache` is requested.
|
||||
|
||||
## Invariants
|
||||
- campaign-aware session root is mandatory.
|
||||
- manifest-driven stage state is durable across runs.
|
||||
- cleanup guardrails prevent destructive root/out-of-scope deletion.
|
||||
249
docs/operations.md
Normal file
249
docs/operations.md
Normal file
@@ -0,0 +1,249 @@
|
||||
# Operations Guide
|
||||
|
||||
Operator workflow for running, recovering, and publishing Narratio sessions.
|
||||
|
||||
For command syntax, see [docs/cli.md](./cli.md). For field-level config, see [docs/config.md](./config.md).
|
||||
|
||||
## Standard Session Workflow
|
||||
|
||||
1. Select pipeline/campaign/session config.
|
||||
2. Validate session readiness:
|
||||
|
||||
```bash
|
||||
narratio session validate 2026-04-04
|
||||
```
|
||||
|
||||
3. (Optional) inspect stage decisions:
|
||||
|
||||
```bash
|
||||
narratio session plan 2026-04-04
|
||||
```
|
||||
|
||||
4. Run the pipeline:
|
||||
|
||||
```bash
|
||||
narratio run 2026-04-04
|
||||
```
|
||||
|
||||
5. Check state:
|
||||
|
||||
```bash
|
||||
narratio session status 2026-04-04
|
||||
```
|
||||
|
||||
## Campaign and Session Selection
|
||||
|
||||
Campaign selection priority:
|
||||
|
||||
- `--campaign-file`
|
||||
- `--campaign`
|
||||
- `pipeline.campaigns.default_campaign_id`
|
||||
|
||||
Session source priority:
|
||||
|
||||
- `--session`
|
||||
- local default search paths
|
||||
- remote session object (S3) when local session file is not found and storage is configured
|
||||
|
||||
## Session Initialization
|
||||
|
||||
Use `session init` to generate a concrete session file for local or remote use.
|
||||
|
||||
Local file:
|
||||
|
||||
```bash
|
||||
narratio session init 2026-04-04 --output ./session.yml --date 2026-04-04 --title "Session 12"
|
||||
```
|
||||
|
||||
Remote session object:
|
||||
|
||||
```bash
|
||||
narratio session init 2026-04-04 --remote --force
|
||||
```
|
||||
|
||||
If `campaign.yml` sets `session_template_file`, `session init` renders it. Template variables must resolve to concrete values.
|
||||
|
||||
## Stage Execution and Resume Behavior
|
||||
|
||||
Canonical stage order:
|
||||
|
||||
1. `prepare`
|
||||
2. `transcribe`
|
||||
3. `merge`
|
||||
4. `polish`
|
||||
5. `normalize`
|
||||
6. `trim`
|
||||
7. `analyze`
|
||||
8. `publish`
|
||||
9. `notify`
|
||||
|
||||
Execution rules:
|
||||
|
||||
- succeeded stages are skipped unless `--force` is set;
|
||||
- `resume` starts at first non-succeeded stage;
|
||||
- force rerunning a succeeded upstream stage marks succeeded downstream stages as `stale`.
|
||||
|
||||
Single-stage execution:
|
||||
|
||||
```bash
|
||||
narratio run-stage normalize 2026-04-04 --force
|
||||
```
|
||||
|
||||
## Artifact Selection
|
||||
|
||||
`--artifacts` can be used on `run`, `resume`, `run-stage`, `analyze`, and `publish`.
|
||||
|
||||
Selection behavior:
|
||||
|
||||
- validates names against `pipeline.scriptorium.artifacts`;
|
||||
- filters analyze execution to selected configured artifacts;
|
||||
- filters publish rules for `narratio.artifact.<name>` sources only;
|
||||
- does not suppress built-in transcript or bounds publish sources.
|
||||
|
||||
## Publish Workflow
|
||||
|
||||
Run publish only:
|
||||
|
||||
```bash
|
||||
narratio publish 2026-04-04
|
||||
```
|
||||
|
||||
Equivalent:
|
||||
|
||||
```bash
|
||||
narratio run-stage publish 2026-04-04 --force
|
||||
```
|
||||
|
||||
Publish commit model:
|
||||
|
||||
- uploads run files under `{session_prefix}/runs/{run_id}/`;
|
||||
- uploads configured published outputs;
|
||||
- uploads `previous/**` cache files when present;
|
||||
- writes `current/manifest.json`;
|
||||
- writes `current/run_id.txt` last.
|
||||
|
||||
`current/run_id.txt` is the remote current-state commit marker.
|
||||
|
||||
## Publish Locks
|
||||
|
||||
Lock sources:
|
||||
|
||||
- static locks in `pipeline.publish.locks`
|
||||
- mutable remote locks in `{session_prefix}/locks.yml`
|
||||
|
||||
Effective lock rules:
|
||||
|
||||
- static and remote locks are merged;
|
||||
- static locks win on source collisions;
|
||||
- locked outputs are intentional skips;
|
||||
- lock add/remove commands mutate only remote lock state.
|
||||
|
||||
Examples:
|
||||
|
||||
```bash
|
||||
narratio session locks 2026-04-04
|
||||
narratio session locks add 2026-04-04 narratio.artifact.session_recap --reason "manual edits" --force
|
||||
narratio session locks remove 2026-04-04 narratio.artifact.session_recap
|
||||
```
|
||||
|
||||
## Restore Workflow
|
||||
|
||||
Use restore when local durable session state is missing or stale and remote committed current state is authoritative.
|
||||
|
||||
Dry run:
|
||||
|
||||
```bash
|
||||
narratio session restore 2026-04-04 --dry-run
|
||||
```
|
||||
|
||||
Apply:
|
||||
|
||||
```bash
|
||||
narratio session restore 2026-04-04
|
||||
```
|
||||
|
||||
Default restore scope:
|
||||
|
||||
- `manifest.json`
|
||||
- `transcripts/**`
|
||||
- `artifacts/**`
|
||||
- `previous/**` when needed by configured previous-session artifact inputs
|
||||
|
||||
Optional:
|
||||
|
||||
- `--include-audio` to include `audio/**`
|
||||
- `--force` to overwrite local conflicts
|
||||
|
||||
Restore writes an execution report at `reports/restore-latest.json`.
|
||||
|
||||
## Local State Layout
|
||||
|
||||
Session root:
|
||||
|
||||
- `{workspace.root}/work/{campaign}/{session_id}`
|
||||
|
||||
Durable session paths:
|
||||
|
||||
- `manifest.json`
|
||||
- `inputs/**`
|
||||
- `audio/**`
|
||||
- `transcripts/**`
|
||||
- `artifacts/**`
|
||||
- `previous/**`
|
||||
- `reports/**`
|
||||
- `logs/**`
|
||||
- `config/**`
|
||||
- `runs/**`
|
||||
|
||||
Run-local layout:
|
||||
|
||||
- `runs/{run_id}/{stage}/outputs`
|
||||
- `runs/{run_id}/{stage}/logs`
|
||||
- `runs/{run_id}/{stage}/reports`
|
||||
- `runs/{run_id}/{stage}/config`
|
||||
- `runs/{run_id}/{stage}/scratch`
|
||||
|
||||
Spool layout (runtime/transient):
|
||||
|
||||
- `{spool.root}/{campaign}/{session_id}/{run_id}/...`
|
||||
- restore audio spool under `{spool.root}/{campaign}/{session_id}/restore/audio`
|
||||
|
||||
Cache layout (durable S3 audio cache):
|
||||
|
||||
- `{cache.root}/s3/{bucket}/...`
|
||||
|
||||
## Cleanup
|
||||
|
||||
Session-scoped cleanup:
|
||||
|
||||
```bash
|
||||
narratio clean 2026-04-04
|
||||
```
|
||||
|
||||
Global cleanup:
|
||||
|
||||
```bash
|
||||
narratio clean --all
|
||||
```
|
||||
|
||||
Dry-run and cache variants:
|
||||
|
||||
```bash
|
||||
narratio clean 2026-04-04 --dry-run --clear-cache
|
||||
narratio clean --all --dry-run --clear-cache
|
||||
```
|
||||
|
||||
Rules:
|
||||
|
||||
- `clean` deletes work/spool session state;
|
||||
- cache is preserved unless `--clear-cache` is set;
|
||||
- automatic post-publish cleanup is gated by successful publish commit plus:
|
||||
- `pipeline.spool.delete_audio_after_publish=true`
|
||||
- `pipeline.workspace.cleanup_after_publish=true`
|
||||
|
||||
## Operational Caveats
|
||||
|
||||
- Local and S3 audio modes are mutually exclusive.
|
||||
- Publish requires prerequisite stages through analyze to be succeeded.
|
||||
- Restore requires configured object storage and committed remote current state.
|
||||
- Storage-backed commands load filesystem secrets before object-store initialization.
|
||||
202
docs/policy/architecture.md
Normal file
202
docs/policy/architecture.md
Normal file
@@ -0,0 +1,202 @@
|
||||
# Narratio Architecture
|
||||
|
||||
## Purpose
|
||||
|
||||
`narratio` is a Go orchestration application for processing D&D session audio into polished transcripts and generated session artifacts.
|
||||
|
||||
This document defines the development principles for the project. It is inward-facing: its audience is developers and LLM coding agents. It should guide future changes, not serve as a complete implementation reference.
|
||||
|
||||
Implemented component details belong under `docs/internal/`.
|
||||
|
||||
## Project Shape
|
||||
|
||||
Narratio is a modular, stage-driven orchestrator.
|
||||
|
||||
It coordinates specialized downstream systems rather than reimplementing their domains:
|
||||
|
||||
- WhisperX handles transcription.
|
||||
- Seriatim handles deterministic transcript merge/normalization/trim behavior.
|
||||
- Audita handles transcript correction and polishing.
|
||||
- Scriptorium handles prompt execution and generated artifacts.
|
||||
|
||||
Narratio owns orchestration, configuration loading, session/run state, local and remote path modeling, manifest persistence, stage sequencing, resume behavior, and publish semantics.
|
||||
|
||||
Narratio should remain explicit and comprehensible. It is not intended to become a generic workflow engine.
|
||||
|
||||
## Core Principles
|
||||
|
||||
### Modular and composable
|
||||
|
||||
Code should be organized around clear responsibilities. Stages, adapters, config loading, manifest persistence, path construction, and storage behavior should remain separable and independently testable.
|
||||
|
||||
### Hexagonal boundaries
|
||||
|
||||
External systems should be isolated behind narrow adapters. Stage logic should depend on Narratio-level interfaces and data structures, not on external SDK types, subprocess argument construction, or transport-specific details.
|
||||
|
||||
### Standard library preference
|
||||
|
||||
Prefer the Go standard library. Add dependencies only when they provide substantial value, are necessary for an external integration, or are a widely used de facto standard.
|
||||
|
||||
Accepted examples include a YAML library for configuration and the AWS SDK for S3-compatible storage.
|
||||
|
||||
### Explicit orchestration
|
||||
|
||||
The pipeline should remain stage-driven and explicit. New behavior should be added through clear stage, adapter, config, or manifest contracts rather than implicit side effects or generic workflow abstraction.
|
||||
|
||||
## Stage Design
|
||||
|
||||
Each stage should have a clear scope of responsibility.
|
||||
|
||||
A stage should define:
|
||||
|
||||
- its purpose;
|
||||
- required input state;
|
||||
- produced output state;
|
||||
- config fields it consumes;
|
||||
- external adapters it uses;
|
||||
- manifest refs it reads or writes;
|
||||
- skip, force, and resume behavior;
|
||||
- failure behavior;
|
||||
- tests that protect its contract.
|
||||
|
||||
Stages should avoid reaching across boundaries. If shared behavior is needed, prefer a helper or service with a narrow interface over duplicating ad hoc logic between stages.
|
||||
|
||||
## Transactionality and Resume
|
||||
|
||||
A stage should behave transactionally.
|
||||
|
||||
A stage is complete only when its outputs have been written, validated, and recorded in the manifest. If a stage fails, Narratio should preserve enough local state for inspection, recovery, and resume.
|
||||
|
||||
A failed or incomplete run must not be treated as successful. Later stages should depend on manifest-recorded success, not merely on incidental files existing on disk.
|
||||
|
||||
## Manifest Model
|
||||
|
||||
The manifest is the durable local ledger for a run.
|
||||
|
||||
It should record:
|
||||
|
||||
- run identity;
|
||||
- stage status;
|
||||
- input and output refs;
|
||||
- logs and generated config refs;
|
||||
- checksums or provenance where useful;
|
||||
- non-secret adapter and publish metadata.
|
||||
|
||||
Resume behavior should be manifest-driven. Filesystem state may be inspected and validated, but it should not replace manifest stage state as the source of run progress.
|
||||
|
||||
## Adapter Boundaries
|
||||
|
||||
Adapters own external integration details.
|
||||
|
||||
Expected boundaries:
|
||||
|
||||
- WhisperX HTTP details stay in the WhisperX adapter.
|
||||
- Seriatim CLI construction stays in the Seriatim adapter.
|
||||
- Audita CLI construction stays in the Audita adapter.
|
||||
- Scriptorium CLI construction stays in the Scriptorium adapter.
|
||||
- Object-storage details stay behind the storage adapter interface.
|
||||
- AWS SDK types stay inside the S3 storage implementation.
|
||||
|
||||
Stage code should express intent in Narratio terms and call adapters through narrow contracts.
|
||||
|
||||
## Configuration Philosophy
|
||||
|
||||
Configuration should be strict, explicit, and operator-friendly.
|
||||
|
||||
Principles:
|
||||
|
||||
- YAML decoding should reject unknown fields.
|
||||
- Defaults should be centralized and testable.
|
||||
- Empty configured values should not silently override meaningful defaults.
|
||||
- Session templating should remain narrow and deterministic.
|
||||
- Template support should serve operator convenience, not become a general configuration language.
|
||||
|
||||
Narratio should not become a secondary configuration system for downstream tools. Seriatim, Audita, and Scriptorium should own their runtime defaults wherever practical. Narratio should pass required stage-contract paths and explicit operator overrides.
|
||||
|
||||
## Path and Storage Discipline
|
||||
|
||||
Local and remote paths are part of Narratio’s application contract.
|
||||
|
||||
Code should use centralized path helpers for workspace, spool, session, run, artifact, log, config, and publish/current paths. Stages should avoid reconstructing canonical paths through scattered string concatenation.
|
||||
|
||||
Storage backends should receive explicit bucket-relative keys. Storage implementations should not infer campaign, session, run, or root-prefix semantics.
|
||||
|
||||
## Publish Invariants
|
||||
|
||||
Publish behavior must preserve a clear commit boundary.
|
||||
|
||||
A remote run is current only after the publish stage has successfully uploaded the run record, required published outputs, `current/manifest.json`, and finally `current/run_id.txt`.
|
||||
|
||||
`current/run_id.txt` is the final remote commit marker and must be written last.
|
||||
|
||||
Failed, incomplete, skipped, or uncommitted publish attempts must not be presented as current remote state. Local cleanup is permitted only after successful publish commit and only when explicitly configured.
|
||||
|
||||
## Security and Privacy
|
||||
|
||||
Narratio handles private campaign material.
|
||||
|
||||
Rules:
|
||||
|
||||
- Do not store raw secrets in pipeline or session YAML.
|
||||
- Use environment variable names or secret-file references for secret handling.
|
||||
- Do not write raw secret values to manifests, logs, generated configs, or publish metadata.
|
||||
- Treat transcripts, generated artifacts, prompts, reports, and logs as potentially sensitive.
|
||||
- Avoid logging transcript or prompt content unless there is a deliberate diagnostic reason.
|
||||
|
||||
## Diagnostics
|
||||
|
||||
Diagnostics should be durable and discoverable, but distinct from canonical outputs.
|
||||
|
||||
Logs, reports, generated invocation/config files, and render-debug files support debugging. Transcript tiers and configured artifacts are pipeline products.
|
||||
|
||||
Manifest refs should preserve that distinction.
|
||||
|
||||
## Determinism
|
||||
|
||||
Where practical, Narratio should prefer deterministic behavior:
|
||||
|
||||
- stable local path layout;
|
||||
- stable remote key layout;
|
||||
- sorted upload order;
|
||||
- predictable generated config files;
|
||||
- repeatable command construction;
|
||||
- tests that do not depend on live external services.
|
||||
|
||||
Run IDs and timestamps may be intentionally variable, but surrounding behavior should remain testable.
|
||||
|
||||
## Testing Expectations
|
||||
|
||||
Core behavior should be testable without live external services.
|
||||
|
||||
Tests should cover:
|
||||
|
||||
- config loading, defaults, and validation;
|
||||
- CLI parsing and command construction;
|
||||
- path helpers;
|
||||
- manifest transitions;
|
||||
- stage success, failure, skip, and resume behavior;
|
||||
- adapter command construction;
|
||||
- fake storage behavior;
|
||||
- publish commit ordering;
|
||||
- example config validity where practical.
|
||||
|
||||
Live S3, WhisperX, LLM, or subprocess integration tests should be explicit integration tests, not required for ordinary unit test runs.
|
||||
|
||||
## Documentation Expectations
|
||||
|
||||
Documentation must follow `docs/documentation/policy.md`.
|
||||
|
||||
Current behavior belongs in user-facing docs and `docs/internal/`. Future, planned, aspirational, experimental, or unimplemented work belongs only under `docs/roadmap/`.
|
||||
|
||||
`docs/architecture.md` should remain concise and principle-focused. It should not duplicate the full config reference, CLI reference, operations guide, or internal stage documentation.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
Narratio is not:
|
||||
|
||||
- a generic DAG or workflow engine;
|
||||
- a replacement configuration layer for Seriatim, Audita, or Scriptorium;
|
||||
- a storage backend abstraction beyond the needs of this pipeline;
|
||||
- a place to embed raw secrets;
|
||||
- a place for stage logic to depend directly on AWS SDK types or downstream tool internals;
|
||||
- a prompt-authoring system.
|
||||
94
docs/policy/development.md
Normal file
94
docs/policy/development.md
Normal file
@@ -0,0 +1,94 @@
|
||||
# Development Guide
|
||||
|
||||
## Purpose
|
||||
Canonical contributor workflow and engineering conventions for implemented Narratio behavior.
|
||||
|
||||
## Repository layout
|
||||
|
||||
- `cmd/narratio/`: CLI entrypoint.
|
||||
- `internal/app/`: command handlers, plan/run/resume orchestration, cleanup gates, secrets loading.
|
||||
- `internal/config/`: strict YAML loading, defaults, and validation.
|
||||
- `internal/stage/`: stage implementations and stage registry/order.
|
||||
- `internal/adapters/`: external boundary adapters (WhisperX, Seriatim, Audita, Scriptorium, storage, notify).
|
||||
- `internal/manifest/`: session/run manifest types and persistence.
|
||||
- `internal/artifacts/`: canonical local/remote path helpers and local artifact store.
|
||||
- `docs/`: canonical documentation set.
|
||||
- `examples/`: maintained config examples used by tests.
|
||||
|
||||
## Build and test commands
|
||||
|
||||
- Run focused CLI behavior checks:
|
||||
|
||||
```bash
|
||||
go test ./internal/app -run TestExecute -v
|
||||
```
|
||||
|
||||
- Run config example load/validate checks:
|
||||
|
||||
```bash
|
||||
go test ./internal/config -run TestExamplesLoadAndValidate -v
|
||||
```
|
||||
|
||||
- Run full test suite:
|
||||
|
||||
```bash
|
||||
go test ./...
|
||||
```
|
||||
|
||||
## Coding conventions
|
||||
|
||||
- Keep orchestration explicit and stage-driven; do not introduce generic workflow/DAG abstractions.
|
||||
- Keep external-system details inside adapter packages; stages should consume Narratio-level contracts only.
|
||||
- Use centralized path helpers from `internal/artifacts` rather than ad hoc path concatenation.
|
||||
- Preserve manifest-driven state transitions (`running`, `succeeded`, `failed`, `skipped`, `stale`) as the source of run progress.
|
||||
- Keep user/operator docs implementation-accurate; planned work belongs only under `docs/roadmap/`.
|
||||
|
||||
For design principles and invariants, see [docs/architecture.md](./architecture.md). For stage/adapter contracts, see [docs/internal/README.md](./internal/README.md).
|
||||
|
||||
## Dependency policy
|
||||
|
||||
- Prefer Go standard library where practical.
|
||||
- Add third-party dependencies only when they provide clear value for required behavior.
|
||||
- Keep dependency additions narrow to the boundary package that needs them.
|
||||
|
||||
## Change playbooks
|
||||
|
||||
### Add config fields
|
||||
|
||||
1. Add fields to config structs in `internal/config`.
|
||||
2. Set defaults in `internal/config/defaults.go` when appropriate.
|
||||
3. Add validation rules in `internal/config/validate.go`.
|
||||
4. Add or update load/validate tests in `internal/config/*_test.go`.
|
||||
5. Update canonical config docs and examples:
|
||||
- [docs/config.md](./config.md)
|
||||
- relevant files under `examples/`
|
||||
|
||||
### Add CLI flags or commands
|
||||
|
||||
1. Update command parsing and behavior in `internal/app`.
|
||||
2. Add or update command tests (`TestExecute` and command-specific tests).
|
||||
3. Update [docs/cli.md](./cli.md) and, if operator workflow changes, [docs/operations.md](./operations.md).
|
||||
|
||||
Remote-storage commands must obtain object storage through the app-level command object-store helper. Do not call `storage.NewObjectStoreFromConfig` directly from command handlers; the helper loads configured filesystem secrets before constructing the storage adapter.
|
||||
|
||||
### Add or modify stages/adapters
|
||||
|
||||
1. Implement stage behavior in `internal/stage` with clear input/output boundaries.
|
||||
2. Keep external transport/subprocess details in `internal/adapters`.
|
||||
3. Preserve manifest and publish-output semantics expected by runner and publish logic.
|
||||
4. Add/update stage and adapter tests.
|
||||
5. Update internal component contracts in `docs/internal/`.
|
||||
|
||||
### Update examples
|
||||
|
||||
1. Keep canonical examples only in `examples/`.
|
||||
2. Ensure examples load and validate through runtime config paths.
|
||||
3. Update `internal/config/load_validate_test.go` as needed.
|
||||
4. Update links in `docs/config.md` if example filenames change.
|
||||
|
||||
### Update docs and roadmap
|
||||
|
||||
1. Keep implemented behavior in canonical docs (`README`, `docs/*.md`, `docs/internal/`).
|
||||
2. Keep planned/unimplemented behavior only in `docs/roadmap/`.
|
||||
3. After completing roadmap items, remove or mark them complete in `docs/roadmap/documentation.md`.
|
||||
4. Run a link/path sweep before finalizing changes.
|
||||
356
docs/policy/documentation.md
Normal file
356
docs/policy/documentation.md
Normal file
@@ -0,0 +1,356 @@
|
||||
# Go Project Documentation Policy
|
||||
|
||||
## Purpose
|
||||
|
||||
Project documentation must help four audiences:
|
||||
|
||||
1. users who need to run the application;
|
||||
2. administrators/operators who need to configure and operate it;
|
||||
3. developers who need to understand and change it safely;
|
||||
4. LLM coding agents that need clear scope, boundaries, and invariants.
|
||||
|
||||
Docs should be accurate, concise, task-oriented, and organized by audience. Prefer links to canonical docs over repetition.
|
||||
|
||||
## Core Rules
|
||||
|
||||
### 1. Keep docs concise
|
||||
|
||||
Each document should cover a defined scope and only the essentials for that scope.
|
||||
|
||||
Avoid:
|
||||
- long background explanations;
|
||||
- repeated reference material;
|
||||
- implementation detail in user-facing docs;
|
||||
- aspirational language outside roadmap docs;
|
||||
- verbose examples where one minimal example is clearer.
|
||||
|
||||
### 2. Document only implemented behavior outside roadmap files
|
||||
|
||||
Unimplemented, planned, aspirational, experimental, or future work may be described only under:
|
||||
|
||||
- `docs/roadmap/`
|
||||
|
||||
No other documentation file, including `README.md`, should describe code, features, modules, stages, commands, config fields, or behaviors that do not currently exist.
|
||||
|
||||
If a feature is partial, non-roadmap docs may describe only the implemented portion and its current boundary.
|
||||
|
||||
### 3. Use canonical homes
|
||||
|
||||
Each type of information should have one canonical location.
|
||||
|
||||
Canonical homes:
|
||||
|
||||
- project purpose and quickstart: `README.md`
|
||||
- development principles: `docs/architecture.md`
|
||||
- configuration reference: `docs/config.md`
|
||||
- CLI reference: `docs/cli.md`
|
||||
- operations and recovery: `docs/operations.md`
|
||||
- troubleshooting: `docs/troubleshooting.md`
|
||||
- implemented internals: `docs/internal/`
|
||||
- future work: `docs/roadmap/`
|
||||
- contributor workflow: `docs/development.md`
|
||||
- copyable examples: `examples/`
|
||||
|
||||
Other files should summarize briefly and link to the canonical source.
|
||||
|
||||
### 4. Keep examples real
|
||||
|
||||
Examples should be valid, maintained, and free of secrets.
|
||||
|
||||
Where practical:
|
||||
- example configs should load successfully;
|
||||
- example commands should match real CLI syntax;
|
||||
- important examples should be covered by tests.
|
||||
|
||||
## Documentation Profiles
|
||||
|
||||
All projects require:
|
||||
|
||||
- `README.md`
|
||||
- `docs/architecture.md`
|
||||
|
||||
Additional docs depend on the project.
|
||||
|
||||
### Small library
|
||||
|
||||
Recommended:
|
||||
- `docs/development.md`, if contributor conventions are non-obvious
|
||||
|
||||
### Simple CLI
|
||||
|
||||
Required:
|
||||
- `docs/cli.md`
|
||||
|
||||
Recommended:
|
||||
- `docs/development.md`
|
||||
|
||||
### Config-driven CLI
|
||||
|
||||
Required:
|
||||
- `docs/cli.md`
|
||||
- `docs/config.md`
|
||||
|
||||
Recommended:
|
||||
- `examples/`
|
||||
- `docs/development.md`
|
||||
|
||||
### Stateful or operator-facing application
|
||||
|
||||
Required:
|
||||
- `docs/cli.md`, if CLI-based
|
||||
- `docs/config.md`, if config-driven
|
||||
- `docs/operations.md`
|
||||
|
||||
Recommended:
|
||||
- `docs/troubleshooting.md`
|
||||
- `examples/`
|
||||
- `docs/development.md`
|
||||
|
||||
### Modular, staged, service-oriented, or orchestration application
|
||||
|
||||
Required:
|
||||
- `docs/cli.md`, if CLI-based
|
||||
- `docs/config.md`, if config-driven
|
||||
- `docs/operations.md`
|
||||
- `docs/internal/`
|
||||
- `docs/development.md`
|
||||
|
||||
Recommended:
|
||||
- `docs/troubleshooting.md`
|
||||
- validated examples under `examples/`
|
||||
|
||||
## Required Documents
|
||||
|
||||
### README.md
|
||||
|
||||
**Audience:** users, administrators, operators
|
||||
|
||||
The README is the outward-facing project orientation page.
|
||||
|
||||
It should include, in order:
|
||||
|
||||
1. concise description;
|
||||
2. elevator pitch;
|
||||
3. shortest useful command or usage example;
|
||||
4. links to targeted docs.
|
||||
|
||||
The README should be short. It is not a manual.
|
||||
|
||||
The “shortest useful command” means the simplest command that performs the project’s core use case. (It does not mean `app --help`.)
|
||||
|
||||
### docs/architecture.md
|
||||
|
||||
**Audience:** developers, LLM coding agents
|
||||
|
||||
`docs/architecture.md` is required for every project.
|
||||
|
||||
It is an inward-facing development policy document. It should describe how the project is intended to be built and changed.
|
||||
|
||||
It should include:
|
||||
|
||||
- project shape;
|
||||
- core design principles;
|
||||
- package and boundary philosophy;
|
||||
- state/persistence philosophy, if applicable;
|
||||
- external integration philosophy, if applicable;
|
||||
- error-handling and logging principles;
|
||||
- testing expectations;
|
||||
- documentation expectations;
|
||||
- architectural invariants;
|
||||
- explicit non-goals, if useful.
|
||||
|
||||
For small projects, this file may be brief. It may simply state that the project is intentionally narrow, monolithic, and dependency-light.
|
||||
|
||||
### docs/config.md
|
||||
|
||||
**Audience:** administrators, operators, advanced users
|
||||
|
||||
Required for applications with configuration files.
|
||||
|
||||
It should include, in order:
|
||||
|
||||
1. config file locations and discovery precedence;
|
||||
2. minimal working config;
|
||||
3. production-oriented config;
|
||||
4. full configuration reference;
|
||||
5. secrets handling, if applicable;
|
||||
6. links to maintained examples.
|
||||
|
||||
The full configuration reference should be canonical.
|
||||
|
||||
### docs/cli.md
|
||||
|
||||
**Audience:** users, administrators, operators
|
||||
|
||||
Required for CLI applications.
|
||||
|
||||
It should include, in order:
|
||||
|
||||
1. shortest useful command;
|
||||
2. command overview;
|
||||
3. complete flag reference;
|
||||
4. common workflows;
|
||||
5. diagnostic or recovery commands, if applicable.
|
||||
|
||||
Explain when commands are useful, not just their syntax.
|
||||
|
||||
### docs/operations.md
|
||||
|
||||
**Audience:** administrators, operators
|
||||
|
||||
Required for applications that maintain state, support resume behavior, run multiple stages, write durable artifacts, use remote storage, or require recovery procedures.
|
||||
|
||||
It should cover:
|
||||
|
||||
- normal workflow;
|
||||
- filesystem layout;
|
||||
- remote storage layout, if applicable;
|
||||
- logs and manifests;
|
||||
- resume/retry behavior;
|
||||
- cleanup behavior;
|
||||
- archive/backup behavior;
|
||||
- safe recovery procedures;
|
||||
- operational caveats.
|
||||
|
||||
### docs/troubleshooting.md
|
||||
|
||||
**Audience:** administrators, operators
|
||||
|
||||
Recommended once recurring failure modes exist.
|
||||
|
||||
Each entry should include:
|
||||
|
||||
- symptom;
|
||||
- likely cause;
|
||||
- diagnostic command or inspection step;
|
||||
- safe fix;
|
||||
- relevant links.
|
||||
|
||||
### docs/development.md
|
||||
|
||||
**Audience:** developers, LLM coding agents
|
||||
|
||||
Required for projects maintained by humans and LLM coding agents.
|
||||
|
||||
It should include:
|
||||
|
||||
- repository layout;
|
||||
- build/test commands;
|
||||
- coding conventions;
|
||||
- dependency policy;
|
||||
- how to add config fields;
|
||||
- how to add CLI flags;
|
||||
- how to add stages/modules/adapters, if applicable;
|
||||
- how to update examples;
|
||||
- documentation update expectations.
|
||||
|
||||
### docs/internal/
|
||||
|
||||
**Audience:** developers, LLM coding agents
|
||||
|
||||
Required for modular, staged, service-oriented, or orchestration projects.
|
||||
|
||||
This directory describes implemented internal components. It is not the roadmap.
|
||||
|
||||
Use one file per major component where useful.
|
||||
|
||||
Each component doc should include:
|
||||
|
||||
1. purpose;
|
||||
2. inputs and outputs;
|
||||
3. boundaries;
|
||||
4. config fields used;
|
||||
5. external adapters used;
|
||||
6. state or manifest behavior, if applicable;
|
||||
7. skip/resume behavior, if applicable;
|
||||
8. failure behavior;
|
||||
9. tests to inspect before changing;
|
||||
10. architectural invariants.
|
||||
|
||||
### docs/roadmap/
|
||||
|
||||
**Audience:** maintainers, developers, LLM coding agents
|
||||
|
||||
This is the only place for planned, future, aspirational, experimental, or unimplemented work.
|
||||
|
||||
Roadmap docs should clearly distinguish:
|
||||
|
||||
- proposed work;
|
||||
- accepted plans;
|
||||
- deferred ideas;
|
||||
- rejected ideas;
|
||||
- implementation prompts or task breakdowns, if useful.
|
||||
|
||||
Roadmap docs should not be confused with current behavior.
|
||||
|
||||
### docs/integrations/
|
||||
|
||||
**Audience:** developers, LLM coding agents
|
||||
|
||||
Required for projects that depend on external CLIs, APIs, services, protocols, or file formats where the integration contract is important to maintain.
|
||||
|
||||
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses.
|
||||
|
||||
Use one file per integration where useful.
|
||||
|
||||
## Examples Directory
|
||||
|
||||
Projects with non-trivial configuration or workflows should include `examples/`.
|
||||
|
||||
Useful examples include:
|
||||
|
||||
- minimal working config;
|
||||
- production-oriented config;
|
||||
- full annotated config;
|
||||
- local development config;
|
||||
- remote/object-storage config;
|
||||
- minimal session/input file.
|
||||
|
||||
Examples should be valid, maintained, tested when practical, and linked from relevant docs.
|
||||
|
||||
## Security and Privacy
|
||||
|
||||
Docs and examples must not include:
|
||||
|
||||
- real API keys;
|
||||
- tokens;
|
||||
- passwords;
|
||||
- private keys;
|
||||
- private environment dumps;
|
||||
- sensitive user data;
|
||||
- raw private transcripts;
|
||||
- private infrastructure details unless intentionally public.
|
||||
|
||||
Document secret-handling mechanisms, not actual secret values.
|
||||
|
||||
## Maintenance Rules
|
||||
|
||||
When docs change, verify the affected behavior.
|
||||
|
||||
Where practical:
|
||||
|
||||
- load example config files in tests;
|
||||
- test CLI examples or command parser behavior;
|
||||
- validate documented flags against real flags;
|
||||
- remove stale references;
|
||||
- update links after renames;
|
||||
- keep roadmap content out of non-roadmap docs.
|
||||
|
||||
If documentation and code disagree, fix the documentation and/or open a roadmap item; do not leave aspirational behavior in current-behavior docs.
|
||||
|
||||
Documentation is complete only when it matches the current code.
|
||||
|
||||
## Documentation Change Checklist
|
||||
|
||||
Before merging documentation changes, verify:
|
||||
|
||||
- README is concise and orientation-focused.
|
||||
- `docs/architecture.md` describes development principles.
|
||||
- Future work appears only under `docs/roadmap/`.
|
||||
- User-facing docs avoid unnecessary internals.
|
||||
- Developer-facing docs preserve boundaries and invariants.
|
||||
- Config examples match the schema.
|
||||
- CLI examples match real commands and flags.
|
||||
- Defaults appear in the canonical config reference.
|
||||
- No secrets or private data are included.
|
||||
- Links are accurate.
|
||||
556
docs/roadmap/audit.md
Normal file
556
docs/roadmap/audit.md
Normal file
@@ -0,0 +1,556 @@
|
||||
# Roadmap: Code Quality and Deduplication Audit
|
||||
|
||||
Status: Draft audit report
|
||||
|
||||
This report is a pre-1.0 implementation audit focused on high-confidence opportunities to simplify, centralize, or clarify Narratio before release. It is intentionally report-only: no refactors are included here.
|
||||
|
||||
The requested `docs/architecture.md` and `docs/development.md` paths do not exist in the current tree. This audit used the current policy documents at `docs/policy/architecture.md` and `docs/policy/development.md`, plus the current user, operator, and internal docs.
|
||||
|
||||
## 1. Executive Summary
|
||||
|
||||
Overall code quality is solid. The codebase has strong package boundaries in the important places: storage adapters expose a narrow object-store interface, AWS SDK types do not leak into app or stage logic, config loading is strict, and pipeline execution remains explicit and stage-driven. Recent pre-1.0 work has also produced useful central points for campaign/session config loading, secret-backed object-store creation, S3 audio caching, transcript artifact naming, local session paths, and S3 key construction.
|
||||
|
||||
The main release risk is not a large architectural flaw. It is policy drift from rapid feature growth. Several public-interface decisions now appear in more than one implementation path: artifact source interpretation, publish-output destination derivation, remote current-state inspection, cleanup safety checks, and session-oriented command parsing. Most of these are correct today, but a future bug fix would likely have to be made in multiple files.
|
||||
|
||||
Top three refactoring targets before 1.0:
|
||||
|
||||
1. Centralize artifact source and publish-output resolution across config validation, publish execution, status/artifacts output, restore, previous-cache hydration, and analyze input resolution.
|
||||
2. Consolidate shared session-command flag parsing and config-loading context for run/resume/run-stage/analyze/publish/restore/clean/session helpers without introducing a generic command framework.
|
||||
3. Finish the publish terminology cleanup internally so public `publish` behavior is not implemented through `archive`-named files, helpers, errors, and tests.
|
||||
|
||||
The codebase appears ready for a limited cleanup pass. No major architecture rewrite is warranted before 1.0.
|
||||
|
||||
## 2. High-Confidence Deduplication Opportunities
|
||||
|
||||
### Artifact Source and Publish Destination Policy Is Split Across Packages
|
||||
|
||||
Affected files/packages:
|
||||
|
||||
- `internal/config/validate.go`
|
||||
- `internal/artifacts/artifact_resolver.go`
|
||||
- `internal/artifacts/catalog.go`
|
||||
- `internal/stage/archive.go`
|
||||
- `internal/app/operator_helpers.go`
|
||||
- `internal/previouscache/previouscache.go`
|
||||
- `internal/stage/analyze.go`
|
||||
|
||||
Duplicated or near-duplicated behavior:
|
||||
|
||||
- Config validation accepts and derives destinations for `pipeline.publish.outputs[]` in `publishSourceKnown` and `derivePublishOutputDest`.
|
||||
- Publish execution derives destinations again in `resolvePublishOutputDest`.
|
||||
- Status and `artifacts list` derive destination display and remote checks in `helperPublishedOutputDest`.
|
||||
- Previous-cache hydration reconstructs candidate artifact locations from manifest outputs, published paths, and configured Scriptorium paths in `artifactRelativePathCandidates`.
|
||||
- Analyze resolves previous-session, built-in, and configured artifact sources separately in `resolveScriptoriumInput`.
|
||||
|
||||
Why it matters:
|
||||
|
||||
Artifact source IDs now define the public contract for analyze inputs, previous-session inputs, publish outputs, locks, status, artifacts listing, restore, and validation. When source interpretation is spread across these packages, it is easy for one path to accept, reject, or resolve a source differently from another.
|
||||
|
||||
Recommended refactor:
|
||||
|
||||
Create one small artifact-source policy layer, likely in `internal/artifacts` or a dependency-light sibling of `internal/artifactmodel`, that can:
|
||||
|
||||
- classify source IDs as built-in, configured artifact, or previous-session configured artifact;
|
||||
- validate a source against the current Scriptorium config;
|
||||
- derive the default published destination for a source;
|
||||
- normalize relative artifact destinations;
|
||||
- return consistent display metadata for status and artifacts output.
|
||||
|
||||
Then update config validation, publish execution, helper commands, previous-cache planning, and analyze input resolution to call that policy instead of deriving partial answers locally.
|
||||
|
||||
Suggested tests:
|
||||
|
||||
- `internal/artifacts`: source classification, configured artifact validation, default destination derivation, relative destination normalization.
|
||||
- `internal/config`: publish outputs and locks validate through the shared policy.
|
||||
- `internal/stage`: publish output resolution preserves locked, optional, required, and selected-artifact behavior.
|
||||
- `internal/app`: `artifacts list`, `status`, and locks use the same source rules as publish.
|
||||
- `internal/previouscache`: previous-session source resolution still checks manifest outputs, published paths, and configured output paths in the intended order.
|
||||
|
||||
Risk level: Medium. The behavior is public, but a table-driven shared policy should reduce risk if introduced behind existing tests.
|
||||
|
||||
### Publish Terminology Cleanup Is Incomplete Internally
|
||||
|
||||
Affected files/packages:
|
||||
|
||||
- `internal/stage/archive.go`
|
||||
- `internal/stage/archive_test.go`
|
||||
- `internal/artifacts/archive_identity.go`
|
||||
- `internal/app/post_archive_cleanup.go`
|
||||
- `internal/app/remote_locks.go`
|
||||
- `internal/app/operator_helpers.go`
|
||||
- tests under `internal/app` and `internal/config`
|
||||
- `internal/adapters/storage/archive.go`
|
||||
|
||||
Duplicated or near-duplicated behavior:
|
||||
|
||||
The public contract now uses `publish`, `published`, and `publish outputs`, but several internal names still use `archive`, `promotion`, or `promoted`. Examples include `archiveStage`, `ResolveArchiveSessionPrefix`, `ResolveArchiveCurrentStateKeys`, `runPostArchiveCleanup`, `staticArchiveLocks`, `normalizeArchiveRelativePath`, and test names such as `TestArchiveUploadsRunRecordPromotionsAndCurrentPointer`.
|
||||
|
||||
Why it matters:
|
||||
|
||||
This is mostly clarity risk, not current behavior risk. However, public docs and config now use publish terminology, while implementation and tests still use old names. This makes code review harder and increases the chance that future work reintroduces old config or command language.
|
||||
|
||||
Recommended refactor:
|
||||
|
||||
Do a mechanical naming cleanup after artifact-source policy is centralized:
|
||||
|
||||
- rename `internal/stage/archive.go` to a publish-oriented file and rename `archiveStage` to `publishStage`;
|
||||
- rename archive identity helpers to publish/current-state helpers while keeping S3 layout unchanged;
|
||||
- rename post-archive cleanup helpers and tests to post-publish cleanup;
|
||||
- update old comments and test failure messages that still say archive/promote when they mean publish/published;
|
||||
- leave the immutable run-history path `runs/{run_id}` unchanged.
|
||||
|
||||
Suggested tests:
|
||||
|
||||
- Existing `internal/stage`, `internal/app`, and `internal/artifacts` tests.
|
||||
- A final term sweep for old terminology, allowing only historical roadmap references and adapter names that are intentionally retained.
|
||||
|
||||
Risk level: Low to Medium. Mostly mechanical, but broad enough to create churn.
|
||||
|
||||
### Session-Oriented CLI Parsing Is Repeated
|
||||
|
||||
Affected files/packages:
|
||||
|
||||
- `internal/app/run.go`
|
||||
- `internal/app/resume.go`
|
||||
- `internal/app/run_stage.go`
|
||||
- `internal/app/restore.go`
|
||||
- `internal/app/clean.go`
|
||||
- `internal/app/operator_helpers.go`
|
||||
- `internal/app/session_args.go`
|
||||
|
||||
Duplicated or near-duplicated behavior:
|
||||
|
||||
Many commands repeat the same flag setup and session ID handling:
|
||||
|
||||
- `--config`, `--campaign`, `--campaign-file`, `--session`, and `--previous-session-id`;
|
||||
- positional session ID extraction;
|
||||
- `--session-id` compatibility through `applyParsedSessionIDArg`;
|
||||
- selected artifact parsing and validation for run/resume/analyze/publish/run-stage;
|
||||
- load through `loadCommandConfig` followed by `config.Validate`.
|
||||
|
||||
Why it matters:
|
||||
|
||||
The command set has recently moved toward `narratio session <subcommand> <session_id>` and shorter top-level convenience commands. Repeated parser setup makes it easy for one command to miss a new flag, use a stale help string, or apply session ID precedence differently.
|
||||
|
||||
Recommended refactor:
|
||||
|
||||
Keep command functions explicit, but add a small internal parser helper for common session-aware commands. Avoid a generic CLI framework. A good target is a helper that returns:
|
||||
|
||||
- common config flags;
|
||||
- resolved positional/flag session ID;
|
||||
- previous session override;
|
||||
- optional selected configured artifacts;
|
||||
- normalized command-specific positional validation.
|
||||
|
||||
`run-stage` can remain special because it has both stage and session positional arguments, but it should reuse the same common flag registration and selected-artifact parsing.
|
||||
|
||||
Suggested tests:
|
||||
|
||||
- Existing app command tests for run, resume, run-stage, analyze, publish, restore, clean, and session subcommands.
|
||||
- Focused tests for positional session ID vs `--session-id` mismatch, missing session ID, and unsupported `--artifacts` by command/stage.
|
||||
|
||||
Risk level: Medium. Refactor is local to app parsing but touches many public commands.
|
||||
|
||||
### Remote Current-State Discovery Is Reimplemented in Several Forms
|
||||
|
||||
Affected files/packages:
|
||||
|
||||
- `internal/app/restore_discovery.go`
|
||||
- `internal/previouscache/previouscache.go`
|
||||
- `internal/app/operator_helpers.go`
|
||||
- `internal/stage/prepare_previous.go`
|
||||
- `internal/app/remote_locks.go`
|
||||
|
||||
Duplicated or near-duplicated behavior:
|
||||
|
||||
Several paths check or download remote current state:
|
||||
|
||||
- restore discovers current run ID and current manifest, validates campaign/session identity, and decodes the manifest;
|
||||
- previous-cache planning repeats current run pointer and manifest checks for the previous session;
|
||||
- session validation checks previous current state with `Exists` calls;
|
||||
- remote lock loading separately checks and downloads `locks.yml`;
|
||||
- remote session fallback lists and downloads `session.yml`.
|
||||
|
||||
Why it matters:
|
||||
|
||||
These workflows are similar but not identical. Some need missing remote state to be an error, while status treats it as state. Still, the low-level sequence of key construction, `Exists`, temp download, decode, and campaign/session/run validation appears multiple times.
|
||||
|
||||
Recommended refactor:
|
||||
|
||||
Extract narrow app-level or artifact-level helpers for remote session state objects, not a generic storage workflow engine. Candidate helpers:
|
||||
|
||||
- download object to temp safely;
|
||||
- load current run pointer and manifest for a supplied session prefix;
|
||||
- validate downloaded current manifest identity;
|
||||
- represent missing current state as a typed error so status can downgrade it while restore/prepare fail.
|
||||
|
||||
Keep `storage.ObjectStore` as the boundary and keep S3 key construction in `internal/artifacts`.
|
||||
|
||||
Suggested tests:
|
||||
|
||||
- `internal/app`: restore current-state discovery, status missing-state behavior, session validate previous-state behavior.
|
||||
- `internal/previouscache`: required vs optional previous artifact behavior with missing current pointers/manifests.
|
||||
- `internal/app`: malformed remote lock/session data still fails closed where publish-capable execution requires it.
|
||||
|
||||
Risk level: Medium. The missing-state policy differs by caller, so the refactor should centralize mechanics and typed outcomes, not final command decisions.
|
||||
|
||||
### Safe Local Deletion Policy Is Duplicated
|
||||
|
||||
Affected files/packages:
|
||||
|
||||
- `internal/app/clean.go`
|
||||
- `internal/app/post_archive_cleanup.go`
|
||||
|
||||
Duplicated or near-duplicated behavior:
|
||||
|
||||
Both `clean` and post-publish cleanup implement scoped deletion checks:
|
||||
|
||||
- reject empty roots/targets;
|
||||
- resolve absolute paths;
|
||||
- refuse root deletion;
|
||||
- refuse deletion outside the configured root;
|
||||
- refuse symlink deletion;
|
||||
- handle missing targets as successful no-ops.
|
||||
|
||||
Why it matters:
|
||||
|
||||
Deletion policy is high-risk code. Even if the current implementations agree, future fixes should not need to be made twice.
|
||||
|
||||
Recommended refactor:
|
||||
|
||||
Extract a small app-level cleanup safety helper, for example `cleanup_target.go`, with functions for:
|
||||
|
||||
- validating a scoped directory target;
|
||||
- validating a scoped file target;
|
||||
- validating removable children under a root.
|
||||
|
||||
Keep command-specific reporting in `clean.go` and manifest metadata handling in post-publish cleanup.
|
||||
|
||||
Suggested tests:
|
||||
|
||||
- Move the existing focused unsafe-path tests to the shared helper.
|
||||
- Preserve `clean` dry-run tests and post-publish cleanup eligibility tests.
|
||||
|
||||
Risk level: Low. This is a contained refactor with clear behavior preservation.
|
||||
|
||||
### Temp Object Download Helper Is Duplicated
|
||||
|
||||
Affected files/packages:
|
||||
|
||||
- `internal/app/restore_discovery.go`
|
||||
- `internal/previouscache/previouscache.go`
|
||||
- `internal/app/remote_locks.go`
|
||||
- `internal/app/config_loader.go`
|
||||
|
||||
Duplicated or near-duplicated behavior:
|
||||
|
||||
Multiple call sites create a temp file, close it, download an object into it, and delete it on error or defer deletion. The app package has one `downloadObjectToTemp`, while `internal/previouscache` has another copy.
|
||||
|
||||
Why it matters:
|
||||
|
||||
Temp-download behavior affects cleanup, error wording, and future hardening. It is not worth abstracting all storage use, but this small operation is repeated enough to centralize.
|
||||
|
||||
Recommended refactor:
|
||||
|
||||
Add a narrow helper close to the storage boundary. Options:
|
||||
|
||||
- `internal/adapters/storage` helper only if it does not learn Narratio session semantics;
|
||||
- `internal/storageutil` if a small internal utility package is acceptable;
|
||||
- app-level helper plus a previouscache dependency inversion if the team wants to avoid a new package.
|
||||
|
||||
The helper should not hide `ObjectStore`; it should only implement safe temp download mechanics.
|
||||
|
||||
Suggested tests:
|
||||
|
||||
- temp file cleanup on failed download;
|
||||
- successful download returns a cleaned temp path;
|
||||
- callers preserve their current contextual error messages.
|
||||
|
||||
Risk level: Low.
|
||||
|
||||
## 3. Medium-Confidence Opportunities
|
||||
|
||||
### Operator Helper Implementation Is Too Broad for One File
|
||||
|
||||
Affected files/packages:
|
||||
|
||||
- `internal/app/operator_helpers.go`
|
||||
|
||||
Duplicated or near-duplicated behavior:
|
||||
|
||||
This 1,100+ line file owns session validation, status, session init, artifacts listing, locks list/add/remove, lock-store mutation, artifact catalog rendering, remote output availability, finding formatting, local input validation, and template rendering.
|
||||
|
||||
Why it matters:
|
||||
|
||||
The code is not inherently wrong, and keeping helper commands in `internal/app` fits the architecture. The issue is discoverability and local coupling. Small changes to one helper command require navigating unrelated helper behavior.
|
||||
|
||||
Recommended refactor:
|
||||
|
||||
Split by command or responsibility:
|
||||
|
||||
- `session_init.go`
|
||||
- `session_validate.go`
|
||||
- `status.go`
|
||||
- `artifacts_list.go`
|
||||
- `locks.go`
|
||||
- `helper_findings.go`
|
||||
- `helper_artifacts.go`
|
||||
|
||||
Do this only after higher-value policy centralization so the file split does not preserve duplicated logic under new names.
|
||||
|
||||
Suggested tests:
|
||||
|
||||
- Existing `internal/app/operator_helpers_test.go` can be split later, but a file split alone should not require behavior changes.
|
||||
|
||||
Risk level: Low.
|
||||
|
||||
### Restore Planning Contains Its Own Remote-to-Local Path Policy
|
||||
|
||||
Affected files/packages:
|
||||
|
||||
- `internal/app/restore_plan.go`
|
||||
|
||||
Duplicated or near-duplicated behavior:
|
||||
|
||||
Restore maps remote session keys back to local session paths in `restoreLocalRelativePathForKey`, with explicit include/exclude rules for `current/`, `runs/`, `logs/`, `reports/`, `config/`, `inputs/`, `transcripts/`, `artifacts/`, `previous/`, and optional `audio/`.
|
||||
|
||||
Why it may be intentional:
|
||||
|
||||
Restore is the only command that should translate an entire remote session prefix into a local session subset. It has command-specific conflict and `--include-audio` semantics.
|
||||
|
||||
Recommended refactor:
|
||||
|
||||
Do not generalize this immediately. If it changes again, move only the remote-key-to-local-restore-scope classifier into a small helper with table-driven tests. Leave restore action classification local to restore.
|
||||
|
||||
Suggested tests:
|
||||
|
||||
- Restore scope tests for every included/excluded root.
|
||||
- Audio-specific conflict behavior remains separate.
|
||||
|
||||
Risk level: Low.
|
||||
|
||||
### Manifest Output Scanning Is Repeated but Mostly Stage-Specific
|
||||
|
||||
Affected files/packages:
|
||||
|
||||
- `internal/artifacts/artifact_resolver.go`
|
||||
- `internal/previouscache/previouscache.go`
|
||||
- `internal/app/runner.go`
|
||||
|
||||
Duplicated or near-duplicated behavior:
|
||||
|
||||
Several call sites inspect manifest stage outputs or metadata to find artifact paths, published paths, run roots, or configured artifact outputs.
|
||||
|
||||
Why it may be intentional:
|
||||
|
||||
Manifest state has different meanings depending on caller: runtime artifact resolution, previous-cache reconstruction, and run summary construction are not the same policy.
|
||||
|
||||
Recommended refactor:
|
||||
|
||||
Avoid a broad manifest-query abstraction before 1.0. Consider adding only narrow helpers for stable metadata reads, such as reading `published_paths` from the publish stage, if the previous-cache and restore paths continue to grow.
|
||||
|
||||
Suggested tests:
|
||||
|
||||
- Existing manifest resolver tests plus previous-cache tests.
|
||||
|
||||
Risk level: Low.
|
||||
|
||||
### Command Output Formatting Could Be More Consistent
|
||||
|
||||
Affected files/packages:
|
||||
|
||||
- `internal/app/operator_helpers.go`
|
||||
- `internal/app/restore_report.go`
|
||||
- `internal/app/restore_plan.go`
|
||||
- `internal/app/clean.go`
|
||||
- `internal/app/plan.go`
|
||||
|
||||
Duplicated or near-duplicated behavior:
|
||||
|
||||
Status, session validate, artifacts list, locks, clean dry-run, restore dry-run, and plan all render text directly with `fmt.Fprintf`.
|
||||
|
||||
Why it may be intentional:
|
||||
|
||||
The output remains text-only and command-specific. A generic renderer would add complexity without much value.
|
||||
|
||||
Recommended refactor:
|
||||
|
||||
Postpone unless user-facing inconsistencies become painful. A small findings renderer already exists for validation-style output; that is enough for now.
|
||||
|
||||
Suggested tests:
|
||||
|
||||
- Snapshot-style output tests only for stable operator-facing lines that support workflows.
|
||||
|
||||
Risk level: Low.
|
||||
|
||||
## 4. Boundary and Responsibility Concerns
|
||||
|
||||
The major boundaries are healthy:
|
||||
|
||||
- `internal/adapters/storage` owns external storage implementation details.
|
||||
- App code creates object stores through `newCommandObjectStore`, which loads filesystem secrets first.
|
||||
- Stage code depends on `storage.ObjectStore`, not AWS SDK types.
|
||||
- `internal/audio` correctly centralizes S3 audio cache materialization without making the storage adapter aware of cache policy.
|
||||
- `internal/artifacts` owns most local paths and S3 keys.
|
||||
|
||||
Concerns to address:
|
||||
|
||||
- Artifact source policy is split between `internal/config`, `internal/artifacts`, `internal/stage`, `internal/app`, and `internal/previouscache`. This is the clearest boundary drift because source IDs are a shared public contract.
|
||||
- `internal/config` currently derives default publish destinations. Validation should be able to call source policy, but the canonical mapping itself should live outside config.
|
||||
- `internal/app/operator_helpers.go` owns artifact catalog rendering and remote published-output state. That is acceptable for formatting, but destination derivation and source classification should move out.
|
||||
- `internal/stage/archive.go` implements the public `publish` stage. This does not violate boundaries, but it creates conceptual drift.
|
||||
|
||||
Recommended home for shared logic:
|
||||
|
||||
- Source classification and destination derivation: `internal/artifacts` or `internal/artifactmodel` plus a small adapter from Scriptorium config.
|
||||
- Remote key construction: continue using `internal/artifacts`.
|
||||
- Object-store initialization: keep in `internal/app`.
|
||||
- Command parsing: keep in `internal/app`.
|
||||
- Stage-specific execution policy: keep in `internal/stage`.
|
||||
|
||||
## 5. Path and Remote Key Construction Review
|
||||
|
||||
Local path construction is mostly centralized:
|
||||
|
||||
- `internal/artifacts/paths.go` owns session work roots, run roots, spool paths, previous-cache paths, and audio cache paths.
|
||||
- Stage code often gets `artifacts.SessionPaths` and joins stage-local files from those roots, which is appropriate.
|
||||
- The previous-cache redundant nested artifact path has already been addressed by `previousArtifactCacheRelativePath`.
|
||||
|
||||
Remote key construction is mostly centralized:
|
||||
|
||||
- `internal/artifacts/s3_keys.go` owns session prefixes, run prefixes, audio prefixes, `session.yml`, `locks.yml`, current manifest/run pointer keys, published output keys, and run-relative keys.
|
||||
- App and stage code call these helpers rather than scattering full S3 key string concatenation.
|
||||
|
||||
Areas needing cleanup:
|
||||
|
||||
- `ResolveArchiveBucket`, `ResolveArchiveSessionPrefix`, `ResolveArchiveRunPrefix`, and `ResolveArchiveCurrentStateKeys` should be renamed to publish/current-state terminology.
|
||||
- `normalizeArchiveRelativePath` exists in both `internal/stage/archive.go` and `internal/previouscache/previouscache.go`; `normalizeHelperArchiveRelativePath` exists in `internal/app/operator_helpers.go`. These should converge into one helper for clean relative artifact destination paths.
|
||||
- `restore_plan.go` owns `normalizeRemoteKey` and remote key scope mapping. That may remain restore-specific, but it should be watched because it overlaps with S3 key normalization helpers.
|
||||
- `downloadObjectToTemp` exists in more than one package and can be centralized.
|
||||
|
||||
## 6. Artifact/Catalog/Source Resolution Review
|
||||
|
||||
Artifact source handling has a strong foundation:
|
||||
|
||||
- Transcript source IDs and paths are centralized in `internal/artifactmodel/transcripts.go`.
|
||||
- Runtime artifact registry and resolver live in `internal/artifacts/artifact_resolver.go`.
|
||||
- Configured artifact source IDs are consistently formed by `artifacts.ConfiguredArtifactSourceID`.
|
||||
- Previous-session source IDs are recognized by `artifacts.PreviousSessionArtifactName`.
|
||||
- The runtime catalog supports built-ins, configured artifacts, selected artifact execution, and availability.
|
||||
|
||||
The remaining issue is that consumers still build their own partial views of this model:
|
||||
|
||||
- config validation validates and derives publish output destinations;
|
||||
- publish execution resolves included outputs, skipped optional outputs, skipped unselected outputs, and locked outputs;
|
||||
- status/artifacts list derives display destinations and remote published state;
|
||||
- previous-cache planning reconstructs candidate remote paths from previous manifests and publish metadata;
|
||||
- analyze input resolution has its own missing-source messages and previous-session behavior.
|
||||
|
||||
Recommendation:
|
||||
|
||||
Make artifact/source resolution the next cleanup target. The goal is not to create one all-purpose resolver. The goal is to centralize the public source vocabulary and destination derivation so each caller can keep its own policy for missing/required/locked behavior.
|
||||
|
||||
## 7. Config and Command-Loading Review
|
||||
|
||||
Config loading is generally consistent:
|
||||
|
||||
- `loadCommandConfig` is the main command path for pipeline, campaign, session, local discovery, and remote session fallback.
|
||||
- `loadPipelineCampaignConfig` covers commands that create session config and therefore cannot load an existing session.
|
||||
- `newCommandObjectStore` correctly centralizes secret-backed object-store creation.
|
||||
- `config.LoadSessionBytesWithOptions` now rejects session templates outside `session init`, preserving strict concrete session loading.
|
||||
|
||||
Intentional differences:
|
||||
|
||||
- `session init` loads only pipeline and campaign because it creates `session.yml`.
|
||||
- `clean --all` loads only pipeline because it is not session-specific.
|
||||
- `status --manifest` remains a compatibility/local-manifest mode.
|
||||
|
||||
Likely accidental drift to clean up:
|
||||
|
||||
- Common flags and help strings are repeated across commands.
|
||||
- Some helper-command messages still say archive where they now mean publish.
|
||||
- `restore` uses `fs.SetOutput(out)` while most other command parsers discard flag package output and wrap errors themselves. This may be intentional for `--help`, but it is a difference worth documenting or standardizing.
|
||||
- App tests and helper names still contain old archive/promotion terminology, making it harder to see which public contract is current.
|
||||
|
||||
## 8. Refactors to Avoid Before 1.0
|
||||
|
||||
Avoid these before release:
|
||||
|
||||
- A generic workflow engine or DAG abstraction. The explicit stage list is a core design choice and is working.
|
||||
- A broad manifest query framework. Add narrow helpers only where repeated policy is clear.
|
||||
- Moving secret loading into storage adapters. Secret loading is app orchestration policy and should stay out of adapters.
|
||||
- Making storage adapters infer campaign/session/root-prefix semantics. They should continue to receive concrete keys.
|
||||
- Replacing command functions with a generic CLI framework. Small shared flag parsers are enough.
|
||||
- Generalizing all file copy/download behavior. S3 audio cache materialization is intentionally special; ordinary restore/download logic has different semantics.
|
||||
- Adding compatibility aliases for old archive/promote or old transcript names during cleanup. The repo has intentionally made hard cutovers.
|
||||
|
||||
## 9. Recommended Implementation Sequence
|
||||
|
||||
1. Centralize relative artifact destination normalization and temp object download helpers.
|
||||
- Scope: low-risk shared helpers for repeated mechanics.
|
||||
- Tests: `internal/artifacts` or helper-package tests, plus existing app/stage tests.
|
||||
|
||||
2. Centralize artifact source and publish-output policy.
|
||||
- Scope: source classification, source validation, default published destination derivation, destination normalization.
|
||||
- Tests: `internal/artifacts`, `internal/config`, `internal/stage -run Publish`, `internal/app -run 'Artifacts|Status|Locks'`, `internal/previouscache`.
|
||||
|
||||
3. Finish publish terminology cleanup.
|
||||
- Scope: rename archive-named files/helpers/tests/comments where they now mean publish; keep S3 layout stable.
|
||||
- Tests: `go test ./internal/stage -v`, `go test ./internal/app -v`, `go test ./internal/artifacts -v`.
|
||||
|
||||
4. Consolidate session-aware command parsing.
|
||||
- Scope: common config/session/artifact flag registration and session ID resolution; no public CLI behavior change.
|
||||
- Tests: app command tests for run, resume, run-stage, analyze, publish, restore, clean, session helpers.
|
||||
|
||||
5. Extract remote current-state mechanics.
|
||||
- Scope: shared helpers for current run pointer/manifest load and identity validation, with typed missing-state errors.
|
||||
- Tests: restore discovery, previous-cache, status, session validate.
|
||||
|
||||
6. Split operator helper implementation by responsibility.
|
||||
- Scope: file organization and small formatting/helper extraction only after policy deduplication.
|
||||
- Tests: existing `internal/app` tests.
|
||||
|
||||
7. Sweep dead transitional terminology and stale tests.
|
||||
- Scope: comments, test names, old strings, internal docs that still say archive/promote where publish is now canonical.
|
||||
- Tests: final `rg` sweeps plus full test run.
|
||||
|
||||
## 10. Test Strategy
|
||||
|
||||
Focused package checks for cleanup work:
|
||||
|
||||
- `go test ./internal/artifacts -v`
|
||||
- `go test ./internal/config -v`
|
||||
- `go test ./internal/stage -run 'Analyze|Publish|Prepare|Restore' -v`
|
||||
- `go test ./internal/app -run 'Run|RunStage|Analyze|Publish|Restore|Clean|Status|Artifacts|Locks|Session' -v`
|
||||
- `go test ./internal/previouscache -v`
|
||||
- `go test ./internal/adapters/storage -v`
|
||||
- `go test ./internal/manifest -v`
|
||||
|
||||
Tests to add or strengthen during follow-up refactors:
|
||||
|
||||
- one table of valid/invalid artifact source IDs used by config validation, publish, locks, status, and analyze;
|
||||
- one table of default published destination derivation for built-in and configured artifacts;
|
||||
- relative destination normalization and path traversal rejection;
|
||||
- shared remote current-state load outcomes: missing pointer, missing manifest, malformed manifest, campaign mismatch, session mismatch, run ID mismatch;
|
||||
- shared cleanup safety helper behavior for files, directories, roots, symlinks, and outside-root paths;
|
||||
- common session command parsing behavior for positional session IDs, `--session-id`, mismatch errors, and unsupported artifacts flags.
|
||||
|
||||
Full validation after each cleanup commit:
|
||||
|
||||
- `go test ./...`
|
||||
|
||||
Useful final searches:
|
||||
|
||||
- `rg -n "archive|promote|promoted|promotion" internal docs examples cmd`
|
||||
- `rg -n "ResolveArchive|archiveStage|post_archive|staticArchive|normalizeArchive" internal`
|
||||
- `rg -n "narratio.transcript.merged|narratio.transcript.full|narratio.transcript.trimmed" internal docs examples`
|
||||
- `rg -n "previous_session_artifact|promote_artifacts|pipeline.archive" internal docs examples`
|
||||
|
||||
## 11. Appendix: Findings Not Worth Acting On
|
||||
|
||||
- Stage-local path joins for files inside a stage run directory are acceptable. They are local implementation details, not shared path policy.
|
||||
- Direct `fmt.Fprintf` output in simple commands is acceptable. A generic renderer would likely obscure behavior.
|
||||
- Restore's remote-session-prefix filtering is command-specific enough to stay local unless restore scope changes again.
|
||||
- `session init` template rendering should remain separate from ordinary session loading. That separation is now a useful safety boundary.
|
||||
- S3 audio cache materialization is already centralized in `internal/audio`; do not fold it into a generic downloader.
|
||||
- Manifest-driven resume behavior should not be abstracted broadly. The explicit runner behavior is easier to audit.
|
||||
294
docs/roadmap/cleanup.md
Normal file
294
docs/roadmap/cleanup.md
Normal file
@@ -0,0 +1,294 @@
|
||||
# Roadmap: Pre-1.0 Code Cleanup
|
||||
|
||||
Status: Implemented (Stages 1-6 complete)
|
||||
|
||||
This roadmap turns the findings in `docs/roadmap/audit.md` into staged cleanup work for the 1.0 release and records completion status for each selected stage.
|
||||
|
||||
The cleanup work must follow the policy documents under `docs/policy/`, especially these invariants:
|
||||
|
||||
- keep Narratio explicit and stage-driven;
|
||||
- do not introduce a generic workflow engine, DAG abstraction, or generic CLI framework;
|
||||
- keep external-system details behind adapters;
|
||||
- do not move campaign/session/root-prefix semantics into storage adapters;
|
||||
- keep AWS SDK types out of app and stage logic;
|
||||
- keep path and remote key construction centralized;
|
||||
- preserve manifest-driven run state;
|
||||
- keep public CLI/config behavior stable unless a stage explicitly says it is an internal naming cleanup.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- Do not change public command syntax, config schema, S3 key layout, manifest schema, or artifact source IDs as part of this cleanup.
|
||||
- Do not add compatibility aliases or migration logic.
|
||||
- Do not rewrite stage execution, manifest state transitions, or adapter contracts.
|
||||
- Do not generalize text output into a generic reporting framework.
|
||||
- Do not fold S3 audio cache behavior into a generic downloader.
|
||||
- Do not move secret loading into storage adapters.
|
||||
|
||||
## Stage 1: Shared Low-Risk Mechanics
|
||||
|
||||
Goal: remove duplicated mechanics that are easy to test and should not affect public behavior.
|
||||
|
||||
Implementation decisions:
|
||||
|
||||
- Add one shared helper for safe relative artifact destination normalization.
|
||||
- It must reject empty paths, absolute paths, `.`, `..`, and traversal outside the artifact/session scope.
|
||||
- It must normalize separators to slash-form for artifact and S3 destination logic.
|
||||
- It must be dependency-light enough to be called from config validation, app helpers, publish execution, and previous-cache planning.
|
||||
- Add one shared object-store temp download helper.
|
||||
- It must take `context.Context`, `storage.ObjectStore`, a key, and a temp-file pattern.
|
||||
- It must create and close the temp file before download, remove the temp file on failed download, and return a cleaned local path on success.
|
||||
- It must not infer bucket, campaign, session, run, or root-prefix semantics.
|
||||
- Extract shared cleanup target validation for local deletion.
|
||||
- Cover scoped directory deletion, scoped file deletion, and removable children under a root.
|
||||
- Preserve existing safety rules: reject empty roots/targets, root deletion, outside-root paths, symlinks, and wrong target types.
|
||||
- Keep command-specific output in `clean` and manifest metadata handling in post-publish cleanup.
|
||||
|
||||
Expected callers:
|
||||
|
||||
- replace duplicate relative destination normalization in publish execution, helper command rendering, and previous-cache planning;
|
||||
- replace duplicate temp download helpers in app and previous-cache code;
|
||||
- replace duplicate scoped deletion validation in clean and post-publish cleanup.
|
||||
|
||||
Tests:
|
||||
|
||||
- Add focused tests for destination normalization and path traversal rejection.
|
||||
- Add temp download tests for success, failed download cleanup, and preserved contextual caller errors.
|
||||
- Add shared cleanup validation tests for directories, files, symlinks, missing targets, root deletion, and outside-root targets.
|
||||
- Run:
|
||||
- `go test ./internal/artifacts -v`
|
||||
- `go test ./internal/adapters/storage -v`
|
||||
- `go test ./internal/app -run 'Clean|Post' -v`
|
||||
- `go test ./...`
|
||||
|
||||
Completion criteria:
|
||||
|
||||
- duplicated low-level mechanics are removed;
|
||||
- public behavior and output are unchanged;
|
||||
- no stage, command, or config semantics move into storage adapters.
|
||||
|
||||
## Stage 2: Artifact Source and Published Output Policy
|
||||
|
||||
Goal: make artifact source IDs and published-output destination derivation a single shared policy.
|
||||
|
||||
Implementation decisions:
|
||||
|
||||
- Introduce `internal/artifactpolicy` as the shared source policy package.
|
||||
- This package is the long-term home because it avoids config/artifacts import cycles.
|
||||
- It may depend on dependency-light model packages, but it must not depend on app, stage, manifest stores, storage adapters, or downstream adapters.
|
||||
- Centralize these behaviors in `internal/artifactpolicy`:
|
||||
- classify source IDs as built-in, configured artifact, or previous-session configured artifact;
|
||||
- parse configured artifact keys from `narratio.artifact.<key>`;
|
||||
- parse previous-session artifact keys from `narratio.previous_session.artifact.<key>`;
|
||||
- validate configured artifact sources against `pipeline.scriptorium.artifacts`;
|
||||
- validate publish lock/output sources;
|
||||
- derive default published destinations for built-in and configured artifact sources;
|
||||
- normalize safe relative published-output destinations.
|
||||
- Update callers to consume the shared policy:
|
||||
- config validation for `publish.outputs` and `publish.locks`;
|
||||
- publish-stage output resolution;
|
||||
- status and `artifacts list` rendering;
|
||||
- locks list/add/remove validation;
|
||||
- analyze input source handling;
|
||||
- previous-cache candidate planning.
|
||||
- Preserve caller-specific policy at call sites.
|
||||
- Required vs optional behavior remains in publish, analyze, restore, and previous-cache callers.
|
||||
- Locked output behavior remains in publish.
|
||||
- Text formatting remains in app commands.
|
||||
- Manifest path scanning remains in artifact/previous-cache logic unless directly tied to source policy.
|
||||
|
||||
Tests:
|
||||
|
||||
- Add `internal/artifactpolicy` table tests for source classification, configured artifact validation, previous-session parsing, default destination derivation, and destination normalization.
|
||||
- Update `internal/config` tests so publish outputs and locks validate through the shared policy.
|
||||
- Update `internal/stage` publish tests for selected, unselected, optional, required, and locked output behavior.
|
||||
- Update `internal/app` tests for status, artifacts list, and locks.
|
||||
- Update `internal/previouscache` tests for previous-session candidate ordering.
|
||||
- Run:
|
||||
- `go test ./internal/artifacts -v`
|
||||
- `go test ./internal/config -v`
|
||||
- `go test ./internal/stage -run 'Analyze|Publish' -v`
|
||||
- `go test ./internal/app -run 'Artifacts|Status|Locks' -v`
|
||||
- `go test ./internal/previouscache -v`
|
||||
- `go test ./...`
|
||||
|
||||
Completion criteria:
|
||||
|
||||
- artifact source vocabulary and destination derivation are no longer reimplemented in config, app, stage, and previous-cache packages;
|
||||
- every caller still owns its own missing/required/optional/locked decision;
|
||||
- public behavior is unchanged.
|
||||
|
||||
## Stage 3: Publish Terminology Cleanup
|
||||
|
||||
Goal: align internal implementation names with the public publish contract.
|
||||
|
||||
Implementation decisions:
|
||||
|
||||
- Rename archive-named internal files, types, helpers, comments, and tests that now implement publish behavior.
|
||||
- Replace names such as:
|
||||
- `archiveStage` with `publishStage`;
|
||||
- `ResolveArchiveSessionPrefix` with publish/current-state terminology;
|
||||
- `ResolveArchiveRunPrefix` with publish/run-history terminology;
|
||||
- `ResolveArchiveCurrentStateKeys` with current-state terminology;
|
||||
- `runPostArchiveCleanup` with post-publish cleanup terminology;
|
||||
- `staticArchiveLocks` with publish lock terminology.
|
||||
- Keep the S3 layout stable:
|
||||
- `{session_prefix}/runs/{run_id}/`;
|
||||
- `{session_prefix}/current/manifest.json`;
|
||||
- `{session_prefix}/current/run_id.txt`;
|
||||
- `{session_prefix}/locks.yml`.
|
||||
- Keep the public stage name `publish`.
|
||||
- Keep old archive/promote references only where they are historical roadmap context or intentionally describe immutable run history.
|
||||
|
||||
Tests and checks:
|
||||
|
||||
- Run:
|
||||
- `go test ./internal/stage -v`
|
||||
- `go test ./internal/app -v`
|
||||
- `go test ./internal/artifacts -v`
|
||||
- `go test ./...`
|
||||
- Run stale-term sweeps:
|
||||
- `rg -n "archive|promote|promoted|promotion" internal docs examples cmd`
|
||||
- `rg -n "ResolveArchive|archiveStage|post_archive|staticArchive|normalizeArchive" internal`
|
||||
|
||||
Completion criteria:
|
||||
|
||||
- public publish behavior is no longer implemented through archive/promote names;
|
||||
- remaining old terms are intentionally historical, test-fixture bucket names, or roadmap-only context;
|
||||
- no config, CLI, manifest, or S3 layout changes are introduced.
|
||||
|
||||
## Stage 4: Session Command Parsing Consolidation
|
||||
|
||||
Goal: reduce command-loading drift while keeping command handlers explicit.
|
||||
|
||||
Implementation decisions:
|
||||
|
||||
- Add a small app-level parser helper for common session-aware commands.
|
||||
- Centralize:
|
||||
- common config flags: `--config`, `--campaign`, `--campaign-file`, `--session`;
|
||||
- positional session ID handling;
|
||||
- `--session-id` compatibility;
|
||||
- `--previous-session-id`;
|
||||
- optional selected-artifact parsing for commands that support it.
|
||||
- Keep command handlers explicit and readable.
|
||||
- Do not introduce a generic CLI framework.
|
||||
- Treat these as intentional special cases:
|
||||
- `session init` loads pipeline and campaign but not session;
|
||||
- `clean --all` loads pipeline only;
|
||||
- `status --manifest` remains local-manifest mode;
|
||||
- `run-stage` keeps its stage-name positional handling but reuses common flag parsing where practical.
|
||||
- Standardize flag help text where commands use the same semantics.
|
||||
|
||||
Tests:
|
||||
|
||||
- Update app command tests for:
|
||||
- positional session ID;
|
||||
- `--session-id`;
|
||||
- positional/flag mismatch;
|
||||
- missing session ID;
|
||||
- `--previous-session-id`;
|
||||
- unsupported `--artifacts` by command/stage;
|
||||
- unchanged behavior for `session init`, `clean --all`, and `status --manifest`.
|
||||
- Run:
|
||||
- `go test ./internal/app -run 'Run|RunStage|Analyze|Publish|Restore|Clean|Session' -v`
|
||||
- `go test ./internal/app -v`
|
||||
- `go test ./...`
|
||||
|
||||
Completion criteria:
|
||||
|
||||
- shared session flag/session ID behavior has one implementation;
|
||||
- command handlers remain command-specific;
|
||||
- public command syntax and output stay unchanged.
|
||||
|
||||
## Stage 5: Remote Current-State Mechanics
|
||||
|
||||
Goal: centralize remote current-state loading mechanics without hiding caller policy.
|
||||
|
||||
Implementation decisions:
|
||||
|
||||
- Extract narrow helpers for remote current state.
|
||||
- Load current run pointer through `storage.ObjectStore`.
|
||||
- Load and decode current manifest through `storage.ObjectStore`.
|
||||
- Validate campaign, session, and run identity when requested by the caller.
|
||||
- Return typed missing-state errors.
|
||||
- Preserve caller policy:
|
||||
- restore treats missing or invalid current state as an error;
|
||||
- previous-cache hydration fails for required previous artifacts and skips optional missing artifacts;
|
||||
- status reports missing remote state as state, not command failure;
|
||||
- session validate emits findings and fails only for error findings.
|
||||
- Keep all remote key construction in `internal/artifacts`.
|
||||
- Keep object-store initialization in `internal/app`.
|
||||
- Do not add storage adapter knowledge of campaigns, sessions, runs, root prefixes, current state, or manifests.
|
||||
|
||||
Tests:
|
||||
|
||||
- Add helper tests for:
|
||||
- missing current run pointer;
|
||||
- missing current manifest;
|
||||
- empty run pointer;
|
||||
- malformed manifest;
|
||||
- campaign mismatch;
|
||||
- session mismatch;
|
||||
- run ID mismatch.
|
||||
- Update restore, previous-cache, status, and session validate tests to prove their caller-specific behavior is unchanged.
|
||||
- Run:
|
||||
- `go test ./internal/app -run 'Restore|Status|SessionValidate' -v`
|
||||
- `go test ./internal/previouscache -v`
|
||||
- `go test ./...`
|
||||
|
||||
Completion criteria:
|
||||
|
||||
- low-level remote current-state mechanics are shared;
|
||||
- missing-state behavior remains caller-specific;
|
||||
- storage adapter boundaries remain unchanged.
|
||||
|
||||
## Stage 6: Operator Helper File Split and Final Sweep
|
||||
|
||||
Goal: improve maintainability after shared policy and mechanics are already centralized.
|
||||
|
||||
Implementation decisions:
|
||||
|
||||
- Split the large operator helper implementation by command or responsibility.
|
||||
- Suggested file grouping:
|
||||
- session init;
|
||||
- session validate;
|
||||
- status;
|
||||
- artifacts list;
|
||||
- locks;
|
||||
- helper findings;
|
||||
- helper artifact rendering.
|
||||
- Do not change command syntax, text output, config loading, remote loading, lock behavior, or artifact catalog behavior during the split.
|
||||
- Keep output formatting text-only and command-specific unless a concrete inconsistency remains after the split.
|
||||
- Update roadmap status notes after each completed stage.
|
||||
|
||||
Tests and checks:
|
||||
|
||||
- Run:
|
||||
- `go test ./internal/app -v`
|
||||
- `go test ./...`
|
||||
- Final searches:
|
||||
- `rg -n "archive|promote|promoted|promotion" internal docs examples cmd`
|
||||
- `rg -n "narratio.transcript.merged|narratio.transcript.full|narratio.transcript.trimmed" internal docs examples`
|
||||
- `rg -n "previous_session_artifact|promote_artifacts|pipeline.archive" internal docs examples`
|
||||
|
||||
Completion criteria:
|
||||
|
||||
- operator helper code is easier to navigate;
|
||||
- stale implementation terminology is removed or intentionally documented;
|
||||
- no behavior changes are introduced by file organization.
|
||||
|
||||
## Overall Validation
|
||||
|
||||
After each implementation stage:
|
||||
|
||||
- run the focused tests listed for that stage;
|
||||
- run `go test ./...`;
|
||||
- run `git status --short`;
|
||||
- update this roadmap to mark the completed stage implemented only after code, tests, and documentation are aligned.
|
||||
|
||||
## Assumptions
|
||||
|
||||
- This roadmap is a cleanup plan, not a feature plan.
|
||||
- Stages may be implemented as separate prompts/commits.
|
||||
- `internal/artifactpolicy` is the chosen home for shared source policy.
|
||||
- Shared object-store temp download helpers must not learn Narratio session semantics.
|
||||
- Public behavior must remain stable unless a stage explicitly says it is internal terminology cleanup.
|
||||
105
docs/roadmap/documentation-stage1-audit.md
Normal file
105
docs/roadmap/documentation-stage1-audit.md
Normal file
@@ -0,0 +1,105 @@
|
||||
# Documentation Pass: Stage 1 Audit
|
||||
|
||||
Status: Completed (2026-05-23)
|
||||
|
||||
## Scope Reviewed
|
||||
|
||||
- All non-policy documentation files under `docs/`
|
||||
- `README.md`
|
||||
- Documentation references to maintained `examples/` files
|
||||
- Documentation-related expectations in tests under `internal/**`
|
||||
|
||||
## File Inventory and Canonical Scope
|
||||
|
||||
| File | Intended audience | Canonical scope (per policy) | Primary source-of-truth anchors |
|
||||
| --- | --- | --- | --- |
|
||||
| `README.md` | Users, operators | Project orientation and links | `cmd/narratio`, `internal/app/commands.go`, docs index files |
|
||||
| `docs/cli.md` | Users, operators | CLI syntax, flags, command workflows | `internal/app/*.go`, `internal/app/*_test.go` |
|
||||
| `docs/config.md` | Operators, advanced users | Config discovery, schema, defaults, examples | `internal/config/*.go`, `internal/config/*_test.go`, `examples/*` |
|
||||
| `docs/operations.md` | Operators | Run/resume/publish/restore/cleanup workflows | `internal/app/runner.go`, `internal/app/restore*.go`, `internal/stage/archive.go`, `internal/artifacts/*.go` |
|
||||
| `docs/troubleshooting.md` | Operators | Failure diagnosis and safe fixes | `internal/app`, `internal/stage`, related tests |
|
||||
| `docs/internal/README.md` | Developers, LLM coding agents | Internal docs index and scope boundaries | `docs/internal/*.md`, policy docs |
|
||||
| `docs/internal/adapters.md` | Developers, LLM coding agents | Adapter boundaries and ownership | `internal/adapters/*`, `internal/stage/*` |
|
||||
| `docs/internal/artifacts.md` | Developers, LLM coding agents | Artifact catalog and source resolution contracts | `internal/artifacts/*`, `internal/stage/analyze.go`, `internal/stage/prepare_previous.go` |
|
||||
| `docs/internal/command-restore.md` | Developers, LLM coding agents | Restore command architecture and contracts | `internal/app/restore*.go`, `internal/app/restore*_test.go` |
|
||||
| `docs/internal/manifest.md` | Developers, LLM coding agents | Session/run manifest contracts and transitions | `internal/manifest/*`, `internal/app/runner.go`, `internal/stage/*` |
|
||||
| `docs/internal/stage-prepare.md` | Developers, LLM coding agents | Prepare stage IO and invariants | `internal/stage/prepare.go`, `internal/stage/prepare*_test.go` |
|
||||
| `docs/internal/stage-transcribe.md` | Developers, LLM coding agents | Transcribe stage IO and invariants | `internal/stage/transcribe.go`, `internal/stage/transcribe_test.go` |
|
||||
| `docs/internal/stage-merge.md` | Developers, LLM coding agents | Merge stage IO and invariants | `internal/stage/merge.go`, `internal/stage/merge_test.go` |
|
||||
| `docs/internal/stage-polish.md` | Developers, LLM coding agents | Polish stage IO and invariants | `internal/stage/polish.go`, `internal/stage/polish_test.go` |
|
||||
| `docs/internal/stage-normalize.md` | Developers, LLM coding agents | Normalize stage IO and invariants | `internal/stage/normalize.go`, `internal/stage/normalize_test.go` |
|
||||
| `docs/internal/stage-trim.md` | Developers, LLM coding agents | Trim stage IO and invariants | `internal/stage/trim.go`, `internal/stage/trim_test.go` |
|
||||
| `docs/internal/stage-analyze.md` | Developers, LLM coding agents | Analyze stage artifact execution and selection | `internal/stage/analyze.go`, `internal/stage/analyze_test.go` |
|
||||
| `docs/internal/stage-publish.md` | Developers, LLM coding agents | Publish-stage commit/upload invariants | `internal/stage/archive.go`, `internal/stage/archive_test.go` |
|
||||
| `docs/internal/storage.md` | Developers, LLM coding agents | Storage adapter contracts and semantics | `internal/adapters/storage/*`, `internal/app/object_store.go` |
|
||||
| `docs/internal/workspace.md` | Developers, LLM coding agents | Local workspace/session/run path model | `internal/artifacts/*`, `internal/app/runner.go`, `internal/stage/run_local.go` |
|
||||
| `docs/integrations/README.md` | Developers, LLM coding agents | Integration docs index | `docs/integrations/*.md` |
|
||||
| `docs/integrations/audita.md` | Developers, integration maintainers | Audita adapter contract | `internal/adapters/audita/*`, `internal/stage/polish.go` |
|
||||
| `docs/integrations/seriatim.md` | Developers, integration maintainers | Seriatim adapter contract | `internal/adapters/seriatim/*`, `internal/stage/merge.go`, `internal/stage/normalize.go`, `internal/stage/trim.go` |
|
||||
| `docs/integrations/scriptorium.md` | Developers, integration maintainers | Scriptorium adapter contract | `internal/adapters/scriptorium/*`, `internal/stage/analyze.go`, `internal/stage/trim.go` |
|
||||
| `docs/roadmap/documentation.md` | Developers, maintainers | Planning and implementation sequencing for documentation pass | N/A (planning artifact) |
|
||||
| `docs/roadmap/documentation-stage1-audit.md` | Developers, maintainers | Stage-1 inventory and source-of-truth audit record | N/A (planning artifact) |
|
||||
|
||||
## Source-of-Truth Mapping Summary
|
||||
|
||||
- CLI behaviors and command names are grounded in `internal/app/commands.go` and command handlers in `internal/app/*.go`.
|
||||
- Stage order and canonical stage names are grounded in `internal/stage/placeholders.go` (`prepare` -> `transcribe` -> `merge` -> `polish` -> `normalize` -> `trim` -> `analyze` -> `publish` -> `notify`).
|
||||
- Publish behavior and current-pointer commit semantics are grounded in `internal/stage/archive.go`.
|
||||
- Config schema/defaults/validation are grounded in `internal/config/*`.
|
||||
- Local/remote paths, publish keys, and workspace layout are grounded in `internal/artifacts/*`.
|
||||
- Restore behavior and report contracts are grounded in `internal/app/restore*.go`.
|
||||
- Maintained examples and schema compatibility are grounded in `examples/*` plus `internal/config/load_validate_test.go` (`TestExamplesLoadAndValidate`).
|
||||
|
||||
## Findings
|
||||
|
||||
### Broken or stale references
|
||||
|
||||
1. `README.md` linked to non-existent files:
|
||||
- `docs/development.md`
|
||||
- `docs/architecture.md`
|
||||
2. `docs/internal/README.md` and `docs/integrations/README.md` linked to non-existent path:
|
||||
- `docs/documentation/policy.md`
|
||||
|
||||
Stage-1 fix applied:
|
||||
- Updated those links to existing policy docs under `docs/policy/`.
|
||||
|
||||
### Stale terminology sweep
|
||||
|
||||
Sweep terms used: `archive`, `promote`, `promoted`, `promote_artifacts`, `run-stage archive`.
|
||||
|
||||
Findings:
|
||||
- User-facing docs in scope did not show obvious stale command examples requiring immediate correction.
|
||||
- Internal code and tests still contain historical `archive` identifiers while user-facing command/stage naming is `publish` (for example, `internal/stage/archive.go` type names). This is acceptable for now but should be normalized deliberately, not incidentally.
|
||||
|
||||
Stage-1 fix applied:
|
||||
- Updated clearly stale publish-related wording in test expectation messages/comments:
|
||||
- `internal/app/commands_test.go`
|
||||
- `internal/app/operator_helpers_test.go`
|
||||
|
||||
### Example path validation
|
||||
|
||||
- All `examples/...` paths referenced from non-policy docs resolve to existing files.
|
||||
- `internal/config/load_validate_test.go` includes `TestExamplesLoadAndValidate` and points to current example files.
|
||||
|
||||
### Roadmap leakage into current-behavior docs
|
||||
|
||||
- No obvious roadmap-only behavior leakage found in non-roadmap docs during this sweep.
|
||||
|
||||
### Duplicate content and scope drift
|
||||
|
||||
- No severe duplication requiring immediate rewrite in this stage.
|
||||
- Existing docs still need full content rewrite for 1.0 readiness in later stages (user/operator first, then internal/integrations), as planned.
|
||||
|
||||
### Canonical-home inconsistency to resolve in rewrite stages
|
||||
|
||||
- Policy canonical-home language names `docs/architecture.md` and `docs/development.md`, while current repository stores those policy documents under `docs/policy/`.
|
||||
- Stage 1 preserves repository behavior by fixing broken links to existing files. Later rewrite stages should converge canonical-home paths and references consistently across docs.
|
||||
|
||||
## Stage-1 Completion Check
|
||||
|
||||
Completed for this stage:
|
||||
- Full non-policy file inventory with audience and scope mapping.
|
||||
- Source-of-truth crosswalk to code/tests.
|
||||
- Stale-term, link, and example-path sweeps.
|
||||
- Documentation-related stale test wording corrections.
|
||||
- Minimal fixes only; broad rewrites intentionally deferred.
|
||||
237
docs/roadmap/documentation.md
Normal file
237
docs/roadmap/documentation.md
Normal file
@@ -0,0 +1,237 @@
|
||||
# Roadmap: 1.0 Documentation Pass
|
||||
|
||||
Status: Completed (2026-05-23)
|
||||
|
||||
## Goal
|
||||
|
||||
Prepare Narratio documentation for a 1.0 release by reviewing and rewriting
|
||||
every non-policy document under `docs/` against the implemented codebase.
|
||||
|
||||
The finished documentation set should be accurate, concise, complete for its
|
||||
audience, and compliant with:
|
||||
|
||||
- `docs/policy/documentation.md`
|
||||
- `docs/policy/architecture.md`
|
||||
- `docs/policy/development.md`
|
||||
|
||||
Do not modify files under `docs/policy/` during this pass.
|
||||
|
||||
Current behavior belongs in canonical docs. Future, planned, aspirational, or
|
||||
unimplemented behavior belongs only under `docs/roadmap/`.
|
||||
|
||||
## Scope
|
||||
|
||||
In scope:
|
||||
|
||||
- `docs/*.md`
|
||||
- `docs/internal/*.md`
|
||||
- `docs/integrations/*.md`
|
||||
- `docs/roadmap/*.md`
|
||||
- documentation references to files under `examples/`
|
||||
- test expectation fixes when the documentation review exposes stale or
|
||||
incorrect doc/example/path expectations
|
||||
|
||||
Out of scope:
|
||||
|
||||
- product/runtime code changes
|
||||
- feature implementation
|
||||
- edits under `docs/policy/`
|
||||
- adding roadmap behavior to current-behavior docs before that behavior is
|
||||
implemented
|
||||
|
||||
## Implementation Stages
|
||||
|
||||
### Stage 1: Inventory and Source-of-Truth Audit
|
||||
|
||||
Status: Completed (2026-05-23)
|
||||
|
||||
Create a file-by-file inventory of all non-policy docs before rewriting.
|
||||
|
||||
Implementation requirements:
|
||||
|
||||
- List every non-policy documentation file and assign its intended audience.
|
||||
- Identify each document's canonical scope using `docs/policy/documentation.md`.
|
||||
- Compare docs against the current code and tests, especially:
|
||||
- `internal/app`
|
||||
- `internal/config`
|
||||
- `internal/stage`
|
||||
- `internal/artifacts`
|
||||
- `examples`
|
||||
- relevant tests under `internal/**`
|
||||
- Record stale terminology, broken links, stale example paths, duplicate
|
||||
content, and roadmap-only behavior that leaked into current-behavior docs.
|
||||
- Record stale test expectations related to docs, examples, paths, or command
|
||||
text.
|
||||
- Do not rewrite content in this stage except obvious broken links or test
|
||||
corrections needed to make documentation validation meaningful.
|
||||
|
||||
Acceptance criteria:
|
||||
|
||||
- The rewrite has a concrete file inventory and source-of-truth map.
|
||||
- The team knows which docs are canonical and which should link elsewhere.
|
||||
- Known stale terms and broken references are identified before broad edits.
|
||||
|
||||
### Stage 2: User and Operator Docs
|
||||
|
||||
Status: Completed (2026-05-23)
|
||||
|
||||
Rewrite the user-facing and operator-facing docs first.
|
||||
|
||||
Implementation requirements:
|
||||
|
||||
- Rewrite these docs as fresh, concise current-behavior references:
|
||||
- `README.md`, if present
|
||||
- `docs/cli.md`
|
||||
- `docs/config.md`
|
||||
- `docs/operations.md`
|
||||
- `docs/troubleshooting.md`
|
||||
- Verify every command, flag, config field, discovery rule, path, and workflow
|
||||
against implemented behavior.
|
||||
- Cover implemented 1.0 behavior, including:
|
||||
- campaign registry selection;
|
||||
- concrete session loading and template-driven `session init`;
|
||||
- session-oriented helper commands;
|
||||
- clean, restore, analyze, and publish workflows;
|
||||
- artifact selection behavior;
|
||||
- locks and published output behavior;
|
||||
- workspace, spool, and cache behavior;
|
||||
- secrets loading and S3-backed operation.
|
||||
- Keep examples short and link to maintained files under `examples/` instead
|
||||
of duplicating large config blocks.
|
||||
- Fix tests only when they assert stale doc paths, example paths, command
|
||||
names, or current-behavior text.
|
||||
|
||||
Acceptance criteria:
|
||||
|
||||
- User/operator docs are task-oriented and match actual CLI/config behavior.
|
||||
- Current-behavior docs do not depend on roadmaps for normal usage.
|
||||
- No current-behavior doc describes unimplemented roadmap items.
|
||||
|
||||
### Stage 3: Internal Developer Docs
|
||||
|
||||
Status: Completed (2026-05-23)
|
||||
|
||||
Rewrite implemented internal component docs after public docs stabilize.
|
||||
|
||||
Implementation requirements:
|
||||
|
||||
- Rewrite:
|
||||
- `docs/internal/README.md`
|
||||
- `docs/internal/adapters.md`
|
||||
- `docs/internal/artifacts.md`
|
||||
- `docs/internal/command-restore.md`
|
||||
- `docs/internal/manifest.md`
|
||||
- `docs/internal/stage-*.md`
|
||||
- `docs/internal/storage.md`
|
||||
- `docs/internal/workspace.md`
|
||||
- Verify stage docs against current stage names, stage ordering, manifest
|
||||
records, declared inputs/outputs, adapters, path helpers, storage behavior,
|
||||
publish/current-state behavior, restore behavior, cache behavior, and
|
||||
workspace cleanup.
|
||||
- Keep implementation details in `docs/internal/`, not in user-facing docs.
|
||||
- Avoid turning internal docs into duplicate config or CLI references; link to
|
||||
canonical docs when needed.
|
||||
|
||||
Acceptance criteria:
|
||||
|
||||
- Internal docs are accurate enough for developers and LLM coding agents to
|
||||
change the system safely.
|
||||
- Stage and adapter boundaries match `docs/policy/architecture.md`.
|
||||
- Manifest, path, storage, and publish invariants are explicit and current.
|
||||
|
||||
### Stage 4: Integrations and Examples
|
||||
|
||||
Status: Completed (2026-05-23)
|
||||
|
||||
Review integration docs and maintained examples after core docs are rewritten.
|
||||
|
||||
Implementation requirements:
|
||||
|
||||
- Rewrite:
|
||||
- `docs/integrations/README.md`
|
||||
- `docs/integrations/audita.md`
|
||||
- `docs/integrations/scriptorium.md`
|
||||
- `docs/integrations/seriatim.md`
|
||||
- Verify integration docs against current adapter contracts and expected
|
||||
downstream tool behavior.
|
||||
- Confirm every referenced example file exists.
|
||||
- Confirm examples match current schema and command usage.
|
||||
- Run or rely on example validation tests.
|
||||
- Fix tests when they reference moved, renamed, or intentionally retired
|
||||
examples.
|
||||
|
||||
Acceptance criteria:
|
||||
|
||||
- Integration docs describe only implemented adapter expectations.
|
||||
- Maintained examples are valid, secret-free, and linked from canonical docs.
|
||||
- Example validation tests reflect the documented example set.
|
||||
|
||||
### Stage 5: Roadmap Cleanup and Final Sweep
|
||||
|
||||
Status: Completed (2026-05-23)
|
||||
|
||||
Clean up roadmap state and run final documentation validation.
|
||||
|
||||
Implementation requirements:
|
||||
|
||||
- Review `docs/roadmap/**` for implemented items that should be marked
|
||||
implemented, retired, or left planned.
|
||||
- Keep historical and planned behavior in roadmaps only.
|
||||
- Run final link/path/term sweeps.
|
||||
- Run validation commands:
|
||||
- `go test ./internal/config -run TestExamplesLoadAndValidate -v`
|
||||
- `go test ./internal/app -run TestExecute -v`
|
||||
- `go test ./...`
|
||||
|
||||
Acceptance criteria:
|
||||
|
||||
- All non-policy docs are current for the 1.0 release.
|
||||
- Roadmaps do not serve as required user/operator documentation.
|
||||
- Tests pass after allowed documentation-related test expectation fixes.
|
||||
|
||||
## Required Checks
|
||||
|
||||
Run searches for stale terminology and references during the pass.
|
||||
|
||||
Stale terminology:
|
||||
|
||||
- `archive`
|
||||
- `promote`
|
||||
- `promoted`
|
||||
- `promote_artifacts`
|
||||
- legacy campaign path/discovery language
|
||||
- old transcript names and paths
|
||||
- removed CLI commands or aliases
|
||||
|
||||
Broken or stale references:
|
||||
|
||||
- missing local doc links;
|
||||
- stale `examples/` paths;
|
||||
- stale internal doc filenames;
|
||||
- references to `docs/policy/**` as editable targets;
|
||||
- command examples that no longer match the CLI.
|
||||
|
||||
Policy checks:
|
||||
|
||||
- Current-behavior docs mention only implemented behavior.
|
||||
- Planned behavior appears only under `docs/roadmap/`.
|
||||
- Docs do not expose raw secrets or recommend storing secrets in config.
|
||||
- Docs use canonical homes:
|
||||
- `docs/config.md` for config schema;
|
||||
- `docs/cli.md` for command syntax;
|
||||
- `docs/operations.md` for operator workflows;
|
||||
- `docs/troubleshooting.md` for failure diagnosis;
|
||||
- `docs/internal/` for implementation contracts;
|
||||
- `docs/integrations/` for downstream tool integration notes;
|
||||
- `docs/roadmap/` for future work.
|
||||
|
||||
## Assumptions
|
||||
|
||||
- `docs/policy/**` is read-only for this documentation pass.
|
||||
- This pass is for 1.0 release readiness, not feature implementation.
|
||||
- Product and runtime code changes are out of scope.
|
||||
- Test fixes are in scope when they correct stale documentation, example, path,
|
||||
command, or current-behavior expectations uncovered during the review.
|
||||
- Roadmap files may remain as planning and historical records.
|
||||
- Current-behavior docs must be sufficient for normal use without requiring
|
||||
readers to consult roadmaps.
|
||||
273
docs/troubleshooting.md
Normal file
273
docs/troubleshooting.md
Normal file
@@ -0,0 +1,273 @@
|
||||
# Troubleshooting
|
||||
|
||||
Operational diagnosis guide for common Narratio failures.
|
||||
|
||||
## Config file not found
|
||||
|
||||
Symptom:
|
||||
|
||||
- command fails to resolve `pipeline.yml`, `campaign.yml`, or `session.yml`.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- missing files in default search paths;
|
||||
- wrong campaign selection;
|
||||
- omitted explicit flags.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio session plan 2026-04-04
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- pass explicit `--config`, `--campaign` or `--campaign-file`, and `--session`.
|
||||
|
||||
## Session template placeholders rejected
|
||||
|
||||
Symptom:
|
||||
|
||||
- load error says session file must be concrete or contains `{{ ... }}` placeholders.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- using template content as runtime session config.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio session validate 2026-04-04 --session /path/session.yml
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- generate concrete session YAML with `narratio session init`.
|
||||
|
||||
## Strict decode or schema validation failure
|
||||
|
||||
Symptom:
|
||||
|
||||
- unknown field / invalid value error during config load.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- stale field name, typo, invalid enum, or invalid duration/path format.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio session plan 2026-04-04 --config /path/pipeline.yml --campaign-file /path/campaign.yml --session /path/session.yml
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- align config with [docs/config.md](./config.md) and maintained files under `examples/`.
|
||||
|
||||
## Audio mode conflict
|
||||
|
||||
Symptom:
|
||||
|
||||
- validation fails on session audio configuration.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- configured both local and S3 session audio inputs.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- use local mode (`audio_dir` or `audio_files`) or S3 mode (`audio_s3.prefix`), not both.
|
||||
|
||||
## `--artifacts` selection error
|
||||
|
||||
Symptom:
|
||||
|
||||
- unknown artifact key or invalid `--artifacts` usage.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- key not defined in `pipeline.scriptorium.artifacts`;
|
||||
- empty list entry (for example trailing comma);
|
||||
- `run-stage` used with non-`analyze`/`publish` target.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- provide only configured keys and use `--artifacts` with supported commands/stages.
|
||||
|
||||
## Previous-session artifact input missing
|
||||
|
||||
Symptom:
|
||||
|
||||
- prepare/analyze fails due to missing required previous-session artifact cache input.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- missing `session.previous_session_id`;
|
||||
- previous artifact not restored/published for source session.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio session validate 2026-04-04
|
||||
narratio session status 2026-04-04
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
```bash
|
||||
narratio session restore 2026-04-04
|
||||
```
|
||||
|
||||
or rerun prepare after correcting session config:
|
||||
|
||||
```bash
|
||||
narratio run-stage prepare 2026-04-04 --force
|
||||
```
|
||||
|
||||
## Session lock conflict (`.lock`)
|
||||
|
||||
Symptom:
|
||||
|
||||
- command fails acquiring session lock.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- another process is running for the same session;
|
||||
- stale lock left by interrupted process.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
ls -l {workspace.root}/work/{campaign}/{session_id}/.lock
|
||||
ps aux | grep narratio
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- wait for active process completion;
|
||||
- remove stale lock only after confirming no live process owns it.
|
||||
|
||||
## Restore conflict without `--force`
|
||||
|
||||
Symptom:
|
||||
|
||||
- restore fails with conflict count.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- local durable files differ from remote restore sources.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio session restore 2026-04-04 --dry-run
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- review conflicts;
|
||||
- rerun with `--force` only when remote state should overwrite local.
|
||||
|
||||
## Restore current-state discovery failure
|
||||
|
||||
Symptom:
|
||||
|
||||
- restore cannot find current pointer or current manifest.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- no committed publish current state;
|
||||
- storage credentials or connectivity failure.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio session status 2026-04-04
|
||||
narratio session restore 2026-04-04 --dry-run
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- resolve storage/auth issue;
|
||||
- republish from healthy local state if current pointer is missing.
|
||||
|
||||
## Publish output failure
|
||||
|
||||
Symptom:
|
||||
|
||||
- publish fails on missing required source, upload error, or commit write.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- required source file not produced;
|
||||
- lock/state expectations mismatch;
|
||||
- remote storage failure.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio session artifacts 2026-04-04 --remote
|
||||
narratio session status 2026-04-04
|
||||
narratio run-stage publish 2026-04-04 --force
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- regenerate missing sources by rerunning prerequisite stages;
|
||||
- correct publish source/destination rules;
|
||||
- retry after storage failure is resolved.
|
||||
|
||||
## Secrets or storage credential failure
|
||||
|
||||
Symptom:
|
||||
|
||||
- object-store command fails at initialization/auth.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- invalid `pipeline.secrets.env_dir`;
|
||||
- missing credential environment variables;
|
||||
- invalid S3 endpoint/bucket settings.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
ls -la /path/to/secrets_dir
|
||||
env | grep -E 'OBJECT_STORAGE|AWS|AUDITA|SCRIPTORIUM'
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- correct secret-file path and permissions;
|
||||
- provide required env vars;
|
||||
- keep secret values out of YAML.
|
||||
|
||||
## S3 audio prepare failure
|
||||
|
||||
Symptom:
|
||||
|
||||
- prepare fails listing/downloading session S3 audio.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- incorrect `session.inputs.audio_s3.prefix`;
|
||||
- no matching `.flac` objects;
|
||||
- storage connectivity or permissions failure.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio run-stage prepare 2026-04-04 --force
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- verify prefix contents and storage access;
|
||||
- keep session audio mode consistent.
|
||||
|
||||
## References
|
||||
|
||||
- [docs/cli.md](./cli.md)
|
||||
- [docs/config.md](./config.md)
|
||||
- [docs/operations.md](./operations.md)
|
||||
- [docs/internal/stage-publish.md](./internal/stage-publish.md)
|
||||
1
examples/campaigns/sample-campaign/autocorrect.yml
Normal file
1
examples/campaigns/sample-campaign/autocorrect.yml
Normal file
@@ -0,0 +1 @@
|
||||
[]
|
||||
@@ -1,10 +1,6 @@
|
||||
session_id: 2026-05-03
|
||||
campaign: sample-campaign
|
||||
date: 2026-05-03
|
||||
title: Sample Session
|
||||
campaign_id: sample-campaign
|
||||
session_template_file: ./session.template.yml
|
||||
inputs:
|
||||
audio_dir: ./audio
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
|
||||
1
examples/campaigns/sample-campaign/glossary.yml
Normal file
1
examples/campaigns/sample-campaign/glossary.yml
Normal file
@@ -0,0 +1 @@
|
||||
[]
|
||||
3
examples/campaigns/sample-campaign/session.template.yml
Normal file
3
examples/campaigns/sample-campaign/session.template.yml
Normal file
@@ -0,0 +1,3 @@
|
||||
session_id: "{{ session_id }}"
|
||||
inputs:
|
||||
audio_dir: ./audio
|
||||
5
examples/campaigns/sample-campaign/speakers.yml
Normal file
5
examples/campaigns/sample-campaign/speakers.yml
Normal file
@@ -0,0 +1,5 @@
|
||||
match:
|
||||
- speaker: "Eric Rakestraw"
|
||||
match:
|
||||
- "Eric_Rakestraw"
|
||||
- "Eric"
|
||||
175
examples/pipeline.full.annotated.yml
Normal file
175
examples/pipeline.full.annotated.yml
Normal file
@@ -0,0 +1,175 @@
|
||||
# Full annotated pipeline example for implemented Narratio config fields.
|
||||
# Values are safe placeholders and must be adapted per environment.
|
||||
|
||||
workspace:
|
||||
# Optional: defaults to /var/lib/narratio.
|
||||
root: /var/lib/narratio/workspace
|
||||
# Optional: remove run-scoped workdir after successful publish commit.
|
||||
cleanup_after_publish: false
|
||||
|
||||
# Optional: local secret file loader (directory of ENV_VAR_NAME files).
|
||||
# secrets:
|
||||
# env_dir: ./secrets
|
||||
|
||||
storage:
|
||||
# Optional storage backend selector; use "s3" for publish + S3 audio workflows.
|
||||
backend: s3
|
||||
s3:
|
||||
# Required when using S3 audio or S3 publish uploads.
|
||||
bucket: my-dnd-archive
|
||||
# Optional; defaults to "dnd".
|
||||
root_prefix: dnd
|
||||
# Optional region/endpoint settings.
|
||||
region: us-east-1
|
||||
endpoint: ""
|
||||
force_path_style: false
|
||||
# Optional; defaults shown explicitly.
|
||||
access_key_id_env: OBJECT_STORAGE_KEY_ID
|
||||
secret_access_key_env: OBJECT_STORAGE_KEY
|
||||
|
||||
campaigns:
|
||||
# Optional; defaults to /usr/local/share/narratio/campaigns.
|
||||
root: /usr/local/share/narratio/campaigns
|
||||
# Optional command default when --campaign is omitted.
|
||||
default_campaign_id: sample-campaign
|
||||
|
||||
spool:
|
||||
# Optional; defaults to /var/spool/narratio.
|
||||
root: /var/spool/narratio
|
||||
# Optional cleanup of run-scoped spool audio after successful publish commit.
|
||||
delete_audio_after_publish: false
|
||||
|
||||
publish:
|
||||
# Optional booleans; defaults are true.
|
||||
enabled: true
|
||||
upload_run: true
|
||||
# Optional publish output rules; sources use Narratio artifact source IDs.
|
||||
outputs:
|
||||
- source: narratio.transcript.final_trimmed
|
||||
dest: transcripts/final.trimmed.json
|
||||
required: true
|
||||
- source: narratio.artifact.session_recap
|
||||
dest: artifacts/session_recap.md
|
||||
required: true
|
||||
- source: narratio.artifact.player_handout
|
||||
dest: artifacts/player_handout.md
|
||||
required: false
|
||||
|
||||
whisperx:
|
||||
# Required.
|
||||
transcribe_url: "https://transcription.example.com/transcribe"
|
||||
# Optional overrides; defaults shown explicitly.
|
||||
language: en
|
||||
timeout: 30m
|
||||
retries: 3
|
||||
retry_delay: 2s
|
||||
concurrency: 2
|
||||
|
||||
seriatim:
|
||||
# Optional overrides; defaults shown explicitly.
|
||||
binary: seriatim
|
||||
timeout: 10m
|
||||
output_schema: seriatim-intermediate
|
||||
coalesce_gap: 3.0
|
||||
report: true
|
||||
env:
|
||||
# Optional advanced tuning; set only when needed.
|
||||
overlap_word_run_gap: 1.0
|
||||
overlap_word_run_reorder_window: 1.0
|
||||
backchannel_max_duration: 2.0
|
||||
filler_max_duration: 1.25
|
||||
|
||||
audita:
|
||||
# Optional overrides; defaults shown explicitly where applicable.
|
||||
binary: audita
|
||||
timeout: 3h
|
||||
llm_api_key_env: AUDITA_LLM_API_KEY
|
||||
modules: [glossary, homophones, spoken_word, grammar]
|
||||
base_url: ""
|
||||
model: ""
|
||||
total_llm_concurrency: 2
|
||||
proposal_llm_concurrency: 1
|
||||
validation_model: ""
|
||||
validation_llm_concurrency: 1
|
||||
transcript_description: ""
|
||||
config_path: /usr/local/etc/audita/config.yml
|
||||
output_schema: audita-v1
|
||||
work_dir_retention: auto
|
||||
report: true
|
||||
|
||||
normalize:
|
||||
# Optional; defaults shown explicitly.
|
||||
output_path: transcripts/final.json
|
||||
output_schema: seriatim-intermediate
|
||||
report: true
|
||||
|
||||
trim:
|
||||
# Keep disabled unless bounds prompt integration is configured.
|
||||
enabled: false
|
||||
output_path: transcripts/final.trimmed.json
|
||||
bounds:
|
||||
prompt_id: dnd.session_bounds
|
||||
profile_id: local-fast
|
||||
transcript_input_name: transcript
|
||||
output_path: reports/session_bounds.json
|
||||
timeout: 10m
|
||||
render_debug: false
|
||||
render_output_path: reports/session_bounds.render.json
|
||||
seriatim:
|
||||
report: false
|
||||
|
||||
scriptorium:
|
||||
binary: scriptorium
|
||||
config_path: /usr/local/etc/scriptorium/config.yml
|
||||
timeout: 10m
|
||||
render_debug: false
|
||||
artifacts:
|
||||
# Configured artifact keys map to source IDs narratio.artifact.<key>.
|
||||
session_recap:
|
||||
enabled: true
|
||||
prompt_id: dnd.session_recap
|
||||
profile_id: local-fast
|
||||
output_path: artifacts/session_recap.md
|
||||
timeout: 10m
|
||||
inputs:
|
||||
transcript:
|
||||
source: narratio.transcript.final_trimmed
|
||||
required: true
|
||||
previous_recap:
|
||||
source: narratio.previous_session.artifact.session_recap
|
||||
required: false
|
||||
vars:
|
||||
session_id: true
|
||||
session_date: true
|
||||
campaign_name: true
|
||||
previous_session_id: true
|
||||
output_kind: session_recap
|
||||
|
||||
# Example dependent artifact:
|
||||
# - depends_on entries use artifact keys.
|
||||
# - narratio.artifact.<key> sources require matching depends_on membership.
|
||||
player_handout:
|
||||
enabled: true
|
||||
depends_on:
|
||||
- session_recap
|
||||
prompt_id: dnd.player_handout
|
||||
profile_id: local-fast
|
||||
output_path: artifacts/player_handout.md
|
||||
timeout: 10m
|
||||
inputs:
|
||||
recap:
|
||||
source: narratio.artifact.session_recap
|
||||
required: true
|
||||
transcript:
|
||||
source: narratio.transcript.final_trimmed
|
||||
required: true
|
||||
vars:
|
||||
session_id: true
|
||||
campaign_name: true
|
||||
output_kind: player_handout
|
||||
|
||||
notification:
|
||||
# Optional notification settings.
|
||||
backend: ""
|
||||
recipient: ""
|
||||
timeout: 30s
|
||||
@@ -1,109 +1,6 @@
|
||||
workspace:
|
||||
root: ./tmp/narratio-workspace
|
||||
|
||||
storage:
|
||||
backend: local
|
||||
campaigns:
|
||||
root: /usr/local/share/narratio/campaigns
|
||||
default_campaign_id: sample-campaign
|
||||
|
||||
whisperx:
|
||||
transcribe_url: "https://transcription.example.com/transcribe"
|
||||
language: "en"
|
||||
timeout: "30m"
|
||||
retries: 3
|
||||
retry_delay: "2s"
|
||||
concurrency: 2
|
||||
|
||||
seriatim:
|
||||
binary: "seriatim"
|
||||
timeout: "10m"
|
||||
output_schema: "seriatim-intermediate"
|
||||
coalesce_gap: 3.0
|
||||
report: true
|
||||
env:
|
||||
overlap_word_run_gap: 1.0
|
||||
overlap_word_run_reorder_window: 1.0
|
||||
backchannel_max_duration: 2.0
|
||||
filler_max_duration: 1.25
|
||||
|
||||
audita:
|
||||
binary: "audita"
|
||||
timeout: "3h"
|
||||
llm_api_key_env: "AUDITA_LLM_API_KEY"
|
||||
modules:
|
||||
- glossary
|
||||
- homophones
|
||||
- glossary
|
||||
- spoken_word
|
||||
- grammar
|
||||
- homophones
|
||||
- glossary
|
||||
base_url: "https://openrouter.ai/api/v1"
|
||||
model: "openrouter/google/gemma-4-31b-it"
|
||||
llm_concurrency: 1
|
||||
validation_model: ""
|
||||
validation_llm_concurrency: 1
|
||||
report: true
|
||||
|
||||
normalize:
|
||||
# Session-workdir-relative when not absolute.
|
||||
output_path: "transcripts/normalized.json"
|
||||
output_schema: "seriatim-intermediate"
|
||||
report: true
|
||||
|
||||
trim:
|
||||
enabled: true
|
||||
# Session-workdir-relative when not absolute.
|
||||
output_path: "transcripts/trimmed.json"
|
||||
bounds:
|
||||
prompt_id: "dnd_session.bounds"
|
||||
# Empty means use prompt default profile.
|
||||
profile_id: ""
|
||||
transcript_input_name: "transcript"
|
||||
output_path: "artifacts/session_bounds.json"
|
||||
timeout: "10m"
|
||||
render_debug: false
|
||||
render_output_path: "artifacts/session_bounds.render.json"
|
||||
seriatim:
|
||||
report: false
|
||||
|
||||
scriptorium:
|
||||
binary: "scriptorium"
|
||||
config_path: "/etc/scriptorium/config.yml"
|
||||
timeout: "10m"
|
||||
render_debug: false
|
||||
artifacts:
|
||||
session_recap:
|
||||
enabled: true
|
||||
prompt_id: "dnd.session_recap"
|
||||
profile_id: "local-quality"
|
||||
output_path: "artifacts/session_recap.md"
|
||||
timeout: "10m"
|
||||
# Optional per-artifact override of global scriptorium.render_debug.
|
||||
# render_debug: true
|
||||
inputs:
|
||||
transcript:
|
||||
# Available transcript sources:
|
||||
# - trimmed_transcript (recommended for session_recap)
|
||||
# - normalized_transcript (recommended for future full-session analysis)
|
||||
# - processed_transcript (raw Audita-polished output)
|
||||
source: "trimmed_transcript"
|
||||
required: true
|
||||
previous_recap:
|
||||
source: "previous_session_artifact"
|
||||
artifact: "session_recap"
|
||||
# Optional: set when previous recap is available.
|
||||
path: ""
|
||||
required: false
|
||||
vars:
|
||||
session_id: true
|
||||
session_date: true
|
||||
campaign_name: true
|
||||
previous_session_id: true
|
||||
output_kind: "session_recap"
|
||||
|
||||
analyzer:
|
||||
timeout: 20m
|
||||
artifacts:
|
||||
output_dir: artifacts
|
||||
|
||||
notification:
|
||||
timeout: 10s
|
||||
|
||||
116
examples/pipeline.production.yml
Normal file
116
examples/pipeline.production.yml
Normal file
@@ -0,0 +1,116 @@
|
||||
workspace:
|
||||
root: /var/lib/narratio/workspace
|
||||
cleanup_after_publish: true
|
||||
|
||||
storage:
|
||||
backend: s3
|
||||
s3:
|
||||
bucket: my-dnd-archive
|
||||
root_prefix: dnd
|
||||
region: us-east-1
|
||||
access_key_id_env: OBJECT_STORAGE_KEY_ID
|
||||
secret_access_key_env: OBJECT_STORAGE_KEY
|
||||
|
||||
campaigns:
|
||||
root: /usr/local/share/narratio/campaigns
|
||||
default_campaign_id: sample-campaign
|
||||
|
||||
spool:
|
||||
root: /var/spool/narratio
|
||||
delete_audio_after_publish: true
|
||||
|
||||
publish:
|
||||
enabled: true
|
||||
upload_run: true
|
||||
outputs:
|
||||
- source: narratio.transcript.final_trimmed
|
||||
dest: transcripts/final.trimmed.json
|
||||
required: true
|
||||
- source: narratio.artifact.session_recap
|
||||
dest: artifacts/session_recap.md
|
||||
required: true
|
||||
- source: narratio.artifact.player_handout
|
||||
dest: artifacts/player_handout.md
|
||||
required: false
|
||||
|
||||
whisperx:
|
||||
transcribe_url: "https://transcription.example.com/transcribe"
|
||||
language: en
|
||||
timeout: 45m
|
||||
retries: 3
|
||||
retry_delay: 3s
|
||||
concurrency: 2
|
||||
|
||||
seriatim:
|
||||
binary: seriatim
|
||||
timeout: 10m
|
||||
output_schema: seriatim-intermediate
|
||||
coalesce_gap: 3.0
|
||||
report: true
|
||||
|
||||
audita:
|
||||
binary: audita
|
||||
timeout: 3h
|
||||
llm_api_key_env: AUDITA_LLM_API_KEY
|
||||
modules: [glossary, homophones, spoken_word, grammar]
|
||||
output_schema: audita-v1
|
||||
work_dir_retention: auto
|
||||
total_llm_concurrency: 2
|
||||
proposal_llm_concurrency: 1
|
||||
validation_llm_concurrency: 1
|
||||
report: true
|
||||
|
||||
normalize:
|
||||
output_path: transcripts/final.json
|
||||
output_schema: seriatim-intermediate
|
||||
report: true
|
||||
|
||||
trim:
|
||||
enabled: false
|
||||
|
||||
scriptorium:
|
||||
binary: scriptorium
|
||||
config_path: /usr/local/etc/scriptorium/config.yml
|
||||
timeout: 10m
|
||||
render_debug: false
|
||||
artifacts:
|
||||
session_recap:
|
||||
enabled: true
|
||||
prompt_id: dnd.session_recap
|
||||
profile_id: local-fast
|
||||
output_path: artifacts/session_recap.md
|
||||
timeout: 10m
|
||||
inputs:
|
||||
transcript:
|
||||
source: narratio.transcript.final_trimmed
|
||||
required: true
|
||||
previous_recap:
|
||||
source: narratio.previous_session.artifact.session_recap
|
||||
required: false
|
||||
vars:
|
||||
session_id: true
|
||||
session_date: true
|
||||
campaign_name: true
|
||||
previous_session_id: true
|
||||
output_kind: session_recap
|
||||
player_handout:
|
||||
enabled: true
|
||||
depends_on:
|
||||
- session_recap
|
||||
prompt_id: dnd.player_handout
|
||||
profile_id: local-fast
|
||||
output_path: artifacts/player_handout.md
|
||||
timeout: 10m
|
||||
inputs:
|
||||
recap:
|
||||
source: narratio.artifact.session_recap
|
||||
required: true
|
||||
transcript:
|
||||
source: narratio.transcript.final_trimmed
|
||||
required: true
|
||||
vars:
|
||||
session_id: true
|
||||
output_kind: player_handout
|
||||
|
||||
notification:
|
||||
timeout: 30s
|
||||
5
examples/session.local-audio.yml
Normal file
5
examples/session.local-audio.yml
Normal file
@@ -0,0 +1,5 @@
|
||||
session_id: 2026-05-03
|
||||
date: 2026-05-03
|
||||
title: Sample Session
|
||||
inputs:
|
||||
audio_dir: ./audio
|
||||
6
examples/session.s3-audio.yml
Normal file
6
examples/session.s3-audio.yml
Normal file
@@ -0,0 +1,6 @@
|
||||
session_id: 2026-05-03
|
||||
date: 2026-05-03
|
||||
title: Sample Session
|
||||
inputs:
|
||||
audio_s3:
|
||||
prefix: audio/
|
||||
3
examples/session.template.yml
Normal file
3
examples/session.template.yml
Normal file
@@ -0,0 +1,3 @@
|
||||
session_id: "{{ session_id }}"
|
||||
inputs:
|
||||
audio_dir: ./audio
|
||||
25
go.mod
25
go.mod
@@ -2,4 +2,27 @@ module gitea.maximumdirect.net/eric/narratio
|
||||
|
||||
go 1.25.0
|
||||
|
||||
require gopkg.in/yaml.v3 v3.0.1
|
||||
require (
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.17
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.16
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
|
||||
github.com/aws/smithy-go v1.25.1
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect
|
||||
)
|
||||
|
||||
36
go.sum
36
go.sum
@@ -1,3 +1,39 @@
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.17 h1:FpL4/758/diKwqbytU0prpuiu60fgXKUWCpDJtApclU=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.17/go.mod h1:OXqUMzgXytfoF9JaKkhrOYsyh72t9G+MJH8mMRaexOE=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 h1:UuSfcORqNSz/ey3VPRS8TcVH2Ikf0/sC+Hdj400QI6U=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23/go.mod h1:+G/OSGiOFnSOkYloKj/9M35s74LgVAdJBSD5lsFfqKg=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 h1:etqBTKY581iwLL/H/S2sVgk3C9lAsTJFeXWFDsDcWOU=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 h1:TdJ+HdzOBhU8+iVAOGUTU63VXopcumCOF1paFulHWZc=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11/go.mod h1:R82ZRExE/nheo0N+T8zHPcLRTcH8MGsnR3BiVGX0TwI=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 h1:7byT8HUWrgoRp6sXjxtZwgOKfhss5fW6SkLBtqzgRoE=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17/go.mod h1:xNWknVi4Ezm1vg1QsB/5EWpAJURq22uqd38U8qKvOJc=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 h1:+1Kl1zx6bWi4X7cKi3VYh29h8BvsCoHQEQ6ST9X8w7w=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21/go.mod h1:4vIRDq+CJB2xFAXZ+YgGUTiEft7oAQlhIs71xcSeuVg=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOItExNM9L1euNuh/fk=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio=
|
||||
github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
|
||||
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
package analyzer
|
||||
|
||||
import "context"
|
||||
|
||||
// NoopRunner is a deterministic no-op analyzer adapter.
|
||||
type NoopRunner struct{}
|
||||
|
||||
// Run returns the requested output path with placeholder metadata.
|
||||
func (n *NoopRunner) Run(ctx context.Context, req AnalyzeRequest) (AnalyzeResult, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return AnalyzeResult{}, err
|
||||
}
|
||||
return AnalyzeResult{ArtifactPath: req.OutputPath, Metadata: map[string]any{"placeholder": true}}, nil
|
||||
}
|
||||
|
||||
// FakeRunner captures analyze requests and returns deterministic responses.
|
||||
type FakeRunner struct {
|
||||
Requests []AnalyzeRequest
|
||||
Err error
|
||||
Result AnalyzeResult
|
||||
}
|
||||
|
||||
// Run records request and returns configured response.
|
||||
func (f *FakeRunner) Run(ctx context.Context, req AnalyzeRequest) (AnalyzeResult, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return AnalyzeResult{}, err
|
||||
}
|
||||
f.Requests = append(f.Requests, req)
|
||||
if f.Err != nil {
|
||||
return AnalyzeResult{}, f.Err
|
||||
}
|
||||
res := f.Result
|
||||
if res.ArtifactPath == "" {
|
||||
res.ArtifactPath = req.OutputPath
|
||||
}
|
||||
if res.Metadata == nil {
|
||||
res.Metadata = map[string]any{"fake": true}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
package analyzer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFakeRunnerCapturesRequestAndReturnsPath(t *testing.T) {
|
||||
fake := &FakeRunner{}
|
||||
req := AnalyzeRequest{ArtifactType: "session-log", OutputPath: "artifacts/session-log.md"}
|
||||
|
||||
res, err := fake.Run(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if len(fake.Requests) != 1 || fake.Requests[0].ArtifactType != "session-log" {
|
||||
t.Fatalf("requests = %#v, want captured request", fake.Requests)
|
||||
}
|
||||
if res.ArtifactPath != req.OutputPath {
|
||||
t.Fatalf("artifact path = %q, want %q", res.ArtifactPath, req.OutputPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFakeRunnerError(t *testing.T) {
|
||||
fake := &FakeRunner{Err: errors.New("boom")}
|
||||
_, err := fake.Run(context.Background(), AnalyzeRequest{})
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
// Package analyzer declares the adapter contract for artifact analysis generation.
|
||||
package analyzer
|
||||
|
||||
import "context"
|
||||
|
||||
// TODO: implement analyzer integration once the analyzer contract is finalized.
|
||||
|
||||
// Runner is the adapter boundary for analyzer invocations.
|
||||
type Runner interface {
|
||||
Run(ctx context.Context, req AnalyzeRequest) (AnalyzeResult, error)
|
||||
}
|
||||
|
||||
// AnalyzeRequest describes one analyzer artifact generation request.
|
||||
type AnalyzeRequest struct {
|
||||
ArtifactType string
|
||||
ProcessedTranscriptPath string
|
||||
ContextReferences []string
|
||||
OutputPath string
|
||||
GeneratedConfigPath string
|
||||
StdoutLogPath string
|
||||
StderrLogPath string
|
||||
}
|
||||
|
||||
// AnalyzeResult describes analyzer output.
|
||||
type AnalyzeResult struct {
|
||||
ArtifactPath string
|
||||
Metadata map[string]any
|
||||
}
|
||||
@@ -14,7 +14,7 @@ func TestFakeRunnerCapturesRequestAndReturnsPath(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
req := PolishRequest{
|
||||
GeneratedConfigPath: filepath.Join(dir, "config", "audita.yml"),
|
||||
OutputProcessedPath: filepath.Join(dir, "transcripts", "processed.json"),
|
||||
OutputProcessedPath: filepath.Join(dir, "transcripts", "polished.json"),
|
||||
StdoutLogPath: filepath.Join(dir, "logs", "audita.stdout.log"),
|
||||
StderrLogPath: filepath.Join(dir, "logs", "audita.stderr.log"),
|
||||
}
|
||||
|
||||
@@ -24,6 +24,12 @@ type PolishRequest struct {
|
||||
Modules []string
|
||||
BaseURL string
|
||||
Model string
|
||||
TranscriptDescription string
|
||||
ConfigPath string
|
||||
OutputSchema string
|
||||
WorkDirRetention string
|
||||
TotalLLMConcurrency *int
|
||||
ProposalLLMConcurrency *int
|
||||
ValidationModel string
|
||||
ValidationLLMConcurrency *int
|
||||
StdoutLogPath string
|
||||
|
||||
@@ -21,7 +21,12 @@ type SubprocessRunnerConfig struct {
|
||||
Modules []string
|
||||
BaseURL string
|
||||
Model string
|
||||
LLMConcurrency *int
|
||||
TranscriptDescription string
|
||||
ConfigPath string
|
||||
OutputSchema string
|
||||
WorkDirRetention string
|
||||
TotalLLMConcurrency *int
|
||||
ProposalLLMConcurrency *int
|
||||
ValidationModel string
|
||||
ValidationLLMConcurrency *int
|
||||
Report bool
|
||||
@@ -35,7 +40,12 @@ type SubprocessRunner struct {
|
||||
modules []string
|
||||
baseURL string
|
||||
model string
|
||||
llmConcurrency *int
|
||||
transcriptDescription string
|
||||
configPath string
|
||||
outputSchema string
|
||||
workDirRetention string
|
||||
totalLLMConcurrency *int
|
||||
proposalLLMConcurrency *int
|
||||
validationModel string
|
||||
validationLLMConcurrency *int
|
||||
report bool
|
||||
@@ -49,7 +59,12 @@ func NewSubprocessRunnerFromConfigValues(
|
||||
modules []string,
|
||||
baseURL string,
|
||||
model string,
|
||||
llmConcurrency *int,
|
||||
transcriptDescription string,
|
||||
configPath string,
|
||||
outputSchema string,
|
||||
workDirRetention string,
|
||||
totalLLMConcurrency *int,
|
||||
proposalLLMConcurrency *int,
|
||||
validationModel string,
|
||||
validationLLMConcurrency *int,
|
||||
report bool,
|
||||
@@ -68,7 +83,12 @@ func NewSubprocessRunnerFromConfigValues(
|
||||
Modules: modules,
|
||||
BaseURL: baseURL,
|
||||
Model: model,
|
||||
LLMConcurrency: llmConcurrency,
|
||||
TranscriptDescription: transcriptDescription,
|
||||
ConfigPath: configPath,
|
||||
OutputSchema: outputSchema,
|
||||
WorkDirRetention: workDirRetention,
|
||||
TotalLLMConcurrency: totalLLMConcurrency,
|
||||
ProposalLLMConcurrency: proposalLLMConcurrency,
|
||||
ValidationModel: validationModel,
|
||||
ValidationLLMConcurrency: validationLLMConcurrency,
|
||||
Report: report,
|
||||
@@ -83,33 +103,39 @@ func NewSubprocessRunner(cfg SubprocessRunnerConfig) (*SubprocessRunner, error)
|
||||
if cfg.Timeout <= 0 {
|
||||
return nil, fmt.Errorf("audita timeout must be > 0")
|
||||
}
|
||||
if len(cfg.Modules) == 0 {
|
||||
return nil, fmt.Errorf("audita modules must include at least one module")
|
||||
}
|
||||
for i, module := range cfg.Modules {
|
||||
if strings.TrimSpace(module) == "" {
|
||||
return nil, fmt.Errorf("audita module at index %d is empty", i)
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(cfg.BaseURL) == "" {
|
||||
return nil, fmt.Errorf("audita base url is required")
|
||||
}
|
||||
u, err := url.Parse(cfg.BaseURL)
|
||||
if err != nil || u.Scheme == "" || u.Host == "" {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("audita base url %q is invalid: %w", cfg.BaseURL, err)
|
||||
if strings.TrimSpace(cfg.BaseURL) != "" {
|
||||
u, err := url.Parse(cfg.BaseURL)
|
||||
if err != nil || u.Scheme == "" || u.Host == "" {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("audita base url %q is invalid: %w", cfg.BaseURL, err)
|
||||
}
|
||||
return nil, fmt.Errorf("audita base url %q is invalid", cfg.BaseURL)
|
||||
}
|
||||
return nil, fmt.Errorf("audita base url %q is invalid", cfg.BaseURL)
|
||||
}
|
||||
if strings.TrimSpace(cfg.Model) == "" {
|
||||
return nil, fmt.Errorf("audita model is required")
|
||||
if cfg.TotalLLMConcurrency != nil && *cfg.TotalLLMConcurrency <= 0 {
|
||||
return nil, fmt.Errorf("audita total llm concurrency must be > 0 when provided")
|
||||
}
|
||||
if cfg.LLMConcurrency != nil && *cfg.LLMConcurrency <= 0 {
|
||||
return nil, fmt.Errorf("audita llm concurrency must be > 0 when provided")
|
||||
if cfg.ProposalLLMConcurrency != nil && *cfg.ProposalLLMConcurrency <= 0 {
|
||||
return nil, fmt.Errorf("audita proposal llm concurrency must be > 0 when provided")
|
||||
}
|
||||
if cfg.ValidationLLMConcurrency != nil && *cfg.ValidationLLMConcurrency <= 0 {
|
||||
return nil, fmt.Errorf("audita validation llm concurrency must be > 0 when provided")
|
||||
}
|
||||
switch strings.TrimSpace(cfg.OutputSchema) {
|
||||
case "", "bare-segments", "audita-v1":
|
||||
default:
|
||||
return nil, fmt.Errorf("audita output schema must be one of: bare-segments, audita-v1")
|
||||
}
|
||||
switch strings.TrimSpace(cfg.WorkDirRetention) {
|
||||
case "", "always", "auto", "never":
|
||||
default:
|
||||
return nil, fmt.Errorf("audita work dir retention must be one of: always, auto, never")
|
||||
}
|
||||
|
||||
modules := make([]string, len(cfg.Modules))
|
||||
for i, m := range cfg.Modules {
|
||||
@@ -123,7 +149,12 @@ func NewSubprocessRunner(cfg SubprocessRunnerConfig) (*SubprocessRunner, error)
|
||||
modules: modules,
|
||||
baseURL: strings.TrimSpace(cfg.BaseURL),
|
||||
model: strings.TrimSpace(cfg.Model),
|
||||
llmConcurrency: cfg.LLMConcurrency,
|
||||
transcriptDescription: strings.TrimSpace(cfg.TranscriptDescription),
|
||||
configPath: strings.TrimSpace(cfg.ConfigPath),
|
||||
outputSchema: strings.TrimSpace(cfg.OutputSchema),
|
||||
workDirRetention: strings.TrimSpace(cfg.WorkDirRetention),
|
||||
totalLLMConcurrency: cfg.TotalLLMConcurrency,
|
||||
proposalLLMConcurrency: cfg.ProposalLLMConcurrency,
|
||||
validationModel: strings.TrimSpace(cfg.ValidationModel),
|
||||
validationLLMConcurrency: cfg.ValidationLLMConcurrency,
|
||||
report: cfg.Report,
|
||||
@@ -152,7 +183,7 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
|
||||
}
|
||||
|
||||
reqModules := req.Modules
|
||||
if len(reqModules) == 0 {
|
||||
if reqModules == nil {
|
||||
reqModules = append([]string(nil), r.modules...)
|
||||
}
|
||||
args := r.buildArgs(req, reqModules)
|
||||
@@ -168,14 +199,9 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
|
||||
env["AUDITA_LLM_API_KEY"] = credential
|
||||
credentialPresent = true
|
||||
}
|
||||
primaryConcurrencyViaEnv := false
|
||||
if r.llmConcurrency != nil {
|
||||
env["AUDITA_LLM_CONCURRENCY"] = strconv.Itoa(*r.llmConcurrency)
|
||||
primaryConcurrencyViaEnv = true
|
||||
}
|
||||
|
||||
if req.GeneratedConfigPath != "" {
|
||||
if err := r.writeInvocationConfig(req, args, reqModules, credentialPresent, primaryConcurrencyViaEnv); err != nil {
|
||||
if err := r.writeInvocationConfig(req, args, reqModules, credentialPresent); err != nil {
|
||||
return PolishResult{}, fmt.Errorf("write audita invocation config %q: %w", req.GeneratedConfigPath, err)
|
||||
}
|
||||
}
|
||||
@@ -196,7 +222,7 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
|
||||
req.StderrLogPath,
|
||||
)
|
||||
wrappedMessage = addSubprocessStreamHint(wrappedMessage, err)
|
||||
return r.failureResult(req, reqModules, runRes, credentialPresent, primaryConcurrencyViaEnv), fmt.Errorf(
|
||||
return r.failureResult(req, reqModules, runRes, credentialPresent), fmt.Errorf(
|
||||
"%s: %w",
|
||||
wrappedMessage,
|
||||
err,
|
||||
@@ -204,11 +230,11 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
|
||||
}
|
||||
|
||||
if err := validateProcessedOutput(req.OutputProcessedPath); err != nil {
|
||||
return r.failureResult(req, reqModules, runRes, credentialPresent, primaryConcurrencyViaEnv), fmt.Errorf("validate audita processed output %q: %w", req.OutputProcessedPath, err)
|
||||
return r.failureResult(req, reqModules, runRes, credentialPresent), fmt.Errorf("validate audita processed output %q: %w", req.OutputProcessedPath, err)
|
||||
}
|
||||
if r.report {
|
||||
if err := validateJSONFile(req.ReportPath); err != nil {
|
||||
return r.failureResult(req, reqModules, runRes, credentialPresent, primaryConcurrencyViaEnv), fmt.Errorf("validate audita report output %q: %w", req.ReportPath, err)
|
||||
return r.failureResult(req, reqModules, runRes, credentialPresent), fmt.Errorf("validate audita report output %q: %w", req.ReportPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -223,21 +249,25 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
|
||||
Duration: runRes.Duration,
|
||||
InvokedBinary: r.binary,
|
||||
Metadata: map[string]any{
|
||||
"adapter": "audita_subprocess",
|
||||
"modules": reqModules,
|
||||
"base_url": r.baseURL,
|
||||
"model": r.model,
|
||||
"validation_model": r.validationModel,
|
||||
"validation_llm_concurrency": r.validationLLMConcurrency,
|
||||
"credential_env_var": r.llmAPIKeyEnv,
|
||||
"credential_present": credentialPresent,
|
||||
"primary_llm_concurrency_via_env": primaryConcurrencyViaEnv,
|
||||
"primary_llm_concurrency_env_name": "AUDITA_LLM_CONCURRENCY",
|
||||
"adapter": "audita_subprocess",
|
||||
"modules": reqModules,
|
||||
"base_url": r.baseURL,
|
||||
"model": r.model,
|
||||
"transcript_description": r.transcriptDescription,
|
||||
"config_path": r.configPath,
|
||||
"output_schema": r.outputSchema,
|
||||
"work_dir_retention": r.workDirRetention,
|
||||
"validation_model": r.validationModel,
|
||||
"total_llm_concurrency": r.totalLLMConcurrency,
|
||||
"proposal_llm_concurrency": r.proposalLLMConcurrency,
|
||||
"validation_llm_concurrency": r.validationLLMConcurrency,
|
||||
"credential_env_var": r.llmAPIKeyEnv,
|
||||
"credential_present": credentialPresent,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *SubprocessRunner) failureResult(req PolishRequest, modules []string, runRes subprocess.RunResult, credentialPresent bool, primaryConcurrencyViaEnv bool) PolishResult {
|
||||
func (r *SubprocessRunner) failureResult(req PolishRequest, modules []string, runRes subprocess.RunResult, credentialPresent bool) PolishResult {
|
||||
return PolishResult{
|
||||
ProcessedTranscriptPath: req.OutputProcessedPath,
|
||||
ReportPath: req.ReportPath,
|
||||
@@ -249,16 +279,20 @@ func (r *SubprocessRunner) failureResult(req PolishRequest, modules []string, ru
|
||||
Duration: runRes.Duration,
|
||||
InvokedBinary: r.binary,
|
||||
Metadata: map[string]any{
|
||||
"adapter": "audita_subprocess",
|
||||
"modules": modules,
|
||||
"base_url": r.baseURL,
|
||||
"model": r.model,
|
||||
"validation_model": r.validationModel,
|
||||
"validation_llm_concurrency": r.validationLLMConcurrency,
|
||||
"credential_env_var": r.llmAPIKeyEnv,
|
||||
"credential_present": credentialPresent,
|
||||
"primary_llm_concurrency_via_env": primaryConcurrencyViaEnv,
|
||||
"primary_llm_concurrency_env_name": "AUDITA_LLM_CONCURRENCY",
|
||||
"adapter": "audita_subprocess",
|
||||
"modules": modules,
|
||||
"base_url": r.baseURL,
|
||||
"model": r.model,
|
||||
"transcript_description": r.transcriptDescription,
|
||||
"config_path": r.configPath,
|
||||
"output_schema": r.outputSchema,
|
||||
"work_dir_retention": r.workDirRetention,
|
||||
"validation_model": r.validationModel,
|
||||
"total_llm_concurrency": r.totalLLMConcurrency,
|
||||
"proposal_llm_concurrency": r.proposalLLMConcurrency,
|
||||
"validation_llm_concurrency": r.validationLLMConcurrency,
|
||||
"credential_env_var": r.llmAPIKeyEnv,
|
||||
"credential_present": credentialPresent,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -269,14 +303,38 @@ func (r *SubprocessRunner) buildArgs(req PolishRequest, modules []string) []stri
|
||||
req.MergedTranscriptPath,
|
||||
"--glossary", req.GlossaryPath,
|
||||
"--output", req.OutputProcessedPath,
|
||||
"--modules", strings.Join(modules, ","),
|
||||
"--base-url", r.baseURL,
|
||||
"--model", r.model,
|
||||
"--work-dir", req.WorkDir,
|
||||
}
|
||||
if r.baseURL != "" {
|
||||
args = append(args, "--base-url", r.baseURL)
|
||||
}
|
||||
if r.model != "" {
|
||||
args = append(args, "--model", r.model)
|
||||
}
|
||||
if len(modules) > 0 {
|
||||
args = append(args, "--modules", strings.Join(modules, ","))
|
||||
}
|
||||
if r.report {
|
||||
args = append(args, "--report-json", req.ReportPath)
|
||||
}
|
||||
if r.transcriptDescription != "" {
|
||||
args = append(args, "--transcript-description", r.transcriptDescription)
|
||||
}
|
||||
if r.configPath != "" {
|
||||
args = append(args, "--config", r.configPath)
|
||||
}
|
||||
if r.outputSchema != "" {
|
||||
args = append(args, "--output-schema", r.outputSchema)
|
||||
}
|
||||
if r.workDirRetention != "" {
|
||||
args = append(args, "--work-dir-retention", r.workDirRetention)
|
||||
}
|
||||
if r.totalLLMConcurrency != nil {
|
||||
args = append(args, "--total-llm-concurrency", strconv.Itoa(*r.totalLLMConcurrency))
|
||||
}
|
||||
if r.proposalLLMConcurrency != nil {
|
||||
args = append(args, "--proposal-llm-concurrency", strconv.Itoa(*r.proposalLLMConcurrency))
|
||||
}
|
||||
if r.validationModel != "" {
|
||||
args = append(args, "--validation-model", r.validationModel)
|
||||
}
|
||||
@@ -286,29 +344,31 @@ func (r *SubprocessRunner) buildArgs(req PolishRequest, modules []string) []stri
|
||||
return args
|
||||
}
|
||||
|
||||
func (r *SubprocessRunner) writeInvocationConfig(req PolishRequest, args []string, modules []string, credentialPresent bool, primaryConcurrencyViaEnv bool) error {
|
||||
func (r *SubprocessRunner) writeInvocationConfig(req PolishRequest, args []string, modules []string, credentialPresent bool) error {
|
||||
payload := map[string]any{
|
||||
"schema": "audita.generated.v1",
|
||||
"binary": r.binary,
|
||||
"args": args,
|
||||
"timeout": r.timeout.String(),
|
||||
"modules": modules,
|
||||
"base_url": r.baseURL,
|
||||
"model": r.model,
|
||||
"validation_model": r.validationModel,
|
||||
"validation_llm_concurrency": r.validationLLMConcurrency,
|
||||
"report_enabled": r.report,
|
||||
"merged_transcript_path": req.MergedTranscriptPath,
|
||||
"glossary_path": req.GlossaryPath,
|
||||
"output_path": req.OutputProcessedPath,
|
||||
"report_path": req.ReportPath,
|
||||
"work_dir": req.WorkDir,
|
||||
"credential_env_var": r.llmAPIKeyEnv,
|
||||
"credential_present": credentialPresent,
|
||||
"primary_llm_concurrency_via_env": primaryConcurrencyViaEnv,
|
||||
}
|
||||
if r.llmConcurrency != nil {
|
||||
payload["llm_concurrency"] = *r.llmConcurrency
|
||||
"schema": "audita.generated.v1",
|
||||
"binary": r.binary,
|
||||
"args": args,
|
||||
"timeout": r.timeout.String(),
|
||||
"modules": modules,
|
||||
"base_url": r.baseURL,
|
||||
"model": r.model,
|
||||
"transcript_description": r.transcriptDescription,
|
||||
"config_path": r.configPath,
|
||||
"output_schema": r.outputSchema,
|
||||
"work_dir_retention": r.workDirRetention,
|
||||
"validation_model": r.validationModel,
|
||||
"total_llm_concurrency": r.totalLLMConcurrency,
|
||||
"proposal_llm_concurrency": r.proposalLLMConcurrency,
|
||||
"validation_llm_concurrency": r.validationLLMConcurrency,
|
||||
"report_enabled": r.report,
|
||||
"merged_transcript_path": req.MergedTranscriptPath,
|
||||
"glossary_path": req.GlossaryPath,
|
||||
"output_path": req.OutputProcessedPath,
|
||||
"report_path": req.ReportPath,
|
||||
"work_dir": req.WorkDir,
|
||||
"credential_env_var": r.llmAPIKeyEnv,
|
||||
"credential_present": credentialPresent,
|
||||
}
|
||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644)
|
||||
}
|
||||
|
||||
@@ -25,7 +25,8 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||
|
||||
wrapper := writeAuditaHelperWrapper(t)
|
||||
llmConcurrency := 1
|
||||
totalLLMConcurrency := 3
|
||||
proposalLLMConcurrency := 2
|
||||
validationLLMConcurrency := 2
|
||||
runner, err := NewSubprocessRunner(SubprocessRunnerConfig{
|
||||
Binary: wrapper,
|
||||
@@ -34,7 +35,12 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
||||
Modules: []string{"glossary", "homophones", "glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
LLMConcurrency: &llmConcurrency,
|
||||
TranscriptDescription: "Campaign Session 42",
|
||||
ConfigPath: "/etc/audita/config.yml",
|
||||
OutputSchema: "audita-v1",
|
||||
WorkDirRetention: "auto",
|
||||
TotalLLMConcurrency: &totalLLMConcurrency,
|
||||
ProposalLLMConcurrency: &proposalLLMConcurrency,
|
||||
ValidationModel: "openrouter/google/gemma-4-31b-it",
|
||||
ValidationLLMConcurrency: &validationLLMConcurrency,
|
||||
Report: true,
|
||||
@@ -46,9 +52,9 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
req := PolishRequest{
|
||||
GeneratedConfigPath: filepath.Join(dir, "audita.generated.yml"),
|
||||
MergedTranscriptPath: filepath.Join(dir, "merged.json"),
|
||||
MergedTranscriptPath: filepath.Join(dir, "base.json"),
|
||||
GlossaryPath: filepath.Join(dir, "glossary.yml"),
|
||||
OutputProcessedPath: filepath.Join(dir, "processed.json"),
|
||||
OutputProcessedPath: filepath.Join(dir, "polished.json"),
|
||||
ReportPath: filepath.Join(dir, "audita.report.json"),
|
||||
WorkDir: filepath.Join(dir, "artifacts", "audita-work"),
|
||||
StdoutLogPath: filepath.Join(dir, "audita.stdout.log"),
|
||||
@@ -93,11 +99,17 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
||||
"process", req.MergedTranscriptPath,
|
||||
"--glossary", req.GlossaryPath,
|
||||
"--output", req.OutputProcessedPath,
|
||||
"--modules", "glossary,homophones,glossary",
|
||||
"--work-dir", req.WorkDir,
|
||||
"--base-url", "https://openrouter.ai/api/v1",
|
||||
"--model", "openrouter/google/gemma-4-31b-it",
|
||||
"--work-dir", req.WorkDir,
|
||||
"--modules", "glossary,homophones,glossary",
|
||||
"--report-json", req.ReportPath,
|
||||
"--transcript-description", "Campaign Session 42",
|
||||
"--config", "/etc/audita/config.yml",
|
||||
"--output-schema", "audita-v1",
|
||||
"--work-dir-retention", "auto",
|
||||
"--total-llm-concurrency", "3",
|
||||
"--proposal-llm-concurrency", "2",
|
||||
"--validation-model", "openrouter/google/gemma-4-31b-it",
|
||||
"--validation-llm-concurrency", "2",
|
||||
}
|
||||
@@ -107,8 +119,8 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
||||
if rec.Env["AUDITA_LLM_API_KEY"] != "super-secret" {
|
||||
t.Fatalf("AUDITA_LLM_API_KEY = %q, want propagated secret", rec.Env["AUDITA_LLM_API_KEY"])
|
||||
}
|
||||
if rec.Env["AUDITA_LLM_CONCURRENCY"] != "1" {
|
||||
t.Fatalf("AUDITA_LLM_CONCURRENCY = %q, want 1", rec.Env["AUDITA_LLM_CONCURRENCY"])
|
||||
if rec.Env["AUDITA_LLM_CONCURRENCY"] != "" {
|
||||
t.Fatalf("AUDITA_LLM_CONCURRENCY = %q, want empty/omitted", rec.Env["AUDITA_LLM_CONCURRENCY"])
|
||||
}
|
||||
|
||||
cfgData, err := os.ReadFile(req.GeneratedConfigPath)
|
||||
@@ -124,16 +136,14 @@ func TestSubprocessRunnerMissingConfiguredCredentialFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
llmConcurrency := 1
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "MISSING_AUDITA_KEY",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
LLMConcurrency: &llmConcurrency,
|
||||
Report: false,
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "MISSING_AUDITA_KEY",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
|
||||
@@ -155,16 +165,14 @@ func TestSubprocessRunnerUnconfiguredCredentialEnvOmitsCredential(t *testing.T)
|
||||
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||
|
||||
llmConcurrency := 1
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
LLMConcurrency: &llmConcurrency,
|
||||
Report: false,
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
|
||||
@@ -211,6 +219,65 @@ func TestSubprocessRunnerInheritsParentEnvironment(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerOmitsModulesFlagWhenNotConfigured(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "success")
|
||||
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "",
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
if _, err := runner.Run(context.Background(), req); err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
rec := readAuditaHelperRecord(t, recordPath)
|
||||
for i := 0; i < len(rec.Args); i++ {
|
||||
if rec.Args[i] == "--modules" {
|
||||
t.Fatalf("args contained --modules unexpectedly: %#v", rec.Args)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerOmitsBaseURLAndModelFlagsWhenNotConfigured(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "success")
|
||||
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "",
|
||||
Report: false,
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
if _, err := runner.Run(context.Background(), req); err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
rec := readAuditaHelperRecord(t, recordPath)
|
||||
for i := 0; i < len(rec.Args); i++ {
|
||||
if rec.Args[i] == "--base-url" {
|
||||
t.Fatalf("args contained --base-url unexpectedly: %#v", rec.Args)
|
||||
}
|
||||
if rec.Args[i] == "--model" {
|
||||
t.Fatalf("args contained --model unexpectedly: %#v", rec.Args)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
@@ -220,16 +287,14 @@ func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
|
||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
|
||||
llmConcurrency := 1
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
LLMConcurrency: &llmConcurrency,
|
||||
Report: true,
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: true,
|
||||
})
|
||||
req := auditaReqForTest(t, true)
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
@@ -256,16 +321,14 @@ func TestSubprocessRunnerSubprocessFailureAddsStderrDescriptorHint(t *testing.T)
|
||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
|
||||
llmConcurrency := 1
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
LLMConcurrency: &llmConcurrency,
|
||||
Report: true,
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: true,
|
||||
})
|
||||
req := auditaReqForTest(t, true)
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
@@ -286,16 +349,14 @@ func TestSubprocessRunnerMissingOutputFails(t *testing.T) {
|
||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
|
||||
llmConcurrency := 1
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
LLMConcurrency: &llmConcurrency,
|
||||
Report: false,
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
@@ -316,16 +377,14 @@ func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) {
|
||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
|
||||
llmConcurrency := 1
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
LLMConcurrency: &llmConcurrency,
|
||||
Report: false,
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
@@ -346,16 +405,14 @@ func TestSubprocessRunnerSegmentsMissingFails(t *testing.T) {
|
||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
|
||||
llmConcurrency := 1
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
LLMConcurrency: &llmConcurrency,
|
||||
Report: false,
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
@@ -376,16 +433,14 @@ func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
|
||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
|
||||
llmConcurrency := 1
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
LLMConcurrency: &llmConcurrency,
|
||||
Report: true,
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: true,
|
||||
})
|
||||
req := auditaReqForTest(t, true)
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
@@ -398,11 +453,11 @@ func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerConstructorValidation(t *testing.T) {
|
||||
_, err := NewSubprocessRunnerFromConfigValues("", "3h", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", nil, "", nil, true)
|
||||
_, err := NewSubprocessRunnerFromConfigValues("", "3h", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", "", "", "", "", nil, nil, "", nil, true)
|
||||
if err == nil {
|
||||
t.Fatal("expected binary validation error")
|
||||
}
|
||||
_, err = NewSubprocessRunnerFromConfigValues("audita", "bad", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", nil, "", nil, true)
|
||||
_, err = NewSubprocessRunnerFromConfigValues("audita", "bad", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", "", "", "", "", nil, nil, "", nil, true)
|
||||
if err == nil {
|
||||
t.Fatal("expected timeout parse error")
|
||||
}
|
||||
@@ -516,7 +571,7 @@ func mustAuditaRunner(t *testing.T, cfg SubprocessRunnerConfig) *SubprocessRunne
|
||||
func auditaReqForTest(t *testing.T, withReport bool) PolishRequest {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
merged := filepath.Join(dir, "merged.json")
|
||||
merged := filepath.Join(dir, "base.json")
|
||||
glossary := filepath.Join(dir, "glossary.yml")
|
||||
writeAuditaTestFile(t, merged, `{"segments":[]}`)
|
||||
writeAuditaTestFile(t, glossary, "terms: []\n")
|
||||
@@ -524,7 +579,7 @@ func auditaReqForTest(t *testing.T, withReport bool) PolishRequest {
|
||||
GeneratedConfigPath: filepath.Join(dir, "audita.generated.yml"),
|
||||
MergedTranscriptPath: merged,
|
||||
GlossaryPath: glossary,
|
||||
OutputProcessedPath: filepath.Join(dir, "processed.json"),
|
||||
OutputProcessedPath: filepath.Join(dir, "polished.json"),
|
||||
WorkDir: filepath.Join(dir, "artifacts", "audita-work"),
|
||||
StdoutLogPath: filepath.Join(dir, "audita.stdout.log"),
|
||||
StderrLogPath: filepath.Join(dir, "audita.stderr.log"),
|
||||
|
||||
@@ -31,7 +31,7 @@ func TestSubprocessRunnerRunSuccessBuildsDeterministicArgsAndCapturesLogs(t *tes
|
||||
ConfigPath: "/etc/scriptorium/config.yml",
|
||||
PromptID: "dnd.session_recap",
|
||||
ProfileID: "local-quality",
|
||||
InputPaths: map[string]string{"transcript": filepath.Join(dir, "processed.json"), "other": filepath.Join(dir, "other.md")},
|
||||
InputPaths: map[string]string{"transcript": filepath.Join(dir, "polished.json"), "other": filepath.Join(dir, "other.md")},
|
||||
Vars: map[string]string{"session_id": "2026-05-03", "campaign_name": "Icewind Dale"},
|
||||
OutputPath: filepath.Join(dir, "artifacts", "session_recap.md"),
|
||||
StdoutLogPath: filepath.Join(dir, "logs", "scriptorium.run.stdout.log"),
|
||||
@@ -180,7 +180,7 @@ func TestSubprocessRunnerRenderSuccess(t *testing.T) {
|
||||
req := RenderArtifactRequest{
|
||||
Binary: wrapper,
|
||||
PromptID: "dnd.session_recap",
|
||||
InputPaths: map[string]string{"transcript": filepath.Join(dir, "processed.json")},
|
||||
InputPaths: map[string]string{"transcript": filepath.Join(dir, "polished.json")},
|
||||
OutputPath: filepath.Join(dir, "artifacts", "session_recap.render.json"),
|
||||
StdoutLogPath: filepath.Join(dir, "logs", "scriptorium.render.stdout.log"),
|
||||
StderrLogPath: filepath.Join(dir, "logs", "scriptorium.render.stderr.log"),
|
||||
@@ -285,7 +285,7 @@ type scriptoriumHelperRecord struct {
|
||||
func runReqForTest(t *testing.T, binary string) RunArtifactRequest {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
transcriptPath := filepath.Join(dir, "processed.json")
|
||||
transcriptPath := filepath.Join(dir, "polished.json")
|
||||
writeScriptoriumFile(t, transcriptPath, `{"segments":[]}`)
|
||||
return RunArtifactRequest{
|
||||
Binary: binary,
|
||||
|
||||
@@ -14,7 +14,7 @@ func TestFakeRunnerCapturesRequestAndReturnsPath(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
req := MergeRequest{
|
||||
GeneratedConfigPath: filepath.Join(dir, "config", "seriatim.yml"),
|
||||
OutputMergedTranscriptPath: filepath.Join(dir, "transcripts", "merged.json"),
|
||||
OutputMergedTranscriptPath: filepath.Join(dir, "transcripts", "base.json"),
|
||||
StdoutLogPath: filepath.Join(dir, "logs", "seriatim.stdout.log"),
|
||||
StderrLogPath: filepath.Join(dir, "logs", "seriatim.stderr.log"),
|
||||
}
|
||||
@@ -57,8 +57,8 @@ func TestFakeRunnerTrimCapturesRequestAndReturnsPath(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
req := TrimRequest{
|
||||
GeneratedConfigPath: filepath.Join(dir, "config", "seriatim.trim.yml"),
|
||||
InputTranscriptPath: filepath.Join(dir, "transcripts", "processed.json"),
|
||||
OutputTrimmedPath: filepath.Join(dir, "transcripts", "trimmed.json"),
|
||||
InputTranscriptPath: filepath.Join(dir, "transcripts", "polished.json"),
|
||||
OutputTrimmedPath: filepath.Join(dir, "transcripts", "final.trimmed.json"),
|
||||
KeepSelector: "1-10",
|
||||
StdoutLogPath: filepath.Join(dir, "logs", "seriatim.trim.stdout.log"),
|
||||
StderrLogPath: filepath.Join(dir, "logs", "seriatim.trim.stderr.log"),
|
||||
@@ -105,8 +105,8 @@ func TestFakeRunnerNormalizeCapturesRequestAndReturnsPath(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
req := NormalizeRequest{
|
||||
GeneratedConfigPath: filepath.Join(dir, "config", "seriatim.normalize.yml"),
|
||||
InputTranscriptPath: filepath.Join(dir, "transcripts", "processed.json"),
|
||||
OutputNormalizedPath: filepath.Join(dir, "transcripts", "normalized.json"),
|
||||
InputTranscriptPath: filepath.Join(dir, "transcripts", "polished.json"),
|
||||
OutputNormalizedPath: filepath.Join(dir, "transcripts", "final.json"),
|
||||
OutputSchema: "seriatim-intermediate",
|
||||
ReportPath: filepath.Join(dir, "artifacts", "seriatim.normalize.report.json"),
|
||||
StdoutLogPath: filepath.Join(dir, "logs", "seriatim.normalize.stdout.log"),
|
||||
|
||||
@@ -50,7 +50,7 @@ func TestSubprocessRunnerSuccessWithReportArgsAndEnv(t *testing.T) {
|
||||
req := MergeRequest{
|
||||
GeneratedConfigPath: filepath.Join(dir, "seriatim.generated.yml"),
|
||||
InputTranscriptPaths: []string{filepath.Join(dir, "a.json"), filepath.Join(dir, "b.json")},
|
||||
OutputMergedTranscriptPath: filepath.Join(dir, "merged.json"),
|
||||
OutputMergedTranscriptPath: filepath.Join(dir, "base.json"),
|
||||
ReportPath: filepath.Join(dir, "seriatim.report.json"),
|
||||
SpeakersPath: filepath.Join(dir, "speakers.yml"),
|
||||
AutocorrectPath: filepath.Join(dir, "autocorrect.yml"),
|
||||
@@ -732,7 +732,7 @@ func mergeReqForTest(t *testing.T, withReport bool) MergeRequest {
|
||||
req := MergeRequest{
|
||||
GeneratedConfigPath: filepath.Join(dir, "seriatim.generated.yml"),
|
||||
InputTranscriptPaths: []string{in1, in2},
|
||||
OutputMergedTranscriptPath: filepath.Join(dir, "merged.json"),
|
||||
OutputMergedTranscriptPath: filepath.Join(dir, "base.json"),
|
||||
StdoutLogPath: filepath.Join(dir, "seriatim.stdout.log"),
|
||||
StderrLogPath: filepath.Join(dir, "seriatim.stderr.log"),
|
||||
}
|
||||
@@ -745,11 +745,11 @@ func mergeReqForTest(t *testing.T, withReport bool) MergeRequest {
|
||||
func trimReqForTest(t *testing.T) TrimRequest {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
input := filepath.Join(dir, "processed.json")
|
||||
input := filepath.Join(dir, "polished.json")
|
||||
writeSeriatimFile(t, input, `{"schema":"seriatim.intermediate.v1","segments":[]}`)
|
||||
return TrimRequest{
|
||||
InputTranscriptPath: input,
|
||||
OutputTrimmedPath: filepath.Join(dir, "trimmed.json"),
|
||||
OutputTrimmedPath: filepath.Join(dir, "final.trimmed.json"),
|
||||
KeepSelector: "5-12",
|
||||
GeneratedConfigPath: filepath.Join(dir, "seriatim.trim.generated.yml"),
|
||||
StdoutLogPath: filepath.Join(dir, "seriatim.trim.stdout.log"),
|
||||
@@ -760,12 +760,12 @@ func trimReqForTest(t *testing.T) TrimRequest {
|
||||
func normalizeReqForTest(t *testing.T, withReport bool) NormalizeRequest {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
input := filepath.Join(dir, "processed.json")
|
||||
input := filepath.Join(dir, "polished.json")
|
||||
writeSeriatimFile(t, input, `{"schema":"audita.processed.v1","segments":[]}`)
|
||||
|
||||
req := NormalizeRequest{
|
||||
InputTranscriptPath: input,
|
||||
OutputNormalizedPath: filepath.Join(dir, "normalized.json"),
|
||||
OutputNormalizedPath: filepath.Join(dir, "final.json"),
|
||||
OutputSchema: "seriatim-intermediate",
|
||||
GeneratedConfigPath: filepath.Join(dir, "seriatim.normalize.generated.yml"),
|
||||
StdoutLogPath: filepath.Join(dir, "seriatim.normalize.stdout.log"),
|
||||
|
||||
29
internal/adapters/storage/factory.go
Normal file
29
internal/adapters/storage/factory.go
Normal file
@@ -0,0 +1,29 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
// NewObjectStoreFromConfig constructs a remote object store from resolved config.
|
||||
func NewObjectStoreFromConfig(ctx context.Context, cfg *config.Config) (ObjectStore, error) {
|
||||
if cfg == nil || cfg.Pipeline == nil {
|
||||
return nil, fmt.Errorf("pipeline config is required")
|
||||
}
|
||||
|
||||
if strings.EqualFold(strings.TrimSpace(cfg.Pipeline.Storage.Backend), "s3") {
|
||||
if cfg.Pipeline.Storage.S3 == nil {
|
||||
return nil, fmt.Errorf("pipeline.storage.s3 is required when pipeline.storage.backend is s3")
|
||||
}
|
||||
return NewS3BackendFromConfig(ctx, *cfg.Pipeline.Storage.S3)
|
||||
}
|
||||
|
||||
if cfg.Pipeline.Storage.S3 != nil && strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket) != "" {
|
||||
return NewS3BackendFromConfig(ctx, *cfg.Pipeline.Storage.S3)
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("no remote object store backend is configured")
|
||||
}
|
||||
56
internal/adapters/storage/factory_test.go
Normal file
56
internal/adapters/storage/factory_test.go
Normal file
@@ -0,0 +1,56 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
func TestNewObjectStoreFromConfigBuildsS3WhenBackendIsS3(t *testing.T) {
|
||||
original := newS3Client
|
||||
t.Cleanup(func() { newS3Client = original })
|
||||
newS3Client = func(_ context.Context, _ s3ClientOptions) (s3API, error) {
|
||||
return &fakeS3API{}, nil
|
||||
}
|
||||
|
||||
store, err := NewObjectStoreFromConfig(context.Background(), &config.Config{
|
||||
Pipeline: &config.PipelineConfig{
|
||||
Storage: config.StorageConfig{
|
||||
Backend: "s3",
|
||||
S3: &config.StorageS3Config{
|
||||
Bucket: "my-archive",
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewObjectStoreFromConfig() error = %v", err)
|
||||
}
|
||||
if _, ok := store.(*S3Backend); !ok {
|
||||
t.Fatalf("store type = %T, want *S3Backend", store)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewObjectStoreFromConfigRequiresS3ConfigWhenBackendIsS3(t *testing.T) {
|
||||
_, err := NewObjectStoreFromConfig(context.Background(), &config.Config{
|
||||
Pipeline: &config.PipelineConfig{
|
||||
Storage: config.StorageConfig{Backend: "s3"},
|
||||
},
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "pipeline.storage.s3 is required") {
|
||||
t.Fatalf("NewObjectStoreFromConfig() error = %v, want missing storage.s3 error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewObjectStoreFromConfigNoRemoteBackendConfigured(t *testing.T) {
|
||||
_, err := NewObjectStoreFromConfig(context.Background(), &config.Config{
|
||||
Pipeline: &config.PipelineConfig{
|
||||
Storage: config.StorageConfig{Backend: "local"},
|
||||
},
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "no remote object store backend is configured") {
|
||||
t.Fatalf("NewObjectStoreFromConfig() error = %v, want no-backend error", err)
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,14 @@
|
||||
package storage
|
||||
|
||||
import "context"
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// NoopBackend is a deterministic no-op archive/storage adapter.
|
||||
type NoopBackend struct{}
|
||||
@@ -18,6 +26,28 @@ type FakeBackend struct {
|
||||
Requests []ArchiveRequest
|
||||
Err error
|
||||
Result ArchiveResult
|
||||
|
||||
Objects map[string]FakeObject
|
||||
Uploads []FakeUploadCall
|
||||
Downloads []FakeDownloadCall
|
||||
|
||||
ListErr error
|
||||
DownloadErr error
|
||||
UploadErr error
|
||||
ExistsErr error
|
||||
}
|
||||
|
||||
// FakeUploadCall captures one upload invocation in call order.
|
||||
type FakeUploadCall struct {
|
||||
LocalPath string
|
||||
Key string
|
||||
Options UploadOptions
|
||||
}
|
||||
|
||||
// FakeDownloadCall captures one download invocation in call order.
|
||||
type FakeDownloadCall struct {
|
||||
Key string
|
||||
LocalPath string
|
||||
}
|
||||
|
||||
// Archive records request and returns configured response.
|
||||
@@ -38,3 +68,152 @@ func (f *FakeBackend) Archive(ctx context.Context, req ArchiveRequest) (ArchiveR
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// FakeObject is a deterministic fake object-store record.
|
||||
type FakeObject struct {
|
||||
Key string
|
||||
Data []byte
|
||||
Metadata map[string]string
|
||||
ETag string
|
||||
LastModified *time.Time
|
||||
}
|
||||
|
||||
// SeedObject inserts or replaces an object in the fake object store.
|
||||
func (f *FakeBackend) SeedObject(obj FakeObject) {
|
||||
if f.Objects == nil {
|
||||
f.Objects = map[string]FakeObject{}
|
||||
}
|
||||
key := normalizeObjectKey(obj.Key)
|
||||
obj.Key = key
|
||||
obj.Data = append([]byte(nil), obj.Data...)
|
||||
obj.Metadata = copyMetadata(obj.Metadata)
|
||||
f.Objects[key] = obj
|
||||
}
|
||||
|
||||
// List returns deterministic prefix-filtered objects.
|
||||
func (f *FakeBackend) List(ctx context.Context, prefix string) ([]ObjectInfo, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if f.ListErr != nil {
|
||||
return nil, f.ListErr
|
||||
}
|
||||
|
||||
normalizedPrefix := normalizeObjectKey(prefix)
|
||||
keys := make([]string, 0, len(f.Objects))
|
||||
for key := range f.Objects {
|
||||
if strings.HasPrefix(key, normalizedPrefix) {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
out := make([]ObjectInfo, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
obj := f.Objects[key]
|
||||
out = append(out, ObjectInfo{
|
||||
Key: obj.Key,
|
||||
Size: int64(len(obj.Data)),
|
||||
ETag: obj.ETag,
|
||||
LastModified: obj.LastModified,
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Download writes one object to a local path.
|
||||
func (f *FakeBackend) Download(ctx context.Context, key, localPath string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if f.DownloadErr != nil {
|
||||
return f.DownloadErr
|
||||
}
|
||||
if strings.TrimSpace(localPath) == "" {
|
||||
return fmt.Errorf("download object: local path is required")
|
||||
}
|
||||
|
||||
obj, ok := f.Objects[normalizeObjectKey(key)]
|
||||
if !ok {
|
||||
return fmt.Errorf("download object %q: %w", key, os.ErrNotExist)
|
||||
}
|
||||
f.Downloads = append(f.Downloads, FakeDownloadCall{
|
||||
Key: normalizeObjectKey(key),
|
||||
LocalPath: localPath,
|
||||
})
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
|
||||
return fmt.Errorf("download object %q: create parent directory: %w", key, err)
|
||||
}
|
||||
if err := os.WriteFile(localPath, obj.Data, 0o644); err != nil {
|
||||
return fmt.Errorf("download object %q: write local file: %w", key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Upload reads a local file and stores it under key.
|
||||
func (f *FakeBackend) Upload(ctx context.Context, localPath, key string, opts UploadOptions) (ObjectInfo, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
if f.UploadErr != nil {
|
||||
return ObjectInfo{}, f.UploadErr
|
||||
}
|
||||
if strings.TrimSpace(localPath) == "" {
|
||||
return ObjectInfo{}, fmt.Errorf("upload object: local path is required")
|
||||
}
|
||||
if strings.TrimSpace(key) == "" {
|
||||
return ObjectInfo{}, fmt.Errorf("upload object: key is required")
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(localPath)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: %w", key, localPath, err)
|
||||
}
|
||||
|
||||
normalizedKey := normalizeObjectKey(key)
|
||||
f.Uploads = append(f.Uploads, FakeUploadCall{
|
||||
LocalPath: localPath,
|
||||
Key: normalizedKey,
|
||||
Options: UploadOptions{
|
||||
Metadata: copyMetadata(opts.Metadata),
|
||||
ContentType: opts.ContentType,
|
||||
},
|
||||
})
|
||||
now := time.Now().UTC()
|
||||
obj := FakeObject{
|
||||
Key: normalizedKey,
|
||||
Data: data,
|
||||
Metadata: copyMetadata(opts.Metadata),
|
||||
LastModified: &now,
|
||||
}
|
||||
f.SeedObject(obj)
|
||||
return ObjectInfo{
|
||||
Key: normalizedKey,
|
||||
Size: int64(len(data)),
|
||||
LastModified: &now,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Exists checks object presence.
|
||||
func (f *FakeBackend) Exists(ctx context.Context, key string) (bool, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if f.ExistsErr != nil {
|
||||
return false, f.ExistsErr
|
||||
}
|
||||
_, ok := f.Objects[normalizeObjectKey(key)]
|
||||
return ok, nil
|
||||
}
|
||||
|
||||
func copyMetadata(in map[string]string) map[string]string {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]string, len(in))
|
||||
for k, v := range in {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -3,6 +3,9 @@ package storage
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -29,3 +32,84 @@ func TestFakeBackendError(t *testing.T) {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFakeBackendListPrefixFiltering(t *testing.T) {
|
||||
fake := &FakeBackend{}
|
||||
fake.SeedObject(FakeObject{Key: "dnd/campaigns/forsaken/audio/a.flac", Data: []byte("a")})
|
||||
fake.SeedObject(FakeObject{Key: "dnd/campaigns/forsaken/audio/b.flac", Data: []byte("b")})
|
||||
fake.SeedObject(FakeObject{Key: "dnd/campaigns/other/audio/c.flac", Data: []byte("c")})
|
||||
|
||||
items, err := fake.List(context.Background(), "dnd/campaigns/forsaken/audio/")
|
||||
if err != nil {
|
||||
t.Fatalf("List() error = %v", err)
|
||||
}
|
||||
if len(items) != 2 {
|
||||
t.Fatalf("List() len = %d, want 2", len(items))
|
||||
}
|
||||
if items[0].Key != "dnd/campaigns/forsaken/audio/a.flac" || items[1].Key != "dnd/campaigns/forsaken/audio/b.flac" {
|
||||
t.Fatalf("List() keys = %#v", items)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFakeBackendDownload(t *testing.T) {
|
||||
fake := &FakeBackend{}
|
||||
fake.SeedObject(FakeObject{Key: "audio/a.flac", Data: []byte("audio-a")})
|
||||
|
||||
dst := filepath.Join(t.TempDir(), "nested", "a.flac")
|
||||
if err := fake.Download(context.Background(), `audio\a.flac`, dst); err != nil {
|
||||
t.Fatalf("Download() error = %v", err)
|
||||
}
|
||||
data, err := os.ReadFile(dst)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile() error = %v", err)
|
||||
}
|
||||
if string(data) != "audio-a" {
|
||||
t.Fatalf("downloaded content = %q, want %q", string(data), "audio-a")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFakeBackendUploadAndExists(t *testing.T) {
|
||||
fake := &FakeBackend{}
|
||||
|
||||
local := filepath.Join(t.TempDir(), "upload.txt")
|
||||
if err := os.WriteFile(local, []byte("payload"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile() error = %v", err)
|
||||
}
|
||||
|
||||
info, err := fake.Upload(context.Background(), local, `runs\id\artifact.txt`, UploadOptions{
|
||||
Metadata: map[string]string{"kind": "artifact"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Upload() error = %v", err)
|
||||
}
|
||||
if info.Key != "runs/id/artifact.txt" {
|
||||
t.Fatalf("Upload() key = %q, want normalized key", info.Key)
|
||||
}
|
||||
|
||||
ok, err := fake.Exists(context.Background(), "runs/id/artifact.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("Exists() error = %v", err)
|
||||
}
|
||||
if !ok {
|
||||
t.Fatal("Exists() = false, want true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFakeBackendObjectErrors(t *testing.T) {
|
||||
fake := &FakeBackend{DownloadErr: errors.New("download fail"), UploadErr: errors.New("upload fail"), ListErr: errors.New("list fail"), ExistsErr: errors.New("exists fail")}
|
||||
|
||||
if _, err := fake.List(context.Background(), "x"); err == nil || !strings.Contains(err.Error(), "list fail") {
|
||||
t.Fatalf("List() error = %v, want list fail", err)
|
||||
}
|
||||
if err := fake.Download(context.Background(), "x", filepath.Join(t.TempDir(), "x")); err == nil || !strings.Contains(err.Error(), "download fail") {
|
||||
t.Fatalf("Download() error = %v, want download fail", err)
|
||||
}
|
||||
local := filepath.Join(t.TempDir(), "x.txt")
|
||||
_ = os.WriteFile(local, []byte("x"), 0o644)
|
||||
if _, err := fake.Upload(context.Background(), local, "x", UploadOptions{}); err == nil || !strings.Contains(err.Error(), "upload fail") {
|
||||
t.Fatalf("Upload() error = %v, want upload fail", err)
|
||||
}
|
||||
if _, err := fake.Exists(context.Background(), "x"); err == nil || !strings.Contains(err.Error(), "exists fail") {
|
||||
t.Fatalf("Exists() error = %v, want exists fail", err)
|
||||
}
|
||||
}
|
||||
|
||||
8
internal/adapters/storage/keys.go
Normal file
8
internal/adapters/storage/keys.go
Normal file
@@ -0,0 +1,8 @@
|
||||
package storage
|
||||
|
||||
import "strings"
|
||||
|
||||
func normalizeObjectKey(key string) string {
|
||||
normalized := strings.ReplaceAll(strings.TrimSpace(key), "\\", "/")
|
||||
return strings.TrimLeft(normalized, "/")
|
||||
}
|
||||
21
internal/adapters/storage/keys_test.go
Normal file
21
internal/adapters/storage/keys_test.go
Normal file
@@ -0,0 +1,21 @@
|
||||
package storage
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestNormalizeObjectKey(t *testing.T) {
|
||||
tests := []struct {
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{in: `dnd\campaigns\forsaken\a.flac`, want: "dnd/campaigns/forsaken/a.flac"},
|
||||
{in: " /dnd/campaigns/forsaken/a.flac ", want: "dnd/campaigns/forsaken/a.flac"},
|
||||
{in: "//dnd/campaigns/forsaken/a.flac", want: "dnd/campaigns/forsaken/a.flac"},
|
||||
{in: "", want: ""},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
if got := normalizeObjectKey(tt.in); got != tt.want {
|
||||
t.Fatalf("normalizeObjectKey(%q) = %q, want %q", tt.in, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
32
internal/adapters/storage/object_store.go
Normal file
32
internal/adapters/storage/object_store.go
Normal file
@@ -0,0 +1,32 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ObjectStore is a remote object storage boundary used by future prepare/archive work.
|
||||
//
|
||||
// Key invariant:
|
||||
// callers pass full bucket-relative object keys. Backend implementations do not
|
||||
// infer Narratio session semantics and do not prepend root prefixes.
|
||||
type ObjectStore interface {
|
||||
List(ctx context.Context, prefix string) ([]ObjectInfo, error)
|
||||
Download(ctx context.Context, key, localPath string) error
|
||||
Upload(ctx context.Context, localPath, key string, opts UploadOptions) (ObjectInfo, error)
|
||||
Exists(ctx context.Context, key string) (bool, error)
|
||||
}
|
||||
|
||||
// ObjectInfo describes one object in remote storage.
|
||||
type ObjectInfo struct {
|
||||
Key string
|
||||
Size int64
|
||||
ETag string
|
||||
LastModified *time.Time
|
||||
}
|
||||
|
||||
// UploadOptions configures optional object upload metadata.
|
||||
type UploadOptions struct {
|
||||
Metadata map[string]string
|
||||
ContentType string
|
||||
}
|
||||
275
internal/adapters/storage/s3_backend.go
Normal file
275
internal/adapters/storage/s3_backend.go
Normal file
@@ -0,0 +1,275 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
awsconfig "github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/aws/smithy-go"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
type s3API interface {
|
||||
ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error)
|
||||
GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error)
|
||||
PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error)
|
||||
HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error)
|
||||
}
|
||||
|
||||
// S3Backend is an ObjectStore implementation backed by S3-compatible APIs.
|
||||
type S3Backend struct {
|
||||
bucket string
|
||||
client s3API
|
||||
}
|
||||
|
||||
type s3ClientOptions struct {
|
||||
Region string
|
||||
Endpoint string
|
||||
ForcePathStyle bool
|
||||
AccessKeyID string
|
||||
SecretKey string
|
||||
}
|
||||
|
||||
var newS3Client = func(ctx context.Context, opts s3ClientOptions) (s3API, error) {
|
||||
loadOpts := make([]func(*awsconfig.LoadOptions) error, 0, 1)
|
||||
if strings.TrimSpace(opts.Region) != "" {
|
||||
loadOpts = append(loadOpts, awsconfig.WithRegion(strings.TrimSpace(opts.Region)))
|
||||
}
|
||||
if strings.TrimSpace(opts.AccessKeyID) != "" && strings.TrimSpace(opts.SecretKey) != "" {
|
||||
loadOpts = append(loadOpts, awsconfig.WithCredentialsProvider(
|
||||
credentials.NewStaticCredentialsProvider(
|
||||
strings.TrimSpace(opts.AccessKeyID),
|
||||
strings.TrimSpace(opts.SecretKey),
|
||||
"",
|
||||
),
|
||||
))
|
||||
}
|
||||
awsCfg, err := awsconfig.LoadDefaultConfig(ctx, loadOpts...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load aws config: %w", err)
|
||||
}
|
||||
|
||||
return s3.NewFromConfig(awsCfg, func(o *s3.Options) {
|
||||
if strings.TrimSpace(opts.Endpoint) != "" {
|
||||
endpoint := strings.TrimSpace(opts.Endpoint)
|
||||
o.BaseEndpoint = &endpoint
|
||||
}
|
||||
o.UsePathStyle = opts.ForcePathStyle
|
||||
}), nil
|
||||
}
|
||||
|
||||
// NewS3BackendFromConfig builds an S3 backend from resolved config.
|
||||
func NewS3BackendFromConfig(ctx context.Context, cfg config.StorageS3Config) (*S3Backend, error) {
|
||||
bucket := strings.TrimSpace(cfg.Bucket)
|
||||
if bucket == "" {
|
||||
return nil, fmt.Errorf("storage.s3.bucket is required")
|
||||
}
|
||||
|
||||
client, err := newS3Client(ctx, s3ClientOptions{
|
||||
Region: cfg.Region,
|
||||
Endpoint: cfg.Endpoint,
|
||||
ForcePathStyle: cfg.ForcePathStyle,
|
||||
AccessKeyID: s3CredentialFromEnv(orDefaultEnvName(cfg.AccessKeyIDEnv, config.DefaultS3AccessKeyIDEnv)),
|
||||
SecretKey: s3CredentialFromEnv(orDefaultEnvName(cfg.SecretKeyEnv, config.DefaultS3SecretAccessKeyEnv)),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build s3 client: %w", err)
|
||||
}
|
||||
|
||||
return &S3Backend{
|
||||
bucket: bucket,
|
||||
client: client,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func s3CredentialFromEnv(envVarName string) string {
|
||||
name := strings.TrimSpace(envVarName)
|
||||
if name == "" {
|
||||
return ""
|
||||
}
|
||||
value, ok := os.LookupEnv(name)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
|
||||
func orDefaultEnvName(name, fallback string) string {
|
||||
trimmed := strings.TrimSpace(name)
|
||||
if trimmed == "" {
|
||||
return fallback
|
||||
}
|
||||
return trimmed
|
||||
}
|
||||
|
||||
// List returns objects under prefix.
|
||||
func (b *S3Backend) List(ctx context.Context, prefix string) ([]ObjectInfo, error) {
|
||||
normalizedPrefix := normalizeObjectKey(prefix)
|
||||
out := make([]ObjectInfo, 0)
|
||||
var token *string
|
||||
|
||||
for {
|
||||
resp, err := b.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
||||
Bucket: &b.bucket,
|
||||
Prefix: &normalizedPrefix,
|
||||
ContinuationToken: token,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list objects under %q: %w", normalizedPrefix, err)
|
||||
}
|
||||
|
||||
for _, item := range resp.Contents {
|
||||
var lastModified *time.Time
|
||||
if item.LastModified != nil {
|
||||
t := *item.LastModified
|
||||
lastModified = &t
|
||||
}
|
||||
out = append(out, ObjectInfo{
|
||||
Key: normalizeObjectKey(valueOrEmpty(item.Key)),
|
||||
Size: valueOrZeroInt64(item.Size),
|
||||
ETag: strings.Trim(valueOrEmpty(item.ETag), "\""),
|
||||
LastModified: lastModified,
|
||||
})
|
||||
}
|
||||
|
||||
if !valueOrFalseBool(resp.IsTruncated) || resp.NextContinuationToken == nil {
|
||||
break
|
||||
}
|
||||
token = resp.NextContinuationToken
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Download retrieves one object to localPath, creating parent directories as needed.
|
||||
func (b *S3Backend) Download(ctx context.Context, key, localPath string) error {
|
||||
normalizedKey := normalizeObjectKey(key)
|
||||
if strings.TrimSpace(localPath) == "" {
|
||||
return fmt.Errorf("download object: local path is required")
|
||||
}
|
||||
|
||||
resp, err := b.client.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: &b.bucket,
|
||||
Key: &normalizedKey,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("download object %q: %w", normalizedKey, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
|
||||
return fmt.Errorf("download object %q: create parent directory: %w", normalizedKey, err)
|
||||
}
|
||||
dst, err := os.Create(localPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("download object %q: create local file: %w", normalizedKey, err)
|
||||
}
|
||||
defer dst.Close()
|
||||
|
||||
if _, err := io.Copy(dst, resp.Body); err != nil {
|
||||
return fmt.Errorf("download object %q: copy body: %w", normalizedKey, err)
|
||||
}
|
||||
if err := dst.Sync(); err != nil {
|
||||
return fmt.Errorf("download object %q: sync local file: %w", normalizedKey, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Upload sends a local file to key.
|
||||
func (b *S3Backend) Upload(ctx context.Context, localPath, key string, opts UploadOptions) (ObjectInfo, error) {
|
||||
normalizedKey := normalizeObjectKey(key)
|
||||
if strings.TrimSpace(localPath) == "" {
|
||||
return ObjectInfo{}, fmt.Errorf("upload object: local path is required")
|
||||
}
|
||||
if normalizedKey == "" {
|
||||
return ObjectInfo{}, fmt.Errorf("upload object: key is required")
|
||||
}
|
||||
|
||||
file, err := os.Open(localPath)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: %w", normalizedKey, localPath, err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
stat, err := file.Stat()
|
||||
if err != nil {
|
||||
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: stat local file: %w", normalizedKey, localPath, err)
|
||||
}
|
||||
|
||||
input := &s3.PutObjectInput{
|
||||
Bucket: &b.bucket,
|
||||
Key: &normalizedKey,
|
||||
Body: file,
|
||||
Metadata: copyMetadata(opts.Metadata),
|
||||
}
|
||||
if strings.TrimSpace(opts.ContentType) != "" {
|
||||
ct := strings.TrimSpace(opts.ContentType)
|
||||
input.ContentType = &ct
|
||||
}
|
||||
|
||||
resp, err := b.client.PutObject(ctx, input)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: %w", normalizedKey, localPath, err)
|
||||
}
|
||||
|
||||
return ObjectInfo{
|
||||
Key: normalizedKey,
|
||||
Size: stat.Size(),
|
||||
ETag: strings.Trim(valueOrEmpty(resp.ETag), "\""),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Exists checks whether one object key exists.
|
||||
func (b *S3Backend) Exists(ctx context.Context, key string) (bool, error) {
|
||||
normalizedKey := normalizeObjectKey(key)
|
||||
_, err := b.client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||
Bucket: &b.bucket,
|
||||
Key: &normalizedKey,
|
||||
})
|
||||
if err == nil {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
var notFound *types.NotFound
|
||||
if errors.As(err, ¬Found) {
|
||||
return false, nil
|
||||
}
|
||||
var apiErr smithy.APIError
|
||||
if errors.As(err, &apiErr) {
|
||||
switch apiErr.ErrorCode() {
|
||||
case "NotFound", "NoSuchKey", "404":
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
return false, fmt.Errorf("head object %q: %w", normalizedKey, err)
|
||||
}
|
||||
|
||||
func valueOrEmpty(v *string) string {
|
||||
if v == nil {
|
||||
return ""
|
||||
}
|
||||
return *v
|
||||
}
|
||||
|
||||
func valueOrZeroInt64(v *int64) int64 {
|
||||
if v == nil {
|
||||
return 0
|
||||
}
|
||||
return *v
|
||||
}
|
||||
|
||||
func valueOrFalseBool(v *bool) bool {
|
||||
if v == nil {
|
||||
return false
|
||||
}
|
||||
return *v
|
||||
}
|
||||
253
internal/adapters/storage/s3_backend_test.go
Normal file
253
internal/adapters/storage/s3_backend_test.go
Normal file
@@ -0,0 +1,253 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/aws/smithy-go"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
type fakeS3API struct {
|
||||
listOut *s3.ListObjectsV2Output
|
||||
listErr error
|
||||
|
||||
getBody io.ReadCloser
|
||||
getErr error
|
||||
|
||||
putOut *s3.PutObjectOutput
|
||||
putErr error
|
||||
|
||||
headErr error
|
||||
|
||||
lastList *s3.ListObjectsV2Input
|
||||
lastGet *s3.GetObjectInput
|
||||
lastPut *s3.PutObjectInput
|
||||
lastHead *s3.HeadObjectInput
|
||||
}
|
||||
|
||||
func (f *fakeS3API) ListObjectsV2(_ context.Context, params *s3.ListObjectsV2Input, _ ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) {
|
||||
f.lastList = params
|
||||
if f.listErr != nil {
|
||||
return nil, f.listErr
|
||||
}
|
||||
if f.listOut == nil {
|
||||
return &s3.ListObjectsV2Output{}, nil
|
||||
}
|
||||
return f.listOut, nil
|
||||
}
|
||||
|
||||
func (f *fakeS3API) GetObject(_ context.Context, params *s3.GetObjectInput, _ ...func(*s3.Options)) (*s3.GetObjectOutput, error) {
|
||||
f.lastGet = params
|
||||
if f.getErr != nil {
|
||||
return nil, f.getErr
|
||||
}
|
||||
body := f.getBody
|
||||
if body == nil {
|
||||
body = io.NopCloser(strings.NewReader(""))
|
||||
}
|
||||
return &s3.GetObjectOutput{Body: body}, nil
|
||||
}
|
||||
|
||||
func (f *fakeS3API) PutObject(_ context.Context, params *s3.PutObjectInput, _ ...func(*s3.Options)) (*s3.PutObjectOutput, error) {
|
||||
f.lastPut = params
|
||||
if f.putErr != nil {
|
||||
return nil, f.putErr
|
||||
}
|
||||
if f.putOut == nil {
|
||||
return &s3.PutObjectOutput{}, nil
|
||||
}
|
||||
return f.putOut, nil
|
||||
}
|
||||
|
||||
func (f *fakeS3API) HeadObject(_ context.Context, params *s3.HeadObjectInput, _ ...func(*s3.Options)) (*s3.HeadObjectOutput, error) {
|
||||
f.lastHead = params
|
||||
if f.headErr != nil {
|
||||
return nil, f.headErr
|
||||
}
|
||||
return &s3.HeadObjectOutput{}, nil
|
||||
}
|
||||
|
||||
func TestS3BackendListAndKeyNormalization(t *testing.T) {
|
||||
lastModified := time.Date(2026, 5, 16, 12, 0, 0, 0, time.UTC)
|
||||
client := &fakeS3API{
|
||||
listOut: &s3.ListObjectsV2Output{
|
||||
Contents: []types.Object{
|
||||
{Key: strPtr(`dnd\campaigns\forsaken\a.flac`), Size: int64Ptr(7), ETag: strPtr(`"abc"`), LastModified: &lastModified},
|
||||
},
|
||||
},
|
||||
}
|
||||
backend := &S3Backend{bucket: "bucket-1", client: client}
|
||||
|
||||
items, err := backend.List(context.Background(), `dnd\campaigns\`)
|
||||
if err != nil {
|
||||
t.Fatalf("List() error = %v", err)
|
||||
}
|
||||
if len(items) != 1 {
|
||||
t.Fatalf("List() len = %d, want 1", len(items))
|
||||
}
|
||||
if items[0].Key != "dnd/campaigns/forsaken/a.flac" {
|
||||
t.Fatalf("List() key = %q, want normalized slash key", items[0].Key)
|
||||
}
|
||||
if items[0].ETag != "abc" {
|
||||
t.Fatalf("List() ETag = %q, want %q", items[0].ETag, "abc")
|
||||
}
|
||||
if client.lastList == nil || *client.lastList.Prefix != "dnd/campaigns/" {
|
||||
t.Fatalf("List() prefix = %#v, want normalized prefix", client.lastList)
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3BackendDownloadCreatesParentDirectory(t *testing.T) {
|
||||
client := &fakeS3API{getBody: io.NopCloser(strings.NewReader("audio"))}
|
||||
backend := &S3Backend{bucket: "bucket-1", client: client}
|
||||
|
||||
dst := filepath.Join(t.TempDir(), "nested", "clip.flac")
|
||||
if err := backend.Download(context.Background(), `audio\clip.flac`, dst); err != nil {
|
||||
t.Fatalf("Download() error = %v", err)
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(dst)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile() error = %v", err)
|
||||
}
|
||||
if string(data) != "audio" {
|
||||
t.Fatalf("downloaded content = %q, want %q", string(data), "audio")
|
||||
}
|
||||
if client.lastGet == nil || *client.lastGet.Key != "audio/clip.flac" {
|
||||
t.Fatalf("GetObject key = %#v, want normalized key", client.lastGet)
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3BackendUploadAndExists(t *testing.T) {
|
||||
client := &fakeS3API{putOut: &s3.PutObjectOutput{ETag: strPtr(`"etag123"`)}}
|
||||
backend := &S3Backend{bucket: "bucket-1", client: client}
|
||||
|
||||
local := filepath.Join(t.TempDir(), "artifact.txt")
|
||||
if err := os.WriteFile(local, []byte("artifact"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile() error = %v", err)
|
||||
}
|
||||
|
||||
info, err := backend.Upload(context.Background(), local, `runs\id\artifact.txt`, UploadOptions{
|
||||
Metadata: map[string]string{"kind": "artifact"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Upload() error = %v", err)
|
||||
}
|
||||
if info.Key != "runs/id/artifact.txt" {
|
||||
t.Fatalf("Upload key = %q, want normalized key", info.Key)
|
||||
}
|
||||
if info.ETag != "etag123" {
|
||||
t.Fatalf("Upload ETag = %q, want %q", info.ETag, "etag123")
|
||||
}
|
||||
if client.lastPut == nil || *client.lastPut.Key != "runs/id/artifact.txt" {
|
||||
t.Fatalf("PutObject key = %#v, want normalized key", client.lastPut)
|
||||
}
|
||||
|
||||
ok, err := backend.Exists(context.Background(), "runs/id/artifact.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("Exists() error = %v", err)
|
||||
}
|
||||
if !ok {
|
||||
t.Fatal("Exists() = false, want true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3BackendUploadMissingLocalFile(t *testing.T) {
|
||||
backend := &S3Backend{bucket: "bucket-1", client: &fakeS3API{}}
|
||||
_, err := backend.Upload(context.Background(), filepath.Join(t.TempDir(), "missing.txt"), "key.txt", UploadOptions{})
|
||||
if err == nil || !strings.Contains(err.Error(), "no such file") {
|
||||
t.Fatalf("Upload() error = %v, want missing local file error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3BackendExistsNotFound(t *testing.T) {
|
||||
backend := &S3Backend{
|
||||
bucket: "bucket-1",
|
||||
client: &fakeS3API{
|
||||
headErr: &smithy.GenericAPIError{Code: "NotFound", Message: "missing"},
|
||||
},
|
||||
}
|
||||
ok, err := backend.Exists(context.Background(), "missing-key")
|
||||
if err != nil {
|
||||
t.Fatalf("Exists() error = %v", err)
|
||||
}
|
||||
if ok {
|
||||
t.Fatal("Exists() = true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewS3BackendFromConfigUsesClientOptions(t *testing.T) {
|
||||
original := newS3Client
|
||||
t.Cleanup(func() { newS3Client = original })
|
||||
t.Setenv("OBJECT_STORAGE_KEY_ID", "id-123")
|
||||
t.Setenv("OBJECT_STORAGE_KEY", "secret-abc")
|
||||
|
||||
var got s3ClientOptions
|
||||
newS3Client = func(_ context.Context, opts s3ClientOptions) (s3API, error) {
|
||||
got = opts
|
||||
return &fakeS3API{}, nil
|
||||
}
|
||||
|
||||
backend, err := NewS3BackendFromConfig(context.Background(), config.StorageS3Config{
|
||||
Bucket: "my-archive",
|
||||
Region: "us-east-1",
|
||||
Endpoint: "http://localhost:9000",
|
||||
ForcePathStyle: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewS3BackendFromConfig() error = %v", err)
|
||||
}
|
||||
if backend.bucket != "my-archive" {
|
||||
t.Fatalf("backend.bucket = %q, want %q", backend.bucket, "my-archive")
|
||||
}
|
||||
if got.Region != "us-east-1" || got.Endpoint != "http://localhost:9000" || !got.ForcePathStyle {
|
||||
t.Fatalf("client options = %#v, want region/endpoint/path-style values", got)
|
||||
}
|
||||
if got.AccessKeyID != "id-123" || got.SecretKey != "secret-abc" {
|
||||
t.Fatalf("client options credentials = %#v, want env-resolved static credentials", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewS3BackendFromConfigRequiresBucket(t *testing.T) {
|
||||
_, err := NewS3BackendFromConfig(context.Background(), config.StorageS3Config{})
|
||||
if err == nil || !strings.Contains(err.Error(), "bucket is required") {
|
||||
t.Fatalf("NewS3BackendFromConfig() error = %v, want bucket validation", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewS3BackendFromConfigFallsBackWhenCredentialEnvMissing(t *testing.T) {
|
||||
original := newS3Client
|
||||
t.Cleanup(func() { newS3Client = original })
|
||||
|
||||
var got s3ClientOptions
|
||||
newS3Client = func(_ context.Context, opts s3ClientOptions) (s3API, error) {
|
||||
got = opts
|
||||
return &fakeS3API{}, nil
|
||||
}
|
||||
|
||||
_, err := NewS3BackendFromConfig(context.Background(), config.StorageS3Config{
|
||||
Bucket: "my-archive",
|
||||
Region: "us-east-1",
|
||||
AccessKeyIDEnv: "MISSING_ACCESS_KEY_ID",
|
||||
SecretKeyEnv: "MISSING_SECRET_KEY",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewS3BackendFromConfig() error = %v", err)
|
||||
}
|
||||
if got.AccessKeyID != "" || got.SecretKey != "" {
|
||||
t.Fatalf("client options credentials = %#v, want empty fallback values", got)
|
||||
}
|
||||
}
|
||||
|
||||
func strPtr(v string) *string { return &v }
|
||||
func int64Ptr(v int64) *int64 { return &v }
|
||||
|
||||
var _ s3API = (*fakeS3API)(nil)
|
||||
36
internal/adapters/storage/temp_download.go
Normal file
36
internal/adapters/storage/temp_download.go
Normal file
@@ -0,0 +1,36 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// DownloadObjectToTemp downloads an object into a temporary file and returns
|
||||
// the cleaned local path.
|
||||
func DownloadObjectToTemp(ctx context.Context, store ObjectStore, key, pattern string) (string, error) {
|
||||
if store == nil {
|
||||
return "", fmt.Errorf("object store is required")
|
||||
}
|
||||
if strings.TrimSpace(pattern) == "" {
|
||||
return "", fmt.Errorf("temp file pattern is required")
|
||||
}
|
||||
|
||||
tmp, err := os.CreateTemp("", pattern)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create temp file: %w", err)
|
||||
}
|
||||
path := tmp.Name()
|
||||
if err := tmp.Close(); err != nil {
|
||||
_ = os.Remove(path)
|
||||
return "", fmt.Errorf("close temp file: %w", err)
|
||||
}
|
||||
|
||||
if err := store.Download(ctx, key, path); err != nil {
|
||||
_ = os.Remove(path)
|
||||
return "", err
|
||||
}
|
||||
return filepath.Clean(path), nil
|
||||
}
|
||||
69
internal/adapters/storage/temp_download_test.go
Normal file
69
internal/adapters/storage/temp_download_test.go
Normal file
@@ -0,0 +1,69 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDownloadObjectToTempSuccess(t *testing.T) {
|
||||
store := &FakeBackend{}
|
||||
store.SeedObject(FakeObject{Key: "sessions/a/current/run_id.txt", Data: []byte("run-123\n")})
|
||||
|
||||
path, err := DownloadObjectToTemp(context.Background(), store, "sessions/a/current/run_id.txt", "narratio-test-*.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("DownloadObjectToTemp() error = %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = os.Remove(path) })
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile() error = %v", err)
|
||||
}
|
||||
if string(data) != "run-123\n" {
|
||||
t.Fatalf("downloaded data = %q, want %q", string(data), "run-123\n")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadObjectToTempFailedDownloadRemovesTempFile(t *testing.T) {
|
||||
sentinel := errors.New("download failed")
|
||||
store := &FakeBackend{DownloadErr: sentinel}
|
||||
pattern := "narratio-test-fail-*.txt"
|
||||
before, err := filepath.Glob(filepath.Join(os.TempDir(), "narratio-test-fail-*.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("Glob(before) error = %v", err)
|
||||
}
|
||||
|
||||
path, err := DownloadObjectToTemp(context.Background(), store, "sessions/a/current/run_id.txt", pattern)
|
||||
if !errors.Is(err, sentinel) {
|
||||
t.Fatalf("DownloadObjectToTemp() error = %v, want %v", err, sentinel)
|
||||
}
|
||||
if strings.TrimSpace(path) != "" {
|
||||
t.Fatalf("DownloadObjectToTemp() path = %q, want empty on failure", path)
|
||||
}
|
||||
after, err := filepath.Glob(filepath.Join(os.TempDir(), "narratio-test-fail-*.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("Glob(after) error = %v", err)
|
||||
}
|
||||
if len(after) != len(before) {
|
||||
t.Fatalf("temp file count changed after failed download: before=%d after=%d", len(before), len(after))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadObjectToTempCallerContextWrappingPreservesCause(t *testing.T) {
|
||||
sentinel := errors.New("object missing")
|
||||
store := &FakeBackend{DownloadErr: sentinel}
|
||||
|
||||
_, err := DownloadObjectToTemp(context.Background(), store, "sessions/a/current/run_id.txt", "narratio-test-*.txt")
|
||||
if err == nil {
|
||||
t.Fatal("DownloadObjectToTemp() error = nil, want error")
|
||||
}
|
||||
err = fmt.Errorf("download run pointer failed: %w", err)
|
||||
if !errors.Is(err, sentinel) {
|
||||
t.Fatalf("wrapped error does not preserve sentinel cause: %v", err)
|
||||
}
|
||||
}
|
||||
66
internal/app/analyze_artifacts.go
Normal file
66
internal/app/analyze_artifacts.go
Normal file
@@ -0,0 +1,66 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
type artifactSelectionFlag struct {
|
||||
values []string
|
||||
}
|
||||
|
||||
func (f *artifactSelectionFlag) String() string {
|
||||
return strings.Join(f.values, ",")
|
||||
}
|
||||
|
||||
func (f *artifactSelectionFlag) Set(value string) error {
|
||||
f.values = append(f.values, value)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *artifactSelectionFlag) Normalize() ([]string, error) {
|
||||
if len(f.values) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
seen := map[string]struct{}{}
|
||||
out := make([]string, 0, len(f.values))
|
||||
for _, raw := range f.values {
|
||||
for _, part := range strings.Split(raw, ",") {
|
||||
name := strings.TrimSpace(part)
|
||||
if name == "" {
|
||||
return nil, fmt.Errorf("artifact names must be non-empty")
|
||||
}
|
||||
if _, ok := seen[name]; ok {
|
||||
continue
|
||||
}
|
||||
seen[name] = struct{}{}
|
||||
out = append(out, name)
|
||||
}
|
||||
}
|
||||
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func validateSelectedArtifacts(cfg *config.Config, selected []string) error {
|
||||
if len(selected) == 0 {
|
||||
return nil
|
||||
}
|
||||
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Scriptorium == nil {
|
||||
return fmt.Errorf("--artifacts requires pipeline.scriptorium.artifacts to be configured")
|
||||
}
|
||||
configured := cfg.Pipeline.Scriptorium.Artifacts
|
||||
if len(configured) == 0 {
|
||||
return fmt.Errorf("--artifacts requires at least one configured artifact in pipeline.scriptorium.artifacts")
|
||||
}
|
||||
for _, name := range selected {
|
||||
if _, ok := configured[name]; !ok {
|
||||
return fmt.Errorf("--artifacts includes unknown artifact %q", name)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
434
internal/app/analyze_artifacts_commands_test.go
Normal file
434
internal/app/analyze_artifacts_commands_test.go
Normal file
@@ -0,0 +1,434 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/stage"
|
||||
)
|
||||
|
||||
func TestExecuteRunStageArtifactsUnsupportedStageFails(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(
|
||||
[]string{"run-stage", "polish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "session_recap"},
|
||||
&stdout,
|
||||
&stderr,
|
||||
)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), `run-stage: --artifacts is only supported for stages "analyze" and "publish"`) {
|
||||
t.Fatalf("stderr = %q, want stage-gating error", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRunStagePublishPropagatesSelectedArtifacts(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
|
||||
var capturedStages []string
|
||||
var capturedArtifacts []string
|
||||
origExecuteStagesFn := executeStagesFn
|
||||
t.Cleanup(func() {
|
||||
executeStagesFn = origExecuteStagesFn
|
||||
})
|
||||
executeStagesFn = func(_ context.Context, _ *config.Config, stages []stage.Stage, opts RunOptions) (*RunSummary, error) {
|
||||
for _, s := range stages {
|
||||
capturedStages = append(capturedStages, s.Name())
|
||||
}
|
||||
capturedArtifacts = append([]string(nil), opts.SelectedArtifacts...)
|
||||
return &RunSummary{ManifestPath: filepath.Join(workspaceRoot, "manifest.json"), Executed: []string{"publish"}}, nil
|
||||
}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(
|
||||
[]string{
|
||||
"run-stage", "publish", "2026-05-03",
|
||||
"--config", pipelinePath,
|
||||
"--campaign-file", campaignPath,
|
||||
"--session", sessionPath,
|
||||
"--artifacts", "session_recap",
|
||||
},
|
||||
&stdout,
|
||||
&stderr,
|
||||
)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if len(capturedStages) != 1 || capturedStages[0] != "publish" {
|
||||
t.Fatalf("captured stages = %#v, want [publish]", capturedStages)
|
||||
}
|
||||
if strings.Join(capturedArtifacts, ",") != "session_recap" {
|
||||
t.Fatalf("captured artifacts = %#v, want [session_recap]", capturedArtifacts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteUnknownArtifactsFailValidation(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(
|
||||
[]string{"run", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "unknown_artifact"},
|
||||
&stdout,
|
||||
&stderr,
|
||||
)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), `run: --artifacts includes unknown artifact "unknown_artifact"`) {
|
||||
t.Fatalf("stderr = %q, want unknown-artifact validation error", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunStageArtifactsDoesNotImplyForce(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||
|
||||
store := &manifest.LocalStore{}
|
||||
seed := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||
seed.MarkStageSucceeded("analyze", time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||
if err := store.Save(context.Background(), manifestPath, seed); err != nil {
|
||||
t.Fatalf("save manifest: %v", err)
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
err := RunStage(
|
||||
context.Background(),
|
||||
[]string{"analyze", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "session_recap,session_recap"},
|
||||
&out,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("RunStage() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(out.String(), "stage=analyze executed=0 skipped=1 force=false") {
|
||||
t.Fatalf("output = %q, want analyze skip without force", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestResumeArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||
|
||||
store := &manifest.LocalStore{}
|
||||
seed := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze", "publish", "notify"} {
|
||||
seed.MarkStageSucceeded(stageName, time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||
}
|
||||
if err := store.Save(context.Background(), manifestPath, seed); err != nil {
|
||||
t.Fatalf("save manifest: %v", err)
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
err := Resume(
|
||||
context.Background(),
|
||||
[]string{"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "session_recap"},
|
||||
&out,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("Resume() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(out.String(), "has no remaining stages") {
|
||||
t.Fatalf("output = %q, want no remaining stages", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteAnalyzeForceRunsAnalyze(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
|
||||
var capturedStages []string
|
||||
var capturedForce bool
|
||||
origExecuteStagesFn := executeStagesFn
|
||||
t.Cleanup(func() {
|
||||
executeStagesFn = origExecuteStagesFn
|
||||
})
|
||||
executeStagesFn = func(_ context.Context, _ *config.Config, stages []stage.Stage, opts RunOptions) (*RunSummary, error) {
|
||||
for _, s := range stages {
|
||||
capturedStages = append(capturedStages, s.Name())
|
||||
}
|
||||
capturedForce = opts.Force
|
||||
return &RunSummary{
|
||||
ManifestPath: filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json"),
|
||||
Executed: []string{"analyze"},
|
||||
}, nil
|
||||
}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(
|
||||
[]string{"analyze", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath},
|
||||
&stdout,
|
||||
&stderr,
|
||||
)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if len(capturedStages) != 1 || capturedStages[0] != "analyze" {
|
||||
t.Fatalf("captured stages = %#v, want [analyze]", capturedStages)
|
||||
}
|
||||
if !capturedForce {
|
||||
t.Fatal("captured force = false, want true")
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "narratio analyze: executed=1 skipped=0 force=true; manifest=") {
|
||||
t.Fatalf("stdout = %q, want analyze summary", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteAnalyzePropagatesSelectedArtifacts(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
|
||||
var capturedArtifacts []string
|
||||
origExecuteStagesFn := executeStagesFn
|
||||
t.Cleanup(func() {
|
||||
executeStagesFn = origExecuteStagesFn
|
||||
})
|
||||
executeStagesFn = func(_ context.Context, _ *config.Config, _ []stage.Stage, opts RunOptions) (*RunSummary, error) {
|
||||
capturedArtifacts = append([]string(nil), opts.SelectedArtifacts...)
|
||||
return &RunSummary{ManifestPath: filepath.Join(workspaceRoot, "manifest.json"), Executed: []string{"analyze"}}, nil
|
||||
}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(
|
||||
[]string{
|
||||
"analyze",
|
||||
"2026-05-03",
|
||||
"--config", pipelinePath,
|
||||
"--campaign-file", campaignPath,
|
||||
"--session", sessionPath,
|
||||
"--artifacts", "player_handout,session_recap",
|
||||
},
|
||||
&stdout,
|
||||
&stderr,
|
||||
)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if strings.Join(capturedArtifacts, ",") != "player_handout,session_recap" {
|
||||
t.Fatalf("captured artifacts = %#v, want sorted selected artifacts", capturedArtifacts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteAnalyzeUnknownArtifactFailsValidation(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(
|
||||
[]string{"analyze", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "unknown_artifact"},
|
||||
&stdout,
|
||||
&stderr,
|
||||
)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), `analyze: --artifacts includes unknown artifact "unknown_artifact"`) {
|
||||
t.Fatalf("stderr = %q, want unknown-artifact validation error", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteAnalyzeRejectsPositionalArgsAndForceFlag(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
args []string
|
||||
want string
|
||||
}{
|
||||
{name: "extra positional", args: []string{"analyze", "2026-05-03", "extra"}, want: "analyze: unexpected positional arguments"},
|
||||
{name: "force flag", args: []string{"analyze", "--force"}, want: "analyze: invalid flags: flag provided but not defined: -force"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(tc.args, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), tc.want) {
|
||||
t.Fatalf("stderr = %q, want %q", stderr.String(), tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteAnalyzeMissingConfigUsesRunStageLoadingPath(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"analyze", "2026-05-03"}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "analyze: no pipeline config path provided and no default pipeline config found; searched:") {
|
||||
t.Fatalf("stderr = %q, want pipeline discovery error", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutePublishForceRunsPublish(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
|
||||
var capturedStages []string
|
||||
var capturedForce bool
|
||||
var capturedArtifacts []string
|
||||
origExecuteStagesFn := executeStagesFn
|
||||
t.Cleanup(func() {
|
||||
executeStagesFn = origExecuteStagesFn
|
||||
})
|
||||
executeStagesFn = func(_ context.Context, _ *config.Config, stages []stage.Stage, opts RunOptions) (*RunSummary, error) {
|
||||
for _, s := range stages {
|
||||
capturedStages = append(capturedStages, s.Name())
|
||||
}
|
||||
capturedForce = opts.Force
|
||||
capturedArtifacts = append([]string(nil), opts.SelectedArtifacts...)
|
||||
return &RunSummary{
|
||||
ManifestPath: filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json"),
|
||||
Executed: []string{"publish"},
|
||||
}, nil
|
||||
}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(
|
||||
[]string{"publish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "session_recap"},
|
||||
&stdout,
|
||||
&stderr,
|
||||
)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if len(capturedStages) != 1 || capturedStages[0] != "publish" {
|
||||
t.Fatalf("captured stages = %#v, want [publish]", capturedStages)
|
||||
}
|
||||
if !capturedForce {
|
||||
t.Fatal("captured force = false, want true")
|
||||
}
|
||||
if strings.Join(capturedArtifacts, ",") != "session_recap" {
|
||||
t.Fatalf("captured artifacts = %#v, want [session_recap]", capturedArtifacts)
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "narratio publish: executed=1 skipped=0 force=true; manifest=") {
|
||||
t.Fatalf("stdout = %q, want publish summary", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutePublishRejectsUnsupportedArgsAndFlags(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
args []string
|
||||
want string
|
||||
}{
|
||||
{name: "extra positional", args: []string{"publish", "2026-05-03", "extra"}, want: "publish: unexpected positional arguments"},
|
||||
{name: "force flag", args: []string{"publish", "--force"}, want: "publish: invalid flags: flag provided but not defined: -force"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(tc.args, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), tc.want) {
|
||||
t.Fatalf("stderr = %q, want %q", stderr.String(), tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutePublishUnknownArtifactFailsValidation(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(
|
||||
[]string{"publish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "unknown_artifact"},
|
||||
&stdout,
|
||||
&stderr,
|
||||
)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), `publish: --artifacts includes unknown artifact "unknown_artifact"`) {
|
||||
t.Fatalf("stderr = %q, want unknown-artifact validation error", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutePublishMissingConfigUsesRunStageLoadingPath(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"publish", "2026-05-03"}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "publish: no pipeline config path provided and no default pipeline config found; searched:") {
|
||||
t.Fatalf("stderr = %q, want pipeline discovery error", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteUsageIncludesAnalyzeAndPublish(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(nil, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "analyze") {
|
||||
t.Fatalf("stderr = %q, want usage to include analyze", stderr.String())
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "publish") {
|
||||
t.Fatalf("stderr = %q, want usage to include publish", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func writeValidConfigFilesWithScriptoriumArtifacts(t *testing.T, workspaceRoot string) (string, string, string) {
|
||||
t.Helper()
|
||||
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
f, err := os.OpenFile(pipelinePath, os.O_APPEND|os.O_WRONLY, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("open pipeline config for append: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
extra := `
|
||||
scriptorium:
|
||||
binary: scriptorium
|
||||
artifacts:
|
||||
session_recap:
|
||||
enabled: true
|
||||
prompt_id: dnd.session_recap
|
||||
output_path: artifacts/session_recap.md
|
||||
player_handout:
|
||||
enabled: true
|
||||
prompt_id: dnd.player_handout
|
||||
output_path: artifacts/player_handout.md
|
||||
depends_on:
|
||||
- session_recap
|
||||
inputs:
|
||||
recap:
|
||||
source: narratio.artifact.session_recap
|
||||
required: true
|
||||
`
|
||||
if _, err := f.WriteString(extra); err != nil {
|
||||
t.Fatalf("append scriptorium config: %v", err)
|
||||
}
|
||||
return pipelinePath, campaignPath, sessionPath
|
||||
}
|
||||
132
internal/app/analyze_artifacts_test.go
Normal file
132
internal/app/analyze_artifacts_test.go
Normal file
@@ -0,0 +1,132 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
func TestArtifactSelectionFlagNormalize(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
inputs []string
|
||||
want []string
|
||||
wantErr string
|
||||
}{
|
||||
{
|
||||
name: "single value",
|
||||
inputs: []string{"session_recap"},
|
||||
want: []string{"session_recap"},
|
||||
},
|
||||
{
|
||||
name: "repeatable and comma separated values are deduped and sorted",
|
||||
inputs: []string{"session_recap,player_handout", "session_recap"},
|
||||
want: []string{"player_handout", "session_recap"},
|
||||
},
|
||||
{
|
||||
name: "empty token fails",
|
||||
inputs: []string{"session_recap,"},
|
||||
wantErr: "artifact names must be non-empty",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
var flag artifactSelectionFlag
|
||||
for _, in := range tt.inputs {
|
||||
if err := flag.Set(in); err != nil {
|
||||
t.Fatalf("Set(%q) error = %v", in, err)
|
||||
}
|
||||
}
|
||||
|
||||
got, err := flag.Normalize()
|
||||
if tt.wantErr != "" {
|
||||
if err == nil {
|
||||
t.Fatalf("Normalize() error = nil, want %q", tt.wantErr)
|
||||
}
|
||||
if err.Error() != tt.wantErr {
|
||||
t.Fatalf("Normalize() error = %q, want %q", err.Error(), tt.wantErr)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("Normalize() error = %v", err)
|
||||
}
|
||||
if len(got) != len(tt.want) {
|
||||
t.Fatalf("Normalize() len = %d, want %d; got=%v", len(got), len(tt.want), got)
|
||||
}
|
||||
for i := range got {
|
||||
if got[i] != tt.want[i] {
|
||||
t.Fatalf("Normalize()[%d] = %q, want %q", i, got[i], tt.want[i])
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateSelectedArtifacts(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
cfg *config.Config
|
||||
selected []string
|
||||
wantErr string
|
||||
}{
|
||||
{
|
||||
name: "empty selection is accepted",
|
||||
cfg: &config.Config{},
|
||||
selected: nil,
|
||||
},
|
||||
{
|
||||
name: "scriptorium required when selected artifacts present",
|
||||
cfg: &config.Config{Pipeline: &config.PipelineConfig{}},
|
||||
selected: []string{"session_recap"},
|
||||
wantErr: "--artifacts requires pipeline.scriptorium.artifacts to be configured",
|
||||
},
|
||||
{
|
||||
name: "unknown selected artifact fails",
|
||||
cfg: &config.Config{
|
||||
Pipeline: &config.PipelineConfig{
|
||||
Scriptorium: &config.ScriptoriumConfig{
|
||||
Artifacts: map[string]config.ScriptoriumArtifactConfig{
|
||||
"session_recap": {Enabled: true, PromptID: "dnd.session_recap", OutputPath: "artifacts/session_recap.md"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
selected: []string{"player_handout"},
|
||||
wantErr: `--artifacts includes unknown artifact "player_handout"`,
|
||||
},
|
||||
{
|
||||
name: "known selected artifacts are accepted",
|
||||
cfg: &config.Config{
|
||||
Pipeline: &config.PipelineConfig{
|
||||
Scriptorium: &config.ScriptoriumConfig{
|
||||
Artifacts: map[string]config.ScriptoriumArtifactConfig{
|
||||
"session_recap": {Enabled: true, PromptID: "dnd.session_recap", OutputPath: "artifacts/session_recap.md"},
|
||||
"player_handout": {Enabled: true, PromptID: "dnd.player_handout", OutputPath: "artifacts/player_handout.md"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
selected: []string{"player_handout", "session_recap"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := validateSelectedArtifacts(tt.cfg, tt.selected)
|
||||
if tt.wantErr != "" {
|
||||
if err == nil {
|
||||
t.Fatalf("error = nil, want %q", tt.wantErr)
|
||||
}
|
||||
if err.Error() != tt.wantErr {
|
||||
t.Fatalf("error = %q, want %q", err.Error(), tt.wantErr)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("error = %v, want nil", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
44
internal/app/campaign_config_path.go
Normal file
44
internal/app/campaign_config_path.go
Normal file
@@ -0,0 +1,44 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
func resolveCampaignConfigPath(pipelineCfg *config.PipelineConfig, campaignIDFlag, campaignFileFlag string) (string, error) {
|
||||
campaignID := strings.TrimSpace(campaignIDFlag)
|
||||
campaignFile := strings.TrimSpace(campaignFileFlag)
|
||||
if campaignID != "" && campaignFile != "" {
|
||||
return "", fmt.Errorf("--campaign and --campaign-file are mutually exclusive")
|
||||
}
|
||||
if campaignFile != "" {
|
||||
return filepath.Clean(campaignFile), nil
|
||||
}
|
||||
if campaignID == "" && pipelineCfg != nil {
|
||||
campaignID = strings.TrimSpace(pipelineCfg.Campaigns.DefaultCampaignID)
|
||||
}
|
||||
if campaignID == "" {
|
||||
return "", fmt.Errorf("no campaign selected; pass --campaign <id> or set pipeline.campaigns.default_campaign_id")
|
||||
}
|
||||
if err := validateCampaignIDToken(campaignID); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if pipelineCfg == nil || strings.TrimSpace(pipelineCfg.Campaigns.Root) == "" {
|
||||
return "", fmt.Errorf("pipeline.campaigns.root is required to select campaign %q", campaignID)
|
||||
}
|
||||
return filepath.Clean(filepath.Join(pipelineCfg.Campaigns.Root, campaignID, "campaign.yml")), nil
|
||||
}
|
||||
|
||||
func validateCampaignIDToken(campaignID string) error {
|
||||
if filepath.IsAbs(campaignID) ||
|
||||
strings.Contains(campaignID, "/") ||
|
||||
strings.Contains(campaignID, `\`) ||
|
||||
campaignID == "." ||
|
||||
campaignID == ".." {
|
||||
return fmt.Errorf("campaign id %q must be a single path segment", campaignID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
84
internal/app/campaign_config_path_test.go
Normal file
84
internal/app/campaign_config_path_test.go
Normal file
@@ -0,0 +1,84 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
func TestResolveCampaignConfigPathCampaignFileWins(t *testing.T) {
|
||||
explicit := filepath.Join(t.TempDir(), "custom-campaign.yml")
|
||||
got, err := resolveCampaignConfigPath(&config.PipelineConfig{}, "", explicit)
|
||||
if err != nil {
|
||||
t.Fatalf("resolveCampaignConfigPath() error = %v", err)
|
||||
}
|
||||
if got != explicit {
|
||||
t.Fatalf("path = %q, want explicit path %q", got, explicit)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveCampaignConfigPathUsesSelectedCampaignID(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
pipelineCfg := &config.PipelineConfig{}
|
||||
pipelineCfg.Campaigns.Root = dir
|
||||
|
||||
got, err := resolveCampaignConfigPath(pipelineCfg, "icewind", "")
|
||||
if err != nil {
|
||||
t.Fatalf("resolveCampaignConfigPath() error = %v", err)
|
||||
}
|
||||
want := filepath.Join(dir, "icewind", "campaign.yml")
|
||||
if got != filepath.Clean(want) {
|
||||
t.Fatalf("path = %q, want %q", got, filepath.Clean(want))
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveCampaignConfigPathUsesDefaultCampaignID(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
pipelineCfg := &config.PipelineConfig{}
|
||||
pipelineCfg.Campaigns.Root = dir
|
||||
pipelineCfg.Campaigns.DefaultCampaignID = "dilfs"
|
||||
|
||||
got, err := resolveCampaignConfigPath(pipelineCfg, "", "")
|
||||
if err != nil {
|
||||
t.Fatalf("resolveCampaignConfigPath() error = %v", err)
|
||||
}
|
||||
want := filepath.Join(dir, "dilfs", "campaign.yml")
|
||||
if got != filepath.Clean(want) {
|
||||
t.Fatalf("path = %q, want %q", got, filepath.Clean(want))
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveCampaignConfigPathRejectsCampaignIDAndFile(t *testing.T) {
|
||||
_, err := resolveCampaignConfigPath(&config.PipelineConfig{}, "dilfs", filepath.Join(t.TempDir(), "campaign.yml"))
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "mutually exclusive") {
|
||||
t.Fatalf("error = %q, want mutual exclusion", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveCampaignConfigPathRequiresCampaignSelection(t *testing.T) {
|
||||
_, err := resolveCampaignConfigPath(&config.PipelineConfig{}, "", "")
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "no campaign selected") {
|
||||
t.Fatalf("error = %q, want missing selection guidance", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveCampaignConfigPathRejectsPathLikeCampaignID(t *testing.T) {
|
||||
pipelineCfg := &config.PipelineConfig{}
|
||||
pipelineCfg.Campaigns.Root = t.TempDir()
|
||||
|
||||
_, err := resolveCampaignConfigPath(pipelineCfg, "../icewind", "")
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "single path segment") {
|
||||
t.Fatalf("error = %q, want path segment guidance", err.Error())
|
||||
}
|
||||
}
|
||||
281
internal/app/clean.go
Normal file
281
internal/app/clean.go
Normal file
@@ -0,0 +1,281 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
// Clean removes local workspace/spool state while preserving durable cache
|
||||
// state unless cache cleanup is explicitly requested.
|
||||
func Clean(ctx context.Context, args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("clean", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
var flags commonConfigFlags
|
||||
var all bool
|
||||
var dryRun bool
|
||||
var clearCache bool
|
||||
addCommonConfigFlags(fs, &flags)
|
||||
fs.BoolVar(&all, "all", false, "clean all local session work/spool state")
|
||||
fs.BoolVar(&dryRun, "dry-run", false, "print cleanup targets without deleting")
|
||||
fs.BoolVar(&clearCache, "clear-cache", false, "also clear durable S3 audio cache entries")
|
||||
if err := parseSessionAwareFlags("clean", fs, args, &flags.sessionID); err != nil {
|
||||
return err
|
||||
}
|
||||
if all {
|
||||
return cleanAllLocal(flags, dryRun, clearCache, out)
|
||||
}
|
||||
return cleanSession(ctx, flags, dryRun, clearCache, out)
|
||||
}
|
||||
|
||||
func cleanSession(ctx context.Context, flags commonConfigFlags, dryRun, clearCache bool, out io.Writer) error {
|
||||
if strings.TrimSpace(flags.sessionID) == "" {
|
||||
return fmt.Errorf("clean: session_id is required unless --all is set")
|
||||
}
|
||||
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
||||
if err != nil {
|
||||
return fmt.Errorf("clean: %w", err)
|
||||
}
|
||||
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
|
||||
return fmt.Errorf("clean: resolved pipeline and session config are required")
|
||||
}
|
||||
campaign := strings.TrimSpace(cfg.Session.Campaign)
|
||||
sessionID := strings.TrimSpace(cfg.Session.SessionID)
|
||||
if campaign == "" || sessionID == "" {
|
||||
return fmt.Errorf("clean: campaign and session_id are required")
|
||||
}
|
||||
|
||||
if dryRun {
|
||||
fmt.Fprintf(out, "Clean plan for %s/%s\n", campaign, sessionID)
|
||||
} else {
|
||||
fmt.Fprintf(out, "Cleaned %s/%s\n", campaign, sessionID)
|
||||
}
|
||||
|
||||
workDir := artifacts.SessionWorkDirForCampaign(cfg.Pipeline.Workspace.Root, campaign, sessionID)
|
||||
spoolDir := artifacts.SessionSpoolDir(cfg.Pipeline.Spool.Root, campaign, sessionID)
|
||||
if err := reportCleanScopedDir(out, cfg.Pipeline.Workspace.Root, workDir, "clean.workspace.session", dryRun); err != nil {
|
||||
return fmt.Errorf("clean: %w", err)
|
||||
}
|
||||
if err := reportCleanScopedDir(out, cfg.Pipeline.Spool.Root, spoolDir, "clean.spool.session", dryRun); err != nil {
|
||||
return fmt.Errorf("clean: %w", err)
|
||||
}
|
||||
|
||||
if clearCache {
|
||||
if err := cleanSessionAudioCache(ctx, cfg, dryRun, out); err != nil {
|
||||
return fmt.Errorf("clean: %w", err)
|
||||
}
|
||||
} else {
|
||||
fmt.Fprintln(out, "Cache: preserved")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func cleanAllLocal(flags commonConfigFlags, dryRun, clearCache bool, out io.Writer) error {
|
||||
if strings.TrimSpace(flags.campaignPath) != "" ||
|
||||
strings.TrimSpace(flags.campaignFilePath) != "" ||
|
||||
strings.TrimSpace(flags.sessionPath) != "" ||
|
||||
strings.TrimSpace(flags.sessionID) != "" ||
|
||||
strings.TrimSpace(flags.previousSessionID) != "" {
|
||||
return fmt.Errorf("clean: --all cannot be combined with --campaign, --campaign-file, --session, a session_id, or --previous-session-id")
|
||||
}
|
||||
resolvedPipelinePath, err := resolvePipelineConfigPath(flags.pipelinePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("clean: %w", err)
|
||||
}
|
||||
pipelineCfg, err := config.LoadPipeline(resolvedPipelinePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("clean: %w", err)
|
||||
}
|
||||
|
||||
if dryRun {
|
||||
fmt.Fprintln(out, "Clean plan for all local sessions")
|
||||
} else {
|
||||
fmt.Fprintln(out, "Cleaned all local sessions")
|
||||
}
|
||||
|
||||
workRoot := filepath.Join(pipelineCfg.Workspace.Root, config.PathWorkDirSegment)
|
||||
if err := reportCleanScopedDir(out, pipelineCfg.Workspace.Root, workRoot, "clean.workspace.all", dryRun); err != nil {
|
||||
return fmt.Errorf("clean: %w", err)
|
||||
}
|
||||
if err := reportCleanRootChildren(out, pipelineCfg.Spool.Root, "clean.spool.all", dryRun); err != nil {
|
||||
return fmt.Errorf("clean: %w", err)
|
||||
}
|
||||
|
||||
if clearCache {
|
||||
if err := cleanAllAudioCache(pipelineCfg, dryRun, out); err != nil {
|
||||
return fmt.Errorf("clean: %w", err)
|
||||
}
|
||||
} else {
|
||||
fmt.Fprintln(out, "Cache: preserved")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func reportCleanScopedDir(out io.Writer, root, target, policy string, dryRun bool) error {
|
||||
dir, err := validateScopedDir(root, target, policy)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if dryRun {
|
||||
if dir.Exists {
|
||||
fmt.Fprintf(out, "Would delete: %s\n", dir.TargetAbs)
|
||||
} else {
|
||||
fmt.Fprintf(out, "Would skip missing: %s\n", dir.TargetAbs)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if !dir.Exists {
|
||||
fmt.Fprintf(out, "Missing: %s\n", dir.TargetAbs)
|
||||
return nil
|
||||
}
|
||||
if err := os.RemoveAll(dir.TargetAbs); err != nil {
|
||||
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, dir.TargetAbs, err)
|
||||
}
|
||||
fmt.Fprintf(out, "Deleted: %s\n", dir.TargetAbs)
|
||||
return nil
|
||||
}
|
||||
|
||||
func reportCleanRootChildren(out io.Writer, root, policy string, dryRun bool) error {
|
||||
rootAbs, entries, err := cleanableRootChildren(root, policy)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
if dryRun {
|
||||
fmt.Fprintf(out, "Would skip empty: %s\n", rootAbs)
|
||||
} else {
|
||||
fmt.Fprintf(out, "Empty: %s\n", rootAbs)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if dryRun {
|
||||
fmt.Fprintf(out, "Would delete: %s\n", entry)
|
||||
continue
|
||||
}
|
||||
if err := os.RemoveAll(entry); err != nil {
|
||||
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, entry, err)
|
||||
}
|
||||
fmt.Fprintf(out, "Deleted: %s\n", entry)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func cleanableRootChildren(root, policy string) (string, []string, error) {
|
||||
rootAbs, exists, err := validateCleanRoot(root, policy)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if !exists {
|
||||
return rootAbs, nil, nil
|
||||
}
|
||||
entries, err := os.ReadDir(rootAbs)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("cleanup policy %s: read root %q: %w", policy, rootAbs, err)
|
||||
}
|
||||
out := make([]string, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
path := filepath.Join(rootAbs, entry.Name())
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("cleanup policy %s: stat child %q: %w", policy, path, err)
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
return "", nil, fmt.Errorf("cleanup policy %s: refusing to delete symlink path %q", policy, path)
|
||||
}
|
||||
out = append(out, path)
|
||||
}
|
||||
return rootAbs, out, nil
|
||||
}
|
||||
|
||||
func cleanSessionAudioCache(ctx context.Context, cfg *config.Config, dryRun bool, out io.Writer) error {
|
||||
if cfg.Session.Inputs.AudioS3 == nil {
|
||||
fmt.Fprintln(out, "Cache: skipped (session does not use audio_s3)")
|
||||
return nil
|
||||
}
|
||||
if cfg.Pipeline.Storage.S3 == nil || strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket) == "" {
|
||||
return fmt.Errorf("clear cache requires pipeline.storage.s3.bucket")
|
||||
}
|
||||
store, err := newCommandObjectStore(ctx, cfg, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("initialize object store for cache cleanup: %w", err)
|
||||
}
|
||||
sessionPrefix := artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
audioPrefix := artifacts.S3AudioPrefix(sessionPrefix, cfg.Session.Inputs.AudioS3.Prefix)
|
||||
objects, err := store.List(ctx, audioPrefix)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list s3 audio objects under %q: %w", audioPrefix, err)
|
||||
}
|
||||
count := 0
|
||||
for _, obj := range objects {
|
||||
key := strings.TrimSpace(obj.Key)
|
||||
if key == "" || strings.HasSuffix(key, "/") || !cleanIsFlac(key) {
|
||||
continue
|
||||
}
|
||||
cachePath, err := artifacts.S3AudioCachePath(cfg.Pipeline.Cache.Root, cfg.Pipeline.Storage.S3.Bucket, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
deleted, err := reportCleanScopedFile(out, cfg.Pipeline.Cache.Root, cachePath, "clean.cache.session", dryRun)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if deleted {
|
||||
count++
|
||||
}
|
||||
}
|
||||
if count == 0 {
|
||||
fmt.Fprintf(out, "Cache: no cached S3 audio files found for %s\n", audioPrefix)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func cleanAllAudioCache(cfg *config.PipelineConfig, dryRun bool, out io.Writer) error {
|
||||
if cfg.Storage.S3 == nil || strings.TrimSpace(cfg.Storage.S3.Bucket) == "" {
|
||||
return fmt.Errorf("clear cache requires pipeline.storage.s3.bucket")
|
||||
}
|
||||
namespaceDir, err := artifacts.S3AudioCacheNamespaceDir(cfg.Cache.Root, cfg.Storage.S3.Bucket, cfg.Storage.S3.RootPrefix)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return reportCleanScopedDir(out, cfg.Cache.Root, namespaceDir, "clean.cache.all", dryRun)
|
||||
}
|
||||
|
||||
func reportCleanScopedFile(out io.Writer, root, target, policy string, dryRun bool) (bool, error) {
|
||||
file, err := validateScopedFile(root, target, policy)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if dryRun {
|
||||
if file.Exists {
|
||||
fmt.Fprintf(out, "Would delete cache file: %s\n", file.TargetAbs)
|
||||
return true, nil
|
||||
}
|
||||
fmt.Fprintf(out, "Would skip missing cache file: %s\n", file.TargetAbs)
|
||||
return false, nil
|
||||
}
|
||||
if !file.Exists {
|
||||
fmt.Fprintf(out, "Missing cache file: %s\n", file.TargetAbs)
|
||||
return false, nil
|
||||
}
|
||||
if err := os.Remove(file.TargetAbs); err != nil {
|
||||
return false, fmt.Errorf("cleanup policy %s: remove %q: %w", policy, file.TargetAbs, err)
|
||||
}
|
||||
fmt.Fprintf(out, "Deleted cache file: %s\n", file.TargetAbs)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func validateScopedFile(root, target, policy string) (scopedDir, error) {
|
||||
return validateScopedTarget(root, target, policy, false)
|
||||
}
|
||||
|
||||
func cleanIsFlac(path string) bool {
|
||||
return strings.EqualFold(filepath.Ext(path), ".flac")
|
||||
}
|
||||
255
internal/app/clean_test.go
Normal file
255
internal/app/clean_test.go
Normal file
@@ -0,0 +1,255 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
)
|
||||
|
||||
func TestExecuteCleanSessionDeletesWorkAndSpoolButPreservesCache(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
workDir := artifacts.SessionWorkDirForCampaign(workspaceRoot, "sample-campaign", "2026-05-03")
|
||||
spoolDir := artifacts.SessionSpoolDir(filepath.Join(workspaceRoot, "spool"), "sample-campaign", "2026-05-03")
|
||||
cachePath, err := artifacts.S3AudioCachePath(filepath.Join(workspaceRoot, "cache"), "test-bucket", "dnd/campaigns/sample-campaign/sessions/2026-05-03/audio/alice.flac")
|
||||
if err != nil {
|
||||
t.Fatalf("S3AudioCachePath() error = %v", err)
|
||||
}
|
||||
mustWriteTestFile(t, filepath.Join(workDir, "manifest.json"), "{}")
|
||||
mustWriteTestFile(t, filepath.Join(spoolDir, "run-1", "audio", "alice.flac"), "audio")
|
||||
mustWriteTestFile(t, cachePath, "cached-audio")
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"clean", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
cleanAssertMissing(t, workDir)
|
||||
cleanAssertMissing(t, spoolDir)
|
||||
cleanAssertExists(t, cachePath)
|
||||
if !strings.Contains(stdout.String(), "Cache: preserved") {
|
||||
t.Fatalf("stdout = %q, want cache preserved", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteCleanSessionDryRunDeletesNothing(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
workDir := artifacts.SessionWorkDirForCampaign(workspaceRoot, "sample-campaign", "2026-05-03")
|
||||
spoolDir := artifacts.SessionSpoolDir(filepath.Join(workspaceRoot, "spool"), "sample-campaign", "2026-05-03")
|
||||
mustWriteTestFile(t, filepath.Join(workDir, "manifest.json"), "{}")
|
||||
mustWriteTestFile(t, filepath.Join(spoolDir, "run-1", "audio", "alice.flac"), "audio")
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"clean", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--dry-run"}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
cleanAssertExists(t, workDir)
|
||||
cleanAssertExists(t, spoolDir)
|
||||
if !strings.Contains(stdout.String(), "Would delete:") {
|
||||
t.Fatalf("stdout = %q, want dry-run delete plan", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteCleanMissingSessionPathsSucceeds(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"clean", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "Missing:") {
|
||||
t.Fatalf("stdout = %q, want missing path output", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteCleanSessionClearCacheRemovesOnlyS3AudioCache(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
if err := os.WriteFile(sessionPath, []byte(`session_id: 2026-05-03
|
||||
inputs:
|
||||
audio_s3:
|
||||
prefix: audio/
|
||||
`), 0o644); err != nil {
|
||||
t.Fatalf("write session: %v", err)
|
||||
}
|
||||
|
||||
audioKey := "dnd/campaigns/sample-campaign/sessions/2026-05-03/audio/alice.flac"
|
||||
fake := &storage.FakeBackend{}
|
||||
fake.SeedObject(storage.FakeObject{Key: audioKey, Data: []byte("audio")})
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||
|
||||
cacheRoot := filepath.Join(workspaceRoot, "cache")
|
||||
cachePath, err := artifacts.S3AudioCachePath(cacheRoot, "test-bucket", audioKey)
|
||||
if err != nil {
|
||||
t.Fatalf("S3AudioCachePath() error = %v", err)
|
||||
}
|
||||
otherCachePath, err := artifacts.S3AudioCachePath(cacheRoot, "test-bucket", "dnd/campaigns/other/sessions/2026-05-03/audio/bob.flac")
|
||||
if err != nil {
|
||||
t.Fatalf("S3AudioCachePath() error = %v", err)
|
||||
}
|
||||
mustWriteTestFile(t, cachePath, "cached-audio")
|
||||
mustWriteTestFile(t, otherCachePath, "other-audio")
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"clean", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--clear-cache"}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
cleanAssertMissing(t, cachePath)
|
||||
cleanAssertExists(t, otherCachePath)
|
||||
if storeInitCalls != 1 {
|
||||
t.Fatalf("object store init calls = %d, want 1", storeInitCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteCleanLocalAudioClearCacheIsNoop(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"clean", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--clear-cache"}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "Cache: skipped (session does not use audio_s3)") {
|
||||
t.Fatalf("stdout = %q, want local audio cache no-op", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteCleanAllDeletesWorkAndSpoolContentsButPreservesCache(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, _, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||
workRoot := filepath.Join(workspaceRoot, "work")
|
||||
spoolRoot := filepath.Join(workspaceRoot, "spool")
|
||||
cachePath := filepath.Join(workspaceRoot, "cache", "keep.txt")
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "sample-campaign", "2026-05-03", "manifest.json"), "{}")
|
||||
mustWriteTestFile(t, filepath.Join(spoolRoot, "sample-campaign", "2026-05-03", "run-1", "audio", "alice.flac"), "audio")
|
||||
mustWriteTestFile(t, cachePath, "cache")
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"clean", "--config", pipelinePath, "--all"}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
cleanAssertMissing(t, workRoot)
|
||||
cleanAssertExists(t, spoolRoot)
|
||||
cleanAssertMissing(t, filepath.Join(spoolRoot, "sample-campaign"))
|
||||
cleanAssertExists(t, cachePath)
|
||||
}
|
||||
|
||||
func TestExecuteCleanAllClearCacheRemovesS3AudioNamespaceOnly(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, _, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||
cacheRoot := filepath.Join(workspaceRoot, "cache")
|
||||
audioCachePath, err := artifacts.S3AudioCachePath(cacheRoot, "test-bucket", "dnd/campaigns/sample-campaign/sessions/2026-05-03/audio/alice.flac")
|
||||
if err != nil {
|
||||
t.Fatalf("S3AudioCachePath() error = %v", err)
|
||||
}
|
||||
otherCachePath, err := artifacts.S3AudioCachePath(cacheRoot, "test-bucket", "other-root/campaigns/sample-campaign/sessions/2026-05-03/audio/alice.flac")
|
||||
if err != nil {
|
||||
t.Fatalf("S3AudioCachePath() error = %v", err)
|
||||
}
|
||||
mustWriteTestFile(t, audioCachePath, "cached-audio")
|
||||
mustWriteTestFile(t, otherCachePath, "other-cache")
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"clean", "--config", pipelinePath, "--all", "--clear-cache"}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
cleanAssertMissing(t, audioCachePath)
|
||||
cleanAssertExists(t, otherCachePath)
|
||||
}
|
||||
|
||||
func TestExecuteCleanAllRejectsSessionScopedFlags(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"clean", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--all"}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "--all cannot be combined") {
|
||||
t.Fatalf("stderr = %q, want --all conflict", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanRequiresSessionID(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"clean"}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "session_id is required unless --all is set") {
|
||||
t.Fatalf("stderr = %q, want missing session-id", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanRejectsUnsafeTargets(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
outside := t.TempDir()
|
||||
if err := reportCleanScopedDir(&bytes.Buffer{}, root, filepath.Join(outside, "target"), "test.outside", false); err == nil {
|
||||
t.Fatal("outside target error = nil, want error")
|
||||
}
|
||||
if err := reportCleanScopedDir(&bytes.Buffer{}, root, root, "test.root", false); err == nil {
|
||||
t.Fatal("root target error = nil, want error")
|
||||
}
|
||||
filePath := filepath.Join(root, "file.txt")
|
||||
mustWriteTestFile(t, filePath, "file")
|
||||
if err := reportCleanScopedDir(&bytes.Buffer{}, root, filePath, "test.file", false); err == nil {
|
||||
t.Fatal("file target error = nil, want error")
|
||||
}
|
||||
symlinkPath := filepath.Join(root, "link")
|
||||
if err := os.Symlink(filepath.Join(root, "missing"), symlinkPath); err != nil {
|
||||
t.Fatalf("Symlink() error = %v", err)
|
||||
}
|
||||
if err := reportCleanScopedDir(&bytes.Buffer{}, root, symlinkPath, "test.symlink", false); err == nil {
|
||||
t.Fatal("symlink target error = nil, want error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClearIsNotCommandAlias(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"clear"}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), `unknown command: "clear"`) {
|
||||
t.Fatalf("stderr = %q, want unknown clear command", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func cleanAssertExists(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
t.Fatalf("expected %q to exist: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func cleanAssertMissing(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
||||
t.Fatalf("expected %q to be missing, stat err=%v", path, err)
|
||||
}
|
||||
}
|
||||
82
internal/app/cleanup_targets.go
Normal file
82
internal/app/cleanup_targets.go
Normal file
@@ -0,0 +1,82 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func validateScopedTarget(root, target, policy string, requireDir bool) (scopedDir, error) {
|
||||
cleanRoot := strings.TrimSpace(root)
|
||||
cleanTarget := strings.TrimSpace(target)
|
||||
if cleanRoot == "" {
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: root path is required", policy)
|
||||
}
|
||||
if cleanTarget == "" {
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: target path is required", policy)
|
||||
}
|
||||
|
||||
rootAbs, err := filepath.Abs(cleanRoot)
|
||||
if err != nil {
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: resolve root %q: %w", policy, cleanRoot, err)
|
||||
}
|
||||
targetAbs, err := filepath.Abs(cleanTarget)
|
||||
if err != nil {
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: resolve target %q: %w", policy, cleanTarget, err)
|
||||
}
|
||||
|
||||
rel, err := filepath.Rel(rootAbs, targetAbs)
|
||||
if err != nil {
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: relative path from %q to %q: %w", policy, rootAbs, targetAbs, err)
|
||||
}
|
||||
if rel == "." {
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: refusing to delete root directory %q", policy, rootAbs)
|
||||
}
|
||||
if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: refusing to delete path outside root: root=%q target=%q", policy, rootAbs, targetAbs)
|
||||
}
|
||||
|
||||
info, err := os.Lstat(targetAbs)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return scopedDir{RootAbs: rootAbs, TargetAbs: targetAbs, Exists: false}, nil
|
||||
}
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: stat target %q: %w", policy, targetAbs, err)
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: refusing to delete symlink path %q", policy, targetAbs)
|
||||
}
|
||||
if requireDir && !info.IsDir() {
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: target %q is not a directory", policy, targetAbs)
|
||||
}
|
||||
if !requireDir && info.IsDir() {
|
||||
return scopedDir{}, fmt.Errorf("cleanup policy %s: target %q is a directory", policy, targetAbs)
|
||||
}
|
||||
return scopedDir{RootAbs: rootAbs, TargetAbs: targetAbs, Exists: true}, nil
|
||||
}
|
||||
|
||||
func validateCleanRoot(root, policy string) (string, bool, error) {
|
||||
cleanRoot := strings.TrimSpace(root)
|
||||
if cleanRoot == "" {
|
||||
return "", false, fmt.Errorf("cleanup policy %s: root path is required", policy)
|
||||
}
|
||||
rootAbs, err := filepath.Abs(cleanRoot)
|
||||
if err != nil {
|
||||
return "", false, fmt.Errorf("cleanup policy %s: resolve root %q: %w", policy, cleanRoot, err)
|
||||
}
|
||||
info, err := os.Lstat(rootAbs)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return rootAbs, false, nil
|
||||
}
|
||||
return "", false, fmt.Errorf("cleanup policy %s: stat root %q: %w", policy, rootAbs, err)
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
return "", false, fmt.Errorf("cleanup policy %s: refusing to clean symlink root %q", policy, rootAbs)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return "", false, fmt.Errorf("cleanup policy %s: root %q is not a directory", policy, rootAbs)
|
||||
}
|
||||
return rootAbs, true, nil
|
||||
}
|
||||
103
internal/app/cleanup_targets_test.go
Normal file
103
internal/app/cleanup_targets_test.go
Normal file
@@ -0,0 +1,103 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCleanValidateScopedDirAndFile(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
dirTarget := filepath.Join(root, "runs", "run-1")
|
||||
fileTarget := filepath.Join(root, "cache", "a.flac")
|
||||
if err := os.MkdirAll(dirTarget, 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll(dirTarget) error = %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(fileTarget), 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll(file parent) error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(fileTarget, []byte("audio"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(fileTarget) error = %v", err)
|
||||
}
|
||||
|
||||
if _, err := validateScopedDir(root, dirTarget, "test.dir"); err != nil {
|
||||
t.Fatalf("validateScopedDir() error = %v", err)
|
||||
}
|
||||
if _, err := validateScopedFile(root, fileTarget, "test.file"); err != nil {
|
||||
t.Fatalf("validateScopedFile() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanValidateScopedTargetSafetyRules(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
outside := t.TempDir()
|
||||
target := filepath.Join(root, "runs", "run-1")
|
||||
if err := os.MkdirAll(target, 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll(target) error = %v", err)
|
||||
}
|
||||
fileTarget := filepath.Join(root, "cache", "a.flac")
|
||||
if err := os.MkdirAll(filepath.Dir(fileTarget), 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll(file parent) error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(fileTarget, []byte("audio"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(fileTarget) error = %v", err)
|
||||
}
|
||||
symlinkTarget := filepath.Join(root, "symlink")
|
||||
if err := os.Symlink(target, symlinkTarget); err != nil {
|
||||
t.Fatalf("Symlink() error = %v", err)
|
||||
}
|
||||
|
||||
if _, err := validateScopedDir(root, root, "test.root"); err == nil || !strings.Contains(err.Error(), "refusing to delete root directory") {
|
||||
t.Fatalf("validateScopedDir(root) error = %v, want root deletion rejection", err)
|
||||
}
|
||||
if _, err := validateScopedDir(root, filepath.Join(outside, "x"), "test.outside"); err == nil || !strings.Contains(err.Error(), "outside root") {
|
||||
t.Fatalf("validateScopedDir(outside) error = %v, want outside-root rejection", err)
|
||||
}
|
||||
if _, err := validateScopedDir(root, fileTarget, "test.file-as-dir"); err == nil || !strings.Contains(err.Error(), "is not a directory") {
|
||||
t.Fatalf("validateScopedDir(file) error = %v, want not-a-directory rejection", err)
|
||||
}
|
||||
if _, err := validateScopedFile(root, target, "test.dir-as-file"); err == nil || !strings.Contains(err.Error(), "is a directory") {
|
||||
t.Fatalf("validateScopedFile(dir) error = %v, want is-a-directory rejection", err)
|
||||
}
|
||||
if _, err := validateScopedDir(root, symlinkTarget, "test.symlink"); err == nil || !strings.Contains(err.Error(), "refusing to delete symlink path") {
|
||||
t.Fatalf("validateScopedDir(symlink) error = %v, want symlink rejection", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanableRootChildrenRejectsSymlinkChild(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
realChild := filepath.Join(root, "runs")
|
||||
if err := os.MkdirAll(realChild, 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll(realChild) error = %v", err)
|
||||
}
|
||||
if err := os.Symlink(realChild, filepath.Join(root, "link")); err != nil {
|
||||
t.Fatalf("Symlink() error = %v", err)
|
||||
}
|
||||
|
||||
_, _, err := cleanableRootChildren(root, "test.root.children")
|
||||
if err == nil || !strings.Contains(err.Error(), "refusing to delete symlink path") {
|
||||
t.Fatalf("cleanableRootChildren() error = %v, want symlink rejection", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanValidateScopedTargetMissing(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
missingDir := filepath.Join(root, "runs", "missing")
|
||||
got, err := validateScopedDir(root, missingDir, "test.missing")
|
||||
if err != nil {
|
||||
t.Fatalf("validateScopedDir(missing) error = %v", err)
|
||||
}
|
||||
if got.Exists {
|
||||
t.Fatalf("validateScopedDir(missing).Exists = true, want false")
|
||||
}
|
||||
|
||||
missingFile := filepath.Join(root, "cache", "missing.flac")
|
||||
got, err = validateScopedFile(root, missingFile, "test.missing.file")
|
||||
if err != nil {
|
||||
t.Fatalf("validateScopedFile(missing) error = %v", err)
|
||||
}
|
||||
if got.Exists {
|
||||
t.Fatalf("validateScopedFile(missing).Exists = true, want false")
|
||||
}
|
||||
}
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
var supportedCommands = []string{"run", "plan", "status", "resume", "run-stage"}
|
||||
var supportedCommands = []string{"run", "run-stage", "resume", "analyze", "publish", "clean", "session"}
|
||||
|
||||
// Execute dispatches CLI commands and returns a process exit code.
|
||||
func Execute(args []string, stdout, stderr io.Writer) int {
|
||||
@@ -24,14 +24,18 @@ func Execute(args []string, stdout, stderr io.Writer) int {
|
||||
switch cmd {
|
||||
case "run":
|
||||
err = Run(ctx, cmdArgs, stdout)
|
||||
case "plan":
|
||||
err = Plan(ctx, cmdArgs, stdout)
|
||||
case "status":
|
||||
err = Status(ctx, cmdArgs, stdout)
|
||||
case "resume":
|
||||
err = Resume(ctx, cmdArgs, stdout)
|
||||
case "run-stage":
|
||||
err = RunStage(ctx, cmdArgs, stdout)
|
||||
case "analyze":
|
||||
err = Analyze(ctx, cmdArgs, stdout)
|
||||
case "publish":
|
||||
err = Publish(ctx, cmdArgs, stdout)
|
||||
case "session":
|
||||
err = Session(ctx, cmdArgs, stdout)
|
||||
case "clean":
|
||||
err = Clean(ctx, cmdArgs, stdout)
|
||||
default:
|
||||
fmt.Fprintf(stderr, "unknown command: %q\n\n", cmd)
|
||||
printUsage(stderr)
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
)
|
||||
|
||||
@@ -23,19 +24,18 @@ func TestExecuteValidCommands(t *testing.T) {
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
||||
manifestPath := writeManifestPathForExecute(t)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
args []string
|
||||
wantOut string
|
||||
}{
|
||||
{name: "run", args: []string{"run", "--config", pipelinePath, "--session", sessionPath}, wantOut: "narratio run: session 2026-05-03; executed=9 skipped=0; manifest="},
|
||||
{name: "plan", args: []string{"plan", "--config", pipelinePath, "--session", sessionPath}, wantOut: "prepare: skip\ntranscribe: skip\nmerge: skip\npolish: skip\nnormalize: skip\ntrim: skip\nanalyze: skip\narchive: skip\nnotify: skip"},
|
||||
{name: "status", args: []string{"status", "--manifest", manifestPath}, wantOut: "session_id: 2026-05-03"},
|
||||
{name: "resume", args: []string{"resume", "--config", pipelinePath, "--session", sessionPath}, wantOut: "narratio resume: session 2026-05-03 has no remaining stages"},
|
||||
{name: "run-stage", args: []string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "polish"}, wantOut: "narratio run-stage: stage=polish executed=0 skipped=1 force=false; manifest="},
|
||||
{name: "run", args: []string{"run", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "narratio run: session 2026-05-03; executed=9 skipped=0; manifest="},
|
||||
{name: "session plan", args: []string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "prepare: skip\ntranscribe: skip\nmerge: skip\npolish: skip\nnormalize: skip\ntrim: skip\nanalyze: skip\npublish: skip\nnotify: skip"},
|
||||
{name: "session status", args: []string{"session", "status", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "Session: 2026-05-03"},
|
||||
{name: "resume", args: []string{"resume", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "narratio resume: session 2026-05-03 has no remaining stages"},
|
||||
{name: "run-stage", args: []string{"run-stage", "polish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "narratio run-stage: stage=polish executed=0 skipped=1 force=false; manifest="},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
@@ -63,12 +63,13 @@ func TestExecuteMissingRequiredFlags(t *testing.T) {
|
||||
args []string
|
||||
want string
|
||||
}{
|
||||
{name: "run missing flags", args: []string{"run"}, want: "run: --config and --session are required"},
|
||||
{name: "plan missing flags", args: []string{"plan"}, want: "plan: --config and --session are required"},
|
||||
{name: "status missing flags", args: []string{"status"}, want: "status: --manifest is required"},
|
||||
{name: "resume missing flags", args: []string{"resume"}, want: "resume: --config and --session are required"},
|
||||
{name: "run-stage missing name", args: []string{"run-stage", "--config", "a", "--session", "b"}, want: "run-stage: expected exactly one stage name"},
|
||||
{name: "run-stage missing config flags", args: []string{"run-stage", "polish"}, want: "run-stage: --config and --session are required"},
|
||||
{name: "run missing session", args: []string{"run"}, want: "run: session_id is required"},
|
||||
{name: "plan old top-level removed", args: []string{"plan"}, want: `unknown command: "plan"`},
|
||||
{name: "status old top-level removed", args: []string{"status"}, want: `unknown command: "status"`},
|
||||
{name: "resume missing session", args: []string{"resume"}, want: "resume: session_id is required"},
|
||||
{name: "run-stage missing name", args: []string{"run-stage", "--config", "a", "--session", "b"}, want: "run-stage: expected stage name and session_id"},
|
||||
{name: "run-stage missing session", args: []string{"run-stage", "polish"}, want: "run-stage: expected stage name and session_id"},
|
||||
{name: "run missing config uses defaults", args: []string{"run", "2026-05-03", "--session", "session.yml"}, want: "run: no pipeline config path provided and no default pipeline config found; searched:"},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
@@ -92,12 +93,12 @@ func TestExecuteMissingRequiredFlags(t *testing.T) {
|
||||
|
||||
func TestExecuteRunStageUnknownFails(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
code := Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "unknown"}, &stdout, &stderr)
|
||||
code := Execute([]string{"run-stage", "unknown", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
@@ -106,16 +107,32 @@ func TestExecuteRunStageUnknownFails(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRunStageNormalizeIsAccepted(t *testing.T) {
|
||||
func TestExecuteRunStageArchiveAliasFails(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
|
||||
workRoot := filepath.Join(workspaceRoot, "work", "2026-05-03")
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "processed.json"), `{"segments":[{"id":1}]}`)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
code := Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "normalize"}, &stdout, &stderr)
|
||||
code := Execute([]string{"run-stage", "archive", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), `unknown stage "archive"`) {
|
||||
t.Fatalf("stderr = %q, want unknown stage alias error", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRunStageNormalizeIsAccepted(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
|
||||
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "polished.json"), `{"segments":[{"id":1}]}`)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
code := Execute([]string{"run-stage", "normalize", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
@@ -134,19 +151,19 @@ func TestExecuteRunStageTranscribeUsesConfiguredWhisperXServer(t *testing.T) {
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
code := Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "prepare"}, &stdout, &stderr)
|
||||
code := Execute([]string{"run-stage", "prepare", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("prepare exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
|
||||
code = Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "--force", "transcribe"}, &stdout, &stderr)
|
||||
code = Execute([]string{"run-stage", "transcribe", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--force"}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("transcribe exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
@@ -154,7 +171,7 @@ func TestExecuteRunStageTranscribeUsesConfiguredWhisperXServer(t *testing.T) {
|
||||
t.Fatal("expected whisperx server to be called at least once")
|
||||
}
|
||||
|
||||
outPath := filepath.Join(workspaceRoot, "work", "2026-05-03", "transcripts", "raw", "alice.json")
|
||||
outPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "transcripts", "raw", "alice.json")
|
||||
data, err := os.ReadFile(outPath)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(%q): %v", outPath, err)
|
||||
@@ -168,6 +185,238 @@ func TestExecuteRunStageTranscribeUsesConfiguredWhisperXServer(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRunStagePolishLoadsCredentialFromSecretsDir(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
configDir := t.TempDir()
|
||||
sessionID := "2026-05-03"
|
||||
secretsDir := filepath.Join(configDir, "secrets")
|
||||
if err := os.MkdirAll(secretsDir, 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll(%q): %v", secretsDir, err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(secretsDir, "OPENROUTER_API_KEY"), []byte("from-secret-file\n"), 0o600); err != nil {
|
||||
t.Fatalf("write OPENROUTER_API_KEY secret file: %v", err)
|
||||
}
|
||||
|
||||
seriatimBinary := writeSeriatimAppTestWrapper(t)
|
||||
auditaBinary := writeAuditaAppTestWrapper(t)
|
||||
t.Setenv("GO_WANT_APP_SERIATIM_HELPER", "1")
|
||||
t.Setenv("GO_WANT_APP_AUDITA_HELPER", "1")
|
||||
|
||||
pipelinePath := filepath.Join(configDir, "pipeline.yml")
|
||||
campaignPath := writeAppTestCampaignConfig(t, configDir)
|
||||
sessionPath := filepath.Join(configDir, "session.yml")
|
||||
pipelineYAML := `workspace:
|
||||
root: ` + workspaceRoot + `
|
||||
storage:
|
||||
backend: local
|
||||
secrets:
|
||||
env_dir: ./secrets
|
||||
whisperx:
|
||||
transcribe_url: https://example.com/transcribe
|
||||
seriatim:
|
||||
binary: ` + seriatimBinary + `
|
||||
audita:
|
||||
binary: ` + auditaBinary + `
|
||||
llm_api_key_env: OPENROUTER_API_KEY
|
||||
notification:
|
||||
timeout: 10s
|
||||
`
|
||||
sessionYAML := `session_id: ` + sessionID + `
|
||||
campaign: sample-campaign
|
||||
inputs:
|
||||
audio_dir: ./audio
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
`
|
||||
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||
t.Fatalf("write pipeline.yml: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
|
||||
t.Fatalf("write session.yml: %v", err)
|
||||
}
|
||||
|
||||
originalWD, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatalf("Getwd(): %v", err)
|
||||
}
|
||||
if err := os.Chdir(configDir); err != nil {
|
||||
t.Fatalf("Chdir(%q): %v", configDir, err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = os.Chdir(originalWD)
|
||||
})
|
||||
|
||||
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", sessionID)
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "base.json"), `{"schema":"seriatim-intermediate","segments":[]}`)
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "[]\n")
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"run-stage", "polish", sessionID, "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--force"}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "stage=polish executed=1 skipped=0") {
|
||||
t.Fatalf("stdout = %q, want polish execution", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRunFailsWhenConfiguredSecretsDirMissing(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
configDir := t.TempDir()
|
||||
pipelinePath := filepath.Join(configDir, "pipeline.yml")
|
||||
campaignPath := writeAppTestCampaignConfig(t, configDir)
|
||||
sessionPath := filepath.Join(configDir, "session.yml")
|
||||
|
||||
pipelineYAML := `workspace:
|
||||
root: ` + workspaceRoot + `
|
||||
storage:
|
||||
backend: local
|
||||
secrets:
|
||||
env_dir: ./missing-secrets
|
||||
whisperx:
|
||||
transcribe_url: https://example.com/transcribe
|
||||
seriatim:
|
||||
binary: seriatim
|
||||
audita:
|
||||
binary: audita
|
||||
notification:
|
||||
timeout: 10s
|
||||
`
|
||||
sessionYAML := `session_id: 2026-05-03
|
||||
campaign: sample-campaign
|
||||
inputs:
|
||||
audio_dir: ./audio
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
`
|
||||
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||
t.Fatalf("write pipeline.yml: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
|
||||
t.Fatalf("write session.yml: %v", err)
|
||||
}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"run", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "read secrets env_dir") {
|
||||
t.Fatalf("stderr = %q, want secrets read-dir error context", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteUsesDefaultPipelineConfigPathWhenConfigFlagOmitted(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"source":"default-config-test","segments":[{"speaker":"alice"}]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
||||
originalDefaults := append([]string(nil), config.DefaultPipelineConfigSearchPaths...)
|
||||
config.DefaultPipelineConfigSearchPaths = []string{pipelinePath}
|
||||
defer func() {
|
||||
config.DefaultPipelineConfigSearchPaths = originalDefaults
|
||||
}()
|
||||
_ = campaignPath
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"run", "2026-05-03", "--session", sessionPath}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "narratio run: session 2026-05-03; executed=9 skipped=0; manifest=") {
|
||||
t.Fatalf("stdout = %q, want successful run output", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteMissingCampaignConfigReportsRegistryPath(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
if err := os.Remove(campaignPath); err != nil {
|
||||
t.Fatalf("remove campaign config: %v", err)
|
||||
}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"run", "2026-05-03", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if stdout.Len() != 0 {
|
||||
t.Fatalf("stdout = %q, want empty", stdout.String())
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "load campaign config") {
|
||||
t.Fatalf("stderr = %q, want campaign discovery failure", stderr.String())
|
||||
}
|
||||
if !strings.Contains(stderr.String(), filepath.ToSlash(filepath.Join("campaigns", "sample-campaign", "campaign.yml"))) {
|
||||
t.Fatalf("stderr = %q, want campaign registry path", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteUsesPipelineDefaultCampaignID(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, _, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"session", "status", "2026-05-03", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "Campaign: sample-campaign") {
|
||||
t.Fatalf("stdout = %q, want default campaign", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteCampaignIDSelectsRegistryCampaign(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
campaignRoot := filepath.Dir(filepath.Dir(campaignPath))
|
||||
otherDir := filepath.Join(campaignRoot, "icewind")
|
||||
mustWriteTestFile(t, filepath.Join(otherDir, "campaign.yml"), `campaign_id: icewind
|
||||
inputs:
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
`)
|
||||
mustWriteTestFile(t, filepath.Join(otherDir, "speakers.yml"), "match:\n - speaker: Alice\n match: [\"alice\"]\n")
|
||||
mustWriteTestFile(t, filepath.Join(otherDir, "autocorrect.yml"), "[]\n")
|
||||
mustWriteTestFile(t, filepath.Join(otherDir, "glossary.yml"), "[]\n")
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"session", "status", "2026-05-03", "--config", pipelinePath, "--campaign", "icewind", "--session", sessionPath}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "Campaign: icewind") {
|
||||
t.Fatalf("stdout = %q, want selected campaign", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRejectsCampaignIDAndCampaignFile(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"session", "status", "2026-05-03", "--config", pipelinePath, "--campaign", "sample-campaign", "--campaign-file", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "mutually exclusive") {
|
||||
t.Fatalf("stderr = %q, want mutually exclusive error", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteInvalidCommand(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
@@ -204,11 +453,14 @@ func TestExecuteMissingCommand(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func writeValidConfigFiles(t *testing.T, workspaceRoot string, transcribeURL ...string) (string, string) {
|
||||
func writeValidConfigFiles(t *testing.T, workspaceRoot string, transcribeURL ...string) (string, string, string) {
|
||||
t.Helper()
|
||||
|
||||
dir := t.TempDir()
|
||||
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||
campaignRoot := filepath.Join(dir, "campaigns")
|
||||
campaignDir := filepath.Join(campaignRoot, "sample-campaign")
|
||||
campaignPath := filepath.Join(campaignDir, "campaign.yml")
|
||||
sessionPath := filepath.Join(dir, "session.yml")
|
||||
url := "https://example.com/transcribe"
|
||||
if len(transcribeURL) > 0 && strings.TrimSpace(transcribeURL[0]) != "" {
|
||||
@@ -222,8 +474,20 @@ func writeValidConfigFiles(t *testing.T, workspaceRoot string, transcribeURL ...
|
||||
|
||||
pipelineYAML := `workspace:
|
||||
root: ` + workspaceRoot + `
|
||||
campaigns:
|
||||
root: ` + campaignRoot + `
|
||||
default_campaign_id: sample-campaign
|
||||
cache:
|
||||
root: ` + filepath.Join(workspaceRoot, "cache") + `
|
||||
spool:
|
||||
root: ` + filepath.Join(workspaceRoot, "spool") + `
|
||||
storage:
|
||||
backend: s3
|
||||
s3:
|
||||
bucket: test-bucket
|
||||
publish:
|
||||
enabled: true
|
||||
upload_run: false
|
||||
whisperx:
|
||||
transcribe_url: ` + url + `
|
||||
timeout: 2s
|
||||
@@ -238,10 +502,6 @@ seriatim:
|
||||
report: true
|
||||
audita:
|
||||
binary: ` + auditaBinary + `
|
||||
analyzer:
|
||||
timeout: 20m
|
||||
artifacts:
|
||||
output_dir: artifacts
|
||||
notification:
|
||||
timeout: 10s
|
||||
`
|
||||
@@ -249,6 +509,9 @@ notification:
|
||||
sessionYAML := `session_id: 2026-05-03
|
||||
inputs:
|
||||
audio_dir: ./audio
|
||||
`
|
||||
campaignYAML := `campaign_id: sample-campaign
|
||||
inputs:
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
@@ -257,16 +520,37 @@ inputs:
|
||||
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||
t.Fatalf("write pipeline config: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(campaignDir, 0o755); err != nil {
|
||||
t.Fatalf("create campaign dir: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(campaignPath, []byte(campaignYAML), 0o644); err != nil {
|
||||
t.Fatalf("write campaign config: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
|
||||
t.Fatalf("write session config: %v", err)
|
||||
}
|
||||
|
||||
mustWriteTestFile(t, filepath.Join(dir, "speakers.yml"), "match:\n - speaker: Alice\n match: [\"alice\"]\n")
|
||||
mustWriteTestFile(t, filepath.Join(dir, "autocorrect.yml"), "[]\n")
|
||||
mustWriteTestFile(t, filepath.Join(dir, "glossary.yml"), "[]\n")
|
||||
mustWriteTestFile(t, filepath.Join(campaignDir, "speakers.yml"), "match:\n - speaker: Alice\n match: [\"alice\"]\n")
|
||||
mustWriteTestFile(t, filepath.Join(campaignDir, "autocorrect.yml"), "[]\n")
|
||||
mustWriteTestFile(t, filepath.Join(campaignDir, "glossary.yml"), "[]\n")
|
||||
mustWriteTestFile(t, filepath.Join(dir, "audio", "alice.flac"), "audio-bytes")
|
||||
|
||||
return pipelinePath, sessionPath
|
||||
return pipelinePath, campaignPath, sessionPath
|
||||
}
|
||||
|
||||
func writeAppTestCampaignConfig(t *testing.T, dir string) string {
|
||||
t.Helper()
|
||||
campaignPath := filepath.Join(dir, "campaign.yml")
|
||||
campaignYAML := `campaign_id: sample-campaign
|
||||
inputs:
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
`
|
||||
if err := os.WriteFile(campaignPath, []byte(campaignYAML), 0o644); err != nil {
|
||||
t.Fatalf("write campaign.yml: %v", err)
|
||||
}
|
||||
return campaignPath
|
||||
}
|
||||
|
||||
func writeManifestPathForExecute(t *testing.T) string {
|
||||
|
||||
157
internal/app/config_loader.go
Normal file
157
internal/app/config_loader.go
Normal file
@@ -0,0 +1,157 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
type pipelineCampaignConfig struct {
|
||||
PipelinePath string
|
||||
CampaignPath string
|
||||
Pipeline *config.PipelineConfig
|
||||
Campaign *config.CampaignConfig
|
||||
}
|
||||
|
||||
func loadCommandConfig(ctx context.Context, pipelineFlag, campaignFlag, campaignFileFlag, sessionFlag string, sessionOpts config.SessionLoadOptions) (*config.Config, error) {
|
||||
base, err := loadPipelineCampaignConfig(pipelineFlag, campaignFlag, campaignFileFlag)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if explicitSession := strings.TrimSpace(sessionFlag); explicitSession != "" {
|
||||
return config.LoadWithSessionOptions(base.PipelinePath, base.CampaignPath, explicitSession, sessionOpts)
|
||||
}
|
||||
|
||||
discoveredSession, err := discoverSessionConfigPathWithCandidates(config.DefaultSessionConfigSearchPaths)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if discoveredSession.Path != "" {
|
||||
return config.LoadWithSessionOptions(base.PipelinePath, base.CampaignPath, discoveredSession.Path, sessionOpts)
|
||||
}
|
||||
|
||||
sessionID := strings.TrimSpace(sessionOpts.SessionID)
|
||||
if sessionID == "" {
|
||||
return nil, missingSessionConfigError(discoveredSession.Searched, "remote session loading requires a session_id")
|
||||
}
|
||||
|
||||
sessionPrefix := artifacts.S3SessionPrefix(base.Pipeline.Storage.S3.RootPrefix, config.CampaignID(base.Campaign), sessionID)
|
||||
remoteKey := artifacts.S3SessionConfigKey(sessionPrefix)
|
||||
partialCfg := &config.Config{
|
||||
Pipeline: base.Pipeline,
|
||||
Campaign: base.Campaign,
|
||||
PipelinePath: base.PipelinePath,
|
||||
CampaignPath: base.CampaignPath,
|
||||
}
|
||||
store, err := newCommandObjectStore(ctx, partialCfg, nil)
|
||||
if err != nil {
|
||||
return nil, missingSessionConfigError(discoveredSession.Searched, fmt.Sprintf("remote session %q unavailable: %v", remoteKey, err))
|
||||
}
|
||||
|
||||
sessionInfo, err := findRemoteSessionConfig(ctx, store, sessionPrefix, remoteKey)
|
||||
if err != nil {
|
||||
return nil, missingSessionConfigError(discoveredSession.Searched, err.Error())
|
||||
}
|
||||
sessionTempPath, err := downloadRemoteSessionConfig(ctx, store, remoteKey)
|
||||
if err != nil {
|
||||
return nil, missingSessionConfigError(discoveredSession.Searched, fmt.Sprintf("remote session %q download failed: %v", remoteKey, err))
|
||||
}
|
||||
sessionBytes, err := os.ReadFile(sessionTempPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read downloaded remote session %q: %w", sessionTempPath, err)
|
||||
}
|
||||
sessionCfg, err := config.LoadSessionBytesWithOptions("s3://"+s3BucketName(base.Pipeline)+"/"+remoteKey, sessionBytes, sessionOpts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return config.Resolve(
|
||||
base.PipelinePath,
|
||||
base.Pipeline,
|
||||
base.CampaignPath,
|
||||
base.Campaign,
|
||||
sessionTempPath,
|
||||
sessionCfg,
|
||||
config.SessionSource{
|
||||
Source: "session_config.s3",
|
||||
LocalPath: sessionTempPath,
|
||||
S3Bucket: s3BucketName(base.Pipeline),
|
||||
S3Key: remoteKey,
|
||||
S3Size: sessionInfo.Size,
|
||||
S3ETag: sessionInfo.ETag,
|
||||
SpoolPath: sessionTempPath,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func loadPipelineCampaignConfig(pipelineFlag, campaignFlag, campaignFileFlag string) (*pipelineCampaignConfig, error) {
|
||||
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelineFlag)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pipelineCfg, err := config.LoadPipeline(resolvedPipelinePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resolvedCampaignPath, err := resolveCampaignConfigPath(pipelineCfg, campaignFlag, campaignFileFlag)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
campaignCfg, err := config.LoadCampaign(resolvedCampaignPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if selectedID := strings.TrimSpace(campaignFlag); selectedID != "" && strings.TrimSpace(campaignFileFlag) == "" {
|
||||
if got := config.CampaignID(campaignCfg); got != selectedID {
|
||||
return nil, fmt.Errorf("campaign config %q invalid: campaign_id %q does not match selected campaign %q", resolvedCampaignPath, got, selectedID)
|
||||
}
|
||||
}
|
||||
return &pipelineCampaignConfig{
|
||||
PipelinePath: resolvedPipelinePath,
|
||||
CampaignPath: resolvedCampaignPath,
|
||||
Pipeline: pipelineCfg,
|
||||
Campaign: campaignCfg,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func findRemoteSessionConfig(ctx context.Context, store storage.ObjectStore, sessionPrefix, remoteKey string) (storage.ObjectInfo, error) {
|
||||
objects, err := store.List(ctx, sessionPrefix)
|
||||
if err != nil {
|
||||
return storage.ObjectInfo{}, fmt.Errorf("remote session %q list failed: %w", remoteKey, err)
|
||||
}
|
||||
for _, obj := range objects {
|
||||
if obj.Key == remoteKey {
|
||||
return obj, nil
|
||||
}
|
||||
}
|
||||
return storage.ObjectInfo{}, fmt.Errorf("remote session %q not found", remoteKey)
|
||||
}
|
||||
|
||||
func downloadRemoteSessionConfig(ctx context.Context, store storage.ObjectStore, remoteKey string) (string, error) {
|
||||
f, err := os.CreateTemp("", "narratio-session-*.yml")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create temp file: %w", err)
|
||||
}
|
||||
path := f.Name()
|
||||
if err := f.Close(); err != nil {
|
||||
return "", fmt.Errorf("close temp file %q: %w", path, err)
|
||||
}
|
||||
if err := store.Download(ctx, remoteKey, path); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return filepath.Clean(path), nil
|
||||
}
|
||||
|
||||
func s3BucketName(cfg *config.PipelineConfig) string {
|
||||
if cfg == nil || cfg.Storage.S3 == nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(cfg.Storage.S3.Bucket)
|
||||
}
|
||||
21
internal/app/object_store.go
Normal file
21
internal/app/object_store.go
Normal file
@@ -0,0 +1,21 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
func newCommandObjectStore(ctx context.Context, cfg *config.Config, logger *slog.Logger) (storage.ObjectStore, error) {
|
||||
if _, err := loadSecretsFromConfig(cfg, logger); err != nil {
|
||||
return nil, fmt.Errorf("load secrets from files: %w", err)
|
||||
}
|
||||
store, err := newObjectStoreFromConfigFn(ctx, cfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("initialize object store backend: %w", err)
|
||||
}
|
||||
return store, nil
|
||||
}
|
||||
165
internal/app/object_store_test.go
Normal file
165
internal/app/object_store_test.go
Normal file
@@ -0,0 +1,165 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
func TestNewCommandObjectStoreLoadsSecretsBeforeFactory(t *testing.T) {
|
||||
accessKeyEnv := "NARRATIO_TEST_COMMAND_STORE_KEY_ID"
|
||||
secretKeyEnv := "NARRATIO_TEST_COMMAND_STORE_SECRET"
|
||||
restoreEnvAfterTest(t, accessKeyEnv, secretKeyEnv)
|
||||
|
||||
secretsDir := t.TempDir()
|
||||
mustWriteSecretFile(t, filepath.Join(secretsDir, accessKeyEnv), "loaded-key-id\n")
|
||||
mustWriteSecretFile(t, filepath.Join(secretsDir, secretKeyEnv), "loaded-secret\n")
|
||||
|
||||
cfg := commandObjectStoreTestConfig(secretsDir)
|
||||
fake := &storage.FakeBackend{}
|
||||
called := false
|
||||
origStoreFn := newObjectStoreFromConfigFn
|
||||
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
called = true
|
||||
if got := os.Getenv(accessKeyEnv); got != "loaded-key-id" {
|
||||
return nil, errors.New("access key was not loaded before object store init")
|
||||
}
|
||||
if got := os.Getenv(secretKeyEnv); got != "loaded-secret" {
|
||||
return nil, errors.New("secret key was not loaded before object store init")
|
||||
}
|
||||
return fake, nil
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
newObjectStoreFromConfigFn = origStoreFn
|
||||
})
|
||||
|
||||
store, err := newCommandObjectStore(context.Background(), cfg, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("newCommandObjectStore() error = %v", err)
|
||||
}
|
||||
if store != fake {
|
||||
t.Fatalf("store = %#v, want fake backend", store)
|
||||
}
|
||||
if !called {
|
||||
t.Fatal("object store factory was not called")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewCommandObjectStorePreservesExistingEnv(t *testing.T) {
|
||||
accessKeyEnv := "NARRATIO_TEST_COMMAND_STORE_EXISTING_KEY_ID"
|
||||
secretKeyEnv := "NARRATIO_TEST_COMMAND_STORE_EXISTING_SECRET"
|
||||
t.Setenv(accessKeyEnv, "existing-key-id")
|
||||
t.Setenv(secretKeyEnv, "existing-secret")
|
||||
|
||||
secretsDir := t.TempDir()
|
||||
mustWriteSecretFile(t, filepath.Join(secretsDir, accessKeyEnv), "file-key-id\n")
|
||||
mustWriteSecretFile(t, filepath.Join(secretsDir, secretKeyEnv), "file-secret\n")
|
||||
|
||||
cfg := commandObjectStoreTestConfig(secretsDir)
|
||||
origStoreFn := newObjectStoreFromConfigFn
|
||||
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
if got := os.Getenv(accessKeyEnv); got != "existing-key-id" {
|
||||
return nil, errors.New("existing access key was overwritten")
|
||||
}
|
||||
if got := os.Getenv(secretKeyEnv); got != "existing-secret" {
|
||||
return nil, errors.New("existing secret key was overwritten")
|
||||
}
|
||||
return &storage.FakeBackend{}, nil
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
newObjectStoreFromConfigFn = origStoreFn
|
||||
})
|
||||
|
||||
if _, err := newCommandObjectStore(context.Background(), cfg, nil); err != nil {
|
||||
t.Fatalf("newCommandObjectStore() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewCommandObjectStoreSecretErrorStopsFactory(t *testing.T) {
|
||||
cfg := commandObjectStoreTestConfig(filepath.Join(t.TempDir(), "missing"))
|
||||
called := false
|
||||
origStoreFn := newObjectStoreFromConfigFn
|
||||
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
called = true
|
||||
return &storage.FakeBackend{}, nil
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
newObjectStoreFromConfigFn = origStoreFn
|
||||
})
|
||||
|
||||
_, err := newCommandObjectStore(context.Background(), cfg, nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
if called {
|
||||
t.Fatal("object store factory was called after secret load failure")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "load secrets from files") {
|
||||
t.Fatalf("error = %q, want secret loading context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewCommandObjectStoreFactoryErrorIsContextual(t *testing.T) {
|
||||
cfg := commandObjectStoreTestConfig("")
|
||||
origStoreFn := newObjectStoreFromConfigFn
|
||||
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
return nil, errors.New("factory boom")
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
newObjectStoreFromConfigFn = origStoreFn
|
||||
})
|
||||
|
||||
_, err := newCommandObjectStore(context.Background(), cfg, nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "initialize object store backend") || !strings.Contains(err.Error(), "factory boom") {
|
||||
t.Fatalf("error = %q, want factory context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func commandObjectStoreTestConfig(secretsDir string) *config.Config {
|
||||
cfg := &config.Config{
|
||||
Pipeline: &config.PipelineConfig{
|
||||
Storage: config.StorageConfig{
|
||||
Backend: "s3",
|
||||
S3: &config.StorageS3Config{
|
||||
Bucket: "test-bucket",
|
||||
AccessKeyIDEnv: "NARRATIO_TEST_COMMAND_STORE_KEY_ID",
|
||||
SecretKeyEnv: "NARRATIO_TEST_COMMAND_STORE_SECRET",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
if strings.TrimSpace(secretsDir) != "" {
|
||||
cfg.Pipeline.Secrets = &config.SecretsConfig{EnvDir: secretsDir}
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
func restoreEnvAfterTest(t *testing.T, names ...string) {
|
||||
t.Helper()
|
||||
originals := make(map[string]string, len(names))
|
||||
present := make(map[string]bool, len(names))
|
||||
for _, name := range names {
|
||||
value, ok := os.LookupEnv(name)
|
||||
originals[name] = value
|
||||
present[name] = ok
|
||||
_ = os.Unsetenv(name)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
for _, name := range names {
|
||||
if present[name] {
|
||||
_ = os.Setenv(name, originals[name])
|
||||
} else {
|
||||
_ = os.Unsetenv(name)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
136
internal/app/operator_artifact_rendering.go
Normal file
136
internal/app/operator_artifact_rendering.go
Normal file
@@ -0,0 +1,136 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
func buildHelperArtifactCatalog(cfg *config.Config) (*artifacts.ArtifactCatalog, error) {
|
||||
catalog := artifacts.NewArtifactCatalog()
|
||||
if err := catalog.RegisterBuiltIns(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
configured := map[string]artifacts.ConfiguredArtifactDefinition{}
|
||||
if cfg.Pipeline.Scriptorium != nil {
|
||||
for key, item := range cfg.Pipeline.Scriptorium.Artifacts {
|
||||
configured[key] = artifacts.ConfiguredArtifactDefinition{Enabled: item.Enabled, OutputPath: item.OutputPath}
|
||||
}
|
||||
}
|
||||
if err := catalog.RegisterConfiguredArtifacts(configured, nil); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return catalog, nil
|
||||
}
|
||||
|
||||
func writeArtifactList(out io.Writer, cfg *config.Config, catalog *artifacts.ArtifactCatalog, locks *effectiveLocks, publishedRemoteState map[string]string) {
|
||||
lockSet := lockSourceSet(locks.All)
|
||||
fmt.Fprintln(out, "Built-in:")
|
||||
for _, id := range []string{
|
||||
artifacts.ArtifactTranscriptBase,
|
||||
artifacts.ArtifactTranscriptPolished,
|
||||
artifacts.ArtifactTranscriptFinal,
|
||||
artifacts.ArtifactTranscriptFinalTrimmed,
|
||||
artifacts.ArtifactBoundsSession,
|
||||
} {
|
||||
writeArtifactLine(out, id, lockSet)
|
||||
}
|
||||
fmt.Fprintln(out, "Configured:")
|
||||
for _, entry := range catalog.ListConfigured() {
|
||||
writeArtifactLine(out, entry.SourceID, lockSet)
|
||||
}
|
||||
fmt.Fprintln(out, "Previous-session:")
|
||||
for _, req := range artifacts.CollectPreviousArtifactRequirements(configuredScriptoriumArtifacts(cfg)) {
|
||||
fmt.Fprintf(out, "- %s required=%t\n", artifactpolicy.PreviousSessionSourceID(req.Name), req.Required)
|
||||
}
|
||||
fmt.Fprintln(out, "Published:")
|
||||
for _, rule := range cfg.Pipeline.Publish.Outputs {
|
||||
writePublishedOutputLine(out, rule, catalog, lockSet, publishedRemoteState)
|
||||
}
|
||||
}
|
||||
|
||||
func writeArtifactLine(out io.Writer, source string, lockSet map[string]config.PublishLockRule) {
|
||||
parts := []string{source}
|
||||
if _, ok := lockSet[source]; ok {
|
||||
parts = append(parts, "locked")
|
||||
}
|
||||
fmt.Fprintf(out, "- %s\n", strings.Join(parts, " "))
|
||||
}
|
||||
|
||||
func writePublishedOutputLine(out io.Writer, rule config.PublishOutputRule, catalog *artifacts.ArtifactCatalog, lockSet map[string]config.PublishLockRule, remoteState map[string]string) {
|
||||
source := strings.TrimSpace(rule.Source)
|
||||
parts := []string{source}
|
||||
if _, ok := lockSet[source]; ok {
|
||||
parts = append(parts, "locked")
|
||||
}
|
||||
dest, showDest, err := helperPublishedOutputDest(rule, catalog)
|
||||
if err != nil {
|
||||
parts = append(parts, "remote=error")
|
||||
fmt.Fprintf(out, "- %s\n", strings.Join(parts, " "))
|
||||
return
|
||||
}
|
||||
if showDest {
|
||||
parts = append(parts, "dest="+dest)
|
||||
}
|
||||
if state := remoteState[publishedOutputRemoteStateKey(source, dest)]; state != "" {
|
||||
parts = append(parts, state)
|
||||
}
|
||||
fmt.Fprintf(out, "- %s\n", strings.Join(parts, " "))
|
||||
}
|
||||
|
||||
func remotePublishedOutputAvailability(ctx context.Context, cfg *config.Config, store storage.ObjectStore, catalog *artifacts.ArtifactCatalog) map[string]string {
|
||||
out := map[string]string{}
|
||||
sessionPrefix := artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
for _, rule := range cfg.Pipeline.Publish.Outputs {
|
||||
source := strings.TrimSpace(rule.Source)
|
||||
dest, _, err := helperPublishedOutputDest(rule, catalog)
|
||||
if err != nil {
|
||||
out[publishedOutputRemoteStateKey(source, "")] = "remote=error"
|
||||
continue
|
||||
}
|
||||
key := artifacts.S3PublishedOutputKey(sessionPrefix, dest)
|
||||
if exists, err := store.Exists(ctx, key); err == nil && exists {
|
||||
out[publishedOutputRemoteStateKey(source, dest)] = "remote=published"
|
||||
} else if err != nil {
|
||||
out[publishedOutputRemoteStateKey(source, dest)] = "remote=error"
|
||||
} else {
|
||||
out[publishedOutputRemoteStateKey(source, dest)] = "remote=missing"
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func helperPublishedOutputDest(rule config.PublishOutputRule, catalog *artifacts.ArtifactCatalog) (string, bool, error) {
|
||||
source := strings.TrimSpace(rule.Source)
|
||||
normalized, err := artifactpolicy.ResolvePublishedDestination(source, rule.Dest, helperConfiguredOutputPathMap(catalog))
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
entry, ok := catalog.Lookup(source)
|
||||
showDest := !ok || strings.TrimSpace(entry.CanonicalRelPath) != normalized
|
||||
return normalized, showDest, nil
|
||||
}
|
||||
|
||||
func helperConfiguredOutputPathMap(catalog *artifacts.ArtifactCatalog) map[string]string {
|
||||
out := map[string]string{}
|
||||
if catalog == nil {
|
||||
return out
|
||||
}
|
||||
for _, entry := range catalog.ListConfigured() {
|
||||
if strings.TrimSpace(entry.ConfiguredKey) == "" {
|
||||
continue
|
||||
}
|
||||
out[entry.ConfiguredKey] = strings.TrimSpace(entry.CanonicalRelPath)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func publishedOutputRemoteStateKey(source, dest string) string {
|
||||
return strings.TrimSpace(source) + "\x00" + strings.TrimSpace(dest)
|
||||
}
|
||||
39
internal/app/operator_artifacts_list.go
Normal file
39
internal/app/operator_artifacts_list.go
Normal file
@@ -0,0 +1,39 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ArtifactsList lists effective artifact sources.
|
||||
func ArtifactsList(ctx context.Context, args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("artifacts list", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
var flags commonConfigFlags
|
||||
var remote bool
|
||||
addCommonConfigFlags(fs, &flags)
|
||||
fs.BoolVar(&remote, "remote", false, "inspect remote publish availability")
|
||||
if err := parseSessionAwareFlags("artifacts list", fs, args, &flags.sessionID); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(flags.sessionID) == "" {
|
||||
return fmt.Errorf("artifacts list: session_id is required")
|
||||
}
|
||||
cfg, store, locks, _, err := loadHelperContext(ctx, flags, remote)
|
||||
if err != nil {
|
||||
return fmt.Errorf("artifacts list: %w", err)
|
||||
}
|
||||
catalog, err := buildHelperArtifactCatalog(cfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("artifacts list: %w", err)
|
||||
}
|
||||
publishedRemoteState := map[string]string{}
|
||||
if remote && store != nil {
|
||||
publishedRemoteState = remotePublishedOutputAvailability(ctx, cfg, store, catalog)
|
||||
}
|
||||
writeArtifactList(out, cfg, catalog, locks, publishedRemoteState)
|
||||
return nil
|
||||
}
|
||||
203
internal/app/operator_findings.go
Normal file
203
internal/app/operator_findings.go
Normal file
@@ -0,0 +1,203 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
)
|
||||
|
||||
type finding struct {
|
||||
Severity string
|
||||
Category string
|
||||
Message string
|
||||
}
|
||||
|
||||
type findingError struct {
|
||||
count int
|
||||
}
|
||||
|
||||
func (e findingError) Error() string {
|
||||
return fmt.Sprintf("%d validation error(s)", e.count)
|
||||
}
|
||||
|
||||
func renderFindings(out io.Writer, campaign, sessionID string, findings []finding) error {
|
||||
if campaign != "" || sessionID != "" {
|
||||
fmt.Fprintf(out, "Campaign: %s\n", campaign)
|
||||
fmt.Fprintf(out, "Session: %s\n\n", sessionID)
|
||||
}
|
||||
errorsCount := 0
|
||||
for _, f := range findings {
|
||||
if f.Severity == "ERROR" {
|
||||
errorsCount++
|
||||
}
|
||||
fmt.Fprintf(out, "%-5s %-10s %s\n", f.Severity, f.Category, f.Message)
|
||||
}
|
||||
if errorsCount > 0 {
|
||||
return findingError{count: errorsCount}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func okFinding(category, msg string) finding { return finding{"OK", category, msg} }
|
||||
func infoFinding(category, msg string) finding { return finding{"INFO", category, msg} }
|
||||
func warnFinding(category, msg string) finding { return finding{"WARN", category, msg} }
|
||||
func errorFinding(category, msg string) finding { return finding{"ERROR", category, msg} }
|
||||
|
||||
func sessionSourceSummary(cfg *config.Config) string {
|
||||
source := cfg.SessionSource.Source
|
||||
if source == "" {
|
||||
source = "session_config"
|
||||
}
|
||||
if cfg.SessionSource.S3Key != "" {
|
||||
return source + " " + cfg.SessionSource.S3Key
|
||||
}
|
||||
return source + " " + cfg.SessionPath
|
||||
}
|
||||
|
||||
func validateStableInputFindings(cfg *config.Config) []finding {
|
||||
items := []struct {
|
||||
name string
|
||||
in config.ResolvedInputFile
|
||||
}{
|
||||
{"speakers", cfg.StableInputs.SpeakersFile},
|
||||
{"autocorrect", cfg.StableInputs.AutocorrectFile},
|
||||
{"glossary", cfg.StableInputs.GlossaryFile},
|
||||
}
|
||||
out := make([]finding, 0, len(items))
|
||||
for _, item := range items {
|
||||
path, err := resolveHelperConfigRelativePath(item.in)
|
||||
if err != nil {
|
||||
out = append(out, errorFinding("inputs", item.name+": "+err.Error()))
|
||||
continue
|
||||
}
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
out = append(out, errorFinding("inputs", fmt.Sprintf("%s missing: %v", item.name, err)))
|
||||
} else {
|
||||
out = append(out, okFinding("inputs", item.name+": "+path))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func resolveHelperConfigRelativePath(input config.ResolvedInputFile) (string, error) {
|
||||
if strings.TrimSpace(input.ConfigPath) == "" {
|
||||
return "", fmt.Errorf("source config path is required")
|
||||
}
|
||||
path := strings.TrimSpace(input.Path)
|
||||
if path == "" {
|
||||
return "", fmt.Errorf("path is required")
|
||||
}
|
||||
if filepath.IsAbs(path) {
|
||||
return filepath.Clean(path), nil
|
||||
}
|
||||
return filepath.Clean(filepath.Join(filepath.Dir(input.ConfigPath), path)), nil
|
||||
}
|
||||
|
||||
func validateLocalAudioFindings(cfg *config.Config) []finding {
|
||||
if cfg.Session.Inputs.AudioS3 != nil {
|
||||
return nil
|
||||
}
|
||||
audioDir := strings.TrimSpace(cfg.Session.Inputs.AudioDir)
|
||||
if audioDir == "" && len(cfg.Session.Inputs.AudioFiles) == 0 {
|
||||
return []finding{errorFinding("audio", "audio_dir, audio_files, or audio_s3 is required")}
|
||||
}
|
||||
base := filepath.Dir(cfg.SessionPath)
|
||||
paths := []string{}
|
||||
if audioDir != "" {
|
||||
dir := audioDir
|
||||
if !filepath.IsAbs(dir) {
|
||||
dir = filepath.Join(base, dir)
|
||||
}
|
||||
matches, err := filepath.Glob(filepath.Join(dir, "*.flac"))
|
||||
if err != nil || len(matches) == 0 {
|
||||
return []finding{errorFinding("audio", "no .flac files found in "+dir)}
|
||||
}
|
||||
paths = append(paths, matches...)
|
||||
}
|
||||
for _, file := range cfg.Session.Inputs.AudioFiles {
|
||||
p := file
|
||||
if !filepath.IsAbs(p) {
|
||||
p = filepath.Join(base, p)
|
||||
}
|
||||
paths = append(paths, p)
|
||||
}
|
||||
for _, p := range paths {
|
||||
if _, err := os.Stat(p); err != nil {
|
||||
return []finding{errorFinding("audio", fmt.Sprintf("audio file missing: %v", err))}
|
||||
}
|
||||
}
|
||||
return []finding{okFinding("audio", fmt.Sprintf("%d local audio file(s)", len(paths)))}
|
||||
}
|
||||
|
||||
func validateRemoteAudioFinding(ctx context.Context, cfg *config.Config, store storage.ObjectStore) finding {
|
||||
sessionPrefix := artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
audioPrefix := artifacts.S3AudioPrefix(sessionPrefix, cfg.Session.Inputs.AudioS3.Prefix)
|
||||
objects, err := store.List(ctx, audioPrefix)
|
||||
if err != nil {
|
||||
return errorFinding("audio", err.Error())
|
||||
}
|
||||
count := 0
|
||||
for _, obj := range objects {
|
||||
if strings.HasSuffix(strings.ToLower(obj.Key), ".flac") {
|
||||
count++
|
||||
}
|
||||
}
|
||||
if count == 0 {
|
||||
return errorFinding("audio", "no remote .flac objects found under "+audioPrefix)
|
||||
}
|
||||
return okFinding("audio", fmt.Sprintf("%d remote .flac object(s)", count))
|
||||
}
|
||||
|
||||
func validatePreviousArtifactFindings(ctx context.Context, cfg *config.Config, store storage.ObjectStore, requirements []artifacts.PreviousArtifactRequirement) []finding {
|
||||
out := []finding{}
|
||||
prefix := artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, cfg.Session.Campaign, cfg.Session.PreviousSessionID)
|
||||
_, err := artifacts.LoadCurrentState(ctx, store, prefix, artifacts.CurrentStateValidation{
|
||||
ExpectedSessionID: strings.TrimSpace(cfg.Session.PreviousSessionID),
|
||||
ExpectedCampaign: strings.TrimSpace(cfg.Session.Campaign),
|
||||
ValidateRunID: true,
|
||||
})
|
||||
if err != nil {
|
||||
out = append(out, errorFinding("previous", fmt.Sprintf("remote %v", err)))
|
||||
return out
|
||||
}
|
||||
for _, req := range requirements {
|
||||
out = append(out, okFinding("previous", fmt.Sprintf("%s required=%t", req.Name, req.Required)))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func loadLocalManifest(ctx context.Context, path string) (*manifest.Manifest, error) {
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
store := &manifest.LocalStore{}
|
||||
return store.Load(ctx, path)
|
||||
}
|
||||
|
||||
func writeStageStatuses(out io.Writer, m *manifest.Manifest) {
|
||||
if m == nil || len(m.Stages) == 0 {
|
||||
fmt.Fprintln(out, "stages: no stages recorded")
|
||||
return
|
||||
}
|
||||
fmt.Fprintln(out, "stages:")
|
||||
names := make([]string, 0, len(m.Stages))
|
||||
for name := range m.Stages {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, name := range names {
|
||||
fmt.Fprintf(out, "- %s: %s\n", name, m.Stages[name].Status)
|
||||
}
|
||||
}
|
||||
131
internal/app/operator_helpers.go
Normal file
131
internal/app/operator_helpers.go
Normal file
@@ -0,0 +1,131 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
)
|
||||
|
||||
type commonConfigFlags struct {
|
||||
pipelinePath string
|
||||
campaignPath string
|
||||
campaignFilePath string
|
||||
sessionPath string
|
||||
sessionID string
|
||||
previousSessionID string
|
||||
}
|
||||
|
||||
func addCommonConfigFlags(fs *flag.FlagSet, flags *commonConfigFlags) {
|
||||
fs.StringVar(&flags.pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
||||
fs.StringVar(&flags.campaignPath, "campaign", "", "campaign ID")
|
||||
fs.StringVar(&flags.campaignFilePath, "campaign-file", "", "path to campaign.yml")
|
||||
fs.StringVar(&flags.sessionPath, "session", "", "path to session.yml")
|
||||
fs.StringVar(&flags.sessionID, "session-id", "", "session identifier")
|
||||
fs.StringVar(&flags.previousSessionID, "previous-session-id", "", "expected previous session identifier")
|
||||
}
|
||||
|
||||
func (f commonConfigFlags) sessionOptions() config.SessionLoadOptions {
|
||||
return config.SessionLoadOptions{
|
||||
SessionID: f.sessionID,
|
||||
PreviousSessionID: f.previousSessionID,
|
||||
}
|
||||
}
|
||||
|
||||
// Session dispatches session helper subcommands.
|
||||
func Session(ctx context.Context, args []string, out io.Writer) error {
|
||||
if len(args) == 0 {
|
||||
return fmt.Errorf("session: expected subcommand: init|validate|status|plan|restore|artifacts|locks")
|
||||
}
|
||||
switch args[0] {
|
||||
case "init":
|
||||
return SessionInit(ctx, args[1:], out)
|
||||
case "validate":
|
||||
return SessionValidate(ctx, args[1:], out)
|
||||
case "status":
|
||||
return Status(ctx, args[1:], out)
|
||||
case "plan":
|
||||
return Plan(ctx, args[1:], out)
|
||||
case "restore":
|
||||
return Restore(ctx, args[1:], out)
|
||||
case "artifacts":
|
||||
return ArtifactsList(ctx, args[1:], out)
|
||||
case "locks":
|
||||
return SessionLocks(ctx, args[1:], out)
|
||||
default:
|
||||
return fmt.Errorf("session: unknown subcommand %q", args[0])
|
||||
}
|
||||
}
|
||||
|
||||
// SessionLocks dispatches session-oriented publish lock list and mutation
|
||||
// helpers while preserving the existing lock implementations.
|
||||
func SessionLocks(ctx context.Context, args []string, out io.Writer) error {
|
||||
if len(args) > 0 && !isCLIFlagToken(args[0]) {
|
||||
switch args[0] {
|
||||
case "add":
|
||||
return LocksAdd(ctx, args[1:], out)
|
||||
case "remove":
|
||||
return LocksRemove(ctx, args[1:], out)
|
||||
}
|
||||
}
|
||||
return LocksList(ctx, args, out)
|
||||
}
|
||||
|
||||
// Artifacts dispatches artifact helper subcommands.
|
||||
func Artifacts(ctx context.Context, args []string, out io.Writer) error {
|
||||
if len(args) == 0 {
|
||||
return fmt.Errorf("artifacts: expected subcommand: list")
|
||||
}
|
||||
switch args[0] {
|
||||
case "list":
|
||||
return ArtifactsList(ctx, args[1:], out)
|
||||
default:
|
||||
return fmt.Errorf("artifacts: unknown subcommand %q", args[0])
|
||||
}
|
||||
}
|
||||
|
||||
func loadHelperContext(ctx context.Context, flags commonConfigFlags, needStore bool) (*config.Config, storage.ObjectStore, *effectiveLocks, *manifest.Manifest, error) {
|
||||
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, err
|
||||
}
|
||||
if err := config.Validate(cfg); err != nil {
|
||||
return nil, nil, nil, nil, err
|
||||
}
|
||||
var store storage.ObjectStore
|
||||
if needStore {
|
||||
store, err = newCommandObjectStore(ctx, cfg, nil)
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, err
|
||||
}
|
||||
} else {
|
||||
store, _ = objectStoreIfConfigured(ctx, cfg)
|
||||
}
|
||||
locks, err := loadEffectiveLocks(ctx, cfg, store)
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, err
|
||||
}
|
||||
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
m, err := loadLocalManifest(ctx, paths.ManifestPath)
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, err
|
||||
}
|
||||
return cfg, store, locks, m, nil
|
||||
}
|
||||
|
||||
func objectStoreIfConfigured(ctx context.Context, cfg *config.Config) (storage.ObjectStore, error) {
|
||||
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Storage.S3 == nil || strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket) == "" {
|
||||
return nil, nil
|
||||
}
|
||||
store, err := newCommandObjectStore(ctx, cfg, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return store, nil
|
||||
}
|
||||
1031
internal/app/operator_helpers_test.go
Normal file
1031
internal/app/operator_helpers_test.go
Normal file
File diff suppressed because it is too large
Load Diff
222
internal/app/operator_locks.go
Normal file
222
internal/app/operator_locks.go
Normal file
@@ -0,0 +1,222 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
// Locks dispatches publish lock list and mutation helpers.
|
||||
func Locks(ctx context.Context, args []string, out io.Writer) error {
|
||||
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||
switch args[0] {
|
||||
case "add":
|
||||
return LocksAdd(ctx, args[1:], out)
|
||||
case "remove":
|
||||
return LocksRemove(ctx, args[1:], out)
|
||||
default:
|
||||
return fmt.Errorf("locks: unknown subcommand %q", args[0])
|
||||
}
|
||||
}
|
||||
return LocksList(ctx, args, out)
|
||||
}
|
||||
|
||||
// LocksList lists effective publish locks.
|
||||
func LocksList(ctx context.Context, args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("locks", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
var flags commonConfigFlags
|
||||
addCommonConfigFlags(fs, &flags)
|
||||
if err := parseSessionAwareFlags("locks", fs, args, &flags.sessionID); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(flags.sessionID) == "" {
|
||||
return fmt.Errorf("locks: session_id is required")
|
||||
}
|
||||
cfg, _, locks, _, err := loadHelperContext(ctx, flags, true)
|
||||
if err != nil {
|
||||
return fmt.Errorf("locks: %w", err)
|
||||
}
|
||||
writeLocks(out, cfg, locks)
|
||||
return nil
|
||||
}
|
||||
|
||||
// LocksAdd adds or updates one remote lock.
|
||||
func LocksAdd(ctx context.Context, args []string, out io.Writer) error {
|
||||
var positionalSessionID string
|
||||
var source string
|
||||
if len(args) >= 2 && !isCLIFlagToken(args[0]) && !isCLIFlagToken(args[1]) {
|
||||
positionalSessionID = strings.TrimSpace(args[0])
|
||||
source = strings.TrimSpace(args[1])
|
||||
args = append([]string(nil), args[2:]...)
|
||||
}
|
||||
fs := flag.NewFlagSet("locks add", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
var flags commonConfigFlags
|
||||
var reason string
|
||||
var force bool
|
||||
addCommonConfigFlags(fs, &flags)
|
||||
fs.StringVar(&reason, "reason", "", "lock reason")
|
||||
fs.BoolVar(&force, "force", false, "update existing remote lock")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return fmt.Errorf("locks add: invalid flags: %w", err)
|
||||
}
|
||||
if source == "" {
|
||||
switch fs.NArg() {
|
||||
case 2:
|
||||
positionalSessionID = strings.TrimSpace(fs.Arg(0))
|
||||
source = strings.TrimSpace(fs.Arg(1))
|
||||
case 1:
|
||||
if strings.TrimSpace(flags.sessionID) == "" {
|
||||
return fmt.Errorf("locks add: expected session_id and source id")
|
||||
}
|
||||
source = strings.TrimSpace(fs.Arg(0))
|
||||
default:
|
||||
return fmt.Errorf("locks add: expected session_id and source id")
|
||||
}
|
||||
} else if fs.NArg() != 0 {
|
||||
return fmt.Errorf("locks add: unexpected positional arguments")
|
||||
}
|
||||
if err := applyPositionalSessionID("locks add", positionalSessionID, &flags.sessionID); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(flags.sessionID) == "" {
|
||||
return fmt.Errorf("locks add: session_id is required")
|
||||
}
|
||||
cfg, store, locks, _, err := loadHelperContext(ctx, flags, true)
|
||||
if err != nil {
|
||||
return fmt.Errorf("locks add: %w", err)
|
||||
}
|
||||
if _, err := config.ValidatePublishLockRules([]config.PublishLockRule{{Source: source}}, cfg.Pipeline.Scriptorium, "locks add"); err != nil {
|
||||
return fmt.Errorf("locks add: %w", err)
|
||||
}
|
||||
if _, ok := lockSourceSet(locks.Static)[source]; ok {
|
||||
return fmt.Errorf("locks add: source %q is locked by pipeline config and cannot be modified remotely", source)
|
||||
}
|
||||
remoteSet := lockSourceSet(locks.Remote)
|
||||
if _, exists := remoteSet[source]; exists && !force {
|
||||
return fmt.Errorf("locks add: remote lock for %q already exists; pass --force to update", source)
|
||||
}
|
||||
remoteSet[source] = config.PublishLockRule{Source: source, Reason: strings.TrimSpace(reason)}
|
||||
remoteLocks := lockMapValues(remoteSet)
|
||||
if _, err := config.ValidatePublishLockRules(remoteLocks, cfg.Pipeline.Scriptorium, "locks"); err != nil {
|
||||
return fmt.Errorf("locks add: %w", err)
|
||||
}
|
||||
if err := uploadRemoteLockStore(ctx, store, locks.Key, &config.PublishLockStore{Locks: remoteLocks}); err != nil {
|
||||
return fmt.Errorf("locks add: %w", err)
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "narratio session locks add: locked %s\n", source)
|
||||
return err
|
||||
}
|
||||
|
||||
// LocksRemove removes one remote lock.
|
||||
func LocksRemove(ctx context.Context, args []string, out io.Writer) error {
|
||||
var positionalSessionID string
|
||||
var source string
|
||||
if len(args) >= 2 && !isCLIFlagToken(args[0]) && !isCLIFlagToken(args[1]) {
|
||||
positionalSessionID = strings.TrimSpace(args[0])
|
||||
source = strings.TrimSpace(args[1])
|
||||
args = append([]string(nil), args[2:]...)
|
||||
}
|
||||
fs := flag.NewFlagSet("locks remove", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
var flags commonConfigFlags
|
||||
addCommonConfigFlags(fs, &flags)
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return fmt.Errorf("locks remove: invalid flags: %w", err)
|
||||
}
|
||||
if source == "" {
|
||||
switch fs.NArg() {
|
||||
case 2:
|
||||
positionalSessionID = strings.TrimSpace(fs.Arg(0))
|
||||
source = strings.TrimSpace(fs.Arg(1))
|
||||
case 1:
|
||||
if strings.TrimSpace(flags.sessionID) == "" {
|
||||
return fmt.Errorf("locks remove: expected session_id and source id")
|
||||
}
|
||||
source = strings.TrimSpace(fs.Arg(0))
|
||||
default:
|
||||
return fmt.Errorf("locks remove: expected session_id and source id")
|
||||
}
|
||||
} else if fs.NArg() != 0 {
|
||||
return fmt.Errorf("locks remove: unexpected positional arguments")
|
||||
}
|
||||
if err := applyPositionalSessionID("locks remove", positionalSessionID, &flags.sessionID); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(flags.sessionID) == "" {
|
||||
return fmt.Errorf("locks remove: session_id is required")
|
||||
}
|
||||
cfg, store, locks, _, err := loadHelperContext(ctx, flags, true)
|
||||
if err != nil {
|
||||
return fmt.Errorf("locks remove: %w", err)
|
||||
}
|
||||
if _, err := config.ValidatePublishLockRules([]config.PublishLockRule{{Source: source}}, cfg.Pipeline.Scriptorium, "locks remove"); err != nil {
|
||||
return fmt.Errorf("locks remove: %w", err)
|
||||
}
|
||||
remoteSet := lockSourceSet(locks.Remote)
|
||||
if _, ok := remoteSet[source]; !ok {
|
||||
if _, static := lockSourceSet(locks.Static)[source]; static {
|
||||
return fmt.Errorf("locks remove: source %q is locked by pipeline config and cannot be unlocked remotely", source)
|
||||
}
|
||||
return fmt.Errorf("locks remove: remote lock for %q does not exist", source)
|
||||
}
|
||||
delete(remoteSet, source)
|
||||
remoteLocks := lockMapValues(remoteSet)
|
||||
if err := uploadRemoteLockStore(ctx, store, locks.Key, &config.PublishLockStore{Locks: remoteLocks}); err != nil {
|
||||
return fmt.Errorf("locks remove: %w", err)
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "narratio session locks remove: unlocked %s\n", source)
|
||||
return err
|
||||
}
|
||||
|
||||
func writeLocks(out io.Writer, cfg *config.Config, locks *effectiveLocks) {
|
||||
if locks == nil || len(locks.All) == 0 {
|
||||
fmt.Fprintln(out, "Publish locks: none")
|
||||
return
|
||||
}
|
||||
fmt.Fprintln(out, "Publish locks:")
|
||||
published := map[string]config.PublishOutputRule{}
|
||||
if cfg != nil && cfg.Pipeline != nil && cfg.Pipeline.Publish != nil {
|
||||
for _, rule := range cfg.Pipeline.Publish.Outputs {
|
||||
published[strings.TrimSpace(rule.Source)] = rule
|
||||
}
|
||||
}
|
||||
staticSet := lockSourceSet(locks.Static)
|
||||
for _, lock := range locks.All {
|
||||
origin := "remote"
|
||||
if _, ok := staticSet[lock.Source]; ok {
|
||||
origin = "pipeline"
|
||||
}
|
||||
promo := "not-published"
|
||||
if _, ok := published[lock.Source]; ok {
|
||||
promo = "published"
|
||||
}
|
||||
reason := strings.TrimSpace(lock.Reason)
|
||||
if reason == "" {
|
||||
reason = "(no reason)"
|
||||
}
|
||||
fmt.Fprintf(out, "- %s origin=%s %s reason=%s\n", lock.Source, origin, promo, reason)
|
||||
}
|
||||
}
|
||||
|
||||
func lockMapValues(in map[string]config.PublishLockRule) []config.PublishLockRule {
|
||||
keys := make([]string, 0, len(in))
|
||||
for key := range in {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
out := make([]config.PublishLockRule, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
item := in[key]
|
||||
item.Source = key
|
||||
item.Reason = strings.TrimSpace(item.Reason)
|
||||
out = append(out, item)
|
||||
}
|
||||
return out
|
||||
}
|
||||
268
internal/app/operator_session_init.go
Normal file
268
internal/app/operator_session_init.go
Normal file
@@ -0,0 +1,268 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// SessionInit creates a local or remote session.yml skeleton.
|
||||
func SessionInit(ctx context.Context, args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("session init", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
var pipelinePath, campaignPath, campaignFilePath, sessionID, previousSessionID, date, title, output, audioS3Prefix, audioDir string
|
||||
var remote, force bool
|
||||
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
||||
fs.StringVar(&campaignPath, "campaign", "", "campaign ID")
|
||||
fs.StringVar(&campaignFilePath, "campaign-file", "", "path to campaign.yml")
|
||||
fs.StringVar(&sessionID, "session-id", "", "session identifier")
|
||||
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier")
|
||||
fs.StringVar(&date, "date", "", "session date")
|
||||
fs.StringVar(&title, "title", "", "session title")
|
||||
fs.StringVar(&output, "output", "", "local output session.yml path")
|
||||
fs.StringVar(&audioS3Prefix, "audio-s3-prefix", "", "session audio S3 prefix")
|
||||
fs.StringVar(&audioDir, "audio-dir", "", "local audio directory")
|
||||
fs.BoolVar(&remote, "remote", false, "write session.yml to S3 session prefix")
|
||||
fs.BoolVar(&force, "force", false, "overwrite existing target")
|
||||
if err := parseSessionAwareFlags("session init", fs, args, &sessionID); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(sessionID) == "" {
|
||||
return fmt.Errorf("session init: session_id is required")
|
||||
}
|
||||
if (strings.TrimSpace(output) == "") == !remote {
|
||||
return fmt.Errorf("session init: specify exactly one target: --output <path> or --remote")
|
||||
}
|
||||
if strings.TrimSpace(audioDir) != "" && strings.TrimSpace(audioS3Prefix) != "" {
|
||||
return fmt.Errorf("session init: --audio-dir and --audio-s3-prefix are mutually exclusive")
|
||||
}
|
||||
|
||||
base, err := loadPipelineCampaignConfig(pipelinePath, campaignPath, campaignFilePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("session init: %w", err)
|
||||
}
|
||||
|
||||
input := sessionInitInput{
|
||||
Campaign: config.CampaignID(base.Campaign),
|
||||
CampaignPath: base.CampaignPath,
|
||||
TemplateFile: base.Campaign.SessionTemplateFile,
|
||||
SessionID: sessionID,
|
||||
PreviousSessionID: previousSessionID,
|
||||
Date: date,
|
||||
Title: title,
|
||||
AudioS3Prefix: audioS3Prefix,
|
||||
AudioDir: audioDir,
|
||||
}
|
||||
data, err := buildSessionInitYAML(input)
|
||||
if err != nil {
|
||||
return fmt.Errorf("session init: %w", err)
|
||||
}
|
||||
label := strings.TrimSpace(output)
|
||||
if label == "" {
|
||||
label = "remote session.yml"
|
||||
}
|
||||
sessionCfg, err := config.LoadSessionBytesWithOptions(label, data, config.SessionLoadOptions{
|
||||
SessionID: sessionID,
|
||||
PreviousSessionID: previousSessionID,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("session init: %w", err)
|
||||
}
|
||||
cfg, err := config.Resolve(base.PipelinePath, base.Pipeline, base.CampaignPath, base.Campaign, label, sessionCfg, config.SessionSource{Source: "session_config", LocalPath: label})
|
||||
if err != nil {
|
||||
return fmt.Errorf("session init: %w", err)
|
||||
}
|
||||
if err := config.Validate(cfg); err != nil {
|
||||
return fmt.Errorf("session init: %w", err)
|
||||
}
|
||||
|
||||
if !remote {
|
||||
if err := writeLocalFile(output, data, force); err != nil {
|
||||
return fmt.Errorf("session init: %w", err)
|
||||
}
|
||||
_, err := fmt.Fprintf(out, "narratio session init: wrote %s\n", filepath.Clean(output))
|
||||
return err
|
||||
}
|
||||
|
||||
store, err := newCommandObjectStore(ctx, cfg, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("session init: %w", err)
|
||||
}
|
||||
sessionPrefix := artifacts.S3SessionPrefix(base.Pipeline.Storage.S3.RootPrefix, config.CampaignID(base.Campaign), sessionID)
|
||||
key := artifacts.S3SessionConfigKey(sessionPrefix)
|
||||
exists, err := store.Exists(ctx, key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("session init: check remote session %q: %w", key, err)
|
||||
}
|
||||
if exists && !force {
|
||||
return fmt.Errorf("session init: remote session %q already exists; pass --force to overwrite", key)
|
||||
}
|
||||
tmp, err := os.CreateTemp("", "narratio-session-init-*.yml")
|
||||
if err != nil {
|
||||
return fmt.Errorf("session init: create temp file: %w", err)
|
||||
}
|
||||
tmpPath := tmp.Name()
|
||||
defer func() { _ = os.Remove(tmpPath) }()
|
||||
if _, err := tmp.Write(data); err != nil {
|
||||
_ = tmp.Close()
|
||||
return fmt.Errorf("session init: write temp file: %w", err)
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return fmt.Errorf("session init: close temp file: %w", err)
|
||||
}
|
||||
if _, err := store.Upload(ctx, tmpPath, key, storage.UploadOptions{ContentType: "application/x-yaml; charset=utf-8"}); err != nil {
|
||||
return fmt.Errorf("session init: upload remote session %q: %w", key, err)
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "narratio session init: wrote s3://%s/%s\n", s3BucketName(base.Pipeline), key)
|
||||
return err
|
||||
}
|
||||
|
||||
func buildSessionYAML(campaign, sessionID, previousSessionID, date, title, audioS3Prefix, audioDir string) ([]byte, error) {
|
||||
if strings.TrimSpace(date) == "" && regexp.MustCompile(`^\d{4}-\d{2}-\d{2}$`).MatchString(strings.TrimSpace(sessionID)) {
|
||||
date = strings.TrimSpace(sessionID)
|
||||
}
|
||||
type audioS3 struct {
|
||||
Prefix string `yaml:"prefix"`
|
||||
}
|
||||
type inputs struct {
|
||||
AudioDir string `yaml:"audio_dir,omitempty"`
|
||||
AudioS3 *audioS3 `yaml:"audio_s3,omitempty"`
|
||||
}
|
||||
type sessionYAML struct {
|
||||
Campaign string `yaml:"campaign"`
|
||||
SessionID string `yaml:"session_id"`
|
||||
PreviousSessionID string `yaml:"previous_session_id,omitempty"`
|
||||
Date string `yaml:"date,omitempty"`
|
||||
Title string `yaml:"title,omitempty"`
|
||||
Inputs inputs `yaml:"inputs"`
|
||||
}
|
||||
in := inputs{AudioDir: strings.TrimSpace(audioDir)}
|
||||
if in.AudioDir == "" {
|
||||
prefix := strings.TrimSpace(audioS3Prefix)
|
||||
if prefix == "" {
|
||||
prefix = "audio/"
|
||||
}
|
||||
in.AudioS3 = &audioS3{Prefix: prefix}
|
||||
}
|
||||
data, err := yaml.Marshal(sessionYAML{
|
||||
Campaign: strings.TrimSpace(campaign),
|
||||
SessionID: strings.TrimSpace(sessionID),
|
||||
PreviousSessionID: strings.TrimSpace(previousSessionID),
|
||||
Date: strings.TrimSpace(date),
|
||||
Title: strings.TrimSpace(title),
|
||||
Inputs: in,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
type sessionInitInput struct {
|
||||
Campaign string
|
||||
CampaignPath string
|
||||
TemplateFile string
|
||||
SessionID string
|
||||
PreviousSessionID string
|
||||
Date string
|
||||
Title string
|
||||
AudioS3Prefix string
|
||||
AudioDir string
|
||||
}
|
||||
|
||||
func buildSessionInitYAML(in sessionInitInput) ([]byte, error) {
|
||||
if strings.TrimSpace(in.TemplateFile) == "" {
|
||||
return buildSessionYAML(in.Campaign, in.SessionID, in.PreviousSessionID, in.Date, in.Title, in.AudioS3Prefix, in.AudioDir)
|
||||
}
|
||||
templatePath := resolveSessionInitTemplatePath(in.CampaignPath, in.TemplateFile)
|
||||
templateBytes, err := os.ReadFile(templatePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read session template %q: %w", templatePath, err)
|
||||
}
|
||||
rendered, err := renderSessionInitTemplate(string(templateBytes), in)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("render session template %q: %w", templatePath, err)
|
||||
}
|
||||
return []byte(rendered), nil
|
||||
}
|
||||
|
||||
func resolveSessionInitTemplatePath(campaignPath, templateFile string) string {
|
||||
templateFile = strings.TrimSpace(templateFile)
|
||||
if filepath.IsAbs(templateFile) {
|
||||
return filepath.Clean(templateFile)
|
||||
}
|
||||
return filepath.Clean(filepath.Join(filepath.Dir(campaignPath), templateFile))
|
||||
}
|
||||
|
||||
var sessionInitTemplatePattern = regexp.MustCompile(`\{\{\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\}\}`)
|
||||
|
||||
func renderSessionInitTemplate(content string, in sessionInitInput) (string, error) {
|
||||
values := map[string]string{
|
||||
"session_id": strings.TrimSpace(in.SessionID),
|
||||
"previous_session_id": strings.TrimSpace(in.PreviousSessionID),
|
||||
"date": strings.TrimSpace(in.Date),
|
||||
"title": strings.TrimSpace(in.Title),
|
||||
"audio_s3_prefix": strings.TrimSpace(in.AudioS3Prefix),
|
||||
"audio_dir": strings.TrimSpace(in.AudioDir),
|
||||
}
|
||||
used := map[string]struct{}{}
|
||||
unknown := map[string]struct{}{}
|
||||
missing := map[string]struct{}{}
|
||||
rendered := sessionInitTemplatePattern.ReplaceAllStringFunc(content, func(match string) string {
|
||||
parts := sessionInitTemplatePattern.FindStringSubmatch(match)
|
||||
if len(parts) < 2 {
|
||||
return match
|
||||
}
|
||||
name := parts[1]
|
||||
value, ok := values[name]
|
||||
if !ok {
|
||||
unknown[name] = struct{}{}
|
||||
return match
|
||||
}
|
||||
used[name] = struct{}{}
|
||||
if value == "" {
|
||||
missing[name] = struct{}{}
|
||||
return match
|
||||
}
|
||||
return value
|
||||
})
|
||||
if len(unknown) > 0 {
|
||||
return "", fmt.Errorf("unsupported template variable(s): %s", sortedStringSet(unknown))
|
||||
}
|
||||
if len(missing) > 0 {
|
||||
return "", fmt.Errorf("missing required template variable value(s): %s", sortedStringSet(missing))
|
||||
}
|
||||
unused := map[string]struct{}{}
|
||||
for _, name := range []string{"previous_session_id", "date", "title", "audio_s3_prefix", "audio_dir"} {
|
||||
if values[name] == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := used[name]; !ok {
|
||||
unused[name] = struct{}{}
|
||||
}
|
||||
}
|
||||
if len(unused) > 0 {
|
||||
return "", fmt.Errorf("unused template variable value(s): %s", sortedStringSet(unused))
|
||||
}
|
||||
return rendered, nil
|
||||
}
|
||||
|
||||
func sortedStringSet(set map[string]struct{}) string {
|
||||
items := make([]string, 0, len(set))
|
||||
for item := range set {
|
||||
items = append(items, item)
|
||||
}
|
||||
sort.Strings(items)
|
||||
return strings.Join(items, ", ")
|
||||
}
|
||||
81
internal/app/operator_session_validate.go
Normal file
81
internal/app/operator_session_validate.go
Normal file
@@ -0,0 +1,81 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
// SessionValidate performs a read-only session preflight.
|
||||
func SessionValidate(ctx context.Context, args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("session validate", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
var flags commonConfigFlags
|
||||
addCommonConfigFlags(fs, &flags)
|
||||
if err := parseSessionAwareFlags("session validate", fs, args, &flags.sessionID); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(flags.sessionID) == "" {
|
||||
return fmt.Errorf("session validate: session_id is required")
|
||||
}
|
||||
|
||||
findings := []finding{}
|
||||
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
||||
if err != nil {
|
||||
findings = append(findings, errorFinding("config", err.Error()))
|
||||
return renderFindings(out, "", "", findings)
|
||||
}
|
||||
if err := config.Validate(cfg); err != nil {
|
||||
findings = append(findings, errorFinding("config", err.Error()))
|
||||
} else {
|
||||
findings = append(findings, okFinding("config", "resolved pipeline, campaign, and session config"))
|
||||
}
|
||||
findings = append(findings, okFinding("session", fmt.Sprintf("session source: %s", sessionSourceSummary(cfg))))
|
||||
|
||||
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
findings = append(findings, validateStableInputFindings(cfg)...)
|
||||
findings = append(findings, validateLocalAudioFindings(cfg)...)
|
||||
|
||||
store, storeErr := objectStoreIfConfigured(ctx, cfg)
|
||||
if storeErr != nil {
|
||||
findings = append(findings, errorFinding("storage", storeErr.Error()))
|
||||
}
|
||||
if cfg.Session.Inputs.AudioS3 != nil {
|
||||
if storeErr != nil {
|
||||
findings = append(findings, errorFinding("audio", "remote audio cannot be checked because storage is unavailable"))
|
||||
} else {
|
||||
findings = append(findings, validateRemoteAudioFinding(ctx, cfg, store))
|
||||
}
|
||||
}
|
||||
|
||||
requirements := artifacts.CollectPreviousArtifactRequirements(configuredScriptoriumArtifacts(cfg))
|
||||
if len(requirements) == 0 {
|
||||
findings = append(findings, okFinding("previous", "no previous-session artifacts required"))
|
||||
} else if strings.TrimSpace(cfg.Session.PreviousSessionID) == "" {
|
||||
findings = append(findings, errorFinding("previous", "previous_session_id is required by configured previous-session artifacts"))
|
||||
} else if storeErr != nil {
|
||||
findings = append(findings, errorFinding("previous", "previous-session artifacts cannot be checked because storage is unavailable"))
|
||||
} else {
|
||||
findings = append(findings, validatePreviousArtifactFindings(ctx, cfg, store, requirements)...)
|
||||
}
|
||||
|
||||
locks, lockErr := loadEffectiveLocks(ctx, cfg, store)
|
||||
if lockErr != nil {
|
||||
findings = append(findings, errorFinding("locks", lockErr.Error()))
|
||||
} else if len(locks.All) == 0 {
|
||||
findings = append(findings, okFinding("locks", "no effective publish locks"))
|
||||
} else {
|
||||
for _, lock := range locks.All {
|
||||
findings = append(findings, warnFinding("locks", fmt.Sprintf("%s locked: %s", lock.Source, strings.TrimSpace(lock.Reason))))
|
||||
}
|
||||
}
|
||||
if paths.ManifestPath != "" {
|
||||
findings = append(findings, infoFinding("workspace", "manifest path: "+paths.ManifestPath))
|
||||
}
|
||||
return renderFindings(out, cfg.Session.Campaign, cfg.Session.SessionID, findings)
|
||||
}
|
||||
88
internal/app/operator_status.go
Normal file
88
internal/app/operator_status.go
Normal file
@@ -0,0 +1,88 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
// Status reports effective local/remote session state.
|
||||
func Status(ctx context.Context, args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("status", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
var flags commonConfigFlags
|
||||
addCommonConfigFlags(fs, &flags)
|
||||
if err := parseSessionAwareFlags("status", fs, args, &flags.sessionID); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(flags.sessionID) == "" {
|
||||
return fmt.Errorf("status: session_id is required")
|
||||
}
|
||||
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
||||
if err != nil {
|
||||
return fmt.Errorf("status: %w", err)
|
||||
}
|
||||
if err := config.Validate(cfg); err != nil {
|
||||
return fmt.Errorf("status: %w", err)
|
||||
}
|
||||
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
fmt.Fprintf(out, "Session: %s\n", cfg.Session.SessionID)
|
||||
fmt.Fprintf(out, "Campaign: %s\n", cfg.Session.Campaign)
|
||||
fmt.Fprintf(out, "Workspace: %s\n", paths.Root)
|
||||
fmt.Fprintf(out, "Session config: %s\n", sessionSourceSummary(cfg))
|
||||
|
||||
if m, err := loadLocalManifest(ctx, paths.ManifestPath); err != nil {
|
||||
fmt.Fprintf(out, "Local manifest: error: %v\n", err)
|
||||
} else if m == nil {
|
||||
fmt.Fprintln(out, "Local manifest: missing")
|
||||
} else {
|
||||
fmt.Fprintf(out, "Local manifest: %s\n", paths.ManifestPath)
|
||||
writeStageStatuses(out, m)
|
||||
}
|
||||
|
||||
store, storeErr := objectStoreIfConfigured(ctx, cfg)
|
||||
if storeErr != nil {
|
||||
fmt.Fprintf(out, "Remote publish: unavailable: %v\n", storeErr)
|
||||
} else if store != nil {
|
||||
current, err := discoverRemoteCurrentStateFn(ctx, cfg, store)
|
||||
if err != nil {
|
||||
fmt.Fprintf(out, "Remote publish: missing or unavailable: %v\n", err)
|
||||
} else {
|
||||
fmt.Fprintf(out, "Remote publish: current run %s\n", current.RunID)
|
||||
fmt.Fprintf(out, "Remote manifest: %s\n", current.CurrentManifestKey)
|
||||
}
|
||||
}
|
||||
|
||||
locks, err := loadEffectiveLocks(ctx, cfg, store)
|
||||
if catalog, catalogErr := buildHelperArtifactCatalog(cfg); catalogErr != nil {
|
||||
fmt.Fprintf(out, "Remote outputs: error: %v\n", catalogErr)
|
||||
} else if storeErr == nil {
|
||||
catalogLocks := locks
|
||||
if err != nil {
|
||||
catalogLocks = &effectiveLocks{
|
||||
Static: staticPublishLocks(cfg),
|
||||
All: staticPublishLocks(cfg),
|
||||
}
|
||||
}
|
||||
publishedRemoteState := map[string]string{}
|
||||
if store != nil {
|
||||
publishedRemoteState = remotePublishedOutputAvailability(ctx, cfg, store, catalog)
|
||||
}
|
||||
fmt.Fprintln(out, "Remote outputs:")
|
||||
writeArtifactList(out, cfg, catalog, catalogLocks, publishedRemoteState)
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintf(out, "Publish locks: error: %v\n", err)
|
||||
} else {
|
||||
writeLocks(out, cfg, locks)
|
||||
}
|
||||
fmt.Fprintln(out, "Next actions:")
|
||||
fmt.Fprintf(out, "- narratio session validate %s\n", cfg.Session.SessionID)
|
||||
fmt.Fprintf(out, "- narratio session restore %s --dry-run\n", cfg.Session.SessionID)
|
||||
return nil
|
||||
}
|
||||
49
internal/app/pipeline_config_path.go
Normal file
49
internal/app/pipeline_config_path.go
Normal file
@@ -0,0 +1,49 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
func resolvePipelineConfigPath(flagValue string) (string, error) {
|
||||
return resolvePipelineConfigPathWithCandidates(flagValue, config.DefaultPipelineConfigSearchPaths)
|
||||
}
|
||||
|
||||
func resolvePipelineConfigPathWithCandidates(flagValue string, candidates []string) (string, error) {
|
||||
if explicit := strings.TrimSpace(flagValue); explicit != "" {
|
||||
return explicit, nil
|
||||
}
|
||||
|
||||
ordered := make([]string, 0, len(candidates))
|
||||
for _, raw := range candidates {
|
||||
path := strings.TrimSpace(raw)
|
||||
if path == "" {
|
||||
continue
|
||||
}
|
||||
ordered = append(ordered, path)
|
||||
info, err := os.Stat(path)
|
||||
if err == nil {
|
||||
if info.IsDir() {
|
||||
continue
|
||||
}
|
||||
return filepath.Clean(path), nil
|
||||
}
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
return "", fmt.Errorf("check default pipeline config %q: %w", path, err)
|
||||
}
|
||||
|
||||
if len(ordered) == 0 {
|
||||
return "", fmt.Errorf("no pipeline config path provided and no default locations configured")
|
||||
}
|
||||
return "", fmt.Errorf(
|
||||
"no pipeline config path provided and no default pipeline config found; searched: %s",
|
||||
strings.Join(ordered, ", "),
|
||||
)
|
||||
}
|
||||
65
internal/app/pipeline_config_path_test.go
Normal file
65
internal/app/pipeline_config_path_test.go
Normal file
@@ -0,0 +1,65 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestResolvePipelineConfigPathWithCandidatesExplicitWins(t *testing.T) {
|
||||
got, err := resolvePipelineConfigPathWithCandidates(" ./custom/pipeline.yml ", []string{"/a", "/b"})
|
||||
if err != nil {
|
||||
t.Fatalf("resolvePipelineConfigPathWithCandidates() error = %v", err)
|
||||
}
|
||||
if got != "./custom/pipeline.yml" {
|
||||
t.Fatalf("resolved path = %q, want explicit path", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineConfigPathWithCandidatesUsesFirstExisting(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
first := filepath.Join(dir, "first.yml")
|
||||
second := filepath.Join(dir, "second.yml")
|
||||
if err := os.WriteFile(second, []byte("workspace:\n root: ./tmp\n"), 0o644); err != nil {
|
||||
t.Fatalf("write second default: %v", err)
|
||||
}
|
||||
|
||||
got, err := resolvePipelineConfigPathWithCandidates("", []string{first, second})
|
||||
if err != nil {
|
||||
t.Fatalf("resolvePipelineConfigPathWithCandidates() error = %v", err)
|
||||
}
|
||||
if got != filepath.Clean(second) {
|
||||
t.Fatalf("resolved path = %q, want %q", got, filepath.Clean(second))
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineConfigPathWithCandidatesPrecedence(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
first := filepath.Join(dir, "first.yml")
|
||||
second := filepath.Join(dir, "second.yml")
|
||||
if err := os.WriteFile(first, []byte("workspace:\n root: ./tmp\n"), 0o644); err != nil {
|
||||
t.Fatalf("write first default: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(second, []byte("workspace:\n root: ./tmp\n"), 0o644); err != nil {
|
||||
t.Fatalf("write second default: %v", err)
|
||||
}
|
||||
|
||||
got, err := resolvePipelineConfigPathWithCandidates("", []string{first, second})
|
||||
if err != nil {
|
||||
t.Fatalf("resolvePipelineConfigPathWithCandidates() error = %v", err)
|
||||
}
|
||||
if got != filepath.Clean(first) {
|
||||
t.Fatalf("resolved path = %q, want first candidate %q", got, filepath.Clean(first))
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineConfigPathWithCandidatesMissing(t *testing.T) {
|
||||
_, err := resolvePipelineConfigPathWithCandidates("", []string{"/does/not/exist/one.yml", "/does/not/exist/two.yml"})
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "no default pipeline config found") {
|
||||
t.Fatalf("error = %q, want missing-defaults context", err.Error())
|
||||
}
|
||||
}
|
||||
@@ -5,9 +5,12 @@ import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/logging"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
)
|
||||
|
||||
@@ -16,33 +19,30 @@ func Plan(ctx context.Context, args []string, out io.Writer) error {
|
||||
fs := flag.NewFlagSet("plan", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
|
||||
var pipelinePath string
|
||||
var sessionPath string
|
||||
var flags commonConfigFlags
|
||||
var force bool
|
||||
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml")
|
||||
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
||||
addCommonConfigFlags(fs, &flags)
|
||||
fs.BoolVar(&force, "force", false, "force stage execution (reserved for future behavior)")
|
||||
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return fmt.Errorf("plan: invalid flags: %w", err)
|
||||
if err := parseSessionAwareFlags("plan", fs, args, &flags.sessionID); err != nil {
|
||||
return err
|
||||
}
|
||||
if fs.NArg() != 0 {
|
||||
return fmt.Errorf("plan: unexpected positional arguments")
|
||||
if flags.sessionID == "" {
|
||||
return fmt.Errorf("plan: session_id is required")
|
||||
}
|
||||
if pipelinePath == "" || sessionPath == "" {
|
||||
return fmt.Errorf("plan: --config and --session are required")
|
||||
}
|
||||
|
||||
cfg, err := config.Load(pipelinePath, sessionPath)
|
||||
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
||||
if err != nil {
|
||||
return fmt.Errorf("plan: %w", err)
|
||||
}
|
||||
if err := config.Validate(cfg); err != nil {
|
||||
return fmt.Errorf("plan: %w", err)
|
||||
}
|
||||
if _, err := loadSecretsFromConfig(cfg, logging.NewLogger(os.Stderr, slog.LevelInfo)); err != nil {
|
||||
return fmt.Errorf("plan: %w", err)
|
||||
}
|
||||
|
||||
store := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root)
|
||||
paths, err := store.EnsureLayout(cfg.Session.SessionID)
|
||||
paths, err := store.EnsureLayoutFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("plan: prepare workdir: %w", err)
|
||||
}
|
||||
@@ -57,7 +57,7 @@ func Plan(ctx context.Context, args []string, out io.Writer) error {
|
||||
|
||||
runCount := 0
|
||||
skipCount := 0
|
||||
if _, err := fmt.Fprintf(out, "narratio plan: workdir prepared at %s\n", paths.Root); err != nil {
|
||||
if _, err := fmt.Fprintf(out, "narratio session plan: workdir prepared at %s\n", paths.Root); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, d := range decisions {
|
||||
|
||||
@@ -15,19 +15,19 @@ import (
|
||||
|
||||
func TestPlanCreatesAndReusesWorkdir(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
|
||||
var out bytes.Buffer
|
||||
args := []string{"--config", pipelinePath, "--session", sessionPath}
|
||||
args := []string{"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}
|
||||
|
||||
if err := Plan(context.Background(), args, &out); err != nil {
|
||||
t.Fatalf("first Plan() error = %v", err)
|
||||
}
|
||||
got := out.String()
|
||||
if !strings.Contains(got, "narratio plan: workdir prepared at") {
|
||||
if !strings.Contains(got, "narratio session plan: workdir prepared at") {
|
||||
t.Fatalf("first output = %q, want workdir prepared", got)
|
||||
}
|
||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze", "archive", "notify"} {
|
||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze", "publish", "notify"} {
|
||||
if !strings.Contains(got, name+": run") {
|
||||
t.Fatalf("first output = %q, missing stage %q", got, name)
|
||||
}
|
||||
@@ -36,7 +36,7 @@ func TestPlanCreatesAndReusesWorkdir(t *testing.T) {
|
||||
t.Fatalf("first output = %q, want totals", got)
|
||||
}
|
||||
|
||||
sessionWorkdir := artifacts.SessionWorkDir(workspaceRoot, "2026-05-03")
|
||||
sessionWorkdir := artifacts.SessionWorkDirForCampaign(workspaceRoot, "sample-campaign", "2026-05-03")
|
||||
expectedDirs := []string{
|
||||
sessionWorkdir,
|
||||
filepath.Join(sessionWorkdir, "inputs"),
|
||||
@@ -55,15 +55,15 @@ func TestPlanCreatesAndReusesWorkdir(t *testing.T) {
|
||||
if err := Plan(context.Background(), args, &out); err != nil {
|
||||
t.Fatalf("second Plan() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(out.String(), "narratio plan: workdir prepared at") {
|
||||
if !strings.Contains(out.String(), "narratio session plan: workdir prepared at") {
|
||||
t.Fatalf("second output = %q, want workdir prepared", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanShowsRunAndSkipFromManifest(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
manifestPath := filepath.Join(workspaceRoot, "work", "2026-05-03", "manifest.json")
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||
|
||||
store := &manifest.LocalStore{}
|
||||
m := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||
@@ -74,7 +74,7 @@ func TestPlanShowsRunAndSkipFromManifest(t *testing.T) {
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
if err := Plan(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath}, &out); err != nil {
|
||||
if err := Plan(context.Background(), []string{"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &out); err != nil {
|
||||
t.Fatalf("Plan() error = %v", err)
|
||||
}
|
||||
got := out.String()
|
||||
@@ -89,6 +89,53 @@ func TestPlanShowsRunAndSkipFromManifest(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanFailsWhenConfiguredSecretsDirMissing(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
configDir := t.TempDir()
|
||||
pipelinePath := filepath.Join(configDir, "pipeline.yml")
|
||||
campaignPath := writeAppTestCampaignConfig(t, configDir)
|
||||
sessionPath := filepath.Join(configDir, "session.yml")
|
||||
|
||||
pipelineYAML := `workspace:
|
||||
root: ` + workspaceRoot + `
|
||||
storage:
|
||||
backend: local
|
||||
secrets:
|
||||
env_dir: ./missing-secrets
|
||||
whisperx:
|
||||
transcribe_url: https://example.com/transcribe
|
||||
seriatim:
|
||||
binary: seriatim
|
||||
audita:
|
||||
binary: audita
|
||||
notification:
|
||||
timeout: 10s
|
||||
`
|
||||
sessionYAML := `session_id: 2026-05-03
|
||||
campaign: sample-campaign
|
||||
inputs:
|
||||
audio_dir: ./audio
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
`
|
||||
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||
t.Fatalf("write pipeline.yml: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
|
||||
t.Fatalf("write session.yml: %v", err)
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
err := Plan(context.Background(), []string{"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &out)
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "read secrets env_dir") {
|
||||
t.Fatalf("error = %q, want secrets read error context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func assertDir(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
info, err := os.Stat(path)
|
||||
|
||||
@@ -4,7 +4,7 @@ import "testing"
|
||||
|
||||
func TestBuildFullPlanOrder(t *testing.T) {
|
||||
got := BuildFullPlan()
|
||||
want := []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze", "archive", "notify"}
|
||||
want := []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze", "publish", "notify"}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("len(plan) = %d, want %d", len(got), len(want))
|
||||
}
|
||||
|
||||
183
internal/app/post_publish_cleanup.go
Normal file
183
internal/app/post_publish_cleanup.go
Normal file
@@ -0,0 +1,183 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
)
|
||||
|
||||
func runPostPublishCleanup(ctx context.Context, env *Env, manifestPath string, m *manifest.Manifest, executed []string) error {
|
||||
if env == nil || env.Config == nil || env.Config.Pipeline == nil || m == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
spoolRequested := env.Config.Pipeline.Spool.DeleteAudioAfterPublish
|
||||
workRequested := env.Config.Pipeline.Workspace.CleanupAfterPublish
|
||||
if !spoolRequested && !workRequested {
|
||||
return nil
|
||||
}
|
||||
|
||||
sr := publishStageRecordForCleanup(m, executed)
|
||||
if sr == nil {
|
||||
return nil
|
||||
}
|
||||
if sr.Metadata == nil {
|
||||
sr.Metadata = map[string]any{}
|
||||
}
|
||||
sr.Metadata["spool_cleanup_requested"] = spoolRequested
|
||||
sr.Metadata["workdir_cleanup_requested"] = workRequested
|
||||
|
||||
eligible, reason := publishCleanupEligible(env.Config, sr)
|
||||
if !eligible {
|
||||
sr.Metadata["cleanup_skipped"] = true
|
||||
sr.Metadata["cleanup_skipped_reason"] = reason
|
||||
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
|
||||
return fmt.Errorf("save manifest cleanup skip metadata %q: %w", manifestPath, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
spoolDir := strings.TrimSpace(m.LocalSpoolDir)
|
||||
if spoolDir == "" {
|
||||
spoolDir = artifacts.SessionSpoolAudioDir(
|
||||
env.Config.Pipeline.Spool.Root,
|
||||
strings.TrimSpace(env.Config.Session.Campaign),
|
||||
strings.TrimSpace(env.Config.Session.SessionID),
|
||||
strings.TrimSpace(m.RunID),
|
||||
)
|
||||
}
|
||||
workDir := strings.TrimSpace(m.LocalWorkDir)
|
||||
if workDir == "" {
|
||||
workDir = artifacts.SessionRunRootForCampaign(
|
||||
env.Config.Pipeline.Workspace.Root,
|
||||
strings.TrimSpace(env.Config.Session.Campaign),
|
||||
strings.TrimSpace(env.Config.Session.SessionID),
|
||||
strings.TrimSpace(m.RunID),
|
||||
)
|
||||
}
|
||||
|
||||
if spoolRequested {
|
||||
if err := removeRunScopedDir(strings.TrimSpace(env.Config.Pipeline.Spool.Root), spoolDir, "pipeline.spool.delete_audio_after_publish"); err != nil {
|
||||
sr.Metadata["cleanup_failed"] = true
|
||||
sr.Metadata["cleanup_failed_policy"] = "pipeline.spool.delete_audio_after_publish"
|
||||
sr.Metadata["cleanup_failed_path"] = spoolDir
|
||||
_ = env.ManifestStore.Save(ctx, manifestPath, m)
|
||||
return err
|
||||
}
|
||||
sr.Metadata["spool_cleanup_deleted"] = filepath.Clean(spoolDir)
|
||||
}
|
||||
|
||||
if !workRequested {
|
||||
sr.Metadata["cleanup_completed"] = true
|
||||
sr.Metadata["cleanup_skipped"] = false
|
||||
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
|
||||
return fmt.Errorf("save manifest cleanup metadata %q: %w", manifestPath, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := removeRunScopedDir(strings.TrimSpace(env.Config.Pipeline.Workspace.Root), workDir, "pipeline.workspace.cleanup_after_publish"); err != nil {
|
||||
sr.Metadata["cleanup_failed"] = true
|
||||
sr.Metadata["cleanup_failed_policy"] = "pipeline.workspace.cleanup_after_publish"
|
||||
sr.Metadata["cleanup_failed_path"] = workDir
|
||||
_ = env.ManifestStore.Save(ctx, manifestPath, m)
|
||||
return err
|
||||
}
|
||||
|
||||
sr.Metadata["workdir_cleanup_deleted"] = filepath.Clean(workDir)
|
||||
sr.Metadata["cleanup_completed"] = true
|
||||
sr.Metadata["cleanup_skipped"] = false
|
||||
return nil
|
||||
}
|
||||
|
||||
func publishStageRecordForCleanup(m *manifest.Manifest, executed []string) *manifest.StageRecord {
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
publishRan := false
|
||||
for _, name := range executed {
|
||||
if name == "publish" {
|
||||
publishRan = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !publishRan {
|
||||
return nil
|
||||
}
|
||||
sr := m.Stages["publish"]
|
||||
if sr == nil || sr.Status != manifest.StatusSucceeded {
|
||||
return nil
|
||||
}
|
||||
return sr
|
||||
}
|
||||
|
||||
func publishCleanupEligible(cfg *config.Config, sr *manifest.StageRecord) (bool, string) {
|
||||
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Publish == nil {
|
||||
return false, "publish configuration is missing"
|
||||
}
|
||||
enabled := true
|
||||
if cfg.Pipeline.Publish.Enabled != nil {
|
||||
enabled = *cfg.Pipeline.Publish.Enabled
|
||||
}
|
||||
if !enabled {
|
||||
return false, "publish.enabled is false"
|
||||
}
|
||||
uploadRun := true
|
||||
if cfg.Pipeline.Publish.UploadRun != nil {
|
||||
uploadRun = *cfg.Pipeline.Publish.UploadRun
|
||||
}
|
||||
if !uploadRun {
|
||||
return false, "publish.upload_run is false"
|
||||
}
|
||||
if sr == nil || sr.Metadata == nil {
|
||||
return false, "publish metadata is missing"
|
||||
}
|
||||
if skipped, _ := sr.Metadata["skipped"].(bool); skipped {
|
||||
return false, "publish stage was skipped"
|
||||
}
|
||||
if uploaded, _ := sr.Metadata["uploaded"].(bool); !uploaded {
|
||||
return false, "publish did not upload run record"
|
||||
}
|
||||
if pointer, _ := sr.Metadata["current_pointer_written"].(bool); !pointer {
|
||||
return false, "publish did not write current pointer"
|
||||
}
|
||||
if strings.TrimSpace(asString(sr.Metadata["current_run_id_key"])) == "" {
|
||||
return false, "publish current run pointer key is missing"
|
||||
}
|
||||
return true, ""
|
||||
}
|
||||
|
||||
type scopedDir struct {
|
||||
RootAbs string
|
||||
TargetAbs string
|
||||
Exists bool
|
||||
}
|
||||
|
||||
func removeRunScopedDir(root, target, policy string) error {
|
||||
dir, err := validateScopedDir(root, target, policy)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !dir.Exists {
|
||||
return nil
|
||||
}
|
||||
if err := os.RemoveAll(dir.TargetAbs); err != nil {
|
||||
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, dir.TargetAbs, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateScopedDir(root, target, policy string) (scopedDir, error) {
|
||||
return validateScopedTarget(root, target, policy, true)
|
||||
}
|
||||
|
||||
func asString(v any) string {
|
||||
s, _ := v.(string)
|
||||
return s
|
||||
}
|
||||
421
internal/app/post_publish_cleanup_test.go
Normal file
421
internal/app/post_publish_cleanup_test.go
Normal file
@@ -0,0 +1,421 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/stage"
|
||||
)
|
||||
|
||||
type publishSuccessStage struct {
|
||||
metadata map[string]any
|
||||
}
|
||||
|
||||
func (publishSuccessStage) Name() string { return "publish" }
|
||||
func (publishSuccessStage) Declares() stage.IODecl { return stage.IODecl{} }
|
||||
func (s publishSuccessStage) Run(_ context.Context, _ *stage.Env, _ *manifest.Manifest) (*stage.StageResult, error) {
|
||||
md := map[string]any{
|
||||
"stage": "publish",
|
||||
"uploaded": true,
|
||||
"current_pointer_written": true,
|
||||
"current_run_id_key": "dnd/campaigns/sample-campaign/sessions/2026-05-03/current/run_id.txt",
|
||||
}
|
||||
for k, v := range s.metadata {
|
||||
md[k] = v
|
||||
}
|
||||
return &stage.StageResult{Metadata: md}, nil
|
||||
}
|
||||
|
||||
type notifyFailStage struct{}
|
||||
|
||||
func (notifyFailStage) Name() string { return "notify" }
|
||||
func (notifyFailStage) Declares() stage.IODecl { return stage.IODecl{} }
|
||||
func (notifyFailStage) Run(_ context.Context, _ *stage.Env, _ *manifest.Manifest) (*stage.StageResult, error) {
|
||||
return nil, errors.New("notify failed")
|
||||
}
|
||||
|
||||
func TestPostPublishCleanupDisabledKeepsLocalDirs(t *testing.T) {
|
||||
cfg, seed := cleanupFixtureConfig(t)
|
||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = false
|
||||
cfg.Pipeline.Workspace.CleanupAfterPublish = false
|
||||
|
||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
|
||||
assertExists(t, seed.spoolAudioDir)
|
||||
assertExists(t, seed.runWorkDir)
|
||||
assertExists(t, seed.localSourceAudio)
|
||||
}
|
||||
|
||||
func TestPostPublishCleanupSpoolOnly(t *testing.T) {
|
||||
cfg, seed := cleanupFixtureConfig(t)
|
||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||
cfg.Pipeline.Workspace.CleanupAfterPublish = false
|
||||
|
||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
|
||||
assertMissing(t, seed.spoolAudioDir)
|
||||
assertExists(t, seed.runWorkDir)
|
||||
assertExists(t, seed.localSourceAudio)
|
||||
}
|
||||
|
||||
func TestPostPublishCleanupWorkdirOnly(t *testing.T) {
|
||||
cfg, seed := cleanupFixtureConfig(t)
|
||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = false
|
||||
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||
|
||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
|
||||
assertExists(t, cfg.Pipeline.Workspace.Root)
|
||||
assertExists(t, seed.otherRunDir)
|
||||
assertExists(t, seed.previousCachePath)
|
||||
assertMissing(t, seed.runWorkDir)
|
||||
assertExists(t, seed.spoolAudioDir)
|
||||
}
|
||||
|
||||
func TestPostPublishCleanupBothPolicies(t *testing.T) {
|
||||
cfg, seed := cleanupFixtureConfig(t)
|
||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||
|
||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
|
||||
assertMissing(t, seed.spoolAudioDir)
|
||||
assertMissing(t, seed.runWorkDir)
|
||||
assertExists(t, seed.otherRunDir)
|
||||
assertExists(t, seed.previousCachePath)
|
||||
}
|
||||
|
||||
func TestPostPublishCleanupNotRunWhenPublishFails(t *testing.T) {
|
||||
cfg, seed := cleanupFixtureConfig(t)
|
||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||
|
||||
_, err := executeStages(context.Background(), cfg, []stage.Stage{failingStage{name: "publish", err: errors.New("publish failed")}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}})
|
||||
if err == nil || !strings.Contains(err.Error(), "stage \"publish\" failed") {
|
||||
t.Fatalf("executeStages() error = %v, want publish failure", err)
|
||||
}
|
||||
|
||||
assertExists(t, seed.spoolAudioDir)
|
||||
assertExists(t, seed.runWorkDir)
|
||||
}
|
||||
|
||||
func TestPostPublishCleanupNotRunWhenPublishSkipped(t *testing.T) {
|
||||
cfg, seed := cleanupFixtureConfig(t)
|
||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||
|
||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{metadata: map[string]any{"skipped": true}}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
|
||||
assertExists(t, seed.spoolAudioDir)
|
||||
assertExists(t, seed.runWorkDir)
|
||||
}
|
||||
|
||||
func TestPostPublishCleanupNotRunWhenCurrentPointerMissing(t *testing.T) {
|
||||
cfg, seed := cleanupFixtureConfig(t)
|
||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||
|
||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{metadata: map[string]any{"current_pointer_written": false}}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
|
||||
assertExists(t, seed.spoolAudioDir)
|
||||
assertExists(t, seed.runWorkDir)
|
||||
}
|
||||
|
||||
func TestPostPublishCleanupNotRunWhenPublishUploadDisabled(t *testing.T) {
|
||||
cfg, seed := cleanupFixtureConfig(t)
|
||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||
cfg.Pipeline.Publish.UploadRun = boolPtr(false)
|
||||
|
||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
|
||||
assertExists(t, seed.spoolAudioDir)
|
||||
assertExists(t, seed.runWorkDir)
|
||||
}
|
||||
|
||||
func TestPostPublishCleanupWaitsUntilAllStagesSucceed(t *testing.T) {
|
||||
cfg, seed := cleanupFixtureConfig(t)
|
||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||
|
||||
_, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{}, notifyFailStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}})
|
||||
if err == nil || !strings.Contains(err.Error(), "stage \"notify\" failed") {
|
||||
t.Fatalf("executeStages() error = %v, want notify failure", err)
|
||||
}
|
||||
|
||||
assertExists(t, seed.spoolAudioDir)
|
||||
assertExists(t, seed.runWorkDir)
|
||||
}
|
||||
|
||||
func TestPostPublishCleanupFailsOnUnsafePath(t *testing.T) {
|
||||
cfg, _ := cleanupFixtureConfig(t)
|
||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||
cfg.Pipeline.Workspace.CleanupAfterPublish = false
|
||||
|
||||
manifestPath := manifestPathFor(cfg)
|
||||
store := &manifest.LocalStore{}
|
||||
m, err := store.Load(context.Background(), manifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
m.LocalSpoolDir = filepath.Join(filepath.Dir(cfg.Pipeline.Spool.Root), "outside-spool")
|
||||
if err := store.Save(context.Background(), manifestPath, m); err != nil {
|
||||
t.Fatalf("Save() error = %v", err)
|
||||
}
|
||||
|
||||
_, err = executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}})
|
||||
if err == nil || !strings.Contains(err.Error(), "refusing to delete path outside root") {
|
||||
t.Fatalf("executeStages() error = %v, want safe-path failure", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostPublishCleanupNotRunWhenOutputIsMissing(t *testing.T) {
|
||||
cfg, seed, runID := publishStageCleanupFixture(t)
|
||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||
cfg.Pipeline.Publish.Outputs = []config.PublishOutputRule{
|
||||
{Source: "narratio.transcript.base", Dest: "transcripts/base.json", Required: boolPtr(true)},
|
||||
}
|
||||
|
||||
publishStageImpl, err := stage.Select("publish")
|
||||
if err != nil {
|
||||
t.Fatalf("Select(publish) error = %v", err)
|
||||
}
|
||||
_, err = executeStages(context.Background(), cfg, []stage.Stage{publishStageImpl}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}})
|
||||
if err == nil || !strings.Contains(err.Error(), "required output source unavailable") {
|
||||
t.Fatalf("executeStages() error = %v, want required output source unavailable failure", err)
|
||||
}
|
||||
|
||||
assertExists(t, seed.spoolAudioDir)
|
||||
assertExists(t, seed.runWorkDir)
|
||||
assertExists(t, filepath.Join(seed.runWorkDir, "manifest.json"))
|
||||
assertExists(t, artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, runID))
|
||||
}
|
||||
|
||||
func TestPostPublishCleanupNotRunWhenCurrentManifestUploadFails(t *testing.T) {
|
||||
cfg, seed, _ := publishStageCleanupFixture(t)
|
||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||
failKey := seed.sessionPrefix + "current/manifest.json"
|
||||
|
||||
publishStageImpl, err := stage.Select("publish")
|
||||
if err != nil {
|
||||
t.Fatalf("Select(publish) error = %v", err)
|
||||
}
|
||||
_, err = executeStages(context.Background(), cfg, []stage.Stage{publishStageImpl}, RunOptions{
|
||||
Env: &Env{ObjectStore: &failKeyStore{delegate: &storage.FakeBackend{}, failKey: failKey}},
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "current manifest") {
|
||||
t.Fatalf("executeStages() error = %v, want current-manifest failure", err)
|
||||
}
|
||||
|
||||
assertExists(t, seed.spoolAudioDir)
|
||||
assertExists(t, seed.runWorkDir)
|
||||
}
|
||||
|
||||
func TestPostPublishCleanupNotRunWhenCurrentPointerUploadFails(t *testing.T) {
|
||||
cfg, seed, _ := publishStageCleanupFixture(t)
|
||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||
failKey := seed.sessionPrefix + "current/run_id.txt"
|
||||
|
||||
publishStageImpl, err := stage.Select("publish")
|
||||
if err != nil {
|
||||
t.Fatalf("Select(publish) error = %v", err)
|
||||
}
|
||||
_, err = executeStages(context.Background(), cfg, []stage.Stage{publishStageImpl}, RunOptions{
|
||||
Env: &Env{ObjectStore: &failKeyStore{delegate: &storage.FakeBackend{}, failKey: failKey}},
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "current run pointer") {
|
||||
t.Fatalf("executeStages() error = %v, want current-run-pointer failure", err)
|
||||
}
|
||||
|
||||
assertExists(t, seed.spoolAudioDir)
|
||||
assertExists(t, seed.runWorkDir)
|
||||
}
|
||||
|
||||
type cleanupSeed struct {
|
||||
runWorkDir string
|
||||
otherRunDir string
|
||||
spoolAudioDir string
|
||||
localSourceAudio string
|
||||
previousCachePath string
|
||||
sessionPrefix string
|
||||
}
|
||||
|
||||
func cleanupFixtureConfig(t *testing.T) (*config.Config, cleanupSeed) {
|
||||
t.Helper()
|
||||
|
||||
cfg := testConfig(t)
|
||||
cfg.Pipeline.Publish = &config.PublishConfig{Enabled: boolPtr(true), UploadRun: boolPtr(true)}
|
||||
cfg.Pipeline.Spool.Root = filepath.Join(t.TempDir(), "spool")
|
||||
|
||||
runID := "20260516T010203Z-1a2b3c4d"
|
||||
runWorkDir := artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, runID)
|
||||
otherRunDir := artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, "20260516T010204Z-5e6f7a8b")
|
||||
spoolAudioDir := artifacts.SessionSpoolAudioDir(cfg.Pipeline.Spool.Root, cfg.Session.Campaign, cfg.Session.SessionID, runID)
|
||||
previousCachePath := artifacts.SessionPreviousArtifactPathForCampaign(
|
||||
cfg.Pipeline.Workspace.Root,
|
||||
cfg.Session.Campaign,
|
||||
cfg.Session.SessionID,
|
||||
"session_recap.md",
|
||||
)
|
||||
|
||||
mustWriteFile(t, filepath.Join(runWorkDir, "manifest.json"), "{}\n")
|
||||
mustWriteFile(t, filepath.Join(runWorkDir, "logs", "stage.log"), "log\n")
|
||||
mustWriteFile(t, filepath.Join(otherRunDir, "logs", "stage.log"), "other\n")
|
||||
mustWriteFile(t, filepath.Join(spoolAudioDir, "speaker.flac"), "flac\n")
|
||||
mustWriteFile(t, previousCachePath, "# previous recap\n")
|
||||
|
||||
localSourceAudio := filepath.Join(filepath.Dir(cfg.SessionPath), "audio", "alice.flac")
|
||||
mustWriteFile(t, localSourceAudio, "source\n")
|
||||
|
||||
seed := manifest.New(cfg.Session.SessionID, time.Now().UTC())
|
||||
seed.Campaign = cfg.Session.Campaign
|
||||
seed.RunID = runID
|
||||
seed.LocalWorkDir = runWorkDir
|
||||
seed.LocalSpoolDir = spoolAudioDir
|
||||
seed.S3Bucket = "my-dnd-archive"
|
||||
seed.S3SessionPrefix = "dnd/campaigns/sample-campaign/sessions/2026-05-03/"
|
||||
seed.S3RunPrefix = seed.S3SessionPrefix + "runs/" + runID + "/"
|
||||
|
||||
store := &manifest.LocalStore{}
|
||||
if err := os.MkdirAll(filepath.Dir(manifestPathFor(cfg)), 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll() error = %v", err)
|
||||
}
|
||||
if err := store.Save(context.Background(), manifestPathFor(cfg), seed); err != nil {
|
||||
t.Fatalf("seed manifest save error = %v", err)
|
||||
}
|
||||
|
||||
return cfg, cleanupSeed{
|
||||
runWorkDir: runWorkDir,
|
||||
otherRunDir: otherRunDir,
|
||||
spoolAudioDir: spoolAudioDir,
|
||||
localSourceAudio: localSourceAudio,
|
||||
previousCachePath: previousCachePath,
|
||||
sessionPrefix: seed.S3SessionPrefix,
|
||||
}
|
||||
}
|
||||
|
||||
func publishStageCleanupFixture(t *testing.T) (*config.Config, cleanupSeed, string) {
|
||||
t.Helper()
|
||||
|
||||
cfg, seed := cleanupFixtureConfig(t)
|
||||
runID := "20260516T010203Z-1a2b3c4d"
|
||||
cfg.Pipeline.Storage.S3 = &config.StorageS3Config{
|
||||
Bucket: "my-dnd-archive",
|
||||
RootPrefix: "dnd",
|
||||
}
|
||||
cfg.Pipeline.Publish = &config.PublishConfig{
|
||||
Enabled: boolPtr(true),
|
||||
UploadRun: boolPtr(true),
|
||||
Outputs: []config.PublishOutputRule{
|
||||
{Source: "narratio.transcript.final_trimmed", Dest: "transcripts/final.trimmed.json", Required: boolPtr(true)},
|
||||
{Source: "narratio.artifact.session_recap", Dest: "artifacts/session_recap.md", Required: boolPtr(true)},
|
||||
},
|
||||
}
|
||||
cfg.Pipeline.Scriptorium = &config.ScriptoriumConfig{
|
||||
Artifacts: map[string]config.ScriptoriumArtifactConfig{
|
||||
"session_recap": {
|
||||
OutputPath: "artifacts/session_recap.md",
|
||||
},
|
||||
},
|
||||
}
|
||||
writePublishFixtureRunFiles(
|
||||
t,
|
||||
seed.runWorkDir,
|
||||
artifacts.SessionWorkDirForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID),
|
||||
)
|
||||
|
||||
store := &manifest.LocalStore{}
|
||||
seedManifest, err := store.Load(context.Background(), manifestPathFor(cfg))
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze"} {
|
||||
seedManifest.MarkStageSucceeded(name, time.Now().UTC(), nil)
|
||||
}
|
||||
seedManifest.S3SessionPrefix = artifacts.S3SessionPrefix("dnd", cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
seedManifest.S3RunPrefix = artifacts.S3RunPrefix(seedManifest.S3SessionPrefix, runID)
|
||||
if err := store.Save(context.Background(), manifestPathFor(cfg), seedManifest); err != nil {
|
||||
t.Fatalf("Save() error = %v", err)
|
||||
}
|
||||
seed.sessionPrefix = seedManifest.S3SessionPrefix
|
||||
return cfg, seed, runID
|
||||
}
|
||||
|
||||
func writePublishFixtureRunFiles(t *testing.T, runWorkDir, sessionRoot string) {
|
||||
t.Helper()
|
||||
mustWriteFile(t, filepath.Join(runWorkDir, "prepare", "inputs", "session.yml"), "session_id: 2026-05-03\n")
|
||||
mustWriteFile(t, filepath.Join(runWorkDir, "transcribe", "outputs", "transcripts", "raw", "speaker.json"), "{}\n")
|
||||
mustWriteFile(t, filepath.Join(runWorkDir, "trim", "outputs", "transcripts", "final.trimmed.json"), "{\"segments\":[]}\n")
|
||||
mustWriteFile(t, filepath.Join(runWorkDir, "analyze", "outputs", "artifacts", "session_recap.md"), "# recap\n")
|
||||
mustWriteFile(t, filepath.Join(runWorkDir, "polish", "reports", "audita.report.json"), "{}\n")
|
||||
mustWriteFile(t, filepath.Join(runWorkDir, "merge", "config", "seriatim.generated.yml"), "key: value\n")
|
||||
mustWriteFile(t, filepath.Join(runWorkDir, "logs", "audita.stderr.log"), "stderr\n")
|
||||
mustWriteFile(t, filepath.Join(runWorkDir, "manifest.json"), "{}\n")
|
||||
|
||||
mustWriteFile(t, filepath.Join(sessionRoot, "transcripts", "final.trimmed.json"), "{\"segments\":[]}\n")
|
||||
mustWriteFile(t, filepath.Join(sessionRoot, "artifacts", "session_recap.md"), "# recap\n")
|
||||
}
|
||||
|
||||
type failKeyStore struct {
|
||||
delegate *storage.FakeBackend
|
||||
failKey string
|
||||
}
|
||||
|
||||
func (s *failKeyStore) List(ctx context.Context, prefix string) ([]storage.ObjectInfo, error) {
|
||||
return s.delegate.List(ctx, prefix)
|
||||
}
|
||||
|
||||
func (s *failKeyStore) Download(ctx context.Context, key, localPath string) error {
|
||||
return s.delegate.Download(ctx, key, localPath)
|
||||
}
|
||||
|
||||
func (s *failKeyStore) Upload(ctx context.Context, localPath, key string, opts storage.UploadOptions) (storage.ObjectInfo, error) {
|
||||
if strings.TrimSpace(key) == strings.TrimSpace(s.failKey) {
|
||||
return storage.ObjectInfo{}, errors.New("forced upload failure")
|
||||
}
|
||||
return s.delegate.Upload(ctx, localPath, key, opts)
|
||||
}
|
||||
|
||||
func (s *failKeyStore) Exists(ctx context.Context, key string) (bool, error) {
|
||||
return s.delegate.Exists(ctx, key)
|
||||
}
|
||||
|
||||
func assertExists(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
t.Fatalf("expected path to exist %q: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func assertMissing(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
||||
t.Fatalf("expected path to be removed %q, stat err=%v", path, err)
|
||||
}
|
||||
}
|
||||
157
internal/app/remote_locks.go
Normal file
157
internal/app/remote_locks.go
Normal file
@@ -0,0 +1,157 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
type effectiveLocks struct {
|
||||
Static []config.PublishLockRule
|
||||
Remote []config.PublishLockRule
|
||||
All []config.PublishLockRule
|
||||
Key string
|
||||
}
|
||||
|
||||
func remoteLocksKey(cfg *config.Config) (string, error) {
|
||||
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
|
||||
return "", fmt.Errorf("resolved config is required")
|
||||
}
|
||||
if cfg.Pipeline.Storage.S3 == nil {
|
||||
return "", fmt.Errorf("pipeline.storage.s3 configuration is required")
|
||||
}
|
||||
sessionPrefix := artifacts.S3SessionPrefix(
|
||||
cfg.Pipeline.Storage.S3.RootPrefix,
|
||||
cfg.Session.Campaign,
|
||||
cfg.Session.SessionID,
|
||||
)
|
||||
return artifacts.S3SessionLocksKey(sessionPrefix), nil
|
||||
}
|
||||
|
||||
func loadRemoteLockStore(ctx context.Context, cfg *config.Config, store storage.ObjectStore) (*config.PublishLockStore, string, error) {
|
||||
key, err := remoteLocksKey(cfg)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
exists, err := store.Exists(ctx, key)
|
||||
if err != nil {
|
||||
return nil, key, fmt.Errorf("check remote locks %q: %w", key, err)
|
||||
}
|
||||
if !exists {
|
||||
return &config.PublishLockStore{}, key, nil
|
||||
}
|
||||
tmp, err := storage.DownloadObjectToTemp(ctx, store, key, "narratio-locks-*.yml")
|
||||
if err != nil {
|
||||
return nil, key, fmt.Errorf("download remote locks %q: %w", key, err)
|
||||
}
|
||||
defer func() { _ = os.Remove(tmp) }()
|
||||
data, err := os.ReadFile(tmp)
|
||||
if err != nil {
|
||||
return nil, key, fmt.Errorf("read remote locks %q: %w", key, err)
|
||||
}
|
||||
lockStore, err := config.LoadPublishLockStoreBytes("s3://"+s3BucketName(cfg.Pipeline)+"/"+key, data, cfg.Pipeline.Scriptorium)
|
||||
if err != nil {
|
||||
return nil, key, err
|
||||
}
|
||||
return lockStore, key, nil
|
||||
}
|
||||
|
||||
func loadEffectiveLocks(ctx context.Context, cfg *config.Config, store storage.ObjectStore) (*effectiveLocks, error) {
|
||||
staticLocks := staticPublishLocks(cfg)
|
||||
if store == nil {
|
||||
return &effectiveLocks{
|
||||
Static: staticLocks,
|
||||
All: append([]config.PublishLockRule(nil), staticLocks...),
|
||||
}, nil
|
||||
}
|
||||
lockStore, key, err := loadRemoteLockStore(ctx, cfg, store)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
remoteLocks := append([]config.PublishLockRule(nil), lockStore.Locks...)
|
||||
return &effectiveLocks{
|
||||
Static: staticLocks,
|
||||
Remote: remoteLocks,
|
||||
All: config.MergePublishLockRules(staticLocks, remoteLocks),
|
||||
Key: key,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func staticPublishLocks(cfg *config.Config) []config.PublishLockRule {
|
||||
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Publish == nil {
|
||||
return nil
|
||||
}
|
||||
return append([]config.PublishLockRule(nil), cfg.Pipeline.Publish.Locks...)
|
||||
}
|
||||
|
||||
func applyEffectiveLocks(cfg *config.Config, locks []config.PublishLockRule) {
|
||||
if cfg == nil || cfg.Pipeline == nil {
|
||||
return
|
||||
}
|
||||
if cfg.Pipeline.Publish == nil {
|
||||
cfg.Pipeline.Publish = &config.PublishConfig{}
|
||||
}
|
||||
cfg.Pipeline.Publish.Locks = append([]config.PublishLockRule(nil), locks...)
|
||||
}
|
||||
|
||||
func uploadRemoteLockStore(ctx context.Context, store storage.ObjectStore, key string, lockStore *config.PublishLockStore) error {
|
||||
data, err := config.MarshalPublishLockStore(lockStore)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmp, err := os.CreateTemp("", "narratio-locks-upload-*.yml")
|
||||
if err != nil {
|
||||
return fmt.Errorf("create lock store temp file: %w", err)
|
||||
}
|
||||
tmpPath := tmp.Name()
|
||||
defer func() { _ = os.Remove(tmpPath) }()
|
||||
if _, err := tmp.Write(data); err != nil {
|
||||
_ = tmp.Close()
|
||||
return fmt.Errorf("write lock store temp file: %w", err)
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return fmt.Errorf("close lock store temp file: %w", err)
|
||||
}
|
||||
if _, err := store.Upload(ctx, tmpPath, key, storage.UploadOptions{ContentType: "application/x-yaml; charset=utf-8"}); err != nil {
|
||||
return fmt.Errorf("upload remote locks %q: %w", key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func lockSourceSet(locks []config.PublishLockRule) map[string]config.PublishLockRule {
|
||||
out := make(map[string]config.PublishLockRule, len(locks))
|
||||
for _, lock := range locks {
|
||||
source := strings.TrimSpace(lock.Source)
|
||||
if source == "" {
|
||||
continue
|
||||
}
|
||||
lock.Source = source
|
||||
lock.Reason = strings.TrimSpace(lock.Reason)
|
||||
out[source] = lock
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func writeLocalFile(path string, data []byte, force bool) error {
|
||||
cleaned := filepath.Clean(strings.TrimSpace(path))
|
||||
if cleaned == "" || cleaned == "." {
|
||||
return fmt.Errorf("output path is required")
|
||||
}
|
||||
if !force {
|
||||
if _, err := os.Stat(cleaned); err == nil {
|
||||
return fmt.Errorf("output file %q already exists; pass --force to overwrite", cleaned)
|
||||
} else if err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("check output file %q: %w", cleaned, err)
|
||||
}
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(cleaned), 0o755); err != nil {
|
||||
return fmt.Errorf("create output directory: %w", err)
|
||||
}
|
||||
return os.WriteFile(cleaned, data, 0o644)
|
||||
}
|
||||
294
internal/app/remote_session_test.go
Normal file
294
internal/app/remote_session_test.go
Normal file
@@ -0,0 +1,294 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
func TestExecuteRemoteSessionFallbackLoadsFromObjectStore(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||
fake := &storage.FakeBackend{}
|
||||
remoteKey := seedRemoteSessionConfig(t, fake, "2026-05-03", `session_id: 2026-05-03
|
||||
inputs:
|
||||
audio_s3:
|
||||
prefix: audio/
|
||||
`)
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if storeInitCalls != 1 {
|
||||
t.Fatalf("object store init calls = %d, want 1", storeInitCalls)
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "narratio session plan: workdir prepared") {
|
||||
t.Fatalf("stdout = %q, want plan output", stdout.String())
|
||||
}
|
||||
if _, ok := fake.Objects[remoteKey]; !ok {
|
||||
t.Fatalf("remote session key %q was not seeded", remoteKey)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRemoteSessionFallbackLoadsSecretsBeforeObjectStoreInit(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||
accessKeyEnv := "NARRATIO_TEST_REMOTE_SESSION_KEY_ID"
|
||||
secretKeyEnv := "NARRATIO_TEST_REMOTE_SESSION_SECRET"
|
||||
restoreEnvAfterTest(t, accessKeyEnv, secretKeyEnv)
|
||||
secretsDir := t.TempDir()
|
||||
mustWriteTestFile(t, filepath.Join(secretsDir, accessKeyEnv), "remote-session-key-id\n")
|
||||
mustWriteTestFile(t, filepath.Join(secretsDir, secretKeyEnv), "remote-session-secret\n")
|
||||
addSecretsToPipelineConfig(t, pipelinePath, secretsDir, accessKeyEnv, secretKeyEnv)
|
||||
|
||||
fake := &storage.FakeBackend{}
|
||||
seedRemoteSessionConfig(t, fake, "2026-05-03", `session_id: 2026-05-03
|
||||
inputs:
|
||||
audio_s3:
|
||||
prefix: audio/
|
||||
`)
|
||||
origStoreFn := newObjectStoreFromConfigFn
|
||||
origSessionDefaults := append([]string(nil), config.DefaultSessionConfigSearchPaths...)
|
||||
config.DefaultSessionConfigSearchPaths = []string{filepath.Join(t.TempDir(), "session.yml")}
|
||||
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
if os.Getenv(accessKeyEnv) != "remote-session-key-id" || os.Getenv(secretKeyEnv) != "remote-session-secret" {
|
||||
return nil, fmt.Errorf("secrets were not loaded before remote session object store init")
|
||||
}
|
||||
return fake, nil
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
newObjectStoreFromConfigFn = origStoreFn
|
||||
config.DefaultSessionConfigSearchPaths = origSessionDefaults
|
||||
})
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteExplicitLocalSessionPrecedenceSkipsRemote(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
fake := &storage.FakeBackend{}
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if storeInitCalls != 0 {
|
||||
t.Fatalf("object store init calls = %d, want 0", storeInitCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteLocalSessionDiscoveryPrecedenceSkipsRemote(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
fake := &storage.FakeBackend{}
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if storeInitCalls != 0 {
|
||||
t.Fatalf("object store init calls = %d, want 0", storeInitCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRemoteSessionMissingObjectFailsClearly(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||
fake := &storage.FakeBackend{}
|
||||
var storeInitCalls int
|
||||
missingSessionPath := filepath.Join(t.TempDir(), "session.yml")
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{missingSessionPath})
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "remote session") || !strings.Contains(stderr.String(), "session.yml") || !strings.Contains(stderr.String(), "not found") {
|
||||
t.Fatalf("stderr = %q, want remote session not found context", stderr.String())
|
||||
}
|
||||
if !strings.Contains(stderr.String(), missingSessionPath) {
|
||||
t.Fatalf("stderr = %q, want local searched path", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRemoteSessionRequiresSessionID(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, &storage.FakeBackend{}, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"session", "plan", "--config", pipelinePath, "--campaign-file", campaignPath}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "plan: session_id is required") {
|
||||
t.Fatalf("stderr = %q, want session_id guidance", stderr.String())
|
||||
}
|
||||
if storeInitCalls != 0 {
|
||||
t.Fatalf("object store init calls = %d, want 0", storeInitCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRemoteSessionStorageInitErrorFailsClearly(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||
origStoreFn := newObjectStoreFromConfigFn
|
||||
origSessionDefaults := append([]string(nil), config.DefaultSessionConfigSearchPaths...)
|
||||
config.DefaultSessionConfigSearchPaths = []string{filepath.Join(t.TempDir(), "session.yml")}
|
||||
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
return nil, errors.New("storage unavailable")
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
newObjectStoreFromConfigFn = origStoreFn
|
||||
config.DefaultSessionConfigSearchPaths = origSessionDefaults
|
||||
})
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "storage unavailable") || !strings.Contains(stderr.String(), "remote session") {
|
||||
t.Fatalf("stderr = %q, want remote storage context", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRemoteSessionMalformedYAMLFailsStrictDecode(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||
fake := &storage.FakeBackend{}
|
||||
seedRemoteSessionConfig(t, fake, "2026-05-03", "session_id: 2026-05-03\nunknown: true\n")
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "strict decode failed") {
|
||||
t.Fatalf("stderr = %q, want strict decode context", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRemoteSessionTemplateFailsConcreteSessionCheck(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||
fake := &storage.FakeBackend{}
|
||||
seedRemoteSessionConfig(t, fake, "2026-05-03", `session_id: "{{ session_id }}"
|
||||
inputs:
|
||||
audio_s3:
|
||||
prefix: audio/
|
||||
`)
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "session.yml must be concrete") || !strings.Contains(stderr.String(), "run narratio session init") {
|
||||
t.Fatalf("stderr = %q, want concrete session guidance", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRemoteSessionMismatchFails(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||
fake := &storage.FakeBackend{}
|
||||
seedRemoteSessionConfig(t, fake, "2026-05-03", "session_id: 2026-05-04\ninputs:\n audio_s3:\n prefix: audio/\n")
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "session_id mismatch") {
|
||||
t.Fatalf("stderr = %q, want session_id mismatch", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func restoreAppConfigTestGlobals(t *testing.T, fake *storage.FakeBackend, storeInitCalls *int, sessionDefaults []string) {
|
||||
t.Helper()
|
||||
origStoreFn := newObjectStoreFromConfigFn
|
||||
origSessionDefaults := append([]string(nil), config.DefaultSessionConfigSearchPaths...)
|
||||
config.DefaultSessionConfigSearchPaths = append([]string(nil), sessionDefaults...)
|
||||
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
if storeInitCalls != nil {
|
||||
(*storeInitCalls)++
|
||||
}
|
||||
return fake, nil
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
newObjectStoreFromConfigFn = origStoreFn
|
||||
config.DefaultSessionConfigSearchPaths = origSessionDefaults
|
||||
})
|
||||
}
|
||||
|
||||
func seedRemoteSessionConfig(t *testing.T, fake *storage.FakeBackend, sessionID, content string) string {
|
||||
t.Helper()
|
||||
sessionPrefix := artifacts.S3SessionPrefix("dnd", "sample-campaign", sessionID)
|
||||
remoteKey := artifacts.S3SessionConfigKey(sessionPrefix)
|
||||
fake.SeedObject(storage.FakeObject{
|
||||
Key: remoteKey,
|
||||
Data: []byte(content),
|
||||
ETag: "remote-session-etag",
|
||||
})
|
||||
return remoteKey
|
||||
}
|
||||
|
||||
func addSecretsToPipelineConfig(t *testing.T, pipelinePath, secretsDir, accessKeyEnv, secretKeyEnv string) {
|
||||
t.Helper()
|
||||
pipelineData, err := os.ReadFile(pipelinePath)
|
||||
if err != nil {
|
||||
t.Fatalf("read pipeline: %v", err)
|
||||
}
|
||||
pipelineYAML := strings.Replace(
|
||||
string(pipelineData),
|
||||
"storage:\n backend: s3\n s3:\n bucket: test-bucket\n",
|
||||
"storage:\n backend: s3\n s3:\n bucket: test-bucket\n access_key_id_env: "+accessKeyEnv+"\n secret_access_key_env: "+secretKeyEnv+"\nsecrets:\n env_dir: "+secretsDir+"\n",
|
||||
1,
|
||||
)
|
||||
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||
t.Fatalf("write pipeline: %v", err)
|
||||
}
|
||||
}
|
||||
138
internal/app/restore.go
Normal file
138
internal/app/restore.go
Normal file
@@ -0,0 +1,138 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/logging"
|
||||
)
|
||||
|
||||
var newObjectStoreFromConfigFn = storage.NewObjectStoreFromConfig
|
||||
var discoverRemoteCurrentStateFn = discoverRemoteCurrentState
|
||||
var buildRestorePlanFn = buildRestorePlan
|
||||
var executeRestorePlanFn = executeRestorePlan
|
||||
|
||||
// Restore validates restore CLI/config inputs and storage preflight for future restore phases.
|
||||
func Restore(ctx context.Context, args []string, out io.Writer) error {
|
||||
positionalSessionID, args := pullLeadingSessionID(args)
|
||||
fs := flag.NewFlagSet("restore", flag.ContinueOnError)
|
||||
fs.SetOutput(out)
|
||||
|
||||
var flags commonConfigFlags
|
||||
var dryRun bool
|
||||
var force bool
|
||||
var includeAudio bool
|
||||
addCommonConfigFlags(fs, &flags)
|
||||
fs.BoolVar(&dryRun, "dry-run", false, "plan restore actions without writing local files")
|
||||
fs.BoolVar(&force, "force", false, "overwrite local conflicts with remote state")
|
||||
fs.BoolVar(&includeAudio, "include-audio", false, "include archived session-level audio objects")
|
||||
fs.Usage = func() {
|
||||
_, _ = fmt.Fprintln(out, "Usage: narratio session restore <session_id> [--config <path>] [--campaign <id>] [--campaign-file <path>] [--session <path>] [--previous-session-id <value>] [--dry-run] [--force] [--include-audio]")
|
||||
_, _ = fmt.Fprintln(out)
|
||||
_, _ = fmt.Fprintln(out, "Flags:")
|
||||
fs.PrintDefaults()
|
||||
}
|
||||
|
||||
if err := fs.Parse(args); err != nil {
|
||||
if errors.Is(err, flag.ErrHelp) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("restore: invalid flags: %w", err)
|
||||
}
|
||||
if err := resolveParsedSessionID("restore", positionalSessionID, fs, &flags.sessionID); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(flags.sessionID) == "" {
|
||||
return fmt.Errorf("restore: session_id is required")
|
||||
}
|
||||
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
||||
if err != nil {
|
||||
return fmt.Errorf("restore: %w", err)
|
||||
}
|
||||
if err := config.Validate(cfg); err != nil {
|
||||
return fmt.Errorf("restore: %w", err)
|
||||
}
|
||||
objectStore, err := newCommandObjectStore(ctx, cfg, logging.NewLogger(os.Stderr, slog.LevelInfo))
|
||||
if err != nil {
|
||||
return fmt.Errorf("restore: %w", err)
|
||||
}
|
||||
current, err := discoverRemoteCurrentStateFn(ctx, cfg, objectStore)
|
||||
if err != nil {
|
||||
return fmt.Errorf("restore: %w", err)
|
||||
}
|
||||
plan, err := buildRestorePlanFn(ctx, cfg, current, objectStore, RestorePlanOptions{
|
||||
IncludeAudio: includeAudio,
|
||||
Force: force,
|
||||
DryRun: dryRun,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("restore: %w", err)
|
||||
}
|
||||
report, err := newRestoreReport(current, plan, RestorePlanOptions{
|
||||
IncludeAudio: includeAudio,
|
||||
Force: force,
|
||||
DryRun: dryRun,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("restore: %w", err)
|
||||
}
|
||||
if dryRun {
|
||||
if err := writeRestoreDryRunSummary(out, report); err != nil {
|
||||
return fmt.Errorf("restore: write plan output: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
artifactStore := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root)
|
||||
if _, err := artifactStore.EnsureLayoutFor(cfg.Session.Campaign, cfg.Session.SessionID); err != nil {
|
||||
return fmt.Errorf("restore: prepare workdir: %w", err)
|
||||
}
|
||||
lock, err := artifactStore.AcquireSessionLockFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("restore: acquire session lock: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
_ = artifactStore.ReleaseSessionLock(lock)
|
||||
}()
|
||||
|
||||
if plan.ConflictCount > 0 && !force {
|
||||
report.setFailed(fmt.Errorf("conflict: %d conflicting path(s)", plan.ConflictCount))
|
||||
if _, reportErr := persistRestoreReport(artifactStore, cfg, report); reportErr != nil {
|
||||
return fmt.Errorf("restore: report failure: %w", reportErr)
|
||||
}
|
||||
return fmt.Errorf(
|
||||
"restore conflict: %d conflicting path(s); rerun with --force to overwrite (download=%d skip_same=%d conflicts=%d)",
|
||||
plan.ConflictCount,
|
||||
plan.DownloadCount,
|
||||
plan.SkipSameCount,
|
||||
plan.ConflictCount,
|
||||
)
|
||||
}
|
||||
|
||||
result, err := executeRestorePlanFn(ctx, cfg, current, plan, report, objectStore)
|
||||
if err != nil {
|
||||
report.setFailed(err)
|
||||
if _, reportErr := persistRestoreReport(artifactStore, cfg, report); reportErr != nil {
|
||||
return fmt.Errorf("restore: execute plan failed (%v) and report write failed (%v)", err, reportErr)
|
||||
}
|
||||
return fmt.Errorf("restore: execute plan: %w", err)
|
||||
}
|
||||
report.Execution.Downloaded = result.DownloadedCount
|
||||
report.setSucceeded()
|
||||
if _, err := persistRestoreReport(artifactStore, cfg, report); err != nil {
|
||||
return fmt.Errorf("restore: write report: %w", err)
|
||||
}
|
||||
if err := writeRestoreSuccessSummary(out, report); err != nil {
|
||||
return fmt.Errorf("restore: write summary: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user