Compare commits
138 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 717451512a | |||
| 3ddb3a947b | |||
| c6632d5576 | |||
| ffc07922c7 | |||
| f3310d4d16 | |||
| 88cee96d8d | |||
| 2fece10215 | |||
| 0658f2f642 | |||
| a51228c803 | |||
| 4491fb5ccd | |||
| 30b905765c | |||
| 03eac70881 | |||
| 0f7e6b979f | |||
| c366912586 | |||
| 9fe44cd00d | |||
| 094b0d2532 | |||
| 98649f4d81 | |||
| 8a559efd5b | |||
| 72deccb4e2 | |||
| 5620fc5bcf | |||
| be57e675e0 | |||
| 3971443831 | |||
| a6b0c33e9f | |||
| 96b886e711 | |||
| 7d584ee6cd | |||
| 572a112c31 | |||
| ea87c335d6 | |||
| 7169ff04df | |||
| ef1f650bc0 | |||
| 0d02cb9fa0 | |||
| 0299b128cf | |||
| d723384888 | |||
| 54228055c8 | |||
| 23ed716450 | |||
| ab59bab044 | |||
| 71395bb076 | |||
| 79737edf79 | |||
| df2c765b7f | |||
| f050b9dd54 | |||
| 9c9cb54339 | |||
| 7657ec3ad6 | |||
| cee52aa092 | |||
| e920f3a8d5 | |||
| 591c529a09 | |||
| 7324c5a686 | |||
| d0936fb022 | |||
| 2aa074c5cf | |||
| 782d0cf3b9 | |||
| 083c01cfa0 | |||
| 2937696024 | |||
| b817a5b772 | |||
| 3022f20beb | |||
| ca1ded1821 | |||
| 3752f3ed28 | |||
| 870c2d69d5 | |||
| 135407ba7c | |||
| 228c348e42 | |||
| a813bd5a50 | |||
| d8f58dce31 | |||
| 7111edeca4 | |||
| 3aae4bbb12 | |||
| b29d8eeb50 | |||
| dffb432537 | |||
| 2dd38c7913 | |||
| bc2ade38d9 | |||
| 5be831eb13 | |||
| cae4d99a89 | |||
| e09dc0512d | |||
| ae82bc1ce0 | |||
| 01eb7aa1aa | |||
| 2ca700195c | |||
| 2b08c34539 | |||
| 79f1fc1e09 | |||
| 9c753270bd | |||
| b907cb01aa | |||
| 7824afd4a5 | |||
| 2a4e1e912c | |||
| dd03c09d75 | |||
| 5bc8e8683f | |||
| 648001a8fe | |||
| 6684774f52 | |||
| f3b63bd5e5 | |||
| 23d6470b0f | |||
| 128449040f | |||
| 02ab106ade | |||
| c128970f58 | |||
| d001baa660 | |||
| c5c35cd3b4 | |||
| 574b1cde6c | |||
| ebb21b9201 | |||
| 958f446387 | |||
| 86caf4b222 | |||
| e38ed8ba97 | |||
| 3e79cf4724 | |||
| 859ae1ae10 | |||
| c63ecbab32 | |||
| 8480b74283 | |||
| 087869f7fa | |||
| 2b2a314d65 | |||
| 08b0f4edc5 | |||
| 571a289296 | |||
| 9f80635b42 | |||
| 11a3e174b6 | |||
| 9c5e5d6dc1 | |||
| c4e87f58c7 | |||
| 37daab7857 | |||
| 2356688cb9 | |||
| 1054b64d9f | |||
| 01fb02426c | |||
| 7dc79e052f | |||
| cb525c0f72 | |||
| 622677d038 | |||
| 550288e008 | |||
| e58e545686 | |||
| 6ff54c5a0f | |||
| 924b5d15c6 | |||
| b065663180 | |||
| 3ba564b00f | |||
| a3986cf0d6 | |||
| 539601bd16 | |||
| 6ca1c8d6b0 | |||
| 4b7b50981b | |||
| 33f7ae8f2e | |||
| d5a9ad38f8 | |||
| 6fbefb9867 | |||
| 1665359486 | |||
| 03f2543927 | |||
| fe9c348092 | |||
| f7f8f1a949 | |||
| d40c91acde | |||
| ed4dcf1ef7 | |||
| 24cce49a70 | |||
| 1e6db89dd4 | |||
| 0454296c81 | |||
| 58c6ab2d54 | |||
| 7995c41675 | |||
| 62551d43a0 | |||
| 8395c12dd3 |
5
.gitignore
vendored
5
.gitignore
vendored
@@ -2,6 +2,8 @@
|
|||||||
.codex
|
.codex
|
||||||
AGENTS.md
|
AGENTS.md
|
||||||
|
|
||||||
|
.DS_Store
|
||||||
|
|
||||||
# ---> Go
|
# ---> Go
|
||||||
# If you prefer the allow list template instead of the deny list, see community template:
|
# If you prefer the allow list template instead of the deny list, see community template:
|
||||||
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
|
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
|
||||||
@@ -22,6 +24,9 @@ AGENTS.md
|
|||||||
# Dependency directories (remove the comment below to include it)
|
# Dependency directories (remove the comment below to include it)
|
||||||
# vendor/
|
# vendor/
|
||||||
|
|
||||||
|
# Go cache
|
||||||
|
.gocache
|
||||||
|
|
||||||
# Go workspace file
|
# Go workspace file
|
||||||
go.work
|
go.work
|
||||||
go.work.sum
|
go.work.sum
|
||||||
|
|||||||
292
README.md
292
README.md
@@ -1,288 +1,22 @@
|
|||||||
# narratio
|
# narratio
|
||||||
|
|
||||||
`narratio` is a Go orchestration application for processing D&D session audio into transcripts and generated artifacts.
|
Narratio is a stage-driven Go orchestrator for turning D&D session audio into polished transcripts and generated artifacts.
|
||||||
|
|
||||||
## Current Implementation
|
It runs a deterministic workflow across `prepare`, `transcribe`, `merge`, `polish`, `normalize`, `trim`, `render`, `analyze`, and `publish`, with manifest-driven continuation and restore support.
|
||||||
|
|
||||||
Implemented now:
|
|
||||||
|
|
||||||
- strict config loading/validation (`pipeline.yml` and `session.yml`)
|
|
||||||
- local workspace/session layout, locking, and manifest persistence
|
|
||||||
- resumable stage control (`run`, `plan`, `resume`, `run-stage`, `status`)
|
|
||||||
- real `prepare`, `transcribe`, `merge`, `polish`, `normalize`, `trim`, and `analyze` stages
|
|
||||||
- real WhisperX, Seriatim, and Audita adapters
|
|
||||||
- real Scriptorium subprocess adapter
|
|
||||||
- optional Scriptorium render diagnostics (`render_debug`)
|
|
||||||
|
|
||||||
Not implemented yet:
|
|
||||||
|
|
||||||
- `archive` stage behavior
|
|
||||||
- `notify` stage behavior
|
|
||||||
- additional analyze artifacts beyond `session_recap`
|
|
||||||
- generic DAG orchestration
|
|
||||||
|
|
||||||
## Config Files
|
|
||||||
|
|
||||||
Narratio expects two YAML files:
|
|
||||||
|
|
||||||
- `pipeline.yml`: pipeline/workspace settings
|
|
||||||
- `session.yml`: per-session settings
|
|
||||||
|
|
||||||
Pipeline config lookup for CLI commands:
|
|
||||||
|
|
||||||
- if `--config <path>` is provided, Narratio uses that path
|
|
||||||
- if `--config` is omitted, Narratio searches in this order:
|
|
||||||
- `/usr/local/etc/narratio/pipeline.yml`
|
|
||||||
- `/etc/narratio/pipeline.yml`
|
|
||||||
|
|
||||||
YAML decoding is strict (`KnownFields(true)`), so unknown fields fail fast.
|
|
||||||
|
|
||||||
## Canonical Stage Order
|
|
||||||
|
|
||||||
1. `prepare`
|
|
||||||
2. `transcribe`
|
|
||||||
3. `merge`
|
|
||||||
4. `polish`
|
|
||||||
5. `normalize`
|
|
||||||
6. `trim`
|
|
||||||
7. `analyze`
|
|
||||||
8. `archive`
|
|
||||||
9. `notify`
|
|
||||||
|
|
||||||
## Transcript Tiers
|
|
||||||
|
|
||||||
- `transcripts/merged.json`: canonical deterministic merged transcript from Seriatim merge
|
|
||||||
- `transcripts/processed.json`: full raw Audita-polished transcript output
|
|
||||||
- `transcripts/normalized.json`: Seriatim-normalized transcript from the normalize stage
|
|
||||||
- `transcripts/trimmed.json`: gameplay-only normalized polished transcript from trim stage
|
|
||||||
|
|
||||||
## Normalize Configuration
|
|
||||||
|
|
||||||
`pipeline.normalize` is optional. When omitted, Narratio defaults to:
|
|
||||||
|
|
||||||
- `output_path: transcripts/normalized.json`
|
|
||||||
- `output_schema: seriatim-intermediate`
|
|
||||||
- `report: true`
|
|
||||||
|
|
||||||
Allowed `normalize.output_schema` values:
|
|
||||||
|
|
||||||
- `seriatim-minimal`
|
|
||||||
- `seriatim-intermediate`
|
|
||||||
- `seriatim-full`
|
|
||||||
|
|
||||||
`normalize.output_path` is treated as session-workdir-relative when not absolute.
|
|
||||||
|
|
||||||
Normalize stage behavior summary:
|
|
||||||
|
|
||||||
- normalize runs after `polish` and before `trim`
|
|
||||||
- normalize resolves `transcripts/processed.json`
|
|
||||||
- normalize runs Seriatim `normalize` to produce `transcripts/normalized.json`
|
|
||||||
- normalize diagnostics are written to:
|
|
||||||
- `artifacts/seriatim.normalize.report.json` (when enabled)
|
|
||||||
- `logs/seriatim.normalize.stdout.log`
|
|
||||||
- `logs/seriatim.normalize.stderr.log`
|
|
||||||
- `config/seriatim.normalize.generated.yml`
|
|
||||||
|
|
||||||
## Trim Configuration
|
|
||||||
|
|
||||||
`pipeline.trim` is optional. If omitted, no trim config is loaded. If `trim.enabled` is omitted, it defaults to `false`.
|
|
||||||
|
|
||||||
When `trim.enabled: true`:
|
|
||||||
|
|
||||||
- `trim.output_path` is required
|
|
||||||
- `trim.bounds.prompt_id` is required
|
|
||||||
- `trim.bounds.transcript_input_name` is required
|
|
||||||
- `trim.bounds.output_path` is required
|
|
||||||
- `trim.bounds.timeout` must be a valid Go duration when provided
|
|
||||||
- `trim.bounds.render_debug: true` requires `trim.bounds.render_output_path`
|
|
||||||
- `trim.bounds.profile_id` may be empty to use the prompt default profile
|
|
||||||
|
|
||||||
Trim paths are treated as session-workdir-relative when not absolute.
|
|
||||||
|
|
||||||
Example trim config:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
trim:
|
|
||||||
enabled: true
|
|
||||||
output_path: "transcripts/trimmed.json"
|
|
||||||
bounds:
|
|
||||||
prompt_id: "dnd_session.bounds"
|
|
||||||
profile_id: ""
|
|
||||||
transcript_input_name: "transcript"
|
|
||||||
output_path: "artifacts/session_bounds.json"
|
|
||||||
timeout: "10m"
|
|
||||||
render_debug: false
|
|
||||||
render_output_path: "artifacts/session_bounds.render.json"
|
|
||||||
seriatim:
|
|
||||||
report: false
|
|
||||||
```
|
|
||||||
|
|
||||||
Trim behavior summary:
|
|
||||||
|
|
||||||
- trim discovers and validates `transcripts/normalized.json`
|
|
||||||
- trim uses Scriptorium bounds (`dnd_session.bounds` by example config) to produce `artifacts/session_bounds.json`
|
|
||||||
- bounds IDs are validated against the same normalized transcript ID space that Seriatim trim will consume
|
|
||||||
- trim converts bounds to Seriatim keep selector (for example `10-868`) and runs Seriatim trim
|
|
||||||
- if trim is disabled, Narratio copies normalized transcript to trimmed transcript and records `trim_action=copy_disabled`
|
|
||||||
|
|
||||||
Trim outputs and diagnostics:
|
|
||||||
|
|
||||||
- `artifacts/session_bounds.json`
|
|
||||||
- `transcripts/trimmed.json`
|
|
||||||
- `logs/scriptorium.bounds.stdout.log`
|
|
||||||
- `logs/scriptorium.bounds.stderr.log`
|
|
||||||
- `config/scriptorium.bounds.generated.yml`
|
|
||||||
- `logs/seriatim.trim.stdout.log`
|
|
||||||
- `logs/seriatim.trim.stderr.log`
|
|
||||||
- `config/seriatim.trim.generated.yml`
|
|
||||||
- optional bounds render-debug outputs:
|
|
||||||
- `artifacts/session_bounds.render.json`
|
|
||||||
- `logs/scriptorium.bounds.render.stdout.log`
|
|
||||||
- `logs/scriptorium.bounds.render.stderr.log`
|
|
||||||
- `config/scriptorium.bounds.render.generated.yml`
|
|
||||||
|
|
||||||
Render-debug files are diagnostics and are not treated as canonical stage output artifact refs.
|
|
||||||
|
|
||||||
## Scriptorium Configuration
|
|
||||||
|
|
||||||
`pipeline.scriptorium` is optional. When present, Narratio validates and uses it for analyze-stage artifact generation.
|
|
||||||
|
|
||||||
Key points:
|
|
||||||
|
|
||||||
- `scriptorium.binary` is required when section is present
|
|
||||||
- `scriptorium.config_path` is optional
|
|
||||||
- `scriptorium.timeout` defaults to `10m` when omitted
|
|
||||||
- `scriptorium.render_debug` enables render diagnostics globally
|
|
||||||
- artifacts are configured under `scriptorium.artifacts` (map shape supports multiple artifacts)
|
|
||||||
- enabled artifacts require `prompt_id` and `output_path`
|
|
||||||
- artifact `render_debug` may override global render setting
|
|
||||||
- `vars` currently support boolean and string values
|
|
||||||
|
|
||||||
Example `session_recap` artifact definition:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
scriptorium:
|
|
||||||
binary: "scriptorium"
|
|
||||||
config_path: "/etc/scriptorium/config.yml"
|
|
||||||
timeout: "10m"
|
|
||||||
render_debug: false
|
|
||||||
|
|
||||||
artifacts:
|
|
||||||
session_recap:
|
|
||||||
enabled: true
|
|
||||||
prompt_id: "dnd.session_recap"
|
|
||||||
profile_id: "local-quality" # optional
|
|
||||||
output_path: "artifacts/session_recap.md"
|
|
||||||
timeout: "10m"
|
|
||||||
# render_debug: true # optional per-artifact override
|
|
||||||
|
|
||||||
inputs:
|
|
||||||
transcript:
|
|
||||||
source: "trimmed_transcript"
|
|
||||||
required: true
|
|
||||||
|
|
||||||
previous_recap:
|
|
||||||
source: "previous_session_artifact"
|
|
||||||
artifact: "session_recap"
|
|
||||||
path: "" # optional; set when available
|
|
||||||
required: false
|
|
||||||
|
|
||||||
vars:
|
|
||||||
session_id: true
|
|
||||||
session_date: true
|
|
||||||
campaign_name: true
|
|
||||||
previous_session_id: true
|
|
||||||
output_kind: "session_recap"
|
|
||||||
```
|
|
||||||
|
|
||||||
Prompt IDs and profile IDs are configuration values. They are not hardcoded in analyze-stage logic.
|
|
||||||
|
|
||||||
Do not put secrets in `pipeline.yml`. If API-key behavior is configured, use env var names only.
|
|
||||||
|
|
||||||
## Scriptorium Runtime Behavior
|
|
||||||
|
|
||||||
Narratio integrates with Scriptorium through the public CLI subprocess contract:
|
|
||||||
|
|
||||||
- generation: `scriptorium run`
|
|
||||||
- diagnostics/testing: `scriptorium render --format json` when `render_debug` is enabled
|
|
||||||
|
|
||||||
For the initial implementation, only `session_recap` generation is supported.
|
|
||||||
|
|
||||||
Analyze-stage session recap behavior:
|
|
||||||
|
|
||||||
- available transcript input sources for configured artifacts: `processed_transcript`, `normalized_transcript`, `trimmed_transcript`
|
|
||||||
- session recap should use gameplay-only transcript input (`source: trimmed_transcript`)
|
|
||||||
- Narratio resolves `trimmed_transcript` from trim manifest output (`transcript_trimmed`) or fallback `transcripts/trimmed.json`
|
|
||||||
- Narratio resolves `normalized_transcript` from normalize manifest output (`transcript_normalized`) or fallback `transcripts/normalized.json`
|
|
||||||
- missing trimmed transcript fails clearly and advises running trim stage first
|
|
||||||
- `normalized_transcript` is the preferred full-transcript source for future table/meta-analysis artifacts
|
|
||||||
- `processed_transcript` remains supported for advanced/debug use cases
|
|
||||||
- optionally includes `previous_recap` when configured and resolvable
|
|
||||||
- omits optional previous recap when unavailable
|
|
||||||
- fails if required inputs are missing
|
|
||||||
- validates output file exists and is non-empty
|
|
||||||
|
|
||||||
Expected session output paths:
|
|
||||||
|
|
||||||
- `artifacts/session_recap.md`
|
|
||||||
- `logs/scriptorium.session_recap.stdout.log`
|
|
||||||
- `logs/scriptorium.session_recap.stderr.log`
|
|
||||||
- `config/scriptorium.session_recap.generated.yml`
|
|
||||||
- `artifacts/session_recap.render.json` when render diagnostics are enabled
|
|
||||||
|
|
||||||
## Examples
|
|
||||||
|
|
||||||
Starter files:
|
|
||||||
|
|
||||||
- `examples/pipeline.minimal.yml`
|
|
||||||
- `examples/session.minimal.yml`
|
|
||||||
- `examples/speakers.yml`
|
|
||||||
|
|
||||||
## Commands
|
|
||||||
|
|
||||||
Run tests:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
go test ./...
|
narratio run 2026-04-04
|
||||||
```
|
```
|
||||||
|
|
||||||
Plan a run:
|
This requires resolvable `pipeline.yml`, `campaign.yml`, and concrete `session.yml` (or explicit config flags).
|
||||||
|
|
||||||
```bash
|
## Documentation
|
||||||
go run ./cmd/narratio plan --session examples/session.minimal.yml
|
|
||||||
```
|
|
||||||
|
|
||||||
Use `--config <path>` to override default pipeline lookup when needed.
|
- [CLI Reference](docs/cli.md)
|
||||||
|
- [Configuration](docs/config.md)
|
||||||
Run full pipeline:
|
- [Operations](docs/operations.md)
|
||||||
|
- [Troubleshooting](docs/troubleshooting.md)
|
||||||
```bash
|
- [Internal Component Contracts](docs/internal/README.md)
|
||||||
go run ./cmd/narratio run --config examples/pipeline.minimal.yml --session examples/session.minimal.yml
|
- [Development Guide](docs/policy/development.md)
|
||||||
```
|
- [Architecture Principles](docs/policy/architecture.md)
|
||||||
|
- [Maintained Examples](examples/)
|
||||||
Run analyze only:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go run ./cmd/narratio run-stage --config examples/pipeline.minimal.yml --session examples/session.minimal.yml analyze
|
|
||||||
```
|
|
||||||
|
|
||||||
## Operational Note
|
|
||||||
|
|
||||||
Checksum-based stale detection is not implemented yet.
|
|
||||||
|
|
||||||
If prepared inputs or prompt/runtime config change, rerun the appropriate upstream stages before relying on downstream artifacts.
|
|
||||||
|
|
||||||
Examples:
|
|
||||||
|
|
||||||
- glossary/autocorrect/speaker-context changes: rerun at least `merge`, `polish`, `normalize`, `trim`, and `analyze`
|
|
||||||
- trim bounds prompt/profile/config changes: rerun at least `normalize`, `trim`, and `analyze`
|
|
||||||
- session recap prompt/profile/input-source changes: rerun `analyze`
|
|
||||||
|
|
||||||
## Roadmap
|
|
||||||
|
|
||||||
Near-term roadmap:
|
|
||||||
|
|
||||||
- extend analyze to additional configured artifacts
|
|
||||||
- support workflows where later artifacts consume earlier generated artifacts
|
|
||||||
- keep orchestration explicit without a generic DAG engine
|
|
||||||
- implement archive and notify backends
|
|
||||||
|
|||||||
323
architecture.md
323
architecture.md
@@ -1,323 +0,0 @@
|
|||||||
# Narratio Architecture
|
|
||||||
|
|
||||||
## 1. Purpose
|
|
||||||
|
|
||||||
`narratio` is a Go orchestrator for D&D session processing. It runs a stage-based local pipeline from audio input through transcript processing and artifact generation, with manifest-based skip/force/resume behavior.
|
|
||||||
|
|
||||||
Narratio integrates with Scriptorium through the **public CLI** (`scriptorium run` and `scriptorium render`) via synchronous subprocess execution.
|
|
||||||
|
|
||||||
## 2. Current Status
|
|
||||||
|
|
||||||
Implemented:
|
|
||||||
|
|
||||||
- strict `pipeline.yml` + `session.yml` loading with strict YAML field checking (`KnownFields(true)`)
|
|
||||||
- local workspace/session layout, lock file handling, artifact path helpers, checksums, and atomic writes
|
|
||||||
- manifest store and stage status transitions for resumable runs
|
|
||||||
- real `prepare`, `transcribe`, `merge`, and `polish` stages
|
|
||||||
- real WhisperX HTTP adapter
|
|
||||||
- real Seriatim subprocess adapter
|
|
||||||
- real Audita subprocess adapter
|
|
||||||
- real Scriptorium subprocess adapter
|
|
||||||
- real `normalize` stage producing `transcripts/normalized.json`
|
|
||||||
- real `trim` stage producing `transcripts/trimmed.json`
|
|
||||||
- real `analyze` stage for initial `session_recap` generation
|
|
||||||
- optional Scriptorium render diagnostics (`render_debug`) before production run
|
|
||||||
|
|
||||||
Still placeholder/future:
|
|
||||||
|
|
||||||
- `archive` stage behavior
|
|
||||||
- `notify` stage behavior
|
|
||||||
- additional Scriptorium artifact types beyond `session_recap`
|
|
||||||
- artifact-to-artifact workflows beyond the initial single-artifact implementation
|
|
||||||
- generic stale detection based on input/config checksums
|
|
||||||
|
|
||||||
## 3. Pipeline and Stage Boundaries
|
|
||||||
|
|
||||||
Canonical stage order:
|
|
||||||
|
|
||||||
1. `prepare`
|
|
||||||
2. `transcribe`
|
|
||||||
3. `merge`
|
|
||||||
4. `polish`
|
|
||||||
5. `normalize`
|
|
||||||
6. `trim`
|
|
||||||
7. `analyze`
|
|
||||||
8. `archive`
|
|
||||||
9. `notify`
|
|
||||||
|
|
||||||
Boundary rules:
|
|
||||||
|
|
||||||
- orchestration logic lives in `internal/app`
|
|
||||||
- stage business logic lives in `internal/stage`
|
|
||||||
- external-tool CLI construction lives in adapter packages
|
|
||||||
- Scriptorium CLI details stay in `internal/adapters/scriptorium`
|
|
||||||
|
|
||||||
## 4. Scriptorium Integration Model
|
|
||||||
|
|
||||||
Integration mode:
|
|
||||||
|
|
||||||
- public CLI subprocesses only (no Scriptorium internal Go packages, no HTTP API)
|
|
||||||
- production generation uses `scriptorium run`
|
|
||||||
- diagnostics/testing render uses `scriptorium render --format json`
|
|
||||||
|
|
||||||
Run invocation shape used by adapter:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
scriptorium run --prompt <prompt_id> --input name=path --out <output_path>
|
|
||||||
```
|
|
||||||
|
|
||||||
Optional flags passed when configured:
|
|
||||||
|
|
||||||
- `--config <path>`
|
|
||||||
- `--profile <profile_id>`
|
|
||||||
- repeated `--var name=value`
|
|
||||||
- repeated `--input name=path`
|
|
||||||
- `--timeout <duration>`
|
|
||||||
- `--api-key-env <ENV_NAME>` when configured
|
|
||||||
|
|
||||||
Render invocation shape used by adapter:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
scriptorium render --prompt <prompt_id> --input name=path --format json --out <render_output_path>
|
|
||||||
```
|
|
||||||
|
|
||||||
Adapter behavior:
|
|
||||||
|
|
||||||
- always passes `--out`
|
|
||||||
- captures stdout/stderr separately
|
|
||||||
- writes generated invocation metadata YAML (redacted, no secrets)
|
|
||||||
- treats exit code `0` as success
|
|
||||||
- treats exit code `1` as failure
|
|
||||||
- treats exit code `2` as failure with `validation_failed=true` and preserves output metadata when available
|
|
||||||
- validates successful output files exist and are non-empty
|
|
||||||
- does not treat non-empty stderr as failure by itself
|
|
||||||
|
|
||||||
## 5. Configuration Contract
|
|
||||||
|
|
||||||
CLI pipeline config path resolution:
|
|
||||||
|
|
||||||
- when `--config <path>` is provided, that path is used
|
|
||||||
- when `--config` is omitted, Narratio searches defaults in order:
|
|
||||||
- `/usr/local/etc/narratio/pipeline.yml`
|
|
||||||
- `/etc/narratio/pipeline.yml`
|
|
||||||
|
|
||||||
`pipeline.scriptorium` is optional. Existing pipelines without Scriptorium continue to work.
|
|
||||||
|
|
||||||
`pipeline.trim` is optional. Existing pipelines without trim config continue to work.
|
|
||||||
|
|
||||||
`pipeline.normalize` is optional. Existing pipelines without normalize config continue to work.
|
|
||||||
|
|
||||||
When `pipeline.normalize` is omitted, defaults are applied:
|
|
||||||
|
|
||||||
- `output_path: transcripts/normalized.json`
|
|
||||||
- `output_schema: seriatim-intermediate`
|
|
||||||
- `report: true`
|
|
||||||
|
|
||||||
When `pipeline.normalize` is present:
|
|
||||||
|
|
||||||
- `output_path` must be non-empty
|
|
||||||
- `output_schema` must be one of `seriatim-minimal`, `seriatim-intermediate`, or `seriatim-full`
|
|
||||||
- relative `output_path` values are session-workdir-relative paths
|
|
||||||
- Seriatim binary settings still come from `pipeline.seriatim`
|
|
||||||
|
|
||||||
When `pipeline.trim` is present:
|
|
||||||
|
|
||||||
- `enabled` is optional and defaults to `false` when omitted
|
|
||||||
- relative `output_path`, `bounds.output_path`, and `bounds.render_output_path` values are session-workdir-relative paths
|
|
||||||
- do not store secrets in trim config values
|
|
||||||
|
|
||||||
When `pipeline.trim.enabled: true`:
|
|
||||||
|
|
||||||
- `output_path` is required and non-empty
|
|
||||||
- `bounds.prompt_id` is required and non-empty
|
|
||||||
- `bounds.transcript_input_name` is required and non-empty
|
|
||||||
- `bounds.output_path` is required and non-empty
|
|
||||||
- `bounds.timeout` must parse as a Go duration when provided
|
|
||||||
- `bounds.render_debug: true` requires non-empty `bounds.render_output_path`
|
|
||||||
- `bounds.profile_id` may be empty to use the prompt default profile
|
|
||||||
- prompt IDs are config values, not hardcoded stage logic
|
|
||||||
|
|
||||||
When `pipeline.scriptorium` is present:
|
|
||||||
|
|
||||||
- `binary` is required and non-empty
|
|
||||||
- `config_path` is optional; when provided it must be non-empty
|
|
||||||
- `timeout` is optional; when provided it must parse as a Go duration
|
|
||||||
- default `timeout` is `10m`
|
|
||||||
- unknown YAML fields fail strict decode
|
|
||||||
|
|
||||||
Artifacts are configured as a map under `pipeline.scriptorium.artifacts` so multiple artifacts are possible in the config shape.
|
|
||||||
|
|
||||||
For each artifact definition:
|
|
||||||
|
|
||||||
- `enabled: true` requires non-empty `prompt_id`
|
|
||||||
- `enabled: true` requires non-empty `output_path`
|
|
||||||
- `timeout` must parse as Go duration when present
|
|
||||||
- optional per-artifact `render_debug` may override global `scriptorium.render_debug`
|
|
||||||
- `inputs` are named and each input requires non-empty `source`
|
|
||||||
- inputs may be optional (`required: false`)
|
|
||||||
- `vars` values currently support `string` and `bool`
|
|
||||||
|
|
||||||
Prompt IDs and profile IDs are configuration values, not hardcoded stage logic.
|
|
||||||
|
|
||||||
Trim config shape:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
trim:
|
|
||||||
enabled: true
|
|
||||||
output_path: "transcripts/trimmed.json"
|
|
||||||
bounds:
|
|
||||||
prompt_id: "dnd_session.bounds"
|
|
||||||
profile_id: ""
|
|
||||||
transcript_input_name: "transcript"
|
|
||||||
output_path: "artifacts/session_bounds.json"
|
|
||||||
timeout: "10m"
|
|
||||||
render_debug: false
|
|
||||||
render_output_path: "artifacts/session_bounds.render.json"
|
|
||||||
seriatim:
|
|
||||||
report: false
|
|
||||||
```
|
|
||||||
|
|
||||||
## 6. Transcript Tiers
|
|
||||||
|
|
||||||
Narratio currently produces and uses four transcript tiers:
|
|
||||||
|
|
||||||
- `transcripts/merged.json`: canonical deterministic merged transcript from Seriatim merge
|
|
||||||
- `transcripts/processed.json`: full raw Audita-polished transcript output (includes pre/post-game content)
|
|
||||||
- `transcripts/normalized.json`: normalized transcript generated by Seriatim normalize
|
|
||||||
- `transcripts/trimmed.json`: gameplay-only normalized polished transcript from trim stage
|
|
||||||
|
|
||||||
Trim reads `transcripts/normalized.json`, validates bounds IDs against that same transcript ID space, and writes `transcripts/trimmed.json`.
|
|
||||||
|
|
||||||
## 7. Normalize Stage (Current Implementation)
|
|
||||||
|
|
||||||
Normalize stage behavior:
|
|
||||||
|
|
||||||
- stage order position: after `polish` and before `trim`
|
|
||||||
- discovers processed transcript from manifest polish outputs (`transcript_processed`) when present, else `work/<session_id>/transcripts/processed.json`
|
|
||||||
- validates processed transcript JSON shape (`segments` array required)
|
|
||||||
- runs Seriatim `normalize` to produce normalized transcript
|
|
||||||
- validates normalized transcript JSON shape (`segments` array required)
|
|
||||||
- validates normalize report JSON when enabled
|
|
||||||
|
|
||||||
Expected normalize outputs and diagnostics:
|
|
||||||
|
|
||||||
- `transcripts/normalized.json`
|
|
||||||
- `artifacts/seriatim.normalize.report.json` (when normalize report is enabled)
|
|
||||||
- `logs/seriatim.normalize.stdout.log`
|
|
||||||
- `logs/seriatim.normalize.stderr.log`
|
|
||||||
- `config/seriatim.normalize.generated.yml`
|
|
||||||
|
|
||||||
## 8. Trim Stage (Current Implementation)
|
|
||||||
|
|
||||||
Trim stage behavior:
|
|
||||||
|
|
||||||
- stage order position: after `normalize` and before `analyze`
|
|
||||||
- discovers normalized transcript from manifest normalize outputs (`transcript_normalized`) when present, else `work/<session_id>/transcripts/normalized.json`
|
|
||||||
- validates normalized transcript JSON shape (`segments` array required)
|
|
||||||
- when `trim.enabled: false` (or trim config omitted), deterministically copies normalized transcript to `transcripts/trimmed.json` and records `trim_action=copy_disabled`
|
|
||||||
- when `trim.enabled: true`:
|
|
||||||
- runs Scriptorium bounds prompt using configured `trim.bounds.prompt_id`
|
|
||||||
- writes bounds output to configured path (typically `artifacts/session_bounds.json`)
|
|
||||||
- parses and validates bounds output against the same normalized transcript being trimmed
|
|
||||||
- converts bounds range to Seriatim keep selector (for example `10-868`)
|
|
||||||
- runs Seriatim `trim` to produce `transcripts/trimmed.json`
|
|
||||||
- supports no-trim bounds actions (`none`/`copy`) by copying normalized transcript unchanged
|
|
||||||
- validates trimmed transcript JSON shape (`segments` array required)
|
|
||||||
|
|
||||||
Expected trim outputs and diagnostics:
|
|
||||||
|
|
||||||
- `artifacts/session_bounds.json`
|
|
||||||
- `transcripts/trimmed.json`
|
|
||||||
- `logs/scriptorium.bounds.stdout.log`
|
|
||||||
- `logs/scriptorium.bounds.stderr.log`
|
|
||||||
- `config/scriptorium.bounds.generated.yml`
|
|
||||||
- `logs/seriatim.trim.stdout.log`
|
|
||||||
- `logs/seriatim.trim.stderr.log`
|
|
||||||
- `config/seriatim.trim.generated.yml`
|
|
||||||
- optional bounds render-debug outputs when enabled:
|
|
||||||
- `artifacts/session_bounds.render.json`
|
|
||||||
- `logs/scriptorium.bounds.render.stdout.log`
|
|
||||||
- `logs/scriptorium.bounds.render.stderr.log`
|
|
||||||
- `config/scriptorium.bounds.render.generated.yml`
|
|
||||||
|
|
||||||
Render-debug files are diagnostics. They are recorded in stage metadata/log/config refs and are not treated as canonical stage output artifact refs.
|
|
||||||
|
|
||||||
## 9. Analyze Stage (Current Implementation)
|
|
||||||
|
|
||||||
The current real analyze implementation supports only `scriptorium.artifacts.session_recap`.
|
|
||||||
|
|
||||||
Behavior:
|
|
||||||
|
|
||||||
- if `pipeline.scriptorium` is missing, analyze returns a skipped result with metadata
|
|
||||||
- if no Scriptorium artifacts are enabled, analyze returns a skipped result with metadata
|
|
||||||
- if enabled artifacts exist but `session_recap` is not enabled, analyze fails clearly
|
|
||||||
- available transcript input sources for configured artifacts: `processed_transcript`, `normalized_transcript`, `trimmed_transcript`
|
|
||||||
- `session_recap` should use `trimmed_transcript` input (`transcripts/trimmed.json`) for in-universe recap generation
|
|
||||||
- `trimmed_transcript` input is resolved from manifest (`trim` output kind `transcript_trimmed`) when available, otherwise fallback path `work/<session_id>/transcripts/trimmed.json`
|
|
||||||
- `normalized_transcript` input is resolved from manifest (`normalize` output kind `transcript_normalized`) when available, otherwise fallback path `work/<session_id>/transcripts/normalized.json`
|
|
||||||
- `processed_transcript` input is resolved from manifest (`polish` output kind `transcript_processed`) when available, otherwise fallback path `work/<session_id>/transcripts/processed.json`
|
|
||||||
- `normalized_transcript` is the preferred full-transcript source for future table/meta-analysis artifacts
|
|
||||||
- `processed_transcript` remains available for advanced/debug use cases
|
|
||||||
- transcript inputs are validated as JSON with top-level `segments` array
|
|
||||||
- configured inputs are resolved by source
|
|
||||||
- optional `previous_recap` is omitted when unavailable
|
|
||||||
- required `previous_recap` fails before invocation when unavailable
|
|
||||||
- vars are built from config + session metadata
|
|
||||||
- `render_debug` controls pre-run `scriptorium render` diagnostics
|
|
||||||
- render failure stops stage before production run
|
|
||||||
- render output is validated as JSON
|
|
||||||
- production call uses Scriptorium adapter `RunArtifact`
|
|
||||||
- successful run output must exist and be non-empty
|
|
||||||
- missing `trimmed_transcript` input for configured `trimmed_transcript` source fails clearly with guidance to run trim stage first
|
|
||||||
- manifest records output refs, logs, generated config paths, and non-secret provenance metadata
|
|
||||||
|
|
||||||
## 10. Session Recap Paths
|
|
||||||
|
|
||||||
Current expected paths for `session_recap`:
|
|
||||||
|
|
||||||
- artifact output: `artifacts/session_recap.md`
|
|
||||||
- run stdout log: `logs/scriptorium.session_recap.stdout.log`
|
|
||||||
- run stderr log: `logs/scriptorium.session_recap.stderr.log`
|
|
||||||
- run generated invocation/config: `config/scriptorium.session_recap.generated.yml`
|
|
||||||
- render output (when enabled): `artifacts/session_recap.render.json`
|
|
||||||
- render stdout log: `logs/scriptorium.session_recap.render.stdout.log`
|
|
||||||
- render stderr log: `logs/scriptorium.session_recap.render.stderr.log`
|
|
||||||
- render generated invocation/config: `config/scriptorium.session_recap.render.generated.yml`
|
|
||||||
|
|
||||||
## 11. Security and Privacy
|
|
||||||
|
|
||||||
- do not store secrets in pipeline YAML, generated invocation YAML, logs, or manifest metadata
|
|
||||||
- if API-key integration is configured, pass env var names only (never raw key values)
|
|
||||||
- avoid logging transcript content or rendered prompt content by default
|
|
||||||
- treat generated artifacts and logs as potentially sensitive session material
|
|
||||||
|
|
||||||
## 12. Operational Caveat (Pre-Stale-Detection)
|
|
||||||
|
|
||||||
Checksum-based stale detection is not implemented yet.
|
|
||||||
|
|
||||||
If prepared inputs or prompt/runtime configuration change (for example glossary files, prompt IDs, profile IDs, or relevant pipeline settings), rerun the appropriate prior stages to refresh downstream artifacts.
|
|
||||||
|
|
||||||
Examples:
|
|
||||||
|
|
||||||
- glossary or autocorrect changes usually require rerunning at least `merge`, `polish`, `normalize`, `trim`, and `analyze`
|
|
||||||
- trim prompt/profile changes require rerunning at least `normalize`, `trim`, and `analyze`
|
|
||||||
- session recap prompt/profile/input-source changes require rerunning `analyze`
|
|
||||||
|
|
||||||
## 13. Roadmap
|
|
||||||
|
|
||||||
Planned next steps:
|
|
||||||
|
|
||||||
- extend analyze beyond `session_recap` to additional configured artifacts
|
|
||||||
- support artifact inputs that consume prior generated artifacts
|
|
||||||
- keep this composable without adding a generic DAG engine in the near term
|
|
||||||
- implement real `archive` backend behavior
|
|
||||||
- implement real `notify` backend behavior
|
|
||||||
- add checksum-based stale detection and stale transitions
|
|
||||||
|
|
||||||
Architectural invariants remain:
|
|
||||||
|
|
||||||
- strict config decoding/validation
|
|
||||||
- manifest-driven run control
|
|
||||||
- clear stage/adapter separation
|
|
||||||
- configuration-driven prompt/profile/input/vars/output mapping
|
|
||||||
- Scriptorium integration through public CLI subprocess contract
|
|
||||||
281
docs/cli.md
Normal file
281
docs/cli.md
Normal file
@@ -0,0 +1,281 @@
|
|||||||
|
# CLI Reference
|
||||||
|
|
||||||
|
## Shortest Useful Command
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio run 2026-04-04
|
||||||
|
```
|
||||||
|
|
||||||
|
This runs the canonical full pipeline for session `2026-04-04`.
|
||||||
|
|
||||||
|
## Command Overview
|
||||||
|
|
||||||
|
Top-level commands:
|
||||||
|
|
||||||
|
- `run <session_id>`: run full stage order.
|
||||||
|
- `run-stage <stage> <session_id>`: run one stage.
|
||||||
|
- `analyze <session_id>`: force-run analyze.
|
||||||
|
- `publish <session_id>`: force-run publish.
|
||||||
|
- `clean <session_id>` or `clean --all`: remove local work/spool state.
|
||||||
|
- `session <subcommand>`: session helper commands.
|
||||||
|
|
||||||
|
Session subcommands:
|
||||||
|
|
||||||
|
- `session init <session_id>`
|
||||||
|
- `session plan <session_id>`
|
||||||
|
- `session validate <session_id>`
|
||||||
|
- `session status <session_id>`
|
||||||
|
- `session restore <session_id>`
|
||||||
|
- `session artifacts <session_id>`
|
||||||
|
- `session locks <session_id>`
|
||||||
|
- `session locks add <session_id> <source>`
|
||||||
|
- `session locks remove <session_id> <source>`
|
||||||
|
|
||||||
|
## Common Config Flags
|
||||||
|
|
||||||
|
Most session-aware commands accept:
|
||||||
|
|
||||||
|
- `--config <pipeline.yml>`
|
||||||
|
- `--campaign <id>`
|
||||||
|
- `--campaign-file <campaign.yml>`
|
||||||
|
- `--session <session.yml>`
|
||||||
|
- `--session-id <session_id>`
|
||||||
|
- `--previous-session-id <session_id>`
|
||||||
|
|
||||||
|
Rules:
|
||||||
|
|
||||||
|
- `--campaign` and `--campaign-file` are mutually exclusive.
|
||||||
|
- `--session` is not used by `session init`.
|
||||||
|
- if both positional `<session_id>` and `--session-id` are provided, values must match.
|
||||||
|
- `clean --all` cannot be combined with campaign/session selectors.
|
||||||
|
|
||||||
|
## Session ID Input Rules
|
||||||
|
|
||||||
|
Session-aware commands accept one of these forms:
|
||||||
|
|
||||||
|
- positional session ID: `... <session_id>`
|
||||||
|
- compatibility flag: `... --session-id <session_id>`
|
||||||
|
|
||||||
|
When both are present, command parsing requires an exact match.
|
||||||
|
|
||||||
|
Commands with additional positionals keep their command-specific order:
|
||||||
|
|
||||||
|
- `run-stage <stage> <session_id>` or `run-stage <stage> --session-id <session_id>`
|
||||||
|
- `session locks add <session_id> <source>` or `session locks add --session-id <session_id> <source>`
|
||||||
|
- `session locks remove <session_id> <source>` or `session locks remove --session-id <session_id> <source>`
|
||||||
|
|
||||||
|
## Command Reference
|
||||||
|
|
||||||
|
### `run`
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio run <session_id> [--force] [--artifacts <name[,name...]>] [...common config flags]
|
||||||
|
```
|
||||||
|
|
||||||
|
Behavior:
|
||||||
|
|
||||||
|
- evaluates full stage order;
|
||||||
|
- skips already-succeeded stages unless `--force` is set;
|
||||||
|
- continues interrupted or partially completed sessions by running non-succeeded stages;
|
||||||
|
- writes session and run manifests.
|
||||||
|
|
||||||
|
### `run-stage`
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio run-stage <stage> <session_id> [--force] [--artifacts <name[,name...]>] [...common config flags]
|
||||||
|
```
|
||||||
|
|
||||||
|
Valid stage names:
|
||||||
|
|
||||||
|
- `prepare`
|
||||||
|
- `transcribe`
|
||||||
|
- `merge`
|
||||||
|
- `polish`
|
||||||
|
- `normalize`
|
||||||
|
- `trim`
|
||||||
|
- `render`
|
||||||
|
- `analyze`
|
||||||
|
- `publish`
|
||||||
|
- `notify`
|
||||||
|
|
||||||
|
Rules:
|
||||||
|
|
||||||
|
- `--artifacts` is accepted only for `analyze` and `publish` stage targets.
|
||||||
|
|
||||||
|
### `analyze`
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio analyze <session_id> [--artifacts <name[,name...]>] [...common config flags]
|
||||||
|
```
|
||||||
|
|
||||||
|
Equivalent to:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio run-stage analyze <session_id> --force [...common config flags]
|
||||||
|
```
|
||||||
|
|
||||||
|
### `publish`
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio publish <session_id> [--artifacts <name[,name...]>] [...common config flags]
|
||||||
|
```
|
||||||
|
|
||||||
|
Equivalent to:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio run-stage publish <session_id> --force [...common config flags]
|
||||||
|
```
|
||||||
|
|
||||||
|
### `clean`
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio clean <session_id> [--dry-run] [--clear-cache] [...common config flags]
|
||||||
|
narratio clean --all [--dry-run] [--clear-cache] [--config <pipeline.yml>]
|
||||||
|
```
|
||||||
|
|
||||||
|
Behavior:
|
||||||
|
|
||||||
|
- session mode removes:
|
||||||
|
- `{workspace.root}/work/{campaign}/{session_id}`
|
||||||
|
- `{spool.root}/{campaign}/{session_id}`
|
||||||
|
- `--all` removes:
|
||||||
|
- `{workspace.root}/work/*`
|
||||||
|
- direct children under `{spool.root}`
|
||||||
|
- cache remains unless `--clear-cache` is provided.
|
||||||
|
|
||||||
|
### `session plan`
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio session plan <session_id> [--force] [...common config flags]
|
||||||
|
```
|
||||||
|
|
||||||
|
Validates config, prepares local workdir layout, and prints run/skip decisions for each stage.
|
||||||
|
|
||||||
|
### `session validate`
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio session validate <session_id> [...common config flags]
|
||||||
|
```
|
||||||
|
|
||||||
|
Read-only preflight checks for config validity, required inputs, audio mode, previous-session requirements, publish outputs, and effective locks.
|
||||||
|
|
||||||
|
### `session status`
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio session status <session_id> [...common config flags]
|
||||||
|
```
|
||||||
|
|
||||||
|
Prints local manifest state and, when storage is available, remote current-state and published-output status.
|
||||||
|
|
||||||
|
### `session init`
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio session init <session_id> --output ./session.yml [options]
|
||||||
|
narratio session init <session_id> --remote [options]
|
||||||
|
```
|
||||||
|
|
||||||
|
Required target selection:
|
||||||
|
|
||||||
|
- exactly one of:
|
||||||
|
- `--output <path>`
|
||||||
|
- `--remote`
|
||||||
|
|
||||||
|
Options:
|
||||||
|
|
||||||
|
- `--config <pipeline.yml>`
|
||||||
|
- `--campaign <id>` or `--campaign-file <campaign.yml>`
|
||||||
|
- `--previous-session-id <id>`
|
||||||
|
- `--date <YYYY-MM-DD>`
|
||||||
|
- `--title <text>`
|
||||||
|
- `--audio-dir <path>`
|
||||||
|
- `--audio-s3-prefix <prefix>`
|
||||||
|
- `--force`
|
||||||
|
|
||||||
|
Rules:
|
||||||
|
|
||||||
|
- `--audio-dir` and `--audio-s3-prefix` are mutually exclusive.
|
||||||
|
- if campaign `session_template_file` is configured, `session init` renders it.
|
||||||
|
- generated session YAML must be concrete (no unresolved `{{ ... }}` placeholders).
|
||||||
|
|
||||||
|
### `session restore`
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio session restore <session_id> [--dry-run] [--force] [--include-audio] [...common config flags]
|
||||||
|
```
|
||||||
|
|
||||||
|
Behavior:
|
||||||
|
|
||||||
|
- discovers committed remote current state;
|
||||||
|
- plans local restores;
|
||||||
|
- writes `reports/restore-latest.json` on execution;
|
||||||
|
- blocks conflicting overwrites unless `--force` is set.
|
||||||
|
|
||||||
|
Default restore scope:
|
||||||
|
|
||||||
|
- `manifest.json`
|
||||||
|
- `transcripts/**`
|
||||||
|
- `artifacts/**`
|
||||||
|
- `previous/**` when required by configured previous-session inputs
|
||||||
|
|
||||||
|
`audio/**` is included only with `--include-audio`.
|
||||||
|
|
||||||
|
### `session artifacts`
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio session artifacts <session_id> [--remote] [...common config flags]
|
||||||
|
```
|
||||||
|
|
||||||
|
Lists effective built-in and configured artifact sources, publish rules, lock state, and optional remote published-state availability.
|
||||||
|
|
||||||
|
### `session locks`
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio session locks <session_id> [...common config flags]
|
||||||
|
narratio session locks add <session_id> <source> [--reason <text>] [--force] [...common config flags]
|
||||||
|
narratio session locks remove <session_id> <source> [...common config flags]
|
||||||
|
```
|
||||||
|
|
||||||
|
Behavior:
|
||||||
|
|
||||||
|
- list mode merges static `pipeline.publish.locks` with remote `{session_prefix}/locks.yml`;
|
||||||
|
- add/remove mutate only remote locks;
|
||||||
|
- static locks from pipeline config cannot be removed by CLI commands.
|
||||||
|
|
||||||
|
## `--artifacts` Selection Rules
|
||||||
|
|
||||||
|
- accepted on `run`, `run-stage`, `analyze`, and `publish`;
|
||||||
|
- names must exist in `pipeline.scriptorium.artifacts`;
|
||||||
|
- empty entries are invalid;
|
||||||
|
- repeated names are deduplicated.
|
||||||
|
|
||||||
|
Effects:
|
||||||
|
|
||||||
|
- filters analyze execution to selected configured artifacts;
|
||||||
|
- filters publish rules that source `narratio.artifact.<name>`;
|
||||||
|
- does not filter built-in transcript/bounds publish sources.
|
||||||
|
|
||||||
|
## Common Workflows
|
||||||
|
|
||||||
|
Run full pipeline:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio run 2026-04-04
|
||||||
|
```
|
||||||
|
|
||||||
|
Dry-run restore plan:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio session restore 2026-04-04 --dry-run
|
||||||
|
```
|
||||||
|
|
||||||
|
Generate a concrete session file from template/default structure:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio session init 2026-04-04 --output ./session.yml --date 2026-04-04 --title "Session 12"
|
||||||
|
```
|
||||||
|
|
||||||
|
Force publish only:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio publish 2026-04-04
|
||||||
|
```
|
||||||
281
docs/config.md
Normal file
281
docs/config.md
Normal file
@@ -0,0 +1,281 @@
|
|||||||
|
# Configuration Reference
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
Narratio resolves three YAML documents:
|
||||||
|
|
||||||
|
- `pipeline.yml`: pipeline/runtime settings
|
||||||
|
- `campaign.yml`: campaign identity and stable input defaults
|
||||||
|
- `session.yml`: session identity, metadata, and audio source selection
|
||||||
|
|
||||||
|
## Discovery and Selection
|
||||||
|
|
||||||
|
### `pipeline.yml`
|
||||||
|
|
||||||
|
When `--config` is omitted, search order is:
|
||||||
|
|
||||||
|
1. `/usr/local/etc/narratio/pipeline.yml`
|
||||||
|
2. `/etc/narratio/pipeline.yml`
|
||||||
|
|
||||||
|
### `campaign.yml`
|
||||||
|
|
||||||
|
Selection rules:
|
||||||
|
|
||||||
|
- if `--campaign-file` is set, use that path;
|
||||||
|
- else if `--campaign <id>` is set, use `{pipeline.campaigns.root}/{id}/campaign.yml`;
|
||||||
|
- else use `{pipeline.campaigns.root}/{pipeline.campaigns.default_campaign_id}/campaign.yml`.
|
||||||
|
|
||||||
|
### `session.yml`
|
||||||
|
|
||||||
|
When `--session` is omitted, local search order is:
|
||||||
|
|
||||||
|
1. `/usr/local/etc/narratio/session.yml`
|
||||||
|
2. `/etc/narratio/session.yml`
|
||||||
|
|
||||||
|
If local session discovery fails and a `session_id` is known, Narratio attempts remote session loading from:
|
||||||
|
|
||||||
|
- `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/session.yml`
|
||||||
|
|
||||||
|
using configured object storage.
|
||||||
|
|
||||||
|
## Validation and Merge Rules
|
||||||
|
|
||||||
|
- YAML decode is strict (`KnownFields(true)`): unknown fields fail load.
|
||||||
|
- Session files must be concrete; unresolved `{{ ... }}` placeholders fail load.
|
||||||
|
- Pipeline defaults are applied before validation.
|
||||||
|
- Campaign and session identities must agree.
|
||||||
|
- Stable files (`speakers_file`, `autocorrect_file`, `glossary_file`, `players_file`, `party_file`) resolve from session overrides when provided, otherwise from campaign defaults.
|
||||||
|
- Exactly one audio mode must be configured in session input:
|
||||||
|
- local (`audio_dir` or `audio_files`), or
|
||||||
|
- S3 (`audio_s3.prefix`).
|
||||||
|
|
||||||
|
## Minimal Working Configuration
|
||||||
|
|
||||||
|
`pipeline.yml`
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
campaigns:
|
||||||
|
root: /usr/local/share/narratio/campaigns
|
||||||
|
default_campaign_id: sample-campaign
|
||||||
|
whisperx:
|
||||||
|
transcribe_url: https://transcription.example.com/transcribe
|
||||||
|
```
|
||||||
|
|
||||||
|
`campaign.yml`
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
campaign_id: sample-campaign
|
||||||
|
inputs:
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
players_file: ./players.yml
|
||||||
|
party_file: ./party.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
`session.yml` (local audio)
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
session_id: 2026-05-03
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
|
```
|
||||||
|
|
||||||
|
## Secrets Handling
|
||||||
|
|
||||||
|
- Do not place raw secrets in YAML.
|
||||||
|
- Use env var names in config (for example `pipeline.audita.llm_api_key_env`).
|
||||||
|
- Optionally load env files from `pipeline.secrets.env_dir`.
|
||||||
|
- Commands that need storage/auth load filesystem secrets before constructing adapters.
|
||||||
|
|
||||||
|
## Publish Configuration Summary
|
||||||
|
|
||||||
|
Publish rules live under `pipeline.publish`.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
publish:
|
||||||
|
enabled: true
|
||||||
|
upload_run: true
|
||||||
|
outputs:
|
||||||
|
- source: narratio.transcript.final_trimmed
|
||||||
|
dest: transcripts/final.trimmed.json
|
||||||
|
required: true
|
||||||
|
- source: narratio.transcript.final_markdown
|
||||||
|
dest: transcripts/final.md
|
||||||
|
required: true
|
||||||
|
- source: narratio.transcript.final_trimmed_markdown
|
||||||
|
dest: transcripts/final.trimmed.md
|
||||||
|
required: true
|
||||||
|
- source: narratio.artifact.session_recap
|
||||||
|
dest: artifacts/session_recap.md
|
||||||
|
required: true
|
||||||
|
locks:
|
||||||
|
- source: narratio.artifact.session_recap
|
||||||
|
reason: manual post-publish edits
|
||||||
|
```
|
||||||
|
|
||||||
|
Rules:
|
||||||
|
|
||||||
|
- `outputs[].source` is required.
|
||||||
|
- `outputs[].dest` may be omitted when derivable from source.
|
||||||
|
- `outputs[].required` defaults to `true`.
|
||||||
|
- static locks (`pipeline.publish.locks`) merge with remote locks (`{session_prefix}/locks.yml`), with static locks taking precedence on duplicates.
|
||||||
|
|
||||||
|
## Full Schema
|
||||||
|
|
||||||
|
### Pipeline
|
||||||
|
|
||||||
|
| Field | Type | Required | Default / Rule |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| `pipeline.workspace.root` | string | No | `/var/lib/narratio` |
|
||||||
|
| `pipeline.workspace.cleanup_after_publish` | bool | No | `false` |
|
||||||
|
| `pipeline.campaigns.root` | string | No | `/usr/local/share/narratio/campaigns` |
|
||||||
|
| `pipeline.campaigns.default_campaign_id` | string | No | empty |
|
||||||
|
| `pipeline.secrets.env_dir` | string | No | empty |
|
||||||
|
| `pipeline.storage.backend` | string | No | empty |
|
||||||
|
| `pipeline.storage.s3.bucket` | string | Conditional | required for S3 session-audio and for publish upload when backend is `s3` |
|
||||||
|
| `pipeline.storage.s3.root_prefix` | string | No | `dnd` |
|
||||||
|
| `pipeline.storage.s3.region` | string | No | empty |
|
||||||
|
| `pipeline.storage.s3.endpoint` | string | No | empty |
|
||||||
|
| `pipeline.storage.s3.force_path_style` | bool | No | `false` |
|
||||||
|
| `pipeline.storage.s3.access_key_id_env` | string | No | `OBJECT_STORAGE_KEY_ID` |
|
||||||
|
| `pipeline.storage.s3.secret_access_key_env` | string | No | `OBJECT_STORAGE_KEY` |
|
||||||
|
| `pipeline.spool.root` | string | No | `/var/spool/narratio` |
|
||||||
|
| `pipeline.spool.delete_audio_after_publish` | bool | No | `false` |
|
||||||
|
| `pipeline.cache.root` | string | No | `/var/cache/narratio` |
|
||||||
|
| `pipeline.cache.s3_audio` | bool | No | `true` |
|
||||||
|
| `pipeline.publish.enabled` | bool | No | `true` |
|
||||||
|
| `pipeline.publish.upload_run` | bool | No | `true` |
|
||||||
|
| `pipeline.publish.outputs[]` | list | No | defaults to final trimmed JSON plus final and final-trimmed Markdown outputs |
|
||||||
|
| `pipeline.publish.outputs[].source` | string | Yes (per rule) | must reference built-in or configured artifact source |
|
||||||
|
| `pipeline.publish.outputs[].dest` | string | Conditional | derived if omitted and source supports derivation |
|
||||||
|
| `pipeline.publish.outputs[].required` | bool | No | `true` |
|
||||||
|
| `pipeline.publish.locks[]` | list | No | empty |
|
||||||
|
| `pipeline.publish.locks[].source` | string | Yes (per lock) | must reference supported publish source |
|
||||||
|
| `pipeline.publish.locks[].reason` | string | No | empty |
|
||||||
|
| `pipeline.whisperx.transcribe_url` | string | Yes | valid URL |
|
||||||
|
| `pipeline.whisperx.language` | string | No | `en` |
|
||||||
|
| `pipeline.whisperx.timeout` | duration | No | `30m` |
|
||||||
|
| `pipeline.whisperx.retries` | int | No | `3` |
|
||||||
|
| `pipeline.whisperx.retry_delay` | duration | No | `2s` |
|
||||||
|
| `pipeline.whisperx.concurrency` | int | No | `2` |
|
||||||
|
| `pipeline.seriatim.binary` | string | No | `seriatim` |
|
||||||
|
| `pipeline.seriatim.timeout` | duration | No | `10m` |
|
||||||
|
| `pipeline.seriatim.output_schema` | string | No | `seriatim-intermediate` |
|
||||||
|
| `pipeline.seriatim.coalesce_gap` | float | No | `3.0` |
|
||||||
|
| `pipeline.seriatim.report` | bool | No | `true` |
|
||||||
|
| `pipeline.seriatim.env.overlap_word_run_gap` | float | No | unset |
|
||||||
|
| `pipeline.seriatim.env.overlap_word_run_reorder_window` | float | No | unset |
|
||||||
|
| `pipeline.seriatim.env.backchannel_max_duration` | float | No | unset |
|
||||||
|
| `pipeline.seriatim.env.filler_max_duration` | float | No | unset |
|
||||||
|
| `pipeline.audita.binary` | string | No | `audita` |
|
||||||
|
| `pipeline.audita.timeout` | duration | No | `3h` |
|
||||||
|
| `pipeline.audita.llm_api_key_env` | string | No | empty |
|
||||||
|
| `pipeline.audita.modules[]` | list[string] | No | empty |
|
||||||
|
| `pipeline.audita.base_url` | string | No | empty |
|
||||||
|
| `pipeline.audita.model` | string | No | empty |
|
||||||
|
| `pipeline.audita.total_llm_concurrency` | int | No | unset |
|
||||||
|
| `pipeline.audita.proposal_llm_concurrency` | int | No | unset |
|
||||||
|
| `pipeline.audita.validation_model` | string | No | empty |
|
||||||
|
| `pipeline.audita.validation_llm_concurrency` | int | No | unset |
|
||||||
|
| `pipeline.audita.transcript_description` | string | No | empty |
|
||||||
|
| `pipeline.audita.config_path` | string | No | empty |
|
||||||
|
| `pipeline.audita.output_schema` | string | No | empty |
|
||||||
|
| `pipeline.audita.work_dir_retention` | string | No | empty |
|
||||||
|
| `pipeline.audita.report` | bool | No | `true` |
|
||||||
|
| `pipeline.normalize.output_path` | string | No | `transcripts/final.json` |
|
||||||
|
| `pipeline.normalize.output_schema` | string | No | `seriatim-intermediate` |
|
||||||
|
| `pipeline.normalize.report` | bool | No | `true` |
|
||||||
|
| `pipeline.trim.enabled` | bool | No | `true` |
|
||||||
|
| `pipeline.trim.output_path` | string | No | `transcripts/final.trimmed.json` |
|
||||||
|
| `pipeline.trim.bounds.prompt_id` | string | No | `dnd.session_bounds` |
|
||||||
|
| `pipeline.trim.bounds.profile_id` | string | No | empty |
|
||||||
|
| `pipeline.trim.bounds.transcript_input_name` | string | No | `transcript` |
|
||||||
|
| `pipeline.trim.bounds.output_path` | string | No | `artifacts/session_bounds.json` |
|
||||||
|
| `pipeline.trim.bounds.timeout` | duration | No | `10m` |
|
||||||
|
| `pipeline.trim.bounds.render_debug` | bool | No | `false` |
|
||||||
|
| `pipeline.trim.bounds.render_output_path` | string | Conditional | required when `render_debug` is true |
|
||||||
|
| `pipeline.trim.seriatim.report` | bool | No | `false` |
|
||||||
|
| `pipeline.render.enabled` | bool | No | `true` |
|
||||||
|
| `pipeline.render.format` | string | No | `markdown` (only supported value) |
|
||||||
|
| `pipeline.render.title` | string | No | empty (falls back to `session.title` when set) |
|
||||||
|
| `pipeline.render.include_timestamps` | bool | No | `true` |
|
||||||
|
| `pipeline.render.include_segment_ids` | bool | No | `true` |
|
||||||
|
| `pipeline.render.include_metadata` | bool | No | `false` |
|
||||||
|
| `pipeline.scriptorium.binary` | string | No | `scriptorium` |
|
||||||
|
| `pipeline.scriptorium.config_path` | string | No | empty |
|
||||||
|
| `pipeline.scriptorium.timeout` | duration | No | `10m` |
|
||||||
|
| `pipeline.scriptorium.render_debug` | bool | No | `false` |
|
||||||
|
| `pipeline.scriptorium.artifacts` | map | No | empty |
|
||||||
|
| `pipeline.notification.backend` | string | No | empty |
|
||||||
|
| `pipeline.notification.recipient` | string | No | empty |
|
||||||
|
| `pipeline.notification.timeout` | duration | No | `30s` |
|
||||||
|
|
||||||
|
### Scriptorium Artifact Entries
|
||||||
|
|
||||||
|
For each `pipeline.scriptorium.artifacts.<name>`:
|
||||||
|
|
||||||
|
| Field | Type | Required | Rule |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| `enabled` | bool | No | `false` if omitted |
|
||||||
|
| `depends_on[]` | list[string] | No | must reference configured artifact keys; no self-reference; enabled graph must be acyclic |
|
||||||
|
| `render_debug` | bool | No | per-artifact override |
|
||||||
|
| `prompt_id` | string | Conditional | required when artifact is enabled |
|
||||||
|
| `profile_id` | string | No | empty |
|
||||||
|
| `output_path` | string | Conditional | required when enabled; also required when referenced by publish/output/input rules |
|
||||||
|
| `timeout` | duration | No | artifact override |
|
||||||
|
| `inputs` | map | No | input key names must be non-empty |
|
||||||
|
| `vars` | map | No | values must be string or bool |
|
||||||
|
|
||||||
|
For each artifact input `pipeline.scriptorium.artifacts.<name>.inputs.<input_name>`:
|
||||||
|
|
||||||
|
| Field | Type | Required | Rule |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| `source` | string | Yes | built-in runtime source, prepared input source, `narratio.artifact.<name>`, or `narratio.previous_session.artifact.<name>` |
|
||||||
|
| `artifact` | string | No | optional passthrough adapter field |
|
||||||
|
| `path` | string | No | optional passthrough adapter field |
|
||||||
|
| `required` | bool | No | optional input requirement |
|
||||||
|
|
||||||
|
### Campaign
|
||||||
|
|
||||||
|
| Field | Type | Required | Notes |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| `campaign_id` | string | Yes | canonical campaign identity |
|
||||||
|
| `session_template_file` | string | No | used by `session init` when set |
|
||||||
|
| `inputs.speakers_file` | string | Yes | stable input default |
|
||||||
|
| `inputs.autocorrect_file` | string | Yes | stable input default |
|
||||||
|
| `inputs.glossary_file` | string | Yes | stable input default |
|
||||||
|
| `inputs.players_file` | string | Yes | stable input default |
|
||||||
|
| `inputs.party_file` | string | Yes | stable input default |
|
||||||
|
|
||||||
|
### Session
|
||||||
|
|
||||||
|
| Field | Type | Required in session file | Notes |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| `session_id` | string | Yes | must match CLI session target when provided |
|
||||||
|
| `previous_session_id` | string | No | must not equal `session_id` |
|
||||||
|
| `campaign` | string | No | filled from `campaign_id` during resolve if omitted |
|
||||||
|
| `date` | string | No | metadata |
|
||||||
|
| `title` | string | No | metadata |
|
||||||
|
| `inputs.speakers_file` | string | No | overrides campaign stable input |
|
||||||
|
| `inputs.autocorrect_file` | string | No | overrides campaign stable input |
|
||||||
|
| `inputs.glossary_file` | string | No | overrides campaign stable input |
|
||||||
|
| `inputs.players_file` | string | No | overrides campaign stable input |
|
||||||
|
| `inputs.party_file` | string | No | overrides campaign stable input |
|
||||||
|
| `inputs.audio_dir` | string | Conditional | local audio mode |
|
||||||
|
| `inputs.audio_files[]` | list[string] | Conditional | local audio mode |
|
||||||
|
| `inputs.audio_s3.prefix` | string | Conditional | S3 audio mode |
|
||||||
|
|
||||||
|
Audio rules:
|
||||||
|
|
||||||
|
- configure local mode (`audio_dir` or `audio_files`) or S3 mode (`audio_s3.prefix`), not both.
|
||||||
|
|
||||||
|
## Maintained Examples
|
||||||
|
|
||||||
|
- `examples/pipeline.minimal.yml`
|
||||||
|
- `examples/pipeline.production.yml`
|
||||||
|
- `examples/pipeline.full.annotated.yml`
|
||||||
|
- `examples/campaigns/sample-campaign/campaign.yml`
|
||||||
|
- `examples/session.local-audio.yml`
|
||||||
|
- `examples/session.s3-audio.yml`
|
||||||
|
- `examples/session.template.yml`
|
||||||
19
docs/integrations/README.md
Normal file
19
docs/integrations/README.md
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
# Integrations Index
|
||||||
|
|
||||||
|
## Audience
|
||||||
|
Developers and coding agents changing Narratio's external integration boundaries.
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
`docs/integrations/` is the implementation-level reference for downstream tool adapter contracts.
|
||||||
|
|
||||||
|
These docs cover what Narratio expects from external tools and what each adapter guarantees back to stage code.
|
||||||
|
|
||||||
|
## Integration Contracts
|
||||||
|
- `audita.md`: transcript polishing adapter (`audita process`).
|
||||||
|
- `seriatim.md`: merge/normalize/trim/render adapter (`seriatim`).
|
||||||
|
- `scriptorium.md`: artifact run/render adapter (`scriptorium run|render`).
|
||||||
|
|
||||||
|
## Related Canonical Docs
|
||||||
|
- `docs/config.md`: operator-facing configuration reference.
|
||||||
|
- `docs/internal/adapters.md`: shared adapter boundary and runner wiring.
|
||||||
|
- `docs/internal/stage-*.md`: stage-specific integration usage.
|
||||||
@@ -1,147 +1,60 @@
|
|||||||
# Audita
|
# Integration: Audita
|
||||||
|
|
||||||
Audita is a framework-first transcript correction application. The public `audita` package provides:
|
## Purpose
|
||||||
|
Define the Audita adapter contract used by the `polish` stage.
|
||||||
|
|
||||||
- deterministic transcript normalization
|
## Adapter Boundary
|
||||||
- token-batched module orchestration
|
Interface:
|
||||||
- concrete `glossary`, `homophones`, `spoken_word`, and `grammar` modules built on reusable proposal / validator contracts
|
- `audita.Runner`
|
||||||
- structured run reporting and work-dir diagnostics
|
- method: `Run(ctx, PolishRequest) (PolishResult, error)`
|
||||||
|
|
||||||
The previous working implementation has been preserved as `audita_prototype` inside this repository. Its full regression suite lives under `tests/audita_prototype`.
|
Primary implementation:
|
||||||
|
- `internal/adapters/audita/SubprocessRunner`
|
||||||
|
|
||||||
## Development
|
Execution mode:
|
||||||
|
- subprocess invocation of `audita process`
|
||||||
|
|
||||||
This project is set up for `uv`.
|
## Request Contract
|
||||||
|
`PolishRequest` carries:
|
||||||
|
- required transcript/glossary/output/work-dir paths;
|
||||||
|
- optional report path (required when report mode is enabled);
|
||||||
|
- generated config and stdout/stderr log paths;
|
||||||
|
- optional module/model/base-url/config/output-schema/concurrency settings.
|
||||||
|
|
||||||
```sh
|
## Result Contract
|
||||||
uv sync --extra dev
|
`PolishResult` returns:
|
||||||
uv run pytest
|
- processed transcript path;
|
||||||
```
|
- optional report path;
|
||||||
|
- work dir and generated-config/log paths;
|
||||||
|
- exit code, duration, binary provenance;
|
||||||
|
- adapter metadata map.
|
||||||
|
|
||||||
## Usage
|
## Validation and Failure Semantics
|
||||||
|
Construction fails for invalid static config values, including:
|
||||||
|
- empty binary;
|
||||||
|
- non-positive timeout;
|
||||||
|
- invalid base URL;
|
||||||
|
- invalid output schema;
|
||||||
|
- invalid work-dir retention value;
|
||||||
|
- invalid concurrency values.
|
||||||
|
|
||||||
Process a transcript with the current framework implementation:
|
Run fails for:
|
||||||
|
- missing required request paths;
|
||||||
|
- missing required credential env var when configured (`llm_api_key_env`);
|
||||||
|
- subprocess execution failure;
|
||||||
|
- invalid processed transcript JSON (`segments` array required);
|
||||||
|
- invalid report JSON when reporting is enabled.
|
||||||
|
|
||||||
```sh
|
Failure results still include output/log/config/exit metadata for diagnostics.
|
||||||
uv run audita process transcript.json --glossary glossary.yaml --output corrected.json
|
|
||||||
```
|
|
||||||
|
|
||||||
The framework currently runs this default module sequence:
|
## Deterministic Behavior
|
||||||
|
- CLI args are built from runner config + request in a fixed order.
|
||||||
|
- Generated invocation YAML (`audita.generated.v1`) is emitted when requested.
|
||||||
|
- Manifest writes are stage-owned; adapter itself is stateless.
|
||||||
|
|
||||||
1. `glossary`
|
## Config Mapping
|
||||||
2. `homophones`
|
Config fields consumed through runner/stage wiring are under `pipeline.audita.*`.
|
||||||
3. `glossary`
|
|
||||||
4. `spoken_word`
|
|
||||||
5. `grammar`
|
|
||||||
|
|
||||||
Resolved run instance names are auto-numbered for repeats, so the default report pipeline is:
|
Maintained example with Audita config:
|
||||||
|
- `examples/pipeline.full.annotated.yml`
|
||||||
1. `glossary_1`
|
- `examples/pipeline.production.yml`
|
||||||
2. `homophones`
|
|
||||||
3. `glossary_2`
|
|
||||||
4. `spoken_word`
|
|
||||||
5. `grammar`
|
|
||||||
|
|
||||||
The default module sequence is fully implemented today:
|
|
||||||
|
|
||||||
- `glossary` proposes glossary-supported acoustic corrections
|
|
||||||
- `homophones` proposes conservative homophone and mistranscription corrections
|
|
||||||
- `spoken_word` proposes conservative dysfluency cleanup
|
|
||||||
- `grammar` proposes punctuation, capitalization, and spacing cleanup only
|
|
||||||
|
|
||||||
To run a custom module sequence, pass `--modules`:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
uv run audita process transcript.json --glossary glossary.yaml --modules grammar --output corrected.json
|
|
||||||
```
|
|
||||||
|
|
||||||
To also write a structured JSON report:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
uv run audita process transcript.json --glossary glossary.yaml --output corrected.json --report-json report.json
|
|
||||||
```
|
|
||||||
|
|
||||||
From a checked-out repository, you can also use the root launcher:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
./audita process transcript.json --glossary glossary.yaml --output corrected.json
|
|
||||||
```
|
|
||||||
|
|
||||||
For a system-wide command, install the source tree under `/usr/local/src/audita`, sync dependencies there, and symlink the root launcher into your `PATH`:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
cd /usr/local/src/audita
|
|
||||||
uv sync --extra dev
|
|
||||||
ln -s /usr/local/src/audita/audita /usr/local/bin/audita
|
|
||||||
audita process transcript.json --glossary glossary.yaml --output corrected.json
|
|
||||||
```
|
|
||||||
|
|
||||||
Without `--output`, Audita writes the corrected transcript JSON to stdout and progress logs to stderr.
|
|
||||||
`--report-json` writes a separate machine-readable run report and never mixes report data into stdout.
|
|
||||||
|
|
||||||
Useful configuration can be supplied by CLI flag or environment variable. CLI flags take precedence over environment variables. Normal runs now require LLM API credentials, because the `glossary`, `homophones`, `spoken_word`, and `grammar` modules make real LLM calls. `AUDITA_LLM_API_KEY` and `--llm-api-key` are the preferred provider-neutral credential surfaces, while `OPENROUTER_API_KEY` remains supported as a backward-compatible fallback.
|
|
||||||
|
|
||||||
| Environment variable | CLI flag | Default | Purpose |
|
|
||||||
| --- | --- | --- | --- |
|
|
||||||
| `AUDITA_MODULES` | `--modules` | `glossary,homophones,glossary,spoken_word,grammar` | Comma-separated logical module keys to run; CLI overrides the environment value |
|
|
||||||
| `AUDITA_LLM_API_KEY` | `--llm-api-key` | unset | Preferred provider-neutral LLM API credential; CLI overrides both environment-key variants |
|
|
||||||
| `AUDITA_VALIDATION_LLM_API_KEY` | `--validation-llm-api-key` | unset | Validation-phase LLM API credential; defaults to the primary LLM API key |
|
|
||||||
| `AUDITA_MODEL` | `--model` | `openrouter/google/gemma-4-31b-it` | LLM model name sent to the configured OpenAI-compatible endpoint |
|
|
||||||
| `AUDITA_VALIDATION_MODEL` | `--validation-model` | unset | Validation-phase LLM model; defaults to `AUDITA_MODEL` |
|
|
||||||
| `AUDITA_BASE_URL` | `--base-url` | `https://openrouter.ai/api/v1` | OpenAI-compatible API base URL |
|
|
||||||
| `AUDITA_VALIDATION_BASE_URL` | `--validation-base-url` | unset | Validation-phase OpenAI-compatible API base URL; defaults to `AUDITA_BASE_URL` |
|
|
||||||
| `AUDITA_LLM_TIMEOUT_SECONDS` | `--llm-timeout-seconds` | `600` | Per-request timeout in seconds for LLM calls to the configured OpenAI-compatible endpoint |
|
|
||||||
| `AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS` | `--validation-llm-timeout-seconds` | unset | Validation-phase per-request timeout in seconds; defaults to `AUDITA_LLM_TIMEOUT_SECONDS` |
|
|
||||||
| `AUDITA_VALIDATION_MAX_PROMPT_TOKENS` | `--validation-max-prompt-tokens` | `2048` | Maximum estimated tokens per validation-phase LLM prompt batch |
|
|
||||||
| `AUDITA_TARGET_SECTIONS` | `--target-sections` | unset | Exact number of contiguous proposal-stage transcript sections; errors if min/max token bounds cannot be satisfied |
|
|
||||||
| `AUDITA_MAX_RETRIES` | `--max-retries` | `3` | Maximum Instructor retries for structured responses |
|
|
||||||
| `AUDITA_VALIDATION_MAX_RETRIES` | `--validation-max-retries` | unset | Validation-phase structured-output retries; defaults to `AUDITA_MAX_RETRIES` |
|
|
||||||
| `AUDITA_VALIDATION_LLM_CONCURRENCY` | `--validation-llm-concurrency` | unset | Validation-phase LLM concurrency; defaults to `AUDITA_LLM_CONCURRENCY` |
|
|
||||||
| `AUDITA_MAX_SECTION_TOKENS` | `--max-section-tokens` | `8192` | Maximum estimated tokens per proposal-stage transcript section |
|
|
||||||
| `AUDITA_MIN_SECTION_TOKENS` | `--min-section-tokens` | `2048` | Minimum estimated tokens per proposal-stage transcript section when balancing for concurrency |
|
|
||||||
| `AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD` | `--glossary-confidence-threshold` | `0.8` | Minimum confidence required for glossary proposals to survive validation |
|
|
||||||
| `AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD` | `--grammar-confidence-threshold` | `0.8` | Minimum confidence required for grammar proposals to survive validation |
|
|
||||||
| `AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD` | `--homophones-confidence-threshold` | `0.8` | Minimum confidence required for homophone proposals to survive validation |
|
|
||||||
| `AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD` | `--spoken-word-confidence-threshold` | `0.8` | Minimum confidence required for spoken-word proposals to survive validation |
|
|
||||||
| `AUDITA_NORMALIZE_MAX_SEGMENT_GAP` | `--normalize-max-segment-gap` | `4.0` | Same-speaker gaps eligible for deterministic merging |
|
|
||||||
| `AUDITA_NORMALIZE_ELLIPSIS_GAP` | `--normalize-ellipsis-gap` | `3.5` | Same-speaker gaps above this value are joined with ` ... ` |
|
|
||||||
| `AUDITA_NORMALIZE_MAX_SEGMENT_DURATION` | `--normalize-max-segment-duration` | `60.0` | Maximum merged segment duration |
|
|
||||||
| `AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS` | `--normalize-max-segment-tokens` | `2048` | Maximum merged segment prompt payload size |
|
|
||||||
| `AUDITA_WORK_DIR` | `--work-dir` | `/tmp/audita` | Per-run scratch diagnostics directory |
|
|
||||||
| `AUDITA_WORK_DIR_RETENTION` | `--work-dir-retention` | `auto` | Whether to retain the per-run work directory: `auto`, `always`, or `never` |
|
|
||||||
|
|
||||||
Set `AUDITA_MODULES=grammar` to run only the grammar module by default, or override it per command with `--modules`.
|
|
||||||
|
|
||||||
Validation-phase LLM settings inherit from the primary `AUDITA_*` LLM settings by default. Set any of the `AUDITA_VALIDATION_*` values only when you want LLM-backed validators to use a different model, endpoint, credential, timeout, retry budget, or concurrency level.
|
|
||||||
|
|
||||||
OpenRouter remains the default out of the box:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
export AUDITA_LLM_API_KEY=your-openrouter-key
|
|
||||||
audita process transcript.json --glossary glossary.yaml --output corrected.json
|
|
||||||
```
|
|
||||||
|
|
||||||
You can point Audita at any OpenAI-compatible endpoint by changing `AUDITA_BASE_URL` and, if needed, `AUDITA_MODEL`. For example, a local vLLM server:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
export AUDITA_LLM_API_KEY=local-dev-key
|
|
||||||
export AUDITA_BASE_URL=http://localhost:8000/v1
|
|
||||||
export AUDITA_MODEL=meta-llama/Llama-3.1-8B-Instruct
|
|
||||||
audita process transcript.json --glossary glossary.yaml --output corrected.json
|
|
||||||
```
|
|
||||||
|
|
||||||
Or the actual OpenAI API:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
export AUDITA_LLM_API_KEY=your-openai-key
|
|
||||||
export AUDITA_BASE_URL=https://api.openai.com/v1
|
|
||||||
export AUDITA_MODEL=gpt-4.1-mini
|
|
||||||
audita process transcript.json --glossary glossary.yaml --output corrected.json
|
|
||||||
```
|
|
||||||
|
|
||||||
`AUDITA_WORK_DIR` stores per-run diagnostics while processing. Under the default `AUDITA_WORK_DIR_RETENTION=auto`, clean successful runs are removed, while failed runs and successful runs with final skipped corrections are preserved. Use `always` to keep every run directory and `never` to remove successful run directories even when skips remain.
|
|
||||||
Failed runs always preserve the run directory and include an authoritative `report.json` alongside normalization and prompt/response diagnostics.
|
|
||||||
|
|
||||||
## Prototype Archive
|
|
||||||
|
|
||||||
The archived prototype remains importable as `audita_prototype` and is still covered by its original regression suite. This is intentional: the new `audita` package is a framework-oriented rewrite, not a thin wrapper around the old code.
|
|
||||||
|
|||||||
@@ -1,339 +1,66 @@
|
|||||||
# Narratio -> Scriptorium CLI Integration
|
# Integration: Scriptorium
|
||||||
|
|
||||||
## 1. Purpose
|
## Purpose
|
||||||
|
Define the Scriptorium adapter contract used by `analyze` and trim-bounds generation in `trim`.
|
||||||
|
|
||||||
This document defines how Narratio should invoke Scriptorium through the **public CLI**.
|
## Adapter Boundary
|
||||||
|
Interface:
|
||||||
|
- `scriptorium.Runner`
|
||||||
|
- methods:
|
||||||
|
- `RunArtifact(ctx, RunArtifactRequest)`
|
||||||
|
- `RenderArtifact(ctx, RenderArtifactRequest)`
|
||||||
|
|
||||||
This is a **subprocess integration contract**, not an internal Go API contract.
|
Primary implementation:
|
||||||
|
- `internal/adapters/scriptorium/SubprocessRunner`
|
||||||
## 2. Assumptions
|
|
||||||
|
|
||||||
- `scriptorium` is installed and available on `PATH`.
|
|
||||||
- Scriptorium is configured with `config.yml`.
|
|
||||||
- `config.yml` provides `prompt_dir`, `profile_dir`, and `schema_dir` as needed.
|
|
||||||
- Prompt and profile libraries are already deployed for the environment.
|
|
||||||
- Narratio provides prepared artifact files (for example polished transcript, glossary, previous recap, campaign notes).
|
|
||||||
- Initial integration is synchronous subprocess execution.
|
|
||||||
- Narratio remains the orchestrator.
|
|
||||||
|
|
||||||
In normal operation, Narratio does not need to pass `--prompt-dir` and `--profile-dir` if they are supplied by Scriptorium config.
|
|
||||||
|
|
||||||
Narratio may pass `--config <PATH>` when it must use a non-default Scriptorium config file.
|
|
||||||
|
|
||||||
## 3. Core Commands Narratio May Call
|
|
||||||
|
|
||||||
Primary commands for subprocess integration:
|
|
||||||
|
|
||||||
|
Execution modes:
|
||||||
- `scriptorium run`
|
- `scriptorium run`
|
||||||
- `scriptorium render`
|
- `scriptorium render`
|
||||||
|
|
||||||
For production generation, use `scriptorium run`.
|
## Request Contract
|
||||||
|
Both request types carry:
|
||||||
`scriptorium render` is for debugging, dry-runs, test assertions, and validating command construction without LLM execution.
|
- binary/config/prompt/profile IDs;
|
||||||
|
- input map and vars map;
|
||||||
Note: `scriptorium serve` and HTTP API exist, but they are not the initial integration path.
|
- output path;
|
||||||
|
- timeout;
|
||||||
## 4. Command Selection Guidance
|
- generated config + stdout/stderr log paths;
|
||||||
|
- optional API-key env var name;
|
||||||
- Use `run` to generate an output artifact.
|
- optional working directory.
|
||||||
- Use `render` to inspect the prepared prompt and effective settings without calling the LLM.
|
|
||||||
- Use `render --format json` when Narratio/tests need structured prepare output.
|
## Result Contract
|
||||||
|
`ArtifactResult` returns:
|
||||||
## 5. Recommended `run` Invocation Shape
|
- output/log/generated-config paths;
|
||||||
|
- exit code and duration;
|
||||||
Production shape:
|
- command mode (`run` or `render`);
|
||||||
|
- prompt/profile provenance;
|
||||||
```bash
|
- `ValidationFailed` marker;
|
||||||
scriptorium run \
|
- metadata map.
|
||||||
--prompt <prompt_id> \
|
|
||||||
--input transcript=<processed-transcript-path> \
|
## Validation and Failure Semantics
|
||||||
--out <output-artifact-path>
|
Request validation fails for:
|
||||||
```
|
- missing binary, prompt id, or output path;
|
||||||
|
- non-positive timeout;
|
||||||
Common optional additions:
|
- empty input/var names;
|
||||||
|
- empty input path values;
|
||||||
- `--config <path>`: use a specific Scriptorium config file.
|
- missing required credential env var when `APIKeyEnv` is set.
|
||||||
- `--profile <profile_id>`: override prompt default profile.
|
|
||||||
- `--var name=value` (repeatable): small metadata values.
|
Run behavior:
|
||||||
- `--input name=path` (repeatable): additional named artifacts.
|
- subprocess errors propagate with context;
|
||||||
- `--timeout <duration>`: per-run timeout override.
|
- `run` exit code `2` is mapped to `ValidationFailed=true`;
|
||||||
- Runtime model override flags (`--llm-base-url`, `--model`, etc.) only for exceptional/operator-directed cases.
|
- successful subprocess still fails if output file is missing or empty.
|
||||||
|
|
||||||
## 6. Recommended `render` Invocation Shape
|
Render behavior:
|
||||||
|
- subprocess errors propagate;
|
||||||
Human-readable debug shape:
|
- output file must exist and be non-empty.
|
||||||
|
|
||||||
```bash
|
## Deterministic Behavior
|
||||||
scriptorium render \
|
- input and var maps are sorted into deterministic `--input` and `--var` CLI args.
|
||||||
--prompt <prompt_id> \
|
- generated invocation YAML (`scriptorium.generated.v1`) is emitted when requested.
|
||||||
--input transcript=<processed-transcript-path> \
|
- adapter is stateless and does not own artifact-selection policy.
|
||||||
--format text
|
|
||||||
```
|
## Config Mapping
|
||||||
|
Config fields consumed through runner/stage wiring are under `pipeline.scriptorium.*` plus per-artifact settings under `pipeline.scriptorium.artifacts.*`.
|
||||||
Structured debug/test shape:
|
|
||||||
|
Maintained examples with Scriptorium config:
|
||||||
```bash
|
- `examples/pipeline.full.annotated.yml`
|
||||||
scriptorium render \
|
- `examples/pipeline.production.yml`
|
||||||
--prompt <prompt_id> \
|
|
||||||
--input transcript=<processed-transcript-path> \
|
|
||||||
--format json \
|
|
||||||
--out <render-debug-path>
|
|
||||||
```
|
|
||||||
|
|
||||||
`render` does **not** call the LLM, does **not** validate model output, and does **not** perform repair.
|
|
||||||
|
|
||||||
## 7. Inputs
|
|
||||||
|
|
||||||
- Pass inputs as repeated `--input name=path` flags.
|
|
||||||
- `name` must match the Prompt Definition input name.
|
|
||||||
- Prefer absolute paths, or paths relative to a working directory controlled by Narratio.
|
|
||||||
- Pass Audita output as the primary transcript input.
|
|
||||||
- Additional inputs may include glossary, previous recap, campaign notes, event logs, final state maps, or other prompt-specific artifacts.
|
|
||||||
- Scriptorium reads input files directly; Narratio does not need to inline file content for CLI use.
|
|
||||||
|
|
||||||
## 8. Variables
|
|
||||||
|
|
||||||
Use repeated `--var name=value` for small metadata values.
|
|
||||||
|
|
||||||
Typical examples:
|
|
||||||
|
|
||||||
- `session_date`
|
|
||||||
- `session_id`
|
|
||||||
- `campaign_name`
|
|
||||||
- `previous_session_id`
|
|
||||||
- `output_kind`
|
|
||||||
|
|
||||||
Large content belongs in input files, not `--var` values.
|
|
||||||
|
|
||||||
## 9. Prompt IDs and Output Artifact Types
|
|
||||||
|
|
||||||
Narratio should treat prompt IDs as configuration, not hardcoded business logic.
|
|
||||||
|
|
||||||
Narratio config may map stage/output names to prompt IDs, for example:
|
|
||||||
|
|
||||||
- session recap prompt
|
|
||||||
- structured event extraction prompt
|
|
||||||
- glossary suggestion prompt
|
|
||||||
- player-facing summary prompt
|
|
||||||
|
|
||||||
Prompt IDs used by Narratio should come from the deployed Scriptorium prompt library.
|
|
||||||
|
|
||||||
## 10. Profiles
|
|
||||||
|
|
||||||
- Prompts may declare `default_profile`.
|
|
||||||
- Narratio may omit `--profile` to use prompt default profile.
|
|
||||||
- Narratio may pass `--profile` to force profile selection.
|
|
||||||
- This enables environment/profile selection like `local-fast`, `local-quality`, `frontier`, `batch`, or test profiles.
|
|
||||||
- Profile names should generally be Narratio configuration values.
|
|
||||||
|
|
||||||
## 11. Runtime Overrides
|
|
||||||
|
|
||||||
Supported runtime override flags:
|
|
||||||
|
|
||||||
- `--llm-base-url`
|
|
||||||
- `--model`
|
|
||||||
- `--api-key-env`
|
|
||||||
- `--temperature`
|
|
||||||
- `--max-tokens`
|
|
||||||
- `--top-p`
|
|
||||||
- `--timeout`
|
|
||||||
|
|
||||||
Guidance:
|
|
||||||
|
|
||||||
- Keep normal model/runtime settings in Execution Profiles.
|
|
||||||
- Use runtime overrides only for explicit per-run exceptions, tests, or operator overrides.
|
|
||||||
- Never pass raw API keys on the command line.
|
|
||||||
- `--api-key-env` names an environment variable; Narratio must ensure that variable is set in subprocess environment.
|
|
||||||
|
|
||||||
## 12. Config Behavior
|
|
||||||
|
|
||||||
- Default config path: `/etc/scriptorium/config.yml`.
|
|
||||||
- `--config <PATH>` overrides default path.
|
|
||||||
- Missing default config is allowed by Scriptorium.
|
|
||||||
- If `--config` is provided explicitly, the file must exist and be valid.
|
|
||||||
- CLI flags override `config.yml`.
|
|
||||||
- `config.yml` overrides built-in application defaults.
|
|
||||||
|
|
||||||
Narratio can either:
|
|
||||||
|
|
||||||
- rely on system default config path, or
|
|
||||||
- carry an explicit config path and pass `--config`.
|
|
||||||
|
|
||||||
## 13. Environment Handling
|
|
||||||
|
|
||||||
Subprocess environment recommendations:
|
|
||||||
|
|
||||||
- Pass through required API-key environment variables referenced by `api_key_env`.
|
|
||||||
- Do not pass raw API keys as CLI arguments.
|
|
||||||
- Avoid logging full environment dumps.
|
|
||||||
- Capture stdout and stderr separately.
|
|
||||||
- Use a controlled working directory.
|
|
||||||
- Prefer absolute artifact paths.
|
|
||||||
|
|
||||||
## 14. Output Handling
|
|
||||||
|
|
||||||
For `scriptorium run`:
|
|
||||||
|
|
||||||
- Use `--out` when Narratio needs durable artifact files.
|
|
||||||
- Without `--out`, artifact content is written to stdout.
|
|
||||||
- Preferred orchestration pattern: always use `--out`, then treat the file as stage output artifact.
|
|
||||||
- Capture stderr for diagnostics.
|
|
||||||
|
|
||||||
For `scriptorium render`:
|
|
||||||
|
|
||||||
- Use `--out` to store render diagnostics.
|
|
||||||
- Use `--format json` when tests need to inspect selected profile, effective runtime settings, input hashes, prompt hash, and rendered messages.
|
|
||||||
|
|
||||||
## 15. Exit Status and Errors
|
|
||||||
|
|
||||||
Current CLI behavior (verified from implementation/tests):
|
|
||||||
|
|
||||||
- `0`: success.
|
|
||||||
- `1`: runtime/parse/config/load/render/generation/IO error.
|
|
||||||
- `2`: run completed but output validation failed (`ValidationFailed`).
|
|
||||||
|
|
||||||
Additional details:
|
|
||||||
|
|
||||||
- On `run`, output artifact write happens before exit code selection. If validation fails, artifact may still be written and exit code is `2`.
|
|
||||||
- `stderr` carries both errors and normal run summary output; non-empty stderr alone does not imply failure.
|
|
||||||
- `render` returns `0` on success and `1` on failures.
|
|
||||||
|
|
||||||
Narratio should treat non-zero exit codes as failed stage execution, but may record generated artifact paths if a run exited `2` and output file exists.
|
|
||||||
|
|
||||||
## 16. Recommended Narratio Integration Pattern
|
|
||||||
|
|
||||||
1. Build CLI args from Narratio stage configuration.
|
|
||||||
2. Use subprocess context cancellation/timeout.
|
|
||||||
3. Pass absolute input paths.
|
|
||||||
4. Pass `--out` to a session-scoped artifact path.
|
|
||||||
5. Add `--var` metadata values.
|
|
||||||
6. Optionally add `--config`.
|
|
||||||
7. Optionally add `--profile`.
|
|
||||||
8. Ensure required API-key env vars are present.
|
|
||||||
9. Run subprocess synchronously.
|
|
||||||
10. Capture stdout/stderr separately.
|
|
||||||
11. On success, store output artifact path and invocation metadata in stage artifacts.
|
|
||||||
12. On failure, store exit code and stderr diagnostics in stage status.
|
|
||||||
|
|
||||||
## 17. Suggested Narratio Configuration Shape
|
|
||||||
|
|
||||||
Illustrative `pipeline.yml` shape:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
scriptorium:
|
|
||||||
binary: scriptorium
|
|
||||||
config_path: /etc/scriptorium/config.yml
|
|
||||||
timeout: 10m
|
|
||||||
render_debug: false
|
|
||||||
artifacts:
|
|
||||||
session_recap:
|
|
||||||
enabled: true
|
|
||||||
prompt_id: dnd.session_recap
|
|
||||||
profile_id: local-quality # optional
|
|
||||||
output_path: artifacts/session_recap.md
|
|
||||||
timeout: 10m
|
|
||||||
render_debug: false # optional artifact override
|
|
||||||
inputs:
|
|
||||||
transcript:
|
|
||||||
source: trimmed_transcript
|
|
||||||
required: true
|
|
||||||
previous_recap:
|
|
||||||
source: previous_session_artifact
|
|
||||||
artifact: session_recap
|
|
||||||
path: "" # optional
|
|
||||||
required: false
|
|
||||||
vars:
|
|
||||||
session_id: true
|
|
||||||
session_date: true
|
|
||||||
campaign_name: true
|
|
||||||
previous_session_id: true
|
|
||||||
output_kind: session_recap
|
|
||||||
```
|
|
||||||
|
|
||||||
The key idea: map Narratio artifact names to prompt ID, optional profile, expected inputs, vars, and output destination.
|
|
||||||
|
|
||||||
## 18. Testing Strategy for Narratio Integration
|
|
||||||
|
|
||||||
- Use `scriptorium render --format json` to verify command construction without LLM calls.
|
|
||||||
- Use dedicated test prompt/profile libraries for integration tests.
|
|
||||||
- Use small fixture transcripts.
|
|
||||||
- Verify missing-input failure behavior.
|
|
||||||
- Verify prompt `default_profile` behavior.
|
|
||||||
- Verify explicit `--profile` override behavior.
|
|
||||||
- Verify `--config` behavior (default and explicit).
|
|
||||||
- Verify output file creation when `--out` is used.
|
|
||||||
- Verify stderr capture on failures.
|
|
||||||
- Avoid real API keys in tests.
|
|
||||||
|
|
||||||
## 19. Security and Privacy Notes
|
|
||||||
|
|
||||||
- Never pass raw API keys on command line.
|
|
||||||
- Do not log full rendered prompts by default; transcripts may contain sensitive content.
|
|
||||||
- Avoid logging prompt content unless explicit debug mode is enabled.
|
|
||||||
- Treat generated artifacts as potentially sensitive.
|
|
||||||
- Use session-scoped, access-controlled output paths.
|
|
||||||
- `api_key_env` names should come from environment management, not embedded secrets.
|
|
||||||
|
|
||||||
## 20. Initial D&D Artifact Generation Examples
|
|
||||||
|
|
||||||
These are examples only. Use prompt IDs from the deployed prompt library.
|
|
||||||
|
|
||||||
Session recap:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
scriptorium run \
|
|
||||||
--prompt dnd.session_recap \
|
|
||||||
--input transcript=/work/session-42/transcript.polished.md \
|
|
||||||
--input glossary=/work/session-42/glossary.yml \
|
|
||||||
--out /work/session-42/artifacts/session_recap.md
|
|
||||||
```
|
|
||||||
|
|
||||||
Structured events:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
scriptorium run \
|
|
||||||
--prompt dnd.structured_events \
|
|
||||||
--input transcript=/work/session-42/transcript.polished.md \
|
|
||||||
--out /work/session-42/artifacts/structured_events.json
|
|
||||||
```
|
|
||||||
|
|
||||||
Glossary suggestions:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
scriptorium run \
|
|
||||||
--prompt dnd.glossary_suggestions \
|
|
||||||
--input transcript=/work/session-42/transcript.polished.md \
|
|
||||||
--input previous_recap=/work/session-41/artifacts/session_recap.md \
|
|
||||||
--out /work/session-42/artifacts/glossary_suggestions.md
|
|
||||||
```
|
|
||||||
|
|
||||||
Player-facing summary:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
scriptorium run \
|
|
||||||
--prompt dnd.player_summary \
|
|
||||||
--input transcript=/work/session-42/transcript.polished.md \
|
|
||||||
--input structured_events=/work/session-42/artifacts/structured_events.json \
|
|
||||||
--out /work/session-42/artifacts/player_summary.md
|
|
||||||
```
|
|
||||||
|
|
||||||
## 21. Non-Goals
|
|
||||||
|
|
||||||
Initial Narratio integration should not:
|
|
||||||
|
|
||||||
- call Scriptorium internal Go packages
|
|
||||||
- use HTTP API as the primary path
|
|
||||||
- expect Scriptorium to read S3 refs directly
|
|
||||||
- make Scriptorium responsible for Narratio stage state
|
|
||||||
- make Scriptorium responsible for notification
|
|
||||||
- require Scriptorium to understand D&D workflow semantics beyond prompt definitions
|
|
||||||
|
|
||||||
## 22. Future Extension Notes
|
|
||||||
|
|
||||||
Possible later extensions:
|
|
||||||
|
|
||||||
- HTTP API integration
|
|
||||||
- S3 artifact references if Scriptorium adds S3 reader support
|
|
||||||
- richer render diagnostics and policy controls
|
|
||||||
- token budgeting/prompt-size checks
|
|
||||||
- batch execution if Scriptorium later adds batch support
|
|
||||||
|
|||||||
@@ -1,403 +1,61 @@
|
|||||||
# seriatim
|
# Integration: Seriatim
|
||||||
|
|
||||||
`seriatim` merges per-speaker WhisperX-style JSON transcripts into a single JSON transcript that preserves speaker identity and chronological order.
|
## Purpose
|
||||||
|
Define the Seriatim adapter contract used by `merge`, `normalize`, `trim`, and `render`.
|
||||||
The current implementation supports the `merge` command. It reads one or more input JSON files, optionally maps each input file to a canonical speaker using `speakers.yml`, sorts all segments by timestamp, detects and resolves overlaps when word-level timing is available, assigns consecutive numeric `id` values, and writes a merged JSON artifact.
|
|
||||||
|
## Adapter Boundary
|
||||||
## Usage
|
Interface:
|
||||||
|
- `seriatim.Runner`
|
||||||
Run from source:
|
- methods:
|
||||||
|
- `Run(ctx, MergeRequest)`
|
||||||
```sh
|
- `Normalize(ctx, NormalizeRequest)`
|
||||||
go run ./cmd/seriatim merge \
|
- `Trim(ctx, TrimRequest)`
|
||||||
--input-file samples/raw/2026-04-19-Eric_Rakestraw.json \
|
- `Render(ctx, RenderRequest)`
|
||||||
--input-file samples/raw/2026-04-19-Mike_Brown.json \
|
|
||||||
--output-file merged.json
|
Primary implementation:
|
||||||
```
|
- `internal/adapters/seriatim/SubprocessRunner`
|
||||||
|
|
||||||
Optional report output:
|
Execution modes:
|
||||||
|
- `seriatim merge`
|
||||||
```sh
|
- `seriatim normalize`
|
||||||
go run ./cmd/seriatim merge \
|
- `seriatim trim`
|
||||||
--input-file eric.json \
|
- `seriatim render`
|
||||||
--input-file mike.json \
|
|
||||||
--output-file merged.json \
|
## Request/Result Contracts
|
||||||
--report-file report.json
|
- `MergeRequest`/`MergeResult`: multi-input merge to base transcript, optional report.
|
||||||
```
|
- `NormalizeRequest`/`NormalizeResult`: transcript normalization with explicit schema.
|
||||||
|
- `TrimRequest`/`TrimResult`: transcript trimming with required keep selector.
|
||||||
## CLI
|
- `RenderRequest`/`RenderResult`: transcript-to-markdown rendering with explicit format and render booleans.
|
||||||
|
|
||||||
```text
|
Results include output/log/config paths, timing, exit code, and metadata.
|
||||||
seriatim merge [flags]
|
|
||||||
```
|
## Validation and Failure Semantics
|
||||||
|
Runner construction validates:
|
||||||
Global flags:
|
- binary presence;
|
||||||
|
- timeout > 0;
|
||||||
| Flag | Description |
|
- supported output schema (`seriatim-minimal|seriatim-intermediate|seriatim-full`);
|
||||||
| --- | --- |
|
- non-negative coalesce gap.
|
||||||
| `--help` | Show command help. |
|
|
||||||
| `--version` | Show application version. Local builds default to `dev`; release builds inject the release version. |
|
Invocation fails on:
|
||||||
|
- missing required request paths/inputs;
|
||||||
`merge` flags:
|
- invalid normalize schema override;
|
||||||
|
- unsupported render format;
|
||||||
| Flag | Required | Default | Description |
|
- subprocess failure;
|
||||||
| --- | --- | --- | --- |
|
- invalid JSON outputs for merge/normalize/trim;
|
||||||
| `--input-file` | Yes | none | Input transcript JSON file. Repeat once per speaker/input file. |
|
- missing `segments` array for normalize/trim transcript outputs;
|
||||||
| `--output-file` | Yes | none | Merged transcript JSON output path. |
|
- empty render output files.
|
||||||
| `--report-file` | No | none | Optional report JSON output path. |
|
|
||||||
| `--speakers` | No | none | Speaker map YAML file. When omitted, input file basenames are used as speaker labels. |
|
When report paths are provided/enabled, report files must parse as JSON.
|
||||||
| `--autocorrect` | No | none | Autocorrect rules YAML file. When omitted, the default `autocorrect` module leaves text unchanged. |
|
|
||||||
| `--input-reader` | No | `json-files` | Input reader module. |
|
## Deterministic Behavior
|
||||||
| `--output-modules` | No | `json` | Comma-separated output modules. |
|
- argument ordering is deterministic per command construction.
|
||||||
| `--output-schema` | No | `seriatim-intermediate` | JSON output contract. Allowed values are `seriatim-minimal`, `seriatim-intermediate`, and `seriatim-full`. If omitted, the runtime default is used; consumers that depend on a specific shape should set this explicitly. |
|
- merge env overrides are explicit (`SERIATIM_*`) and only emitted when configured.
|
||||||
| `--preprocessing-modules` | No | `validate-raw,normalize-speakers,trim-text` | Comma-separated preprocessing modules, evaluated in order. |
|
- generated invocation YAML (`seriatim.generated.v1`) is emitted when requested.
|
||||||
| `--postprocessing-modules` | No | `detect-overlaps,resolve-overlaps,backchannel,filler,resolve-danglers,coalesce,detect-overlaps,autocorrect,assign-ids,validate-output` | Comma-separated postprocessing modules, evaluated in order. |
|
- adapter does not write manifests or choose stage inputs.
|
||||||
| `--coalesce-gap` | No | `3.0` | Maximum same-speaker gap in seconds for `coalesce`; also used as the `resolve-overlaps` context window. Must be a non-negative float. |
|
|
||||||
|
## Config Mapping
|
||||||
Environment variables:
|
Config fields consumed through runner/stage wiring are under `pipeline.seriatim.*` and `pipeline.render.*`.
|
||||||
|
|
||||||
| Environment Variable | Default | Description |
|
Maintained examples with Seriatim config:
|
||||||
| --- | --- | --- |
|
- `examples/pipeline.full.annotated.yml`
|
||||||
| `SERIATIM_OUTPUT_SCHEMA` | `seriatim-intermediate` | Output schema used when `--output-schema` is not explicitly provided. Allowed values are `seriatim-minimal`, `seriatim-intermediate`, and `seriatim-full`. The CLI flag takes precedence. |
|
- `examples/pipeline.production.yml`
|
||||||
| `SERIATIM_OVERLAP_WORD_RUN_GAP` | `1.0` | Maximum gap in seconds between adjacent timed words when `resolve-overlaps` builds word-run replacement segments. Must be a positive float. |
|
|
||||||
| `SERIATIM_OVERLAP_WORD_RUN_REORDER_WINDOW` | `1.0` | Near-start window in seconds for ordering replacement word runs shortest-first. Must be a positive float. |
|
|
||||||
| `SERIATIM_BACKCHANNEL_MAX_DURATION` | `2.0` | Maximum duration in seconds for `backchannel` classification. Must be a positive float. |
|
|
||||||
| `SERIATIM_FILLER_MAX_DURATION` | `1.25` | Maximum duration in seconds for `filler` classification. Must be a positive float. |
|
|
||||||
|
|
||||||
## Input JSON Format
|
|
||||||
|
|
||||||
Each input file must be valid JSON with a top-level `segments` array. The current parser accepts the WhisperX segment subset needed for merging:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"segments": [
|
|
||||||
{
|
|
||||||
"start": 1.25,
|
|
||||||
"end": 3.5,
|
|
||||||
"text": "Hello there.",
|
|
||||||
"words": [
|
|
||||||
{"word": "Hello", "start": 1.25, "end": 1.55, "score": 0.98},
|
|
||||||
{"word": "there.", "start": 1.7, "end": 2.0}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Required segment fields:
|
|
||||||
|
|
||||||
- `start`: number, must be `>= 0`.
|
|
||||||
- `end`: number, must be `>= start`.
|
|
||||||
- `text`: string.
|
|
||||||
|
|
||||||
Optional word fields:
|
|
||||||
|
|
||||||
- `words`: array of word timing objects.
|
|
||||||
- `words[].word`: string.
|
|
||||||
- `words[].start`: optional number, must be `>= 0` when present.
|
|
||||||
- `words[].end`: optional number, must be `>= start` when present with `start`.
|
|
||||||
- `words[].score`: optional number.
|
|
||||||
- `words[].speaker`: optional raw speaker label string.
|
|
||||||
|
|
||||||
Word-level timing is preserved internally for overlap resolution. If a word is missing `start` or `end`, seriatim keeps the word text, emits a warning in the optional report, and does not use that word as a timing anchor. Word timing is not emitted in the final JSON artifact.
|
|
||||||
|
|
||||||
## Speaker Map Format
|
|
||||||
|
|
||||||
`speakers.yml` maps input files to canonical speaker names using ordered substring rules:
|
|
||||||
|
|
||||||
This file is optional. If `--speakers` is omitted, `seriatim` uses each input file basename as the segment speaker label.
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
match:
|
|
||||||
- speaker: "Eric Rakestraw"
|
|
||||||
match:
|
|
||||||
- "Eric_Rakestraw"
|
|
||||||
- "Eric"
|
|
||||||
|
|
||||||
- speaker: "Mike Brown"
|
|
||||||
match:
|
|
||||||
- "Mike_Brown"
|
|
||||||
- "mb"
|
|
||||||
```
|
|
||||||
|
|
||||||
For each `--input-file`, `seriatim` takes the file basename and evaluates the rules in order. The first rule with a matching substring wins, and no later rules are evaluated.
|
|
||||||
|
|
||||||
For example, this input:
|
|
||||||
|
|
||||||
```text
|
|
||||||
samples/raw/2026-04-19-Eric_Rakestraw.json
|
|
||||||
```
|
|
||||||
|
|
||||||
matches this rule because the basename contains `Eric_Rakestraw`:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
- speaker: "Eric Rakestraw"
|
|
||||||
match:
|
|
||||||
- "Eric_Rakestraw"
|
|
||||||
```
|
|
||||||
|
|
||||||
Important details:
|
|
||||||
|
|
||||||
- Matching is against the input file basename, not the full path.
|
|
||||||
- Matching is case-insensitive.
|
|
||||||
- Rules are evaluated from first to last.
|
|
||||||
- Each rule must have a non-empty `speaker`.
|
|
||||||
- Each rule must have at least one non-empty `match` string.
|
|
||||||
- Duplicate speaker names are invalid.
|
|
||||||
- Every input file must match at least one rule or the command fails.
|
|
||||||
|
|
||||||
Deprecated old format:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
inputs:
|
|
||||||
eric.json:
|
|
||||||
speaker: "Eric Rakestraw"
|
|
||||||
```
|
|
||||||
|
|
||||||
The old `inputs:` direct mapping format is no longer supported.
|
|
||||||
|
|
||||||
## Output JSON Format
|
|
||||||
|
|
||||||
`--output-modules json` controls the writer. `--output-schema` controls the JSON contract that writer serializes.
|
|
||||||
|
|
||||||
The named schemas are stable public contracts. If a consumer depends on a specific shape, it should request that schema explicitly at runtime. The runtime default selection may change in a future release.
|
|
||||||
|
|
||||||
The `seriatim-intermediate` schema is the current default selection when neither `--output-schema` nor `SERIATIM_OUTPUT_SCHEMA` is set. It stays close to the minimal schema, but adds optional `categories` on each segment:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"metadata": {
|
|
||||||
"application": "seriatim",
|
|
||||||
"version": "dev",
|
|
||||||
"output_schema": "seriatim-intermediate"
|
|
||||||
},
|
|
||||||
"segments": [
|
|
||||||
{
|
|
||||||
"id": 1,
|
|
||||||
"start": 1.25,
|
|
||||||
"end": 3.5,
|
|
||||||
"speaker": "Eric Rakestraw",
|
|
||||||
"text": "Hello there.",
|
|
||||||
"categories": ["backchannel"]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
The `seriatim-full` schema uses the full seriatim envelope:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"metadata": {
|
|
||||||
"application": "seriatim",
|
|
||||||
"version": "dev",
|
|
||||||
"input_reader": "json-files",
|
|
||||||
"input_files": ["eric.json", "mike.json"],
|
|
||||||
"preprocessing_modules": ["validate-raw", "normalize-speakers", "trim-text"],
|
|
||||||
"postprocessing_modules": ["detect-overlaps", "resolve-overlaps", "backchannel", "filler", "resolve-danglers", "coalesce", "detect-overlaps", "autocorrect", "assign-ids", "validate-output"],
|
|
||||||
"output_modules": ["json"]
|
|
||||||
},
|
|
||||||
"segments": [
|
|
||||||
{
|
|
||||||
"id": 1,
|
|
||||||
"source": "eric.json",
|
|
||||||
"source_segment_index": 0,
|
|
||||||
"speaker": "Eric Rakestraw",
|
|
||||||
"start": 1.25,
|
|
||||||
"end": 3.5,
|
|
||||||
"text": "Hello there.",
|
|
||||||
"overlap_group_id": 1
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": 2,
|
|
||||||
"source": "eric.json",
|
|
||||||
"source_ref": "word-run:1:1:1",
|
|
||||||
"derived_from": ["eric.json#0"],
|
|
||||||
"speaker": "Eric Rakestraw",
|
|
||||||
"start": 2.0,
|
|
||||||
"end": 2.5,
|
|
||||||
"text": "Resolved word run",
|
|
||||||
"categories": ["backchannel"]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"overlap_groups": [
|
|
||||||
{
|
|
||||||
"id": 1,
|
|
||||||
"start": 1.25,
|
|
||||||
"end": 4.0,
|
|
||||||
"segments": ["eric.json#0", "mike.json#0"],
|
|
||||||
"speakers": ["Eric Rakestraw", "Mike Brown"],
|
|
||||||
"class": "unknown",
|
|
||||||
"resolution": "unresolved"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
The `seriatim-minimal` schema emits minimal metadata and compact ordered segments:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"metadata": {
|
|
||||||
"application": "seriatim",
|
|
||||||
"version": "dev",
|
|
||||||
"output_schema": "seriatim-minimal"
|
|
||||||
},
|
|
||||||
"segments": [
|
|
||||||
{
|
|
||||||
"id": 1,
|
|
||||||
"start": 1.25,
|
|
||||||
"end": 3.5,
|
|
||||||
"speaker": "Eric Rakestraw",
|
|
||||||
"text": "Hello there."
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Minimal output intentionally omits categories, overlap groups, source/provenance fields, and pipeline configuration metadata.
|
|
||||||
|
|
||||||
Intermediate output intentionally omits overlap groups and source/provenance fields, but keeps optional `categories` and minimal metadata.
|
|
||||||
|
|
||||||
Segments are sorted deterministically by:
|
|
||||||
|
|
||||||
```text
|
|
||||||
(start, end, source, source_segment_index/source_ref, speaker)
|
|
||||||
```
|
|
||||||
|
|
||||||
Final segment IDs are assigned after sorting and start at `1`.
|
|
||||||
|
|
||||||
The public Go output contract is available from:
|
|
||||||
|
|
||||||
```go
|
|
||||||
import "gitea.maximumdirect.net/eric/seriatim/schema"
|
|
||||||
```
|
|
||||||
|
|
||||||
The same package embeds machine-readable JSON Schemas in `schema/full-output.schema.json`, `schema/intermediate-output.schema.json`, and `schema/minimal-output.schema.json`. The default `validate-output` postprocessor validates the selected output shape and verifies final segment IDs are present, sequential, and start at `1`.
|
|
||||||
|
|
||||||
## Overlap Detection
|
|
||||||
|
|
||||||
The default postprocessing pipeline detects overlapping segment groups.
|
|
||||||
|
|
||||||
Overlap behavior:
|
|
||||||
|
|
||||||
- A strict timing overlap is required: `next.start < current_group_end`.
|
|
||||||
- Segments that only touch at a boundary are not grouped.
|
|
||||||
- Groups require at least two distinct speakers.
|
|
||||||
- Transitive overlaps are grouped together.
|
|
||||||
- Segments in detected groups receive `overlap_group_id`.
|
|
||||||
- `overlap_groups[].segments` contains stable references in `source#source_segment_index` format.
|
|
||||||
- `class` is currently `unknown`.
|
|
||||||
- `resolution` is `unresolved` until `resolve-overlaps` replaces the group.
|
|
||||||
|
|
||||||
## Overlap Resolution
|
|
||||||
|
|
||||||
The default postprocessing pipeline runs `detect-overlaps`, then `resolve-overlaps`, then `backchannel`, then `filler`, then `resolve-danglers`, then `coalesce`, then a second `detect-overlaps` pass.
|
|
||||||
|
|
||||||
For each detected overlap group, `resolve-overlaps` uses preserved WhisperX word timing to build smaller word-run replacement segments:
|
|
||||||
|
|
||||||
- The resolution window expands the detected overlap group by `--coalesce-gap` seconds on both sides.
|
|
||||||
- Nearby same-speaker context segments are included when they intersect the expanded window and their start or end is within `--coalesce-gap` of the original overlap boundary.
|
|
||||||
- Once a segment is selected for replacement, all timed words from that segment participate in word-run construction; the window controls segment selection, not per-word clipping.
|
|
||||||
- Context segments that are part of another detected overlap group are not pulled into the current group.
|
|
||||||
- Untimed words are included in replacement text in original word order when nearby timed words create a replacement run.
|
|
||||||
- Untimed words do not affect replacement segment start/end times or word-run gap splitting.
|
|
||||||
- Words for the same speaker are merged into one run when the gap between adjacent words is no greater than `SERIATIM_OVERLAP_WORD_RUN_GAP`.
|
|
||||||
- The default word-run gap is `1.0` seconds.
|
|
||||||
- Set `SERIATIM_OVERLAP_WORD_RUN_GAP` to a positive number of seconds to override the default.
|
|
||||||
- Near-start replacement word runs are reordered so shorter segments come first when adjacent starts are within `SERIATIM_OVERLAP_WORD_RUN_REORDER_WINDOW`.
|
|
||||||
- The default word-run reorder window is `1.0` seconds.
|
|
||||||
- Set `SERIATIM_OVERLAP_WORD_RUN_REORDER_WINDOW` to a positive number of seconds to override the default.
|
|
||||||
- Replacement segment text is built by joining word text with single spaces.
|
|
||||||
- Replacement segments include `source_ref` and `derived_from`.
|
|
||||||
- Replacement segments omit `source_segment_index` because they are derived from one or more original segments.
|
|
||||||
- Resolved overlap groups are removed before the second detection pass.
|
|
||||||
- Replacement segments are left without `overlap_group_id` until the second detection pass annotates any remaining overlap.
|
|
||||||
- If a speaker has no usable word timing in a group, that speaker's original segment is kept.
|
|
||||||
- If no speakers in a group have usable word timing, the original group and annotations remain unchanged.
|
|
||||||
|
|
||||||
## Backchannels
|
|
||||||
|
|
||||||
The default pipeline runs `backchannel` before `coalesce`. It tags short acknowledgement segments with:
|
|
||||||
|
|
||||||
```json
|
|
||||||
"categories": ["backchannel"]
|
|
||||||
```
|
|
||||||
|
|
||||||
Backchannel matching is case-insensitive, ignores punctuation for matching and word-count purposes, trims surrounding whitespace, and requires a matching acknowledgement phrase, no more than three whitespace-delimited words, and duration no greater than `SERIATIM_BACKCHANNEL_MAX_DURATION` seconds. The default maximum duration is `2.0` seconds.
|
|
||||||
|
|
||||||
## Fillers
|
|
||||||
|
|
||||||
The default pipeline runs `filler` after `backchannel` and before `coalesce`. It tags short filler utterances with:
|
|
||||||
|
|
||||||
```json
|
|
||||||
"categories": ["filler"]
|
|
||||||
```
|
|
||||||
|
|
||||||
Filler matching is case-insensitive, ignores punctuation for matching and word-count purposes, trims surrounding whitespace, and requires only filler tokens such as `um`, `uh`, `er`, `erm`, `ah`, `eh`, `hmm`, `mm`, or repeated combinations of those tokens. Matching segments must contain no more than three whitespace-delimited words and have duration no greater than `SERIATIM_FILLER_MAX_DURATION` seconds. The default maximum duration is `1.25` seconds.
|
|
||||||
|
|
||||||
## Dangler Resolution
|
|
||||||
|
|
||||||
The default pipeline runs `resolve-danglers` before `coalesce` and before the second overlap detection pass. It repairs short derived fragments when they share provenance with a nearby segment:
|
|
||||||
|
|
||||||
- Dangling-end fragments have no more than two words and end in punctuation.
|
|
||||||
- Dangling-start fragments have no more than two words.
|
|
||||||
- Matching uses same-speaker segments with any shared `derived_from` value.
|
|
||||||
- Merged segments use `source_ref` values such as `resolve-danglers:1`, keep the target segment's transcript position, and union `derived_from`.
|
|
||||||
|
|
||||||
## Coalescing
|
|
||||||
|
|
||||||
The default pipeline runs `coalesce` after `resolve-danglers` and before the second overlap detection pass. It merges adjacent same-speaker segments in the transcript's current order when `next.start - current.end <= --coalesce-gap`.
|
|
||||||
|
|
||||||
Coalesced segments use `source_ref` values such as `coalesce:1`, include `derived_from`, and omit `source_segment_index`.
|
|
||||||
|
|
||||||
Different-speaker backchannel and filler segments do not block coalescing of surrounding same-speaker segments. Same-speaker backchannel and filler segments are merged normally when they are within `--coalesce-gap`. When same-speaker segments are coalesced, any `backchannel` or `filler` category from the merged inputs is dropped from the coalesced segment.
|
|
||||||
|
|
||||||
## Autocorrect
|
|
||||||
|
|
||||||
Autocorrect is included in the default postprocessing pipeline. If `--autocorrect` is omitted, the module leaves transcript text unchanged and records a skip event in the optional report.
|
|
||||||
|
|
||||||
Enable corrections by passing `--autocorrect`:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go run ./cmd/seriatim merge \
|
|
||||||
--input-file input.json \
|
|
||||||
--autocorrect autocorrect.yml \
|
|
||||||
--output-file merged.json
|
|
||||||
```
|
|
||||||
|
|
||||||
`autocorrect.yml` format:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
autocorrect:
|
|
||||||
- target: "Hrank"
|
|
||||||
match:
|
|
||||||
- "hrank"
|
|
||||||
- "Frank"
|
|
||||||
|
|
||||||
- target: "Mike Brown"
|
|
||||||
match:
|
|
||||||
- "Mike Pat"
|
|
||||||
```
|
|
||||||
|
|
||||||
Matching behavior:
|
|
||||||
|
|
||||||
- Matching is case-sensitive.
|
|
||||||
- Matches apply only to whole tokens, not substrings inside larger words.
|
|
||||||
- Punctuation and whitespace can surround a match.
|
|
||||||
- Multi-word and hyphenated matches are supported.
|
|
||||||
- Duplicate match strings are invalid, including duplicates across separate rules.
|
|
||||||
|
|
||||||
## Current Limitations
|
|
||||||
|
|
||||||
- Only JSON input is supported.
|
|
||||||
- Overlap resolution depends on WhisperX word timing; groups without usable word timing remain unresolved.
|
|
||||||
- Alternate output formats are not implemented yet.
|
|
||||||
|
|
||||||
## Release Builds
|
|
||||||
|
|
||||||
Local builds record version metadata as `dev`. Release builds should inject the release version with `ldflags`:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go build -ldflags "-X gitea.maximumdirect.net/eric/seriatim/internal/buildinfo.Version=v1.0.0" ./cmd/seriatim
|
|
||||||
```
|
|
||||||
|
|||||||
45
docs/internal/README.md
Normal file
45
docs/internal/README.md
Normal file
@@ -0,0 +1,45 @@
|
|||||||
|
# Internal Documentation Index
|
||||||
|
|
||||||
|
## Audience
|
||||||
|
Developers and coding agents changing Narratio internals.
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
`docs/internal/` documents implemented internal contracts: stage boundaries, manifest/state behavior, artifact resolution, restore behavior, storage boundaries, and workspace invariants.
|
||||||
|
|
||||||
|
User and operator behavior belongs in:
|
||||||
|
- `docs/cli.md`
|
||||||
|
- `docs/config.md`
|
||||||
|
- `docs/operations.md`
|
||||||
|
- `docs/troubleshooting.md`
|
||||||
|
|
||||||
|
## Pipeline Stage Set
|
||||||
|
Canonical stage order from `internal/stage.All()`:
|
||||||
|
1. `prepare`
|
||||||
|
2. `transcribe`
|
||||||
|
3. `merge`
|
||||||
|
4. `polish`
|
||||||
|
5. `normalize`
|
||||||
|
6. `trim`
|
||||||
|
7. `render`
|
||||||
|
8. `analyze`
|
||||||
|
9. `publish`
|
||||||
|
10. `notify` (placeholder)
|
||||||
|
|
||||||
|
`notify` is currently a placeholder stage with optional notifier call behavior; it has no persisted pipeline outputs.
|
||||||
|
|
||||||
|
## Internal Component Docs
|
||||||
|
- `adapters.md`: external adapter boundaries and default runtime wiring.
|
||||||
|
- `artifacts.md`: canonical source IDs, runtime catalog behavior, and resolution rules.
|
||||||
|
- `manifest.md`: session and run manifest contracts.
|
||||||
|
- `storage.md`: object-store interface and S3 implementation behavior.
|
||||||
|
- `workspace.md`: local session layout, run-local layout, and cleanup guardrails.
|
||||||
|
- `command-restore.md`: restore discovery, planning, execution, and reporting.
|
||||||
|
- `stage-prepare.md`
|
||||||
|
- `stage-transcribe.md`
|
||||||
|
- `stage-merge.md`
|
||||||
|
- `stage-polish.md`
|
||||||
|
- `stage-normalize.md`
|
||||||
|
- `stage-trim.md`
|
||||||
|
- `stage-render.md`
|
||||||
|
- `stage-analyze.md`
|
||||||
|
- `stage-publish.md`
|
||||||
49
docs/internal/adapters.md
Normal file
49
docs/internal/adapters.md
Normal file
@@ -0,0 +1,49 @@
|
|||||||
|
# Internal: Adapters
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
Define external integration boundaries and default adapter wiring used by app/stage orchestration.
|
||||||
|
|
||||||
|
## Adapter Boundaries
|
||||||
|
Narratio stage logic depends on adapter interfaces, not transport-specific details.
|
||||||
|
|
||||||
|
Primary adapters:
|
||||||
|
- `whisperx.Client`
|
||||||
|
- `seriatim.Runner`
|
||||||
|
- `audita.Runner`
|
||||||
|
- `scriptorium.Runner`
|
||||||
|
- `storage.ObjectStore`
|
||||||
|
- `notify.Sender`
|
||||||
|
|
||||||
|
## Ownership
|
||||||
|
Adapters own:
|
||||||
|
- HTTP/subprocess/SDK argument and transport details.
|
||||||
|
- Backend-specific request/response mapping.
|
||||||
|
|
||||||
|
Adapters do not own:
|
||||||
|
- stage ordering/skip/force logic;
|
||||||
|
- manifest transitions;
|
||||||
|
- canonical path policy.
|
||||||
|
|
||||||
|
## Default Wiring
|
||||||
|
`internal/app/runner.go` initializes default adapters when not injected:
|
||||||
|
- WhisperX HTTP client from pipeline config.
|
||||||
|
- Seriatim subprocess runner.
|
||||||
|
- Audita subprocess runner.
|
||||||
|
- Scriptorium subprocess runner.
|
||||||
|
- Noop notifier (`notify.NoopSender`).
|
||||||
|
- Object store only when required by selected stages/config.
|
||||||
|
|
||||||
|
Object-store construction goes through `newCommandObjectStore`, which loads configured filesystem secrets before adapter initialization.
|
||||||
|
|
||||||
|
## Failure Semantics
|
||||||
|
- Constructor errors fail stage execution setup early.
|
||||||
|
- Runtime adapter errors propagate to stage code and then manifest failure handling.
|
||||||
|
- Subprocess adapters persist stage logs/generated configs through stage-managed paths.
|
||||||
|
|
||||||
|
## Test Surfaces
|
||||||
|
- `internal/adapters/whisperx/http_test.go`
|
||||||
|
- `internal/adapters/seriatim/subprocess_test.go`
|
||||||
|
- `internal/adapters/audita/subprocess_test.go`
|
||||||
|
- `internal/adapters/scriptorium/subprocess_test.go`
|
||||||
|
- `internal/adapters/storage/*_test.go`
|
||||||
|
- `internal/app/runner_test.go`
|
||||||
115
docs/internal/artifacts.md
Normal file
115
docs/internal/artifacts.md
Normal file
@@ -0,0 +1,115 @@
|
|||||||
|
# Internal: Artifacts
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
Define canonical artifact IDs, runtime catalog behavior, source resolution rules, and shared current-state mechanics used by app and previous-cache code.
|
||||||
|
|
||||||
|
## Built-in Source IDs
|
||||||
|
|
||||||
|
- `narratio.transcript.base` -> `transcripts/base.json` (`merge`)
|
||||||
|
- `narratio.transcript.polished` -> `transcripts/polished.json` (`polish`)
|
||||||
|
- `narratio.transcript.final` -> `transcripts/final.json` (`normalize`)
|
||||||
|
- `narratio.transcript.final_trimmed` -> `transcripts/final.trimmed.json` (`trim`)
|
||||||
|
- `narratio.transcript.final_markdown` -> `transcripts/final.md` (`render`)
|
||||||
|
- `narratio.transcript.final_trimmed_markdown` -> `transcripts/final.trimmed.md` (`render`)
|
||||||
|
- `narratio.bounds.session` -> `artifacts/session_bounds.json` (`trim`)
|
||||||
|
|
||||||
|
## Configured and Previous-Session Sources
|
||||||
|
|
||||||
|
- configured source ID format: `narratio.artifact.<artifact_key>`
|
||||||
|
- previous-session source ID format: `narratio.previous_session.artifact.<artifact_key>`
|
||||||
|
|
||||||
|
Both formats are validated by strict source-policy rules.
|
||||||
|
|
||||||
|
## Runtime Catalog
|
||||||
|
|
||||||
|
`ArtifactCatalog` tracks:
|
||||||
|
|
||||||
|
- `planned`: source registered for run context;
|
||||||
|
- `executable`: selected and enabled for analyze execution;
|
||||||
|
- `available`: local file exists and validates;
|
||||||
|
- `provenance`: availability source.
|
||||||
|
|
||||||
|
Current provenance values:
|
||||||
|
|
||||||
|
- `generated.current_analyze_run`
|
||||||
|
- `filesystem.disabled_artifact_output`
|
||||||
|
- `manifest.inputs.previous_cache`
|
||||||
|
- `current_session.previous_cache`
|
||||||
|
|
||||||
|
## Resolution Rules
|
||||||
|
|
||||||
|
Built-ins:
|
||||||
|
|
||||||
|
1. manifest producer outputs (when present)
|
||||||
|
2. canonical session-path fallback
|
||||||
|
|
||||||
|
Configured sources (`narratio.artifact.*`):
|
||||||
|
|
||||||
|
- resolve only through runtime catalog availability.
|
||||||
|
|
||||||
|
Previous-session sources (`narratio.previous_session.artifact.*`):
|
||||||
|
|
||||||
|
- resolve only from local `previous/` cache state;
|
||||||
|
- prefer manifest-backed previous-input paths;
|
||||||
|
- fallback to existing previous-cache filesystem paths.
|
||||||
|
|
||||||
|
Validation by content type:
|
||||||
|
|
||||||
|
- transcript JSON built-ins: JSON with top-level `segments` array;
|
||||||
|
- transcript Markdown built-ins: non-empty text file;
|
||||||
|
- bounds built-in: valid JSON;
|
||||||
|
- configured/previous-session artifact files: non-empty text file.
|
||||||
|
|
||||||
|
## Previous Requirement Collection
|
||||||
|
|
||||||
|
`CollectPreviousArtifactRequirements`:
|
||||||
|
|
||||||
|
- scans enabled configured artifacts only;
|
||||||
|
- extracts only canonical previous-session sources;
|
||||||
|
- deduplicates by artifact key;
|
||||||
|
- merges required and optional references (required wins);
|
||||||
|
- returns deterministic ordering and source locations.
|
||||||
|
|
||||||
|
## Current-State Helpers
|
||||||
|
|
||||||
|
Artifacts package owns shared remote current-state loading mechanics used by restore, status/validate checks, and previous-cache planning.
|
||||||
|
|
||||||
|
Core helpers:
|
||||||
|
|
||||||
|
- `LoadCurrentRunPointer`
|
||||||
|
- `LoadCurrentManifest`
|
||||||
|
- `LoadCurrentState`
|
||||||
|
- `ValidateCurrentStateIdentity`
|
||||||
|
|
||||||
|
Typed missing-state errors:
|
||||||
|
|
||||||
|
- `CurrentRunPointerMissingError` (`ErrCurrentRunPointerMissing`)
|
||||||
|
- `CurrentManifestMissingError` (`ErrCurrentManifestMissing`)
|
||||||
|
|
||||||
|
Identity validation supports caller-provided expectations:
|
||||||
|
|
||||||
|
- expected campaign;
|
||||||
|
- expected session ID;
|
||||||
|
- expected run ID, or pointer/manifest run-ID consistency check.
|
||||||
|
|
||||||
|
Caller policy is intentionally outside artifacts helpers:
|
||||||
|
|
||||||
|
- some callers fail on missing current state;
|
||||||
|
- some callers downgrade missing state to status/findings;
|
||||||
|
- some callers skip optional behavior when state is missing.
|
||||||
|
|
||||||
|
## Key Path Helpers
|
||||||
|
|
||||||
|
`internal/artifacts/paths.go` and S3-key helpers define canonical helpers for:
|
||||||
|
|
||||||
|
- session/work/run paths;
|
||||||
|
- previous-cache paths;
|
||||||
|
- spool/cache paths;
|
||||||
|
- S3 session/run/current-state key layout.
|
||||||
|
|
||||||
|
## Invariants
|
||||||
|
|
||||||
|
- source ID formats are stable contracts;
|
||||||
|
- artifact resolution is deterministic and manifest-aware;
|
||||||
|
- previous-session source resolution in `analyze` is local-only;
|
||||||
|
- remote current-state key construction remains centralized in artifacts helpers.
|
||||||
84
docs/internal/command-restore.md
Normal file
84
docs/internal/command-restore.md
Normal file
@@ -0,0 +1,84 @@
|
|||||||
|
# Internal: Command Restore
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
Define the implemented `narratio session restore` command contract:
|
||||||
|
|
||||||
|
- committed remote current-state discovery;
|
||||||
|
- deterministic restore planning;
|
||||||
|
- safe local install semantics;
|
||||||
|
- durable restore reporting.
|
||||||
|
|
||||||
|
## Discovery Contract
|
||||||
|
|
||||||
|
Restore resolves remote committed state from the session publish current pointers:
|
||||||
|
|
||||||
|
- `current/run_id.txt` (required, non-empty);
|
||||||
|
- `current/manifest.json` (required, decodable).
|
||||||
|
|
||||||
|
Current-state discovery uses shared artifacts-level mechanics and validates identity against the resolved request config:
|
||||||
|
|
||||||
|
- campaign must match;
|
||||||
|
- session ID must match.
|
||||||
|
|
||||||
|
Restore treats any missing or invalid remote current state as a command error.
|
||||||
|
|
||||||
|
## Planning Contract
|
||||||
|
|
||||||
|
Restore planner action kinds:
|
||||||
|
|
||||||
|
- `download`;
|
||||||
|
- `skip_same`;
|
||||||
|
- `conflict`.
|
||||||
|
|
||||||
|
Planner behavior:
|
||||||
|
|
||||||
|
- remote list scope is the resolved session prefix;
|
||||||
|
- remote-to-local mapping is traversal-safe;
|
||||||
|
- actions are sorted deterministically by local relative path.
|
||||||
|
|
||||||
|
Restore scope from current remote state:
|
||||||
|
|
||||||
|
- include `manifest.json`;
|
||||||
|
- include `transcripts/**`;
|
||||||
|
- include `artifacts/**`;
|
||||||
|
- include `audio/**` only with `--include-audio`.
|
||||||
|
|
||||||
|
Explicit exclusions from current remote state mapping:
|
||||||
|
|
||||||
|
- `current/**`;
|
||||||
|
- `runs/**`;
|
||||||
|
- `logs/**`;
|
||||||
|
- `reports/**`;
|
||||||
|
- `config/**`;
|
||||||
|
- `inputs/**`;
|
||||||
|
- `previous/**`.
|
||||||
|
|
||||||
|
Previous-cache files are planned separately through `previouscache.BuildPlan` when configured previous-session requirements exist.
|
||||||
|
|
||||||
|
## Execution Contract
|
||||||
|
|
||||||
|
Execution order and safety:
|
||||||
|
|
||||||
|
- non-manifest downloads happen before manifest install;
|
||||||
|
- `manifest.json` installs last;
|
||||||
|
- downloads use sibling temp files plus atomic rename;
|
||||||
|
- manifest replacement is validated before rename;
|
||||||
|
- failed installs do not roll back files already written in the same execution.
|
||||||
|
|
||||||
|
Audio restore path:
|
||||||
|
|
||||||
|
- uses `audio.MaterializeS3Audio`;
|
||||||
|
- integrates spool and S3 audio cache paths;
|
||||||
|
- supports cache-hit reuse without object redownload.
|
||||||
|
|
||||||
|
## Reporting Contract
|
||||||
|
|
||||||
|
- `--dry-run`: prints summary only; no local writes.
|
||||||
|
- non-dry-run: writes `reports/restore-latest.json`.
|
||||||
|
- report includes plan counts, per-action status, and execution failures.
|
||||||
|
|
||||||
|
## Invariants
|
||||||
|
|
||||||
|
- restore uses committed remote current state as authority;
|
||||||
|
- `current/run_id.txt` is the remote publish commit marker;
|
||||||
|
- restore does not execute pipeline stages.
|
||||||
57
docs/internal/manifest.md
Normal file
57
docs/internal/manifest.md
Normal file
@@ -0,0 +1,57 @@
|
|||||||
|
# Internal: Manifest
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
Define durable session state (`manifest.json`) and invocation state (`runs/{run_id}/manifest.json`) contracts.
|
||||||
|
|
||||||
|
## Session Manifest
|
||||||
|
Path:
|
||||||
|
- `{workspace.root}/work/{campaign}/{session_id}/manifest.json`
|
||||||
|
|
||||||
|
Primary model (`manifest.Manifest`):
|
||||||
|
- identity (`session_id`, `campaign`, `run_id`)
|
||||||
|
- local path metadata (`local_workdir`, `local_spool_dir`)
|
||||||
|
- remote identity metadata (`s3_bucket`, `s3_session_prefix`, `s3_run_prefix`)
|
||||||
|
- `inputs` records
|
||||||
|
- durable `artifacts` records
|
||||||
|
- per-stage `stages` map
|
||||||
|
|
||||||
|
Stage status enum:
|
||||||
|
- `pending`
|
||||||
|
- `running`
|
||||||
|
- `succeeded`
|
||||||
|
- `failed`
|
||||||
|
- `skipped`
|
||||||
|
- `stale`
|
||||||
|
- `interrupted`
|
||||||
|
|
||||||
|
## Run Manifest
|
||||||
|
Path:
|
||||||
|
- `{workspace.root}/work/{campaign}/{session_id}/runs/{run_id}/manifest.json`
|
||||||
|
|
||||||
|
Run model (`manifest.RunManifest`):
|
||||||
|
- invocation identity and `force` flag
|
||||||
|
- requested stages
|
||||||
|
- per-stage action (`run` or `skip`)
|
||||||
|
- per-stage status
|
||||||
|
- overall run status (`running`, `succeeded`, `failed`)
|
||||||
|
|
||||||
|
## Persistence Semantics
|
||||||
|
`manifest.LocalStore`:
|
||||||
|
- validates loaded documents;
|
||||||
|
- normalizes missing maps/stage records;
|
||||||
|
- writes atomically via temp file + rename;
|
||||||
|
- updates `updated_at` on save.
|
||||||
|
|
||||||
|
## Execution Semantics
|
||||||
|
Runner updates both manifests per stage transition:
|
||||||
|
- mark running
|
||||||
|
- mark succeeded/failed/skipped
|
||||||
|
- persist logs/generated config refs and metadata
|
||||||
|
|
||||||
|
Session manifest is the authoritative stage-progress ledger across invocations.
|
||||||
|
Run manifest is invocation-scoped audit state.
|
||||||
|
|
||||||
|
## Invariants
|
||||||
|
- stage resume/skip decisions are session-manifest driven.
|
||||||
|
- force reruns stale downstream succeeded stages.
|
||||||
|
- run manifest does not replace session manifest as progress authority.
|
||||||
42
docs/internal/stage-analyze.md
Normal file
42
docs/internal/stage-analyze.md
Normal file
@@ -0,0 +1,42 @@
|
|||||||
|
# Stage: analyze
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
Execute selected configured Scriptorium artifacts in dependency order and materialize outputs.
|
||||||
|
|
||||||
|
## Inputs
|
||||||
|
- configured artifacts from `pipeline.scriptorium.artifacts`
|
||||||
|
- optional selected artifact filter (`--artifacts`)
|
||||||
|
- built-in/configured/previous-session source references in artifact inputs
|
||||||
|
|
||||||
|
Supported source families:
|
||||||
|
- built-ins: `narratio.transcript.*`, `narratio.bounds.session`
|
||||||
|
- prepared stable inputs: `narratio.input.players`, `narratio.input.party`, `narratio.input.glossary`
|
||||||
|
- configured artifacts: `narratio.artifact.<key>`
|
||||||
|
- previous-session cache: `narratio.previous_session.artifact.<key>`
|
||||||
|
|
||||||
|
## Outputs
|
||||||
|
- one materialized output per executed configured artifact (`output_path`)
|
||||||
|
- stage metadata describing selected/generated/reused artifacts
|
||||||
|
|
||||||
|
## Key Behavior
|
||||||
|
- skips with metadata when Scriptorium config is missing or no executable artifacts remain.
|
||||||
|
- builds runtime artifact catalog (built-ins + configured artifacts).
|
||||||
|
- marks non-executable configured artifacts as reusable when output files already exist.
|
||||||
|
- validates selected artifact dependency order (cycle-safe topo ordering).
|
||||||
|
- resolves required/optional inputs per artifact source definition.
|
||||||
|
- resolves prepared stable input sources from `inputs/*.yml` materialized by `prepare`.
|
||||||
|
- resolves previous-session sources from local `previous/` cache only.
|
||||||
|
- runs optional render-debug, then artifact execution.
|
||||||
|
- validates non-empty output files and materializes canonical outputs.
|
||||||
|
|
||||||
|
## Failure Semantics
|
||||||
|
- required missing configured/previous-session inputs fail.
|
||||||
|
- missing required prepared stable input source includes prepare rerun guidance.
|
||||||
|
- missing required previous-session source includes prepare rerun guidance.
|
||||||
|
- missing required `narratio.transcript.final_markdown` or `narratio.transcript.final_trimmed_markdown` inputs includes render rerun guidance.
|
||||||
|
- dependency cycles or unavailable required dependencies fail.
|
||||||
|
- adapter validation failures fail stage.
|
||||||
|
|
||||||
|
## Invariants
|
||||||
|
- `analyze` performs no remote storage calls for previous-session source resolution.
|
||||||
|
- output provenance and metadata are deterministic per execution.
|
||||||
25
docs/internal/stage-merge.md
Normal file
25
docs/internal/stage-merge.md
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
# Stage: merge
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
Normalize raw transcript inputs and merge into base transcript via Seriatim.
|
||||||
|
|
||||||
|
## Inputs
|
||||||
|
- `transcripts/raw/*.json`
|
||||||
|
- `inputs/speakers.yml`
|
||||||
|
- `inputs/autocorrect.yml`
|
||||||
|
|
||||||
|
## Outputs
|
||||||
|
- `transcripts/base.json`
|
||||||
|
- optional `artifacts/seriatim.report.json`
|
||||||
|
|
||||||
|
## Key Behavior
|
||||||
|
- discovers and validates raw transcript inputs.
|
||||||
|
- normalizes each raw transcript (`seriatim.Normalize`) into run-local scratch output.
|
||||||
|
- merges normalized inputs (`seriatim.Run`) into base transcript.
|
||||||
|
- validates merged transcript and optional report JSON.
|
||||||
|
- materializes canonical outputs and records stage logs/generated configs.
|
||||||
|
|
||||||
|
## Invariants
|
||||||
|
- merge always consumes normalized forms of raw inputs.
|
||||||
|
- base transcript must validate before stage success.
|
||||||
|
- report output is config-gated.
|
||||||
22
docs/internal/stage-normalize.md
Normal file
22
docs/internal/stage-normalize.md
Normal file
@@ -0,0 +1,22 @@
|
|||||||
|
# Stage: normalize
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
Normalize polished transcript into final transcript using Seriatim.
|
||||||
|
|
||||||
|
## Inputs
|
||||||
|
- `transcripts/polished.json`
|
||||||
|
|
||||||
|
## Outputs
|
||||||
|
- `transcripts/final.json` (or configured normalize output path)
|
||||||
|
- optional `artifacts/seriatim.normalize.report.json`
|
||||||
|
|
||||||
|
## Key Behavior
|
||||||
|
- resolves polished transcript from manifest outputs/canonical fallback.
|
||||||
|
- applies `pipeline.normalize` config or default normalize config.
|
||||||
|
- runs Seriatim normalize with configured timeout/binary.
|
||||||
|
- validates normalized transcript and optional report.
|
||||||
|
- materializes canonical outputs and records logs/generated configs.
|
||||||
|
|
||||||
|
## Invariants
|
||||||
|
- final transcript must validate as processed transcript JSON (`segments` array).
|
||||||
|
- normalize defaults are applied when `pipeline.normalize` is unset.
|
||||||
23
docs/internal/stage-polish.md
Normal file
23
docs/internal/stage-polish.md
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
# Stage: polish
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
Run Audita polishing on base transcript and produce polished transcript.
|
||||||
|
|
||||||
|
## Inputs
|
||||||
|
- `transcripts/base.json`
|
||||||
|
- `inputs/glossary.yml`
|
||||||
|
|
||||||
|
## Outputs
|
||||||
|
- `transcripts/polished.json`
|
||||||
|
- optional `artifacts/audita.report.json`
|
||||||
|
|
||||||
|
## Key Behavior
|
||||||
|
- resolves base transcript from merge outputs/canonical fallback.
|
||||||
|
- invokes Audita with configured model/module/runtime options.
|
||||||
|
- validates processed transcript structure (`segments` array required).
|
||||||
|
- validates optional report JSON.
|
||||||
|
- materializes canonical outputs; records logs/generated config and adapter metadata.
|
||||||
|
|
||||||
|
## Invariants
|
||||||
|
- polished transcript schema validation is mandatory.
|
||||||
|
- report output is config-gated.
|
||||||
44
docs/internal/stage-prepare.md
Normal file
44
docs/internal/stage-prepare.md
Normal file
@@ -0,0 +1,44 @@
|
|||||||
|
# Stage: prepare
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
Materialize canonical current-session inputs before processing stages.
|
||||||
|
|
||||||
|
## Inputs
|
||||||
|
- resolved `campaign.yml`, `session.yml`, and pipeline config
|
||||||
|
- stable input files (`speakers`, `autocorrect`, `glossary`, `players`, `party`)
|
||||||
|
- audio source:
|
||||||
|
- local `audio_dir`/`audio_files`, or
|
||||||
|
- S3 `audio_s3.prefix`
|
||||||
|
- enabled configured artifact input requirements for previous-session sources
|
||||||
|
|
||||||
|
## Outputs
|
||||||
|
- `inputs/campaign.yml`
|
||||||
|
- `inputs/session.yml`
|
||||||
|
- `inputs/pipeline.resolved.yml`
|
||||||
|
- `inputs/speakers.yml`
|
||||||
|
- `inputs/autocorrect.yml`
|
||||||
|
- `inputs/glossary.yml`
|
||||||
|
- `inputs/players.yml`
|
||||||
|
- `inputs/party.yml`
|
||||||
|
- `audio/*.flac`
|
||||||
|
- optional `previous/manifest.json`
|
||||||
|
- optional `previous/artifacts/**`
|
||||||
|
- deterministic `manifest.inputs` entries (checksums + provenance)
|
||||||
|
|
||||||
|
## Key Behavior
|
||||||
|
- validates required config/store state.
|
||||||
|
- enforces local audio vs S3 audio mutual exclusivity.
|
||||||
|
- materializes S3 audio through spool/cache-aware logic.
|
||||||
|
- scans enabled configured artifact inputs for `narratio.previous_session.artifact.*` requirements.
|
||||||
|
- when previous requirements exist:
|
||||||
|
- clears managed `previous/` state;
|
||||||
|
- builds previous-cache remote plan;
|
||||||
|
- downloads previous manifest/artifacts;
|
||||||
|
- records previous inputs in `manifest.inputs`.
|
||||||
|
|
||||||
|
Required previous-session inputs fail when unavailable; optional missing inputs are skipped.
|
||||||
|
|
||||||
|
## Invariants
|
||||||
|
- only `prepare` hydrates canonical `previous/` cache state.
|
||||||
|
- managed previous artifacts are stored under `previous/artifacts/**` without duplicate `artifacts/artifacts/` nesting.
|
||||||
|
- `manifest.inputs` ordering is deterministic (`kind`, `path`).
|
||||||
44
docs/internal/stage-publish.md
Normal file
44
docs/internal/stage-publish.md
Normal file
@@ -0,0 +1,44 @@
|
|||||||
|
# Stage: publish
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
Upload run/session outputs to object storage and atomically advance remote current state.
|
||||||
|
|
||||||
|
## Inputs
|
||||||
|
- successful prerequisite stages: `prepare`, `transcribe`, `merge`, `polish`, `normalize`, `trim`, `render`, `analyze`
|
||||||
|
- run root `runs/{run_id}/**`
|
||||||
|
- publish output rules (`pipeline.publish.outputs`)
|
||||||
|
- effective publish locks (static + remote merged lock set)
|
||||||
|
- local `previous/**` files when present
|
||||||
|
|
||||||
|
## Outputs
|
||||||
|
- uploaded run files under remote `runs/{run_id}/...` (excluding `audio/**`)
|
||||||
|
- uploaded selected publish outputs under session prefix
|
||||||
|
- uploaded `previous/**` files under session prefix when present
|
||||||
|
- uploaded `current/manifest.json`
|
||||||
|
- uploaded `current/run_id.txt` written last
|
||||||
|
|
||||||
|
## Key Behavior
|
||||||
|
- stage can self-skip when publish disabled or run upload disabled.
|
||||||
|
- validates prerequisite stage success and object-store availability.
|
||||||
|
- collects deterministic run file list plus run `manifest.json`.
|
||||||
|
- resolves publish output sources through runtime artifact catalog and manifest-aware resolution.
|
||||||
|
- selected artifact filter applies to configured artifact sources only.
|
||||||
|
- locked outputs are skipped intentionally (including required ones).
|
||||||
|
- optional missing outputs are skipped; required missing unlocked outputs fail.
|
||||||
|
- writes remote current manifest before current run pointer.
|
||||||
|
|
||||||
|
## Metadata Signals
|
||||||
|
Includes counts/lists for:
|
||||||
|
- run uploads
|
||||||
|
- published output uploads
|
||||||
|
- previous uploads
|
||||||
|
- skipped optional outputs
|
||||||
|
- skipped unselected outputs
|
||||||
|
- locked outputs
|
||||||
|
- current-state key paths
|
||||||
|
- `current_pointer_written`
|
||||||
|
|
||||||
|
## Invariants
|
||||||
|
- `current/run_id.txt` is the remote commit marker and is written last.
|
||||||
|
- run upload excludes `audio/**`.
|
||||||
|
- publish locks are not overridden by `--force`.
|
||||||
29
docs/internal/stage-render.md
Normal file
29
docs/internal/stage-render.md
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
# Stage: render
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
Render Markdown transcript artifacts from normalized JSON transcripts via Seriatim.
|
||||||
|
|
||||||
|
## Inputs
|
||||||
|
- `narratio.transcript.final` (`transcripts/final.json`)
|
||||||
|
- `narratio.transcript.final_trimmed` (`transcripts/final.trimmed.json`)
|
||||||
|
|
||||||
|
## Outputs
|
||||||
|
- `narratio.transcript.final_markdown` -> `transcripts/final.md`
|
||||||
|
- `narratio.transcript.final_trimmed_markdown` -> `transcripts/final.trimmed.md`
|
||||||
|
|
||||||
|
## Key Behavior
|
||||||
|
- uses `pipeline.render` settings (enabled/format/title/booleans).
|
||||||
|
- resolves inputs manifest-first, then canonical fallback.
|
||||||
|
- writes run-local outputs first, then materializes canonical session outputs.
|
||||||
|
- records input provenance, output paths, adapter metadata, logs, and generated config refs.
|
||||||
|
- skips with stage metadata when `pipeline.render.enabled=false`.
|
||||||
|
|
||||||
|
## Failure Semantics
|
||||||
|
- missing normalized input fails with normalize rerun guidance.
|
||||||
|
- missing trimmed input fails with trim rerun guidance.
|
||||||
|
- adapter/subprocess failure fails stage.
|
||||||
|
- empty render output files fail validation.
|
||||||
|
|
||||||
|
## Invariants
|
||||||
|
- only `format: markdown` is supported.
|
||||||
|
- render stage owns production of built-in Markdown transcript sources.
|
||||||
22
docs/internal/stage-transcribe.md
Normal file
22
docs/internal/stage-transcribe.md
Normal file
@@ -0,0 +1,22 @@
|
|||||||
|
# Stage: transcribe
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
Generate raw per-speaker transcripts from prepared audio using WhisperX.
|
||||||
|
|
||||||
|
## Inputs
|
||||||
|
- `audio/*.flac` from `prepare`
|
||||||
|
|
||||||
|
## Outputs
|
||||||
|
- `transcripts/raw/<speaker>.json`
|
||||||
|
|
||||||
|
## Key Behavior
|
||||||
|
- discovers prepared audio from manifest inputs or canonical audio directory.
|
||||||
|
- derives speaker ID from `.flac` basename.
|
||||||
|
- runs WhisperX with configured concurrency/retry settings.
|
||||||
|
- validates each output as JSON.
|
||||||
|
- writes run-local outputs then materializes canonical transcript outputs.
|
||||||
|
|
||||||
|
## Invariants
|
||||||
|
- speaker basenames must be unique.
|
||||||
|
- output path returned by adapter must match requested output path.
|
||||||
|
- each successful output is validated before stage success.
|
||||||
28
docs/internal/stage-trim.md
Normal file
28
docs/internal/stage-trim.md
Normal file
@@ -0,0 +1,28 @@
|
|||||||
|
# Stage: trim
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
Produce a final-trimmed transcript. By default, the stage generates bounds and applies a bounds-driven trim.
|
||||||
|
|
||||||
|
## Inputs
|
||||||
|
- `transcripts/final.json`
|
||||||
|
|
||||||
|
## Outputs
|
||||||
|
- `transcripts/final.trimmed.json` (or configured trim output path)
|
||||||
|
- when trim enabled: `artifacts/session_bounds.json`
|
||||||
|
|
||||||
|
## Key Behavior
|
||||||
|
When `trim.enabled=true`:
|
||||||
|
- runs Scriptorium bounds artifact generation;
|
||||||
|
- optionally runs render-debug output generation;
|
||||||
|
- validates bounds payload against transcript;
|
||||||
|
- derives keep selector;
|
||||||
|
- either copies unchanged transcript or runs Seriatim trim;
|
||||||
|
- validates trimmed transcript and materializes bounds output.
|
||||||
|
|
||||||
|
When `trim.enabled=false`:
|
||||||
|
- copies normalized transcript to trimmed output.
|
||||||
|
|
||||||
|
## Invariants
|
||||||
|
- normalized transcript is required input.
|
||||||
|
- bounds output exists only in enabled trim path.
|
||||||
|
- render-debug output is diagnostic and not a declared stage output.
|
||||||
34
docs/internal/storage.md
Normal file
34
docs/internal/storage.md
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
# Internal: Storage
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
Document remote object-store contracts and S3 implementation behavior.
|
||||||
|
|
||||||
|
## Primary Contract
|
||||||
|
`storage.ObjectStore` interface:
|
||||||
|
- `List(ctx, prefix)`
|
||||||
|
- `Download(ctx, key, localPath)`
|
||||||
|
- `Upload(ctx, localPath, key, opts)`
|
||||||
|
- `Exists(ctx, key)`
|
||||||
|
|
||||||
|
Key invariant:
|
||||||
|
- callers pass full bucket-relative keys;
|
||||||
|
- storage implementations do not infer campaign/session/run prefixes.
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
`NewObjectStoreFromConfig` currently supports S3-backed stores from `pipeline.storage.*` config.
|
||||||
|
|
||||||
|
S3 constructor behavior:
|
||||||
|
- requires configured bucket;
|
||||||
|
- uses region/endpoint/path-style options when set;
|
||||||
|
- resolves credentials from configured env var names (with defaults).
|
||||||
|
|
||||||
|
## S3 Backend Behavior
|
||||||
|
- normalizes object keys.
|
||||||
|
- `List` paginates and returns normalized `ObjectInfo`.
|
||||||
|
- `Download` writes local files with parent directory creation.
|
||||||
|
- `Upload` streams local file and returns remote metadata.
|
||||||
|
- `Exists` maps not-found responses to `false`.
|
||||||
|
|
||||||
|
## Invariants
|
||||||
|
- storage layer is stateless regarding manifest/stage progression.
|
||||||
|
- publish ordering semantics are owned by stage/app code, not storage adapters.
|
||||||
57
docs/internal/workspace.md
Normal file
57
docs/internal/workspace.md
Normal file
@@ -0,0 +1,57 @@
|
|||||||
|
# Internal: Workspace
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
Define local session layout, run-local stage layout, and cleanup guardrails.
|
||||||
|
|
||||||
|
## Canonical Session Layout
|
||||||
|
Session root:
|
||||||
|
- `{workspace.root}/work/{campaign}/{session_id}`
|
||||||
|
|
||||||
|
Core directories/files:
|
||||||
|
- `inputs/`
|
||||||
|
- `audio/`
|
||||||
|
- `transcripts/`
|
||||||
|
- `artifacts/`
|
||||||
|
- `reports/`
|
||||||
|
- `logs/`
|
||||||
|
- `config/`
|
||||||
|
- `current/`
|
||||||
|
- `runs/`
|
||||||
|
- `previous/`
|
||||||
|
- `manifest.json`
|
||||||
|
- `.lock`
|
||||||
|
|
||||||
|
`previous/` reserved files:
|
||||||
|
- `previous/manifest.json`
|
||||||
|
- `previous/artifacts/**`
|
||||||
|
|
||||||
|
## Run-Local Stage Layout
|
||||||
|
When run context is available, stages use:
|
||||||
|
- `runs/{run_id}/{stage}/outputs/`
|
||||||
|
- `runs/{run_id}/{stage}/logs/`
|
||||||
|
- `runs/{run_id}/{stage}/reports/`
|
||||||
|
- `runs/{run_id}/{stage}/config/`
|
||||||
|
- `runs/{run_id}/{stage}/scratch/`
|
||||||
|
|
||||||
|
Run-local outputs are materialized back into canonical session paths before stage success.
|
||||||
|
`previous/**` writes are never redirected to run-local output paths.
|
||||||
|
|
||||||
|
## Locking
|
||||||
|
`artifacts.LocalStore` enforces single-writer session lock via `.lock` file (`ErrLockConflict` on contention).
|
||||||
|
|
||||||
|
## Cleanup Semantics
|
||||||
|
Automatic post-publish cleanup:
|
||||||
|
- only runs when publish actually executed and succeeded;
|
||||||
|
- requires `uploaded=true` and `current_pointer_written=true` metadata;
|
||||||
|
- respects `pipeline.spool.delete_audio_after_publish` and `pipeline.workspace.cleanup_after_publish`;
|
||||||
|
- refuses unsafe deletes (root delete, out-of-root delete, symlink paths).
|
||||||
|
|
||||||
|
Manual clean command:
|
||||||
|
- `clean <session_id>` removes session work and spool subtree.
|
||||||
|
- `clean --all` removes all workspace work and spool children.
|
||||||
|
- durable cache is preserved unless `--clear-cache` is requested.
|
||||||
|
|
||||||
|
## Invariants
|
||||||
|
- campaign-aware session root is mandatory.
|
||||||
|
- manifest-driven stage state is durable across runs.
|
||||||
|
- cleanup guardrails prevent destructive root/out-of-scope deletion.
|
||||||
253
docs/operations.md
Normal file
253
docs/operations.md
Normal file
@@ -0,0 +1,253 @@
|
|||||||
|
# Operations Guide
|
||||||
|
|
||||||
|
Operator workflow for running, recovering, and publishing Narratio sessions.
|
||||||
|
|
||||||
|
For command syntax, see [docs/cli.md](./cli.md). For field-level config, see [docs/config.md](./config.md).
|
||||||
|
|
||||||
|
## Campaign and Session Selection
|
||||||
|
|
||||||
|
Campaign selection priority:
|
||||||
|
|
||||||
|
- `--campaign-file`
|
||||||
|
- `--campaign`
|
||||||
|
- `pipeline.campaigns.default_campaign_id`
|
||||||
|
|
||||||
|
Session source priority:
|
||||||
|
|
||||||
|
- `--session`
|
||||||
|
- local default search paths
|
||||||
|
- remote session object (S3) when local session file is not found and storage is configured
|
||||||
|
|
||||||
|
## Session Initialization
|
||||||
|
|
||||||
|
Use `session init` to generate a concrete session file for local or remote use.
|
||||||
|
|
||||||
|
Local file:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio session init 2026-04-04 --output ./session.yml --date 2026-04-04 --title "Session 12"
|
||||||
|
```
|
||||||
|
|
||||||
|
Remote session object:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio session init 2026-04-04 --remote --force
|
||||||
|
```
|
||||||
|
|
||||||
|
If `campaign.yml` sets `session_template_file`, `session init` renders it. Template variables must resolve to concrete values.
|
||||||
|
|
||||||
|
Campaigns must provide stable input files for speakers, autocorrect, glossary, players, and party. Session files may override those paths for one session. The `prepare` stage materializes them under `inputs/`; configured Scriptorium artifacts can reference prepared `players`, `party`, and `glossary` files with `narratio.input.players`, `narratio.input.party`, and `narratio.input.glossary`.
|
||||||
|
|
||||||
|
## Standard Session Workflow
|
||||||
|
|
||||||
|
1. Select pipeline/campaign/session config.
|
||||||
|
2. Validate session readiness:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio session validate 2026-04-04
|
||||||
|
```
|
||||||
|
|
||||||
|
3. (Optional) inspect stage decisions:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio session plan 2026-04-04
|
||||||
|
```
|
||||||
|
|
||||||
|
4. Run the pipeline:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio run 2026-04-04
|
||||||
|
```
|
||||||
|
|
||||||
|
5. Check state:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio session status 2026-04-04
|
||||||
|
```
|
||||||
|
|
||||||
|
## Stage Execution and Continuation Behavior
|
||||||
|
|
||||||
|
Canonical stage order:
|
||||||
|
|
||||||
|
1. `prepare`
|
||||||
|
2. `transcribe`
|
||||||
|
3. `merge`
|
||||||
|
4. `polish`
|
||||||
|
5. `normalize`
|
||||||
|
6. `trim`
|
||||||
|
7. `render`
|
||||||
|
8. `analyze`
|
||||||
|
9. `publish`
|
||||||
|
10. `notify`
|
||||||
|
|
||||||
|
Execution rules:
|
||||||
|
|
||||||
|
- succeeded stages are skipped unless `--force` is set;
|
||||||
|
- `run` continues interrupted or partially completed sessions by running non-succeeded stages;
|
||||||
|
- force rerunning a succeeded upstream stage marks succeeded downstream stages as `stale`.
|
||||||
|
|
||||||
|
Single-stage execution:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio run-stage normalize 2026-04-04 --force
|
||||||
|
```
|
||||||
|
|
||||||
|
## Artifact Selection
|
||||||
|
|
||||||
|
`--artifacts` can be used on `run`, `run-stage`, `analyze`, and `publish`.
|
||||||
|
|
||||||
|
Selection behavior:
|
||||||
|
|
||||||
|
- validates names against `pipeline.scriptorium.artifacts`;
|
||||||
|
- filters analyze execution to selected configured artifacts;
|
||||||
|
- filters publish rules for `narratio.artifact.<name>` sources only;
|
||||||
|
- does not suppress built-in transcript or bounds publish sources.
|
||||||
|
|
||||||
|
## Publish Workflow
|
||||||
|
|
||||||
|
Run publish only:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio publish 2026-04-04
|
||||||
|
```
|
||||||
|
|
||||||
|
Equivalent:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio run-stage publish 2026-04-04 --force
|
||||||
|
```
|
||||||
|
|
||||||
|
Publish commit model:
|
||||||
|
|
||||||
|
- uploads run files under `{session_prefix}/runs/{run_id}/`;
|
||||||
|
- uploads configured published outputs;
|
||||||
|
- uploads `previous/**` cache files when present;
|
||||||
|
- writes `current/manifest.json`;
|
||||||
|
- writes `current/run_id.txt` last.
|
||||||
|
|
||||||
|
`current/run_id.txt` is the remote current-state commit marker.
|
||||||
|
|
||||||
|
## Publish Locks
|
||||||
|
|
||||||
|
Lock sources:
|
||||||
|
|
||||||
|
- static locks in `pipeline.publish.locks`
|
||||||
|
- mutable remote locks in `{session_prefix}/locks.yml`
|
||||||
|
|
||||||
|
Effective lock rules:
|
||||||
|
|
||||||
|
- static and remote locks are merged;
|
||||||
|
- static locks win on source collisions;
|
||||||
|
- locked outputs are intentional skips;
|
||||||
|
- lock add/remove commands mutate only remote lock state.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio session locks 2026-04-04
|
||||||
|
narratio session locks add 2026-04-04 narratio.artifact.session_recap --reason "manual edits" --force
|
||||||
|
narratio session locks remove 2026-04-04 narratio.artifact.session_recap
|
||||||
|
```
|
||||||
|
|
||||||
|
## Restore Workflow
|
||||||
|
|
||||||
|
Use restore when local durable session state is missing or stale and remote committed current state is authoritative.
|
||||||
|
|
||||||
|
Dry run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio session restore 2026-04-04 --dry-run
|
||||||
|
```
|
||||||
|
|
||||||
|
Apply:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio session restore 2026-04-04
|
||||||
|
```
|
||||||
|
|
||||||
|
Default restore scope:
|
||||||
|
|
||||||
|
- `manifest.json`
|
||||||
|
- `transcripts/**`
|
||||||
|
- `artifacts/**`
|
||||||
|
- `previous/**` when needed by configured previous-session artifact inputs
|
||||||
|
|
||||||
|
Optional:
|
||||||
|
|
||||||
|
- `--include-audio` to include `audio/**`
|
||||||
|
- `--force` to overwrite local conflicts
|
||||||
|
|
||||||
|
Restore writes an execution report at `reports/restore-latest.json`.
|
||||||
|
|
||||||
|
## Local State Layout
|
||||||
|
|
||||||
|
Session root:
|
||||||
|
|
||||||
|
- `{workspace.root}/work/{campaign}/{session_id}`
|
||||||
|
|
||||||
|
Durable session paths:
|
||||||
|
|
||||||
|
- `manifest.json`
|
||||||
|
- `inputs/**`
|
||||||
|
- `audio/**`
|
||||||
|
- `transcripts/**`
|
||||||
|
- `artifacts/**`
|
||||||
|
- `previous/**`
|
||||||
|
- `reports/**`
|
||||||
|
- `logs/**`
|
||||||
|
- `config/**`
|
||||||
|
- `runs/**`
|
||||||
|
|
||||||
|
Run-local layout:
|
||||||
|
|
||||||
|
- `runs/{run_id}/{stage}/outputs`
|
||||||
|
- `runs/{run_id}/{stage}/logs`
|
||||||
|
- `runs/{run_id}/{stage}/reports`
|
||||||
|
- `runs/{run_id}/{stage}/config`
|
||||||
|
- `runs/{run_id}/{stage}/scratch`
|
||||||
|
|
||||||
|
Spool layout (runtime/transient):
|
||||||
|
|
||||||
|
- `{spool.root}/{campaign}/{session_id}/{run_id}/...`
|
||||||
|
- restore audio spool under `{spool.root}/{campaign}/{session_id}/restore/audio`
|
||||||
|
|
||||||
|
Cache layout (durable S3 audio cache):
|
||||||
|
|
||||||
|
- `{cache.root}/s3/{bucket}/...`
|
||||||
|
|
||||||
|
## Cleanup
|
||||||
|
|
||||||
|
Session-scoped cleanup:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio clean 2026-04-04
|
||||||
|
```
|
||||||
|
|
||||||
|
Global cleanup:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio clean --all
|
||||||
|
```
|
||||||
|
|
||||||
|
Dry-run and cache variants:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio clean 2026-04-04 --dry-run --clear-cache
|
||||||
|
narratio clean --all --dry-run --clear-cache
|
||||||
|
```
|
||||||
|
|
||||||
|
Rules:
|
||||||
|
|
||||||
|
- `clean` deletes work/spool session state;
|
||||||
|
- cache is preserved unless `--clear-cache` is set;
|
||||||
|
- automatic post-publish cleanup is gated by successful publish commit plus:
|
||||||
|
- `pipeline.spool.delete_audio_after_publish=true`
|
||||||
|
- `pipeline.workspace.cleanup_after_publish=true`
|
||||||
|
|
||||||
|
## Operational Caveats
|
||||||
|
|
||||||
|
- Local and S3 audio modes are mutually exclusive.
|
||||||
|
- Publish requires prerequisite stages through `render` and `analyze` to be succeeded.
|
||||||
|
- Markdown publish defaults require render outputs (`transcripts/final.md` and `transcripts/final.trimmed.md`).
|
||||||
|
- Restore requires configured object storage and committed remote current state.
|
||||||
|
- Storage-backed commands load filesystem secrets before object-store initialization.
|
||||||
202
docs/policy/architecture.md
Normal file
202
docs/policy/architecture.md
Normal file
@@ -0,0 +1,202 @@
|
|||||||
|
# Narratio Architecture
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
`narratio` is a Go orchestration application for processing D&D session audio into polished transcripts and generated session artifacts.
|
||||||
|
|
||||||
|
This document defines the development principles for the project. It is inward-facing: its audience is developers and LLM coding agents. It should guide future changes, not serve as a complete implementation reference.
|
||||||
|
|
||||||
|
Implemented component details belong under `docs/internal/`.
|
||||||
|
|
||||||
|
## Project Shape
|
||||||
|
|
||||||
|
Narratio is a modular, stage-driven orchestrator.
|
||||||
|
|
||||||
|
It coordinates specialized downstream systems rather than reimplementing their domains:
|
||||||
|
|
||||||
|
- WhisperX handles transcription.
|
||||||
|
- Seriatim handles deterministic transcript merge/normalization/trim behavior.
|
||||||
|
- Audita handles transcript correction and polishing.
|
||||||
|
- Scriptorium handles prompt execution and generated artifacts.
|
||||||
|
|
||||||
|
Narratio owns orchestration, configuration loading, session/run state, local and remote path modeling, manifest persistence, stage sequencing, resume behavior, and publish semantics.
|
||||||
|
|
||||||
|
Narratio should remain explicit and comprehensible. It is not intended to become a generic workflow engine.
|
||||||
|
|
||||||
|
## Core Principles
|
||||||
|
|
||||||
|
### Modular and composable
|
||||||
|
|
||||||
|
Code should be organized around clear responsibilities. Stages, adapters, config loading, manifest persistence, path construction, and storage behavior should remain separable and independently testable.
|
||||||
|
|
||||||
|
### Hexagonal boundaries
|
||||||
|
|
||||||
|
External systems should be isolated behind narrow adapters. Stage logic should depend on Narratio-level interfaces and data structures, not on external SDK types, subprocess argument construction, or transport-specific details.
|
||||||
|
|
||||||
|
### Standard library preference
|
||||||
|
|
||||||
|
Prefer the Go standard library. Add dependencies only when they provide substantial value, are necessary for an external integration, or are a widely used de facto standard.
|
||||||
|
|
||||||
|
Accepted examples include a YAML library for configuration and the AWS SDK for S3-compatible storage.
|
||||||
|
|
||||||
|
### Explicit orchestration
|
||||||
|
|
||||||
|
The pipeline should remain stage-driven and explicit. New behavior should be added through clear stage, adapter, config, or manifest contracts rather than implicit side effects or generic workflow abstraction.
|
||||||
|
|
||||||
|
## Stage Design
|
||||||
|
|
||||||
|
Each stage should have a clear scope of responsibility.
|
||||||
|
|
||||||
|
A stage should define:
|
||||||
|
|
||||||
|
- its purpose;
|
||||||
|
- required input state;
|
||||||
|
- produced output state;
|
||||||
|
- config fields it consumes;
|
||||||
|
- external adapters it uses;
|
||||||
|
- manifest refs it reads or writes;
|
||||||
|
- skip, force, and resume behavior;
|
||||||
|
- failure behavior;
|
||||||
|
- tests that protect its contract.
|
||||||
|
|
||||||
|
Stages should avoid reaching across boundaries. If shared behavior is needed, prefer a helper or service with a narrow interface over duplicating ad hoc logic between stages.
|
||||||
|
|
||||||
|
## Transactionality and Resume
|
||||||
|
|
||||||
|
A stage should behave transactionally.
|
||||||
|
|
||||||
|
A stage is complete only when its outputs have been written, validated, and recorded in the manifest. If a stage fails, Narratio should preserve enough local state for inspection, recovery, and resume.
|
||||||
|
|
||||||
|
A failed or incomplete run must not be treated as successful. Later stages should depend on manifest-recorded success, not merely on incidental files existing on disk.
|
||||||
|
|
||||||
|
## Manifest Model
|
||||||
|
|
||||||
|
The manifest is the durable local ledger for a run.
|
||||||
|
|
||||||
|
It should record:
|
||||||
|
|
||||||
|
- run identity;
|
||||||
|
- stage status;
|
||||||
|
- input and output refs;
|
||||||
|
- logs and generated config refs;
|
||||||
|
- checksums or provenance where useful;
|
||||||
|
- non-secret adapter and publish metadata.
|
||||||
|
|
||||||
|
Resume behavior should be manifest-driven. Filesystem state may be inspected and validated, but it should not replace manifest stage state as the source of run progress.
|
||||||
|
|
||||||
|
## Adapter Boundaries
|
||||||
|
|
||||||
|
Adapters own external integration details.
|
||||||
|
|
||||||
|
Expected boundaries:
|
||||||
|
|
||||||
|
- WhisperX HTTP details stay in the WhisperX adapter.
|
||||||
|
- Seriatim CLI construction stays in the Seriatim adapter.
|
||||||
|
- Audita CLI construction stays in the Audita adapter.
|
||||||
|
- Scriptorium CLI construction stays in the Scriptorium adapter.
|
||||||
|
- Object-storage details stay behind the storage adapter interface.
|
||||||
|
- AWS SDK types stay inside the S3 storage implementation.
|
||||||
|
|
||||||
|
Stage code should express intent in Narratio terms and call adapters through narrow contracts.
|
||||||
|
|
||||||
|
## Configuration Philosophy
|
||||||
|
|
||||||
|
Configuration should be strict, explicit, and operator-friendly.
|
||||||
|
|
||||||
|
Principles:
|
||||||
|
|
||||||
|
- YAML decoding should reject unknown fields.
|
||||||
|
- Defaults should be centralized and testable.
|
||||||
|
- Empty configured values should not silently override meaningful defaults.
|
||||||
|
- Session templating should remain narrow and deterministic.
|
||||||
|
- Template support should serve operator convenience, not become a general configuration language.
|
||||||
|
|
||||||
|
Narratio should not become a secondary configuration system for downstream tools. Seriatim, Audita, and Scriptorium should own their runtime defaults wherever practical. Narratio should pass required stage-contract paths and explicit operator overrides.
|
||||||
|
|
||||||
|
## Path and Storage Discipline
|
||||||
|
|
||||||
|
Local and remote paths are part of Narratio’s application contract.
|
||||||
|
|
||||||
|
Code should use centralized path helpers for workspace, spool, session, run, artifact, log, config, and publish/current paths. Stages should avoid reconstructing canonical paths through scattered string concatenation.
|
||||||
|
|
||||||
|
Storage backends should receive explicit bucket-relative keys. Storage implementations should not infer campaign, session, run, or root-prefix semantics.
|
||||||
|
|
||||||
|
## Publish Invariants
|
||||||
|
|
||||||
|
Publish behavior must preserve a clear commit boundary.
|
||||||
|
|
||||||
|
A remote run is current only after the publish stage has successfully uploaded the run record, required published outputs, `current/manifest.json`, and finally `current/run_id.txt`.
|
||||||
|
|
||||||
|
`current/run_id.txt` is the final remote commit marker and must be written last.
|
||||||
|
|
||||||
|
Failed, incomplete, skipped, or uncommitted publish attempts must not be presented as current remote state. Local cleanup is permitted only after successful publish commit and only when explicitly configured.
|
||||||
|
|
||||||
|
## Security and Privacy
|
||||||
|
|
||||||
|
Narratio handles private campaign material.
|
||||||
|
|
||||||
|
Rules:
|
||||||
|
|
||||||
|
- Do not store raw secrets in pipeline or session YAML.
|
||||||
|
- Use environment variable names or secret-file references for secret handling.
|
||||||
|
- Do not write raw secret values to manifests, logs, generated configs, or publish metadata.
|
||||||
|
- Treat transcripts, generated artifacts, prompts, reports, and logs as potentially sensitive.
|
||||||
|
- Avoid logging transcript or prompt content unless there is a deliberate diagnostic reason.
|
||||||
|
|
||||||
|
## Diagnostics
|
||||||
|
|
||||||
|
Diagnostics should be durable and discoverable, but distinct from canonical outputs.
|
||||||
|
|
||||||
|
Logs, reports, generated invocation/config files, and render-debug files support debugging. Transcript tiers and configured artifacts are pipeline products.
|
||||||
|
|
||||||
|
Manifest refs should preserve that distinction.
|
||||||
|
|
||||||
|
## Determinism
|
||||||
|
|
||||||
|
Where practical, Narratio should prefer deterministic behavior:
|
||||||
|
|
||||||
|
- stable local path layout;
|
||||||
|
- stable remote key layout;
|
||||||
|
- sorted upload order;
|
||||||
|
- predictable generated config files;
|
||||||
|
- repeatable command construction;
|
||||||
|
- tests that do not depend on live external services.
|
||||||
|
|
||||||
|
Run IDs and timestamps may be intentionally variable, but surrounding behavior should remain testable.
|
||||||
|
|
||||||
|
## Testing Expectations
|
||||||
|
|
||||||
|
Core behavior should be testable without live external services.
|
||||||
|
|
||||||
|
Tests should cover:
|
||||||
|
|
||||||
|
- config loading, defaults, and validation;
|
||||||
|
- CLI parsing and command construction;
|
||||||
|
- path helpers;
|
||||||
|
- manifest transitions;
|
||||||
|
- stage success, failure, skip, and resume behavior;
|
||||||
|
- adapter command construction;
|
||||||
|
- fake storage behavior;
|
||||||
|
- publish commit ordering;
|
||||||
|
- example config validity where practical.
|
||||||
|
|
||||||
|
Live S3, WhisperX, LLM, or subprocess integration tests should be explicit integration tests, not required for ordinary unit test runs.
|
||||||
|
|
||||||
|
## Documentation Expectations
|
||||||
|
|
||||||
|
Documentation must follow `docs/documentation/policy.md`.
|
||||||
|
|
||||||
|
Current behavior belongs in user-facing docs and `docs/internal/`. Future, planned, aspirational, experimental, or unimplemented work belongs only under `docs/roadmap/`.
|
||||||
|
|
||||||
|
`docs/architecture.md` should remain concise and principle-focused. It should not duplicate the full config reference, CLI reference, operations guide, or internal stage documentation.
|
||||||
|
|
||||||
|
## Non-Goals
|
||||||
|
|
||||||
|
Narratio is not:
|
||||||
|
|
||||||
|
- a generic DAG or workflow engine;
|
||||||
|
- a replacement configuration layer for Seriatim, Audita, or Scriptorium;
|
||||||
|
- a storage backend abstraction beyond the needs of this pipeline;
|
||||||
|
- a place to embed raw secrets;
|
||||||
|
- a place for stage logic to depend directly on AWS SDK types or downstream tool internals;
|
||||||
|
- a prompt-authoring system.
|
||||||
94
docs/policy/development.md
Normal file
94
docs/policy/development.md
Normal file
@@ -0,0 +1,94 @@
|
|||||||
|
# Development Guide
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
Canonical contributor workflow and engineering conventions for implemented Narratio behavior.
|
||||||
|
|
||||||
|
## Repository layout
|
||||||
|
|
||||||
|
- `cmd/narratio/`: CLI entrypoint.
|
||||||
|
- `internal/app/`: command handlers, run/stage orchestration, cleanup gates, secrets loading.
|
||||||
|
- `internal/config/`: strict YAML loading, defaults, and validation.
|
||||||
|
- `internal/stage/`: stage implementations and stage registry/order.
|
||||||
|
- `internal/adapters/`: external boundary adapters (WhisperX, Seriatim, Audita, Scriptorium, storage, notify).
|
||||||
|
- `internal/manifest/`: session/run manifest types and persistence.
|
||||||
|
- `internal/artifacts/`: canonical local/remote path helpers and local artifact store.
|
||||||
|
- `docs/`: canonical documentation set.
|
||||||
|
- `examples/`: maintained config examples used by tests.
|
||||||
|
|
||||||
|
## Build and test commands
|
||||||
|
|
||||||
|
- Run focused CLI behavior checks:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
go test ./internal/app -run TestExecute -v
|
||||||
|
```
|
||||||
|
|
||||||
|
- Run config example load/validate checks:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
go test ./internal/config -run TestExamplesLoadAndValidate -v
|
||||||
|
```
|
||||||
|
|
||||||
|
- Run full test suite:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
go test ./...
|
||||||
|
```
|
||||||
|
|
||||||
|
## Coding conventions
|
||||||
|
|
||||||
|
- Keep orchestration explicit and stage-driven; do not introduce generic workflow/DAG abstractions.
|
||||||
|
- Keep external-system details inside adapter packages; stages should consume Narratio-level contracts only.
|
||||||
|
- Use centralized path helpers from `internal/artifacts` rather than ad hoc path concatenation.
|
||||||
|
- Preserve manifest-driven state transitions (`running`, `succeeded`, `failed`, `skipped`, `stale`) as the source of run progress.
|
||||||
|
- Keep user/operator docs implementation-accurate; planned work belongs only under `docs/roadmap/`.
|
||||||
|
|
||||||
|
For design principles and invariants, see [docs/architecture.md](./architecture.md). For stage/adapter contracts, see [docs/internal/README.md](./internal/README.md).
|
||||||
|
|
||||||
|
## Dependency policy
|
||||||
|
|
||||||
|
- Prefer Go standard library where practical.
|
||||||
|
- Add third-party dependencies only when they provide clear value for required behavior.
|
||||||
|
- Keep dependency additions narrow to the boundary package that needs them.
|
||||||
|
|
||||||
|
## Change playbooks
|
||||||
|
|
||||||
|
### Add config fields
|
||||||
|
|
||||||
|
1. Add fields to config structs in `internal/config`.
|
||||||
|
2. Set defaults in `internal/config/defaults.go` when appropriate.
|
||||||
|
3. Add validation rules in `internal/config/validate.go`.
|
||||||
|
4. Add or update load/validate tests in `internal/config/*_test.go`.
|
||||||
|
5. Update canonical config docs and examples:
|
||||||
|
- [docs/config.md](./config.md)
|
||||||
|
- relevant files under `examples/`
|
||||||
|
|
||||||
|
### Add CLI flags or commands
|
||||||
|
|
||||||
|
1. Update command parsing and behavior in `internal/app`.
|
||||||
|
2. Add or update command tests (`TestExecute` and command-specific tests).
|
||||||
|
3. Update [docs/cli.md](./cli.md) and, if operator workflow changes, [docs/operations.md](./operations.md).
|
||||||
|
|
||||||
|
Remote-storage commands must obtain object storage through the app-level command object-store helper. Do not call `storage.NewObjectStoreFromConfig` directly from command handlers; the helper loads configured filesystem secrets before constructing the storage adapter.
|
||||||
|
|
||||||
|
### Add or modify stages/adapters
|
||||||
|
|
||||||
|
1. Implement stage behavior in `internal/stage` with clear input/output boundaries.
|
||||||
|
2. Keep external transport/subprocess details in `internal/adapters`.
|
||||||
|
3. Preserve manifest and publish-output semantics expected by runner and publish logic.
|
||||||
|
4. Add/update stage and adapter tests.
|
||||||
|
5. Update internal component contracts in `docs/internal/`.
|
||||||
|
|
||||||
|
### Update examples
|
||||||
|
|
||||||
|
1. Keep canonical examples only in `examples/`.
|
||||||
|
2. Ensure examples load and validate through runtime config paths.
|
||||||
|
3. Update `internal/config/load_validate_test.go` as needed.
|
||||||
|
4. Update links in `docs/config.md` if example filenames change.
|
||||||
|
|
||||||
|
### Update docs and roadmap
|
||||||
|
|
||||||
|
1. Keep implemented behavior in canonical docs (`README`, `docs/*.md`, `docs/internal/`).
|
||||||
|
2. Keep planned/unimplemented behavior only in `docs/roadmap/`.
|
||||||
|
3. After completing roadmap items, remove or mark them complete in `docs/roadmap/documentation.md`.
|
||||||
|
4. Run a link/path sweep before finalizing changes.
|
||||||
356
docs/policy/documentation.md
Normal file
356
docs/policy/documentation.md
Normal file
@@ -0,0 +1,356 @@
|
|||||||
|
# Go Project Documentation Policy
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
Project documentation must help four audiences:
|
||||||
|
|
||||||
|
1. users who need to run the application;
|
||||||
|
2. administrators/operators who need to configure and operate it;
|
||||||
|
3. developers who need to understand and change it safely;
|
||||||
|
4. LLM coding agents that need clear scope, boundaries, and invariants.
|
||||||
|
|
||||||
|
Docs should be accurate, concise, task-oriented, and organized by audience. Prefer links to canonical docs over repetition.
|
||||||
|
|
||||||
|
## Core Rules
|
||||||
|
|
||||||
|
### 1. Keep docs concise
|
||||||
|
|
||||||
|
Each document should cover a defined scope and only the essentials for that scope.
|
||||||
|
|
||||||
|
Avoid:
|
||||||
|
- long background explanations;
|
||||||
|
- repeated reference material;
|
||||||
|
- implementation detail in user-facing docs;
|
||||||
|
- aspirational language outside roadmap docs;
|
||||||
|
- verbose examples where one minimal example is clearer.
|
||||||
|
|
||||||
|
### 2. Document only implemented behavior outside roadmap files
|
||||||
|
|
||||||
|
Unimplemented, planned, aspirational, experimental, or future work may be described only under:
|
||||||
|
|
||||||
|
- `docs/roadmap/`
|
||||||
|
|
||||||
|
No other documentation file, including `README.md`, should describe code, features, modules, stages, commands, config fields, or behaviors that do not currently exist.
|
||||||
|
|
||||||
|
If a feature is partial, non-roadmap docs may describe only the implemented portion and its current boundary.
|
||||||
|
|
||||||
|
### 3. Use canonical homes
|
||||||
|
|
||||||
|
Each type of information should have one canonical location.
|
||||||
|
|
||||||
|
Canonical homes:
|
||||||
|
|
||||||
|
- project purpose and quickstart: `README.md`
|
||||||
|
- development principles: `docs/architecture.md`
|
||||||
|
- configuration reference: `docs/config.md`
|
||||||
|
- CLI reference: `docs/cli.md`
|
||||||
|
- operations and recovery: `docs/operations.md`
|
||||||
|
- troubleshooting: `docs/troubleshooting.md`
|
||||||
|
- implemented internals: `docs/internal/`
|
||||||
|
- future work: `docs/roadmap/`
|
||||||
|
- contributor workflow: `docs/development.md`
|
||||||
|
- copyable examples: `examples/`
|
||||||
|
|
||||||
|
Other files should summarize briefly and link to the canonical source.
|
||||||
|
|
||||||
|
### 4. Keep examples real
|
||||||
|
|
||||||
|
Examples should be valid, maintained, and free of secrets.
|
||||||
|
|
||||||
|
Where practical:
|
||||||
|
- example configs should load successfully;
|
||||||
|
- example commands should match real CLI syntax;
|
||||||
|
- important examples should be covered by tests.
|
||||||
|
|
||||||
|
## Documentation Profiles
|
||||||
|
|
||||||
|
All projects require:
|
||||||
|
|
||||||
|
- `README.md`
|
||||||
|
- `docs/architecture.md`
|
||||||
|
|
||||||
|
Additional docs depend on the project.
|
||||||
|
|
||||||
|
### Small library
|
||||||
|
|
||||||
|
Recommended:
|
||||||
|
- `docs/development.md`, if contributor conventions are non-obvious
|
||||||
|
|
||||||
|
### Simple CLI
|
||||||
|
|
||||||
|
Required:
|
||||||
|
- `docs/cli.md`
|
||||||
|
|
||||||
|
Recommended:
|
||||||
|
- `docs/development.md`
|
||||||
|
|
||||||
|
### Config-driven CLI
|
||||||
|
|
||||||
|
Required:
|
||||||
|
- `docs/cli.md`
|
||||||
|
- `docs/config.md`
|
||||||
|
|
||||||
|
Recommended:
|
||||||
|
- `examples/`
|
||||||
|
- `docs/development.md`
|
||||||
|
|
||||||
|
### Stateful or operator-facing application
|
||||||
|
|
||||||
|
Required:
|
||||||
|
- `docs/cli.md`, if CLI-based
|
||||||
|
- `docs/config.md`, if config-driven
|
||||||
|
- `docs/operations.md`
|
||||||
|
|
||||||
|
Recommended:
|
||||||
|
- `docs/troubleshooting.md`
|
||||||
|
- `examples/`
|
||||||
|
- `docs/development.md`
|
||||||
|
|
||||||
|
### Modular, staged, service-oriented, or orchestration application
|
||||||
|
|
||||||
|
Required:
|
||||||
|
- `docs/cli.md`, if CLI-based
|
||||||
|
- `docs/config.md`, if config-driven
|
||||||
|
- `docs/operations.md`
|
||||||
|
- `docs/internal/`
|
||||||
|
- `docs/development.md`
|
||||||
|
|
||||||
|
Recommended:
|
||||||
|
- `docs/troubleshooting.md`
|
||||||
|
- validated examples under `examples/`
|
||||||
|
|
||||||
|
## Required Documents
|
||||||
|
|
||||||
|
### README.md
|
||||||
|
|
||||||
|
**Audience:** users, administrators, operators
|
||||||
|
|
||||||
|
The README is the outward-facing project orientation page.
|
||||||
|
|
||||||
|
It should include, in order:
|
||||||
|
|
||||||
|
1. concise description;
|
||||||
|
2. elevator pitch;
|
||||||
|
3. shortest useful command or usage example;
|
||||||
|
4. links to targeted docs.
|
||||||
|
|
||||||
|
The README should be short. It is not a manual.
|
||||||
|
|
||||||
|
The “shortest useful command” means the simplest command that performs the project’s core use case. (It does not mean `app --help`.)
|
||||||
|
|
||||||
|
### docs/architecture.md
|
||||||
|
|
||||||
|
**Audience:** developers, LLM coding agents
|
||||||
|
|
||||||
|
`docs/architecture.md` is required for every project.
|
||||||
|
|
||||||
|
It is an inward-facing development policy document. It should describe how the project is intended to be built and changed.
|
||||||
|
|
||||||
|
It should include:
|
||||||
|
|
||||||
|
- project shape;
|
||||||
|
- core design principles;
|
||||||
|
- package and boundary philosophy;
|
||||||
|
- state/persistence philosophy, if applicable;
|
||||||
|
- external integration philosophy, if applicable;
|
||||||
|
- error-handling and logging principles;
|
||||||
|
- testing expectations;
|
||||||
|
- documentation expectations;
|
||||||
|
- architectural invariants;
|
||||||
|
- explicit non-goals, if useful.
|
||||||
|
|
||||||
|
For small projects, this file may be brief. It may simply state that the project is intentionally narrow, monolithic, and dependency-light.
|
||||||
|
|
||||||
|
### docs/config.md
|
||||||
|
|
||||||
|
**Audience:** administrators, operators, advanced users
|
||||||
|
|
||||||
|
Required for applications with configuration files.
|
||||||
|
|
||||||
|
It should include, in order:
|
||||||
|
|
||||||
|
1. config file locations and discovery precedence;
|
||||||
|
2. minimal working config;
|
||||||
|
3. production-oriented config;
|
||||||
|
4. full configuration reference;
|
||||||
|
5. secrets handling, if applicable;
|
||||||
|
6. links to maintained examples.
|
||||||
|
|
||||||
|
The full configuration reference should be canonical.
|
||||||
|
|
||||||
|
### docs/cli.md
|
||||||
|
|
||||||
|
**Audience:** users, administrators, operators
|
||||||
|
|
||||||
|
Required for CLI applications.
|
||||||
|
|
||||||
|
It should include, in order:
|
||||||
|
|
||||||
|
1. shortest useful command;
|
||||||
|
2. command overview;
|
||||||
|
3. complete flag reference;
|
||||||
|
4. common workflows;
|
||||||
|
5. diagnostic or recovery commands, if applicable.
|
||||||
|
|
||||||
|
Explain when commands are useful, not just their syntax.
|
||||||
|
|
||||||
|
### docs/operations.md
|
||||||
|
|
||||||
|
**Audience:** administrators, operators
|
||||||
|
|
||||||
|
Required for applications that maintain state, support resume behavior, run multiple stages, write durable artifacts, use remote storage, or require recovery procedures.
|
||||||
|
|
||||||
|
It should cover:
|
||||||
|
|
||||||
|
- normal workflow;
|
||||||
|
- filesystem layout;
|
||||||
|
- remote storage layout, if applicable;
|
||||||
|
- logs and manifests;
|
||||||
|
- resume/retry behavior;
|
||||||
|
- cleanup behavior;
|
||||||
|
- archive/backup behavior;
|
||||||
|
- safe recovery procedures;
|
||||||
|
- operational caveats.
|
||||||
|
|
||||||
|
### docs/troubleshooting.md
|
||||||
|
|
||||||
|
**Audience:** administrators, operators
|
||||||
|
|
||||||
|
Recommended once recurring failure modes exist.
|
||||||
|
|
||||||
|
Each entry should include:
|
||||||
|
|
||||||
|
- symptom;
|
||||||
|
- likely cause;
|
||||||
|
- diagnostic command or inspection step;
|
||||||
|
- safe fix;
|
||||||
|
- relevant links.
|
||||||
|
|
||||||
|
### docs/development.md
|
||||||
|
|
||||||
|
**Audience:** developers, LLM coding agents
|
||||||
|
|
||||||
|
Required for projects maintained by humans and LLM coding agents.
|
||||||
|
|
||||||
|
It should include:
|
||||||
|
|
||||||
|
- repository layout;
|
||||||
|
- build/test commands;
|
||||||
|
- coding conventions;
|
||||||
|
- dependency policy;
|
||||||
|
- how to add config fields;
|
||||||
|
- how to add CLI flags;
|
||||||
|
- how to add stages/modules/adapters, if applicable;
|
||||||
|
- how to update examples;
|
||||||
|
- documentation update expectations.
|
||||||
|
|
||||||
|
### docs/internal/
|
||||||
|
|
||||||
|
**Audience:** developers, LLM coding agents
|
||||||
|
|
||||||
|
Required for modular, staged, service-oriented, or orchestration projects.
|
||||||
|
|
||||||
|
This directory describes implemented internal components. It is not the roadmap.
|
||||||
|
|
||||||
|
Use one file per major component where useful.
|
||||||
|
|
||||||
|
Each component doc should include:
|
||||||
|
|
||||||
|
1. purpose;
|
||||||
|
2. inputs and outputs;
|
||||||
|
3. boundaries;
|
||||||
|
4. config fields used;
|
||||||
|
5. external adapters used;
|
||||||
|
6. state or manifest behavior, if applicable;
|
||||||
|
7. skip/resume behavior, if applicable;
|
||||||
|
8. failure behavior;
|
||||||
|
9. tests to inspect before changing;
|
||||||
|
10. architectural invariants.
|
||||||
|
|
||||||
|
### docs/roadmap/
|
||||||
|
|
||||||
|
**Audience:** maintainers, developers, LLM coding agents
|
||||||
|
|
||||||
|
This is the only place for planned, future, aspirational, experimental, or unimplemented work.
|
||||||
|
|
||||||
|
Roadmap docs should clearly distinguish:
|
||||||
|
|
||||||
|
- proposed work;
|
||||||
|
- accepted plans;
|
||||||
|
- deferred ideas;
|
||||||
|
- rejected ideas;
|
||||||
|
- implementation prompts or task breakdowns, if useful.
|
||||||
|
|
||||||
|
Roadmap docs should not be confused with current behavior.
|
||||||
|
|
||||||
|
### docs/integrations/
|
||||||
|
|
||||||
|
**Audience:** developers, LLM coding agents
|
||||||
|
|
||||||
|
Required for projects that depend on external CLIs, APIs, services, protocols, or file formats where the integration contract is important to maintain.
|
||||||
|
|
||||||
|
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses.
|
||||||
|
|
||||||
|
Use one file per integration where useful.
|
||||||
|
|
||||||
|
## Examples Directory
|
||||||
|
|
||||||
|
Projects with non-trivial configuration or workflows should include `examples/`.
|
||||||
|
|
||||||
|
Useful examples include:
|
||||||
|
|
||||||
|
- minimal working config;
|
||||||
|
- production-oriented config;
|
||||||
|
- full annotated config;
|
||||||
|
- local development config;
|
||||||
|
- remote/object-storage config;
|
||||||
|
- minimal session/input file.
|
||||||
|
|
||||||
|
Examples should be valid, maintained, tested when practical, and linked from relevant docs.
|
||||||
|
|
||||||
|
## Security and Privacy
|
||||||
|
|
||||||
|
Docs and examples must not include:
|
||||||
|
|
||||||
|
- real API keys;
|
||||||
|
- tokens;
|
||||||
|
- passwords;
|
||||||
|
- private keys;
|
||||||
|
- private environment dumps;
|
||||||
|
- sensitive user data;
|
||||||
|
- raw private transcripts;
|
||||||
|
- private infrastructure details unless intentionally public.
|
||||||
|
|
||||||
|
Document secret-handling mechanisms, not actual secret values.
|
||||||
|
|
||||||
|
## Maintenance Rules
|
||||||
|
|
||||||
|
When docs change, verify the affected behavior.
|
||||||
|
|
||||||
|
Where practical:
|
||||||
|
|
||||||
|
- load example config files in tests;
|
||||||
|
- test CLI examples or command parser behavior;
|
||||||
|
- validate documented flags against real flags;
|
||||||
|
- remove stale references;
|
||||||
|
- update links after renames;
|
||||||
|
- keep roadmap content out of non-roadmap docs.
|
||||||
|
|
||||||
|
If documentation and code disagree, fix the documentation and/or open a roadmap item; do not leave aspirational behavior in current-behavior docs.
|
||||||
|
|
||||||
|
Documentation is complete only when it matches the current code.
|
||||||
|
|
||||||
|
## Documentation Change Checklist
|
||||||
|
|
||||||
|
Before merging documentation changes, verify:
|
||||||
|
|
||||||
|
- README is concise and orientation-focused.
|
||||||
|
- `docs/architecture.md` describes development principles.
|
||||||
|
- Future work appears only under `docs/roadmap/`.
|
||||||
|
- User-facing docs avoid unnecessary internals.
|
||||||
|
- Developer-facing docs preserve boundaries and invariants.
|
||||||
|
- Config examples match the schema.
|
||||||
|
- CLI examples match real commands and flags.
|
||||||
|
- Defaults appear in the canonical config reference.
|
||||||
|
- No secrets or private data are included.
|
||||||
|
- Links are accurate.
|
||||||
300
docs/troubleshooting.md
Normal file
300
docs/troubleshooting.md
Normal file
@@ -0,0 +1,300 @@
|
|||||||
|
# Troubleshooting
|
||||||
|
|
||||||
|
Operational diagnosis guide for common Narratio failures.
|
||||||
|
|
||||||
|
## Config file not found
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- command fails to resolve `pipeline.yml`, `campaign.yml`, or `session.yml`.
|
||||||
|
|
||||||
|
Likely causes:
|
||||||
|
|
||||||
|
- missing files in default search paths;
|
||||||
|
- wrong campaign selection;
|
||||||
|
- omitted explicit flags.
|
||||||
|
|
||||||
|
Diagnostics:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio session plan 2026-04-04
|
||||||
|
```
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- pass explicit `--config`, `--campaign` or `--campaign-file`, and `--session`.
|
||||||
|
|
||||||
|
## Session template placeholders rejected
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- load error says session file must be concrete or contains `{{ ... }}` placeholders.
|
||||||
|
|
||||||
|
Likely cause:
|
||||||
|
|
||||||
|
- using template content as runtime session config.
|
||||||
|
|
||||||
|
Diagnostics:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio session validate 2026-04-04 --session /path/session.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- generate concrete session YAML with `narratio session init`.
|
||||||
|
|
||||||
|
## Strict decode or schema validation failure
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- unknown field / invalid value error during config load.
|
||||||
|
|
||||||
|
Likely cause:
|
||||||
|
|
||||||
|
- stale field name, typo, invalid enum, or invalid duration/path format.
|
||||||
|
|
||||||
|
Diagnostics:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio session plan 2026-04-04 --config /path/pipeline.yml --campaign-file /path/campaign.yml --session /path/session.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- align config with [docs/config.md](./config.md) and maintained files under `examples/`.
|
||||||
|
|
||||||
|
## Audio mode conflict
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- validation fails on session audio configuration.
|
||||||
|
|
||||||
|
Likely cause:
|
||||||
|
|
||||||
|
- configured both local and S3 session audio inputs.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- use local mode (`audio_dir` or `audio_files`) or S3 mode (`audio_s3.prefix`), not both.
|
||||||
|
|
||||||
|
## `--artifacts` selection error
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- unknown artifact key or invalid `--artifacts` usage.
|
||||||
|
|
||||||
|
Likely causes:
|
||||||
|
|
||||||
|
- key not defined in `pipeline.scriptorium.artifacts`;
|
||||||
|
- empty list entry (for example trailing comma);
|
||||||
|
- `run-stage` used with non-`analyze`/`publish` target.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- provide only configured keys and use `--artifacts` with supported commands/stages.
|
||||||
|
|
||||||
|
## Previous-session artifact input missing
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- prepare/analyze fails due to missing required previous-session artifact cache input.
|
||||||
|
|
||||||
|
Likely causes:
|
||||||
|
|
||||||
|
- missing `session.previous_session_id`;
|
||||||
|
- previous artifact not restored/published for source session.
|
||||||
|
|
||||||
|
Diagnostics:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio session validate 2026-04-04
|
||||||
|
narratio session status 2026-04-04
|
||||||
|
```
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio session restore 2026-04-04
|
||||||
|
```
|
||||||
|
|
||||||
|
or rerun prepare after correcting session config:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio run-stage prepare 2026-04-04 --force
|
||||||
|
```
|
||||||
|
|
||||||
|
## Session lock conflict (`.lock`)
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- command fails acquiring session lock.
|
||||||
|
|
||||||
|
Likely causes:
|
||||||
|
|
||||||
|
- another process is running for the same session;
|
||||||
|
- stale lock left by interrupted process.
|
||||||
|
|
||||||
|
Diagnostics:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
ls -l {workspace.root}/work/{campaign}/{session_id}/.lock
|
||||||
|
ps aux | grep narratio
|
||||||
|
```
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- wait for active process completion;
|
||||||
|
- remove stale lock only after confirming no live process owns it.
|
||||||
|
|
||||||
|
## Restore conflict without `--force`
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- restore fails with conflict count.
|
||||||
|
|
||||||
|
Likely cause:
|
||||||
|
|
||||||
|
- local durable files differ from remote restore sources.
|
||||||
|
|
||||||
|
Diagnostics:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio session restore 2026-04-04 --dry-run
|
||||||
|
```
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- review conflicts;
|
||||||
|
- rerun with `--force` only when remote state should overwrite local.
|
||||||
|
|
||||||
|
## Restore current-state discovery failure
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- restore cannot find current pointer or current manifest.
|
||||||
|
|
||||||
|
Likely causes:
|
||||||
|
|
||||||
|
- no committed publish current state;
|
||||||
|
- storage credentials or connectivity failure.
|
||||||
|
|
||||||
|
Diagnostics:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio session status 2026-04-04
|
||||||
|
narratio session restore 2026-04-04 --dry-run
|
||||||
|
```
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- resolve storage/auth issue;
|
||||||
|
- republish from healthy local state if current pointer is missing.
|
||||||
|
|
||||||
|
## Publish output failure
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- publish fails on missing required source, upload error, or commit write.
|
||||||
|
|
||||||
|
Likely causes:
|
||||||
|
|
||||||
|
- required source file not produced;
|
||||||
|
- lock/state expectations mismatch;
|
||||||
|
- remote storage failure.
|
||||||
|
|
||||||
|
Diagnostics:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio session artifacts 2026-04-04 --remote
|
||||||
|
narratio session status 2026-04-04
|
||||||
|
narratio run-stage publish 2026-04-04 --force
|
||||||
|
```
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- regenerate missing sources by rerunning prerequisite stages;
|
||||||
|
- correct publish source/destination rules;
|
||||||
|
- retry after storage failure is resolved.
|
||||||
|
|
||||||
|
## Render markdown source missing
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- analyze or publish fails because `narratio.transcript.final_markdown` or `narratio.transcript.final_trimmed_markdown` is unavailable.
|
||||||
|
|
||||||
|
Likely causes:
|
||||||
|
|
||||||
|
- render stage was not executed after transcript changes;
|
||||||
|
- render stage failed before producing canonical markdown outputs.
|
||||||
|
|
||||||
|
Diagnostics:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio session status 2026-04-04
|
||||||
|
narratio run-stage render 2026-04-04 --force
|
||||||
|
```
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- rerun render and then retry downstream stage(s):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio run-stage render 2026-04-04 --force
|
||||||
|
narratio run-stage analyze 2026-04-04 --force
|
||||||
|
```
|
||||||
|
|
||||||
|
## Secrets or storage credential failure
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- object-store command fails at initialization/auth.
|
||||||
|
|
||||||
|
Likely causes:
|
||||||
|
|
||||||
|
- invalid `pipeline.secrets.env_dir`;
|
||||||
|
- missing credential environment variables;
|
||||||
|
- invalid S3 endpoint/bucket settings.
|
||||||
|
|
||||||
|
Diagnostics:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
ls -la /path/to/secrets_dir
|
||||||
|
env | grep -E 'OBJECT_STORAGE|AWS|AUDITA|SCRIPTORIUM'
|
||||||
|
```
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- correct secret-file path and permissions;
|
||||||
|
- provide required env vars;
|
||||||
|
- keep secret values out of YAML.
|
||||||
|
|
||||||
|
## S3 audio prepare failure
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- prepare fails listing/downloading session S3 audio.
|
||||||
|
|
||||||
|
Likely causes:
|
||||||
|
|
||||||
|
- incorrect `session.inputs.audio_s3.prefix`;
|
||||||
|
- no matching `.flac` objects;
|
||||||
|
- storage connectivity or permissions failure.
|
||||||
|
|
||||||
|
Diagnostics:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio run-stage prepare 2026-04-04 --force
|
||||||
|
```
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- verify prefix contents and storage access;
|
||||||
|
- keep session audio mode consistent.
|
||||||
|
|
||||||
|
## References
|
||||||
|
|
||||||
|
- [docs/cli.md](./cli.md)
|
||||||
|
- [docs/config.md](./config.md)
|
||||||
|
- [docs/operations.md](./operations.md)
|
||||||
|
- [docs/internal/stage-publish.md](./internal/stage-publish.md)
|
||||||
1
examples/campaigns/sample-campaign/autocorrect.yml
Normal file
1
examples/campaigns/sample-campaign/autocorrect.yml
Normal file
@@ -0,0 +1 @@
|
|||||||
|
[]
|
||||||
8
examples/campaigns/sample-campaign/campaign.yml
Normal file
8
examples/campaigns/sample-campaign/campaign.yml
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
campaign_id: sample-campaign
|
||||||
|
session_template_file: ./session.template.yml
|
||||||
|
inputs:
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
players_file: ./players.yml
|
||||||
|
party_file: ./party.yml
|
||||||
1
examples/campaigns/sample-campaign/glossary.yml
Normal file
1
examples/campaigns/sample-campaign/glossary.yml
Normal file
@@ -0,0 +1 @@
|
|||||||
|
[]
|
||||||
2
examples/campaigns/sample-campaign/party.yml
Normal file
2
examples/campaigns/sample-campaign/party.yml
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
- name: Example Hero
|
||||||
|
type: pc
|
||||||
2
examples/campaigns/sample-campaign/players.yml
Normal file
2
examples/campaigns/sample-campaign/players.yml
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
- name: Example Player
|
||||||
|
role: player
|
||||||
3
examples/campaigns/sample-campaign/session.template.yml
Normal file
3
examples/campaigns/sample-campaign/session.template.yml
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
session_id: "{{ session_id }}"
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
5
examples/campaigns/sample-campaign/speakers.yml
Normal file
5
examples/campaigns/sample-campaign/speakers.yml
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
match:
|
||||||
|
- speaker: "Eric Rakestraw"
|
||||||
|
match:
|
||||||
|
- "Eric_Rakestraw"
|
||||||
|
- "Eric"
|
||||||
189
examples/pipeline.full.annotated.yml
Normal file
189
examples/pipeline.full.annotated.yml
Normal file
@@ -0,0 +1,189 @@
|
|||||||
|
# Full annotated pipeline example for implemented Narratio config fields.
|
||||||
|
# Values are safe placeholders and must be adapted per environment.
|
||||||
|
|
||||||
|
workspace:
|
||||||
|
# Optional: defaults to /var/lib/narratio.
|
||||||
|
root: /var/lib/narratio/workspace
|
||||||
|
# Optional: remove run-scoped workdir after successful publish commit.
|
||||||
|
cleanup_after_publish: false
|
||||||
|
|
||||||
|
# Optional: local secret file loader (directory of ENV_VAR_NAME files).
|
||||||
|
# secrets:
|
||||||
|
# env_dir: ./secrets
|
||||||
|
|
||||||
|
storage:
|
||||||
|
# Optional storage backend selector; use "s3" for publish + S3 audio workflows.
|
||||||
|
backend: s3
|
||||||
|
s3:
|
||||||
|
# Required when using S3 audio or S3 publish uploads.
|
||||||
|
bucket: my-dnd-archive
|
||||||
|
# Optional; defaults to "dnd".
|
||||||
|
root_prefix: dnd
|
||||||
|
# Optional region/endpoint settings.
|
||||||
|
region: us-east-1
|
||||||
|
endpoint: ""
|
||||||
|
force_path_style: false
|
||||||
|
# Optional; defaults shown explicitly.
|
||||||
|
access_key_id_env: OBJECT_STORAGE_KEY_ID
|
||||||
|
secret_access_key_env: OBJECT_STORAGE_KEY
|
||||||
|
|
||||||
|
campaigns:
|
||||||
|
# Optional; defaults to /usr/local/share/narratio/campaigns.
|
||||||
|
root: /usr/local/share/narratio/campaigns
|
||||||
|
# Optional command default when --campaign is omitted.
|
||||||
|
default_campaign_id: sample-campaign
|
||||||
|
|
||||||
|
spool:
|
||||||
|
# Optional; defaults to /var/spool/narratio.
|
||||||
|
root: /var/spool/narratio
|
||||||
|
# Optional cleanup of run-scoped spool audio after successful publish commit.
|
||||||
|
delete_audio_after_publish: false
|
||||||
|
|
||||||
|
publish:
|
||||||
|
# Optional booleans; defaults are true.
|
||||||
|
enabled: true
|
||||||
|
upload_run: true
|
||||||
|
# Optional publish output rules; sources use Narratio artifact source IDs.
|
||||||
|
outputs:
|
||||||
|
- source: narratio.transcript.final_trimmed
|
||||||
|
dest: transcripts/final.trimmed.json
|
||||||
|
required: true
|
||||||
|
- source: narratio.transcript.final_markdown
|
||||||
|
dest: transcripts/final.md
|
||||||
|
required: true
|
||||||
|
- source: narratio.transcript.final_trimmed_markdown
|
||||||
|
dest: transcripts/final.trimmed.md
|
||||||
|
required: true
|
||||||
|
- source: narratio.artifact.session_recap
|
||||||
|
dest: artifacts/session_recap.md
|
||||||
|
required: true
|
||||||
|
- source: narratio.artifact.player_handout
|
||||||
|
dest: artifacts/player_handout.md
|
||||||
|
required: false
|
||||||
|
|
||||||
|
whisperx:
|
||||||
|
# Required.
|
||||||
|
transcribe_url: "https://transcription.example.com/transcribe"
|
||||||
|
# Optional overrides; defaults shown explicitly.
|
||||||
|
language: en
|
||||||
|
timeout: 30m
|
||||||
|
retries: 3
|
||||||
|
retry_delay: 2s
|
||||||
|
concurrency: 2
|
||||||
|
|
||||||
|
seriatim:
|
||||||
|
# Optional overrides; defaults shown explicitly.
|
||||||
|
binary: seriatim
|
||||||
|
timeout: 10m
|
||||||
|
output_schema: seriatim-intermediate
|
||||||
|
coalesce_gap: 3.0
|
||||||
|
report: true
|
||||||
|
env:
|
||||||
|
# Optional advanced tuning; set only when needed.
|
||||||
|
overlap_word_run_gap: 1.0
|
||||||
|
overlap_word_run_reorder_window: 1.0
|
||||||
|
backchannel_max_duration: 2.0
|
||||||
|
filler_max_duration: 1.25
|
||||||
|
|
||||||
|
audita:
|
||||||
|
# Optional overrides; defaults shown explicitly where applicable.
|
||||||
|
binary: audita
|
||||||
|
timeout: 3h
|
||||||
|
llm_api_key_env: AUDITA_LLM_API_KEY
|
||||||
|
modules: [glossary, homophones, spoken_word, grammar]
|
||||||
|
base_url: ""
|
||||||
|
model: ""
|
||||||
|
total_llm_concurrency: 2
|
||||||
|
proposal_llm_concurrency: 1
|
||||||
|
validation_model: ""
|
||||||
|
validation_llm_concurrency: 1
|
||||||
|
transcript_description: ""
|
||||||
|
config_path: /usr/local/etc/audita/config.yml
|
||||||
|
output_schema: audita-v1
|
||||||
|
work_dir_retention: auto
|
||||||
|
report: true
|
||||||
|
|
||||||
|
normalize:
|
||||||
|
# Optional; defaults shown explicitly.
|
||||||
|
output_path: transcripts/final.json
|
||||||
|
output_schema: seriatim-intermediate
|
||||||
|
report: true
|
||||||
|
|
||||||
|
trim:
|
||||||
|
# Optional; defaults shown explicitly.
|
||||||
|
enabled: true
|
||||||
|
output_path: transcripts/final.trimmed.json
|
||||||
|
bounds:
|
||||||
|
prompt_id: dnd.session_bounds
|
||||||
|
profile_id: ""
|
||||||
|
transcript_input_name: transcript
|
||||||
|
output_path: artifacts/session_bounds.json
|
||||||
|
timeout: 10m
|
||||||
|
render_debug: false
|
||||||
|
seriatim:
|
||||||
|
report: false
|
||||||
|
|
||||||
|
scriptorium:
|
||||||
|
binary: scriptorium
|
||||||
|
config_path: /usr/local/etc/scriptorium/config.yml
|
||||||
|
timeout: 10m
|
||||||
|
render_debug: false
|
||||||
|
artifacts:
|
||||||
|
# Configured artifact keys map to source IDs narratio.artifact.<key>.
|
||||||
|
session_recap:
|
||||||
|
enabled: true
|
||||||
|
prompt_id: dnd.session_recap
|
||||||
|
profile_id: local-fast
|
||||||
|
output_path: artifacts/session_recap.md
|
||||||
|
timeout: 10m
|
||||||
|
inputs:
|
||||||
|
transcript:
|
||||||
|
source: narratio.transcript.final_trimmed
|
||||||
|
required: true
|
||||||
|
previous_recap:
|
||||||
|
source: narratio.previous_session.artifact.session_recap
|
||||||
|
required: false
|
||||||
|
players:
|
||||||
|
source: narratio.input.players
|
||||||
|
required: true
|
||||||
|
party:
|
||||||
|
source: narratio.input.party
|
||||||
|
required: true
|
||||||
|
glossary:
|
||||||
|
source: narratio.input.glossary
|
||||||
|
required: false
|
||||||
|
vars:
|
||||||
|
session_id: true
|
||||||
|
session_date: true
|
||||||
|
campaign_name: true
|
||||||
|
previous_session_id: true
|
||||||
|
output_kind: session_recap
|
||||||
|
|
||||||
|
# Example dependent artifact:
|
||||||
|
# - depends_on entries use artifact keys.
|
||||||
|
# - narratio.artifact.<key> sources require matching depends_on membership.
|
||||||
|
player_handout:
|
||||||
|
enabled: true
|
||||||
|
depends_on:
|
||||||
|
- session_recap
|
||||||
|
prompt_id: dnd.player_handout
|
||||||
|
profile_id: local-fast
|
||||||
|
output_path: artifacts/player_handout.md
|
||||||
|
timeout: 10m
|
||||||
|
inputs:
|
||||||
|
recap:
|
||||||
|
source: narratio.artifact.session_recap
|
||||||
|
required: true
|
||||||
|
transcript:
|
||||||
|
source: narratio.transcript.final_trimmed
|
||||||
|
required: true
|
||||||
|
vars:
|
||||||
|
session_id: true
|
||||||
|
campaign_name: true
|
||||||
|
output_kind: player_handout
|
||||||
|
|
||||||
|
notification:
|
||||||
|
# Optional notification settings.
|
||||||
|
backend: ""
|
||||||
|
recipient: ""
|
||||||
|
timeout: 30s
|
||||||
@@ -1,109 +1,6 @@
|
|||||||
workspace:
|
campaigns:
|
||||||
root: ./tmp/narratio-workspace
|
root: /usr/local/share/narratio/campaigns
|
||||||
|
default_campaign_id: sample-campaign
|
||||||
storage:
|
|
||||||
backend: local
|
|
||||||
|
|
||||||
whisperx:
|
whisperx:
|
||||||
transcribe_url: "https://transcription.example.com/transcribe"
|
transcribe_url: "https://transcription.example.com/transcribe"
|
||||||
language: "en"
|
|
||||||
timeout: "30m"
|
|
||||||
retries: 3
|
|
||||||
retry_delay: "2s"
|
|
||||||
concurrency: 2
|
|
||||||
|
|
||||||
seriatim:
|
|
||||||
binary: "seriatim"
|
|
||||||
timeout: "10m"
|
|
||||||
output_schema: "seriatim-intermediate"
|
|
||||||
coalesce_gap: 3.0
|
|
||||||
report: true
|
|
||||||
env:
|
|
||||||
overlap_word_run_gap: 1.0
|
|
||||||
overlap_word_run_reorder_window: 1.0
|
|
||||||
backchannel_max_duration: 2.0
|
|
||||||
filler_max_duration: 1.25
|
|
||||||
|
|
||||||
audita:
|
|
||||||
binary: "audita"
|
|
||||||
timeout: "3h"
|
|
||||||
llm_api_key_env: "AUDITA_LLM_API_KEY"
|
|
||||||
modules:
|
|
||||||
- glossary
|
|
||||||
- homophones
|
|
||||||
- glossary
|
|
||||||
- spoken_word
|
|
||||||
- grammar
|
|
||||||
- homophones
|
|
||||||
- glossary
|
|
||||||
base_url: "https://openrouter.ai/api/v1"
|
|
||||||
model: "openrouter/google/gemma-4-31b-it"
|
|
||||||
llm_concurrency: 1
|
|
||||||
validation_model: ""
|
|
||||||
validation_llm_concurrency: 1
|
|
||||||
report: true
|
|
||||||
|
|
||||||
normalize:
|
|
||||||
# Session-workdir-relative when not absolute.
|
|
||||||
output_path: "transcripts/normalized.json"
|
|
||||||
output_schema: "seriatim-intermediate"
|
|
||||||
report: true
|
|
||||||
|
|
||||||
trim:
|
|
||||||
enabled: true
|
|
||||||
# Session-workdir-relative when not absolute.
|
|
||||||
output_path: "transcripts/trimmed.json"
|
|
||||||
bounds:
|
|
||||||
prompt_id: "dnd_session.bounds"
|
|
||||||
# Empty means use prompt default profile.
|
|
||||||
profile_id: ""
|
|
||||||
transcript_input_name: "transcript"
|
|
||||||
output_path: "artifacts/session_bounds.json"
|
|
||||||
timeout: "10m"
|
|
||||||
render_debug: false
|
|
||||||
render_output_path: "artifacts/session_bounds.render.json"
|
|
||||||
seriatim:
|
|
||||||
report: false
|
|
||||||
|
|
||||||
scriptorium:
|
|
||||||
binary: "scriptorium"
|
|
||||||
config_path: "/etc/scriptorium/config.yml"
|
|
||||||
timeout: "10m"
|
|
||||||
render_debug: false
|
|
||||||
artifacts:
|
|
||||||
session_recap:
|
|
||||||
enabled: true
|
|
||||||
prompt_id: "dnd.session_recap"
|
|
||||||
profile_id: "local-quality"
|
|
||||||
output_path: "artifacts/session_recap.md"
|
|
||||||
timeout: "10m"
|
|
||||||
# Optional per-artifact override of global scriptorium.render_debug.
|
|
||||||
# render_debug: true
|
|
||||||
inputs:
|
|
||||||
transcript:
|
|
||||||
# Available transcript sources:
|
|
||||||
# - trimmed_transcript (recommended for session_recap)
|
|
||||||
# - normalized_transcript (recommended for future full-session analysis)
|
|
||||||
# - processed_transcript (raw Audita-polished output)
|
|
||||||
source: "trimmed_transcript"
|
|
||||||
required: true
|
|
||||||
previous_recap:
|
|
||||||
source: "previous_session_artifact"
|
|
||||||
artifact: "session_recap"
|
|
||||||
# Optional: set when previous recap is available.
|
|
||||||
path: ""
|
|
||||||
required: false
|
|
||||||
vars:
|
|
||||||
session_id: true
|
|
||||||
session_date: true
|
|
||||||
campaign_name: true
|
|
||||||
previous_session_id: true
|
|
||||||
output_kind: "session_recap"
|
|
||||||
|
|
||||||
analyzer:
|
|
||||||
timeout: 20m
|
|
||||||
artifacts:
|
|
||||||
output_dir: artifacts
|
|
||||||
|
|
||||||
notification:
|
|
||||||
timeout: 10s
|
|
||||||
|
|||||||
128
examples/pipeline.production.yml
Normal file
128
examples/pipeline.production.yml
Normal file
@@ -0,0 +1,128 @@
|
|||||||
|
workspace:
|
||||||
|
root: /var/lib/narratio/workspace
|
||||||
|
cleanup_after_publish: true
|
||||||
|
|
||||||
|
storage:
|
||||||
|
backend: s3
|
||||||
|
s3:
|
||||||
|
bucket: my-dnd-archive
|
||||||
|
root_prefix: dnd
|
||||||
|
region: us-east-1
|
||||||
|
access_key_id_env: OBJECT_STORAGE_KEY_ID
|
||||||
|
secret_access_key_env: OBJECT_STORAGE_KEY
|
||||||
|
|
||||||
|
campaigns:
|
||||||
|
root: /usr/local/share/narratio/campaigns
|
||||||
|
default_campaign_id: sample-campaign
|
||||||
|
|
||||||
|
spool:
|
||||||
|
root: /var/spool/narratio
|
||||||
|
delete_audio_after_publish: true
|
||||||
|
|
||||||
|
publish:
|
||||||
|
enabled: true
|
||||||
|
upload_run: true
|
||||||
|
outputs:
|
||||||
|
- source: narratio.transcript.final_trimmed
|
||||||
|
dest: transcripts/final.trimmed.json
|
||||||
|
required: true
|
||||||
|
- source: narratio.transcript.final_markdown
|
||||||
|
dest: transcripts/final.md
|
||||||
|
required: true
|
||||||
|
- source: narratio.transcript.final_trimmed_markdown
|
||||||
|
dest: transcripts/final.trimmed.md
|
||||||
|
required: true
|
||||||
|
- source: narratio.artifact.session_recap
|
||||||
|
dest: artifacts/session_recap.md
|
||||||
|
required: true
|
||||||
|
- source: narratio.artifact.player_handout
|
||||||
|
dest: artifacts/player_handout.md
|
||||||
|
required: false
|
||||||
|
|
||||||
|
whisperx:
|
||||||
|
transcribe_url: "https://transcription.example.com/transcribe"
|
||||||
|
language: en
|
||||||
|
timeout: 45m
|
||||||
|
retries: 3
|
||||||
|
retry_delay: 3s
|
||||||
|
concurrency: 2
|
||||||
|
|
||||||
|
seriatim:
|
||||||
|
binary: seriatim
|
||||||
|
timeout: 10m
|
||||||
|
output_schema: seriatim-intermediate
|
||||||
|
coalesce_gap: 3.0
|
||||||
|
report: true
|
||||||
|
|
||||||
|
audita:
|
||||||
|
binary: audita
|
||||||
|
timeout: 3h
|
||||||
|
llm_api_key_env: AUDITA_LLM_API_KEY
|
||||||
|
modules: [glossary, homophones, spoken_word, grammar]
|
||||||
|
output_schema: audita-v1
|
||||||
|
work_dir_retention: auto
|
||||||
|
total_llm_concurrency: 2
|
||||||
|
proposal_llm_concurrency: 1
|
||||||
|
validation_llm_concurrency: 1
|
||||||
|
report: true
|
||||||
|
|
||||||
|
normalize:
|
||||||
|
output_path: transcripts/final.json
|
||||||
|
output_schema: seriatim-intermediate
|
||||||
|
report: true
|
||||||
|
|
||||||
|
scriptorium:
|
||||||
|
binary: scriptorium
|
||||||
|
config_path: /usr/local/etc/scriptorium/config.yml
|
||||||
|
timeout: 10m
|
||||||
|
render_debug: false
|
||||||
|
artifacts:
|
||||||
|
session_recap:
|
||||||
|
enabled: true
|
||||||
|
prompt_id: dnd.session_recap
|
||||||
|
profile_id: local-fast
|
||||||
|
output_path: artifacts/session_recap.md
|
||||||
|
timeout: 10m
|
||||||
|
inputs:
|
||||||
|
transcript:
|
||||||
|
source: narratio.transcript.final_trimmed
|
||||||
|
required: true
|
||||||
|
previous_recap:
|
||||||
|
source: narratio.previous_session.artifact.session_recap
|
||||||
|
required: false
|
||||||
|
players:
|
||||||
|
source: narratio.input.players
|
||||||
|
required: true
|
||||||
|
party:
|
||||||
|
source: narratio.input.party
|
||||||
|
required: true
|
||||||
|
glossary:
|
||||||
|
source: narratio.input.glossary
|
||||||
|
required: false
|
||||||
|
vars:
|
||||||
|
session_id: true
|
||||||
|
session_date: true
|
||||||
|
campaign_name: true
|
||||||
|
previous_session_id: true
|
||||||
|
output_kind: session_recap
|
||||||
|
player_handout:
|
||||||
|
enabled: true
|
||||||
|
depends_on:
|
||||||
|
- session_recap
|
||||||
|
prompt_id: dnd.player_handout
|
||||||
|
profile_id: local-fast
|
||||||
|
output_path: artifacts/player_handout.md
|
||||||
|
timeout: 10m
|
||||||
|
inputs:
|
||||||
|
recap:
|
||||||
|
source: narratio.artifact.session_recap
|
||||||
|
required: true
|
||||||
|
transcript:
|
||||||
|
source: narratio.transcript.final_trimmed
|
||||||
|
required: true
|
||||||
|
vars:
|
||||||
|
session_id: true
|
||||||
|
output_kind: player_handout
|
||||||
|
|
||||||
|
notification:
|
||||||
|
timeout: 30s
|
||||||
5
examples/session.local-audio.yml
Normal file
5
examples/session.local-audio.yml
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
session_id: 2026-05-03
|
||||||
|
date: 2026-05-03
|
||||||
|
title: Sample Session
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
@@ -1,10 +0,0 @@
|
|||||||
session_id: 2026-05-03
|
|
||||||
campaign: sample-campaign
|
|
||||||
date: 2026-05-03
|
|
||||||
title: Sample Session
|
|
||||||
inputs:
|
|
||||||
audio_dir: ./audio
|
|
||||||
speakers_file: ./speakers.yml
|
|
||||||
autocorrect_file: ./autocorrect.yml
|
|
||||||
glossary_file: ./glossary.yml
|
|
||||||
|
|
||||||
6
examples/session.s3-audio.yml
Normal file
6
examples/session.s3-audio.yml
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
session_id: 2026-05-03
|
||||||
|
date: 2026-05-03
|
||||||
|
title: Sample Session
|
||||||
|
inputs:
|
||||||
|
audio_s3:
|
||||||
|
prefix: audio/
|
||||||
3
examples/session.template.yml
Normal file
3
examples/session.template.yml
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
session_id: "{{ session_id }}"
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
25
go.mod
25
go.mod
@@ -2,4 +2,27 @@ module gitea.maximumdirect.net/eric/narratio
|
|||||||
|
|
||||||
go 1.25.0
|
go 1.25.0
|
||||||
|
|
||||||
require gopkg.in/yaml.v3 v3.0.1
|
require (
|
||||||
|
github.com/aws/aws-sdk-go-v2/config v1.32.17
|
||||||
|
github.com/aws/aws-sdk-go-v2/credentials v1.19.16
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
|
||||||
|
github.com/aws/smithy-go v1.25.1
|
||||||
|
gopkg.in/yaml.v3 v3.0.1
|
||||||
|
)
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/aws/aws-sdk-go-v2 v1.41.7 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect
|
||||||
|
)
|
||||||
|
|||||||
36
go.sum
36
go.sum
@@ -1,3 +1,39 @@
|
|||||||
|
github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=
|
||||||
|
github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=
|
||||||
|
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho=
|
||||||
|
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY=
|
||||||
|
github.com/aws/aws-sdk-go-v2/config v1.32.17 h1:FpL4/758/diKwqbytU0prpuiu60fgXKUWCpDJtApclU=
|
||||||
|
github.com/aws/aws-sdk-go-v2/config v1.32.17/go.mod h1:OXqUMzgXytfoF9JaKkhrOYsyh72t9G+MJH8mMRaexOE=
|
||||||
|
github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU=
|
||||||
|
github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug=
|
||||||
|
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 h1:UuSfcORqNSz/ey3VPRS8TcVH2Ikf0/sC+Hdj400QI6U=
|
||||||
|
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23/go.mod h1:+G/OSGiOFnSOkYloKj/9M35s74LgVAdJBSD5lsFfqKg=
|
||||||
|
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0=
|
||||||
|
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA=
|
||||||
|
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo=
|
||||||
|
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk=
|
||||||
|
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA=
|
||||||
|
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 h1:etqBTKY581iwLL/H/S2sVgk3C9lAsTJFeXWFDsDcWOU=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 h1:TdJ+HdzOBhU8+iVAOGUTU63VXopcumCOF1paFulHWZc=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11/go.mod h1:R82ZRExE/nheo0N+T8zHPcLRTcH8MGsnR3BiVGX0TwI=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 h1:7byT8HUWrgoRp6sXjxtZwgOKfhss5fW6SkLBtqzgRoE=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17/go.mod h1:xNWknVi4Ezm1vg1QsB/5EWpAJURq22uqd38U8qKvOJc=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 h1:+1Kl1zx6bWi4X7cKi3VYh29h8BvsCoHQEQ6ST9X8w7w=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21/go.mod h1:4vIRDq+CJB2xFAXZ+YgGUTiEft7oAQlhIs71xcSeuVg=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOItExNM9L1euNuh/fk=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio=
|
||||||
|
github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
|
||||||
|
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||||
|
|||||||
@@ -1,40 +0,0 @@
|
|||||||
package analyzer
|
|
||||||
|
|
||||||
import "context"
|
|
||||||
|
|
||||||
// NoopRunner is a deterministic no-op analyzer adapter.
|
|
||||||
type NoopRunner struct{}
|
|
||||||
|
|
||||||
// Run returns the requested output path with placeholder metadata.
|
|
||||||
func (n *NoopRunner) Run(ctx context.Context, req AnalyzeRequest) (AnalyzeResult, error) {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return AnalyzeResult{}, err
|
|
||||||
}
|
|
||||||
return AnalyzeResult{ArtifactPath: req.OutputPath, Metadata: map[string]any{"placeholder": true}}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// FakeRunner captures analyze requests and returns deterministic responses.
|
|
||||||
type FakeRunner struct {
|
|
||||||
Requests []AnalyzeRequest
|
|
||||||
Err error
|
|
||||||
Result AnalyzeResult
|
|
||||||
}
|
|
||||||
|
|
||||||
// Run records request and returns configured response.
|
|
||||||
func (f *FakeRunner) Run(ctx context.Context, req AnalyzeRequest) (AnalyzeResult, error) {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return AnalyzeResult{}, err
|
|
||||||
}
|
|
||||||
f.Requests = append(f.Requests, req)
|
|
||||||
if f.Err != nil {
|
|
||||||
return AnalyzeResult{}, f.Err
|
|
||||||
}
|
|
||||||
res := f.Result
|
|
||||||
if res.ArtifactPath == "" {
|
|
||||||
res.ArtifactPath = req.OutputPath
|
|
||||||
}
|
|
||||||
if res.Metadata == nil {
|
|
||||||
res.Metadata = map[string]any{"fake": true}
|
|
||||||
}
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
@@ -1,31 +0,0 @@
|
|||||||
package analyzer
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestFakeRunnerCapturesRequestAndReturnsPath(t *testing.T) {
|
|
||||||
fake := &FakeRunner{}
|
|
||||||
req := AnalyzeRequest{ArtifactType: "session-log", OutputPath: "artifacts/session-log.md"}
|
|
||||||
|
|
||||||
res, err := fake.Run(context.Background(), req)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Run() error = %v", err)
|
|
||||||
}
|
|
||||||
if len(fake.Requests) != 1 || fake.Requests[0].ArtifactType != "session-log" {
|
|
||||||
t.Fatalf("requests = %#v, want captured request", fake.Requests)
|
|
||||||
}
|
|
||||||
if res.ArtifactPath != req.OutputPath {
|
|
||||||
t.Fatalf("artifact path = %q, want %q", res.ArtifactPath, req.OutputPath)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFakeRunnerError(t *testing.T) {
|
|
||||||
fake := &FakeRunner{Err: errors.New("boom")}
|
|
||||||
_, err := fake.Run(context.Background(), AnalyzeRequest{})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected error, got nil")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,28 +0,0 @@
|
|||||||
// Package analyzer declares the adapter contract for artifact analysis generation.
|
|
||||||
package analyzer
|
|
||||||
|
|
||||||
import "context"
|
|
||||||
|
|
||||||
// TODO: implement analyzer integration once the analyzer contract is finalized.
|
|
||||||
|
|
||||||
// Runner is the adapter boundary for analyzer invocations.
|
|
||||||
type Runner interface {
|
|
||||||
Run(ctx context.Context, req AnalyzeRequest) (AnalyzeResult, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// AnalyzeRequest describes one analyzer artifact generation request.
|
|
||||||
type AnalyzeRequest struct {
|
|
||||||
ArtifactType string
|
|
||||||
ProcessedTranscriptPath string
|
|
||||||
ContextReferences []string
|
|
||||||
OutputPath string
|
|
||||||
GeneratedConfigPath string
|
|
||||||
StdoutLogPath string
|
|
||||||
StderrLogPath string
|
|
||||||
}
|
|
||||||
|
|
||||||
// AnalyzeResult describes analyzer output.
|
|
||||||
type AnalyzeResult struct {
|
|
||||||
ArtifactPath string
|
|
||||||
Metadata map[string]any
|
|
||||||
}
|
|
||||||
@@ -14,7 +14,7 @@ func TestFakeRunnerCapturesRequestAndReturnsPath(t *testing.T) {
|
|||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
req := PolishRequest{
|
req := PolishRequest{
|
||||||
GeneratedConfigPath: filepath.Join(dir, "config", "audita.yml"),
|
GeneratedConfigPath: filepath.Join(dir, "config", "audita.yml"),
|
||||||
OutputProcessedPath: filepath.Join(dir, "transcripts", "processed.json"),
|
OutputProcessedPath: filepath.Join(dir, "transcripts", "polished.json"),
|
||||||
StdoutLogPath: filepath.Join(dir, "logs", "audita.stdout.log"),
|
StdoutLogPath: filepath.Join(dir, "logs", "audita.stdout.log"),
|
||||||
StderrLogPath: filepath.Join(dir, "logs", "audita.stderr.log"),
|
StderrLogPath: filepath.Join(dir, "logs", "audita.stderr.log"),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -24,6 +24,12 @@ type PolishRequest struct {
|
|||||||
Modules []string
|
Modules []string
|
||||||
BaseURL string
|
BaseURL string
|
||||||
Model string
|
Model string
|
||||||
|
TranscriptDescription string
|
||||||
|
ConfigPath string
|
||||||
|
OutputSchema string
|
||||||
|
WorkDirRetention string
|
||||||
|
TotalLLMConcurrency *int
|
||||||
|
ProposalLLMConcurrency *int
|
||||||
ValidationModel string
|
ValidationModel string
|
||||||
ValidationLLMConcurrency *int
|
ValidationLLMConcurrency *int
|
||||||
StdoutLogPath string
|
StdoutLogPath string
|
||||||
|
|||||||
@@ -21,7 +21,12 @@ type SubprocessRunnerConfig struct {
|
|||||||
Modules []string
|
Modules []string
|
||||||
BaseURL string
|
BaseURL string
|
||||||
Model string
|
Model string
|
||||||
LLMConcurrency *int
|
TranscriptDescription string
|
||||||
|
ConfigPath string
|
||||||
|
OutputSchema string
|
||||||
|
WorkDirRetention string
|
||||||
|
TotalLLMConcurrency *int
|
||||||
|
ProposalLLMConcurrency *int
|
||||||
ValidationModel string
|
ValidationModel string
|
||||||
ValidationLLMConcurrency *int
|
ValidationLLMConcurrency *int
|
||||||
Report bool
|
Report bool
|
||||||
@@ -35,7 +40,12 @@ type SubprocessRunner struct {
|
|||||||
modules []string
|
modules []string
|
||||||
baseURL string
|
baseURL string
|
||||||
model string
|
model string
|
||||||
llmConcurrency *int
|
transcriptDescription string
|
||||||
|
configPath string
|
||||||
|
outputSchema string
|
||||||
|
workDirRetention string
|
||||||
|
totalLLMConcurrency *int
|
||||||
|
proposalLLMConcurrency *int
|
||||||
validationModel string
|
validationModel string
|
||||||
validationLLMConcurrency *int
|
validationLLMConcurrency *int
|
||||||
report bool
|
report bool
|
||||||
@@ -49,7 +59,12 @@ func NewSubprocessRunnerFromConfigValues(
|
|||||||
modules []string,
|
modules []string,
|
||||||
baseURL string,
|
baseURL string,
|
||||||
model string,
|
model string,
|
||||||
llmConcurrency *int,
|
transcriptDescription string,
|
||||||
|
configPath string,
|
||||||
|
outputSchema string,
|
||||||
|
workDirRetention string,
|
||||||
|
totalLLMConcurrency *int,
|
||||||
|
proposalLLMConcurrency *int,
|
||||||
validationModel string,
|
validationModel string,
|
||||||
validationLLMConcurrency *int,
|
validationLLMConcurrency *int,
|
||||||
report bool,
|
report bool,
|
||||||
@@ -68,7 +83,12 @@ func NewSubprocessRunnerFromConfigValues(
|
|||||||
Modules: modules,
|
Modules: modules,
|
||||||
BaseURL: baseURL,
|
BaseURL: baseURL,
|
||||||
Model: model,
|
Model: model,
|
||||||
LLMConcurrency: llmConcurrency,
|
TranscriptDescription: transcriptDescription,
|
||||||
|
ConfigPath: configPath,
|
||||||
|
OutputSchema: outputSchema,
|
||||||
|
WorkDirRetention: workDirRetention,
|
||||||
|
TotalLLMConcurrency: totalLLMConcurrency,
|
||||||
|
ProposalLLMConcurrency: proposalLLMConcurrency,
|
||||||
ValidationModel: validationModel,
|
ValidationModel: validationModel,
|
||||||
ValidationLLMConcurrency: validationLLMConcurrency,
|
ValidationLLMConcurrency: validationLLMConcurrency,
|
||||||
Report: report,
|
Report: report,
|
||||||
@@ -83,17 +103,12 @@ func NewSubprocessRunner(cfg SubprocessRunnerConfig) (*SubprocessRunner, error)
|
|||||||
if cfg.Timeout <= 0 {
|
if cfg.Timeout <= 0 {
|
||||||
return nil, fmt.Errorf("audita timeout must be > 0")
|
return nil, fmt.Errorf("audita timeout must be > 0")
|
||||||
}
|
}
|
||||||
if len(cfg.Modules) == 0 {
|
|
||||||
return nil, fmt.Errorf("audita modules must include at least one module")
|
|
||||||
}
|
|
||||||
for i, module := range cfg.Modules {
|
for i, module := range cfg.Modules {
|
||||||
if strings.TrimSpace(module) == "" {
|
if strings.TrimSpace(module) == "" {
|
||||||
return nil, fmt.Errorf("audita module at index %d is empty", i)
|
return nil, fmt.Errorf("audita module at index %d is empty", i)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(cfg.BaseURL) == "" {
|
if strings.TrimSpace(cfg.BaseURL) != "" {
|
||||||
return nil, fmt.Errorf("audita base url is required")
|
|
||||||
}
|
|
||||||
u, err := url.Parse(cfg.BaseURL)
|
u, err := url.Parse(cfg.BaseURL)
|
||||||
if err != nil || u.Scheme == "" || u.Host == "" {
|
if err != nil || u.Scheme == "" || u.Host == "" {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -101,15 +116,26 @@ func NewSubprocessRunner(cfg SubprocessRunnerConfig) (*SubprocessRunner, error)
|
|||||||
}
|
}
|
||||||
return nil, fmt.Errorf("audita base url %q is invalid", cfg.BaseURL)
|
return nil, fmt.Errorf("audita base url %q is invalid", cfg.BaseURL)
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(cfg.Model) == "" {
|
|
||||||
return nil, fmt.Errorf("audita model is required")
|
|
||||||
}
|
}
|
||||||
if cfg.LLMConcurrency != nil && *cfg.LLMConcurrency <= 0 {
|
if cfg.TotalLLMConcurrency != nil && *cfg.TotalLLMConcurrency <= 0 {
|
||||||
return nil, fmt.Errorf("audita llm concurrency must be > 0 when provided")
|
return nil, fmt.Errorf("audita total llm concurrency must be > 0 when provided")
|
||||||
|
}
|
||||||
|
if cfg.ProposalLLMConcurrency != nil && *cfg.ProposalLLMConcurrency <= 0 {
|
||||||
|
return nil, fmt.Errorf("audita proposal llm concurrency must be > 0 when provided")
|
||||||
}
|
}
|
||||||
if cfg.ValidationLLMConcurrency != nil && *cfg.ValidationLLMConcurrency <= 0 {
|
if cfg.ValidationLLMConcurrency != nil && *cfg.ValidationLLMConcurrency <= 0 {
|
||||||
return nil, fmt.Errorf("audita validation llm concurrency must be > 0 when provided")
|
return nil, fmt.Errorf("audita validation llm concurrency must be > 0 when provided")
|
||||||
}
|
}
|
||||||
|
switch strings.TrimSpace(cfg.OutputSchema) {
|
||||||
|
case "", "bare-segments", "audita-v1":
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("audita output schema must be one of: bare-segments, audita-v1")
|
||||||
|
}
|
||||||
|
switch strings.TrimSpace(cfg.WorkDirRetention) {
|
||||||
|
case "", "always", "auto", "never":
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("audita work dir retention must be one of: always, auto, never")
|
||||||
|
}
|
||||||
|
|
||||||
modules := make([]string, len(cfg.Modules))
|
modules := make([]string, len(cfg.Modules))
|
||||||
for i, m := range cfg.Modules {
|
for i, m := range cfg.Modules {
|
||||||
@@ -123,7 +149,12 @@ func NewSubprocessRunner(cfg SubprocessRunnerConfig) (*SubprocessRunner, error)
|
|||||||
modules: modules,
|
modules: modules,
|
||||||
baseURL: strings.TrimSpace(cfg.BaseURL),
|
baseURL: strings.TrimSpace(cfg.BaseURL),
|
||||||
model: strings.TrimSpace(cfg.Model),
|
model: strings.TrimSpace(cfg.Model),
|
||||||
llmConcurrency: cfg.LLMConcurrency,
|
transcriptDescription: strings.TrimSpace(cfg.TranscriptDescription),
|
||||||
|
configPath: strings.TrimSpace(cfg.ConfigPath),
|
||||||
|
outputSchema: strings.TrimSpace(cfg.OutputSchema),
|
||||||
|
workDirRetention: strings.TrimSpace(cfg.WorkDirRetention),
|
||||||
|
totalLLMConcurrency: cfg.TotalLLMConcurrency,
|
||||||
|
proposalLLMConcurrency: cfg.ProposalLLMConcurrency,
|
||||||
validationModel: strings.TrimSpace(cfg.ValidationModel),
|
validationModel: strings.TrimSpace(cfg.ValidationModel),
|
||||||
validationLLMConcurrency: cfg.ValidationLLMConcurrency,
|
validationLLMConcurrency: cfg.ValidationLLMConcurrency,
|
||||||
report: cfg.Report,
|
report: cfg.Report,
|
||||||
@@ -152,7 +183,7 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
|
|||||||
}
|
}
|
||||||
|
|
||||||
reqModules := req.Modules
|
reqModules := req.Modules
|
||||||
if len(reqModules) == 0 {
|
if reqModules == nil {
|
||||||
reqModules = append([]string(nil), r.modules...)
|
reqModules = append([]string(nil), r.modules...)
|
||||||
}
|
}
|
||||||
args := r.buildArgs(req, reqModules)
|
args := r.buildArgs(req, reqModules)
|
||||||
@@ -168,14 +199,9 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
|
|||||||
env["AUDITA_LLM_API_KEY"] = credential
|
env["AUDITA_LLM_API_KEY"] = credential
|
||||||
credentialPresent = true
|
credentialPresent = true
|
||||||
}
|
}
|
||||||
primaryConcurrencyViaEnv := false
|
|
||||||
if r.llmConcurrency != nil {
|
|
||||||
env["AUDITA_LLM_CONCURRENCY"] = strconv.Itoa(*r.llmConcurrency)
|
|
||||||
primaryConcurrencyViaEnv = true
|
|
||||||
}
|
|
||||||
|
|
||||||
if req.GeneratedConfigPath != "" {
|
if req.GeneratedConfigPath != "" {
|
||||||
if err := r.writeInvocationConfig(req, args, reqModules, credentialPresent, primaryConcurrencyViaEnv); err != nil {
|
if err := r.writeInvocationConfig(req, args, reqModules, credentialPresent); err != nil {
|
||||||
return PolishResult{}, fmt.Errorf("write audita invocation config %q: %w", req.GeneratedConfigPath, err)
|
return PolishResult{}, fmt.Errorf("write audita invocation config %q: %w", req.GeneratedConfigPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -196,7 +222,7 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
|
|||||||
req.StderrLogPath,
|
req.StderrLogPath,
|
||||||
)
|
)
|
||||||
wrappedMessage = addSubprocessStreamHint(wrappedMessage, err)
|
wrappedMessage = addSubprocessStreamHint(wrappedMessage, err)
|
||||||
return r.failureResult(req, reqModules, runRes, credentialPresent, primaryConcurrencyViaEnv), fmt.Errorf(
|
return r.failureResult(req, reqModules, runRes, credentialPresent), fmt.Errorf(
|
||||||
"%s: %w",
|
"%s: %w",
|
||||||
wrappedMessage,
|
wrappedMessage,
|
||||||
err,
|
err,
|
||||||
@@ -204,11 +230,11 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
|
|||||||
}
|
}
|
||||||
|
|
||||||
if err := validateProcessedOutput(req.OutputProcessedPath); err != nil {
|
if err := validateProcessedOutput(req.OutputProcessedPath); err != nil {
|
||||||
return r.failureResult(req, reqModules, runRes, credentialPresent, primaryConcurrencyViaEnv), fmt.Errorf("validate audita processed output %q: %w", req.OutputProcessedPath, err)
|
return r.failureResult(req, reqModules, runRes, credentialPresent), fmt.Errorf("validate audita processed output %q: %w", req.OutputProcessedPath, err)
|
||||||
}
|
}
|
||||||
if r.report {
|
if r.report {
|
||||||
if err := validateJSONFile(req.ReportPath); err != nil {
|
if err := validateJSONFile(req.ReportPath); err != nil {
|
||||||
return r.failureResult(req, reqModules, runRes, credentialPresent, primaryConcurrencyViaEnv), fmt.Errorf("validate audita report output %q: %w", req.ReportPath, err)
|
return r.failureResult(req, reqModules, runRes, credentialPresent), fmt.Errorf("validate audita report output %q: %w", req.ReportPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -227,17 +253,21 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
|
|||||||
"modules": reqModules,
|
"modules": reqModules,
|
||||||
"base_url": r.baseURL,
|
"base_url": r.baseURL,
|
||||||
"model": r.model,
|
"model": r.model,
|
||||||
|
"transcript_description": r.transcriptDescription,
|
||||||
|
"config_path": r.configPath,
|
||||||
|
"output_schema": r.outputSchema,
|
||||||
|
"work_dir_retention": r.workDirRetention,
|
||||||
"validation_model": r.validationModel,
|
"validation_model": r.validationModel,
|
||||||
|
"total_llm_concurrency": r.totalLLMConcurrency,
|
||||||
|
"proposal_llm_concurrency": r.proposalLLMConcurrency,
|
||||||
"validation_llm_concurrency": r.validationLLMConcurrency,
|
"validation_llm_concurrency": r.validationLLMConcurrency,
|
||||||
"credential_env_var": r.llmAPIKeyEnv,
|
"credential_env_var": r.llmAPIKeyEnv,
|
||||||
"credential_present": credentialPresent,
|
"credential_present": credentialPresent,
|
||||||
"primary_llm_concurrency_via_env": primaryConcurrencyViaEnv,
|
|
||||||
"primary_llm_concurrency_env_name": "AUDITA_LLM_CONCURRENCY",
|
|
||||||
},
|
},
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *SubprocessRunner) failureResult(req PolishRequest, modules []string, runRes subprocess.RunResult, credentialPresent bool, primaryConcurrencyViaEnv bool) PolishResult {
|
func (r *SubprocessRunner) failureResult(req PolishRequest, modules []string, runRes subprocess.RunResult, credentialPresent bool) PolishResult {
|
||||||
return PolishResult{
|
return PolishResult{
|
||||||
ProcessedTranscriptPath: req.OutputProcessedPath,
|
ProcessedTranscriptPath: req.OutputProcessedPath,
|
||||||
ReportPath: req.ReportPath,
|
ReportPath: req.ReportPath,
|
||||||
@@ -253,12 +283,16 @@ func (r *SubprocessRunner) failureResult(req PolishRequest, modules []string, ru
|
|||||||
"modules": modules,
|
"modules": modules,
|
||||||
"base_url": r.baseURL,
|
"base_url": r.baseURL,
|
||||||
"model": r.model,
|
"model": r.model,
|
||||||
|
"transcript_description": r.transcriptDescription,
|
||||||
|
"config_path": r.configPath,
|
||||||
|
"output_schema": r.outputSchema,
|
||||||
|
"work_dir_retention": r.workDirRetention,
|
||||||
"validation_model": r.validationModel,
|
"validation_model": r.validationModel,
|
||||||
|
"total_llm_concurrency": r.totalLLMConcurrency,
|
||||||
|
"proposal_llm_concurrency": r.proposalLLMConcurrency,
|
||||||
"validation_llm_concurrency": r.validationLLMConcurrency,
|
"validation_llm_concurrency": r.validationLLMConcurrency,
|
||||||
"credential_env_var": r.llmAPIKeyEnv,
|
"credential_env_var": r.llmAPIKeyEnv,
|
||||||
"credential_present": credentialPresent,
|
"credential_present": credentialPresent,
|
||||||
"primary_llm_concurrency_via_env": primaryConcurrencyViaEnv,
|
|
||||||
"primary_llm_concurrency_env_name": "AUDITA_LLM_CONCURRENCY",
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -269,14 +303,38 @@ func (r *SubprocessRunner) buildArgs(req PolishRequest, modules []string) []stri
|
|||||||
req.MergedTranscriptPath,
|
req.MergedTranscriptPath,
|
||||||
"--glossary", req.GlossaryPath,
|
"--glossary", req.GlossaryPath,
|
||||||
"--output", req.OutputProcessedPath,
|
"--output", req.OutputProcessedPath,
|
||||||
"--modules", strings.Join(modules, ","),
|
|
||||||
"--base-url", r.baseURL,
|
|
||||||
"--model", r.model,
|
|
||||||
"--work-dir", req.WorkDir,
|
"--work-dir", req.WorkDir,
|
||||||
}
|
}
|
||||||
|
if r.baseURL != "" {
|
||||||
|
args = append(args, "--base-url", r.baseURL)
|
||||||
|
}
|
||||||
|
if r.model != "" {
|
||||||
|
args = append(args, "--model", r.model)
|
||||||
|
}
|
||||||
|
if len(modules) > 0 {
|
||||||
|
args = append(args, "--modules", strings.Join(modules, ","))
|
||||||
|
}
|
||||||
if r.report {
|
if r.report {
|
||||||
args = append(args, "--report-json", req.ReportPath)
|
args = append(args, "--report-json", req.ReportPath)
|
||||||
}
|
}
|
||||||
|
if r.transcriptDescription != "" {
|
||||||
|
args = append(args, "--transcript-description", r.transcriptDescription)
|
||||||
|
}
|
||||||
|
if r.configPath != "" {
|
||||||
|
args = append(args, "--config", r.configPath)
|
||||||
|
}
|
||||||
|
if r.outputSchema != "" {
|
||||||
|
args = append(args, "--output-schema", r.outputSchema)
|
||||||
|
}
|
||||||
|
if r.workDirRetention != "" {
|
||||||
|
args = append(args, "--work-dir-retention", r.workDirRetention)
|
||||||
|
}
|
||||||
|
if r.totalLLMConcurrency != nil {
|
||||||
|
args = append(args, "--total-llm-concurrency", strconv.Itoa(*r.totalLLMConcurrency))
|
||||||
|
}
|
||||||
|
if r.proposalLLMConcurrency != nil {
|
||||||
|
args = append(args, "--proposal-llm-concurrency", strconv.Itoa(*r.proposalLLMConcurrency))
|
||||||
|
}
|
||||||
if r.validationModel != "" {
|
if r.validationModel != "" {
|
||||||
args = append(args, "--validation-model", r.validationModel)
|
args = append(args, "--validation-model", r.validationModel)
|
||||||
}
|
}
|
||||||
@@ -286,7 +344,7 @@ func (r *SubprocessRunner) buildArgs(req PolishRequest, modules []string) []stri
|
|||||||
return args
|
return args
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *SubprocessRunner) writeInvocationConfig(req PolishRequest, args []string, modules []string, credentialPresent bool, primaryConcurrencyViaEnv bool) error {
|
func (r *SubprocessRunner) writeInvocationConfig(req PolishRequest, args []string, modules []string, credentialPresent bool) error {
|
||||||
payload := map[string]any{
|
payload := map[string]any{
|
||||||
"schema": "audita.generated.v1",
|
"schema": "audita.generated.v1",
|
||||||
"binary": r.binary,
|
"binary": r.binary,
|
||||||
@@ -295,7 +353,13 @@ func (r *SubprocessRunner) writeInvocationConfig(req PolishRequest, args []strin
|
|||||||
"modules": modules,
|
"modules": modules,
|
||||||
"base_url": r.baseURL,
|
"base_url": r.baseURL,
|
||||||
"model": r.model,
|
"model": r.model,
|
||||||
|
"transcript_description": r.transcriptDescription,
|
||||||
|
"config_path": r.configPath,
|
||||||
|
"output_schema": r.outputSchema,
|
||||||
|
"work_dir_retention": r.workDirRetention,
|
||||||
"validation_model": r.validationModel,
|
"validation_model": r.validationModel,
|
||||||
|
"total_llm_concurrency": r.totalLLMConcurrency,
|
||||||
|
"proposal_llm_concurrency": r.proposalLLMConcurrency,
|
||||||
"validation_llm_concurrency": r.validationLLMConcurrency,
|
"validation_llm_concurrency": r.validationLLMConcurrency,
|
||||||
"report_enabled": r.report,
|
"report_enabled": r.report,
|
||||||
"merged_transcript_path": req.MergedTranscriptPath,
|
"merged_transcript_path": req.MergedTranscriptPath,
|
||||||
@@ -305,10 +369,6 @@ func (r *SubprocessRunner) writeInvocationConfig(req PolishRequest, args []strin
|
|||||||
"work_dir": req.WorkDir,
|
"work_dir": req.WorkDir,
|
||||||
"credential_env_var": r.llmAPIKeyEnv,
|
"credential_env_var": r.llmAPIKeyEnv,
|
||||||
"credential_present": credentialPresent,
|
"credential_present": credentialPresent,
|
||||||
"primary_llm_concurrency_via_env": primaryConcurrencyViaEnv,
|
|
||||||
}
|
|
||||||
if r.llmConcurrency != nil {
|
|
||||||
payload["llm_concurrency"] = *r.llmConcurrency
|
|
||||||
}
|
}
|
||||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644)
|
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -25,7 +25,8 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
|||||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||||
|
|
||||||
wrapper := writeAuditaHelperWrapper(t)
|
wrapper := writeAuditaHelperWrapper(t)
|
||||||
llmConcurrency := 1
|
totalLLMConcurrency := 3
|
||||||
|
proposalLLMConcurrency := 2
|
||||||
validationLLMConcurrency := 2
|
validationLLMConcurrency := 2
|
||||||
runner, err := NewSubprocessRunner(SubprocessRunnerConfig{
|
runner, err := NewSubprocessRunner(SubprocessRunnerConfig{
|
||||||
Binary: wrapper,
|
Binary: wrapper,
|
||||||
@@ -34,7 +35,12 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
|||||||
Modules: []string{"glossary", "homophones", "glossary"},
|
Modules: []string{"glossary", "homophones", "glossary"},
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
LLMConcurrency: &llmConcurrency,
|
TranscriptDescription: "Campaign Session 42",
|
||||||
|
ConfigPath: "/etc/audita/config.yml",
|
||||||
|
OutputSchema: "audita-v1",
|
||||||
|
WorkDirRetention: "auto",
|
||||||
|
TotalLLMConcurrency: &totalLLMConcurrency,
|
||||||
|
ProposalLLMConcurrency: &proposalLLMConcurrency,
|
||||||
ValidationModel: "openrouter/google/gemma-4-31b-it",
|
ValidationModel: "openrouter/google/gemma-4-31b-it",
|
||||||
ValidationLLMConcurrency: &validationLLMConcurrency,
|
ValidationLLMConcurrency: &validationLLMConcurrency,
|
||||||
Report: true,
|
Report: true,
|
||||||
@@ -46,9 +52,9 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
|||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
req := PolishRequest{
|
req := PolishRequest{
|
||||||
GeneratedConfigPath: filepath.Join(dir, "audita.generated.yml"),
|
GeneratedConfigPath: filepath.Join(dir, "audita.generated.yml"),
|
||||||
MergedTranscriptPath: filepath.Join(dir, "merged.json"),
|
MergedTranscriptPath: filepath.Join(dir, "base.json"),
|
||||||
GlossaryPath: filepath.Join(dir, "glossary.yml"),
|
GlossaryPath: filepath.Join(dir, "glossary.yml"),
|
||||||
OutputProcessedPath: filepath.Join(dir, "processed.json"),
|
OutputProcessedPath: filepath.Join(dir, "polished.json"),
|
||||||
ReportPath: filepath.Join(dir, "audita.report.json"),
|
ReportPath: filepath.Join(dir, "audita.report.json"),
|
||||||
WorkDir: filepath.Join(dir, "artifacts", "audita-work"),
|
WorkDir: filepath.Join(dir, "artifacts", "audita-work"),
|
||||||
StdoutLogPath: filepath.Join(dir, "audita.stdout.log"),
|
StdoutLogPath: filepath.Join(dir, "audita.stdout.log"),
|
||||||
@@ -93,11 +99,17 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
|||||||
"process", req.MergedTranscriptPath,
|
"process", req.MergedTranscriptPath,
|
||||||
"--glossary", req.GlossaryPath,
|
"--glossary", req.GlossaryPath,
|
||||||
"--output", req.OutputProcessedPath,
|
"--output", req.OutputProcessedPath,
|
||||||
"--modules", "glossary,homophones,glossary",
|
"--work-dir", req.WorkDir,
|
||||||
"--base-url", "https://openrouter.ai/api/v1",
|
"--base-url", "https://openrouter.ai/api/v1",
|
||||||
"--model", "openrouter/google/gemma-4-31b-it",
|
"--model", "openrouter/google/gemma-4-31b-it",
|
||||||
"--work-dir", req.WorkDir,
|
"--modules", "glossary,homophones,glossary",
|
||||||
"--report-json", req.ReportPath,
|
"--report-json", req.ReportPath,
|
||||||
|
"--transcript-description", "Campaign Session 42",
|
||||||
|
"--config", "/etc/audita/config.yml",
|
||||||
|
"--output-schema", "audita-v1",
|
||||||
|
"--work-dir-retention", "auto",
|
||||||
|
"--total-llm-concurrency", "3",
|
||||||
|
"--proposal-llm-concurrency", "2",
|
||||||
"--validation-model", "openrouter/google/gemma-4-31b-it",
|
"--validation-model", "openrouter/google/gemma-4-31b-it",
|
||||||
"--validation-llm-concurrency", "2",
|
"--validation-llm-concurrency", "2",
|
||||||
}
|
}
|
||||||
@@ -107,8 +119,8 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
|||||||
if rec.Env["AUDITA_LLM_API_KEY"] != "super-secret" {
|
if rec.Env["AUDITA_LLM_API_KEY"] != "super-secret" {
|
||||||
t.Fatalf("AUDITA_LLM_API_KEY = %q, want propagated secret", rec.Env["AUDITA_LLM_API_KEY"])
|
t.Fatalf("AUDITA_LLM_API_KEY = %q, want propagated secret", rec.Env["AUDITA_LLM_API_KEY"])
|
||||||
}
|
}
|
||||||
if rec.Env["AUDITA_LLM_CONCURRENCY"] != "1" {
|
if rec.Env["AUDITA_LLM_CONCURRENCY"] != "" {
|
||||||
t.Fatalf("AUDITA_LLM_CONCURRENCY = %q, want 1", rec.Env["AUDITA_LLM_CONCURRENCY"])
|
t.Fatalf("AUDITA_LLM_CONCURRENCY = %q, want empty/omitted", rec.Env["AUDITA_LLM_CONCURRENCY"])
|
||||||
}
|
}
|
||||||
|
|
||||||
cfgData, err := os.ReadFile(req.GeneratedConfigPath)
|
cfgData, err := os.ReadFile(req.GeneratedConfigPath)
|
||||||
@@ -124,7 +136,6 @@ func TestSubprocessRunnerMissingConfiguredCredentialFails(t *testing.T) {
|
|||||||
if runtime.GOOS == "windows" {
|
if runtime.GOOS == "windows" {
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
t.Skip("helper wrapper script uses /bin/sh")
|
||||||
}
|
}
|
||||||
llmConcurrency := 1
|
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
@@ -132,7 +143,6 @@ func TestSubprocessRunnerMissingConfiguredCredentialFails(t *testing.T) {
|
|||||||
Modules: []string{"glossary"},
|
Modules: []string{"glossary"},
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
LLMConcurrency: &llmConcurrency,
|
|
||||||
Report: false,
|
Report: false,
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, false)
|
req := auditaReqForTest(t, false)
|
||||||
@@ -155,7 +165,6 @@ func TestSubprocessRunnerUnconfiguredCredentialEnvOmitsCredential(t *testing.T)
|
|||||||
recordPath := filepath.Join(t.TempDir(), "record.json")
|
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||||
|
|
||||||
llmConcurrency := 1
|
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
@@ -163,7 +172,6 @@ func TestSubprocessRunnerUnconfiguredCredentialEnvOmitsCredential(t *testing.T)
|
|||||||
Modules: []string{"glossary"},
|
Modules: []string{"glossary"},
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
LLMConcurrency: &llmConcurrency,
|
|
||||||
Report: false,
|
Report: false,
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, false)
|
req := auditaReqForTest(t, false)
|
||||||
@@ -211,6 +219,65 @@ func TestSubprocessRunnerInheritsParentEnvironment(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSubprocessRunnerOmitsModulesFlagWhenNotConfigured(t *testing.T) {
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
t.Skip("helper wrapper script uses /bin/sh")
|
||||||
|
}
|
||||||
|
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||||
|
t.Setenv("AUDITA_HELPER_MODE", "success")
|
||||||
|
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||||
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||||
|
|
||||||
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
|
LLMAPIKeyEnv: "",
|
||||||
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
|
Report: false,
|
||||||
|
})
|
||||||
|
req := auditaReqForTest(t, false)
|
||||||
|
if _, err := runner.Run(context.Background(), req); err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
rec := readAuditaHelperRecord(t, recordPath)
|
||||||
|
for i := 0; i < len(rec.Args); i++ {
|
||||||
|
if rec.Args[i] == "--modules" {
|
||||||
|
t.Fatalf("args contained --modules unexpectedly: %#v", rec.Args)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessRunnerOmitsBaseURLAndModelFlagsWhenNotConfigured(t *testing.T) {
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
t.Skip("helper wrapper script uses /bin/sh")
|
||||||
|
}
|
||||||
|
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||||
|
t.Setenv("AUDITA_HELPER_MODE", "success")
|
||||||
|
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||||
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||||
|
|
||||||
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
|
LLMAPIKeyEnv: "",
|
||||||
|
Report: false,
|
||||||
|
})
|
||||||
|
req := auditaReqForTest(t, false)
|
||||||
|
if _, err := runner.Run(context.Background(), req); err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
rec := readAuditaHelperRecord(t, recordPath)
|
||||||
|
for i := 0; i < len(rec.Args); i++ {
|
||||||
|
if rec.Args[i] == "--base-url" {
|
||||||
|
t.Fatalf("args contained --base-url unexpectedly: %#v", rec.Args)
|
||||||
|
}
|
||||||
|
if rec.Args[i] == "--model" {
|
||||||
|
t.Fatalf("args contained --model unexpectedly: %#v", rec.Args)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
|
func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
if runtime.GOOS == "windows" {
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
t.Skip("helper wrapper script uses /bin/sh")
|
||||||
@@ -220,7 +287,6 @@ func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
|
|||||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
|
|
||||||
llmConcurrency := 1
|
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
@@ -228,7 +294,6 @@ func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
|
|||||||
Modules: []string{"glossary"},
|
Modules: []string{"glossary"},
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
LLMConcurrency: &llmConcurrency,
|
|
||||||
Report: true,
|
Report: true,
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, true)
|
req := auditaReqForTest(t, true)
|
||||||
@@ -256,7 +321,6 @@ func TestSubprocessRunnerSubprocessFailureAddsStderrDescriptorHint(t *testing.T)
|
|||||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
|
|
||||||
llmConcurrency := 1
|
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
@@ -264,7 +328,6 @@ func TestSubprocessRunnerSubprocessFailureAddsStderrDescriptorHint(t *testing.T)
|
|||||||
Modules: []string{"glossary"},
|
Modules: []string{"glossary"},
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
LLMConcurrency: &llmConcurrency,
|
|
||||||
Report: true,
|
Report: true,
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, true)
|
req := auditaReqForTest(t, true)
|
||||||
@@ -286,7 +349,6 @@ func TestSubprocessRunnerMissingOutputFails(t *testing.T) {
|
|||||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
|
|
||||||
llmConcurrency := 1
|
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
@@ -294,7 +356,6 @@ func TestSubprocessRunnerMissingOutputFails(t *testing.T) {
|
|||||||
Modules: []string{"glossary"},
|
Modules: []string{"glossary"},
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
LLMConcurrency: &llmConcurrency,
|
|
||||||
Report: false,
|
Report: false,
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, false)
|
req := auditaReqForTest(t, false)
|
||||||
@@ -316,7 +377,6 @@ func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) {
|
|||||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
|
|
||||||
llmConcurrency := 1
|
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
@@ -324,7 +384,6 @@ func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) {
|
|||||||
Modules: []string{"glossary"},
|
Modules: []string{"glossary"},
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
LLMConcurrency: &llmConcurrency,
|
|
||||||
Report: false,
|
Report: false,
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, false)
|
req := auditaReqForTest(t, false)
|
||||||
@@ -346,7 +405,6 @@ func TestSubprocessRunnerSegmentsMissingFails(t *testing.T) {
|
|||||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
|
|
||||||
llmConcurrency := 1
|
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
@@ -354,7 +412,6 @@ func TestSubprocessRunnerSegmentsMissingFails(t *testing.T) {
|
|||||||
Modules: []string{"glossary"},
|
Modules: []string{"glossary"},
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
LLMConcurrency: &llmConcurrency,
|
|
||||||
Report: false,
|
Report: false,
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, false)
|
req := auditaReqForTest(t, false)
|
||||||
@@ -376,7 +433,6 @@ func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
|
|||||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
|
|
||||||
llmConcurrency := 1
|
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
@@ -384,7 +440,6 @@ func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
|
|||||||
Modules: []string{"glossary"},
|
Modules: []string{"glossary"},
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
LLMConcurrency: &llmConcurrency,
|
|
||||||
Report: true,
|
Report: true,
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, true)
|
req := auditaReqForTest(t, true)
|
||||||
@@ -398,11 +453,11 @@ func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerConstructorValidation(t *testing.T) {
|
func TestSubprocessRunnerConstructorValidation(t *testing.T) {
|
||||||
_, err := NewSubprocessRunnerFromConfigValues("", "3h", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", nil, "", nil, true)
|
_, err := NewSubprocessRunnerFromConfigValues("", "3h", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", "", "", "", "", nil, nil, "", nil, true)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected binary validation error")
|
t.Fatal("expected binary validation error")
|
||||||
}
|
}
|
||||||
_, err = NewSubprocessRunnerFromConfigValues("audita", "bad", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", nil, "", nil, true)
|
_, err = NewSubprocessRunnerFromConfigValues("audita", "bad", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", "", "", "", "", nil, nil, "", nil, true)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected timeout parse error")
|
t.Fatal("expected timeout parse error")
|
||||||
}
|
}
|
||||||
@@ -516,7 +571,7 @@ func mustAuditaRunner(t *testing.T, cfg SubprocessRunnerConfig) *SubprocessRunne
|
|||||||
func auditaReqForTest(t *testing.T, withReport bool) PolishRequest {
|
func auditaReqForTest(t *testing.T, withReport bool) PolishRequest {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
merged := filepath.Join(dir, "merged.json")
|
merged := filepath.Join(dir, "base.json")
|
||||||
glossary := filepath.Join(dir, "glossary.yml")
|
glossary := filepath.Join(dir, "glossary.yml")
|
||||||
writeAuditaTestFile(t, merged, `{"segments":[]}`)
|
writeAuditaTestFile(t, merged, `{"segments":[]}`)
|
||||||
writeAuditaTestFile(t, glossary, "terms: []\n")
|
writeAuditaTestFile(t, glossary, "terms: []\n")
|
||||||
@@ -524,7 +579,7 @@ func auditaReqForTest(t *testing.T, withReport bool) PolishRequest {
|
|||||||
GeneratedConfigPath: filepath.Join(dir, "audita.generated.yml"),
|
GeneratedConfigPath: filepath.Join(dir, "audita.generated.yml"),
|
||||||
MergedTranscriptPath: merged,
|
MergedTranscriptPath: merged,
|
||||||
GlossaryPath: glossary,
|
GlossaryPath: glossary,
|
||||||
OutputProcessedPath: filepath.Join(dir, "processed.json"),
|
OutputProcessedPath: filepath.Join(dir, "polished.json"),
|
||||||
WorkDir: filepath.Join(dir, "artifacts", "audita-work"),
|
WorkDir: filepath.Join(dir, "artifacts", "audita-work"),
|
||||||
StdoutLogPath: filepath.Join(dir, "audita.stdout.log"),
|
StdoutLogPath: filepath.Join(dir, "audita.stdout.log"),
|
||||||
StderrLogPath: filepath.Join(dir, "audita.stderr.log"),
|
StderrLogPath: filepath.Join(dir, "audita.stderr.log"),
|
||||||
|
|||||||
@@ -31,7 +31,7 @@ func TestSubprocessRunnerRunSuccessBuildsDeterministicArgsAndCapturesLogs(t *tes
|
|||||||
ConfigPath: "/etc/scriptorium/config.yml",
|
ConfigPath: "/etc/scriptorium/config.yml",
|
||||||
PromptID: "dnd.session_recap",
|
PromptID: "dnd.session_recap",
|
||||||
ProfileID: "local-quality",
|
ProfileID: "local-quality",
|
||||||
InputPaths: map[string]string{"transcript": filepath.Join(dir, "processed.json"), "other": filepath.Join(dir, "other.md")},
|
InputPaths: map[string]string{"transcript": filepath.Join(dir, "polished.json"), "other": filepath.Join(dir, "other.md")},
|
||||||
Vars: map[string]string{"session_id": "2026-05-03", "campaign_name": "Icewind Dale"},
|
Vars: map[string]string{"session_id": "2026-05-03", "campaign_name": "Icewind Dale"},
|
||||||
OutputPath: filepath.Join(dir, "artifacts", "session_recap.md"),
|
OutputPath: filepath.Join(dir, "artifacts", "session_recap.md"),
|
||||||
StdoutLogPath: filepath.Join(dir, "logs", "scriptorium.run.stdout.log"),
|
StdoutLogPath: filepath.Join(dir, "logs", "scriptorium.run.stdout.log"),
|
||||||
@@ -180,7 +180,7 @@ func TestSubprocessRunnerRenderSuccess(t *testing.T) {
|
|||||||
req := RenderArtifactRequest{
|
req := RenderArtifactRequest{
|
||||||
Binary: wrapper,
|
Binary: wrapper,
|
||||||
PromptID: "dnd.session_recap",
|
PromptID: "dnd.session_recap",
|
||||||
InputPaths: map[string]string{"transcript": filepath.Join(dir, "processed.json")},
|
InputPaths: map[string]string{"transcript": filepath.Join(dir, "polished.json")},
|
||||||
OutputPath: filepath.Join(dir, "artifacts", "session_recap.render.json"),
|
OutputPath: filepath.Join(dir, "artifacts", "session_recap.render.json"),
|
||||||
StdoutLogPath: filepath.Join(dir, "logs", "scriptorium.render.stdout.log"),
|
StdoutLogPath: filepath.Join(dir, "logs", "scriptorium.render.stdout.log"),
|
||||||
StderrLogPath: filepath.Join(dir, "logs", "scriptorium.render.stderr.log"),
|
StderrLogPath: filepath.Join(dir, "logs", "scriptorium.render.stderr.log"),
|
||||||
@@ -285,7 +285,7 @@ type scriptoriumHelperRecord struct {
|
|||||||
func runReqForTest(t *testing.T, binary string) RunArtifactRequest {
|
func runReqForTest(t *testing.T, binary string) RunArtifactRequest {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
transcriptPath := filepath.Join(dir, "processed.json")
|
transcriptPath := filepath.Join(dir, "polished.json")
|
||||||
writeScriptoriumFile(t, transcriptPath, `{"segments":[]}`)
|
writeScriptoriumFile(t, transcriptPath, `{"segments":[]}`)
|
||||||
return RunArtifactRequest{
|
return RunArtifactRequest{
|
||||||
Binary: binary,
|
Binary: binary,
|
||||||
|
|||||||
@@ -68,6 +68,26 @@ func (n *NoopRunner) Normalize(ctx context.Context, req NormalizeRequest) (Norma
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Render returns the requested output path with placeholder metadata.
|
||||||
|
func (n *NoopRunner) Render(ctx context.Context, req RenderRequest) (RenderResult, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return RenderResult{}, err
|
||||||
|
}
|
||||||
|
if err := materializeRenderPlaceholders(req); err != nil {
|
||||||
|
return RenderResult{}, err
|
||||||
|
}
|
||||||
|
return RenderResult{
|
||||||
|
OutputRenderedPath: req.OutputRenderedPath,
|
||||||
|
StdoutLogPath: req.StdoutLogPath,
|
||||||
|
StderrLogPath: req.StderrLogPath,
|
||||||
|
GeneratedConfigPath: req.GeneratedConfigPath,
|
||||||
|
InvokedBinary: "noop",
|
||||||
|
Format: req.Format,
|
||||||
|
Title: req.Title,
|
||||||
|
Metadata: map[string]any{"placeholder": true},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
// FakeRunner captures merge requests and returns deterministic responses.
|
// FakeRunner captures merge requests and returns deterministic responses.
|
||||||
type FakeRunner struct {
|
type FakeRunner struct {
|
||||||
Requests []MergeRequest
|
Requests []MergeRequest
|
||||||
@@ -79,6 +99,9 @@ type FakeRunner struct {
|
|||||||
TrimRequests []TrimRequest
|
TrimRequests []TrimRequest
|
||||||
TrimErr error
|
TrimErr error
|
||||||
TrimResult TrimResult
|
TrimResult TrimResult
|
||||||
|
RenderRequests []RenderRequest
|
||||||
|
RenderErr error
|
||||||
|
RenderResult RenderResult
|
||||||
}
|
}
|
||||||
|
|
||||||
// Run records request and returns configured response.
|
// Run records request and returns configured response.
|
||||||
@@ -195,6 +218,46 @@ func (f *FakeRunner) Normalize(ctx context.Context, req NormalizeRequest) (Norma
|
|||||||
return res, nil
|
return res, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Render records request and returns configured response.
|
||||||
|
func (f *FakeRunner) Render(ctx context.Context, req RenderRequest) (RenderResult, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return RenderResult{}, err
|
||||||
|
}
|
||||||
|
f.RenderRequests = append(f.RenderRequests, req)
|
||||||
|
if f.RenderErr != nil {
|
||||||
|
return RenderResult{}, f.RenderErr
|
||||||
|
}
|
||||||
|
if err := materializeRenderPlaceholders(req); err != nil {
|
||||||
|
return RenderResult{}, err
|
||||||
|
}
|
||||||
|
res := f.RenderResult
|
||||||
|
if res.OutputRenderedPath == "" {
|
||||||
|
res.OutputRenderedPath = req.OutputRenderedPath
|
||||||
|
}
|
||||||
|
if res.StdoutLogPath == "" {
|
||||||
|
res.StdoutLogPath = req.StdoutLogPath
|
||||||
|
}
|
||||||
|
if res.StderrLogPath == "" {
|
||||||
|
res.StderrLogPath = req.StderrLogPath
|
||||||
|
}
|
||||||
|
if res.GeneratedConfigPath == "" {
|
||||||
|
res.GeneratedConfigPath = req.GeneratedConfigPath
|
||||||
|
}
|
||||||
|
if res.InvokedBinary == "" {
|
||||||
|
res.InvokedBinary = "fake"
|
||||||
|
}
|
||||||
|
if res.Format == "" {
|
||||||
|
res.Format = req.Format
|
||||||
|
}
|
||||||
|
if res.Title == "" {
|
||||||
|
res.Title = req.Title
|
||||||
|
}
|
||||||
|
if res.Metadata == nil {
|
||||||
|
res.Metadata = map[string]any{"fake": true}
|
||||||
|
}
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
func materializePlaceholders(req MergeRequest) error {
|
func materializePlaceholders(req MergeRequest) error {
|
||||||
if req.OutputMergedTranscriptPath != "" {
|
if req.OutputMergedTranscriptPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.OutputMergedTranscriptPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), 0o644); err != nil {
|
if err := subprocess.WriteFileAtomic(req.OutputMergedTranscriptPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), 0o644); err != nil {
|
||||||
@@ -301,3 +364,39 @@ func materializeNormalizePlaceholders(req NormalizeRequest) error {
|
|||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func materializeRenderPlaceholders(req RenderRequest) error {
|
||||||
|
if req.OutputRenderedPath != "" {
|
||||||
|
if err := subprocess.WriteFileAtomic(req.OutputRenderedPath, []byte("# Transcript\n\nRendered markdown placeholder.\n"), 0o644); err != nil {
|
||||||
|
return fmt.Errorf("write rendered transcript %q: %w", req.OutputRenderedPath, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if req.GeneratedConfigPath != "" {
|
||||||
|
payload := map[string]any{
|
||||||
|
"schema": "seriatim.generated.v1",
|
||||||
|
"placeholder": true,
|
||||||
|
"command": "render",
|
||||||
|
"input_path": req.InputTranscriptPath,
|
||||||
|
"output_path": req.OutputRenderedPath,
|
||||||
|
"format": req.Format,
|
||||||
|
"title": req.Title,
|
||||||
|
"include_timestamps": req.IncludeTimestamps,
|
||||||
|
"include_segment_ids": req.IncludeSegmentIDs,
|
||||||
|
"include_metadata": req.IncludeMetadata,
|
||||||
|
}
|
||||||
|
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644); err != nil {
|
||||||
|
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if req.StdoutLogPath != "" {
|
||||||
|
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake render stdout placeholder\n"), 0o644); err != nil {
|
||||||
|
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if req.StderrLogPath != "" {
|
||||||
|
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake render stderr placeholder\n"), 0o644); err != nil {
|
||||||
|
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ func TestFakeRunnerCapturesRequestAndReturnsPath(t *testing.T) {
|
|||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
req := MergeRequest{
|
req := MergeRequest{
|
||||||
GeneratedConfigPath: filepath.Join(dir, "config", "seriatim.yml"),
|
GeneratedConfigPath: filepath.Join(dir, "config", "seriatim.yml"),
|
||||||
OutputMergedTranscriptPath: filepath.Join(dir, "transcripts", "merged.json"),
|
OutputMergedTranscriptPath: filepath.Join(dir, "transcripts", "base.json"),
|
||||||
StdoutLogPath: filepath.Join(dir, "logs", "seriatim.stdout.log"),
|
StdoutLogPath: filepath.Join(dir, "logs", "seriatim.stdout.log"),
|
||||||
StderrLogPath: filepath.Join(dir, "logs", "seriatim.stderr.log"),
|
StderrLogPath: filepath.Join(dir, "logs", "seriatim.stderr.log"),
|
||||||
}
|
}
|
||||||
@@ -57,8 +57,8 @@ func TestFakeRunnerTrimCapturesRequestAndReturnsPath(t *testing.T) {
|
|||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
req := TrimRequest{
|
req := TrimRequest{
|
||||||
GeneratedConfigPath: filepath.Join(dir, "config", "seriatim.trim.yml"),
|
GeneratedConfigPath: filepath.Join(dir, "config", "seriatim.trim.yml"),
|
||||||
InputTranscriptPath: filepath.Join(dir, "transcripts", "processed.json"),
|
InputTranscriptPath: filepath.Join(dir, "transcripts", "polished.json"),
|
||||||
OutputTrimmedPath: filepath.Join(dir, "transcripts", "trimmed.json"),
|
OutputTrimmedPath: filepath.Join(dir, "transcripts", "final.trimmed.json"),
|
||||||
KeepSelector: "1-10",
|
KeepSelector: "1-10",
|
||||||
StdoutLogPath: filepath.Join(dir, "logs", "seriatim.trim.stdout.log"),
|
StdoutLogPath: filepath.Join(dir, "logs", "seriatim.trim.stdout.log"),
|
||||||
StderrLogPath: filepath.Join(dir, "logs", "seriatim.trim.stderr.log"),
|
StderrLogPath: filepath.Join(dir, "logs", "seriatim.trim.stderr.log"),
|
||||||
@@ -105,8 +105,8 @@ func TestFakeRunnerNormalizeCapturesRequestAndReturnsPath(t *testing.T) {
|
|||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
req := NormalizeRequest{
|
req := NormalizeRequest{
|
||||||
GeneratedConfigPath: filepath.Join(dir, "config", "seriatim.normalize.yml"),
|
GeneratedConfigPath: filepath.Join(dir, "config", "seriatim.normalize.yml"),
|
||||||
InputTranscriptPath: filepath.Join(dir, "transcripts", "processed.json"),
|
InputTranscriptPath: filepath.Join(dir, "transcripts", "polished.json"),
|
||||||
OutputNormalizedPath: filepath.Join(dir, "transcripts", "normalized.json"),
|
OutputNormalizedPath: filepath.Join(dir, "transcripts", "final.json"),
|
||||||
OutputSchema: "seriatim-intermediate",
|
OutputSchema: "seriatim-intermediate",
|
||||||
ReportPath: filepath.Join(dir, "artifacts", "seriatim.normalize.report.json"),
|
ReportPath: filepath.Join(dir, "artifacts", "seriatim.normalize.report.json"),
|
||||||
StdoutLogPath: filepath.Join(dir, "logs", "seriatim.normalize.stdout.log"),
|
StdoutLogPath: filepath.Join(dir, "logs", "seriatim.normalize.stdout.log"),
|
||||||
@@ -148,3 +148,58 @@ func TestFakeRunnerNormalizeError(t *testing.T) {
|
|||||||
t.Fatal("expected error, got nil")
|
t.Fatal("expected error, got nil")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFakeRunnerRenderCapturesRequestAndReturnsPath(t *testing.T) {
|
||||||
|
fake := &FakeRunner{}
|
||||||
|
dir := t.TempDir()
|
||||||
|
req := RenderRequest{
|
||||||
|
GeneratedConfigPath: filepath.Join(dir, "config", "seriatim.render.yml"),
|
||||||
|
InputTranscriptPath: filepath.Join(dir, "transcripts", "final.trimmed.json"),
|
||||||
|
OutputRenderedPath: filepath.Join(dir, "transcripts", "final.trimmed.md"),
|
||||||
|
Format: "markdown",
|
||||||
|
Title: "Session render",
|
||||||
|
IncludeTimestamps: true,
|
||||||
|
IncludeSegmentIDs: false,
|
||||||
|
IncludeMetadata: true,
|
||||||
|
StdoutLogPath: filepath.Join(dir, "logs", "seriatim.render.stdout.log"),
|
||||||
|
StderrLogPath: filepath.Join(dir, "logs", "seriatim.render.stderr.log"),
|
||||||
|
}
|
||||||
|
|
||||||
|
res, err := fake.Render(context.Background(), req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Render() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(fake.RenderRequests) != 1 || fake.RenderRequests[0].GeneratedConfigPath == "" {
|
||||||
|
t.Fatalf("render requests = %#v, want captured request", fake.RenderRequests)
|
||||||
|
}
|
||||||
|
if res.OutputRenderedPath != req.OutputRenderedPath {
|
||||||
|
t.Fatalf("rendered path = %q, want %q", res.OutputRenderedPath, req.OutputRenderedPath)
|
||||||
|
}
|
||||||
|
if res.Format != req.Format {
|
||||||
|
t.Fatalf("format = %q, want %q", res.Format, req.Format)
|
||||||
|
}
|
||||||
|
if res.Title != req.Title {
|
||||||
|
t.Fatalf("title = %q, want %q", res.Title, req.Title)
|
||||||
|
}
|
||||||
|
|
||||||
|
cfgData, err := os.ReadFile(req.GeneratedConfigPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read generated config: %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(cfgData), "command: render") {
|
||||||
|
t.Fatalf("generated config = %q, want render command marker", string(cfgData))
|
||||||
|
}
|
||||||
|
for _, path := range []string{req.StdoutLogPath, req.StderrLogPath, req.OutputRenderedPath} {
|
||||||
|
if _, err := os.Stat(path); err != nil {
|
||||||
|
t.Fatalf("expected file %q to exist: %v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFakeRunnerRenderError(t *testing.T) {
|
||||||
|
fake := &FakeRunner{RenderErr: errors.New("boom")}
|
||||||
|
_, err := fake.Render(context.Background(), RenderRequest{})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error, got nil")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
// Package seriatim declares the adapter contract for transcript merge/normalize/trim execution.
|
// Package seriatim declares the adapter contract for transcript merge/normalize/trim/render execution.
|
||||||
package seriatim
|
package seriatim
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -6,11 +6,12 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Runner is the adapter boundary for seriatim merge/normalize/trim invocations.
|
// Runner is the adapter boundary for seriatim merge/normalize/trim/render invocations.
|
||||||
type Runner interface {
|
type Runner interface {
|
||||||
Run(ctx context.Context, req MergeRequest) (MergeResult, error)
|
Run(ctx context.Context, req MergeRequest) (MergeResult, error)
|
||||||
Normalize(ctx context.Context, req NormalizeRequest) (NormalizeResult, error)
|
Normalize(ctx context.Context, req NormalizeRequest) (NormalizeResult, error)
|
||||||
Trim(ctx context.Context, req TrimRequest) (TrimResult, error)
|
Trim(ctx context.Context, req TrimRequest) (TrimResult, error)
|
||||||
|
Render(ctx context.Context, req RenderRequest) (RenderResult, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// MergeRequest describes a seriatim merge invocation.
|
// MergeRequest describes a seriatim merge invocation.
|
||||||
@@ -90,3 +91,33 @@ type TrimResult struct {
|
|||||||
KeepSelector string
|
KeepSelector string
|
||||||
Metadata map[string]any
|
Metadata map[string]any
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RenderRequest describes a seriatim render invocation.
|
||||||
|
type RenderRequest struct {
|
||||||
|
Binary string
|
||||||
|
InputTranscriptPath string
|
||||||
|
OutputRenderedPath string
|
||||||
|
Format string
|
||||||
|
Title string
|
||||||
|
IncludeTimestamps bool
|
||||||
|
IncludeSegmentIDs bool
|
||||||
|
IncludeMetadata bool
|
||||||
|
StdoutLogPath string
|
||||||
|
StderrLogPath string
|
||||||
|
GeneratedConfigPath string
|
||||||
|
Timeout time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// RenderResult describes a render output.
|
||||||
|
type RenderResult struct {
|
||||||
|
OutputRenderedPath string
|
||||||
|
StdoutLogPath string
|
||||||
|
StderrLogPath string
|
||||||
|
GeneratedConfigPath string
|
||||||
|
ExitCode int
|
||||||
|
Duration time.Duration
|
||||||
|
InvokedBinary string
|
||||||
|
Format string
|
||||||
|
Title string
|
||||||
|
Metadata map[string]any
|
||||||
|
}
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
"unicode/utf8"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
||||||
)
|
)
|
||||||
@@ -384,6 +385,96 @@ func (r *SubprocessRunner) Normalize(ctx context.Context, req NormalizeRequest)
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Render executes Seriatim render with deterministic flags and validates non-empty text output.
|
||||||
|
func (r *SubprocessRunner) Render(ctx context.Context, req RenderRequest) (RenderResult, error) {
|
||||||
|
if r == nil {
|
||||||
|
return RenderResult{}, fmt.Errorf("seriatim subprocess runner is nil")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(req.InputTranscriptPath) == "" {
|
||||||
|
return RenderResult{}, fmt.Errorf("seriatim render input path is required")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(req.OutputRenderedPath) == "" {
|
||||||
|
return RenderResult{}, fmt.Errorf("seriatim render output path is required")
|
||||||
|
}
|
||||||
|
format := strings.TrimSpace(req.Format)
|
||||||
|
if format == "" {
|
||||||
|
format = "markdown"
|
||||||
|
}
|
||||||
|
if format != "markdown" {
|
||||||
|
return RenderResult{}, fmt.Errorf("seriatim render format %q is unsupported", req.Format)
|
||||||
|
}
|
||||||
|
|
||||||
|
binary := r.binary
|
||||||
|
if strings.TrimSpace(req.Binary) != "" {
|
||||||
|
binary = strings.TrimSpace(req.Binary)
|
||||||
|
}
|
||||||
|
|
||||||
|
timeout := r.timeout
|
||||||
|
if req.Timeout < 0 {
|
||||||
|
return RenderResult{}, fmt.Errorf("seriatim render timeout must be >= 0")
|
||||||
|
}
|
||||||
|
if req.Timeout > 0 {
|
||||||
|
timeout = req.Timeout
|
||||||
|
}
|
||||||
|
|
||||||
|
args := buildRenderArgs(req, format)
|
||||||
|
if req.GeneratedConfigPath != "" {
|
||||||
|
if err := writeRenderInvocationConfig(req, args, binary, timeout, format); err != nil {
|
||||||
|
return RenderResult{}, fmt.Errorf("write seriatim render invocation config %q: %w", req.GeneratedConfigPath, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
runRes, err := subprocess.Run(ctx, subprocess.RunRequest{
|
||||||
|
Executable: binary,
|
||||||
|
Args: args,
|
||||||
|
Timeout: timeout,
|
||||||
|
StdoutLogPath: req.StdoutLogPath,
|
||||||
|
StderrLogPath: req.StderrLogPath,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return RenderResult{
|
||||||
|
OutputRenderedPath: req.OutputRenderedPath,
|
||||||
|
StdoutLogPath: req.StdoutLogPath,
|
||||||
|
StderrLogPath: req.StderrLogPath,
|
||||||
|
GeneratedConfigPath: req.GeneratedConfigPath,
|
||||||
|
ExitCode: runRes.ExitCode,
|
||||||
|
Duration: runRes.Duration,
|
||||||
|
InvokedBinary: binary,
|
||||||
|
Format: format,
|
||||||
|
Title: req.Title,
|
||||||
|
}, fmt.Errorf("run seriatim render (binary=%q): %w", binary, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := validateNonEmptyTextFile(req.OutputRenderedPath); err != nil {
|
||||||
|
return RenderResult{
|
||||||
|
OutputRenderedPath: req.OutputRenderedPath,
|
||||||
|
StdoutLogPath: req.StdoutLogPath,
|
||||||
|
StderrLogPath: req.StderrLogPath,
|
||||||
|
GeneratedConfigPath: req.GeneratedConfigPath,
|
||||||
|
ExitCode: runRes.ExitCode,
|
||||||
|
Duration: runRes.Duration,
|
||||||
|
InvokedBinary: binary,
|
||||||
|
Format: format,
|
||||||
|
Title: req.Title,
|
||||||
|
}, fmt.Errorf("validate seriatim rendered output %q: %w", req.OutputRenderedPath, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return RenderResult{
|
||||||
|
OutputRenderedPath: req.OutputRenderedPath,
|
||||||
|
StdoutLogPath: req.StdoutLogPath,
|
||||||
|
StderrLogPath: req.StderrLogPath,
|
||||||
|
GeneratedConfigPath: req.GeneratedConfigPath,
|
||||||
|
ExitCode: runRes.ExitCode,
|
||||||
|
Duration: runRes.Duration,
|
||||||
|
InvokedBinary: binary,
|
||||||
|
Format: format,
|
||||||
|
Title: req.Title,
|
||||||
|
Metadata: map[string]any{
|
||||||
|
"adapter": "seriatim_subprocess",
|
||||||
|
},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (r *SubprocessRunner) buildMergeArgs(req MergeRequest) []string {
|
func (r *SubprocessRunner) buildMergeArgs(req MergeRequest) []string {
|
||||||
args := []string{"merge"}
|
args := []string{"merge"}
|
||||||
|
|
||||||
@@ -480,6 +571,22 @@ func buildNormalizeArgs(req NormalizeRequest, outputSchema string) []string {
|
|||||||
return args
|
return args
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func buildRenderArgs(req RenderRequest, format string) []string {
|
||||||
|
args := []string{
|
||||||
|
"render",
|
||||||
|
"--input-file", req.InputTranscriptPath,
|
||||||
|
"--output-file", req.OutputRenderedPath,
|
||||||
|
"--format", format,
|
||||||
|
"--include-timestamps=" + strconv.FormatBool(req.IncludeTimestamps),
|
||||||
|
"--include-segment-ids=" + strconv.FormatBool(req.IncludeSegmentIDs),
|
||||||
|
"--include-metadata=" + strconv.FormatBool(req.IncludeMetadata),
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(req.Title) != "" {
|
||||||
|
args = append(args, "--title", req.Title)
|
||||||
|
}
|
||||||
|
return args
|
||||||
|
}
|
||||||
|
|
||||||
func writeTrimInvocationConfig(req TrimRequest, args []string, binary string, timeout time.Duration) error {
|
func writeTrimInvocationConfig(req TrimRequest, args []string, binary string, timeout time.Duration) error {
|
||||||
payload := map[string]any{
|
payload := map[string]any{
|
||||||
"schema": "seriatim.generated.v1",
|
"schema": "seriatim.generated.v1",
|
||||||
@@ -509,6 +616,24 @@ func writeNormalizeInvocationConfig(req NormalizeRequest, args []string, binary
|
|||||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644)
|
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func writeRenderInvocationConfig(req RenderRequest, args []string, binary string, timeout time.Duration, format string) error {
|
||||||
|
payload := map[string]any{
|
||||||
|
"schema": "seriatim.generated.v1",
|
||||||
|
"command": "render",
|
||||||
|
"binary": binary,
|
||||||
|
"args": args,
|
||||||
|
"timeout": timeout.String(),
|
||||||
|
"input_path": req.InputTranscriptPath,
|
||||||
|
"output_path": req.OutputRenderedPath,
|
||||||
|
"format": format,
|
||||||
|
"title": req.Title,
|
||||||
|
"include_timestamps": req.IncludeTimestamps,
|
||||||
|
"include_segment_ids": req.IncludeSegmentIDs,
|
||||||
|
"include_metadata": req.IncludeMetadata,
|
||||||
|
}
|
||||||
|
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644)
|
||||||
|
}
|
||||||
|
|
||||||
func validateJSONFile(path string) error {
|
func validateJSONFile(path string) error {
|
||||||
data, err := os.ReadFile(path)
|
data, err := os.ReadFile(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -541,3 +666,20 @@ func validateJSONFileWithSegments(path string) error {
|
|||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validateNonEmptyTextFile(path string) error {
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("read file: %w", err)
|
||||||
|
}
|
||||||
|
if len(data) == 0 {
|
||||||
|
return fmt.Errorf("file is empty")
|
||||||
|
}
|
||||||
|
if !utf8.Valid(data) {
|
||||||
|
return fmt.Errorf("file is not valid utf-8 text")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(string(data)) == "" {
|
||||||
|
return fmt.Errorf("file has no non-whitespace content")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -50,7 +50,7 @@ func TestSubprocessRunnerSuccessWithReportArgsAndEnv(t *testing.T) {
|
|||||||
req := MergeRequest{
|
req := MergeRequest{
|
||||||
GeneratedConfigPath: filepath.Join(dir, "seriatim.generated.yml"),
|
GeneratedConfigPath: filepath.Join(dir, "seriatim.generated.yml"),
|
||||||
InputTranscriptPaths: []string{filepath.Join(dir, "a.json"), filepath.Join(dir, "b.json")},
|
InputTranscriptPaths: []string{filepath.Join(dir, "a.json"), filepath.Join(dir, "b.json")},
|
||||||
OutputMergedTranscriptPath: filepath.Join(dir, "merged.json"),
|
OutputMergedTranscriptPath: filepath.Join(dir, "base.json"),
|
||||||
ReportPath: filepath.Join(dir, "seriatim.report.json"),
|
ReportPath: filepath.Join(dir, "seriatim.report.json"),
|
||||||
SpeakersPath: filepath.Join(dir, "speakers.yml"),
|
SpeakersPath: filepath.Join(dir, "speakers.yml"),
|
||||||
AutocorrectPath: filepath.Join(dir, "autocorrect.yml"),
|
AutocorrectPath: filepath.Join(dir, "autocorrect.yml"),
|
||||||
@@ -569,6 +569,156 @@ func TestSubprocessRunnerNormalizeInvalidReportJSONFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSubprocessRunnerRenderSuccessInvocationAndProvenance(t *testing.T) {
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
t.Skip("helper wrapper script uses /bin/sh")
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
|
t.Setenv("SERIATIM_HELPER_MODE", "render_success")
|
||||||
|
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||||
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", recordPath)
|
||||||
|
|
||||||
|
wrapper := writeHelperWrapper(t)
|
||||||
|
runner := mustRunner(t, wrapper, false)
|
||||||
|
req := renderReqForTest(t)
|
||||||
|
|
||||||
|
res, err := runner.Render(context.Background(), req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Render() error = %v", err)
|
||||||
|
}
|
||||||
|
if res.OutputRenderedPath != req.OutputRenderedPath {
|
||||||
|
t.Fatalf("OutputRenderedPath = %q, want %q", res.OutputRenderedPath, req.OutputRenderedPath)
|
||||||
|
}
|
||||||
|
if res.Format != req.Format {
|
||||||
|
t.Fatalf("Format = %q, want %q", res.Format, req.Format)
|
||||||
|
}
|
||||||
|
if res.Title != req.Title {
|
||||||
|
t.Fatalf("Title = %q, want %q", res.Title, req.Title)
|
||||||
|
}
|
||||||
|
if res.InvokedBinary != wrapper {
|
||||||
|
t.Fatalf("InvokedBinary = %q, want %q", res.InvokedBinary, wrapper)
|
||||||
|
}
|
||||||
|
if res.ExitCode != 0 {
|
||||||
|
t.Fatalf("ExitCode = %d, want 0", res.ExitCode)
|
||||||
|
}
|
||||||
|
if res.Duration <= 0 {
|
||||||
|
t.Fatalf("Duration = %s, want >0", res.Duration)
|
||||||
|
}
|
||||||
|
if res.Metadata == nil || res.Metadata["adapter"] != "seriatim_subprocess" {
|
||||||
|
t.Fatalf("Metadata = %#v, want adapter marker", res.Metadata)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := os.Stat(req.OutputRenderedPath); err != nil {
|
||||||
|
t.Fatalf("rendered output missing: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(req.StdoutLogPath); err != nil {
|
||||||
|
t.Fatalf("stdout log missing: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(req.StderrLogPath); err != nil {
|
||||||
|
t.Fatalf("stderr log missing: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(req.GeneratedConfigPath); err != nil {
|
||||||
|
t.Fatalf("generated config missing: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
rec := readHelperRecord(t, recordPath)
|
||||||
|
wantArgs := []string{
|
||||||
|
"render",
|
||||||
|
"--input-file", req.InputTranscriptPath,
|
||||||
|
"--output-file", req.OutputRenderedPath,
|
||||||
|
"--format", req.Format,
|
||||||
|
"--include-timestamps=true",
|
||||||
|
"--include-segment-ids=true",
|
||||||
|
"--include-metadata=false",
|
||||||
|
"--title", req.Title,
|
||||||
|
}
|
||||||
|
if strings.Join(rec.Args, "\n") != strings.Join(wantArgs, "\n") {
|
||||||
|
t.Fatalf("args = %#v, want %#v", rec.Args, wantArgs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessRunnerRenderWithoutTitleOmitsTitleArg(t *testing.T) {
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
t.Skip("helper wrapper script uses /bin/sh")
|
||||||
|
}
|
||||||
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
|
t.Setenv("SERIATIM_HELPER_MODE", "render_success")
|
||||||
|
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||||
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", recordPath)
|
||||||
|
|
||||||
|
runner := mustRunner(t, writeHelperWrapper(t), false)
|
||||||
|
req := renderReqForTest(t)
|
||||||
|
req.Title = ""
|
||||||
|
if _, err := runner.Render(context.Background(), req); err != nil {
|
||||||
|
t.Fatalf("Render() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
rec := readHelperRecord(t, recordPath)
|
||||||
|
for i := 0; i < len(rec.Args); i++ {
|
||||||
|
if rec.Args[i] == "--title" {
|
||||||
|
t.Fatalf("args = %#v, did not expect --title", rec.Args)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessRunnerRenderSubprocessFailure(t *testing.T) {
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
t.Skip("helper wrapper script uses /bin/sh")
|
||||||
|
}
|
||||||
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
|
t.Setenv("SERIATIM_HELPER_MODE", "fail")
|
||||||
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
|
|
||||||
|
runner := mustRunner(t, writeHelperWrapper(t), false)
|
||||||
|
req := renderReqForTest(t)
|
||||||
|
_, err := runner.Render(context.Background(), req)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Render() error = nil, want non-nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "run seriatim render") {
|
||||||
|
t.Fatalf("error = %q, want subprocess context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessRunnerRenderMissingOutputFails(t *testing.T) {
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
t.Skip("helper wrapper script uses /bin/sh")
|
||||||
|
}
|
||||||
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
|
t.Setenv("SERIATIM_HELPER_MODE", "missing_output")
|
||||||
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
|
|
||||||
|
runner := mustRunner(t, writeHelperWrapper(t), false)
|
||||||
|
req := renderReqForTest(t)
|
||||||
|
_, err := runner.Render(context.Background(), req)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Render() error = nil, want non-nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "validate seriatim rendered output") {
|
||||||
|
t.Fatalf("error = %q, want output validation context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessRunnerRenderEmptyOutputFails(t *testing.T) {
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
t.Skip("helper wrapper script uses /bin/sh")
|
||||||
|
}
|
||||||
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
|
t.Setenv("SERIATIM_HELPER_MODE", "render_empty_output")
|
||||||
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
|
|
||||||
|
runner := mustRunner(t, writeHelperWrapper(t), false)
|
||||||
|
req := renderReqForTest(t)
|
||||||
|
_, err := runner.Render(context.Background(), req)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Render() error = nil, want non-nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "file is empty") {
|
||||||
|
t.Fatalf("error = %q, want empty-file validation", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerConstructorValidation(t *testing.T) {
|
func TestSubprocessRunnerConstructorValidation(t *testing.T) {
|
||||||
_, err := NewSubprocessRunnerFromConfigValues("", "10m", "seriatim-intermediate", nil, true, EnvConfig{})
|
_, err := NewSubprocessRunnerFromConfigValues("", "10m", "seriatim-intermediate", nil, true, EnvConfig{})
|
||||||
if err == nil {
|
if err == nil {
|
||||||
@@ -702,6 +852,14 @@ func TestSeriatimSubprocessHelper(t *testing.T) {
|
|||||||
case "normalize_report_missing":
|
case "normalize_report_missing":
|
||||||
writeSeriatimHelperFile(outputPath, `{"schema":"seriatim.intermediate.v1","segments":[]}`)
|
writeSeriatimHelperFile(outputPath, `{"schema":"seriatim.intermediate.v1","segments":[]}`)
|
||||||
os.Exit(0)
|
os.Exit(0)
|
||||||
|
case "render_success":
|
||||||
|
writeSeriatimHelperFile(outputPath, "# Rendered transcript\n\nHello.\n")
|
||||||
|
_, _ = os.Stdout.WriteString("seriatim helper render stdout\n")
|
||||||
|
_, _ = os.Stderr.WriteString("seriatim helper render stderr\n")
|
||||||
|
os.Exit(0)
|
||||||
|
case "render_empty_output":
|
||||||
|
writeSeriatimHelperFile(outputPath, "")
|
||||||
|
os.Exit(0)
|
||||||
default:
|
default:
|
||||||
_, _ = os.Stderr.WriteString(fmt.Sprintf("unknown helper mode %q\n", mode))
|
_, _ = os.Stderr.WriteString(fmt.Sprintf("unknown helper mode %q\n", mode))
|
||||||
os.Exit(2)
|
os.Exit(2)
|
||||||
@@ -732,7 +890,7 @@ func mergeReqForTest(t *testing.T, withReport bool) MergeRequest {
|
|||||||
req := MergeRequest{
|
req := MergeRequest{
|
||||||
GeneratedConfigPath: filepath.Join(dir, "seriatim.generated.yml"),
|
GeneratedConfigPath: filepath.Join(dir, "seriatim.generated.yml"),
|
||||||
InputTranscriptPaths: []string{in1, in2},
|
InputTranscriptPaths: []string{in1, in2},
|
||||||
OutputMergedTranscriptPath: filepath.Join(dir, "merged.json"),
|
OutputMergedTranscriptPath: filepath.Join(dir, "base.json"),
|
||||||
StdoutLogPath: filepath.Join(dir, "seriatim.stdout.log"),
|
StdoutLogPath: filepath.Join(dir, "seriatim.stdout.log"),
|
||||||
StderrLogPath: filepath.Join(dir, "seriatim.stderr.log"),
|
StderrLogPath: filepath.Join(dir, "seriatim.stderr.log"),
|
||||||
}
|
}
|
||||||
@@ -745,11 +903,11 @@ func mergeReqForTest(t *testing.T, withReport bool) MergeRequest {
|
|||||||
func trimReqForTest(t *testing.T) TrimRequest {
|
func trimReqForTest(t *testing.T) TrimRequest {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
input := filepath.Join(dir, "processed.json")
|
input := filepath.Join(dir, "polished.json")
|
||||||
writeSeriatimFile(t, input, `{"schema":"seriatim.intermediate.v1","segments":[]}`)
|
writeSeriatimFile(t, input, `{"schema":"seriatim.intermediate.v1","segments":[]}`)
|
||||||
return TrimRequest{
|
return TrimRequest{
|
||||||
InputTranscriptPath: input,
|
InputTranscriptPath: input,
|
||||||
OutputTrimmedPath: filepath.Join(dir, "trimmed.json"),
|
OutputTrimmedPath: filepath.Join(dir, "final.trimmed.json"),
|
||||||
KeepSelector: "5-12",
|
KeepSelector: "5-12",
|
||||||
GeneratedConfigPath: filepath.Join(dir, "seriatim.trim.generated.yml"),
|
GeneratedConfigPath: filepath.Join(dir, "seriatim.trim.generated.yml"),
|
||||||
StdoutLogPath: filepath.Join(dir, "seriatim.trim.stdout.log"),
|
StdoutLogPath: filepath.Join(dir, "seriatim.trim.stdout.log"),
|
||||||
@@ -760,12 +918,12 @@ func trimReqForTest(t *testing.T) TrimRequest {
|
|||||||
func normalizeReqForTest(t *testing.T, withReport bool) NormalizeRequest {
|
func normalizeReqForTest(t *testing.T, withReport bool) NormalizeRequest {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
input := filepath.Join(dir, "processed.json")
|
input := filepath.Join(dir, "polished.json")
|
||||||
writeSeriatimFile(t, input, `{"schema":"audita.processed.v1","segments":[]}`)
|
writeSeriatimFile(t, input, `{"schema":"audita.processed.v1","segments":[]}`)
|
||||||
|
|
||||||
req := NormalizeRequest{
|
req := NormalizeRequest{
|
||||||
InputTranscriptPath: input,
|
InputTranscriptPath: input,
|
||||||
OutputNormalizedPath: filepath.Join(dir, "normalized.json"),
|
OutputNormalizedPath: filepath.Join(dir, "final.json"),
|
||||||
OutputSchema: "seriatim-intermediate",
|
OutputSchema: "seriatim-intermediate",
|
||||||
GeneratedConfigPath: filepath.Join(dir, "seriatim.normalize.generated.yml"),
|
GeneratedConfigPath: filepath.Join(dir, "seriatim.normalize.generated.yml"),
|
||||||
StdoutLogPath: filepath.Join(dir, "seriatim.normalize.stdout.log"),
|
StdoutLogPath: filepath.Join(dir, "seriatim.normalize.stdout.log"),
|
||||||
@@ -777,6 +935,25 @@ func normalizeReqForTest(t *testing.T, withReport bool) NormalizeRequest {
|
|||||||
return req
|
return req
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func renderReqForTest(t *testing.T) RenderRequest {
|
||||||
|
t.Helper()
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := filepath.Join(dir, "final.trimmed.json")
|
||||||
|
writeSeriatimFile(t, input, `{"schema":"seriatim.intermediate.v1","segments":[]}`)
|
||||||
|
return RenderRequest{
|
||||||
|
InputTranscriptPath: input,
|
||||||
|
OutputRenderedPath: filepath.Join(dir, "final.trimmed.md"),
|
||||||
|
Format: "markdown",
|
||||||
|
Title: "Session 42",
|
||||||
|
IncludeTimestamps: true,
|
||||||
|
IncludeSegmentIDs: true,
|
||||||
|
IncludeMetadata: false,
|
||||||
|
GeneratedConfigPath: filepath.Join(dir, "seriatim.render.generated.yml"),
|
||||||
|
StdoutLogPath: filepath.Join(dir, "seriatim.render.stdout.log"),
|
||||||
|
StderrLogPath: filepath.Join(dir, "seriatim.render.stderr.log"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func mustRunner(t *testing.T, binary string, report bool) *SubprocessRunner {
|
func mustRunner(t *testing.T, binary string, report bool) *SubprocessRunner {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
coalesce := 3.0
|
coalesce := 3.0
|
||||||
|
|||||||
@@ -1,31 +0,0 @@
|
|||||||
// Package storage declares archive/storage backend adapter boundaries.
|
|
||||||
package storage
|
|
||||||
|
|
||||||
import "context"
|
|
||||||
|
|
||||||
// TODO: implement remote storage/archive backends (S3/SFTP/etc.).
|
|
||||||
|
|
||||||
// Backend is the adapter boundary for archive/storage operations.
|
|
||||||
type Backend interface {
|
|
||||||
Archive(ctx context.Context, req ArchiveRequest) (ArchiveResult, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ArchiveItem describes one item to archive.
|
|
||||||
type ArchiveItem struct {
|
|
||||||
Kind string
|
|
||||||
LocalPath string
|
|
||||||
RemoteKey string
|
|
||||||
}
|
|
||||||
|
|
||||||
// ArchiveRequest describes one archive operation.
|
|
||||||
type ArchiveRequest struct {
|
|
||||||
SessionID string
|
|
||||||
ManifestPath string
|
|
||||||
Items []ArchiveItem
|
|
||||||
}
|
|
||||||
|
|
||||||
// ArchiveResult describes archive operation output.
|
|
||||||
type ArchiveResult struct {
|
|
||||||
Archived []ArchiveItem
|
|
||||||
Metadata map[string]any
|
|
||||||
}
|
|
||||||
29
internal/adapters/storage/factory.go
Normal file
29
internal/adapters/storage/factory.go
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
// NewObjectStoreFromConfig constructs a remote object store from resolved config.
|
||||||
|
func NewObjectStoreFromConfig(ctx context.Context, cfg *config.Config) (ObjectStore, error) {
|
||||||
|
if cfg == nil || cfg.Pipeline == nil {
|
||||||
|
return nil, fmt.Errorf("pipeline config is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
if strings.EqualFold(strings.TrimSpace(cfg.Pipeline.Storage.Backend), "s3") {
|
||||||
|
if cfg.Pipeline.Storage.S3 == nil {
|
||||||
|
return nil, fmt.Errorf("pipeline.storage.s3 is required when pipeline.storage.backend is s3")
|
||||||
|
}
|
||||||
|
return NewS3BackendFromConfig(ctx, *cfg.Pipeline.Storage.S3)
|
||||||
|
}
|
||||||
|
|
||||||
|
if cfg.Pipeline.Storage.S3 != nil && strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket) != "" {
|
||||||
|
return NewS3BackendFromConfig(ctx, *cfg.Pipeline.Storage.S3)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil, fmt.Errorf("no remote object store backend is configured")
|
||||||
|
}
|
||||||
56
internal/adapters/storage/factory_test.go
Normal file
56
internal/adapters/storage/factory_test.go
Normal file
@@ -0,0 +1,56 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNewObjectStoreFromConfigBuildsS3WhenBackendIsS3(t *testing.T) {
|
||||||
|
original := newS3Client
|
||||||
|
t.Cleanup(func() { newS3Client = original })
|
||||||
|
newS3Client = func(_ context.Context, _ s3ClientOptions) (s3API, error) {
|
||||||
|
return &fakeS3API{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
store, err := NewObjectStoreFromConfig(context.Background(), &config.Config{
|
||||||
|
Pipeline: &config.PipelineConfig{
|
||||||
|
Storage: config.StorageConfig{
|
||||||
|
Backend: "s3",
|
||||||
|
S3: &config.StorageS3Config{
|
||||||
|
Bucket: "my-archive",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewObjectStoreFromConfig() error = %v", err)
|
||||||
|
}
|
||||||
|
if _, ok := store.(*S3Backend); !ok {
|
||||||
|
t.Fatalf("store type = %T, want *S3Backend", store)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewObjectStoreFromConfigRequiresS3ConfigWhenBackendIsS3(t *testing.T) {
|
||||||
|
_, err := NewObjectStoreFromConfig(context.Background(), &config.Config{
|
||||||
|
Pipeline: &config.PipelineConfig{
|
||||||
|
Storage: config.StorageConfig{Backend: "s3"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "pipeline.storage.s3 is required") {
|
||||||
|
t.Fatalf("NewObjectStoreFromConfig() error = %v, want missing storage.s3 error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewObjectStoreFromConfigNoRemoteBackendConfigured(t *testing.T) {
|
||||||
|
_, err := NewObjectStoreFromConfig(context.Background(), &config.Config{
|
||||||
|
Pipeline: &config.PipelineConfig{
|
||||||
|
Storage: config.StorageConfig{Backend: "local"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "no remote object store backend is configured") {
|
||||||
|
t.Fatalf("NewObjectStoreFromConfig() error = %v, want no-backend error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,40 +1,185 @@
|
|||||||
package storage
|
package storage
|
||||||
|
|
||||||
import "context"
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
// NoopBackend is a deterministic no-op archive/storage adapter.
|
// FakeBackend provides a deterministic in-memory object store for tests.
|
||||||
type NoopBackend struct{}
|
|
||||||
|
|
||||||
// Archive returns the requested items as archived with placeholder metadata.
|
|
||||||
func (n *NoopBackend) Archive(ctx context.Context, req ArchiveRequest) (ArchiveResult, error) {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return ArchiveResult{}, err
|
|
||||||
}
|
|
||||||
return ArchiveResult{Archived: append([]ArchiveItem(nil), req.Items...), Metadata: map[string]any{"placeholder": true}}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// FakeBackend captures archive requests and returns deterministic responses.
|
|
||||||
type FakeBackend struct {
|
type FakeBackend struct {
|
||||||
Requests []ArchiveRequest
|
Objects map[string]FakeObject
|
||||||
Err error
|
Uploads []FakeUploadCall
|
||||||
Result ArchiveResult
|
Downloads []FakeDownloadCall
|
||||||
|
|
||||||
|
ListErr error
|
||||||
|
DownloadErr error
|
||||||
|
UploadErr error
|
||||||
|
ExistsErr error
|
||||||
}
|
}
|
||||||
|
|
||||||
// Archive records request and returns configured response.
|
// FakeUploadCall captures one upload invocation in call order.
|
||||||
func (f *FakeBackend) Archive(ctx context.Context, req ArchiveRequest) (ArchiveResult, error) {
|
type FakeUploadCall struct {
|
||||||
if err := ctx.Err(); err != nil {
|
LocalPath string
|
||||||
return ArchiveResult{}, err
|
Key string
|
||||||
}
|
Options UploadOptions
|
||||||
f.Requests = append(f.Requests, req)
|
}
|
||||||
if f.Err != nil {
|
|
||||||
return ArchiveResult{}, f.Err
|
// FakeDownloadCall captures one download invocation in call order.
|
||||||
}
|
type FakeDownloadCall struct {
|
||||||
res := f.Result
|
Key string
|
||||||
if res.Archived == nil {
|
LocalPath string
|
||||||
res.Archived = append([]ArchiveItem(nil), req.Items...)
|
}
|
||||||
}
|
|
||||||
if res.Metadata == nil {
|
// FakeObject is a deterministic fake object-store record.
|
||||||
res.Metadata = map[string]any{"fake": true}
|
type FakeObject struct {
|
||||||
}
|
Key string
|
||||||
return res, nil
|
Data []byte
|
||||||
|
Metadata map[string]string
|
||||||
|
ETag string
|
||||||
|
LastModified *time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// SeedObject inserts or replaces an object in the fake object store.
|
||||||
|
func (f *FakeBackend) SeedObject(obj FakeObject) {
|
||||||
|
if f.Objects == nil {
|
||||||
|
f.Objects = map[string]FakeObject{}
|
||||||
|
}
|
||||||
|
key := normalizeObjectKey(obj.Key)
|
||||||
|
obj.Key = key
|
||||||
|
obj.Data = append([]byte(nil), obj.Data...)
|
||||||
|
obj.Metadata = copyMetadata(obj.Metadata)
|
||||||
|
f.Objects[key] = obj
|
||||||
|
}
|
||||||
|
|
||||||
|
// List returns deterministic prefix-filtered objects.
|
||||||
|
func (f *FakeBackend) List(ctx context.Context, prefix string) ([]ObjectInfo, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if f.ListErr != nil {
|
||||||
|
return nil, f.ListErr
|
||||||
|
}
|
||||||
|
|
||||||
|
normalizedPrefix := normalizeObjectKey(prefix)
|
||||||
|
keys := make([]string, 0, len(f.Objects))
|
||||||
|
for key := range f.Objects {
|
||||||
|
if strings.HasPrefix(key, normalizedPrefix) {
|
||||||
|
keys = append(keys, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
|
||||||
|
out := make([]ObjectInfo, 0, len(keys))
|
||||||
|
for _, key := range keys {
|
||||||
|
obj := f.Objects[key]
|
||||||
|
out = append(out, ObjectInfo{
|
||||||
|
Key: obj.Key,
|
||||||
|
Size: int64(len(obj.Data)),
|
||||||
|
ETag: obj.ETag,
|
||||||
|
LastModified: obj.LastModified,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Download writes one object to a local path.
|
||||||
|
func (f *FakeBackend) Download(ctx context.Context, key, localPath string) error {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if f.DownloadErr != nil {
|
||||||
|
return f.DownloadErr
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(localPath) == "" {
|
||||||
|
return fmt.Errorf("download object: local path is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
obj, ok := f.Objects[normalizeObjectKey(key)]
|
||||||
|
if !ok {
|
||||||
|
return fmt.Errorf("download object %q: %w", key, os.ErrNotExist)
|
||||||
|
}
|
||||||
|
f.Downloads = append(f.Downloads, FakeDownloadCall{
|
||||||
|
Key: normalizeObjectKey(key),
|
||||||
|
LocalPath: localPath,
|
||||||
|
})
|
||||||
|
|
||||||
|
if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
|
||||||
|
return fmt.Errorf("download object %q: create parent directory: %w", key, err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(localPath, obj.Data, 0o644); err != nil {
|
||||||
|
return fmt.Errorf("download object %q: write local file: %w", key, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Upload reads a local file and stores it under key.
|
||||||
|
func (f *FakeBackend) Upload(ctx context.Context, localPath, key string, opts UploadOptions) (ObjectInfo, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return ObjectInfo{}, err
|
||||||
|
}
|
||||||
|
if f.UploadErr != nil {
|
||||||
|
return ObjectInfo{}, f.UploadErr
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(localPath) == "" {
|
||||||
|
return ObjectInfo{}, fmt.Errorf("upload object: local path is required")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(key) == "" {
|
||||||
|
return ObjectInfo{}, fmt.Errorf("upload object: key is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := os.ReadFile(localPath)
|
||||||
|
if err != nil {
|
||||||
|
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: %w", key, localPath, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
normalizedKey := normalizeObjectKey(key)
|
||||||
|
f.Uploads = append(f.Uploads, FakeUploadCall{
|
||||||
|
LocalPath: localPath,
|
||||||
|
Key: normalizedKey,
|
||||||
|
Options: UploadOptions{
|
||||||
|
Metadata: copyMetadata(opts.Metadata),
|
||||||
|
ContentType: opts.ContentType,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
now := time.Now().UTC()
|
||||||
|
obj := FakeObject{
|
||||||
|
Key: normalizedKey,
|
||||||
|
Data: data,
|
||||||
|
Metadata: copyMetadata(opts.Metadata),
|
||||||
|
LastModified: &now,
|
||||||
|
}
|
||||||
|
f.SeedObject(obj)
|
||||||
|
return ObjectInfo{
|
||||||
|
Key: normalizedKey,
|
||||||
|
Size: int64(len(data)),
|
||||||
|
LastModified: &now,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Exists checks object presence.
|
||||||
|
func (f *FakeBackend) Exists(ctx context.Context, key string) (bool, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
if f.ExistsErr != nil {
|
||||||
|
return false, f.ExistsErr
|
||||||
|
}
|
||||||
|
_, ok := f.Objects[normalizeObjectKey(key)]
|
||||||
|
return ok, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyMetadata(in map[string]string) map[string]string {
|
||||||
|
if len(in) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make(map[string]string, len(in))
|
||||||
|
for k, v := range in {
|
||||||
|
out[k] = v
|
||||||
|
}
|
||||||
|
return out
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,29 +3,89 @@ package storage
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestFakeBackendCapturesRequestAndReturnsItems(t *testing.T) {
|
func TestFakeBackendListPrefixFiltering(t *testing.T) {
|
||||||
fake := &FakeBackend{}
|
fake := &FakeBackend{}
|
||||||
req := ArchiveRequest{SessionID: "s1", Items: []ArchiveItem{{Kind: "artifact", LocalPath: "artifacts/log.md"}}}
|
fake.SeedObject(FakeObject{Key: "dnd/campaigns/forsaken/audio/a.flac", Data: []byte("a")})
|
||||||
|
fake.SeedObject(FakeObject{Key: "dnd/campaigns/forsaken/audio/b.flac", Data: []byte("b")})
|
||||||
|
fake.SeedObject(FakeObject{Key: "dnd/campaigns/other/audio/c.flac", Data: []byte("c")})
|
||||||
|
|
||||||
res, err := fake.Archive(context.Background(), req)
|
items, err := fake.List(context.Background(), "dnd/campaigns/forsaken/audio/")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Archive() error = %v", err)
|
t.Fatalf("List() error = %v", err)
|
||||||
}
|
}
|
||||||
if len(fake.Requests) != 1 || fake.Requests[0].SessionID != "s1" {
|
if len(items) != 2 {
|
||||||
t.Fatalf("requests = %#v, want captured request", fake.Requests)
|
t.Fatalf("List() len = %d, want 2", len(items))
|
||||||
}
|
}
|
||||||
if len(res.Archived) != 1 {
|
if items[0].Key != "dnd/campaigns/forsaken/audio/a.flac" || items[1].Key != "dnd/campaigns/forsaken/audio/b.flac" {
|
||||||
t.Fatalf("archived len = %d, want 1", len(res.Archived))
|
t.Fatalf("List() keys = %#v", items)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFakeBackendError(t *testing.T) {
|
func TestFakeBackendDownload(t *testing.T) {
|
||||||
fake := &FakeBackend{Err: errors.New("boom")}
|
fake := &FakeBackend{}
|
||||||
_, err := fake.Archive(context.Background(), ArchiveRequest{})
|
fake.SeedObject(FakeObject{Key: "audio/a.flac", Data: []byte("audio-a")})
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected error, got nil")
|
dst := filepath.Join(t.TempDir(), "nested", "a.flac")
|
||||||
|
if err := fake.Download(context.Background(), `audio\a.flac`, dst); err != nil {
|
||||||
|
t.Fatalf("Download() error = %v", err)
|
||||||
|
}
|
||||||
|
data, err := os.ReadFile(dst)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadFile() error = %v", err)
|
||||||
|
}
|
||||||
|
if string(data) != "audio-a" {
|
||||||
|
t.Fatalf("downloaded content = %q, want %q", string(data), "audio-a")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFakeBackendUploadAndExists(t *testing.T) {
|
||||||
|
fake := &FakeBackend{}
|
||||||
|
|
||||||
|
local := filepath.Join(t.TempDir(), "upload.txt")
|
||||||
|
if err := os.WriteFile(local, []byte("payload"), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
info, err := fake.Upload(context.Background(), local, `runs\id\artifact.txt`, UploadOptions{
|
||||||
|
Metadata: map[string]string{"kind": "artifact"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Upload() error = %v", err)
|
||||||
|
}
|
||||||
|
if info.Key != "runs/id/artifact.txt" {
|
||||||
|
t.Fatalf("Upload() key = %q, want normalized key", info.Key)
|
||||||
|
}
|
||||||
|
|
||||||
|
ok, err := fake.Exists(context.Background(), "runs/id/artifact.txt")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Exists() error = %v", err)
|
||||||
|
}
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("Exists() = false, want true")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFakeBackendObjectErrors(t *testing.T) {
|
||||||
|
fake := &FakeBackend{DownloadErr: errors.New("download fail"), UploadErr: errors.New("upload fail"), ListErr: errors.New("list fail"), ExistsErr: errors.New("exists fail")}
|
||||||
|
|
||||||
|
if _, err := fake.List(context.Background(), "x"); err == nil || !strings.Contains(err.Error(), "list fail") {
|
||||||
|
t.Fatalf("List() error = %v, want list fail", err)
|
||||||
|
}
|
||||||
|
if err := fake.Download(context.Background(), "x", filepath.Join(t.TempDir(), "x")); err == nil || !strings.Contains(err.Error(), "download fail") {
|
||||||
|
t.Fatalf("Download() error = %v, want download fail", err)
|
||||||
|
}
|
||||||
|
local := filepath.Join(t.TempDir(), "x.txt")
|
||||||
|
_ = os.WriteFile(local, []byte("x"), 0o644)
|
||||||
|
if _, err := fake.Upload(context.Background(), local, "x", UploadOptions{}); err == nil || !strings.Contains(err.Error(), "upload fail") {
|
||||||
|
t.Fatalf("Upload() error = %v, want upload fail", err)
|
||||||
|
}
|
||||||
|
if _, err := fake.Exists(context.Background(), "x"); err == nil || !strings.Contains(err.Error(), "exists fail") {
|
||||||
|
t.Fatalf("Exists() error = %v, want exists fail", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
8
internal/adapters/storage/keys.go
Normal file
8
internal/adapters/storage/keys.go
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import "strings"
|
||||||
|
|
||||||
|
func normalizeObjectKey(key string) string {
|
||||||
|
normalized := strings.ReplaceAll(strings.TrimSpace(key), "\\", "/")
|
||||||
|
return strings.TrimLeft(normalized, "/")
|
||||||
|
}
|
||||||
21
internal/adapters/storage/keys_test.go
Normal file
21
internal/adapters/storage/keys_test.go
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestNormalizeObjectKey(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
in string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{in: `dnd\campaigns\forsaken\a.flac`, want: "dnd/campaigns/forsaken/a.flac"},
|
||||||
|
{in: " /dnd/campaigns/forsaken/a.flac ", want: "dnd/campaigns/forsaken/a.flac"},
|
||||||
|
{in: "//dnd/campaigns/forsaken/a.flac", want: "dnd/campaigns/forsaken/a.flac"},
|
||||||
|
{in: "", want: ""},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
if got := normalizeObjectKey(tt.in); got != tt.want {
|
||||||
|
t.Fatalf("normalizeObjectKey(%q) = %q, want %q", tt.in, got, tt.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
32
internal/adapters/storage/object_store.go
Normal file
32
internal/adapters/storage/object_store.go
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ObjectStore is a remote object storage boundary used by prepare, restore, and publish work.
|
||||||
|
//
|
||||||
|
// Key invariant:
|
||||||
|
// callers pass full bucket-relative object keys. Backend implementations do not
|
||||||
|
// infer Narratio session semantics and do not prepend root prefixes.
|
||||||
|
type ObjectStore interface {
|
||||||
|
List(ctx context.Context, prefix string) ([]ObjectInfo, error)
|
||||||
|
Download(ctx context.Context, key, localPath string) error
|
||||||
|
Upload(ctx context.Context, localPath, key string, opts UploadOptions) (ObjectInfo, error)
|
||||||
|
Exists(ctx context.Context, key string) (bool, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ObjectInfo describes one object in remote storage.
|
||||||
|
type ObjectInfo struct {
|
||||||
|
Key string
|
||||||
|
Size int64
|
||||||
|
ETag string
|
||||||
|
LastModified *time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// UploadOptions configures optional object upload metadata.
|
||||||
|
type UploadOptions struct {
|
||||||
|
Metadata map[string]string
|
||||||
|
ContentType string
|
||||||
|
}
|
||||||
275
internal/adapters/storage/s3_backend.go
Normal file
275
internal/adapters/storage/s3_backend.go
Normal file
@@ -0,0 +1,275 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
awsconfig "github.com/aws/aws-sdk-go-v2/config"
|
||||||
|
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||||
|
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||||
|
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||||
|
"github.com/aws/smithy-go"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
type s3API interface {
|
||||||
|
ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error)
|
||||||
|
GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error)
|
||||||
|
PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error)
|
||||||
|
HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// S3Backend is an ObjectStore implementation backed by S3-compatible APIs.
|
||||||
|
type S3Backend struct {
|
||||||
|
bucket string
|
||||||
|
client s3API
|
||||||
|
}
|
||||||
|
|
||||||
|
type s3ClientOptions struct {
|
||||||
|
Region string
|
||||||
|
Endpoint string
|
||||||
|
ForcePathStyle bool
|
||||||
|
AccessKeyID string
|
||||||
|
SecretKey string
|
||||||
|
}
|
||||||
|
|
||||||
|
var newS3Client = func(ctx context.Context, opts s3ClientOptions) (s3API, error) {
|
||||||
|
loadOpts := make([]func(*awsconfig.LoadOptions) error, 0, 1)
|
||||||
|
if strings.TrimSpace(opts.Region) != "" {
|
||||||
|
loadOpts = append(loadOpts, awsconfig.WithRegion(strings.TrimSpace(opts.Region)))
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(opts.AccessKeyID) != "" && strings.TrimSpace(opts.SecretKey) != "" {
|
||||||
|
loadOpts = append(loadOpts, awsconfig.WithCredentialsProvider(
|
||||||
|
credentials.NewStaticCredentialsProvider(
|
||||||
|
strings.TrimSpace(opts.AccessKeyID),
|
||||||
|
strings.TrimSpace(opts.SecretKey),
|
||||||
|
"",
|
||||||
|
),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
awsCfg, err := awsconfig.LoadDefaultConfig(ctx, loadOpts...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("load aws config: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return s3.NewFromConfig(awsCfg, func(o *s3.Options) {
|
||||||
|
if strings.TrimSpace(opts.Endpoint) != "" {
|
||||||
|
endpoint := strings.TrimSpace(opts.Endpoint)
|
||||||
|
o.BaseEndpoint = &endpoint
|
||||||
|
}
|
||||||
|
o.UsePathStyle = opts.ForcePathStyle
|
||||||
|
}), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewS3BackendFromConfig builds an S3 backend from resolved config.
|
||||||
|
func NewS3BackendFromConfig(ctx context.Context, cfg config.StorageS3Config) (*S3Backend, error) {
|
||||||
|
bucket := strings.TrimSpace(cfg.Bucket)
|
||||||
|
if bucket == "" {
|
||||||
|
return nil, fmt.Errorf("storage.s3.bucket is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
client, err := newS3Client(ctx, s3ClientOptions{
|
||||||
|
Region: cfg.Region,
|
||||||
|
Endpoint: cfg.Endpoint,
|
||||||
|
ForcePathStyle: cfg.ForcePathStyle,
|
||||||
|
AccessKeyID: s3CredentialFromEnv(orDefaultEnvName(cfg.AccessKeyIDEnv, config.DefaultS3AccessKeyIDEnv)),
|
||||||
|
SecretKey: s3CredentialFromEnv(orDefaultEnvName(cfg.SecretKeyEnv, config.DefaultS3SecretAccessKeyEnv)),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("build s3 client: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &S3Backend{
|
||||||
|
bucket: bucket,
|
||||||
|
client: client,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func s3CredentialFromEnv(envVarName string) string {
|
||||||
|
name := strings.TrimSpace(envVarName)
|
||||||
|
if name == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
value, ok := os.LookupEnv(name)
|
||||||
|
if !ok {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(value)
|
||||||
|
}
|
||||||
|
|
||||||
|
func orDefaultEnvName(name, fallback string) string {
|
||||||
|
trimmed := strings.TrimSpace(name)
|
||||||
|
if trimmed == "" {
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
return trimmed
|
||||||
|
}
|
||||||
|
|
||||||
|
// List returns objects under prefix.
|
||||||
|
func (b *S3Backend) List(ctx context.Context, prefix string) ([]ObjectInfo, error) {
|
||||||
|
normalizedPrefix := normalizeObjectKey(prefix)
|
||||||
|
out := make([]ObjectInfo, 0)
|
||||||
|
var token *string
|
||||||
|
|
||||||
|
for {
|
||||||
|
resp, err := b.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
||||||
|
Bucket: &b.bucket,
|
||||||
|
Prefix: &normalizedPrefix,
|
||||||
|
ContinuationToken: token,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("list objects under %q: %w", normalizedPrefix, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, item := range resp.Contents {
|
||||||
|
var lastModified *time.Time
|
||||||
|
if item.LastModified != nil {
|
||||||
|
t := *item.LastModified
|
||||||
|
lastModified = &t
|
||||||
|
}
|
||||||
|
out = append(out, ObjectInfo{
|
||||||
|
Key: normalizeObjectKey(valueOrEmpty(item.Key)),
|
||||||
|
Size: valueOrZeroInt64(item.Size),
|
||||||
|
ETag: strings.Trim(valueOrEmpty(item.ETag), "\""),
|
||||||
|
LastModified: lastModified,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
if !valueOrFalseBool(resp.IsTruncated) || resp.NextContinuationToken == nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
token = resp.NextContinuationToken
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Download retrieves one object to localPath, creating parent directories as needed.
|
||||||
|
func (b *S3Backend) Download(ctx context.Context, key, localPath string) error {
|
||||||
|
normalizedKey := normalizeObjectKey(key)
|
||||||
|
if strings.TrimSpace(localPath) == "" {
|
||||||
|
return fmt.Errorf("download object: local path is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := b.client.GetObject(ctx, &s3.GetObjectInput{
|
||||||
|
Bucket: &b.bucket,
|
||||||
|
Key: &normalizedKey,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("download object %q: %w", normalizedKey, err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
|
||||||
|
return fmt.Errorf("download object %q: create parent directory: %w", normalizedKey, err)
|
||||||
|
}
|
||||||
|
dst, err := os.Create(localPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("download object %q: create local file: %w", normalizedKey, err)
|
||||||
|
}
|
||||||
|
defer dst.Close()
|
||||||
|
|
||||||
|
if _, err := io.Copy(dst, resp.Body); err != nil {
|
||||||
|
return fmt.Errorf("download object %q: copy body: %w", normalizedKey, err)
|
||||||
|
}
|
||||||
|
if err := dst.Sync(); err != nil {
|
||||||
|
return fmt.Errorf("download object %q: sync local file: %w", normalizedKey, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Upload sends a local file to key.
|
||||||
|
func (b *S3Backend) Upload(ctx context.Context, localPath, key string, opts UploadOptions) (ObjectInfo, error) {
|
||||||
|
normalizedKey := normalizeObjectKey(key)
|
||||||
|
if strings.TrimSpace(localPath) == "" {
|
||||||
|
return ObjectInfo{}, fmt.Errorf("upload object: local path is required")
|
||||||
|
}
|
||||||
|
if normalizedKey == "" {
|
||||||
|
return ObjectInfo{}, fmt.Errorf("upload object: key is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
file, err := os.Open(localPath)
|
||||||
|
if err != nil {
|
||||||
|
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: %w", normalizedKey, localPath, err)
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
stat, err := file.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: stat local file: %w", normalizedKey, localPath, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
input := &s3.PutObjectInput{
|
||||||
|
Bucket: &b.bucket,
|
||||||
|
Key: &normalizedKey,
|
||||||
|
Body: file,
|
||||||
|
Metadata: copyMetadata(opts.Metadata),
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(opts.ContentType) != "" {
|
||||||
|
ct := strings.TrimSpace(opts.ContentType)
|
||||||
|
input.ContentType = &ct
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := b.client.PutObject(ctx, input)
|
||||||
|
if err != nil {
|
||||||
|
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: %w", normalizedKey, localPath, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return ObjectInfo{
|
||||||
|
Key: normalizedKey,
|
||||||
|
Size: stat.Size(),
|
||||||
|
ETag: strings.Trim(valueOrEmpty(resp.ETag), "\""),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Exists checks whether one object key exists.
|
||||||
|
func (b *S3Backend) Exists(ctx context.Context, key string) (bool, error) {
|
||||||
|
normalizedKey := normalizeObjectKey(key)
|
||||||
|
_, err := b.client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||||
|
Bucket: &b.bucket,
|
||||||
|
Key: &normalizedKey,
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var notFound *types.NotFound
|
||||||
|
if errors.As(err, ¬Found) {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
var apiErr smithy.APIError
|
||||||
|
if errors.As(err, &apiErr) {
|
||||||
|
switch apiErr.ErrorCode() {
|
||||||
|
case "NotFound", "NoSuchKey", "404":
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false, fmt.Errorf("head object %q: %w", normalizedKey, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func valueOrEmpty(v *string) string {
|
||||||
|
if v == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return *v
|
||||||
|
}
|
||||||
|
|
||||||
|
func valueOrZeroInt64(v *int64) int64 {
|
||||||
|
if v == nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return *v
|
||||||
|
}
|
||||||
|
|
||||||
|
func valueOrFalseBool(v *bool) bool {
|
||||||
|
if v == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return *v
|
||||||
|
}
|
||||||
253
internal/adapters/storage/s3_backend_test.go
Normal file
253
internal/adapters/storage/s3_backend_test.go
Normal file
@@ -0,0 +1,253 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||||
|
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||||
|
"github.com/aws/smithy-go"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
type fakeS3API struct {
|
||||||
|
listOut *s3.ListObjectsV2Output
|
||||||
|
listErr error
|
||||||
|
|
||||||
|
getBody io.ReadCloser
|
||||||
|
getErr error
|
||||||
|
|
||||||
|
putOut *s3.PutObjectOutput
|
||||||
|
putErr error
|
||||||
|
|
||||||
|
headErr error
|
||||||
|
|
||||||
|
lastList *s3.ListObjectsV2Input
|
||||||
|
lastGet *s3.GetObjectInput
|
||||||
|
lastPut *s3.PutObjectInput
|
||||||
|
lastHead *s3.HeadObjectInput
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeS3API) ListObjectsV2(_ context.Context, params *s3.ListObjectsV2Input, _ ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) {
|
||||||
|
f.lastList = params
|
||||||
|
if f.listErr != nil {
|
||||||
|
return nil, f.listErr
|
||||||
|
}
|
||||||
|
if f.listOut == nil {
|
||||||
|
return &s3.ListObjectsV2Output{}, nil
|
||||||
|
}
|
||||||
|
return f.listOut, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeS3API) GetObject(_ context.Context, params *s3.GetObjectInput, _ ...func(*s3.Options)) (*s3.GetObjectOutput, error) {
|
||||||
|
f.lastGet = params
|
||||||
|
if f.getErr != nil {
|
||||||
|
return nil, f.getErr
|
||||||
|
}
|
||||||
|
body := f.getBody
|
||||||
|
if body == nil {
|
||||||
|
body = io.NopCloser(strings.NewReader(""))
|
||||||
|
}
|
||||||
|
return &s3.GetObjectOutput{Body: body}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeS3API) PutObject(_ context.Context, params *s3.PutObjectInput, _ ...func(*s3.Options)) (*s3.PutObjectOutput, error) {
|
||||||
|
f.lastPut = params
|
||||||
|
if f.putErr != nil {
|
||||||
|
return nil, f.putErr
|
||||||
|
}
|
||||||
|
if f.putOut == nil {
|
||||||
|
return &s3.PutObjectOutput{}, nil
|
||||||
|
}
|
||||||
|
return f.putOut, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeS3API) HeadObject(_ context.Context, params *s3.HeadObjectInput, _ ...func(*s3.Options)) (*s3.HeadObjectOutput, error) {
|
||||||
|
f.lastHead = params
|
||||||
|
if f.headErr != nil {
|
||||||
|
return nil, f.headErr
|
||||||
|
}
|
||||||
|
return &s3.HeadObjectOutput{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestS3BackendListAndKeyNormalization(t *testing.T) {
|
||||||
|
lastModified := time.Date(2026, 5, 16, 12, 0, 0, 0, time.UTC)
|
||||||
|
client := &fakeS3API{
|
||||||
|
listOut: &s3.ListObjectsV2Output{
|
||||||
|
Contents: []types.Object{
|
||||||
|
{Key: strPtr(`dnd\campaigns\forsaken\a.flac`), Size: int64Ptr(7), ETag: strPtr(`"abc"`), LastModified: &lastModified},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
backend := &S3Backend{bucket: "bucket-1", client: client}
|
||||||
|
|
||||||
|
items, err := backend.List(context.Background(), `dnd\campaigns\`)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("List() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(items) != 1 {
|
||||||
|
t.Fatalf("List() len = %d, want 1", len(items))
|
||||||
|
}
|
||||||
|
if items[0].Key != "dnd/campaigns/forsaken/a.flac" {
|
||||||
|
t.Fatalf("List() key = %q, want normalized slash key", items[0].Key)
|
||||||
|
}
|
||||||
|
if items[0].ETag != "abc" {
|
||||||
|
t.Fatalf("List() ETag = %q, want %q", items[0].ETag, "abc")
|
||||||
|
}
|
||||||
|
if client.lastList == nil || *client.lastList.Prefix != "dnd/campaigns/" {
|
||||||
|
t.Fatalf("List() prefix = %#v, want normalized prefix", client.lastList)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestS3BackendDownloadCreatesParentDirectory(t *testing.T) {
|
||||||
|
client := &fakeS3API{getBody: io.NopCloser(strings.NewReader("audio"))}
|
||||||
|
backend := &S3Backend{bucket: "bucket-1", client: client}
|
||||||
|
|
||||||
|
dst := filepath.Join(t.TempDir(), "nested", "clip.flac")
|
||||||
|
if err := backend.Download(context.Background(), `audio\clip.flac`, dst); err != nil {
|
||||||
|
t.Fatalf("Download() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := os.ReadFile(dst)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadFile() error = %v", err)
|
||||||
|
}
|
||||||
|
if string(data) != "audio" {
|
||||||
|
t.Fatalf("downloaded content = %q, want %q", string(data), "audio")
|
||||||
|
}
|
||||||
|
if client.lastGet == nil || *client.lastGet.Key != "audio/clip.flac" {
|
||||||
|
t.Fatalf("GetObject key = %#v, want normalized key", client.lastGet)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestS3BackendUploadAndExists(t *testing.T) {
|
||||||
|
client := &fakeS3API{putOut: &s3.PutObjectOutput{ETag: strPtr(`"etag123"`)}}
|
||||||
|
backend := &S3Backend{bucket: "bucket-1", client: client}
|
||||||
|
|
||||||
|
local := filepath.Join(t.TempDir(), "artifact.txt")
|
||||||
|
if err := os.WriteFile(local, []byte("artifact"), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
info, err := backend.Upload(context.Background(), local, `runs\id\artifact.txt`, UploadOptions{
|
||||||
|
Metadata: map[string]string{"kind": "artifact"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Upload() error = %v", err)
|
||||||
|
}
|
||||||
|
if info.Key != "runs/id/artifact.txt" {
|
||||||
|
t.Fatalf("Upload key = %q, want normalized key", info.Key)
|
||||||
|
}
|
||||||
|
if info.ETag != "etag123" {
|
||||||
|
t.Fatalf("Upload ETag = %q, want %q", info.ETag, "etag123")
|
||||||
|
}
|
||||||
|
if client.lastPut == nil || *client.lastPut.Key != "runs/id/artifact.txt" {
|
||||||
|
t.Fatalf("PutObject key = %#v, want normalized key", client.lastPut)
|
||||||
|
}
|
||||||
|
|
||||||
|
ok, err := backend.Exists(context.Background(), "runs/id/artifact.txt")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Exists() error = %v", err)
|
||||||
|
}
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("Exists() = false, want true")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestS3BackendUploadMissingLocalFile(t *testing.T) {
|
||||||
|
backend := &S3Backend{bucket: "bucket-1", client: &fakeS3API{}}
|
||||||
|
_, err := backend.Upload(context.Background(), filepath.Join(t.TempDir(), "missing.txt"), "key.txt", UploadOptions{})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "no such file") {
|
||||||
|
t.Fatalf("Upload() error = %v, want missing local file error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestS3BackendExistsNotFound(t *testing.T) {
|
||||||
|
backend := &S3Backend{
|
||||||
|
bucket: "bucket-1",
|
||||||
|
client: &fakeS3API{
|
||||||
|
headErr: &smithy.GenericAPIError{Code: "NotFound", Message: "missing"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
ok, err := backend.Exists(context.Background(), "missing-key")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Exists() error = %v", err)
|
||||||
|
}
|
||||||
|
if ok {
|
||||||
|
t.Fatal("Exists() = true, want false")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewS3BackendFromConfigUsesClientOptions(t *testing.T) {
|
||||||
|
original := newS3Client
|
||||||
|
t.Cleanup(func() { newS3Client = original })
|
||||||
|
t.Setenv("OBJECT_STORAGE_KEY_ID", "id-123")
|
||||||
|
t.Setenv("OBJECT_STORAGE_KEY", "secret-abc")
|
||||||
|
|
||||||
|
var got s3ClientOptions
|
||||||
|
newS3Client = func(_ context.Context, opts s3ClientOptions) (s3API, error) {
|
||||||
|
got = opts
|
||||||
|
return &fakeS3API{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
backend, err := NewS3BackendFromConfig(context.Background(), config.StorageS3Config{
|
||||||
|
Bucket: "my-archive",
|
||||||
|
Region: "us-east-1",
|
||||||
|
Endpoint: "http://localhost:9000",
|
||||||
|
ForcePathStyle: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewS3BackendFromConfig() error = %v", err)
|
||||||
|
}
|
||||||
|
if backend.bucket != "my-archive" {
|
||||||
|
t.Fatalf("backend.bucket = %q, want %q", backend.bucket, "my-archive")
|
||||||
|
}
|
||||||
|
if got.Region != "us-east-1" || got.Endpoint != "http://localhost:9000" || !got.ForcePathStyle {
|
||||||
|
t.Fatalf("client options = %#v, want region/endpoint/path-style values", got)
|
||||||
|
}
|
||||||
|
if got.AccessKeyID != "id-123" || got.SecretKey != "secret-abc" {
|
||||||
|
t.Fatalf("client options credentials = %#v, want env-resolved static credentials", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewS3BackendFromConfigRequiresBucket(t *testing.T) {
|
||||||
|
_, err := NewS3BackendFromConfig(context.Background(), config.StorageS3Config{})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "bucket is required") {
|
||||||
|
t.Fatalf("NewS3BackendFromConfig() error = %v, want bucket validation", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewS3BackendFromConfigFallsBackWhenCredentialEnvMissing(t *testing.T) {
|
||||||
|
original := newS3Client
|
||||||
|
t.Cleanup(func() { newS3Client = original })
|
||||||
|
|
||||||
|
var got s3ClientOptions
|
||||||
|
newS3Client = func(_ context.Context, opts s3ClientOptions) (s3API, error) {
|
||||||
|
got = opts
|
||||||
|
return &fakeS3API{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := NewS3BackendFromConfig(context.Background(), config.StorageS3Config{
|
||||||
|
Bucket: "my-archive",
|
||||||
|
Region: "us-east-1",
|
||||||
|
AccessKeyIDEnv: "MISSING_ACCESS_KEY_ID",
|
||||||
|
SecretKeyEnv: "MISSING_SECRET_KEY",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewS3BackendFromConfig() error = %v", err)
|
||||||
|
}
|
||||||
|
if got.AccessKeyID != "" || got.SecretKey != "" {
|
||||||
|
t.Fatalf("client options credentials = %#v, want empty fallback values", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func strPtr(v string) *string { return &v }
|
||||||
|
func int64Ptr(v int64) *int64 { return &v }
|
||||||
|
|
||||||
|
var _ s3API = (*fakeS3API)(nil)
|
||||||
36
internal/adapters/storage/temp_download.go
Normal file
36
internal/adapters/storage/temp_download.go
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// DownloadObjectToTemp downloads an object into a temporary file and returns
|
||||||
|
// the cleaned local path.
|
||||||
|
func DownloadObjectToTemp(ctx context.Context, store ObjectStore, key, pattern string) (string, error) {
|
||||||
|
if store == nil {
|
||||||
|
return "", fmt.Errorf("object store is required")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(pattern) == "" {
|
||||||
|
return "", fmt.Errorf("temp file pattern is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
tmp, err := os.CreateTemp("", pattern)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("create temp file: %w", err)
|
||||||
|
}
|
||||||
|
path := tmp.Name()
|
||||||
|
if err := tmp.Close(); err != nil {
|
||||||
|
_ = os.Remove(path)
|
||||||
|
return "", fmt.Errorf("close temp file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := store.Download(ctx, key, path); err != nil {
|
||||||
|
_ = os.Remove(path)
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return filepath.Clean(path), nil
|
||||||
|
}
|
||||||
69
internal/adapters/storage/temp_download_test.go
Normal file
69
internal/adapters/storage/temp_download_test.go
Normal file
@@ -0,0 +1,69 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDownloadObjectToTempSuccess(t *testing.T) {
|
||||||
|
store := &FakeBackend{}
|
||||||
|
store.SeedObject(FakeObject{Key: "sessions/a/current/run_id.txt", Data: []byte("run-123\n")})
|
||||||
|
|
||||||
|
path, err := DownloadObjectToTemp(context.Background(), store, "sessions/a/current/run_id.txt", "narratio-test-*.txt")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("DownloadObjectToTemp() error = %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = os.Remove(path) })
|
||||||
|
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadFile() error = %v", err)
|
||||||
|
}
|
||||||
|
if string(data) != "run-123\n" {
|
||||||
|
t.Fatalf("downloaded data = %q, want %q", string(data), "run-123\n")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDownloadObjectToTempFailedDownloadRemovesTempFile(t *testing.T) {
|
||||||
|
sentinel := errors.New("download failed")
|
||||||
|
store := &FakeBackend{DownloadErr: sentinel}
|
||||||
|
pattern := "narratio-test-fail-*.txt"
|
||||||
|
before, err := filepath.Glob(filepath.Join(os.TempDir(), "narratio-test-fail-*.txt"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Glob(before) error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
path, err := DownloadObjectToTemp(context.Background(), store, "sessions/a/current/run_id.txt", pattern)
|
||||||
|
if !errors.Is(err, sentinel) {
|
||||||
|
t.Fatalf("DownloadObjectToTemp() error = %v, want %v", err, sentinel)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(path) != "" {
|
||||||
|
t.Fatalf("DownloadObjectToTemp() path = %q, want empty on failure", path)
|
||||||
|
}
|
||||||
|
after, err := filepath.Glob(filepath.Join(os.TempDir(), "narratio-test-fail-*.txt"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Glob(after) error = %v", err)
|
||||||
|
}
|
||||||
|
if len(after) != len(before) {
|
||||||
|
t.Fatalf("temp file count changed after failed download: before=%d after=%d", len(before), len(after))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDownloadObjectToTempCallerContextWrappingPreservesCause(t *testing.T) {
|
||||||
|
sentinel := errors.New("object missing")
|
||||||
|
store := &FakeBackend{DownloadErr: sentinel}
|
||||||
|
|
||||||
|
_, err := DownloadObjectToTemp(context.Background(), store, "sessions/a/current/run_id.txt", "narratio-test-*.txt")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("DownloadObjectToTemp() error = nil, want error")
|
||||||
|
}
|
||||||
|
err = fmt.Errorf("download run pointer failed: %w", err)
|
||||||
|
if !errors.Is(err, sentinel) {
|
||||||
|
t.Fatalf("wrapped error does not preserve sentinel cause: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -145,7 +145,7 @@ func TestHTTPClientDoesNotRetryOnNonRetryableStatus(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHTTPClientInvalidJSONFailsAndDoesNotPromote(t *testing.T) {
|
func TestHTTPClientInvalidJSONFailsAndDoesNotInstallOutput(t *testing.T) {
|
||||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
_, _ = w.Write([]byte(`not-json`))
|
_, _ = w.Write([]byte(`not-json`))
|
||||||
}))
|
}))
|
||||||
|
|||||||
66
internal/app/analyze_artifacts.go
Normal file
66
internal/app/analyze_artifacts.go
Normal file
@@ -0,0 +1,66 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
type artifactSelectionFlag struct {
|
||||||
|
values []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *artifactSelectionFlag) String() string {
|
||||||
|
return strings.Join(f.values, ",")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *artifactSelectionFlag) Set(value string) error {
|
||||||
|
f.values = append(f.values, value)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *artifactSelectionFlag) Normalize() ([]string, error) {
|
||||||
|
if len(f.values) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
seen := map[string]struct{}{}
|
||||||
|
out := make([]string, 0, len(f.values))
|
||||||
|
for _, raw := range f.values {
|
||||||
|
for _, part := range strings.Split(raw, ",") {
|
||||||
|
name := strings.TrimSpace(part)
|
||||||
|
if name == "" {
|
||||||
|
return nil, fmt.Errorf("artifact names must be non-empty")
|
||||||
|
}
|
||||||
|
if _, ok := seen[name]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[name] = struct{}{}
|
||||||
|
out = append(out, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sort.Strings(out)
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateSelectedArtifacts(cfg *config.Config, selected []string) error {
|
||||||
|
if len(selected) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Scriptorium == nil {
|
||||||
|
return fmt.Errorf("--artifacts requires pipeline.scriptorium.artifacts to be configured")
|
||||||
|
}
|
||||||
|
configured := cfg.Pipeline.Scriptorium.Artifacts
|
||||||
|
if len(configured) == 0 {
|
||||||
|
return fmt.Errorf("--artifacts requires at least one configured artifact in pipeline.scriptorium.artifacts")
|
||||||
|
}
|
||||||
|
for _, name := range selected {
|
||||||
|
if _, ok := configured[name]; !ok {
|
||||||
|
return fmt.Errorf("--artifacts includes unknown artifact %q", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
434
internal/app/analyze_artifacts_commands_test.go
Normal file
434
internal/app/analyze_artifacts_commands_test.go
Normal file
@@ -0,0 +1,434 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/stage"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestExecuteRunStageArtifactsUnsupportedStageFails(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute(
|
||||||
|
[]string{"run-stage", "polish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "session_recap"},
|
||||||
|
&stdout,
|
||||||
|
&stderr,
|
||||||
|
)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), `run-stage: --artifacts is only supported for stages "analyze" and "publish"`) {
|
||||||
|
t.Fatalf("stderr = %q, want stage-gating error", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteRunStagePublishPropagatesSelectedArtifacts(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||||
|
|
||||||
|
var capturedStages []string
|
||||||
|
var capturedArtifacts []string
|
||||||
|
origExecuteStagesFn := executeStagesFn
|
||||||
|
t.Cleanup(func() {
|
||||||
|
executeStagesFn = origExecuteStagesFn
|
||||||
|
})
|
||||||
|
executeStagesFn = func(_ context.Context, _ *config.Config, stages []stage.Stage, opts RunOptions) (*RunSummary, error) {
|
||||||
|
for _, s := range stages {
|
||||||
|
capturedStages = append(capturedStages, s.Name())
|
||||||
|
}
|
||||||
|
capturedArtifacts = append([]string(nil), opts.SelectedArtifacts...)
|
||||||
|
return &RunSummary{ManifestPath: filepath.Join(workspaceRoot, "manifest.json"), Executed: []string{"publish"}}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute(
|
||||||
|
[]string{
|
||||||
|
"run-stage", "publish", "2026-05-03",
|
||||||
|
"--config", pipelinePath,
|
||||||
|
"--campaign-file", campaignPath,
|
||||||
|
"--session", sessionPath,
|
||||||
|
"--artifacts", "session_recap",
|
||||||
|
},
|
||||||
|
&stdout,
|
||||||
|
&stderr,
|
||||||
|
)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
if len(capturedStages) != 1 || capturedStages[0] != "publish" {
|
||||||
|
t.Fatalf("captured stages = %#v, want [publish]", capturedStages)
|
||||||
|
}
|
||||||
|
if strings.Join(capturedArtifacts, ",") != "session_recap" {
|
||||||
|
t.Fatalf("captured artifacts = %#v, want [session_recap]", capturedArtifacts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteUnknownArtifactsFailValidation(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute(
|
||||||
|
[]string{"run", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "unknown_artifact"},
|
||||||
|
&stdout,
|
||||||
|
&stderr,
|
||||||
|
)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), `run: --artifacts includes unknown artifact "unknown_artifact"`) {
|
||||||
|
t.Fatalf("stderr = %q, want unknown-artifact validation error", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunStageArtifactsDoesNotImplyForce(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||||
|
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||||
|
|
||||||
|
store := &manifest.LocalStore{}
|
||||||
|
seed := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||||
|
seed.MarkStageSucceeded("analyze", time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||||
|
if err := store.Save(context.Background(), manifestPath, seed); err != nil {
|
||||||
|
t.Fatalf("save manifest: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
err := RunStage(
|
||||||
|
context.Background(),
|
||||||
|
[]string{"analyze", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "session_recap,session_recap"},
|
||||||
|
&out,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("RunStage() error = %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out.String(), "stage=analyze executed=0 skipped=1 force=false") {
|
||||||
|
t.Fatalf("output = %q, want analyze skip without force", out.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||||
|
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||||
|
|
||||||
|
store := &manifest.LocalStore{}
|
||||||
|
seed := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||||
|
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze", "publish", "notify"} {
|
||||||
|
seed.MarkStageSucceeded(stageName, time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||||
|
}
|
||||||
|
if err := store.Save(context.Background(), manifestPath, seed); err != nil {
|
||||||
|
t.Fatalf("save manifest: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
err := Run(
|
||||||
|
context.Background(),
|
||||||
|
[]string{"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "session_recap"},
|
||||||
|
&out,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out.String(), "executed=0 skipped=10") {
|
||||||
|
t.Fatalf("output = %q, want all stages skipped", out.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteAnalyzeForceRunsAnalyze(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||||
|
|
||||||
|
var capturedStages []string
|
||||||
|
var capturedForce bool
|
||||||
|
origExecuteStagesFn := executeStagesFn
|
||||||
|
t.Cleanup(func() {
|
||||||
|
executeStagesFn = origExecuteStagesFn
|
||||||
|
})
|
||||||
|
executeStagesFn = func(_ context.Context, _ *config.Config, stages []stage.Stage, opts RunOptions) (*RunSummary, error) {
|
||||||
|
for _, s := range stages {
|
||||||
|
capturedStages = append(capturedStages, s.Name())
|
||||||
|
}
|
||||||
|
capturedForce = opts.Force
|
||||||
|
return &RunSummary{
|
||||||
|
ManifestPath: filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json"),
|
||||||
|
Executed: []string{"analyze"},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute(
|
||||||
|
[]string{"analyze", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath},
|
||||||
|
&stdout,
|
||||||
|
&stderr,
|
||||||
|
)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
if len(capturedStages) != 1 || capturedStages[0] != "analyze" {
|
||||||
|
t.Fatalf("captured stages = %#v, want [analyze]", capturedStages)
|
||||||
|
}
|
||||||
|
if !capturedForce {
|
||||||
|
t.Fatal("captured force = false, want true")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout.String(), "narratio analyze: executed=1 skipped=0 force=true; manifest=") {
|
||||||
|
t.Fatalf("stdout = %q, want analyze summary", stdout.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteAnalyzePropagatesSelectedArtifacts(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||||
|
|
||||||
|
var capturedArtifacts []string
|
||||||
|
origExecuteStagesFn := executeStagesFn
|
||||||
|
t.Cleanup(func() {
|
||||||
|
executeStagesFn = origExecuteStagesFn
|
||||||
|
})
|
||||||
|
executeStagesFn = func(_ context.Context, _ *config.Config, _ []stage.Stage, opts RunOptions) (*RunSummary, error) {
|
||||||
|
capturedArtifacts = append([]string(nil), opts.SelectedArtifacts...)
|
||||||
|
return &RunSummary{ManifestPath: filepath.Join(workspaceRoot, "manifest.json"), Executed: []string{"analyze"}}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute(
|
||||||
|
[]string{
|
||||||
|
"analyze",
|
||||||
|
"2026-05-03",
|
||||||
|
"--config", pipelinePath,
|
||||||
|
"--campaign-file", campaignPath,
|
||||||
|
"--session", sessionPath,
|
||||||
|
"--artifacts", "player_handout,session_recap",
|
||||||
|
},
|
||||||
|
&stdout,
|
||||||
|
&stderr,
|
||||||
|
)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
if strings.Join(capturedArtifacts, ",") != "player_handout,session_recap" {
|
||||||
|
t.Fatalf("captured artifacts = %#v, want sorted selected artifacts", capturedArtifacts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteAnalyzeUnknownArtifactFailsValidation(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute(
|
||||||
|
[]string{"analyze", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "unknown_artifact"},
|
||||||
|
&stdout,
|
||||||
|
&stderr,
|
||||||
|
)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), `analyze: --artifacts includes unknown artifact "unknown_artifact"`) {
|
||||||
|
t.Fatalf("stderr = %q, want unknown-artifact validation error", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteAnalyzeRejectsPositionalArgsAndForceFlag(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
args []string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "extra positional", args: []string{"analyze", "2026-05-03", "extra"}, want: "analyze: unexpected positional arguments"},
|
||||||
|
{name: "force flag", args: []string{"analyze", "--force"}, want: "analyze: invalid flags: flag provided but not defined: -force"},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute(tc.args, &stdout, &stderr)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), tc.want) {
|
||||||
|
t.Fatalf("stderr = %q, want %q", stderr.String(), tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteAnalyzeMissingConfigUsesRunStageLoadingPath(t *testing.T) {
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"analyze", "2026-05-03"}, &stdout, &stderr)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "analyze: no pipeline config path provided and no default pipeline config found; searched:") {
|
||||||
|
t.Fatalf("stderr = %q, want pipeline discovery error", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecutePublishForceRunsPublish(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||||
|
|
||||||
|
var capturedStages []string
|
||||||
|
var capturedForce bool
|
||||||
|
var capturedArtifacts []string
|
||||||
|
origExecuteStagesFn := executeStagesFn
|
||||||
|
t.Cleanup(func() {
|
||||||
|
executeStagesFn = origExecuteStagesFn
|
||||||
|
})
|
||||||
|
executeStagesFn = func(_ context.Context, _ *config.Config, stages []stage.Stage, opts RunOptions) (*RunSummary, error) {
|
||||||
|
for _, s := range stages {
|
||||||
|
capturedStages = append(capturedStages, s.Name())
|
||||||
|
}
|
||||||
|
capturedForce = opts.Force
|
||||||
|
capturedArtifacts = append([]string(nil), opts.SelectedArtifacts...)
|
||||||
|
return &RunSummary{
|
||||||
|
ManifestPath: filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json"),
|
||||||
|
Executed: []string{"publish"},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute(
|
||||||
|
[]string{"publish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "session_recap"},
|
||||||
|
&stdout,
|
||||||
|
&stderr,
|
||||||
|
)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
if len(capturedStages) != 1 || capturedStages[0] != "publish" {
|
||||||
|
t.Fatalf("captured stages = %#v, want [publish]", capturedStages)
|
||||||
|
}
|
||||||
|
if !capturedForce {
|
||||||
|
t.Fatal("captured force = false, want true")
|
||||||
|
}
|
||||||
|
if strings.Join(capturedArtifacts, ",") != "session_recap" {
|
||||||
|
t.Fatalf("captured artifacts = %#v, want [session_recap]", capturedArtifacts)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout.String(), "narratio publish: executed=1 skipped=0 force=true; manifest=") {
|
||||||
|
t.Fatalf("stdout = %q, want publish summary", stdout.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecutePublishRejectsUnsupportedArgsAndFlags(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
args []string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "extra positional", args: []string{"publish", "2026-05-03", "extra"}, want: "publish: unexpected positional arguments"},
|
||||||
|
{name: "force flag", args: []string{"publish", "--force"}, want: "publish: invalid flags: flag provided but not defined: -force"},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute(tc.args, &stdout, &stderr)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), tc.want) {
|
||||||
|
t.Fatalf("stderr = %q, want %q", stderr.String(), tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecutePublishUnknownArtifactFailsValidation(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute(
|
||||||
|
[]string{"publish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "unknown_artifact"},
|
||||||
|
&stdout,
|
||||||
|
&stderr,
|
||||||
|
)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), `publish: --artifacts includes unknown artifact "unknown_artifact"`) {
|
||||||
|
t.Fatalf("stderr = %q, want unknown-artifact validation error", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecutePublishMissingConfigUsesRunStageLoadingPath(t *testing.T) {
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"publish", "2026-05-03"}, &stdout, &stderr)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "publish: no pipeline config path provided and no default pipeline config found; searched:") {
|
||||||
|
t.Fatalf("stderr = %q, want pipeline discovery error", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteUsageIncludesAnalyzeAndPublish(t *testing.T) {
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute(nil, &stdout, &stderr)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "analyze") {
|
||||||
|
t.Fatalf("stderr = %q, want usage to include analyze", stderr.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "publish") {
|
||||||
|
t.Fatalf("stderr = %q, want usage to include publish", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeValidConfigFilesWithScriptoriumArtifacts(t *testing.T, workspaceRoot string) (string, string, string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
f, err := os.OpenFile(pipelinePath, os.O_APPEND|os.O_WRONLY, 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("open pipeline config for append: %v", err)
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
|
||||||
|
extra := `
|
||||||
|
scriptorium:
|
||||||
|
binary: scriptorium
|
||||||
|
artifacts:
|
||||||
|
session_recap:
|
||||||
|
enabled: true
|
||||||
|
prompt_id: dnd.session_recap
|
||||||
|
output_path: artifacts/session_recap.md
|
||||||
|
player_handout:
|
||||||
|
enabled: true
|
||||||
|
prompt_id: dnd.player_handout
|
||||||
|
output_path: artifacts/player_handout.md
|
||||||
|
depends_on:
|
||||||
|
- session_recap
|
||||||
|
inputs:
|
||||||
|
recap:
|
||||||
|
source: narratio.artifact.session_recap
|
||||||
|
required: true
|
||||||
|
`
|
||||||
|
if _, err := f.WriteString(extra); err != nil {
|
||||||
|
t.Fatalf("append scriptorium config: %v", err)
|
||||||
|
}
|
||||||
|
return pipelinePath, campaignPath, sessionPath
|
||||||
|
}
|
||||||
132
internal/app/analyze_artifacts_test.go
Normal file
132
internal/app/analyze_artifacts_test.go
Normal file
@@ -0,0 +1,132 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestArtifactSelectionFlagNormalize(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
inputs []string
|
||||||
|
want []string
|
||||||
|
wantErr string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "single value",
|
||||||
|
inputs: []string{"session_recap"},
|
||||||
|
want: []string{"session_recap"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "repeatable and comma separated values are deduped and sorted",
|
||||||
|
inputs: []string{"session_recap,player_handout", "session_recap"},
|
||||||
|
want: []string{"player_handout", "session_recap"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty token fails",
|
||||||
|
inputs: []string{"session_recap,"},
|
||||||
|
wantErr: "artifact names must be non-empty",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
var flag artifactSelectionFlag
|
||||||
|
for _, in := range tt.inputs {
|
||||||
|
if err := flag.Set(in); err != nil {
|
||||||
|
t.Fatalf("Set(%q) error = %v", in, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := flag.Normalize()
|
||||||
|
if tt.wantErr != "" {
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("Normalize() error = nil, want %q", tt.wantErr)
|
||||||
|
}
|
||||||
|
if err.Error() != tt.wantErr {
|
||||||
|
t.Fatalf("Normalize() error = %q, want %q", err.Error(), tt.wantErr)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(got) != len(tt.want) {
|
||||||
|
t.Fatalf("Normalize() len = %d, want %d; got=%v", len(got), len(tt.want), got)
|
||||||
|
}
|
||||||
|
for i := range got {
|
||||||
|
if got[i] != tt.want[i] {
|
||||||
|
t.Fatalf("Normalize()[%d] = %q, want %q", i, got[i], tt.want[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateSelectedArtifacts(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
cfg *config.Config
|
||||||
|
selected []string
|
||||||
|
wantErr string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "empty selection is accepted",
|
||||||
|
cfg: &config.Config{},
|
||||||
|
selected: nil,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "scriptorium required when selected artifacts present",
|
||||||
|
cfg: &config.Config{Pipeline: &config.PipelineConfig{}},
|
||||||
|
selected: []string{"session_recap"},
|
||||||
|
wantErr: "--artifacts requires pipeline.scriptorium.artifacts to be configured",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "unknown selected artifact fails",
|
||||||
|
cfg: &config.Config{
|
||||||
|
Pipeline: &config.PipelineConfig{
|
||||||
|
Scriptorium: &config.ScriptoriumConfig{
|
||||||
|
Artifacts: map[string]config.ScriptoriumArtifactConfig{
|
||||||
|
"session_recap": {Enabled: true, PromptID: "dnd.session_recap", OutputPath: "artifacts/session_recap.md"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
selected: []string{"player_handout"},
|
||||||
|
wantErr: `--artifacts includes unknown artifact "player_handout"`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "known selected artifacts are accepted",
|
||||||
|
cfg: &config.Config{
|
||||||
|
Pipeline: &config.PipelineConfig{
|
||||||
|
Scriptorium: &config.ScriptoriumConfig{
|
||||||
|
Artifacts: map[string]config.ScriptoriumArtifactConfig{
|
||||||
|
"session_recap": {Enabled: true, PromptID: "dnd.session_recap", OutputPath: "artifacts/session_recap.md"},
|
||||||
|
"player_handout": {Enabled: true, PromptID: "dnd.player_handout", OutputPath: "artifacts/player_handout.md"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
selected: []string{"player_handout", "session_recap"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
err := validateSelectedArtifacts(tt.cfg, tt.selected)
|
||||||
|
if tt.wantErr != "" {
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("error = nil, want %q", tt.wantErr)
|
||||||
|
}
|
||||||
|
if err.Error() != tt.wantErr {
|
||||||
|
t.Fatalf("error = %q, want %q", err.Error(), tt.wantErr)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
44
internal/app/campaign_config_path.go
Normal file
44
internal/app/campaign_config_path.go
Normal file
@@ -0,0 +1,44 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func resolveCampaignConfigPath(pipelineCfg *config.PipelineConfig, campaignIDFlag, campaignFileFlag string) (string, error) {
|
||||||
|
campaignID := strings.TrimSpace(campaignIDFlag)
|
||||||
|
campaignFile := strings.TrimSpace(campaignFileFlag)
|
||||||
|
if campaignID != "" && campaignFile != "" {
|
||||||
|
return "", fmt.Errorf("--campaign and --campaign-file are mutually exclusive")
|
||||||
|
}
|
||||||
|
if campaignFile != "" {
|
||||||
|
return filepath.Clean(campaignFile), nil
|
||||||
|
}
|
||||||
|
if campaignID == "" && pipelineCfg != nil {
|
||||||
|
campaignID = strings.TrimSpace(pipelineCfg.Campaigns.DefaultCampaignID)
|
||||||
|
}
|
||||||
|
if campaignID == "" {
|
||||||
|
return "", fmt.Errorf("no campaign selected; pass --campaign <id> or set pipeline.campaigns.default_campaign_id")
|
||||||
|
}
|
||||||
|
if err := validateCampaignIDToken(campaignID); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if pipelineCfg == nil || strings.TrimSpace(pipelineCfg.Campaigns.Root) == "" {
|
||||||
|
return "", fmt.Errorf("pipeline.campaigns.root is required to select campaign %q", campaignID)
|
||||||
|
}
|
||||||
|
return filepath.Clean(filepath.Join(pipelineCfg.Campaigns.Root, campaignID, "campaign.yml")), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateCampaignIDToken(campaignID string) error {
|
||||||
|
if filepath.IsAbs(campaignID) ||
|
||||||
|
strings.Contains(campaignID, "/") ||
|
||||||
|
strings.Contains(campaignID, `\`) ||
|
||||||
|
campaignID == "." ||
|
||||||
|
campaignID == ".." {
|
||||||
|
return fmt.Errorf("campaign id %q must be a single path segment", campaignID)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
84
internal/app/campaign_config_path_test.go
Normal file
84
internal/app/campaign_config_path_test.go
Normal file
@@ -0,0 +1,84 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestResolveCampaignConfigPathCampaignFileWins(t *testing.T) {
|
||||||
|
explicit := filepath.Join(t.TempDir(), "custom-campaign.yml")
|
||||||
|
got, err := resolveCampaignConfigPath(&config.PipelineConfig{}, "", explicit)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolveCampaignConfigPath() error = %v", err)
|
||||||
|
}
|
||||||
|
if got != explicit {
|
||||||
|
t.Fatalf("path = %q, want explicit path %q", got, explicit)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveCampaignConfigPathUsesSelectedCampaignID(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
pipelineCfg := &config.PipelineConfig{}
|
||||||
|
pipelineCfg.Campaigns.Root = dir
|
||||||
|
|
||||||
|
got, err := resolveCampaignConfigPath(pipelineCfg, "icewind", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolveCampaignConfigPath() error = %v", err)
|
||||||
|
}
|
||||||
|
want := filepath.Join(dir, "icewind", "campaign.yml")
|
||||||
|
if got != filepath.Clean(want) {
|
||||||
|
t.Fatalf("path = %q, want %q", got, filepath.Clean(want))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveCampaignConfigPathUsesDefaultCampaignID(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
pipelineCfg := &config.PipelineConfig{}
|
||||||
|
pipelineCfg.Campaigns.Root = dir
|
||||||
|
pipelineCfg.Campaigns.DefaultCampaignID = "dilfs"
|
||||||
|
|
||||||
|
got, err := resolveCampaignConfigPath(pipelineCfg, "", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolveCampaignConfigPath() error = %v", err)
|
||||||
|
}
|
||||||
|
want := filepath.Join(dir, "dilfs", "campaign.yml")
|
||||||
|
if got != filepath.Clean(want) {
|
||||||
|
t.Fatalf("path = %q, want %q", got, filepath.Clean(want))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveCampaignConfigPathRejectsCampaignIDAndFile(t *testing.T) {
|
||||||
|
_, err := resolveCampaignConfigPath(&config.PipelineConfig{}, "dilfs", filepath.Join(t.TempDir(), "campaign.yml"))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "mutually exclusive") {
|
||||||
|
t.Fatalf("error = %q, want mutual exclusion", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveCampaignConfigPathRequiresCampaignSelection(t *testing.T) {
|
||||||
|
_, err := resolveCampaignConfigPath(&config.PipelineConfig{}, "", "")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "no campaign selected") {
|
||||||
|
t.Fatalf("error = %q, want missing selection guidance", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveCampaignConfigPathRejectsPathLikeCampaignID(t *testing.T) {
|
||||||
|
pipelineCfg := &config.PipelineConfig{}
|
||||||
|
pipelineCfg.Campaigns.Root = t.TempDir()
|
||||||
|
|
||||||
|
_, err := resolveCampaignConfigPath(pipelineCfg, "../icewind", "")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "single path segment") {
|
||||||
|
t.Fatalf("error = %q, want path segment guidance", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
281
internal/app/clean.go
Normal file
281
internal/app/clean.go
Normal file
@@ -0,0 +1,281 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Clean removes local workspace/spool state while preserving durable cache
|
||||||
|
// state unless cache cleanup is explicitly requested.
|
||||||
|
func Clean(ctx context.Context, args []string, out io.Writer) error {
|
||||||
|
fs := flag.NewFlagSet("clean", flag.ContinueOnError)
|
||||||
|
fs.SetOutput(io.Discard)
|
||||||
|
var flags commonConfigFlags
|
||||||
|
var all bool
|
||||||
|
var dryRun bool
|
||||||
|
var clearCache bool
|
||||||
|
addCommonConfigFlags(fs, &flags)
|
||||||
|
fs.BoolVar(&all, "all", false, "clean all local session work/spool state")
|
||||||
|
fs.BoolVar(&dryRun, "dry-run", false, "print cleanup targets without deleting")
|
||||||
|
fs.BoolVar(&clearCache, "clear-cache", false, "also clear durable S3 audio cache entries")
|
||||||
|
if err := parseSessionAwareFlags("clean", fs, args, &flags.sessionID); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if all {
|
||||||
|
return cleanAllLocal(flags, dryRun, clearCache, out)
|
||||||
|
}
|
||||||
|
return cleanSession(ctx, flags, dryRun, clearCache, out)
|
||||||
|
}
|
||||||
|
|
||||||
|
func cleanSession(ctx context.Context, flags commonConfigFlags, dryRun, clearCache bool, out io.Writer) error {
|
||||||
|
if strings.TrimSpace(flags.sessionID) == "" {
|
||||||
|
return fmt.Errorf("clean: session_id is required unless --all is set")
|
||||||
|
}
|
||||||
|
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("clean: %w", err)
|
||||||
|
}
|
||||||
|
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
|
||||||
|
return fmt.Errorf("clean: resolved pipeline and session config are required")
|
||||||
|
}
|
||||||
|
campaign := strings.TrimSpace(cfg.Session.Campaign)
|
||||||
|
sessionID := strings.TrimSpace(cfg.Session.SessionID)
|
||||||
|
if campaign == "" || sessionID == "" {
|
||||||
|
return fmt.Errorf("clean: campaign and session_id are required")
|
||||||
|
}
|
||||||
|
|
||||||
|
if dryRun {
|
||||||
|
fmt.Fprintf(out, "Clean plan for %s/%s\n", campaign, sessionID)
|
||||||
|
} else {
|
||||||
|
fmt.Fprintf(out, "Cleaned %s/%s\n", campaign, sessionID)
|
||||||
|
}
|
||||||
|
|
||||||
|
workDir := artifacts.SessionWorkDirForCampaign(cfg.Pipeline.Workspace.Root, campaign, sessionID)
|
||||||
|
spoolDir := artifacts.SessionSpoolDir(cfg.Pipeline.Spool.Root, campaign, sessionID)
|
||||||
|
if err := reportCleanScopedDir(out, cfg.Pipeline.Workspace.Root, workDir, "clean.workspace.session", dryRun); err != nil {
|
||||||
|
return fmt.Errorf("clean: %w", err)
|
||||||
|
}
|
||||||
|
if err := reportCleanScopedDir(out, cfg.Pipeline.Spool.Root, spoolDir, "clean.spool.session", dryRun); err != nil {
|
||||||
|
return fmt.Errorf("clean: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if clearCache {
|
||||||
|
if err := cleanSessionAudioCache(ctx, cfg, dryRun, out); err != nil {
|
||||||
|
return fmt.Errorf("clean: %w", err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
fmt.Fprintln(out, "Cache: preserved")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cleanAllLocal(flags commonConfigFlags, dryRun, clearCache bool, out io.Writer) error {
|
||||||
|
if strings.TrimSpace(flags.campaignPath) != "" ||
|
||||||
|
strings.TrimSpace(flags.campaignFilePath) != "" ||
|
||||||
|
strings.TrimSpace(flags.sessionPath) != "" ||
|
||||||
|
strings.TrimSpace(flags.sessionID) != "" ||
|
||||||
|
strings.TrimSpace(flags.previousSessionID) != "" {
|
||||||
|
return fmt.Errorf("clean: --all cannot be combined with --campaign, --campaign-file, --session, a session_id, or --previous-session-id")
|
||||||
|
}
|
||||||
|
resolvedPipelinePath, err := resolvePipelineConfigPath(flags.pipelinePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("clean: %w", err)
|
||||||
|
}
|
||||||
|
pipelineCfg, err := config.LoadPipeline(resolvedPipelinePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("clean: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if dryRun {
|
||||||
|
fmt.Fprintln(out, "Clean plan for all local sessions")
|
||||||
|
} else {
|
||||||
|
fmt.Fprintln(out, "Cleaned all local sessions")
|
||||||
|
}
|
||||||
|
|
||||||
|
workRoot := filepath.Join(pipelineCfg.Workspace.Root, config.PathWorkDirSegment)
|
||||||
|
if err := reportCleanScopedDir(out, pipelineCfg.Workspace.Root, workRoot, "clean.workspace.all", dryRun); err != nil {
|
||||||
|
return fmt.Errorf("clean: %w", err)
|
||||||
|
}
|
||||||
|
if err := reportCleanRootChildren(out, pipelineCfg.Spool.Root, "clean.spool.all", dryRun); err != nil {
|
||||||
|
return fmt.Errorf("clean: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if clearCache {
|
||||||
|
if err := cleanAllAudioCache(pipelineCfg, dryRun, out); err != nil {
|
||||||
|
return fmt.Errorf("clean: %w", err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
fmt.Fprintln(out, "Cache: preserved")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func reportCleanScopedDir(out io.Writer, root, target, policy string, dryRun bool) error {
|
||||||
|
dir, err := validateScopedDir(root, target, policy)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if dryRun {
|
||||||
|
if dir.Exists {
|
||||||
|
fmt.Fprintf(out, "Would delete: %s\n", dir.TargetAbs)
|
||||||
|
} else {
|
||||||
|
fmt.Fprintf(out, "Would skip missing: %s\n", dir.TargetAbs)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if !dir.Exists {
|
||||||
|
fmt.Fprintf(out, "Missing: %s\n", dir.TargetAbs)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := os.RemoveAll(dir.TargetAbs); err != nil {
|
||||||
|
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, dir.TargetAbs, err)
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "Deleted: %s\n", dir.TargetAbs)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func reportCleanRootChildren(out io.Writer, root, policy string, dryRun bool) error {
|
||||||
|
rootAbs, entries, err := cleanableRootChildren(root, policy)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(entries) == 0 {
|
||||||
|
if dryRun {
|
||||||
|
fmt.Fprintf(out, "Would skip empty: %s\n", rootAbs)
|
||||||
|
} else {
|
||||||
|
fmt.Fprintf(out, "Empty: %s\n", rootAbs)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for _, entry := range entries {
|
||||||
|
if dryRun {
|
||||||
|
fmt.Fprintf(out, "Would delete: %s\n", entry)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := os.RemoveAll(entry); err != nil {
|
||||||
|
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, entry, err)
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "Deleted: %s\n", entry)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cleanableRootChildren(root, policy string) (string, []string, error) {
|
||||||
|
rootAbs, exists, err := validateCleanRoot(root, policy)
|
||||||
|
if err != nil {
|
||||||
|
return "", nil, err
|
||||||
|
}
|
||||||
|
if !exists {
|
||||||
|
return rootAbs, nil, nil
|
||||||
|
}
|
||||||
|
entries, err := os.ReadDir(rootAbs)
|
||||||
|
if err != nil {
|
||||||
|
return "", nil, fmt.Errorf("cleanup policy %s: read root %q: %w", policy, rootAbs, err)
|
||||||
|
}
|
||||||
|
out := make([]string, 0, len(entries))
|
||||||
|
for _, entry := range entries {
|
||||||
|
path := filepath.Join(rootAbs, entry.Name())
|
||||||
|
info, err := os.Lstat(path)
|
||||||
|
if err != nil {
|
||||||
|
return "", nil, fmt.Errorf("cleanup policy %s: stat child %q: %w", policy, path, err)
|
||||||
|
}
|
||||||
|
if info.Mode()&os.ModeSymlink != 0 {
|
||||||
|
return "", nil, fmt.Errorf("cleanup policy %s: refusing to delete symlink path %q", policy, path)
|
||||||
|
}
|
||||||
|
out = append(out, path)
|
||||||
|
}
|
||||||
|
return rootAbs, out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cleanSessionAudioCache(ctx context.Context, cfg *config.Config, dryRun bool, out io.Writer) error {
|
||||||
|
if cfg.Session.Inputs.AudioS3 == nil {
|
||||||
|
fmt.Fprintln(out, "Cache: skipped (session does not use audio_s3)")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if cfg.Pipeline.Storage.S3 == nil || strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket) == "" {
|
||||||
|
return fmt.Errorf("clear cache requires pipeline.storage.s3.bucket")
|
||||||
|
}
|
||||||
|
store, err := newCommandObjectStore(ctx, cfg, nil)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("initialize object store for cache cleanup: %w", err)
|
||||||
|
}
|
||||||
|
sessionPrefix := artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
audioPrefix := artifacts.S3AudioPrefix(sessionPrefix, cfg.Session.Inputs.AudioS3.Prefix)
|
||||||
|
objects, err := store.List(ctx, audioPrefix)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("list s3 audio objects under %q: %w", audioPrefix, err)
|
||||||
|
}
|
||||||
|
count := 0
|
||||||
|
for _, obj := range objects {
|
||||||
|
key := strings.TrimSpace(obj.Key)
|
||||||
|
if key == "" || strings.HasSuffix(key, "/") || !cleanIsFlac(key) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
cachePath, err := artifacts.S3AudioCachePath(cfg.Pipeline.Cache.Root, cfg.Pipeline.Storage.S3.Bucket, key)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
deleted, err := reportCleanScopedFile(out, cfg.Pipeline.Cache.Root, cachePath, "clean.cache.session", dryRun)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if deleted {
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if count == 0 {
|
||||||
|
fmt.Fprintf(out, "Cache: no cached S3 audio files found for %s\n", audioPrefix)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cleanAllAudioCache(cfg *config.PipelineConfig, dryRun bool, out io.Writer) error {
|
||||||
|
if cfg.Storage.S3 == nil || strings.TrimSpace(cfg.Storage.S3.Bucket) == "" {
|
||||||
|
return fmt.Errorf("clear cache requires pipeline.storage.s3.bucket")
|
||||||
|
}
|
||||||
|
namespaceDir, err := artifacts.S3AudioCacheNamespaceDir(cfg.Cache.Root, cfg.Storage.S3.Bucket, cfg.Storage.S3.RootPrefix)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return reportCleanScopedDir(out, cfg.Cache.Root, namespaceDir, "clean.cache.all", dryRun)
|
||||||
|
}
|
||||||
|
|
||||||
|
func reportCleanScopedFile(out io.Writer, root, target, policy string, dryRun bool) (bool, error) {
|
||||||
|
file, err := validateScopedFile(root, target, policy)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
if dryRun {
|
||||||
|
if file.Exists {
|
||||||
|
fmt.Fprintf(out, "Would delete cache file: %s\n", file.TargetAbs)
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "Would skip missing cache file: %s\n", file.TargetAbs)
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
if !file.Exists {
|
||||||
|
fmt.Fprintf(out, "Missing cache file: %s\n", file.TargetAbs)
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
if err := os.Remove(file.TargetAbs); err != nil {
|
||||||
|
return false, fmt.Errorf("cleanup policy %s: remove %q: %w", policy, file.TargetAbs, err)
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "Deleted cache file: %s\n", file.TargetAbs)
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateScopedFile(root, target, policy string) (scopedDir, error) {
|
||||||
|
return validateScopedTarget(root, target, policy, false)
|
||||||
|
}
|
||||||
|
|
||||||
|
func cleanIsFlac(path string) bool {
|
||||||
|
return strings.EqualFold(filepath.Ext(path), ".flac")
|
||||||
|
}
|
||||||
255
internal/app/clean_test.go
Normal file
255
internal/app/clean_test.go
Normal file
@@ -0,0 +1,255 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestExecuteCleanSessionDeletesWorkAndSpoolButPreservesCache(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
workDir := artifacts.SessionWorkDirForCampaign(workspaceRoot, "sample-campaign", "2026-05-03")
|
||||||
|
spoolDir := artifacts.SessionSpoolDir(filepath.Join(workspaceRoot, "spool"), "sample-campaign", "2026-05-03")
|
||||||
|
cachePath, err := artifacts.S3AudioCachePath(filepath.Join(workspaceRoot, "cache"), "test-bucket", "dnd/campaigns/sample-campaign/sessions/2026-05-03/audio/alice.flac")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("S3AudioCachePath() error = %v", err)
|
||||||
|
}
|
||||||
|
mustWriteTestFile(t, filepath.Join(workDir, "manifest.json"), "{}")
|
||||||
|
mustWriteTestFile(t, filepath.Join(spoolDir, "run-1", "audio", "alice.flac"), "audio")
|
||||||
|
mustWriteTestFile(t, cachePath, "cached-audio")
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"clean", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
cleanAssertMissing(t, workDir)
|
||||||
|
cleanAssertMissing(t, spoolDir)
|
||||||
|
cleanAssertExists(t, cachePath)
|
||||||
|
if !strings.Contains(stdout.String(), "Cache: preserved") {
|
||||||
|
t.Fatalf("stdout = %q, want cache preserved", stdout.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteCleanSessionDryRunDeletesNothing(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
workDir := artifacts.SessionWorkDirForCampaign(workspaceRoot, "sample-campaign", "2026-05-03")
|
||||||
|
spoolDir := artifacts.SessionSpoolDir(filepath.Join(workspaceRoot, "spool"), "sample-campaign", "2026-05-03")
|
||||||
|
mustWriteTestFile(t, filepath.Join(workDir, "manifest.json"), "{}")
|
||||||
|
mustWriteTestFile(t, filepath.Join(spoolDir, "run-1", "audio", "alice.flac"), "audio")
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"clean", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--dry-run"}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
cleanAssertExists(t, workDir)
|
||||||
|
cleanAssertExists(t, spoolDir)
|
||||||
|
if !strings.Contains(stdout.String(), "Would delete:") {
|
||||||
|
t.Fatalf("stdout = %q, want dry-run delete plan", stdout.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteCleanMissingSessionPathsSucceeds(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"clean", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout.String(), "Missing:") {
|
||||||
|
t.Fatalf("stdout = %q, want missing path output", stdout.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteCleanSessionClearCacheRemovesOnlyS3AudioCache(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
if err := os.WriteFile(sessionPath, []byte(`session_id: 2026-05-03
|
||||||
|
inputs:
|
||||||
|
audio_s3:
|
||||||
|
prefix: audio/
|
||||||
|
`), 0o644); err != nil {
|
||||||
|
t.Fatalf("write session: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
audioKey := "dnd/campaigns/sample-campaign/sessions/2026-05-03/audio/alice.flac"
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: audioKey, Data: []byte("audio")})
|
||||||
|
var storeInitCalls int
|
||||||
|
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||||
|
|
||||||
|
cacheRoot := filepath.Join(workspaceRoot, "cache")
|
||||||
|
cachePath, err := artifacts.S3AudioCachePath(cacheRoot, "test-bucket", audioKey)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("S3AudioCachePath() error = %v", err)
|
||||||
|
}
|
||||||
|
otherCachePath, err := artifacts.S3AudioCachePath(cacheRoot, "test-bucket", "dnd/campaigns/other/sessions/2026-05-03/audio/bob.flac")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("S3AudioCachePath() error = %v", err)
|
||||||
|
}
|
||||||
|
mustWriteTestFile(t, cachePath, "cached-audio")
|
||||||
|
mustWriteTestFile(t, otherCachePath, "other-audio")
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"clean", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--clear-cache"}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
cleanAssertMissing(t, cachePath)
|
||||||
|
cleanAssertExists(t, otherCachePath)
|
||||||
|
if storeInitCalls != 1 {
|
||||||
|
t.Fatalf("object store init calls = %d, want 1", storeInitCalls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteCleanLocalAudioClearCacheIsNoop(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"clean", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--clear-cache"}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout.String(), "Cache: skipped (session does not use audio_s3)") {
|
||||||
|
t.Fatalf("stdout = %q, want local audio cache no-op", stdout.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteCleanAllDeletesWorkAndSpoolContentsButPreservesCache(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, _, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
workRoot := filepath.Join(workspaceRoot, "work")
|
||||||
|
spoolRoot := filepath.Join(workspaceRoot, "spool")
|
||||||
|
cachePath := filepath.Join(workspaceRoot, "cache", "keep.txt")
|
||||||
|
mustWriteTestFile(t, filepath.Join(workRoot, "sample-campaign", "2026-05-03", "manifest.json"), "{}")
|
||||||
|
mustWriteTestFile(t, filepath.Join(spoolRoot, "sample-campaign", "2026-05-03", "run-1", "audio", "alice.flac"), "audio")
|
||||||
|
mustWriteTestFile(t, cachePath, "cache")
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"clean", "--config", pipelinePath, "--all"}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
cleanAssertMissing(t, workRoot)
|
||||||
|
cleanAssertExists(t, spoolRoot)
|
||||||
|
cleanAssertMissing(t, filepath.Join(spoolRoot, "sample-campaign"))
|
||||||
|
cleanAssertExists(t, cachePath)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteCleanAllClearCacheRemovesS3AudioNamespaceOnly(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, _, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
cacheRoot := filepath.Join(workspaceRoot, "cache")
|
||||||
|
audioCachePath, err := artifacts.S3AudioCachePath(cacheRoot, "test-bucket", "dnd/campaigns/sample-campaign/sessions/2026-05-03/audio/alice.flac")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("S3AudioCachePath() error = %v", err)
|
||||||
|
}
|
||||||
|
otherCachePath, err := artifacts.S3AudioCachePath(cacheRoot, "test-bucket", "other-root/campaigns/sample-campaign/sessions/2026-05-03/audio/alice.flac")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("S3AudioCachePath() error = %v", err)
|
||||||
|
}
|
||||||
|
mustWriteTestFile(t, audioCachePath, "cached-audio")
|
||||||
|
mustWriteTestFile(t, otherCachePath, "other-cache")
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"clean", "--config", pipelinePath, "--all", "--clear-cache"}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
cleanAssertMissing(t, audioCachePath)
|
||||||
|
cleanAssertExists(t, otherCachePath)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteCleanAllRejectsSessionScopedFlags(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"clean", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--all"}, &stdout, &stderr)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "--all cannot be combined") {
|
||||||
|
t.Fatalf("stderr = %q, want --all conflict", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCleanRequiresSessionID(t *testing.T) {
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"clean"}, &stdout, &stderr)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "session_id is required unless --all is set") {
|
||||||
|
t.Fatalf("stderr = %q, want missing session-id", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCleanRejectsUnsafeTargets(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
outside := t.TempDir()
|
||||||
|
if err := reportCleanScopedDir(&bytes.Buffer{}, root, filepath.Join(outside, "target"), "test.outside", false); err == nil {
|
||||||
|
t.Fatal("outside target error = nil, want error")
|
||||||
|
}
|
||||||
|
if err := reportCleanScopedDir(&bytes.Buffer{}, root, root, "test.root", false); err == nil {
|
||||||
|
t.Fatal("root target error = nil, want error")
|
||||||
|
}
|
||||||
|
filePath := filepath.Join(root, "file.txt")
|
||||||
|
mustWriteTestFile(t, filePath, "file")
|
||||||
|
if err := reportCleanScopedDir(&bytes.Buffer{}, root, filePath, "test.file", false); err == nil {
|
||||||
|
t.Fatal("file target error = nil, want error")
|
||||||
|
}
|
||||||
|
symlinkPath := filepath.Join(root, "link")
|
||||||
|
if err := os.Symlink(filepath.Join(root, "missing"), symlinkPath); err != nil {
|
||||||
|
t.Fatalf("Symlink() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := reportCleanScopedDir(&bytes.Buffer{}, root, symlinkPath, "test.symlink", false); err == nil {
|
||||||
|
t.Fatal("symlink target error = nil, want error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClearIsNotCommandAlias(t *testing.T) {
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"clear"}, &stdout, &stderr)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), `unknown command: "clear"`) {
|
||||||
|
t.Fatalf("stderr = %q, want unknown clear command", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func cleanAssertExists(t *testing.T, path string) {
|
||||||
|
t.Helper()
|
||||||
|
if _, err := os.Stat(path); err != nil {
|
||||||
|
t.Fatalf("expected %q to exist: %v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func cleanAssertMissing(t *testing.T, path string) {
|
||||||
|
t.Helper()
|
||||||
|
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("expected %q to be missing, stat err=%v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
82
internal/app/cleanup_targets.go
Normal file
82
internal/app/cleanup_targets.go
Normal file
@@ -0,0 +1,82 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
func validateScopedTarget(root, target, policy string, requireDir bool) (scopedDir, error) {
|
||||||
|
cleanRoot := strings.TrimSpace(root)
|
||||||
|
cleanTarget := strings.TrimSpace(target)
|
||||||
|
if cleanRoot == "" {
|
||||||
|
return scopedDir{}, fmt.Errorf("cleanup policy %s: root path is required", policy)
|
||||||
|
}
|
||||||
|
if cleanTarget == "" {
|
||||||
|
return scopedDir{}, fmt.Errorf("cleanup policy %s: target path is required", policy)
|
||||||
|
}
|
||||||
|
|
||||||
|
rootAbs, err := filepath.Abs(cleanRoot)
|
||||||
|
if err != nil {
|
||||||
|
return scopedDir{}, fmt.Errorf("cleanup policy %s: resolve root %q: %w", policy, cleanRoot, err)
|
||||||
|
}
|
||||||
|
targetAbs, err := filepath.Abs(cleanTarget)
|
||||||
|
if err != nil {
|
||||||
|
return scopedDir{}, fmt.Errorf("cleanup policy %s: resolve target %q: %w", policy, cleanTarget, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
rel, err := filepath.Rel(rootAbs, targetAbs)
|
||||||
|
if err != nil {
|
||||||
|
return scopedDir{}, fmt.Errorf("cleanup policy %s: relative path from %q to %q: %w", policy, rootAbs, targetAbs, err)
|
||||||
|
}
|
||||||
|
if rel == "." {
|
||||||
|
return scopedDir{}, fmt.Errorf("cleanup policy %s: refusing to delete root directory %q", policy, rootAbs)
|
||||||
|
}
|
||||||
|
if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
|
||||||
|
return scopedDir{}, fmt.Errorf("cleanup policy %s: refusing to delete path outside root: root=%q target=%q", policy, rootAbs, targetAbs)
|
||||||
|
}
|
||||||
|
|
||||||
|
info, err := os.Lstat(targetAbs)
|
||||||
|
if err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return scopedDir{RootAbs: rootAbs, TargetAbs: targetAbs, Exists: false}, nil
|
||||||
|
}
|
||||||
|
return scopedDir{}, fmt.Errorf("cleanup policy %s: stat target %q: %w", policy, targetAbs, err)
|
||||||
|
}
|
||||||
|
if info.Mode()&os.ModeSymlink != 0 {
|
||||||
|
return scopedDir{}, fmt.Errorf("cleanup policy %s: refusing to delete symlink path %q", policy, targetAbs)
|
||||||
|
}
|
||||||
|
if requireDir && !info.IsDir() {
|
||||||
|
return scopedDir{}, fmt.Errorf("cleanup policy %s: target %q is not a directory", policy, targetAbs)
|
||||||
|
}
|
||||||
|
if !requireDir && info.IsDir() {
|
||||||
|
return scopedDir{}, fmt.Errorf("cleanup policy %s: target %q is a directory", policy, targetAbs)
|
||||||
|
}
|
||||||
|
return scopedDir{RootAbs: rootAbs, TargetAbs: targetAbs, Exists: true}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateCleanRoot(root, policy string) (string, bool, error) {
|
||||||
|
cleanRoot := strings.TrimSpace(root)
|
||||||
|
if cleanRoot == "" {
|
||||||
|
return "", false, fmt.Errorf("cleanup policy %s: root path is required", policy)
|
||||||
|
}
|
||||||
|
rootAbs, err := filepath.Abs(cleanRoot)
|
||||||
|
if err != nil {
|
||||||
|
return "", false, fmt.Errorf("cleanup policy %s: resolve root %q: %w", policy, cleanRoot, err)
|
||||||
|
}
|
||||||
|
info, err := os.Lstat(rootAbs)
|
||||||
|
if err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return rootAbs, false, nil
|
||||||
|
}
|
||||||
|
return "", false, fmt.Errorf("cleanup policy %s: stat root %q: %w", policy, rootAbs, err)
|
||||||
|
}
|
||||||
|
if info.Mode()&os.ModeSymlink != 0 {
|
||||||
|
return "", false, fmt.Errorf("cleanup policy %s: refusing to clean symlink root %q", policy, rootAbs)
|
||||||
|
}
|
||||||
|
if !info.IsDir() {
|
||||||
|
return "", false, fmt.Errorf("cleanup policy %s: root %q is not a directory", policy, rootAbs)
|
||||||
|
}
|
||||||
|
return rootAbs, true, nil
|
||||||
|
}
|
||||||
103
internal/app/cleanup_targets_test.go
Normal file
103
internal/app/cleanup_targets_test.go
Normal file
@@ -0,0 +1,103 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCleanValidateScopedDirAndFile(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
dirTarget := filepath.Join(root, "runs", "run-1")
|
||||||
|
fileTarget := filepath.Join(root, "cache", "a.flac")
|
||||||
|
if err := os.MkdirAll(dirTarget, 0o755); err != nil {
|
||||||
|
t.Fatalf("MkdirAll(dirTarget) error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Dir(fileTarget), 0o755); err != nil {
|
||||||
|
t.Fatalf("MkdirAll(file parent) error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(fileTarget, []byte("audio"), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(fileTarget) error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := validateScopedDir(root, dirTarget, "test.dir"); err != nil {
|
||||||
|
t.Fatalf("validateScopedDir() error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := validateScopedFile(root, fileTarget, "test.file"); err != nil {
|
||||||
|
t.Fatalf("validateScopedFile() error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCleanValidateScopedTargetSafetyRules(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
outside := t.TempDir()
|
||||||
|
target := filepath.Join(root, "runs", "run-1")
|
||||||
|
if err := os.MkdirAll(target, 0o755); err != nil {
|
||||||
|
t.Fatalf("MkdirAll(target) error = %v", err)
|
||||||
|
}
|
||||||
|
fileTarget := filepath.Join(root, "cache", "a.flac")
|
||||||
|
if err := os.MkdirAll(filepath.Dir(fileTarget), 0o755); err != nil {
|
||||||
|
t.Fatalf("MkdirAll(file parent) error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(fileTarget, []byte("audio"), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(fileTarget) error = %v", err)
|
||||||
|
}
|
||||||
|
symlinkTarget := filepath.Join(root, "symlink")
|
||||||
|
if err := os.Symlink(target, symlinkTarget); err != nil {
|
||||||
|
t.Fatalf("Symlink() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := validateScopedDir(root, root, "test.root"); err == nil || !strings.Contains(err.Error(), "refusing to delete root directory") {
|
||||||
|
t.Fatalf("validateScopedDir(root) error = %v, want root deletion rejection", err)
|
||||||
|
}
|
||||||
|
if _, err := validateScopedDir(root, filepath.Join(outside, "x"), "test.outside"); err == nil || !strings.Contains(err.Error(), "outside root") {
|
||||||
|
t.Fatalf("validateScopedDir(outside) error = %v, want outside-root rejection", err)
|
||||||
|
}
|
||||||
|
if _, err := validateScopedDir(root, fileTarget, "test.file-as-dir"); err == nil || !strings.Contains(err.Error(), "is not a directory") {
|
||||||
|
t.Fatalf("validateScopedDir(file) error = %v, want not-a-directory rejection", err)
|
||||||
|
}
|
||||||
|
if _, err := validateScopedFile(root, target, "test.dir-as-file"); err == nil || !strings.Contains(err.Error(), "is a directory") {
|
||||||
|
t.Fatalf("validateScopedFile(dir) error = %v, want is-a-directory rejection", err)
|
||||||
|
}
|
||||||
|
if _, err := validateScopedDir(root, symlinkTarget, "test.symlink"); err == nil || !strings.Contains(err.Error(), "refusing to delete symlink path") {
|
||||||
|
t.Fatalf("validateScopedDir(symlink) error = %v, want symlink rejection", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCleanableRootChildrenRejectsSymlinkChild(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
realChild := filepath.Join(root, "runs")
|
||||||
|
if err := os.MkdirAll(realChild, 0o755); err != nil {
|
||||||
|
t.Fatalf("MkdirAll(realChild) error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.Symlink(realChild, filepath.Join(root, "link")); err != nil {
|
||||||
|
t.Fatalf("Symlink() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, _, err := cleanableRootChildren(root, "test.root.children")
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "refusing to delete symlink path") {
|
||||||
|
t.Fatalf("cleanableRootChildren() error = %v, want symlink rejection", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCleanValidateScopedTargetMissing(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
missingDir := filepath.Join(root, "runs", "missing")
|
||||||
|
got, err := validateScopedDir(root, missingDir, "test.missing")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("validateScopedDir(missing) error = %v", err)
|
||||||
|
}
|
||||||
|
if got.Exists {
|
||||||
|
t.Fatalf("validateScopedDir(missing).Exists = true, want false")
|
||||||
|
}
|
||||||
|
|
||||||
|
missingFile := filepath.Join(root, "cache", "missing.flac")
|
||||||
|
got, err = validateScopedFile(root, missingFile, "test.missing.file")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("validateScopedFile(missing) error = %v", err)
|
||||||
|
}
|
||||||
|
if got.Exists {
|
||||||
|
t.Fatalf("validateScopedFile(missing).Exists = true, want false")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -7,7 +7,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
var supportedCommands = []string{"run", "plan", "status", "resume", "run-stage"}
|
var supportedCommands = []string{"run", "run-stage", "analyze", "publish", "clean", "session"}
|
||||||
|
|
||||||
// Execute dispatches CLI commands and returns a process exit code.
|
// Execute dispatches CLI commands and returns a process exit code.
|
||||||
func Execute(args []string, stdout, stderr io.Writer) int {
|
func Execute(args []string, stdout, stderr io.Writer) int {
|
||||||
@@ -24,14 +24,16 @@ func Execute(args []string, stdout, stderr io.Writer) int {
|
|||||||
switch cmd {
|
switch cmd {
|
||||||
case "run":
|
case "run":
|
||||||
err = Run(ctx, cmdArgs, stdout)
|
err = Run(ctx, cmdArgs, stdout)
|
||||||
case "plan":
|
|
||||||
err = Plan(ctx, cmdArgs, stdout)
|
|
||||||
case "status":
|
|
||||||
err = Status(ctx, cmdArgs, stdout)
|
|
||||||
case "resume":
|
|
||||||
err = Resume(ctx, cmdArgs, stdout)
|
|
||||||
case "run-stage":
|
case "run-stage":
|
||||||
err = RunStage(ctx, cmdArgs, stdout)
|
err = RunStage(ctx, cmdArgs, stdout)
|
||||||
|
case "analyze":
|
||||||
|
err = Analyze(ctx, cmdArgs, stdout)
|
||||||
|
case "publish":
|
||||||
|
err = Publish(ctx, cmdArgs, stdout)
|
||||||
|
case "session":
|
||||||
|
err = Session(ctx, cmdArgs, stdout)
|
||||||
|
case "clean":
|
||||||
|
err = Clean(ctx, cmdArgs, stdout)
|
||||||
default:
|
default:
|
||||||
fmt.Fprintf(stderr, "unknown command: %q\n\n", cmd)
|
fmt.Fprintf(stderr, "unknown command: %q\n\n", cmd)
|
||||||
printUsage(stderr)
|
printUsage(stderr)
|
||||||
|
|||||||
@@ -24,19 +24,17 @@ func TestExecuteValidCommands(t *testing.T) {
|
|||||||
}))
|
}))
|
||||||
defer srv.Close()
|
defer srv.Close()
|
||||||
|
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
||||||
manifestPath := writeManifestPathForExecute(t)
|
|
||||||
|
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
name string
|
name string
|
||||||
args []string
|
args []string
|
||||||
wantOut string
|
wantOut string
|
||||||
}{
|
}{
|
||||||
{name: "run", args: []string{"run", "--config", pipelinePath, "--session", sessionPath}, wantOut: "narratio run: session 2026-05-03; executed=9 skipped=0; manifest="},
|
{name: "run", args: []string{"run", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "narratio run: session 2026-05-03; executed=10 skipped=0; manifest="},
|
||||||
{name: "plan", args: []string{"plan", "--config", pipelinePath, "--session", sessionPath}, wantOut: "prepare: skip\ntranscribe: skip\nmerge: skip\npolish: skip\nnormalize: skip\ntrim: skip\nanalyze: skip\narchive: skip\nnotify: skip"},
|
{name: "session plan", args: []string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "prepare: skip\ntranscribe: skip\nmerge: skip\npolish: skip\nnormalize: skip\ntrim: skip\nrender: skip\nanalyze: skip\npublish: skip\nnotify: skip"},
|
||||||
{name: "status", args: []string{"status", "--manifest", manifestPath}, wantOut: "session_id: 2026-05-03"},
|
{name: "session status", args: []string{"session", "status", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "Session: 2026-05-03"},
|
||||||
{name: "resume", args: []string{"resume", "--config", pipelinePath, "--session", sessionPath}, wantOut: "narratio resume: session 2026-05-03 has no remaining stages"},
|
{name: "run-stage", args: []string{"run-stage", "polish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "narratio run-stage: stage=polish executed=0 skipped=1 force=false; manifest="},
|
||||||
{name: "run-stage", args: []string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "polish"}, wantOut: "narratio run-stage: stage=polish executed=0 skipped=1 force=false; manifest="},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range cases {
|
for _, tc := range cases {
|
||||||
@@ -64,13 +62,13 @@ func TestExecuteMissingRequiredFlags(t *testing.T) {
|
|||||||
args []string
|
args []string
|
||||||
want string
|
want string
|
||||||
}{
|
}{
|
||||||
{name: "run missing flags", args: []string{"run"}, want: "run: --session is required"},
|
{name: "run missing session", args: []string{"run"}, want: "run: session_id is required"},
|
||||||
{name: "plan missing flags", args: []string{"plan"}, want: "plan: --session is required"},
|
{name: "plan old top-level removed", args: []string{"plan"}, want: `unknown command: "plan"`},
|
||||||
{name: "status missing flags", args: []string{"status"}, want: "status: --manifest is required"},
|
{name: "status old top-level removed", args: []string{"status"}, want: `unknown command: "status"`},
|
||||||
{name: "resume missing flags", args: []string{"resume"}, want: "resume: --session is required"},
|
{name: "resume removed", args: []string{"resume"}, want: `unknown command: "resume"`},
|
||||||
{name: "run-stage missing name", args: []string{"run-stage", "--config", "a", "--session", "b"}, want: "run-stage: expected exactly one stage name"},
|
{name: "run-stage missing name", args: []string{"run-stage", "--config", "a", "--session", "b"}, want: "run-stage: expected stage name and session_id"},
|
||||||
{name: "run-stage missing config flags", args: []string{"run-stage", "polish"}, want: "run-stage: --session is required"},
|
{name: "run-stage missing session", args: []string{"run-stage", "polish"}, want: "run-stage: expected stage name and session_id"},
|
||||||
{name: "run missing config uses defaults", args: []string{"run", "--session", "session.yml"}, want: "run: no pipeline config path provided and no default pipeline config found; searched:"},
|
{name: "run missing config uses defaults", args: []string{"run", "2026-05-03", "--session", "session.yml"}, want: "run: no pipeline config path provided and no default pipeline config found; searched:"},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range cases {
|
for _, tc := range cases {
|
||||||
@@ -94,12 +92,12 @@ func TestExecuteMissingRequiredFlags(t *testing.T) {
|
|||||||
|
|
||||||
func TestExecuteRunStageUnknownFails(t *testing.T) {
|
func TestExecuteRunStageUnknownFails(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
code := Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "unknown"}, &stdout, &stderr)
|
code := Execute([]string{"run-stage", "unknown", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||||
if code == 0 {
|
if code == 0 {
|
||||||
t.Fatal("exit code = 0, want non-zero")
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
}
|
}
|
||||||
@@ -108,16 +106,32 @@ func TestExecuteRunStageUnknownFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExecuteRunStageNormalizeIsAccepted(t *testing.T) {
|
func TestExecuteRunStageArchiveAliasFails(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
|
||||||
workRoot := filepath.Join(workspaceRoot, "work", "2026-05-03")
|
|
||||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "processed.json"), `{"segments":[{"id":1}]}`)
|
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
code := Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "normalize"}, &stdout, &stderr)
|
code := Execute([]string{"run-stage", "archive", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), `unknown stage "archive"`) {
|
||||||
|
t.Fatalf("stderr = %q, want unknown stage alias error", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteRunStageNormalizeIsAccepted(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
|
||||||
|
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||||
|
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "polished.json"), `{"segments":[{"id":1}]}`)
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := Execute([]string{"run-stage", "normalize", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
@@ -136,19 +150,19 @@ func TestExecuteRunStageTranscribeUsesConfiguredWhisperXServer(t *testing.T) {
|
|||||||
}))
|
}))
|
||||||
defer srv.Close()
|
defer srv.Close()
|
||||||
|
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
code := Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "prepare"}, &stdout, &stderr)
|
code := Execute([]string{"run-stage", "prepare", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("prepare exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("prepare exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
stdout.Reset()
|
stdout.Reset()
|
||||||
stderr.Reset()
|
stderr.Reset()
|
||||||
|
|
||||||
code = Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "--force", "transcribe"}, &stdout, &stderr)
|
code = Execute([]string{"run-stage", "transcribe", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--force"}, &stdout, &stderr)
|
||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("transcribe exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("transcribe exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
@@ -156,7 +170,7 @@ func TestExecuteRunStageTranscribeUsesConfiguredWhisperXServer(t *testing.T) {
|
|||||||
t.Fatal("expected whisperx server to be called at least once")
|
t.Fatal("expected whisperx server to be called at least once")
|
||||||
}
|
}
|
||||||
|
|
||||||
outPath := filepath.Join(workspaceRoot, "work", "2026-05-03", "transcripts", "raw", "alice.json")
|
outPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "transcripts", "raw", "alice.json")
|
||||||
data, err := os.ReadFile(outPath)
|
data, err := os.ReadFile(outPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("ReadFile(%q): %v", outPath, err)
|
t.Fatalf("ReadFile(%q): %v", outPath, err)
|
||||||
@@ -170,6 +184,135 @@ func TestExecuteRunStageTranscribeUsesConfiguredWhisperXServer(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExecuteRunStagePolishLoadsCredentialFromSecretsDir(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
configDir := t.TempDir()
|
||||||
|
sessionID := "2026-05-03"
|
||||||
|
secretsDir := filepath.Join(configDir, "secrets")
|
||||||
|
if err := os.MkdirAll(secretsDir, 0o755); err != nil {
|
||||||
|
t.Fatalf("MkdirAll(%q): %v", secretsDir, err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(secretsDir, "OPENROUTER_API_KEY"), []byte("from-secret-file\n"), 0o600); err != nil {
|
||||||
|
t.Fatalf("write OPENROUTER_API_KEY secret file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
seriatimBinary := writeSeriatimAppTestWrapper(t)
|
||||||
|
auditaBinary := writeAuditaAppTestWrapper(t)
|
||||||
|
t.Setenv("GO_WANT_APP_SERIATIM_HELPER", "1")
|
||||||
|
t.Setenv("GO_WANT_APP_AUDITA_HELPER", "1")
|
||||||
|
|
||||||
|
pipelinePath := filepath.Join(configDir, "pipeline.yml")
|
||||||
|
campaignPath := writeAppTestCampaignConfig(t, configDir)
|
||||||
|
sessionPath := filepath.Join(configDir, "session.yml")
|
||||||
|
pipelineYAML := `workspace:
|
||||||
|
root: ` + workspaceRoot + `
|
||||||
|
storage:
|
||||||
|
backend: local
|
||||||
|
secrets:
|
||||||
|
env_dir: ./secrets
|
||||||
|
whisperx:
|
||||||
|
transcribe_url: https://example.com/transcribe
|
||||||
|
seriatim:
|
||||||
|
binary: ` + seriatimBinary + `
|
||||||
|
audita:
|
||||||
|
binary: ` + auditaBinary + `
|
||||||
|
llm_api_key_env: OPENROUTER_API_KEY
|
||||||
|
notification:
|
||||||
|
timeout: 10s
|
||||||
|
`
|
||||||
|
sessionYAML := `session_id: ` + sessionID + `
|
||||||
|
campaign: sample-campaign
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
players_file: ./players.yml
|
||||||
|
party_file: ./party.yml
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write pipeline.yml: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write session.yml: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
originalWD, err := os.Getwd()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Getwd(): %v", err)
|
||||||
|
}
|
||||||
|
if err := os.Chdir(configDir); err != nil {
|
||||||
|
t.Fatalf("Chdir(%q): %v", configDir, err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
_ = os.Chdir(originalWD)
|
||||||
|
})
|
||||||
|
|
||||||
|
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", sessionID)
|
||||||
|
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "base.json"), `{"schema":"seriatim-intermediate","segments":[]}`)
|
||||||
|
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "[]\n")
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"run-stage", "polish", sessionID, "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--force"}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout.String(), "stage=polish executed=1 skipped=0") {
|
||||||
|
t.Fatalf("stdout = %q, want polish execution", stdout.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteRunFailsWhenConfiguredSecretsDirMissing(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
configDir := t.TempDir()
|
||||||
|
pipelinePath := filepath.Join(configDir, "pipeline.yml")
|
||||||
|
campaignPath := writeAppTestCampaignConfig(t, configDir)
|
||||||
|
sessionPath := filepath.Join(configDir, "session.yml")
|
||||||
|
|
||||||
|
pipelineYAML := `workspace:
|
||||||
|
root: ` + workspaceRoot + `
|
||||||
|
storage:
|
||||||
|
backend: local
|
||||||
|
secrets:
|
||||||
|
env_dir: ./missing-secrets
|
||||||
|
whisperx:
|
||||||
|
transcribe_url: https://example.com/transcribe
|
||||||
|
seriatim:
|
||||||
|
binary: seriatim
|
||||||
|
audita:
|
||||||
|
binary: audita
|
||||||
|
notification:
|
||||||
|
timeout: 10s
|
||||||
|
`
|
||||||
|
sessionYAML := `session_id: 2026-05-03
|
||||||
|
campaign: sample-campaign
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
players_file: ./players.yml
|
||||||
|
party_file: ./party.yml
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write pipeline.yml: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write session.yml: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"run", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "read secrets env_dir") {
|
||||||
|
t.Fatalf("stderr = %q, want secrets read-dir error context", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExecuteUsesDefaultPipelineConfigPathWhenConfigFlagOmitted(t *testing.T) {
|
func TestExecuteUsesDefaultPipelineConfigPathWhenConfigFlagOmitted(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
@@ -178,24 +321,109 @@ func TestExecuteUsesDefaultPipelineConfigPathWhenConfigFlagOmitted(t *testing.T)
|
|||||||
}))
|
}))
|
||||||
defer srv.Close()
|
defer srv.Close()
|
||||||
|
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
||||||
originalDefaults := append([]string(nil), config.DefaultPipelineConfigSearchPaths...)
|
originalDefaults := append([]string(nil), config.DefaultPipelineConfigSearchPaths...)
|
||||||
config.DefaultPipelineConfigSearchPaths = []string{pipelinePath}
|
config.DefaultPipelineConfigSearchPaths = []string{pipelinePath}
|
||||||
defer func() {
|
defer func() {
|
||||||
config.DefaultPipelineConfigSearchPaths = originalDefaults
|
config.DefaultPipelineConfigSearchPaths = originalDefaults
|
||||||
}()
|
}()
|
||||||
|
_ = campaignPath
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute([]string{"run", "--session", sessionPath}, &stdout, &stderr)
|
code := Execute([]string{"run", "2026-05-03", "--session", sessionPath}, &stdout, &stderr)
|
||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
if !strings.Contains(stdout.String(), "narratio run: session 2026-05-03; executed=9 skipped=0; manifest=") {
|
if !strings.Contains(stdout.String(), "narratio run: session 2026-05-03; executed=10 skipped=0; manifest=") {
|
||||||
t.Fatalf("stdout = %q, want successful run output", stdout.String())
|
t.Fatalf("stdout = %q, want successful run output", stdout.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExecuteMissingCampaignConfigReportsRegistryPath(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
if err := os.Remove(campaignPath); err != nil {
|
||||||
|
t.Fatalf("remove campaign config: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"run", "2026-05-03", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if stdout.Len() != 0 {
|
||||||
|
t.Fatalf("stdout = %q, want empty", stdout.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "load campaign config") {
|
||||||
|
t.Fatalf("stderr = %q, want campaign discovery failure", stderr.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), filepath.ToSlash(filepath.Join("campaigns", "sample-campaign", "campaign.yml"))) {
|
||||||
|
t.Fatalf("stderr = %q, want campaign registry path", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteUsesPipelineDefaultCampaignID(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, _, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"session", "status", "2026-05-03", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout.String(), "Campaign: sample-campaign") {
|
||||||
|
t.Fatalf("stdout = %q, want default campaign", stdout.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteCampaignIDSelectsRegistryCampaign(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
campaignRoot := filepath.Dir(filepath.Dir(campaignPath))
|
||||||
|
otherDir := filepath.Join(campaignRoot, "icewind")
|
||||||
|
mustWriteTestFile(t, filepath.Join(otherDir, "campaign.yml"), `campaign_id: icewind
|
||||||
|
inputs:
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
players_file: ./players.yml
|
||||||
|
party_file: ./party.yml
|
||||||
|
`)
|
||||||
|
mustWriteTestFile(t, filepath.Join(otherDir, "speakers.yml"), "match:\n - speaker: Alice\n match: [\"alice\"]\n")
|
||||||
|
mustWriteTestFile(t, filepath.Join(otherDir, "autocorrect.yml"), "[]\n")
|
||||||
|
mustWriteTestFile(t, filepath.Join(otherDir, "glossary.yml"), "[]\n")
|
||||||
|
mustWriteTestFile(t, filepath.Join(otherDir, "players.yml"), "[]\n")
|
||||||
|
mustWriteTestFile(t, filepath.Join(otherDir, "party.yml"), "[]\n")
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"session", "status", "2026-05-03", "--config", pipelinePath, "--campaign", "icewind", "--session", sessionPath}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout.String(), "Campaign: icewind") {
|
||||||
|
t.Fatalf("stdout = %q, want selected campaign", stdout.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteRejectsCampaignIDAndCampaignFile(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"session", "status", "2026-05-03", "--config", pipelinePath, "--campaign", "sample-campaign", "--campaign-file", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "mutually exclusive") {
|
||||||
|
t.Fatalf("stderr = %q, want mutually exclusive error", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExecuteInvalidCommand(t *testing.T) {
|
func TestExecuteInvalidCommand(t *testing.T) {
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
@@ -232,26 +460,44 @@ func TestExecuteMissingCommand(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeValidConfigFiles(t *testing.T, workspaceRoot string, transcribeURL ...string) (string, string) {
|
func writeValidConfigFiles(t *testing.T, workspaceRoot string, transcribeURL ...string) (string, string, string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||||
|
campaignRoot := filepath.Join(dir, "campaigns")
|
||||||
|
campaignDir := filepath.Join(campaignRoot, "sample-campaign")
|
||||||
|
campaignPath := filepath.Join(campaignDir, "campaign.yml")
|
||||||
sessionPath := filepath.Join(dir, "session.yml")
|
sessionPath := filepath.Join(dir, "session.yml")
|
||||||
url := "https://example.com/transcribe"
|
url := "https://example.com/transcribe"
|
||||||
if len(transcribeURL) > 0 && strings.TrimSpace(transcribeURL[0]) != "" {
|
if len(transcribeURL) > 0 && strings.TrimSpace(transcribeURL[0]) != "" {
|
||||||
url = transcribeURL[0]
|
url = transcribeURL[0]
|
||||||
}
|
}
|
||||||
seriatimBinary := writeSeriatimAppTestWrapper(t)
|
seriatimBinary := writeSeriatimAppTestWrapper(t)
|
||||||
|
scriptoriumBinary := writeScriptoriumAppTestWrapper(t)
|
||||||
auditaBinary := writeAuditaAppTestWrapper(t)
|
auditaBinary := writeAuditaAppTestWrapper(t)
|
||||||
t.Setenv("GO_WANT_APP_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_APP_SERIATIM_HELPER", "1")
|
||||||
|
t.Setenv("GO_WANT_APP_SCRIPTORIUM_HELPER", "1")
|
||||||
t.Setenv("GO_WANT_APP_AUDITA_HELPER", "1")
|
t.Setenv("GO_WANT_APP_AUDITA_HELPER", "1")
|
||||||
t.Setenv("AUDITA_LLM_API_KEY", "test-audita-key")
|
t.Setenv("AUDITA_LLM_API_KEY", "test-audita-key")
|
||||||
|
t.Setenv("PATH", filepath.Dir(scriptoriumBinary)+string(os.PathListSeparator)+os.Getenv("PATH"))
|
||||||
|
|
||||||
pipelineYAML := `workspace:
|
pipelineYAML := `workspace:
|
||||||
root: ` + workspaceRoot + `
|
root: ` + workspaceRoot + `
|
||||||
|
campaigns:
|
||||||
|
root: ` + campaignRoot + `
|
||||||
|
default_campaign_id: sample-campaign
|
||||||
|
cache:
|
||||||
|
root: ` + filepath.Join(workspaceRoot, "cache") + `
|
||||||
|
spool:
|
||||||
|
root: ` + filepath.Join(workspaceRoot, "spool") + `
|
||||||
storage:
|
storage:
|
||||||
backend: s3
|
backend: s3
|
||||||
|
s3:
|
||||||
|
bucket: test-bucket
|
||||||
|
publish:
|
||||||
|
enabled: true
|
||||||
|
upload_run: false
|
||||||
whisperx:
|
whisperx:
|
||||||
transcribe_url: ` + url + `
|
transcribe_url: ` + url + `
|
||||||
timeout: 2s
|
timeout: 2s
|
||||||
@@ -266,10 +512,6 @@ seriatim:
|
|||||||
report: true
|
report: true
|
||||||
audita:
|
audita:
|
||||||
binary: ` + auditaBinary + `
|
binary: ` + auditaBinary + `
|
||||||
analyzer:
|
|
||||||
timeout: 20m
|
|
||||||
artifacts:
|
|
||||||
output_dir: artifacts
|
|
||||||
notification:
|
notification:
|
||||||
timeout: 10s
|
timeout: 10s
|
||||||
`
|
`
|
||||||
@@ -277,24 +519,54 @@ notification:
|
|||||||
sessionYAML := `session_id: 2026-05-03
|
sessionYAML := `session_id: 2026-05-03
|
||||||
inputs:
|
inputs:
|
||||||
audio_dir: ./audio
|
audio_dir: ./audio
|
||||||
|
`
|
||||||
|
campaignYAML := `campaign_id: sample-campaign
|
||||||
|
inputs:
|
||||||
speakers_file: ./speakers.yml
|
speakers_file: ./speakers.yml
|
||||||
autocorrect_file: ./autocorrect.yml
|
autocorrect_file: ./autocorrect.yml
|
||||||
glossary_file: ./glossary.yml
|
glossary_file: ./glossary.yml
|
||||||
|
players_file: ./players.yml
|
||||||
|
party_file: ./party.yml
|
||||||
`
|
`
|
||||||
|
|
||||||
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||||
t.Fatalf("write pipeline config: %v", err)
|
t.Fatalf("write pipeline config: %v", err)
|
||||||
}
|
}
|
||||||
|
if err := os.MkdirAll(campaignDir, 0o755); err != nil {
|
||||||
|
t.Fatalf("create campaign dir: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(campaignPath, []byte(campaignYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write campaign config: %v", err)
|
||||||
|
}
|
||||||
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
|
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
|
||||||
t.Fatalf("write session config: %v", err)
|
t.Fatalf("write session config: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
mustWriteTestFile(t, filepath.Join(dir, "speakers.yml"), "match:\n - speaker: Alice\n match: [\"alice\"]\n")
|
mustWriteTestFile(t, filepath.Join(campaignDir, "speakers.yml"), "match:\n - speaker: Alice\n match: [\"alice\"]\n")
|
||||||
mustWriteTestFile(t, filepath.Join(dir, "autocorrect.yml"), "[]\n")
|
mustWriteTestFile(t, filepath.Join(campaignDir, "autocorrect.yml"), "[]\n")
|
||||||
mustWriteTestFile(t, filepath.Join(dir, "glossary.yml"), "[]\n")
|
mustWriteTestFile(t, filepath.Join(campaignDir, "glossary.yml"), "[]\n")
|
||||||
|
mustWriteTestFile(t, filepath.Join(campaignDir, "players.yml"), "[]\n")
|
||||||
|
mustWriteTestFile(t, filepath.Join(campaignDir, "party.yml"), "[]\n")
|
||||||
mustWriteTestFile(t, filepath.Join(dir, "audio", "alice.flac"), "audio-bytes")
|
mustWriteTestFile(t, filepath.Join(dir, "audio", "alice.flac"), "audio-bytes")
|
||||||
|
|
||||||
return pipelinePath, sessionPath
|
return pipelinePath, campaignPath, sessionPath
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeAppTestCampaignConfig(t *testing.T, dir string) string {
|
||||||
|
t.Helper()
|
||||||
|
campaignPath := filepath.Join(dir, "campaign.yml")
|
||||||
|
campaignYAML := `campaign_id: sample-campaign
|
||||||
|
inputs:
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
players_file: ./players.yml
|
||||||
|
party_file: ./party.yml
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(campaignPath, []byte(campaignYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write campaign.yml: %v", err)
|
||||||
|
}
|
||||||
|
return campaignPath
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeManifestPathForExecute(t *testing.T) string {
|
func writeManifestPathForExecute(t *testing.T) string {
|
||||||
@@ -336,6 +608,60 @@ func writeSeriatimAppTestWrapper(t *testing.T) string {
|
|||||||
return path
|
return path
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func writeScriptoriumAppTestWrapper(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
exe, err := os.Executable()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("os.Executable() error = %v", err)
|
||||||
|
}
|
||||||
|
path := filepath.Join(t.TempDir(), "scriptorium")
|
||||||
|
content := "#!/bin/sh\nexec \"" + exe + "\" -test.run=TestScriptoriumAppHelper -- \"$@\"\n"
|
||||||
|
if err := os.WriteFile(path, []byte(content), 0o755); err != nil {
|
||||||
|
t.Fatalf("WriteFile(%q): %v", path, err)
|
||||||
|
}
|
||||||
|
return path
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestScriptoriumAppHelper(t *testing.T) {
|
||||||
|
if os.Getenv("GO_WANT_APP_SCRIPTORIUM_HELPER") != "1" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
args := os.Args
|
||||||
|
start := -1
|
||||||
|
for i := range args {
|
||||||
|
if args[i] == "--" {
|
||||||
|
start = i + 1
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if start < 0 || start >= len(args) {
|
||||||
|
_, _ = os.Stderr.WriteString("missing -- args separator\n")
|
||||||
|
os.Exit(2)
|
||||||
|
}
|
||||||
|
runArgs := args[start:]
|
||||||
|
|
||||||
|
outputPath := appSeriatimFlagValue(runArgs, "--out")
|
||||||
|
if strings.TrimSpace(outputPath) == "" {
|
||||||
|
outputPath = appSeriatimFlagValue(runArgs, "--output")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(outputPath) == "" {
|
||||||
|
_, _ = os.Stderr.WriteString("missing output flag\n")
|
||||||
|
os.Exit(2)
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Dir(outputPath), 0o755); err != nil {
|
||||||
|
_, _ = os.Stderr.WriteString(fmt.Sprintf("mkdir output dir: %v\n", err))
|
||||||
|
os.Exit(2)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(outputPath, []byte(`{"trim_action":"copy","warnings":[]}`), 0o644); err != nil {
|
||||||
|
_, _ = os.Stderr.WriteString(fmt.Sprintf("write output: %v\n", err))
|
||||||
|
os.Exit(2)
|
||||||
|
}
|
||||||
|
_, _ = os.Stdout.WriteString("scriptorium helper stdout\n")
|
||||||
|
_, _ = os.Stderr.WriteString("scriptorium helper stderr\n")
|
||||||
|
os.Exit(0)
|
||||||
|
}
|
||||||
|
|
||||||
func TestSeriatimAppHelper(t *testing.T) {
|
func TestSeriatimAppHelper(t *testing.T) {
|
||||||
if os.Getenv("GO_WANT_APP_SERIATIM_HELPER") != "1" {
|
if os.Getenv("GO_WANT_APP_SERIATIM_HELPER") != "1" {
|
||||||
return
|
return
|
||||||
|
|||||||
141
internal/app/config_loader.go
Normal file
141
internal/app/config_loader.go
Normal file
@@ -0,0 +1,141 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
type pipelineCampaignConfig struct {
|
||||||
|
PipelinePath string
|
||||||
|
CampaignPath string
|
||||||
|
Pipeline *config.PipelineConfig
|
||||||
|
Campaign *config.CampaignConfig
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadCommandConfig(ctx context.Context, pipelineFlag, campaignFlag, campaignFileFlag, sessionFlag string, sessionOpts config.SessionLoadOptions) (*config.Config, error) {
|
||||||
|
base, err := loadPipelineCampaignConfig(pipelineFlag, campaignFlag, campaignFileFlag)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if explicitSession := strings.TrimSpace(sessionFlag); explicitSession != "" {
|
||||||
|
return config.LoadWithSessionOptions(base.PipelinePath, base.CampaignPath, explicitSession, sessionOpts)
|
||||||
|
}
|
||||||
|
|
||||||
|
discoveredSession, err := discoverSessionConfigPathWithCandidates(config.DefaultSessionConfigSearchPaths)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if discoveredSession.Path != "" {
|
||||||
|
return config.LoadWithSessionOptions(base.PipelinePath, base.CampaignPath, discoveredSession.Path, sessionOpts)
|
||||||
|
}
|
||||||
|
|
||||||
|
sessionID := strings.TrimSpace(sessionOpts.SessionID)
|
||||||
|
if sessionID == "" {
|
||||||
|
return nil, missingSessionConfigError(discoveredSession.Searched, "remote session loading requires a session_id")
|
||||||
|
}
|
||||||
|
|
||||||
|
sessionPrefix := artifacts.S3SessionPrefix(base.Pipeline.Storage.S3.RootPrefix, config.CampaignID(base.Campaign), sessionID)
|
||||||
|
remoteKey := artifacts.S3SessionConfigKey(sessionPrefix)
|
||||||
|
partialCfg := &config.Config{
|
||||||
|
Pipeline: base.Pipeline,
|
||||||
|
Campaign: base.Campaign,
|
||||||
|
PipelinePath: base.PipelinePath,
|
||||||
|
CampaignPath: base.CampaignPath,
|
||||||
|
}
|
||||||
|
store, err := newCommandObjectStore(ctx, partialCfg, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, missingSessionConfigError(discoveredSession.Searched, fmt.Sprintf("remote session %q unavailable: %v", remoteKey, err))
|
||||||
|
}
|
||||||
|
|
||||||
|
sessionInfo, err := findRemoteSessionConfig(ctx, store, sessionPrefix, remoteKey)
|
||||||
|
if err != nil {
|
||||||
|
return nil, missingSessionConfigError(discoveredSession.Searched, err.Error())
|
||||||
|
}
|
||||||
|
sessionTempPath, err := storage.DownloadObjectToTemp(ctx, store, remoteKey, "narratio-session-*.yml")
|
||||||
|
if err != nil {
|
||||||
|
return nil, missingSessionConfigError(discoveredSession.Searched, fmt.Sprintf("remote session %q download failed: %v", remoteKey, err))
|
||||||
|
}
|
||||||
|
sessionBytes, err := os.ReadFile(sessionTempPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("read downloaded remote session %q: %w", sessionTempPath, err)
|
||||||
|
}
|
||||||
|
sessionCfg, err := config.LoadSessionBytesWithOptions("s3://"+s3BucketName(base.Pipeline)+"/"+remoteKey, sessionBytes, sessionOpts)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return config.Resolve(
|
||||||
|
base.PipelinePath,
|
||||||
|
base.Pipeline,
|
||||||
|
base.CampaignPath,
|
||||||
|
base.Campaign,
|
||||||
|
sessionTempPath,
|
||||||
|
sessionCfg,
|
||||||
|
config.SessionSource{
|
||||||
|
Source: "session_config.s3",
|
||||||
|
LocalPath: sessionTempPath,
|
||||||
|
S3Bucket: s3BucketName(base.Pipeline),
|
||||||
|
S3Key: remoteKey,
|
||||||
|
S3Size: sessionInfo.Size,
|
||||||
|
S3ETag: sessionInfo.ETag,
|
||||||
|
SpoolPath: sessionTempPath,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadPipelineCampaignConfig(pipelineFlag, campaignFlag, campaignFileFlag string) (*pipelineCampaignConfig, error) {
|
||||||
|
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelineFlag)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
pipelineCfg, err := config.LoadPipeline(resolvedPipelinePath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
resolvedCampaignPath, err := resolveCampaignConfigPath(pipelineCfg, campaignFlag, campaignFileFlag)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
campaignCfg, err := config.LoadCampaign(resolvedCampaignPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if selectedID := strings.TrimSpace(campaignFlag); selectedID != "" && strings.TrimSpace(campaignFileFlag) == "" {
|
||||||
|
if got := config.CampaignID(campaignCfg); got != selectedID {
|
||||||
|
return nil, fmt.Errorf("campaign config %q invalid: campaign_id %q does not match selected campaign %q", resolvedCampaignPath, got, selectedID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return &pipelineCampaignConfig{
|
||||||
|
PipelinePath: resolvedPipelinePath,
|
||||||
|
CampaignPath: resolvedCampaignPath,
|
||||||
|
Pipeline: pipelineCfg,
|
||||||
|
Campaign: campaignCfg,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func findRemoteSessionConfig(ctx context.Context, store storage.ObjectStore, sessionPrefix, remoteKey string) (storage.ObjectInfo, error) {
|
||||||
|
objects, err := store.List(ctx, sessionPrefix)
|
||||||
|
if err != nil {
|
||||||
|
return storage.ObjectInfo{}, fmt.Errorf("remote session %q list failed: %w", remoteKey, err)
|
||||||
|
}
|
||||||
|
for _, obj := range objects {
|
||||||
|
if obj.Key == remoteKey {
|
||||||
|
return obj, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return storage.ObjectInfo{}, fmt.Errorf("remote session %q not found", remoteKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
func s3BucketName(cfg *config.PipelineConfig) string {
|
||||||
|
if cfg == nil || cfg.Storage.S3 == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(cfg.Storage.S3.Bucket)
|
||||||
|
}
|
||||||
21
internal/app/object_store.go
Normal file
21
internal/app/object_store.go
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newCommandObjectStore(ctx context.Context, cfg *config.Config, logger *slog.Logger) (storage.ObjectStore, error) {
|
||||||
|
if _, err := loadSecretsFromConfig(cfg, logger); err != nil {
|
||||||
|
return nil, fmt.Errorf("load secrets from files: %w", err)
|
||||||
|
}
|
||||||
|
store, err := newObjectStoreFromConfigFn(ctx, cfg)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("initialize object store backend: %w", err)
|
||||||
|
}
|
||||||
|
return store, nil
|
||||||
|
}
|
||||||
165
internal/app/object_store_test.go
Normal file
165
internal/app/object_store_test.go
Normal file
@@ -0,0 +1,165 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNewCommandObjectStoreLoadsSecretsBeforeFactory(t *testing.T) {
|
||||||
|
accessKeyEnv := "NARRATIO_TEST_COMMAND_STORE_KEY_ID"
|
||||||
|
secretKeyEnv := "NARRATIO_TEST_COMMAND_STORE_SECRET"
|
||||||
|
restoreEnvAfterTest(t, accessKeyEnv, secretKeyEnv)
|
||||||
|
|
||||||
|
secretsDir := t.TempDir()
|
||||||
|
mustWriteSecretFile(t, filepath.Join(secretsDir, accessKeyEnv), "loaded-key-id\n")
|
||||||
|
mustWriteSecretFile(t, filepath.Join(secretsDir, secretKeyEnv), "loaded-secret\n")
|
||||||
|
|
||||||
|
cfg := commandObjectStoreTestConfig(secretsDir)
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
called := false
|
||||||
|
origStoreFn := newObjectStoreFromConfigFn
|
||||||
|
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||||
|
called = true
|
||||||
|
if got := os.Getenv(accessKeyEnv); got != "loaded-key-id" {
|
||||||
|
return nil, errors.New("access key was not loaded before object store init")
|
||||||
|
}
|
||||||
|
if got := os.Getenv(secretKeyEnv); got != "loaded-secret" {
|
||||||
|
return nil, errors.New("secret key was not loaded before object store init")
|
||||||
|
}
|
||||||
|
return fake, nil
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
newObjectStoreFromConfigFn = origStoreFn
|
||||||
|
})
|
||||||
|
|
||||||
|
store, err := newCommandObjectStore(context.Background(), cfg, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("newCommandObjectStore() error = %v", err)
|
||||||
|
}
|
||||||
|
if store != fake {
|
||||||
|
t.Fatalf("store = %#v, want fake backend", store)
|
||||||
|
}
|
||||||
|
if !called {
|
||||||
|
t.Fatal("object store factory was not called")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewCommandObjectStorePreservesExistingEnv(t *testing.T) {
|
||||||
|
accessKeyEnv := "NARRATIO_TEST_COMMAND_STORE_EXISTING_KEY_ID"
|
||||||
|
secretKeyEnv := "NARRATIO_TEST_COMMAND_STORE_EXISTING_SECRET"
|
||||||
|
t.Setenv(accessKeyEnv, "existing-key-id")
|
||||||
|
t.Setenv(secretKeyEnv, "existing-secret")
|
||||||
|
|
||||||
|
secretsDir := t.TempDir()
|
||||||
|
mustWriteSecretFile(t, filepath.Join(secretsDir, accessKeyEnv), "file-key-id\n")
|
||||||
|
mustWriteSecretFile(t, filepath.Join(secretsDir, secretKeyEnv), "file-secret\n")
|
||||||
|
|
||||||
|
cfg := commandObjectStoreTestConfig(secretsDir)
|
||||||
|
origStoreFn := newObjectStoreFromConfigFn
|
||||||
|
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||||
|
if got := os.Getenv(accessKeyEnv); got != "existing-key-id" {
|
||||||
|
return nil, errors.New("existing access key was overwritten")
|
||||||
|
}
|
||||||
|
if got := os.Getenv(secretKeyEnv); got != "existing-secret" {
|
||||||
|
return nil, errors.New("existing secret key was overwritten")
|
||||||
|
}
|
||||||
|
return &storage.FakeBackend{}, nil
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
newObjectStoreFromConfigFn = origStoreFn
|
||||||
|
})
|
||||||
|
|
||||||
|
if _, err := newCommandObjectStore(context.Background(), cfg, nil); err != nil {
|
||||||
|
t.Fatalf("newCommandObjectStore() error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewCommandObjectStoreSecretErrorStopsFactory(t *testing.T) {
|
||||||
|
cfg := commandObjectStoreTestConfig(filepath.Join(t.TempDir(), "missing"))
|
||||||
|
called := false
|
||||||
|
origStoreFn := newObjectStoreFromConfigFn
|
||||||
|
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||||
|
called = true
|
||||||
|
return &storage.FakeBackend{}, nil
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
newObjectStoreFromConfigFn = origStoreFn
|
||||||
|
})
|
||||||
|
|
||||||
|
_, err := newCommandObjectStore(context.Background(), cfg, nil)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error, got nil")
|
||||||
|
}
|
||||||
|
if called {
|
||||||
|
t.Fatal("object store factory was called after secret load failure")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "load secrets from files") {
|
||||||
|
t.Fatalf("error = %q, want secret loading context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewCommandObjectStoreFactoryErrorIsContextual(t *testing.T) {
|
||||||
|
cfg := commandObjectStoreTestConfig("")
|
||||||
|
origStoreFn := newObjectStoreFromConfigFn
|
||||||
|
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||||
|
return nil, errors.New("factory boom")
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
newObjectStoreFromConfigFn = origStoreFn
|
||||||
|
})
|
||||||
|
|
||||||
|
_, err := newCommandObjectStore(context.Background(), cfg, nil)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "initialize object store backend") || !strings.Contains(err.Error(), "factory boom") {
|
||||||
|
t.Fatalf("error = %q, want factory context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func commandObjectStoreTestConfig(secretsDir string) *config.Config {
|
||||||
|
cfg := &config.Config{
|
||||||
|
Pipeline: &config.PipelineConfig{
|
||||||
|
Storage: config.StorageConfig{
|
||||||
|
Backend: "s3",
|
||||||
|
S3: &config.StorageS3Config{
|
||||||
|
Bucket: "test-bucket",
|
||||||
|
AccessKeyIDEnv: "NARRATIO_TEST_COMMAND_STORE_KEY_ID",
|
||||||
|
SecretKeyEnv: "NARRATIO_TEST_COMMAND_STORE_SECRET",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(secretsDir) != "" {
|
||||||
|
cfg.Pipeline.Secrets = &config.SecretsConfig{EnvDir: secretsDir}
|
||||||
|
}
|
||||||
|
return cfg
|
||||||
|
}
|
||||||
|
|
||||||
|
func restoreEnvAfterTest(t *testing.T, names ...string) {
|
||||||
|
t.Helper()
|
||||||
|
originals := make(map[string]string, len(names))
|
||||||
|
present := make(map[string]bool, len(names))
|
||||||
|
for _, name := range names {
|
||||||
|
value, ok := os.LookupEnv(name)
|
||||||
|
originals[name] = value
|
||||||
|
present[name] = ok
|
||||||
|
_ = os.Unsetenv(name)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
for _, name := range names {
|
||||||
|
if present[name] {
|
||||||
|
_ = os.Setenv(name, originals[name])
|
||||||
|
} else {
|
||||||
|
_ = os.Unsetenv(name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
131
internal/app/operator_artifact_rendering.go
Normal file
131
internal/app/operator_artifact_rendering.go
Normal file
@@ -0,0 +1,131 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func buildHelperArtifactCatalog(cfg *config.Config) (*artifacts.ArtifactCatalog, error) {
|
||||||
|
catalog := artifacts.NewArtifactCatalog()
|
||||||
|
if err := catalog.RegisterBuiltIns(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
configured := map[string]artifacts.ConfiguredArtifactDefinition{}
|
||||||
|
if cfg.Pipeline.Scriptorium != nil {
|
||||||
|
for key, item := range cfg.Pipeline.Scriptorium.Artifacts {
|
||||||
|
configured[key] = artifacts.ConfiguredArtifactDefinition{Enabled: item.Enabled, OutputPath: item.OutputPath}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := catalog.RegisterConfiguredArtifacts(configured, nil); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return catalog, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeArtifactList(out io.Writer, cfg *config.Config, catalog *artifacts.ArtifactCatalog, locks *effectiveLocks, publishedRemoteState map[string]string) {
|
||||||
|
lockSet := lockSourceSet(locks.All)
|
||||||
|
fmt.Fprintln(out, "Built-in:")
|
||||||
|
for _, transcript := range artifacts.RuntimeTranscriptArtifacts() {
|
||||||
|
writeArtifactLine(out, transcript.SourceID, lockSet)
|
||||||
|
}
|
||||||
|
writeArtifactLine(out, artifacts.ArtifactBoundsSession, lockSet)
|
||||||
|
fmt.Fprintln(out, "Configured:")
|
||||||
|
for _, entry := range catalog.ListConfigured() {
|
||||||
|
writeArtifactLine(out, entry.SourceID, lockSet)
|
||||||
|
}
|
||||||
|
fmt.Fprintln(out, "Previous-session:")
|
||||||
|
for _, req := range artifacts.CollectPreviousArtifactRequirements(configuredScriptoriumArtifacts(cfg)) {
|
||||||
|
fmt.Fprintf(out, "- %s required=%t\n", artifactpolicy.PreviousSessionSourceID(req.Name), req.Required)
|
||||||
|
}
|
||||||
|
fmt.Fprintln(out, "Published:")
|
||||||
|
for _, rule := range cfg.Pipeline.Publish.Outputs {
|
||||||
|
writePublishedOutputLine(out, rule, catalog, lockSet, publishedRemoteState)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeArtifactLine(out io.Writer, source string, lockSet map[string]config.PublishLockRule) {
|
||||||
|
parts := []string{source}
|
||||||
|
if _, ok := lockSet[source]; ok {
|
||||||
|
parts = append(parts, "locked")
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "- %s\n", strings.Join(parts, " "))
|
||||||
|
}
|
||||||
|
|
||||||
|
func writePublishedOutputLine(out io.Writer, rule config.PublishOutputRule, catalog *artifacts.ArtifactCatalog, lockSet map[string]config.PublishLockRule, remoteState map[string]string) {
|
||||||
|
source := strings.TrimSpace(rule.Source)
|
||||||
|
parts := []string{source}
|
||||||
|
if _, ok := lockSet[source]; ok {
|
||||||
|
parts = append(parts, "locked")
|
||||||
|
}
|
||||||
|
dest, showDest, err := helperPublishedOutputDest(rule, catalog)
|
||||||
|
if err != nil {
|
||||||
|
parts = append(parts, "remote=error")
|
||||||
|
fmt.Fprintf(out, "- %s\n", strings.Join(parts, " "))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if showDest {
|
||||||
|
parts = append(parts, "dest="+dest)
|
||||||
|
}
|
||||||
|
if state := remoteState[publishedOutputRemoteStateKey(source, dest)]; state != "" {
|
||||||
|
parts = append(parts, state)
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "- %s\n", strings.Join(parts, " "))
|
||||||
|
}
|
||||||
|
|
||||||
|
func remotePublishedOutputAvailability(ctx context.Context, cfg *config.Config, store storage.ObjectStore, catalog *artifacts.ArtifactCatalog) map[string]string {
|
||||||
|
out := map[string]string{}
|
||||||
|
sessionPrefix := artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
for _, rule := range cfg.Pipeline.Publish.Outputs {
|
||||||
|
source := strings.TrimSpace(rule.Source)
|
||||||
|
dest, _, err := helperPublishedOutputDest(rule, catalog)
|
||||||
|
if err != nil {
|
||||||
|
out[publishedOutputRemoteStateKey(source, "")] = "remote=error"
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
key := artifacts.S3PublishedOutputKey(sessionPrefix, dest)
|
||||||
|
if exists, err := store.Exists(ctx, key); err == nil && exists {
|
||||||
|
out[publishedOutputRemoteStateKey(source, dest)] = "remote=published"
|
||||||
|
} else if err != nil {
|
||||||
|
out[publishedOutputRemoteStateKey(source, dest)] = "remote=error"
|
||||||
|
} else {
|
||||||
|
out[publishedOutputRemoteStateKey(source, dest)] = "remote=missing"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func helperPublishedOutputDest(rule config.PublishOutputRule, catalog *artifacts.ArtifactCatalog) (string, bool, error) {
|
||||||
|
source := strings.TrimSpace(rule.Source)
|
||||||
|
normalized, err := artifactpolicy.ResolvePublishedDestination(source, rule.Dest, helperConfiguredOutputPathMap(catalog))
|
||||||
|
if err != nil {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
entry, ok := catalog.Lookup(source)
|
||||||
|
showDest := !ok || strings.TrimSpace(entry.CanonicalRelPath) != normalized
|
||||||
|
return normalized, showDest, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func helperConfiguredOutputPathMap(catalog *artifacts.ArtifactCatalog) map[string]string {
|
||||||
|
out := map[string]string{}
|
||||||
|
if catalog == nil {
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
for _, entry := range catalog.ListConfigured() {
|
||||||
|
if strings.TrimSpace(entry.ConfiguredKey) == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out[entry.ConfiguredKey] = strings.TrimSpace(entry.CanonicalRelPath)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func publishedOutputRemoteStateKey(source, dest string) string {
|
||||||
|
return strings.TrimSpace(source) + "\x00" + strings.TrimSpace(dest)
|
||||||
|
}
|
||||||
39
internal/app/operator_artifacts_list.go
Normal file
39
internal/app/operator_artifacts_list.go
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ArtifactsList lists effective artifact sources.
|
||||||
|
func ArtifactsList(ctx context.Context, args []string, out io.Writer) error {
|
||||||
|
fs := flag.NewFlagSet("artifacts list", flag.ContinueOnError)
|
||||||
|
fs.SetOutput(io.Discard)
|
||||||
|
var flags commonConfigFlags
|
||||||
|
var remote bool
|
||||||
|
addCommonConfigFlags(fs, &flags)
|
||||||
|
fs.BoolVar(&remote, "remote", false, "inspect remote publish availability")
|
||||||
|
if err := parseSessionAwareFlags("artifacts list", fs, args, &flags.sessionID); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(flags.sessionID) == "" {
|
||||||
|
return fmt.Errorf("artifacts list: session_id is required")
|
||||||
|
}
|
||||||
|
cfg, store, locks, _, err := loadHelperContext(ctx, flags, remote)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("artifacts list: %w", err)
|
||||||
|
}
|
||||||
|
catalog, err := buildHelperArtifactCatalog(cfg)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("artifacts list: %w", err)
|
||||||
|
}
|
||||||
|
publishedRemoteState := map[string]string{}
|
||||||
|
if remote && store != nil {
|
||||||
|
publishedRemoteState = remotePublishedOutputAvailability(ctx, cfg, store, catalog)
|
||||||
|
}
|
||||||
|
writeArtifactList(out, cfg, catalog, locks, publishedRemoteState)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
140
internal/app/operator_findings.go
Normal file
140
internal/app/operator_findings.go
Normal file
@@ -0,0 +1,140 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
|
)
|
||||||
|
|
||||||
|
type finding struct {
|
||||||
|
Severity string
|
||||||
|
Category string
|
||||||
|
Message string
|
||||||
|
}
|
||||||
|
|
||||||
|
type findingError struct {
|
||||||
|
count int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e findingError) Error() string {
|
||||||
|
return fmt.Sprintf("%d validation error(s)", e.count)
|
||||||
|
}
|
||||||
|
|
||||||
|
func renderFindings(out io.Writer, campaign, sessionID string, findings []finding) error {
|
||||||
|
if campaign != "" || sessionID != "" {
|
||||||
|
fmt.Fprintf(out, "Campaign: %s\n", campaign)
|
||||||
|
fmt.Fprintf(out, "Session: %s\n\n", sessionID)
|
||||||
|
}
|
||||||
|
errorsCount := 0
|
||||||
|
for _, f := range findings {
|
||||||
|
if f.Severity == "ERROR" {
|
||||||
|
errorsCount++
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "%-5s %-10s %s\n", f.Severity, f.Category, f.Message)
|
||||||
|
}
|
||||||
|
if errorsCount > 0 {
|
||||||
|
return findingError{count: errorsCount}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func okFinding(category, msg string) finding { return finding{"OK", category, msg} }
|
||||||
|
func infoFinding(category, msg string) finding { return finding{"INFO", category, msg} }
|
||||||
|
func warnFinding(category, msg string) finding { return finding{"WARN", category, msg} }
|
||||||
|
func errorFinding(category, msg string) finding { return finding{"ERROR", category, msg} }
|
||||||
|
|
||||||
|
func sessionSourceSummary(cfg *config.Config) string {
|
||||||
|
source := cfg.SessionSource.Source
|
||||||
|
if source == "" {
|
||||||
|
source = "session_config"
|
||||||
|
}
|
||||||
|
if cfg.SessionSource.S3Key != "" {
|
||||||
|
return source + " " + cfg.SessionSource.S3Key
|
||||||
|
}
|
||||||
|
return source + " " + cfg.SessionPath
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateStableInputFindings(cfg *config.Config) []finding {
|
||||||
|
checks := inspectStableInputs(cfg)
|
||||||
|
out := make([]finding, 0, len(checks))
|
||||||
|
for _, check := range checks {
|
||||||
|
if check.Err != nil {
|
||||||
|
msg := check.Name + ": " + check.Err.Error()
|
||||||
|
if strings.TrimSpace(check.Path) != "" {
|
||||||
|
msg = fmt.Sprintf("%s missing: %v", check.Name, check.Err)
|
||||||
|
}
|
||||||
|
out = append(out, errorFinding("inputs", msg))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, okFinding("inputs", check.Name+": "+check.Path))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveHelperConfigRelativePath(input config.ResolvedInputFile) (string, error) {
|
||||||
|
if strings.TrimSpace(input.ConfigPath) == "" {
|
||||||
|
return "", fmt.Errorf("source config path is required")
|
||||||
|
}
|
||||||
|
path := strings.TrimSpace(input.Path)
|
||||||
|
if path == "" {
|
||||||
|
return "", fmt.Errorf("path is required")
|
||||||
|
}
|
||||||
|
if filepath.IsAbs(path) {
|
||||||
|
return filepath.Clean(path), nil
|
||||||
|
}
|
||||||
|
return filepath.Clean(filepath.Join(filepath.Dir(input.ConfigPath), path)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateLocalAudioFindings(cfg *config.Config) []finding {
|
||||||
|
check := inspectLocalAudioPresence(cfg)
|
||||||
|
if !check.Checked {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if check.Err != nil {
|
||||||
|
return []finding{errorFinding("audio", check.Err.Error())}
|
||||||
|
}
|
||||||
|
return []finding{okFinding("audio", fmt.Sprintf("%d local audio file(s)", len(check.Paths)))}
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateRemoteAudioFinding(ctx context.Context, cfg *config.Config, store storage.ObjectStore) finding {
|
||||||
|
check := inspectRemoteAudioPresence(ctx, cfg, store)
|
||||||
|
if check.Err != nil {
|
||||||
|
return errorFinding("audio", check.Err.Error())
|
||||||
|
}
|
||||||
|
return okFinding("audio", fmt.Sprintf("%d remote .flac object(s)", len(check.Keys)))
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadLocalManifest(ctx context.Context, path string) (*manifest.Manifest, error) {
|
||||||
|
if _, err := os.Stat(path); err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
store := &manifest.LocalStore{}
|
||||||
|
return store.Load(ctx, path)
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeStageStatuses(out io.Writer, m *manifest.Manifest) {
|
||||||
|
if m == nil || len(m.Stages) == 0 {
|
||||||
|
fmt.Fprintln(out, "stages: no stages recorded")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fmt.Fprintln(out, "stages:")
|
||||||
|
names := make([]string, 0, len(m.Stages))
|
||||||
|
for name := range m.Stages {
|
||||||
|
names = append(names, name)
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
for _, name := range names {
|
||||||
|
fmt.Fprintf(out, "- %s: %s\n", name, m.Stages[name].Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
131
internal/app/operator_helpers.go
Normal file
131
internal/app/operator_helpers.go
Normal file
@@ -0,0 +1,131 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
|
)
|
||||||
|
|
||||||
|
type commonConfigFlags struct {
|
||||||
|
pipelinePath string
|
||||||
|
campaignPath string
|
||||||
|
campaignFilePath string
|
||||||
|
sessionPath string
|
||||||
|
sessionID string
|
||||||
|
previousSessionID string
|
||||||
|
}
|
||||||
|
|
||||||
|
func addCommonConfigFlags(fs *flag.FlagSet, flags *commonConfigFlags) {
|
||||||
|
fs.StringVar(&flags.pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
||||||
|
fs.StringVar(&flags.campaignPath, "campaign", "", "campaign ID")
|
||||||
|
fs.StringVar(&flags.campaignFilePath, "campaign-file", "", "path to campaign.yml")
|
||||||
|
fs.StringVar(&flags.sessionPath, "session", "", "path to session.yml")
|
||||||
|
fs.StringVar(&flags.sessionID, "session-id", "", "session identifier")
|
||||||
|
fs.StringVar(&flags.previousSessionID, "previous-session-id", "", "expected previous session identifier")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f commonConfigFlags) sessionOptions() config.SessionLoadOptions {
|
||||||
|
return config.SessionLoadOptions{
|
||||||
|
SessionID: f.sessionID,
|
||||||
|
PreviousSessionID: f.previousSessionID,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Session dispatches session helper subcommands.
|
||||||
|
func Session(ctx context.Context, args []string, out io.Writer) error {
|
||||||
|
if len(args) == 0 {
|
||||||
|
return fmt.Errorf("session: expected subcommand: init|validate|status|plan|restore|artifacts|locks")
|
||||||
|
}
|
||||||
|
switch args[0] {
|
||||||
|
case "init":
|
||||||
|
return SessionInit(ctx, args[1:], out)
|
||||||
|
case "validate":
|
||||||
|
return SessionValidate(ctx, args[1:], out)
|
||||||
|
case "status":
|
||||||
|
return Status(ctx, args[1:], out)
|
||||||
|
case "plan":
|
||||||
|
return Plan(ctx, args[1:], out)
|
||||||
|
case "restore":
|
||||||
|
return Restore(ctx, args[1:], out)
|
||||||
|
case "artifacts":
|
||||||
|
return ArtifactsList(ctx, args[1:], out)
|
||||||
|
case "locks":
|
||||||
|
return SessionLocks(ctx, args[1:], out)
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("session: unknown subcommand %q", args[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SessionLocks dispatches session-oriented publish lock list and mutation
|
||||||
|
// helpers while preserving the existing lock implementations.
|
||||||
|
func SessionLocks(ctx context.Context, args []string, out io.Writer) error {
|
||||||
|
if len(args) > 0 && !isCLIFlagToken(args[0]) {
|
||||||
|
switch args[0] {
|
||||||
|
case "add":
|
||||||
|
return LocksAdd(ctx, args[1:], out)
|
||||||
|
case "remove":
|
||||||
|
return LocksRemove(ctx, args[1:], out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return LocksList(ctx, args, out)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Artifacts dispatches artifact helper subcommands.
|
||||||
|
func Artifacts(ctx context.Context, args []string, out io.Writer) error {
|
||||||
|
if len(args) == 0 {
|
||||||
|
return fmt.Errorf("artifacts: expected subcommand: list")
|
||||||
|
}
|
||||||
|
switch args[0] {
|
||||||
|
case "list":
|
||||||
|
return ArtifactsList(ctx, args[1:], out)
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("artifacts: unknown subcommand %q", args[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadHelperContext(ctx context.Context, flags commonConfigFlags, needStore bool) (*config.Config, storage.ObjectStore, *effectiveLocks, *manifest.Manifest, error) {
|
||||||
|
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, nil, nil, err
|
||||||
|
}
|
||||||
|
if err := config.Validate(cfg); err != nil {
|
||||||
|
return nil, nil, nil, nil, err
|
||||||
|
}
|
||||||
|
var store storage.ObjectStore
|
||||||
|
if needStore {
|
||||||
|
store, err = newCommandObjectStore(ctx, cfg, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, nil, nil, err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
store, _ = objectStoreIfConfigured(ctx, cfg)
|
||||||
|
}
|
||||||
|
locks, err := loadEffectiveLocks(ctx, cfg, store)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, nil, nil, err
|
||||||
|
}
|
||||||
|
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
m, err := loadLocalManifest(ctx, paths.ManifestPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, nil, nil, err
|
||||||
|
}
|
||||||
|
return cfg, store, locks, m, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func objectStoreIfConfigured(ctx context.Context, cfg *config.Config) (storage.ObjectStore, error) {
|
||||||
|
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Storage.S3 == nil || strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket) == "" {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
store, err := newCommandObjectStore(ctx, cfg, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return store, nil
|
||||||
|
}
|
||||||
1114
internal/app/operator_helpers_test.go
Normal file
1114
internal/app/operator_helpers_test.go
Normal file
File diff suppressed because it is too large
Load Diff
276
internal/app/operator_inspection.go
Normal file
276
internal/app/operator_inspection.go
Normal file
@@ -0,0 +1,276 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
type stableInputCheck struct {
|
||||||
|
Name string
|
||||||
|
Path string
|
||||||
|
Err error
|
||||||
|
}
|
||||||
|
|
||||||
|
type localAudioCheck struct {
|
||||||
|
Checked bool
|
||||||
|
Paths []string
|
||||||
|
Err error
|
||||||
|
}
|
||||||
|
|
||||||
|
type remoteAudioCheck struct {
|
||||||
|
Checked bool
|
||||||
|
Prefix string
|
||||||
|
Keys []string
|
||||||
|
Err error
|
||||||
|
}
|
||||||
|
|
||||||
|
type previousArtifactReadiness struct {
|
||||||
|
Requirements []artifacts.PreviousArtifactRequirement
|
||||||
|
MissingID bool
|
||||||
|
Err error
|
||||||
|
}
|
||||||
|
|
||||||
|
type remoteCurrentStateCheck struct {
|
||||||
|
State *RemoteCurrentState
|
||||||
|
Err error
|
||||||
|
}
|
||||||
|
|
||||||
|
type effectiveLocksCheck struct {
|
||||||
|
Locks *effectiveLocks
|
||||||
|
Err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func inspectStableInputs(cfg *config.Config) []stableInputCheck {
|
||||||
|
items := []struct {
|
||||||
|
name string
|
||||||
|
in config.ResolvedInputFile
|
||||||
|
}{
|
||||||
|
{name: "speakers", in: cfg.StableInputs.SpeakersFile},
|
||||||
|
{name: "autocorrect", in: cfg.StableInputs.AutocorrectFile},
|
||||||
|
{name: "glossary", in: cfg.StableInputs.GlossaryFile},
|
||||||
|
{name: "players", in: cfg.StableInputs.PlayersFile},
|
||||||
|
{name: "party", in: cfg.StableInputs.PartyFile},
|
||||||
|
}
|
||||||
|
out := make([]stableInputCheck, 0, len(items))
|
||||||
|
for _, item := range items {
|
||||||
|
path, err := resolveHelperConfigRelativePath(item.in)
|
||||||
|
if err != nil {
|
||||||
|
out = append(out, stableInputCheck{Name: item.name, Err: err})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(path); err != nil {
|
||||||
|
out = append(out, stableInputCheck{Name: item.name, Path: path, Err: err})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, stableInputCheck{Name: item.name, Path: path})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func inspectLocalAudioPresence(cfg *config.Config) localAudioCheck {
|
||||||
|
if cfg.Session.Inputs.AudioS3 != nil {
|
||||||
|
return localAudioCheck{}
|
||||||
|
}
|
||||||
|
|
||||||
|
sessionDir := filepath.Dir(cfg.SessionPath)
|
||||||
|
resolved, err := resolveLocalInspectionAudioPaths(sessionDir, cfg.Session.Inputs)
|
||||||
|
if err != nil {
|
||||||
|
return localAudioCheck{Checked: true, Err: err}
|
||||||
|
}
|
||||||
|
return localAudioCheck{
|
||||||
|
Checked: true,
|
||||||
|
Paths: resolved,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func inspectRemoteAudioPresence(ctx context.Context, cfg *config.Config, store storage.ObjectStore) remoteAudioCheck {
|
||||||
|
if cfg.Session.Inputs.AudioS3 == nil {
|
||||||
|
return remoteAudioCheck{}
|
||||||
|
}
|
||||||
|
if store == nil {
|
||||||
|
return remoteAudioCheck{Checked: true, Err: fmt.Errorf("storage backend is required for remote audio checks")}
|
||||||
|
}
|
||||||
|
|
||||||
|
sessionPrefix := artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
audioPrefix := artifacts.S3AudioPrefix(sessionPrefix, cfg.Session.Inputs.AudioS3.Prefix)
|
||||||
|
objects, err := store.List(ctx, audioPrefix)
|
||||||
|
if err != nil {
|
||||||
|
return remoteAudioCheck{Checked: true, Prefix: audioPrefix, Err: err}
|
||||||
|
}
|
||||||
|
|
||||||
|
keys := make([]string, 0, len(objects))
|
||||||
|
seenBase := map[string]string{}
|
||||||
|
for _, obj := range objects {
|
||||||
|
key := strings.TrimSpace(obj.Key)
|
||||||
|
if key == "" || strings.HasSuffix(key, "/") || !isInspectionFlacPath(key) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
base := path.Base(key)
|
||||||
|
if prev, exists := seenBase[base]; exists && prev != key {
|
||||||
|
return remoteAudioCheck{
|
||||||
|
Checked: true,
|
||||||
|
Prefix: audioPrefix,
|
||||||
|
Err: fmt.Errorf("duplicate s3 audio basename %q from %q and %q", base, prev, key),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
seenBase[base] = key
|
||||||
|
keys = append(keys, key)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
if len(keys) == 0 {
|
||||||
|
return remoteAudioCheck{
|
||||||
|
Checked: true,
|
||||||
|
Prefix: audioPrefix,
|
||||||
|
Err: fmt.Errorf("no .flac files found under s3 audio prefix %q", audioPrefix),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return remoteAudioCheck{
|
||||||
|
Checked: true,
|
||||||
|
Prefix: audioPrefix,
|
||||||
|
Keys: keys,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func inspectPreviousArtifactReadiness(
|
||||||
|
ctx context.Context,
|
||||||
|
cfg *config.Config,
|
||||||
|
store storage.ObjectStore,
|
||||||
|
requirements []artifacts.PreviousArtifactRequirement,
|
||||||
|
) previousArtifactReadiness {
|
||||||
|
out := previousArtifactReadiness{
|
||||||
|
Requirements: append([]artifacts.PreviousArtifactRequirement(nil), requirements...),
|
||||||
|
}
|
||||||
|
if len(requirements) == 0 {
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(cfg.Session.PreviousSessionID) == "" {
|
||||||
|
out.MissingID = true
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
if store == nil {
|
||||||
|
out.Err = fmt.Errorf("previous-session artifacts cannot be checked because storage is unavailable")
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
prefix := artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, cfg.Session.Campaign, cfg.Session.PreviousSessionID)
|
||||||
|
if _, err := artifacts.LoadCurrentState(ctx, store, prefix, artifacts.CurrentStateValidation{
|
||||||
|
ExpectedSessionID: strings.TrimSpace(cfg.Session.PreviousSessionID),
|
||||||
|
ExpectedCampaign: strings.TrimSpace(cfg.Session.Campaign),
|
||||||
|
ValidateRunID: true,
|
||||||
|
}); err != nil {
|
||||||
|
out.Err = fmt.Errorf("remote %v", err)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func inspectRemoteCurrentState(ctx context.Context, cfg *config.Config, store storage.ObjectStore) remoteCurrentStateCheck {
|
||||||
|
if store == nil {
|
||||||
|
return remoteCurrentStateCheck{}
|
||||||
|
}
|
||||||
|
current, err := discoverRemoteCurrentStateFn(ctx, cfg, store)
|
||||||
|
if err != nil {
|
||||||
|
return remoteCurrentStateCheck{Err: err}
|
||||||
|
}
|
||||||
|
return remoteCurrentStateCheck{State: current}
|
||||||
|
}
|
||||||
|
|
||||||
|
func inspectEffectiveLocks(ctx context.Context, cfg *config.Config, store storage.ObjectStore) effectiveLocksCheck {
|
||||||
|
locks, err := loadEffectiveLocks(ctx, cfg, store)
|
||||||
|
if err != nil {
|
||||||
|
return effectiveLocksCheck{Err: err}
|
||||||
|
}
|
||||||
|
return effectiveLocksCheck{Locks: locks}
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveLocalInspectionAudioPaths(sessionDir string, inputs config.SessionInputsConfig) ([]string, error) {
|
||||||
|
if len(inputs.AudioFiles) > 0 {
|
||||||
|
out := make([]string, 0, len(inputs.AudioFiles))
|
||||||
|
seenBase := map[string]string{}
|
||||||
|
for _, item := range inputs.AudioFiles {
|
||||||
|
resolved, err := resolveInspectionPath(sessionDir, item)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if !isInspectionFlacPath(resolved) {
|
||||||
|
return nil, fmt.Errorf("audio file %q must have .flac extension", resolved)
|
||||||
|
}
|
||||||
|
if err := requireInspectionFile(resolved, "audio file"); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
base := filepath.Base(resolved)
|
||||||
|
if prev, exists := seenBase[base]; exists && prev != resolved {
|
||||||
|
return nil, fmt.Errorf("duplicate audio basename %q from %q and %q", base, prev, resolved)
|
||||||
|
}
|
||||||
|
seenBase[base] = resolved
|
||||||
|
out = append(out, resolved)
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
audioDir, err := resolveInspectionPath(sessionDir, inputs.AudioDir)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
entries, err := os.ReadDir(audioDir)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("read audio directory %q: %w", audioDir, err)
|
||||||
|
}
|
||||||
|
out := make([]string, 0, len(entries))
|
||||||
|
for _, entry := range entries {
|
||||||
|
if entry.IsDir() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
full := filepath.Join(audioDir, entry.Name())
|
||||||
|
if !isInspectionFlacPath(full) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := requireInspectionFile(full, "audio file"); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out = append(out, full)
|
||||||
|
}
|
||||||
|
if len(out) == 0 {
|
||||||
|
return nil, fmt.Errorf("no .flac files found in audio directory %q", audioDir)
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveInspectionPath(baseDir, inputPath string) (string, error) {
|
||||||
|
pathValue := strings.TrimSpace(inputPath)
|
||||||
|
if pathValue == "" {
|
||||||
|
return "", fmt.Errorf("path is required")
|
||||||
|
}
|
||||||
|
if filepath.IsAbs(pathValue) {
|
||||||
|
return filepath.Clean(pathValue), nil
|
||||||
|
}
|
||||||
|
return filepath.Clean(filepath.Join(baseDir, pathValue)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func requireInspectionFile(path, label string) error {
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
if err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return fmt.Errorf("%s %q does not exist", label, path)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("stat %s %q: %w", label, path, err)
|
||||||
|
}
|
||||||
|
if info.IsDir() {
|
||||||
|
return fmt.Errorf("%s %q is a directory", label, path)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func isInspectionFlacPath(path string) bool {
|
||||||
|
return strings.EqualFold(filepath.Ext(strings.TrimSpace(path)), ".flac")
|
||||||
|
}
|
||||||
172
internal/app/operator_locks.go
Normal file
172
internal/app/operator_locks.go
Normal file
@@ -0,0 +1,172 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Locks dispatches publish lock list and mutation helpers.
|
||||||
|
func Locks(ctx context.Context, args []string, out io.Writer) error {
|
||||||
|
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||||
|
switch args[0] {
|
||||||
|
case "add":
|
||||||
|
return LocksAdd(ctx, args[1:], out)
|
||||||
|
case "remove":
|
||||||
|
return LocksRemove(ctx, args[1:], out)
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("locks: unknown subcommand %q", args[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return LocksList(ctx, args, out)
|
||||||
|
}
|
||||||
|
|
||||||
|
// LocksList lists effective publish locks.
|
||||||
|
func LocksList(ctx context.Context, args []string, out io.Writer) error {
|
||||||
|
fs := flag.NewFlagSet("locks", flag.ContinueOnError)
|
||||||
|
fs.SetOutput(io.Discard)
|
||||||
|
var flags commonConfigFlags
|
||||||
|
addCommonConfigFlags(fs, &flags)
|
||||||
|
if err := parseSessionAwareFlags("locks", fs, args, &flags.sessionID); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(flags.sessionID) == "" {
|
||||||
|
return fmt.Errorf("locks: session_id is required")
|
||||||
|
}
|
||||||
|
cfg, _, locks, _, err := loadHelperContext(ctx, flags, true)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("locks: %w", err)
|
||||||
|
}
|
||||||
|
writeLocks(out, cfg, locks)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// LocksAdd adds or updates one remote lock.
|
||||||
|
func LocksAdd(ctx context.Context, args []string, out io.Writer) error {
|
||||||
|
fs := flag.NewFlagSet("locks add", flag.ContinueOnError)
|
||||||
|
fs.SetOutput(io.Discard)
|
||||||
|
var flags commonConfigFlags
|
||||||
|
var reason string
|
||||||
|
var force bool
|
||||||
|
addCommonConfigFlags(fs, &flags)
|
||||||
|
fs.StringVar(&reason, "reason", "", "lock reason")
|
||||||
|
fs.BoolVar(&force, "force", false, "update existing remote lock")
|
||||||
|
source, err := parseSessionIDAndOnePositionalArg("locks add", "source id", fs, args, &flags.sessionID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(flags.sessionID) == "" {
|
||||||
|
return fmt.Errorf("locks add: session_id is required")
|
||||||
|
}
|
||||||
|
cfg, store, locks, _, err := loadHelperContext(ctx, flags, true)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("locks add: %w", err)
|
||||||
|
}
|
||||||
|
if _, err := config.ValidatePublishLockRules([]config.PublishLockRule{{Source: source}}, cfg.Pipeline.Scriptorium, "locks add"); err != nil {
|
||||||
|
return fmt.Errorf("locks add: %w", err)
|
||||||
|
}
|
||||||
|
if _, ok := lockSourceSet(locks.Static)[source]; ok {
|
||||||
|
return fmt.Errorf("locks add: source %q is locked by pipeline config and cannot be modified remotely", source)
|
||||||
|
}
|
||||||
|
remoteSet := lockSourceSet(locks.Remote)
|
||||||
|
if _, exists := remoteSet[source]; exists && !force {
|
||||||
|
return fmt.Errorf("locks add: remote lock for %q already exists; pass --force to update", source)
|
||||||
|
}
|
||||||
|
remoteSet[source] = config.PublishLockRule{Source: source, Reason: strings.TrimSpace(reason)}
|
||||||
|
remoteLocks := lockMapValues(remoteSet)
|
||||||
|
if _, err := config.ValidatePublishLockRules(remoteLocks, cfg.Pipeline.Scriptorium, "locks"); err != nil {
|
||||||
|
return fmt.Errorf("locks add: %w", err)
|
||||||
|
}
|
||||||
|
if err := uploadRemoteLockStore(ctx, store, locks.Key, &config.PublishLockStore{Locks: remoteLocks}); err != nil {
|
||||||
|
return fmt.Errorf("locks add: %w", err)
|
||||||
|
}
|
||||||
|
_, err = fmt.Fprintf(out, "narratio session locks add: locked %s\n", source)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// LocksRemove removes one remote lock.
|
||||||
|
func LocksRemove(ctx context.Context, args []string, out io.Writer) error {
|
||||||
|
fs := flag.NewFlagSet("locks remove", flag.ContinueOnError)
|
||||||
|
fs.SetOutput(io.Discard)
|
||||||
|
var flags commonConfigFlags
|
||||||
|
addCommonConfigFlags(fs, &flags)
|
||||||
|
source, err := parseSessionIDAndOnePositionalArg("locks remove", "source id", fs, args, &flags.sessionID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(flags.sessionID) == "" {
|
||||||
|
return fmt.Errorf("locks remove: session_id is required")
|
||||||
|
}
|
||||||
|
cfg, store, locks, _, err := loadHelperContext(ctx, flags, true)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("locks remove: %w", err)
|
||||||
|
}
|
||||||
|
if _, err := config.ValidatePublishLockRules([]config.PublishLockRule{{Source: source}}, cfg.Pipeline.Scriptorium, "locks remove"); err != nil {
|
||||||
|
return fmt.Errorf("locks remove: %w", err)
|
||||||
|
}
|
||||||
|
remoteSet := lockSourceSet(locks.Remote)
|
||||||
|
if _, ok := remoteSet[source]; !ok {
|
||||||
|
if _, static := lockSourceSet(locks.Static)[source]; static {
|
||||||
|
return fmt.Errorf("locks remove: source %q is locked by pipeline config and cannot be unlocked remotely", source)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("locks remove: remote lock for %q does not exist", source)
|
||||||
|
}
|
||||||
|
delete(remoteSet, source)
|
||||||
|
remoteLocks := lockMapValues(remoteSet)
|
||||||
|
if err := uploadRemoteLockStore(ctx, store, locks.Key, &config.PublishLockStore{Locks: remoteLocks}); err != nil {
|
||||||
|
return fmt.Errorf("locks remove: %w", err)
|
||||||
|
}
|
||||||
|
_, err = fmt.Fprintf(out, "narratio session locks remove: unlocked %s\n", source)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeLocks(out io.Writer, cfg *config.Config, locks *effectiveLocks) {
|
||||||
|
if locks == nil || len(locks.All) == 0 {
|
||||||
|
fmt.Fprintln(out, "Publish locks: none")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fmt.Fprintln(out, "Publish locks:")
|
||||||
|
published := map[string]config.PublishOutputRule{}
|
||||||
|
if cfg != nil && cfg.Pipeline != nil && cfg.Pipeline.Publish != nil {
|
||||||
|
for _, rule := range cfg.Pipeline.Publish.Outputs {
|
||||||
|
published[strings.TrimSpace(rule.Source)] = rule
|
||||||
|
}
|
||||||
|
}
|
||||||
|
staticSet := lockSourceSet(locks.Static)
|
||||||
|
for _, lock := range locks.All {
|
||||||
|
origin := "remote"
|
||||||
|
if _, ok := staticSet[lock.Source]; ok {
|
||||||
|
origin = "pipeline"
|
||||||
|
}
|
||||||
|
promo := "not-published"
|
||||||
|
if _, ok := published[lock.Source]; ok {
|
||||||
|
promo = "published"
|
||||||
|
}
|
||||||
|
reason := strings.TrimSpace(lock.Reason)
|
||||||
|
if reason == "" {
|
||||||
|
reason = "(no reason)"
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "- %s origin=%s %s reason=%s\n", lock.Source, origin, promo, reason)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func lockMapValues(in map[string]config.PublishLockRule) []config.PublishLockRule {
|
||||||
|
keys := make([]string, 0, len(in))
|
||||||
|
for key := range in {
|
||||||
|
keys = append(keys, key)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
out := make([]config.PublishLockRule, 0, len(keys))
|
||||||
|
for _, key := range keys {
|
||||||
|
item := in[key]
|
||||||
|
item.Source = key
|
||||||
|
item.Reason = strings.TrimSpace(item.Reason)
|
||||||
|
out = append(out, item)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
268
internal/app/operator_session_init.go
Normal file
268
internal/app/operator_session_init.go
Normal file
@@ -0,0 +1,268 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
"gopkg.in/yaml.v3"
|
||||||
|
)
|
||||||
|
|
||||||
|
// SessionInit creates a local or remote session.yml skeleton.
|
||||||
|
func SessionInit(ctx context.Context, args []string, out io.Writer) error {
|
||||||
|
fs := flag.NewFlagSet("session init", flag.ContinueOnError)
|
||||||
|
fs.SetOutput(io.Discard)
|
||||||
|
var pipelinePath, campaignPath, campaignFilePath, sessionID, previousSessionID, date, title, output, audioS3Prefix, audioDir string
|
||||||
|
var remote, force bool
|
||||||
|
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
||||||
|
fs.StringVar(&campaignPath, "campaign", "", "campaign ID")
|
||||||
|
fs.StringVar(&campaignFilePath, "campaign-file", "", "path to campaign.yml")
|
||||||
|
fs.StringVar(&sessionID, "session-id", "", "session identifier")
|
||||||
|
fs.StringVar(&previousSessionID, "previous-session-id", "", "previous session identifier")
|
||||||
|
fs.StringVar(&date, "date", "", "session date")
|
||||||
|
fs.StringVar(&title, "title", "", "session title")
|
||||||
|
fs.StringVar(&output, "output", "", "local output session.yml path")
|
||||||
|
fs.StringVar(&audioS3Prefix, "audio-s3-prefix", "", "session audio S3 prefix")
|
||||||
|
fs.StringVar(&audioDir, "audio-dir", "", "local audio directory")
|
||||||
|
fs.BoolVar(&remote, "remote", false, "write session.yml to S3 session prefix")
|
||||||
|
fs.BoolVar(&force, "force", false, "overwrite existing target")
|
||||||
|
if err := parseSessionAwareFlags("session init", fs, args, &sessionID); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(sessionID) == "" {
|
||||||
|
return fmt.Errorf("session init: session_id is required")
|
||||||
|
}
|
||||||
|
if (strings.TrimSpace(output) == "") == !remote {
|
||||||
|
return fmt.Errorf("session init: specify exactly one target: --output <path> or --remote")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(audioDir) != "" && strings.TrimSpace(audioS3Prefix) != "" {
|
||||||
|
return fmt.Errorf("session init: --audio-dir and --audio-s3-prefix are mutually exclusive")
|
||||||
|
}
|
||||||
|
|
||||||
|
base, err := loadPipelineCampaignConfig(pipelinePath, campaignPath, campaignFilePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("session init: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
input := sessionInitInput{
|
||||||
|
Campaign: config.CampaignID(base.Campaign),
|
||||||
|
CampaignPath: base.CampaignPath,
|
||||||
|
TemplateFile: base.Campaign.SessionTemplateFile,
|
||||||
|
SessionID: sessionID,
|
||||||
|
PreviousSessionID: previousSessionID,
|
||||||
|
Date: date,
|
||||||
|
Title: title,
|
||||||
|
AudioS3Prefix: audioS3Prefix,
|
||||||
|
AudioDir: audioDir,
|
||||||
|
}
|
||||||
|
data, err := buildSessionInitYAML(input)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("session init: %w", err)
|
||||||
|
}
|
||||||
|
label := strings.TrimSpace(output)
|
||||||
|
if label == "" {
|
||||||
|
label = "remote session.yml"
|
||||||
|
}
|
||||||
|
sessionCfg, err := config.LoadSessionBytesWithOptions(label, data, config.SessionLoadOptions{
|
||||||
|
SessionID: sessionID,
|
||||||
|
PreviousSessionID: previousSessionID,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("session init: %w", err)
|
||||||
|
}
|
||||||
|
cfg, err := config.Resolve(base.PipelinePath, base.Pipeline, base.CampaignPath, base.Campaign, label, sessionCfg, config.SessionSource{Source: "session_config", LocalPath: label})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("session init: %w", err)
|
||||||
|
}
|
||||||
|
if err := config.Validate(cfg); err != nil {
|
||||||
|
return fmt.Errorf("session init: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !remote {
|
||||||
|
if err := writeLocalFile(output, data, force); err != nil {
|
||||||
|
return fmt.Errorf("session init: %w", err)
|
||||||
|
}
|
||||||
|
_, err := fmt.Fprintf(out, "narratio session init: wrote %s\n", filepath.Clean(output))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
store, err := newCommandObjectStore(ctx, cfg, nil)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("session init: %w", err)
|
||||||
|
}
|
||||||
|
sessionPrefix := artifacts.S3SessionPrefix(base.Pipeline.Storage.S3.RootPrefix, config.CampaignID(base.Campaign), sessionID)
|
||||||
|
key := artifacts.S3SessionConfigKey(sessionPrefix)
|
||||||
|
exists, err := store.Exists(ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("session init: check remote session %q: %w", key, err)
|
||||||
|
}
|
||||||
|
if exists && !force {
|
||||||
|
return fmt.Errorf("session init: remote session %q already exists; pass --force to overwrite", key)
|
||||||
|
}
|
||||||
|
tmp, err := os.CreateTemp("", "narratio-session-init-*.yml")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("session init: create temp file: %w", err)
|
||||||
|
}
|
||||||
|
tmpPath := tmp.Name()
|
||||||
|
defer func() { _ = os.Remove(tmpPath) }()
|
||||||
|
if _, err := tmp.Write(data); err != nil {
|
||||||
|
_ = tmp.Close()
|
||||||
|
return fmt.Errorf("session init: write temp file: %w", err)
|
||||||
|
}
|
||||||
|
if err := tmp.Close(); err != nil {
|
||||||
|
return fmt.Errorf("session init: close temp file: %w", err)
|
||||||
|
}
|
||||||
|
if _, err := store.Upload(ctx, tmpPath, key, storage.UploadOptions{ContentType: "application/x-yaml; charset=utf-8"}); err != nil {
|
||||||
|
return fmt.Errorf("session init: upload remote session %q: %w", key, err)
|
||||||
|
}
|
||||||
|
_, err = fmt.Fprintf(out, "narratio session init: wrote s3://%s/%s\n", s3BucketName(base.Pipeline), key)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildSessionYAML(campaign, sessionID, previousSessionID, date, title, audioS3Prefix, audioDir string) ([]byte, error) {
|
||||||
|
if strings.TrimSpace(date) == "" && regexp.MustCompile(`^\d{4}-\d{2}-\d{2}$`).MatchString(strings.TrimSpace(sessionID)) {
|
||||||
|
date = strings.TrimSpace(sessionID)
|
||||||
|
}
|
||||||
|
type audioS3 struct {
|
||||||
|
Prefix string `yaml:"prefix"`
|
||||||
|
}
|
||||||
|
type inputs struct {
|
||||||
|
AudioDir string `yaml:"audio_dir,omitempty"`
|
||||||
|
AudioS3 *audioS3 `yaml:"audio_s3,omitempty"`
|
||||||
|
}
|
||||||
|
type sessionYAML struct {
|
||||||
|
Campaign string `yaml:"campaign"`
|
||||||
|
SessionID string `yaml:"session_id"`
|
||||||
|
PreviousSessionID string `yaml:"previous_session_id,omitempty"`
|
||||||
|
Date string `yaml:"date,omitempty"`
|
||||||
|
Title string `yaml:"title,omitempty"`
|
||||||
|
Inputs inputs `yaml:"inputs"`
|
||||||
|
}
|
||||||
|
in := inputs{AudioDir: strings.TrimSpace(audioDir)}
|
||||||
|
if in.AudioDir == "" {
|
||||||
|
prefix := strings.TrimSpace(audioS3Prefix)
|
||||||
|
if prefix == "" {
|
||||||
|
prefix = "audio/"
|
||||||
|
}
|
||||||
|
in.AudioS3 = &audioS3{Prefix: prefix}
|
||||||
|
}
|
||||||
|
data, err := yaml.Marshal(sessionYAML{
|
||||||
|
Campaign: strings.TrimSpace(campaign),
|
||||||
|
SessionID: strings.TrimSpace(sessionID),
|
||||||
|
PreviousSessionID: strings.TrimSpace(previousSessionID),
|
||||||
|
Date: strings.TrimSpace(date),
|
||||||
|
Title: strings.TrimSpace(title),
|
||||||
|
Inputs: in,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return data, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type sessionInitInput struct {
|
||||||
|
Campaign string
|
||||||
|
CampaignPath string
|
||||||
|
TemplateFile string
|
||||||
|
SessionID string
|
||||||
|
PreviousSessionID string
|
||||||
|
Date string
|
||||||
|
Title string
|
||||||
|
AudioS3Prefix string
|
||||||
|
AudioDir string
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildSessionInitYAML(in sessionInitInput) ([]byte, error) {
|
||||||
|
if strings.TrimSpace(in.TemplateFile) == "" {
|
||||||
|
return buildSessionYAML(in.Campaign, in.SessionID, in.PreviousSessionID, in.Date, in.Title, in.AudioS3Prefix, in.AudioDir)
|
||||||
|
}
|
||||||
|
templatePath := resolveSessionInitTemplatePath(in.CampaignPath, in.TemplateFile)
|
||||||
|
templateBytes, err := os.ReadFile(templatePath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("read session template %q: %w", templatePath, err)
|
||||||
|
}
|
||||||
|
rendered, err := renderSessionInitTemplate(string(templateBytes), in)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("render session template %q: %w", templatePath, err)
|
||||||
|
}
|
||||||
|
return []byte(rendered), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveSessionInitTemplatePath(campaignPath, templateFile string) string {
|
||||||
|
templateFile = strings.TrimSpace(templateFile)
|
||||||
|
if filepath.IsAbs(templateFile) {
|
||||||
|
return filepath.Clean(templateFile)
|
||||||
|
}
|
||||||
|
return filepath.Clean(filepath.Join(filepath.Dir(campaignPath), templateFile))
|
||||||
|
}
|
||||||
|
|
||||||
|
var sessionInitTemplatePattern = regexp.MustCompile(`\{\{\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\}\}`)
|
||||||
|
|
||||||
|
func renderSessionInitTemplate(content string, in sessionInitInput) (string, error) {
|
||||||
|
values := map[string]string{
|
||||||
|
"session_id": strings.TrimSpace(in.SessionID),
|
||||||
|
"previous_session_id": strings.TrimSpace(in.PreviousSessionID),
|
||||||
|
"date": strings.TrimSpace(in.Date),
|
||||||
|
"title": strings.TrimSpace(in.Title),
|
||||||
|
"audio_s3_prefix": strings.TrimSpace(in.AudioS3Prefix),
|
||||||
|
"audio_dir": strings.TrimSpace(in.AudioDir),
|
||||||
|
}
|
||||||
|
used := map[string]struct{}{}
|
||||||
|
unknown := map[string]struct{}{}
|
||||||
|
missing := map[string]struct{}{}
|
||||||
|
rendered := sessionInitTemplatePattern.ReplaceAllStringFunc(content, func(match string) string {
|
||||||
|
parts := sessionInitTemplatePattern.FindStringSubmatch(match)
|
||||||
|
if len(parts) < 2 {
|
||||||
|
return match
|
||||||
|
}
|
||||||
|
name := parts[1]
|
||||||
|
value, ok := values[name]
|
||||||
|
if !ok {
|
||||||
|
unknown[name] = struct{}{}
|
||||||
|
return match
|
||||||
|
}
|
||||||
|
used[name] = struct{}{}
|
||||||
|
if value == "" {
|
||||||
|
missing[name] = struct{}{}
|
||||||
|
return match
|
||||||
|
}
|
||||||
|
return value
|
||||||
|
})
|
||||||
|
if len(unknown) > 0 {
|
||||||
|
return "", fmt.Errorf("unsupported template variable(s): %s", sortedStringSet(unknown))
|
||||||
|
}
|
||||||
|
if len(missing) > 0 {
|
||||||
|
return "", fmt.Errorf("missing required template variable value(s): %s", sortedStringSet(missing))
|
||||||
|
}
|
||||||
|
unused := map[string]struct{}{}
|
||||||
|
for _, name := range []string{"previous_session_id", "date", "title", "audio_s3_prefix", "audio_dir"} {
|
||||||
|
if values[name] == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, ok := used[name]; !ok {
|
||||||
|
unused[name] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(unused) > 0 {
|
||||||
|
return "", fmt.Errorf("unused template variable value(s): %s", sortedStringSet(unused))
|
||||||
|
}
|
||||||
|
return rendered, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func sortedStringSet(set map[string]struct{}) string {
|
||||||
|
items := make([]string, 0, len(set))
|
||||||
|
for item := range set {
|
||||||
|
items = append(items, item)
|
||||||
|
}
|
||||||
|
sort.Strings(items)
|
||||||
|
return strings.Join(items, ", ")
|
||||||
|
}
|
||||||
84
internal/app/operator_session_validate.go
Normal file
84
internal/app/operator_session_validate.go
Normal file
@@ -0,0 +1,84 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
// SessionValidate performs a read-only session preflight.
|
||||||
|
func SessionValidate(ctx context.Context, args []string, out io.Writer) error {
|
||||||
|
fs := flag.NewFlagSet("session validate", flag.ContinueOnError)
|
||||||
|
fs.SetOutput(io.Discard)
|
||||||
|
var flags commonConfigFlags
|
||||||
|
addCommonConfigFlags(fs, &flags)
|
||||||
|
if err := parseSessionAwareFlags("session validate", fs, args, &flags.sessionID); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(flags.sessionID) == "" {
|
||||||
|
return fmt.Errorf("session validate: session_id is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
findings := []finding{}
|
||||||
|
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
||||||
|
if err != nil {
|
||||||
|
findings = append(findings, errorFinding("config", err.Error()))
|
||||||
|
return renderFindings(out, "", "", findings)
|
||||||
|
}
|
||||||
|
if err := config.Validate(cfg); err != nil {
|
||||||
|
findings = append(findings, errorFinding("config", err.Error()))
|
||||||
|
} else {
|
||||||
|
findings = append(findings, okFinding("config", "resolved pipeline, campaign, and session config"))
|
||||||
|
}
|
||||||
|
findings = append(findings, okFinding("session", fmt.Sprintf("session source: %s", sessionSourceSummary(cfg))))
|
||||||
|
|
||||||
|
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
findings = append(findings, validateStableInputFindings(cfg)...)
|
||||||
|
findings = append(findings, validateLocalAudioFindings(cfg)...)
|
||||||
|
|
||||||
|
store, storeErr := objectStoreIfConfigured(ctx, cfg)
|
||||||
|
if storeErr != nil {
|
||||||
|
findings = append(findings, errorFinding("storage", storeErr.Error()))
|
||||||
|
}
|
||||||
|
if cfg.Session.Inputs.AudioS3 != nil {
|
||||||
|
if storeErr != nil {
|
||||||
|
findings = append(findings, errorFinding("audio", "remote audio cannot be checked because storage is unavailable"))
|
||||||
|
} else {
|
||||||
|
findings = append(findings, validateRemoteAudioFinding(ctx, cfg, store))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
requirements := artifacts.CollectPreviousArtifactRequirements(configuredScriptoriumArtifacts(cfg))
|
||||||
|
previous := inspectPreviousArtifactReadiness(ctx, cfg, store, requirements)
|
||||||
|
if len(previous.Requirements) == 0 {
|
||||||
|
findings = append(findings, okFinding("previous", "no previous-session artifacts required"))
|
||||||
|
} else if previous.MissingID {
|
||||||
|
findings = append(findings, errorFinding("previous", "previous_session_id is required by configured previous-session artifacts"))
|
||||||
|
} else if previous.Err != nil {
|
||||||
|
findings = append(findings, errorFinding("previous", previous.Err.Error()))
|
||||||
|
} else {
|
||||||
|
for _, req := range previous.Requirements {
|
||||||
|
findings = append(findings, okFinding("previous", fmt.Sprintf("%s required=%t", req.Name, req.Required)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
locks := inspectEffectiveLocks(ctx, cfg, store)
|
||||||
|
if locks.Err != nil {
|
||||||
|
findings = append(findings, errorFinding("locks", locks.Err.Error()))
|
||||||
|
} else if len(locks.Locks.All) == 0 {
|
||||||
|
findings = append(findings, okFinding("locks", "no effective publish locks"))
|
||||||
|
} else {
|
||||||
|
for _, lock := range locks.Locks.All {
|
||||||
|
findings = append(findings, warnFinding("locks", fmt.Sprintf("%s locked: %s", lock.Source, strings.TrimSpace(lock.Reason))))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if paths.ManifestPath != "" {
|
||||||
|
findings = append(findings, infoFinding("workspace", "manifest path: "+paths.ManifestPath))
|
||||||
|
}
|
||||||
|
return renderFindings(out, cfg.Session.Campaign, cfg.Session.SessionID, findings)
|
||||||
|
}
|
||||||
166
internal/app/operator_status.go
Normal file
166
internal/app/operator_status.go
Normal file
@@ -0,0 +1,166 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Status reports effective local/remote session state.
|
||||||
|
func Status(ctx context.Context, args []string, out io.Writer) error {
|
||||||
|
fs := flag.NewFlagSet("status", flag.ContinueOnError)
|
||||||
|
fs.SetOutput(io.Discard)
|
||||||
|
var flags commonConfigFlags
|
||||||
|
addCommonConfigFlags(fs, &flags)
|
||||||
|
if err := parseSessionAwareFlags("status", fs, args, &flags.sessionID); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(flags.sessionID) == "" {
|
||||||
|
return fmt.Errorf("status: session_id is required")
|
||||||
|
}
|
||||||
|
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("status: %w", err)
|
||||||
|
}
|
||||||
|
if err := config.Validate(cfg); err != nil {
|
||||||
|
return fmt.Errorf("status: %w", err)
|
||||||
|
}
|
||||||
|
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
fmt.Fprintf(out, "Session: %s\n", cfg.Session.SessionID)
|
||||||
|
fmt.Fprintf(out, "Campaign: %s\n", cfg.Session.Campaign)
|
||||||
|
fmt.Fprintf(out, "Workspace: %s\n", paths.Root)
|
||||||
|
fmt.Fprintf(out, "Session config: %s\n", sessionSourceSummary(cfg))
|
||||||
|
writeStatusStableInputs(out, inspectStableInputs(cfg))
|
||||||
|
writeStatusLocalAudio(out, inspectLocalAudioPresence(cfg))
|
||||||
|
|
||||||
|
if m, err := loadLocalManifest(ctx, paths.ManifestPath); err != nil {
|
||||||
|
fmt.Fprintf(out, "Local manifest: error: %v\n", err)
|
||||||
|
} else if m == nil {
|
||||||
|
fmt.Fprintln(out, "Local manifest: missing")
|
||||||
|
} else {
|
||||||
|
fmt.Fprintf(out, "Local manifest: %s\n", paths.ManifestPath)
|
||||||
|
writeStageStatuses(out, m)
|
||||||
|
}
|
||||||
|
|
||||||
|
store, storeErr := objectStoreIfConfigured(ctx, cfg)
|
||||||
|
if storeErr != nil {
|
||||||
|
fmt.Fprintf(out, "Remote publish: unavailable: %v\n", storeErr)
|
||||||
|
} else if store != nil {
|
||||||
|
current := inspectRemoteCurrentState(ctx, cfg, store)
|
||||||
|
if current.Err != nil {
|
||||||
|
fmt.Fprintf(out, "Remote publish: missing or unavailable: %v\n", current.Err)
|
||||||
|
} else {
|
||||||
|
fmt.Fprintf(out, "Remote publish: current run %s\n", current.State.RunID)
|
||||||
|
fmt.Fprintf(out, "Remote manifest: %s\n", current.State.CurrentManifestKey)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
writeStatusRemoteAudio(ctx, out, cfg, store, storeErr)
|
||||||
|
writeStatusPreviousArtifacts(out, inspectPreviousArtifactReadiness(
|
||||||
|
ctx,
|
||||||
|
cfg,
|
||||||
|
store,
|
||||||
|
artifacts.CollectPreviousArtifactRequirements(configuredScriptoriumArtifacts(cfg)),
|
||||||
|
))
|
||||||
|
|
||||||
|
lockChecks := inspectEffectiveLocks(ctx, cfg, store)
|
||||||
|
locks := lockChecks.Locks
|
||||||
|
lockErr := lockChecks.Err
|
||||||
|
if catalog, catalogErr := buildHelperArtifactCatalog(cfg); catalogErr != nil {
|
||||||
|
fmt.Fprintf(out, "Remote outputs: error: %v\n", catalogErr)
|
||||||
|
} else if storeErr == nil {
|
||||||
|
catalogLocks := locks
|
||||||
|
if lockErr != nil {
|
||||||
|
catalogLocks = &effectiveLocks{
|
||||||
|
Static: staticPublishLocks(cfg),
|
||||||
|
All: staticPublishLocks(cfg),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
publishedRemoteState := map[string]string{}
|
||||||
|
if store != nil {
|
||||||
|
publishedRemoteState = remotePublishedOutputAvailability(ctx, cfg, store, catalog)
|
||||||
|
}
|
||||||
|
fmt.Fprintln(out, "Remote outputs:")
|
||||||
|
writeArtifactList(out, cfg, catalog, catalogLocks, publishedRemoteState)
|
||||||
|
}
|
||||||
|
if lockErr != nil {
|
||||||
|
fmt.Fprintf(out, "Publish locks: error: %v\n", lockErr)
|
||||||
|
} else {
|
||||||
|
writeLocks(out, cfg, locks)
|
||||||
|
}
|
||||||
|
fmt.Fprintln(out, "Next actions:")
|
||||||
|
fmt.Fprintf(out, "- narratio session validate %s\n", cfg.Session.SessionID)
|
||||||
|
fmt.Fprintf(out, "- narratio session restore %s --dry-run\n", cfg.Session.SessionID)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeStatusStableInputs(out io.Writer, checks []stableInputCheck) {
|
||||||
|
if len(checks) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, check := range checks {
|
||||||
|
if check.Err != nil {
|
||||||
|
if strings.TrimSpace(check.Path) != "" {
|
||||||
|
fmt.Fprintf(out, "Stable input %s: unavailable: %v\n", check.Name, check.Err)
|
||||||
|
} else {
|
||||||
|
fmt.Fprintf(out, "Stable input %s: unavailable: %s\n", check.Name, check.Err.Error())
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "Stable input %s: %s\n", check.Name, check.Path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeStatusLocalAudio(out io.Writer, check localAudioCheck) {
|
||||||
|
if !check.Checked {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if check.Err != nil {
|
||||||
|
fmt.Fprintf(out, "Local audio: unavailable: %v\n", check.Err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "Local audio: %d file(s)\n", len(check.Paths))
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeStatusRemoteAudio(ctx context.Context, out io.Writer, cfg *config.Config, store storage.ObjectStore, storeErr error) {
|
||||||
|
if cfg.Session.Inputs.AudioS3 == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if storeErr != nil {
|
||||||
|
fmt.Fprintf(out, "Remote audio: unavailable: %v\n", storeErr)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
check := inspectRemoteAudioPresence(ctx, cfg, store)
|
||||||
|
if check.Err != nil {
|
||||||
|
fmt.Fprintf(out, "Remote audio: unavailable: %v\n", check.Err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "Remote audio: %d .flac object(s)\n", len(check.Keys))
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeStatusPreviousArtifacts(out io.Writer, readiness previousArtifactReadiness) {
|
||||||
|
if len(readiness.Requirements) == 0 {
|
||||||
|
fmt.Fprintln(out, "Previous-session artifacts: not required")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if readiness.MissingID {
|
||||||
|
fmt.Fprintln(out, "Previous-session artifacts: unavailable: previous_session_id is required by configured previous-session artifacts")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if readiness.Err != nil {
|
||||||
|
fmt.Fprintf(out, "Previous-session artifacts: unavailable: %v\n", readiness.Err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
names := make([]string, 0, len(readiness.Requirements))
|
||||||
|
for _, req := range readiness.Requirements {
|
||||||
|
names = append(names, fmt.Sprintf("%s(required=%t)", req.Name, req.Required))
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
fmt.Fprintf(out, "Previous-session artifacts: ready: %s\n", strings.Join(names, ", "))
|
||||||
|
}
|
||||||
@@ -5,9 +5,12 @@ import (
|
|||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/logging"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -16,38 +19,30 @@ func Plan(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
fs := flag.NewFlagSet("plan", flag.ContinueOnError)
|
fs := flag.NewFlagSet("plan", flag.ContinueOnError)
|
||||||
fs.SetOutput(io.Discard)
|
fs.SetOutput(io.Discard)
|
||||||
|
|
||||||
var pipelinePath string
|
var flags commonConfigFlags
|
||||||
var sessionPath string
|
|
||||||
var force bool
|
var force bool
|
||||||
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
addCommonConfigFlags(fs, &flags)
|
||||||
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
|
||||||
fs.BoolVar(&force, "force", false, "force stage execution (reserved for future behavior)")
|
fs.BoolVar(&force, "force", false, "force stage execution (reserved for future behavior)")
|
||||||
|
|
||||||
if err := fs.Parse(args); err != nil {
|
if err := parseSessionAwareFlags("plan", fs, args, &flags.sessionID); err != nil {
|
||||||
return fmt.Errorf("plan: invalid flags: %w", err)
|
return err
|
||||||
}
|
}
|
||||||
if fs.NArg() != 0 {
|
if flags.sessionID == "" {
|
||||||
return fmt.Errorf("plan: unexpected positional arguments")
|
return fmt.Errorf("plan: session_id is required")
|
||||||
}
|
}
|
||||||
if sessionPath == "" {
|
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
||||||
return fmt.Errorf("plan: --session is required")
|
|
||||||
}
|
|
||||||
|
|
||||||
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("plan: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg, err := config.Load(resolvedPipelinePath, sessionPath)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("plan: %w", err)
|
return fmt.Errorf("plan: %w", err)
|
||||||
}
|
}
|
||||||
if err := config.Validate(cfg); err != nil {
|
if err := config.Validate(cfg); err != nil {
|
||||||
return fmt.Errorf("plan: %w", err)
|
return fmt.Errorf("plan: %w", err)
|
||||||
}
|
}
|
||||||
|
if _, err := loadSecretsFromConfig(cfg, logging.NewLogger(os.Stderr, slog.LevelInfo)); err != nil {
|
||||||
|
return fmt.Errorf("plan: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
store := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root)
|
store := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root)
|
||||||
paths, err := store.EnsureLayout(cfg.Session.SessionID)
|
paths, err := store.EnsureLayoutFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("plan: prepare workdir: %w", err)
|
return fmt.Errorf("plan: prepare workdir: %w", err)
|
||||||
}
|
}
|
||||||
@@ -62,7 +57,7 @@ func Plan(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
|
|
||||||
runCount := 0
|
runCount := 0
|
||||||
skipCount := 0
|
skipCount := 0
|
||||||
if _, err := fmt.Fprintf(out, "narratio plan: workdir prepared at %s\n", paths.Root); err != nil {
|
if _, err := fmt.Fprintf(out, "narratio session plan: workdir prepared at %s\n", paths.Root); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
for _, d := range decisions {
|
for _, d := range decisions {
|
||||||
|
|||||||
@@ -15,28 +15,28 @@ import (
|
|||||||
|
|
||||||
func TestPlanCreatesAndReusesWorkdir(t *testing.T) {
|
func TestPlanCreatesAndReusesWorkdir(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
args := []string{"--config", pipelinePath, "--session", sessionPath}
|
args := []string{"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}
|
||||||
|
|
||||||
if err := Plan(context.Background(), args, &out); err != nil {
|
if err := Plan(context.Background(), args, &out); err != nil {
|
||||||
t.Fatalf("first Plan() error = %v", err)
|
t.Fatalf("first Plan() error = %v", err)
|
||||||
}
|
}
|
||||||
got := out.String()
|
got := out.String()
|
||||||
if !strings.Contains(got, "narratio plan: workdir prepared at") {
|
if !strings.Contains(got, "narratio session plan: workdir prepared at") {
|
||||||
t.Fatalf("first output = %q, want workdir prepared", got)
|
t.Fatalf("first output = %q, want workdir prepared", got)
|
||||||
}
|
}
|
||||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze", "archive", "notify"} {
|
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze", "publish", "notify"} {
|
||||||
if !strings.Contains(got, name+": run") {
|
if !strings.Contains(got, name+": run") {
|
||||||
t.Fatalf("first output = %q, missing stage %q", got, name)
|
t.Fatalf("first output = %q, missing stage %q", got, name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !strings.Contains(got, "totals: run=9 skip=0") {
|
if !strings.Contains(got, "totals: run=10 skip=0") {
|
||||||
t.Fatalf("first output = %q, want totals", got)
|
t.Fatalf("first output = %q, want totals", got)
|
||||||
}
|
}
|
||||||
|
|
||||||
sessionWorkdir := artifacts.SessionWorkDir(workspaceRoot, "2026-05-03")
|
sessionWorkdir := artifacts.SessionWorkDirForCampaign(workspaceRoot, "sample-campaign", "2026-05-03")
|
||||||
expectedDirs := []string{
|
expectedDirs := []string{
|
||||||
sessionWorkdir,
|
sessionWorkdir,
|
||||||
filepath.Join(sessionWorkdir, "inputs"),
|
filepath.Join(sessionWorkdir, "inputs"),
|
||||||
@@ -55,15 +55,15 @@ func TestPlanCreatesAndReusesWorkdir(t *testing.T) {
|
|||||||
if err := Plan(context.Background(), args, &out); err != nil {
|
if err := Plan(context.Background(), args, &out); err != nil {
|
||||||
t.Fatalf("second Plan() error = %v", err)
|
t.Fatalf("second Plan() error = %v", err)
|
||||||
}
|
}
|
||||||
if !strings.Contains(out.String(), "narratio plan: workdir prepared at") {
|
if !strings.Contains(out.String(), "narratio session plan: workdir prepared at") {
|
||||||
t.Fatalf("second output = %q, want workdir prepared", out.String())
|
t.Fatalf("second output = %q, want workdir prepared", out.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPlanShowsRunAndSkipFromManifest(t *testing.T) {
|
func TestPlanShowsRunAndSkipFromManifest(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
manifestPath := filepath.Join(workspaceRoot, "work", "2026-05-03", "manifest.json")
|
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||||
|
|
||||||
store := &manifest.LocalStore{}
|
store := &manifest.LocalStore{}
|
||||||
m := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
m := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||||
@@ -74,7 +74,7 @@ func TestPlanShowsRunAndSkipFromManifest(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
if err := Plan(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath}, &out); err != nil {
|
if err := Plan(context.Background(), []string{"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &out); err != nil {
|
||||||
t.Fatalf("Plan() error = %v", err)
|
t.Fatalf("Plan() error = %v", err)
|
||||||
}
|
}
|
||||||
got := out.String()
|
got := out.String()
|
||||||
@@ -84,8 +84,57 @@ func TestPlanShowsRunAndSkipFromManifest(t *testing.T) {
|
|||||||
if !strings.Contains(got, "trim: run") {
|
if !strings.Contains(got, "trim: run") {
|
||||||
t.Fatalf("output = %q, want trim run", got)
|
t.Fatalf("output = %q, want trim run", got)
|
||||||
}
|
}
|
||||||
if !strings.Contains(got, "totals: run=7 skip=2") {
|
if !strings.Contains(got, "totals: run=8 skip=2") {
|
||||||
t.Fatalf("output = %q, want totals run=7 skip=2", got)
|
t.Fatalf("output = %q, want totals run=8 skip=2", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPlanFailsWhenConfiguredSecretsDirMissing(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
configDir := t.TempDir()
|
||||||
|
pipelinePath := filepath.Join(configDir, "pipeline.yml")
|
||||||
|
campaignPath := writeAppTestCampaignConfig(t, configDir)
|
||||||
|
sessionPath := filepath.Join(configDir, "session.yml")
|
||||||
|
|
||||||
|
pipelineYAML := `workspace:
|
||||||
|
root: ` + workspaceRoot + `
|
||||||
|
storage:
|
||||||
|
backend: local
|
||||||
|
secrets:
|
||||||
|
env_dir: ./missing-secrets
|
||||||
|
whisperx:
|
||||||
|
transcribe_url: https://example.com/transcribe
|
||||||
|
seriatim:
|
||||||
|
binary: seriatim
|
||||||
|
audita:
|
||||||
|
binary: audita
|
||||||
|
notification:
|
||||||
|
timeout: 10s
|
||||||
|
`
|
||||||
|
sessionYAML := `session_id: 2026-05-03
|
||||||
|
campaign: sample-campaign
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
players_file: ./players.yml
|
||||||
|
party_file: ./party.yml
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write pipeline.yml: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write session.yml: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
err := Plan(context.Background(), []string{"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &out)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "read secrets env_dir") {
|
||||||
|
t.Fatalf("error = %q, want secrets read error context", err.Error())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import "testing"
|
|||||||
|
|
||||||
func TestBuildFullPlanOrder(t *testing.T) {
|
func TestBuildFullPlanOrder(t *testing.T) {
|
||||||
got := BuildFullPlan()
|
got := BuildFullPlan()
|
||||||
want := []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze", "archive", "notify"}
|
want := []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze", "publish", "notify"}
|
||||||
if len(got) != len(want) {
|
if len(got) != len(want) {
|
||||||
t.Fatalf("len(plan) = %d, want %d", len(got), len(want))
|
t.Fatalf("len(plan) = %d, want %d", len(got), len(want))
|
||||||
}
|
}
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user