Compare commits
34 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 37daab7857 | |||
| 2356688cb9 | |||
| 1054b64d9f | |||
| 01fb02426c | |||
| 7dc79e052f | |||
| cb525c0f72 | |||
| 622677d038 | |||
| 550288e008 | |||
| e58e545686 | |||
| 6ff54c5a0f | |||
| 924b5d15c6 | |||
| b065663180 | |||
| 3ba564b00f | |||
| a3986cf0d6 | |||
| 539601bd16 | |||
| 6ca1c8d6b0 | |||
| 4b7b50981b | |||
| 33f7ae8f2e | |||
| d5a9ad38f8 | |||
| 6fbefb9867 | |||
| 1665359486 | |||
| 03f2543927 | |||
| fe9c348092 | |||
| f7f8f1a949 | |||
| d40c91acde | |||
| ed4dcf1ef7 | |||
| 24cce49a70 | |||
| 1e6db89dd4 | |||
| 0454296c81 | |||
| 58c6ab2d54 | |||
| 7995c41675 | |||
| 62551d43a0 | |||
| 8395c12dd3 | |||
| dc8e1040f2 |
5
.gitignore
vendored
5
.gitignore
vendored
@@ -2,6 +2,8 @@
|
|||||||
.codex
|
.codex
|
||||||
AGENTS.md
|
AGENTS.md
|
||||||
|
|
||||||
|
.DS_Store
|
||||||
|
|
||||||
# ---> Go
|
# ---> Go
|
||||||
# If you prefer the allow list template instead of the deny list, see community template:
|
# If you prefer the allow list template instead of the deny list, see community template:
|
||||||
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
|
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
|
||||||
@@ -22,6 +24,9 @@ AGENTS.md
|
|||||||
# Dependency directories (remove the comment below to include it)
|
# Dependency directories (remove the comment below to include it)
|
||||||
# vendor/
|
# vendor/
|
||||||
|
|
||||||
|
# Go cache
|
||||||
|
.gocache
|
||||||
|
|
||||||
# Go workspace file
|
# Go workspace file
|
||||||
go.work
|
go.work
|
||||||
go.work.sum
|
go.work.sum
|
||||||
|
|||||||
200
README.md
200
README.md
@@ -16,7 +16,6 @@ Implemented now:
|
|||||||
|
|
||||||
Not implemented yet:
|
Not implemented yet:
|
||||||
|
|
||||||
- `archive` stage behavior
|
|
||||||
- `notify` stage behavior
|
- `notify` stage behavior
|
||||||
- additional analyze artifacts beyond `session_recap`
|
- additional analyze artifacts beyond `session_recap`
|
||||||
- generic DAG orchestration
|
- generic DAG orchestration
|
||||||
@@ -28,8 +27,127 @@ Narratio expects two YAML files:
|
|||||||
- `pipeline.yml`: pipeline/workspace settings
|
- `pipeline.yml`: pipeline/workspace settings
|
||||||
- `session.yml`: per-session settings
|
- `session.yml`: per-session settings
|
||||||
|
|
||||||
|
Pipeline config lookup for CLI commands:
|
||||||
|
|
||||||
|
- if `--config <path>` is provided, Narratio uses that path
|
||||||
|
- if `--config` is omitted, Narratio searches in this order:
|
||||||
|
- `/usr/local/etc/narratio/pipeline.yml`
|
||||||
|
- `/etc/narratio/pipeline.yml`
|
||||||
|
|
||||||
|
Session config lookup for CLI commands:
|
||||||
|
|
||||||
|
- if `--session <path>` is provided, Narratio uses that path
|
||||||
|
- if `--session` is omitted, Narratio searches in this order:
|
||||||
|
- `./session.yml`
|
||||||
|
- `/usr/local/etc/narratio/session.yml`
|
||||||
|
- `/etc/narratio/session.yml`
|
||||||
|
|
||||||
|
Session template support:
|
||||||
|
|
||||||
|
- Narratio renders `session.yml` templates before strict YAML decode.
|
||||||
|
- `--session-id <value>` provides the `session_id` template variable.
|
||||||
|
- Supported placeholder forms:
|
||||||
|
- `{{session_id}}`
|
||||||
|
- `{{ session_id }}`
|
||||||
|
- unresolved template placeholders fail with a clear error.
|
||||||
|
- strict YAML validation still runs after rendering.
|
||||||
|
- concrete `session.yml` files without templates remain fully supported.
|
||||||
|
|
||||||
|
Optional secrets-from-files config:
|
||||||
|
|
||||||
|
- `pipeline.secrets.env_dir` may point to a directory of secret files
|
||||||
|
- each top-level file with an env-var-style name is loaded as an environment variable:
|
||||||
|
- file name = env var name
|
||||||
|
- file contents = env var value (trailing newline/CRLF trimmed)
|
||||||
|
- process environment wins: existing env vars are not overwritten
|
||||||
|
- if configured, Narratio fails fast when `env_dir` is missing/unreadable
|
||||||
|
- relative `env_dir` values resolve from Narratio’s current working directory
|
||||||
|
|
||||||
YAML decoding is strict (`KnownFields(true)`), so unknown fields fail fast.
|
YAML decoding is strict (`KnownFields(true)`), so unknown fields fail fast.
|
||||||
|
|
||||||
|
Maintainer note: application defaults are centralized in [`internal/config/defaults.go`](internal/config/defaults.go).
|
||||||
|
|
||||||
|
## Storage And Archive Foundations
|
||||||
|
|
||||||
|
Narratio now includes configuration and path-model foundations for archive support, plus implemented prepare-stage S3 audio input.
|
||||||
|
|
||||||
|
Implemented foundations:
|
||||||
|
|
||||||
|
- `pipeline.storage.s3` config shape (`bucket`, `root_prefix`, `region`, `endpoint`, `force_path_style`, `access_key_id_env`, `secret_access_key_env`)
|
||||||
|
- `pipeline.spool` config shape (`root`, `delete_audio_after_archive`)
|
||||||
|
- `pipeline.archive` config shape (`enabled`, `upload_run`, `promote_artifacts`)
|
||||||
|
- promotion-rule validation (`from`/`to` required, relative-only paths, traversal rejected)
|
||||||
|
- `session.campaign` requirement for campaign-aware path construction
|
||||||
|
- optional `session.inputs.audio_s3.prefix` modeling and prepare-stage S3 audio download
|
||||||
|
- run ID generation and S3/local path helper foundations
|
||||||
|
- manifest run/path identity fields
|
||||||
|
|
||||||
|
Current defaults:
|
||||||
|
|
||||||
|
- `pipeline.storage.s3.root_prefix`: `dnd`
|
||||||
|
- `pipeline.storage.s3.access_key_id_env`: `OBJECT_STORAGE_KEY_ID`
|
||||||
|
- `pipeline.storage.s3.secret_access_key_env`: `OBJECT_STORAGE_KEY`
|
||||||
|
- `pipeline.workspace.cleanup_after_archive`: `false`
|
||||||
|
- `pipeline.spool.root`: `/var/spool/narratio`
|
||||||
|
- `pipeline.spool.delete_audio_after_archive`: `false`
|
||||||
|
- `pipeline.archive.enabled`: `true`
|
||||||
|
- `pipeline.archive.upload_run`: `true`
|
||||||
|
- default `pipeline.archive.promote_artifacts`:
|
||||||
|
- `transcripts/trimmed.json` -> `transcripts/trimmed.json` (`required: true`)
|
||||||
|
- `artifacts/session_recap.md` -> `artifacts/session_recap.md` (`required: true`)
|
||||||
|
|
||||||
|
Current boundaries:
|
||||||
|
|
||||||
|
- local development audio (`audio_dir` / `audio_files`) still works
|
||||||
|
- `audio_dir`/`audio_files` and `audio_s3` are mutually exclusive
|
||||||
|
- real S3-compatible backend now exists in the storage adapter package
|
||||||
|
- storage backend tests use fake storage and do not require live S3
|
||||||
|
- archive uploads successful run records under `runs/{run_id}/`
|
||||||
|
- archive does not upload local audio by default
|
||||||
|
- archive uploads promoted outputs to session-level keys using `archive.promote_artifacts`
|
||||||
|
- archive uploads `current/manifest.json`
|
||||||
|
- archive uploads `current/run_id.txt` last as the effective commit marker
|
||||||
|
- required missing promotions fail archive
|
||||||
|
- optional missing promotions are skipped and recorded
|
||||||
|
- cleanup remains conservative and opt-in:
|
||||||
|
- `pipeline.spool.delete_audio_after_archive: true` removes only the run-scoped spool audio directory after successful archive commit
|
||||||
|
- `pipeline.workspace.cleanup_after_archive: true` removes only the run-scoped local workdir after successful archive commit
|
||||||
|
- cleanup executes only after all selected stages for the command invocation succeed
|
||||||
|
- cleanup does not run for failed, incomplete, skipped, or unarchived runs
|
||||||
|
- local development `audio_dir`/`audio_files` source inputs are never deleted by spool cleanup
|
||||||
|
- S3 credentials are resolved from configured env-var names when both are present; if either is missing, Narratio falls back to the AWS SDK default credential chain
|
||||||
|
|
||||||
|
S3 input details and current boundaries are documented in [docs/s3-audio-input.md](docs/s3-audio-input.md).
|
||||||
|
|
||||||
|
## Remote Storage Backend
|
||||||
|
|
||||||
|
Narratio includes an object-store backend layer for future prepare/archive work:
|
||||||
|
|
||||||
|
- `List(ctx, prefix)`
|
||||||
|
- `Download(ctx, key, localPath)`
|
||||||
|
- `Upload(ctx, localPath, key, opts)`
|
||||||
|
- `Exists(ctx, key)`
|
||||||
|
|
||||||
|
Implemented backends:
|
||||||
|
|
||||||
|
- fake storage backend for deterministic tests
|
||||||
|
- S3-compatible backend built from `pipeline.storage.s3`
|
||||||
|
|
||||||
|
Key invariant:
|
||||||
|
|
||||||
|
- callers pass full bucket-relative object keys
|
||||||
|
- storage backends do not prepend `root_prefix` and do not infer session/campaign paths
|
||||||
|
|
||||||
|
Current boundary:
|
||||||
|
|
||||||
|
- `prepare` uses `List` + `Download` through the backend when `session.inputs.audio_s3` is configured
|
||||||
|
- `archive` uses `Upload` through the backend for successful run-record uploads
|
||||||
|
- `archive` also uses `Upload` for promotion writes and current pointers
|
||||||
|
- no failed or incomplete runs are uploaded
|
||||||
|
- local audio is not re-uploaded by default
|
||||||
|
|
||||||
|
Archive run-upload details and boundaries are documented in [docs/archive-storage.md](docs/archive-storage.md).
|
||||||
|
|
||||||
## Canonical Stage Order
|
## Canonical Stage Order
|
||||||
|
|
||||||
1. `prepare`
|
1. `prepare`
|
||||||
@@ -49,6 +167,52 @@ YAML decoding is strict (`KnownFields(true)`), so unknown fields fail fast.
|
|||||||
- `transcripts/normalized.json`: Seriatim-normalized transcript from the normalize stage
|
- `transcripts/normalized.json`: Seriatim-normalized transcript from the normalize stage
|
||||||
- `transcripts/trimmed.json`: gameplay-only normalized polished transcript from trim stage
|
- `transcripts/trimmed.json`: gameplay-only normalized polished transcript from trim stage
|
||||||
|
|
||||||
|
## Seriatim Configuration
|
||||||
|
|
||||||
|
`pipeline.seriatim` configures the Seriatim subprocess adapter used by `merge`, `normalize`, and `trim`.
|
||||||
|
|
||||||
|
Minimal behavior:
|
||||||
|
|
||||||
|
- `pipeline.seriatim` may be omitted entirely.
|
||||||
|
- when omitted, Narratio defaults to:
|
||||||
|
- `binary: seriatim`
|
||||||
|
- `timeout: 10m`
|
||||||
|
- `output_schema: seriatim-intermediate`
|
||||||
|
- `coalesce_gap: 3.0`
|
||||||
|
- `report: true`
|
||||||
|
|
||||||
|
Optional overrides in `pipeline.seriatim` continue to work, including explicit binary paths and advanced `env` tuning values.
|
||||||
|
|
||||||
|
## Audita Configuration
|
||||||
|
|
||||||
|
`pipeline.audita` configures the real Audita subprocess adapter used by `polish`.
|
||||||
|
|
||||||
|
Minimal behavior:
|
||||||
|
|
||||||
|
- `pipeline.audita` may be omitted entirely.
|
||||||
|
- when omitted, Narratio defaults to:
|
||||||
|
- `binary: audita`
|
||||||
|
- `timeout: 3h`
|
||||||
|
- `report: true`
|
||||||
|
|
||||||
|
Optional:
|
||||||
|
|
||||||
|
- `llm_api_key_env` (when set, Narratio requires that env var and passes it to Audita as `AUDITA_LLM_API_KEY`)
|
||||||
|
- `modules` override list (when empty/omitted, Narratio does not pass `--modules`)
|
||||||
|
- `base_url` (when omitted, Narratio does not pass `--base-url`; Audita runtime defaults/config may apply)
|
||||||
|
- `model` (when omitted, Narratio does not pass `--model`; Audita runtime defaults/config may apply)
|
||||||
|
- `transcript_description`
|
||||||
|
- `config_path`
|
||||||
|
- `output_schema` (`bare-segments` or `audita-v1`)
|
||||||
|
- `work_dir_retention` (`always`, `auto`, or `never`)
|
||||||
|
- `total_llm_concurrency` (> 0 when provided)
|
||||||
|
- `proposal_llm_concurrency` (> 0 when provided)
|
||||||
|
- `validation_model`
|
||||||
|
- `validation_llm_concurrency` (> 0 when provided)
|
||||||
|
- `report` (defaults to `true`)
|
||||||
|
|
||||||
|
Narratio passes only configured optional Audita flags. Omitted optional values are left to Audita runtime defaults/config.
|
||||||
|
|
||||||
## Normalize Configuration
|
## Normalize Configuration
|
||||||
|
|
||||||
`pipeline.normalize` is optional. When omitted, Narratio defaults to:
|
`pipeline.normalize` is optional. When omitted, Narratio defaults to:
|
||||||
@@ -142,7 +306,7 @@ Render-debug files are diagnostics and are not treated as canonical stage output
|
|||||||
|
|
||||||
Key points:
|
Key points:
|
||||||
|
|
||||||
- `scriptorium.binary` is required when section is present
|
- `scriptorium.binary` defaults to `scriptorium` when section is present
|
||||||
- `scriptorium.config_path` is optional
|
- `scriptorium.config_path` is optional
|
||||||
- `scriptorium.timeout` defaults to `10m` when omitted
|
- `scriptorium.timeout` defaults to `10m` when omitted
|
||||||
- `scriptorium.render_debug` enables render diagnostics globally
|
- `scriptorium.render_debug` enables render diagnostics globally
|
||||||
@@ -192,6 +356,8 @@ Prompt IDs and profile IDs are configuration values. They are not hardcoded in a
|
|||||||
|
|
||||||
Do not put secrets in `pipeline.yml`. If API-key behavior is configured, use env var names only.
|
Do not put secrets in `pipeline.yml`. If API-key behavior is configured, use env var names only.
|
||||||
|
|
||||||
|
If `pipeline.secrets.env_dir` is configured, keep only references and secret files there; secret values are still not written to manifests, generated configs, or Narratio-managed logs.
|
||||||
|
|
||||||
## Scriptorium Runtime Behavior
|
## Scriptorium Runtime Behavior
|
||||||
|
|
||||||
Narratio integrates with Scriptorium through the public CLI subprocess contract:
|
Narratio integrates with Scriptorium through the public CLI subprocess contract:
|
||||||
@@ -203,10 +369,16 @@ For the initial implementation, only `session_recap` generation is supported.
|
|||||||
|
|
||||||
Analyze-stage session recap behavior:
|
Analyze-stage session recap behavior:
|
||||||
|
|
||||||
- available transcript input sources for configured artifacts: `processed_transcript`, `normalized_transcript`, `trimmed_transcript`
|
- preferred transcript artifact source IDs:
|
||||||
|
- `narratio.transcript.polished`
|
||||||
|
- `narratio.transcript.full`
|
||||||
|
- `narratio.transcript.trimmed`
|
||||||
|
- backward-compatible aliases remain supported:
|
||||||
|
- `processed_transcript`
|
||||||
|
- `normalized_transcript`
|
||||||
|
- `trimmed_transcript`
|
||||||
- session recap should use gameplay-only transcript input (`source: trimmed_transcript`)
|
- session recap should use gameplay-only transcript input (`source: trimmed_transcript`)
|
||||||
- Narratio resolves `trimmed_transcript` from trim manifest output (`transcript_trimmed`) or fallback `transcripts/trimmed.json`
|
- Narratio resolves transcript inputs from the artifact resolver (manifest producer outputs first, then canonical session paths)
|
||||||
- Narratio resolves `normalized_transcript` from normalize manifest output (`transcript_normalized`) or fallback `transcripts/normalized.json`
|
|
||||||
- missing trimmed transcript fails clearly and advises running trim stage first
|
- missing trimmed transcript fails clearly and advises running trim stage first
|
||||||
- `normalized_transcript` is the preferred full-transcript source for future table/meta-analysis artifacts
|
- `normalized_transcript` is the preferred full-transcript source for future table/meta-analysis artifacts
|
||||||
- `processed_transcript` remains supported for advanced/debug use cases
|
- `processed_transcript` remains supported for advanced/debug use cases
|
||||||
@@ -228,7 +400,9 @@ Expected session output paths:
|
|||||||
Starter files:
|
Starter files:
|
||||||
|
|
||||||
- `examples/pipeline.minimal.yml`
|
- `examples/pipeline.minimal.yml`
|
||||||
|
- `examples/pipeline.audita-overrides.yml`
|
||||||
- `examples/session.minimal.yml`
|
- `examples/session.minimal.yml`
|
||||||
|
- `examples/session.template.yml`
|
||||||
- `examples/speakers.yml`
|
- `examples/speakers.yml`
|
||||||
|
|
||||||
## Commands
|
## Commands
|
||||||
@@ -242,7 +416,15 @@ go test ./...
|
|||||||
Plan a run:
|
Plan a run:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
go run ./cmd/narratio plan --config examples/pipeline.minimal.yml --session examples/session.minimal.yml
|
go run ./cmd/narratio plan --session examples/session.minimal.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
Use `--config <path>` to override default pipeline lookup when needed.
|
||||||
|
|
||||||
|
Run with a discoverable session template:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
go run ./cmd/narratio run --session-id 2026-04-04
|
||||||
```
|
```
|
||||||
|
|
||||||
Run full pipeline:
|
Run full pipeline:
|
||||||
@@ -257,6 +439,12 @@ Run analyze only:
|
|||||||
go run ./cmd/narratio run-stage --config examples/pipeline.minimal.yml --session examples/session.minimal.yml analyze
|
go run ./cmd/narratio run-stage --config examples/pipeline.minimal.yml --session examples/session.minimal.yml analyze
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Resume with a template session ID:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
go run ./cmd/narratio resume --config examples/pipeline.minimal.yml --session examples/session.template.yml --session-id 2026-04-04
|
||||||
|
```
|
||||||
|
|
||||||
## Operational Note
|
## Operational Note
|
||||||
|
|
||||||
Checksum-based stale detection is not implemented yet.
|
Checksum-based stale detection is not implemented yet.
|
||||||
|
|||||||
190
architecture.md
190
architecture.md
@@ -22,10 +22,24 @@ Implemented:
|
|||||||
- real `trim` stage producing `transcripts/trimmed.json`
|
- real `trim` stage producing `transcripts/trimmed.json`
|
||||||
- real `analyze` stage for initial `session_recap` generation
|
- real `analyze` stage for initial `session_recap` generation
|
||||||
- optional Scriptorium render diagnostics (`render_debug`) before production run
|
- optional Scriptorium render diagnostics (`render_debug`) before production run
|
||||||
|
- storage/archive configuration and validation foundations for:
|
||||||
|
- `pipeline.storage.s3`
|
||||||
|
- `pipeline.spool`
|
||||||
|
- `pipeline.archive` promotion rules
|
||||||
|
- `session.inputs.audio_s3`
|
||||||
|
- run identity and path-model foundations:
|
||||||
|
- run ID generation (`YYYYMMDDTHHMMSSZ-xxxxxxxx`)
|
||||||
|
- S3 session/run/current key builders
|
||||||
|
- campaign/session/run local work/spool path helpers
|
||||||
|
- manifest run/path identity fields (`campaign`, `run_id`, local and S3 prefixes)
|
||||||
|
- remote storage backend layer:
|
||||||
|
- narrow object-store interface (`List`, `Download`, `Upload`, `Exists`)
|
||||||
|
- fake storage backend for deterministic tests (no network dependency)
|
||||||
|
- S3-compatible backend using AWS SDK v2
|
||||||
|
- config-based object-store construction helper
|
||||||
|
|
||||||
Still placeholder/future:
|
Still placeholder/future:
|
||||||
|
|
||||||
- `archive` stage behavior
|
|
||||||
- `notify` stage behavior
|
- `notify` stage behavior
|
||||||
- additional Scriptorium artifact types beyond `session_recap`
|
- additional Scriptorium artifact types beyond `session_recap`
|
||||||
- artifact-to-artifact workflows beyond the initial single-artifact implementation
|
- artifact-to-artifact workflows beyond the initial single-artifact implementation
|
||||||
@@ -94,12 +108,183 @@ Adapter behavior:
|
|||||||
|
|
||||||
## 5. Configuration Contract
|
## 5. Configuration Contract
|
||||||
|
|
||||||
|
CLI pipeline config path resolution:
|
||||||
|
|
||||||
|
- when `--config <path>` is provided, that path is used
|
||||||
|
- when `--config` is omitted, Narratio searches defaults in order:
|
||||||
|
- `/usr/local/etc/narratio/pipeline.yml`
|
||||||
|
- `/etc/narratio/pipeline.yml`
|
||||||
|
- default values are centralized in `internal/config/defaults.go`
|
||||||
|
|
||||||
|
CLI session config path resolution:
|
||||||
|
|
||||||
|
- when `--session <path>` is provided, that path is used
|
||||||
|
- when `--session` is omitted, Narratio searches defaults in order:
|
||||||
|
- `./session.yml`
|
||||||
|
- `/usr/local/etc/narratio/session.yml`
|
||||||
|
- `/etc/narratio/session.yml`
|
||||||
|
|
||||||
|
Session template rendering:
|
||||||
|
|
||||||
|
- session templates are rendered before strict YAML decode
|
||||||
|
- `--session-id <value>` provides the `session_id` template variable
|
||||||
|
- supported placeholders:
|
||||||
|
- `{{session_id}}`
|
||||||
|
- `{{ session_id }}`
|
||||||
|
- unresolved placeholders fail clearly
|
||||||
|
- strict `KnownFields(true)` YAML validation still applies after rendering
|
||||||
|
- if rendered `session.session_id` conflicts with `--session-id`, load fails clearly
|
||||||
|
|
||||||
|
Optional pipeline secrets directory:
|
||||||
|
|
||||||
|
- `pipeline.secrets.env_dir` enables loading environment variables from local files before command execution
|
||||||
|
- file name = env var name; file contents = env var value (trailing newline/CRLF trimmed)
|
||||||
|
- only env-var-style file names are considered; other entries are ignored
|
||||||
|
- existing process environment values are preserved (not overwritten)
|
||||||
|
- if configured, unreadable/missing `env_dir` fails command execution early
|
||||||
|
- relative `env_dir` values are resolved from current working directory
|
||||||
|
|
||||||
|
Storage and archive foundations:
|
||||||
|
|
||||||
|
- `pipeline.storage.s3` is available for modeling S3 coordinates:
|
||||||
|
- `bucket`
|
||||||
|
- `root_prefix` (default `dnd`)
|
||||||
|
- `region`
|
||||||
|
- `endpoint`
|
||||||
|
- `force_path_style` (default `false`)
|
||||||
|
- `access_key_id_env` (default `OBJECT_STORAGE_KEY_ID`)
|
||||||
|
- `secret_access_key_env` (default `OBJECT_STORAGE_KEY`)
|
||||||
|
- `pipeline.spool.root` defaults to `/var/spool/narratio`
|
||||||
|
- `pipeline.workspace.cleanup_after_archive` defaults to `false`
|
||||||
|
- `pipeline.spool.delete_audio_after_archive` defaults to `false`
|
||||||
|
- `pipeline.archive` is optional and defaults to:
|
||||||
|
- `enabled: true`
|
||||||
|
- `upload_run: true`
|
||||||
|
- default `promote_artifacts`:
|
||||||
|
- `transcripts/trimmed.json`
|
||||||
|
- `artifacts/session_recap.md`
|
||||||
|
- archive promotion rules enforce safe relative paths:
|
||||||
|
- `from` and `to` are required
|
||||||
|
- absolute paths are rejected
|
||||||
|
- traversal segments such as `..` are rejected
|
||||||
|
|
||||||
|
Session input foundations:
|
||||||
|
|
||||||
|
- `session.campaign` is required
|
||||||
|
- local audio remains supported through `session.inputs.audio_dir` or `session.inputs.audio_files`
|
||||||
|
- optional S3 audio input shape is `session.inputs.audio_s3.prefix`
|
||||||
|
- `audio_dir`/`audio_files` and `audio_s3` are mutually exclusive
|
||||||
|
- when `audio_s3` is configured, `prepare` lists and downloads `.flac` objects through the object-store backend
|
||||||
|
|
||||||
|
Cross-config validation scope:
|
||||||
|
|
||||||
|
- `pipeline.storage.s3.bucket` is required only when an S3-dependent feature is explicitly configured (for current foundations, that includes `session.inputs.audio_s3`, and archive upload intent when using `storage.backend: s3`)
|
||||||
|
- no AWS credential values are stored in Narratio config; only env-var names are configured
|
||||||
|
- when both configured credential env vars resolve to non-empty values, the S3 backend uses them as static credentials
|
||||||
|
- when either configured credential value is missing, the S3 backend falls back to the AWS SDK default credential chain
|
||||||
|
|
||||||
|
Remote object-store backend scope:
|
||||||
|
|
||||||
|
- remote storage APIs are isolated to `internal/adapters/storage`
|
||||||
|
- AWS SDK types remain contained within the S3 backend implementation package
|
||||||
|
- S3 key/session path semantics remain outside the backend, with this invariant:
|
||||||
|
- callers pass full bucket-relative object keys
|
||||||
|
- backend methods do not prepend `root_prefix` or infer campaign/session/run paths
|
||||||
|
- `prepare` now uses object-store `List` and `Download` for S3 audio input
|
||||||
|
- `archive` now uses object-store `Upload` for successful run-record upload under the run prefix
|
||||||
|
- `archive` now uses object-store `Upload` for promoted outputs and current pointers
|
||||||
|
|
||||||
|
Prepare S3 audio behavior (implemented):
|
||||||
|
|
||||||
|
- compute session prefix as `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/`
|
||||||
|
- resolve `session.inputs.audio_s3.prefix` under that session prefix
|
||||||
|
- list objects under the computed audio prefix and filter `.flac` keys
|
||||||
|
- fail clearly when no `.flac` objects are found
|
||||||
|
- download selected objects to spool audio path:
|
||||||
|
- `{spool.root}/{campaign}/{session_id}/{run_id}/audio/`
|
||||||
|
- materialize audio files into workdir audio path:
|
||||||
|
- `{workspace.root}/work/{campaign}/{session_id}/{run_id}/audio/`
|
||||||
|
- record S3 provenance in manifest input records (bucket/key/metadata/local paths/checksum)
|
||||||
|
- no AWS SDK types are used in stage code; storage implementation details stay in storage adapter packages
|
||||||
|
|
||||||
|
Archive publishing behavior (implemented):
|
||||||
|
|
||||||
|
- `archive` verifies prerequisite stage success before upload:
|
||||||
|
- `prepare`, `transcribe`, `merge`, `polish`, `normalize`, `trim`, `analyze`
|
||||||
|
- only successful/completed runs are uploaded
|
||||||
|
- uploaded run record destination is:
|
||||||
|
- `{root_prefix}/campaigns/{campaign}/sessions/{session_id}/runs/{run_id}/`
|
||||||
|
- uploaded existing local paths include:
|
||||||
|
- `inputs/`, `transcripts/`, `artifacts/`, optional `reports/`, `config/`, `logs/`, and `manifest.json`
|
||||||
|
- local `audio/` is intentionally excluded from upload by default
|
||||||
|
- file upload order is deterministic (sorted relative paths)
|
||||||
|
- `archive.enabled: false` and `archive.upload_run: false` skip upload cleanly
|
||||||
|
- stage metadata records non-secret upload context:
|
||||||
|
- run upload details, promoted output details, current manifest key, current pointer key
|
||||||
|
- no secrets, transcript contents, prompt contents, or environment dumps
|
||||||
|
- promotion rules:
|
||||||
|
- `from` resolves from local workdir
|
||||||
|
- `to` resolves under session-level S3 root
|
||||||
|
- missing required source fails archive
|
||||||
|
- missing optional source is skipped and recorded
|
||||||
|
- default promoted outputs:
|
||||||
|
- `transcripts/trimmed.json`
|
||||||
|
- `artifacts/session_recap.md`
|
||||||
|
- current pointers:
|
||||||
|
- `current/manifest.json` uploaded after run upload and promotions
|
||||||
|
- `current/run_id.txt` uploaded last with `{run_id}\n`
|
||||||
|
- `current/run_id.txt` is the effective commit marker
|
||||||
|
- if promotion or current-manifest upload fails, archive returns failure and does not write `current/run_id.txt`
|
||||||
|
- failed/incomplete runs remain local and are not uploaded
|
||||||
|
- post-archive local cleanup (implemented, opt-in):
|
||||||
|
- cleanup runs only after archive succeeded and wrote `current/run_id.txt`
|
||||||
|
- cleanup is executed after all selected stages in the command invocation succeed (for example, a later `notify` failure leaves local files intact)
|
||||||
|
- `pipeline.spool.delete_audio_after_archive: true` removes only `{spool.root}/{campaign}/{session_id}/{run_id}/audio/`
|
||||||
|
- `pipeline.workspace.cleanup_after_archive: true` removes only `{workspace.root}/work/{campaign}/{session_id}/{run_id}/`
|
||||||
|
- cleanup does not run when archive is skipped/disabled/fails or when run upload is disabled
|
||||||
|
- local development `audio_dir`/`audio_files` inputs are never removed by spool cleanup
|
||||||
|
|
||||||
`pipeline.scriptorium` is optional. Existing pipelines without Scriptorium continue to work.
|
`pipeline.scriptorium` is optional. Existing pipelines without Scriptorium continue to work.
|
||||||
|
|
||||||
`pipeline.trim` is optional. Existing pipelines without trim config continue to work.
|
`pipeline.trim` is optional. Existing pipelines without trim config continue to work.
|
||||||
|
|
||||||
`pipeline.normalize` is optional. Existing pipelines without normalize config continue to work.
|
`pipeline.normalize` is optional. Existing pipelines without normalize config continue to work.
|
||||||
|
|
||||||
|
`pipeline.audita` drives the real Audita subprocess adapter for the `polish` stage.
|
||||||
|
|
||||||
|
Audita defaulted fields:
|
||||||
|
|
||||||
|
- `binary` defaults to `audita`
|
||||||
|
- `timeout` defaults to `3h`
|
||||||
|
- `report` defaults to `true`
|
||||||
|
|
||||||
|
Audita optional fields:
|
||||||
|
|
||||||
|
- `llm_api_key_env` (enforced only when configured)
|
||||||
|
- `modules` override list (when omitted/empty, Narratio does not pass `--modules`)
|
||||||
|
- `base_url` (when omitted, Narratio does not pass `--base-url`)
|
||||||
|
- `model` (when omitted, Narratio does not pass `--model`)
|
||||||
|
- `transcript_description`
|
||||||
|
- `config_path`
|
||||||
|
- `output_schema` (`bare-segments` or `audita-v1`)
|
||||||
|
- `work_dir_retention` (`always`, `auto`, `never`)
|
||||||
|
- `total_llm_concurrency` (> 0 when provided)
|
||||||
|
- `proposal_llm_concurrency` (> 0 when provided)
|
||||||
|
- `validation_model`
|
||||||
|
- `validation_llm_concurrency` (> 0 when provided)
|
||||||
|
- `report` override
|
||||||
|
|
||||||
|
Narratio passes only configured optional Audita flags; omitted optional values defer to Audita runtime defaults/config.
|
||||||
|
|
||||||
|
Seriatim defaults:
|
||||||
|
|
||||||
|
- `pipeline.seriatim` may be omitted
|
||||||
|
- `binary` defaults to `seriatim`
|
||||||
|
- `timeout` defaults to `10m`
|
||||||
|
- `output_schema` defaults to `seriatim-intermediate`
|
||||||
|
- `coalesce_gap` defaults to `3.0`
|
||||||
|
- `report` defaults to `true`
|
||||||
|
|
||||||
When `pipeline.normalize` is omitted, defaults are applied:
|
When `pipeline.normalize` is omitted, defaults are applied:
|
||||||
|
|
||||||
- `output_path: transcripts/normalized.json`
|
- `output_path: transcripts/normalized.json`
|
||||||
@@ -132,7 +317,7 @@ When `pipeline.trim.enabled: true`:
|
|||||||
|
|
||||||
When `pipeline.scriptorium` is present:
|
When `pipeline.scriptorium` is present:
|
||||||
|
|
||||||
- `binary` is required and non-empty
|
- `binary` defaults to `scriptorium` when omitted
|
||||||
- `config_path` is optional; when provided it must be non-empty
|
- `config_path` is optional; when provided it must be non-empty
|
||||||
- `timeout` is optional; when provided it must parse as a Go duration
|
- `timeout` is optional; when provided it must parse as a Go duration
|
||||||
- default `timeout` is `10m`
|
- default `timeout` is `10m`
|
||||||
@@ -281,6 +466,7 @@ Current expected paths for `session_recap`:
|
|||||||
|
|
||||||
- do not store secrets in pipeline YAML, generated invocation YAML, logs, or manifest metadata
|
- do not store secrets in pipeline YAML, generated invocation YAML, logs, or manifest metadata
|
||||||
- if API-key integration is configured, pass env var names only (never raw key values)
|
- if API-key integration is configured, pass env var names only (never raw key values)
|
||||||
|
- with `pipeline.secrets.env_dir`, secret file values are loaded into process env only and are not persisted in manifest metadata or generated configs
|
||||||
- avoid logging transcript content or rendered prompt content by default
|
- avoid logging transcript content or rendered prompt content by default
|
||||||
- treat generated artifacts and logs as potentially sensitive session material
|
- treat generated artifacts and logs as potentially sensitive session material
|
||||||
|
|
||||||
|
|||||||
130
docs/development/workspace-implementation-plan.md
Normal file
130
docs/development/workspace-implementation-plan.md
Normal file
@@ -0,0 +1,130 @@
|
|||||||
|
# Workspace Architecture Implementation Plan (Status)
|
||||||
|
|
||||||
|
This document tracks the implemented workspace architecture and remaining work for v1.0.
|
||||||
|
|
||||||
|
## Current Architecture (Implemented)
|
||||||
|
|
||||||
|
Narratio now uses a canonical campaign-aware local layout:
|
||||||
|
|
||||||
|
```text
|
||||||
|
{workspace.root}/work/{campaign_id}/{session_id}/
|
||||||
|
manifest.json
|
||||||
|
current/
|
||||||
|
manifest.json
|
||||||
|
run_id.txt
|
||||||
|
inputs/
|
||||||
|
transcripts/
|
||||||
|
artifacts/
|
||||||
|
reports/
|
||||||
|
logs/
|
||||||
|
config/
|
||||||
|
runs/
|
||||||
|
{run_id}/
|
||||||
|
manifest.json
|
||||||
|
{stage}/
|
||||||
|
outputs/
|
||||||
|
logs/
|
||||||
|
reports/
|
||||||
|
config/
|
||||||
|
scratch/
|
||||||
|
```
|
||||||
|
|
||||||
|
Core behavior:
|
||||||
|
|
||||||
|
- Session manifest remains the skip/resume source of truth.
|
||||||
|
- Each invocation creates a run manifest at `runs/{run_id}/manifest.json`.
|
||||||
|
- Stage execution writes run-local artifacts and promotes durable outputs to canonical session paths.
|
||||||
|
- Archive uploads run records under `runs/{run_id}/`, applies promotion rules, then publishes `current/manifest.json` and `current/run_id.txt`.
|
||||||
|
- Analyze input resolution uses centralized artifact IDs with alias support.
|
||||||
|
- Forced upstream reruns mark downstream succeeded stages `stale` so later runs do not skip stale outputs.
|
||||||
|
|
||||||
|
## Section 4 Sequence Status
|
||||||
|
|
||||||
|
### Step 1: Campaign-aware session path model
|
||||||
|
|
||||||
|
Status: complete.
|
||||||
|
|
||||||
|
Implemented:
|
||||||
|
|
||||||
|
- Campaign-aware session and run path helpers.
|
||||||
|
- Campaign-aware artifact-store layout APIs.
|
||||||
|
- Canonical session manifest pathing under `work/{campaign}/{session}`.
|
||||||
|
|
||||||
|
### Step 2: Session manifest + run manifest scaffolding
|
||||||
|
|
||||||
|
Status: complete.
|
||||||
|
|
||||||
|
Implemented:
|
||||||
|
|
||||||
|
- Invocation-scoped run manifest type and store methods.
|
||||||
|
- Runner creates/saves run manifests per invocation.
|
||||||
|
- Session manifest remains authoritative for idempotent stage skipping.
|
||||||
|
|
||||||
|
### Step 3: Run-local stage execution + promotion
|
||||||
|
|
||||||
|
Status: complete.
|
||||||
|
|
||||||
|
Implemented:
|
||||||
|
|
||||||
|
- Run-local stage directory layout under `runs/{run_id}/{stage}`.
|
||||||
|
- Shared helpers for run-local output mapping and promotion to canonical durable paths.
|
||||||
|
- Producer run provenance recorded on durable artifact outputs.
|
||||||
|
|
||||||
|
### Step 4: Archive alignment
|
||||||
|
|
||||||
|
Status: complete.
|
||||||
|
|
||||||
|
Implemented:
|
||||||
|
|
||||||
|
- Canonical run-root/session-root resolution.
|
||||||
|
- Deterministic run-file collection and promotion source resolution.
|
||||||
|
- Current-pointer publication ordering retained (`current/manifest.json` then `current/run_id.txt`).
|
||||||
|
|
||||||
|
### Step 5: Artifact registry/resolver (analyze first consumer)
|
||||||
|
|
||||||
|
Status: complete.
|
||||||
|
|
||||||
|
Implemented:
|
||||||
|
|
||||||
|
- Central artifact resolver with canonical IDs:
|
||||||
|
- `narratio.transcript.merged`
|
||||||
|
- `narratio.transcript.polished`
|
||||||
|
- `narratio.transcript.full`
|
||||||
|
- `narratio.transcript.trimmed`
|
||||||
|
- `narratio.bounds.session`
|
||||||
|
- `narratio.artifact.session_recap`
|
||||||
|
- Backward-compatible aliases:
|
||||||
|
- `processed_transcript`
|
||||||
|
- `normalized_transcript`
|
||||||
|
- `trimmed_transcript`
|
||||||
|
- Analyze stage switched to resolver-based source resolution.
|
||||||
|
|
||||||
|
### Step 6: Minimal downstream invalidation for forced reruns
|
||||||
|
|
||||||
|
Status: complete.
|
||||||
|
|
||||||
|
Implemented:
|
||||||
|
|
||||||
|
- Deterministic downstream invalidation based on canonical stage order.
|
||||||
|
- On forced successful rerun of stage `X`, downstream succeeded stages are marked `stale`.
|
||||||
|
- Resume and non-forced runs naturally re-execute stale stages.
|
||||||
|
|
||||||
|
### Step 7: Legacy layout migration strategy
|
||||||
|
|
||||||
|
Status: intentionally skipped.
|
||||||
|
|
||||||
|
Decision:
|
||||||
|
|
||||||
|
- Automatic migration and legacy fallback compatibility are intentionally not implemented.
|
||||||
|
- The codebase targets canonical-only local layout behavior.
|
||||||
|
- Legacy local workspace state, if present, should be recreated or migrated manually outside Narratio.
|
||||||
|
|
||||||
|
## Remaining Work (v1.0)
|
||||||
|
|
||||||
|
No required workspace/run-history migration steps remain from Section 4.
|
||||||
|
|
||||||
|
Possible future enhancements (non-blocking):
|
||||||
|
|
||||||
|
- Full checksum/input-graph stale detection.
|
||||||
|
- Optional retention-policy expansion for run-history cleanup.
|
||||||
|
- Broader artifact-resolver adoption across additional stage consumers.
|
||||||
744
docs/development/workspace.md
Normal file
744
docs/development/workspace.md
Normal file
@@ -0,0 +1,744 @@
|
|||||||
|
# Narratio Workspace, Run History, and Artifact Resolution Architecture
|
||||||
|
|
||||||
|
## 1. Purpose
|
||||||
|
|
||||||
|
This document defines the intended v1.0 architecture for Narratio's local workspace layout, run history model, durable session outputs, manifest responsibilities, and artifact resolution contract.
|
||||||
|
|
||||||
|
Narratio is an idempotent session orchestrator. The command:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio run --session-id 2026-05-07
|
||||||
|
```
|
||||||
|
|
||||||
|
means "bring the identified session to its desired completed state." It does **not** mean "always create an entirely new independent output tree and ignore prior session state."
|
||||||
|
|
||||||
|
This distinction drives the architecture:
|
||||||
|
|
||||||
|
* A **session** is the durable domain object and idempotency boundary.
|
||||||
|
* A **run** is an execution attempt that may update the session's durable state.
|
||||||
|
* Durable outputs live at the session level.
|
||||||
|
* Run-specific outputs, logs, generated configs, scratch files, and diagnostics live under `runs/{run_id}/`.
|
||||||
|
* Successful stage outputs are promoted from run-local locations into canonical session-level locations.
|
||||||
|
* The session manifest records current durable state.
|
||||||
|
* Run manifests record execution history and debugging/provenance details.
|
||||||
|
|
||||||
|
This model intentionally mirrors the S3 archive model: session-level current artifacts are distinct from run-record history.
|
||||||
|
|
||||||
|
## 2. Core Concepts
|
||||||
|
|
||||||
|
### 2.1 Session
|
||||||
|
|
||||||
|
A session is the stable unit of work identified by `campaign_id` and `session_id`.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
|
||||||
|
```text
|
||||||
|
campaign_id = dilfs
|
||||||
|
session_id = 2026-05-07
|
||||||
|
```
|
||||||
|
|
||||||
|
The session directory represents the current durable local state for that session. Re-running Narratio for the same session should consult this state, skip already-completed stages by default, and produce no changes unless work is incomplete, stale, forced, or explicitly selected.
|
||||||
|
|
||||||
|
### 2.2 Run
|
||||||
|
|
||||||
|
A run is a particular execution attempt identified by a generated `run_id`, for example:
|
||||||
|
|
||||||
|
```text
|
||||||
|
20260517T174748Z-abcd1234
|
||||||
|
```
|
||||||
|
|
||||||
|
A run may execute all stages or only a sparse subset of stages. Sparse runs are expected and desirable when the user invokes `--force`, `run-stage`, or a stage-limited command.
|
||||||
|
|
||||||
|
Run directories are provenance/debug records. They should reflect what actually happened during that invocation, not a synthetic complete pipeline layout.
|
||||||
|
|
||||||
|
### 2.3 Durable Output
|
||||||
|
|
||||||
|
A durable output is a canonical session-level artifact intended for later stages, user consumption, archive promotion, or future idempotency decisions.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
|
||||||
|
```text
|
||||||
|
transcripts/merged.json
|
||||||
|
transcripts/processed.json
|
||||||
|
transcripts/normalized.json
|
||||||
|
transcripts/trimmed.json
|
||||||
|
artifacts/session_recap.md
|
||||||
|
```
|
||||||
|
|
||||||
|
Durable outputs live directly under the session directory, not under a particular run directory.
|
||||||
|
|
||||||
|
### 2.4 Run-Local Output
|
||||||
|
|
||||||
|
A run-local output is the file initially produced by a stage during a specific run. After validation, durable outputs are promoted from run-local paths to session-level canonical paths.
|
||||||
|
|
||||||
|
Run-local outputs, logs, generated configs, reports, and scratch files should remain under:
|
||||||
|
|
||||||
|
```text
|
||||||
|
runs/{run_id}/{stage}/...
|
||||||
|
```
|
||||||
|
|
||||||
|
## 3. Local Workspace Layout
|
||||||
|
|
||||||
|
The canonical local workspace layout is:
|
||||||
|
|
||||||
|
```text
|
||||||
|
{workspace.root}/work/{campaign_id}/{session_id}/
|
||||||
|
manifest.json
|
||||||
|
current/
|
||||||
|
manifest.json
|
||||||
|
run_id.txt
|
||||||
|
inputs/
|
||||||
|
transcripts/
|
||||||
|
artifacts/
|
||||||
|
reports/
|
||||||
|
logs/
|
||||||
|
config/
|
||||||
|
runs/
|
||||||
|
{run_id}/
|
||||||
|
manifest.json
|
||||||
|
prepare/
|
||||||
|
transcribe/
|
||||||
|
merge/
|
||||||
|
polish/
|
||||||
|
normalize/
|
||||||
|
trim/
|
||||||
|
analyze/
|
||||||
|
archive/
|
||||||
|
notify/
|
||||||
|
```
|
||||||
|
|
||||||
|
Not every directory must exist at all times. Directories should be created idempotently when needed.
|
||||||
|
|
||||||
|
### 3.1 Session Root
|
||||||
|
|
||||||
|
The session root is:
|
||||||
|
|
||||||
|
```text
|
||||||
|
{workspace.root}/work/{campaign_id}/{session_id}/
|
||||||
|
```
|
||||||
|
|
||||||
|
The session root is the stable local home for the session. It is the default base for resolving canonical artifact paths.
|
||||||
|
|
||||||
|
The only files that should live directly in the session root are core session-state files, primarily:
|
||||||
|
|
||||||
|
```text
|
||||||
|
manifest.json
|
||||||
|
```
|
||||||
|
|
||||||
|
Lock files may also be session-root scoped if the implementation uses file locks there, but transient locks should not be treated as durable artifacts.
|
||||||
|
|
||||||
|
### 3.2 Session-Level Canonical Directories
|
||||||
|
|
||||||
|
The following directories contain current durable session state:
|
||||||
|
|
||||||
|
```text
|
||||||
|
inputs/
|
||||||
|
transcripts/
|
||||||
|
artifacts/
|
||||||
|
reports/
|
||||||
|
logs/
|
||||||
|
config/
|
||||||
|
current/
|
||||||
|
```
|
||||||
|
|
||||||
|
Recommended meanings:
|
||||||
|
|
||||||
|
| Directory | Purpose |
|
||||||
|
| -------------- | ----------------------------------------------------------------------------- |
|
||||||
|
| `inputs/` | Materialized or copied input files used by the current durable session state. |
|
||||||
|
| `transcripts/` | Canonical transcript tiers. |
|
||||||
|
| `artifacts/` | User-facing and machine-readable generated artifacts. |
|
||||||
|
| `reports/` | Canonical stage reports worth preserving at the session level. |
|
||||||
|
| `logs/` | Optional session-level logs or promoted/latest logs. |
|
||||||
|
| `config/` | Optional session-level generated config snapshots or promoted/latest configs. |
|
||||||
|
| `current/` | Current published session pointers, mirroring the archive backend. |
|
||||||
|
|
||||||
|
Canonical durable outputs should use stable paths under these directories.
|
||||||
|
|
||||||
|
### 3.3 Run History Directory
|
||||||
|
|
||||||
|
Run history lives under:
|
||||||
|
|
||||||
|
```text
|
||||||
|
{workspace.root}/work/{campaign_id}/{session_id}/runs/{run_id}/
|
||||||
|
```
|
||||||
|
|
||||||
|
Each run directory records what happened during that invocation. A run may contain all stage directories or only a sparse subset.
|
||||||
|
|
||||||
|
Example full run:
|
||||||
|
|
||||||
|
```text
|
||||||
|
runs/20260517T174748Z-abcd1234/
|
||||||
|
manifest.json
|
||||||
|
prepare/
|
||||||
|
transcribe/
|
||||||
|
merge/
|
||||||
|
polish/
|
||||||
|
normalize/
|
||||||
|
trim/
|
||||||
|
analyze/
|
||||||
|
archive/
|
||||||
|
notify/
|
||||||
|
```
|
||||||
|
|
||||||
|
Example sparse forced analyze run:
|
||||||
|
|
||||||
|
```text
|
||||||
|
runs/20260518T030000Z-efgh5678/
|
||||||
|
manifest.json
|
||||||
|
analyze/
|
||||||
|
```
|
||||||
|
|
||||||
|
Example sparse polish-through-analyze rerun:
|
||||||
|
|
||||||
|
```text
|
||||||
|
runs/20260518T041500Z-a1b2c3d4/
|
||||||
|
manifest.json
|
||||||
|
polish/
|
||||||
|
normalize/
|
||||||
|
trim/
|
||||||
|
analyze/
|
||||||
|
```
|
||||||
|
|
||||||
|
Run directories should not create stage folders for stages that were not selected, executed, skipped, or otherwise considered during that run unless there is a clear diagnostic reason to do so.
|
||||||
|
|
||||||
|
### 3.4 Stage Run-Local Directories
|
||||||
|
|
||||||
|
Each stage receives a run-local directory:
|
||||||
|
|
||||||
|
```text
|
||||||
|
runs/{run_id}/{stage}/
|
||||||
|
```
|
||||||
|
|
||||||
|
Within that stage directory, the stage may use subdirectories such as:
|
||||||
|
|
||||||
|
```text
|
||||||
|
outputs/
|
||||||
|
logs/
|
||||||
|
reports/
|
||||||
|
config/
|
||||||
|
scratch/
|
||||||
|
```
|
||||||
|
|
||||||
|
For example:
|
||||||
|
|
||||||
|
```text
|
||||||
|
runs/{run_id}/polish/
|
||||||
|
outputs/transcripts/processed.json
|
||||||
|
reports/audita.polish.report.json
|
||||||
|
logs/stdout.log
|
||||||
|
logs/stderr.log
|
||||||
|
config/audita.polish.generated.yml
|
||||||
|
scratch/
|
||||||
|
```
|
||||||
|
|
||||||
|
The exact internal layout of a stage directory may vary by stage, but it should be deterministic, documented, and generated through centralized path helpers rather than ad hoc path joins.
|
||||||
|
|
||||||
|
## 4. Promotion Model
|
||||||
|
|
||||||
|
Narratio uses stage-level promotion with immediate promotion after successful validation.
|
||||||
|
|
||||||
|
The stage lifecycle is:
|
||||||
|
|
||||||
|
1. Resolve required inputs from the current session state and/or run-local context.
|
||||||
|
2. Create the run-local stage directory.
|
||||||
|
3. Execute the stage, writing outputs under `runs/{run_id}/{stage}/...`.
|
||||||
|
4. Validate run-local outputs.
|
||||||
|
5. Promote durable outputs into session-level canonical paths.
|
||||||
|
6. Update the session manifest.
|
||||||
|
7. Update the run manifest.
|
||||||
|
|
||||||
|
Promotion means an atomic or effectively atomic copy/rename from a run-local path to a session-level canonical path.
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
```text
|
||||||
|
runs/{run_id}/polish/outputs/transcripts/processed.json
|
||||||
|
```
|
||||||
|
|
||||||
|
is promoted to:
|
||||||
|
|
||||||
|
```text
|
||||||
|
transcripts/processed.json
|
||||||
|
```
|
||||||
|
|
||||||
|
Promotion should be safe and deterministic:
|
||||||
|
|
||||||
|
* Validate before promotion.
|
||||||
|
* Write promoted files atomically where possible.
|
||||||
|
* Never leave partially written durable outputs.
|
||||||
|
* Record the producing `run_id` in the session manifest.
|
||||||
|
* Preserve run-local files for debugging unless retention policy deletes them.
|
||||||
|
|
||||||
|
## 5. Promotion Policy: Option A
|
||||||
|
|
||||||
|
Narratio uses immediate stage-level promotion.
|
||||||
|
|
||||||
|
If a selected stage succeeds, its durable outputs are promoted immediately, even if a later selected stage fails.
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio run --session-id 2026-05-07 --force --stages polish,normalize,trim,analyze
|
||||||
|
```
|
||||||
|
|
||||||
|
If `polish` succeeds and `normalize` fails:
|
||||||
|
|
||||||
|
* `transcripts/processed.json` may be updated from the new run.
|
||||||
|
* `normalize`, `trim`, and `analyze` should not be marked succeeded for the new input state.
|
||||||
|
* Downstream outputs may now be stale relative to the newly promoted polished transcript.
|
||||||
|
|
||||||
|
This policy is simpler, transparent, and consistent with stage-level resumability. It does require explicit stale/invalidation handling.
|
||||||
|
|
||||||
|
## 6. Stale and Invalidation Semantics
|
||||||
|
|
||||||
|
Full checksum-based stale detection may be implemented later. Before that exists, Narratio should still use a simple deterministic invalidation rule for forced or explicit upstream reruns.
|
||||||
|
|
||||||
|
When a stage is successfully re-executed and promoted, downstream stages should be marked stale unless they are also re-executed successfully in the same command invocation.
|
||||||
|
|
||||||
|
Example stage order:
|
||||||
|
|
||||||
|
```text
|
||||||
|
prepare -> transcribe -> merge -> polish -> normalize -> trim -> analyze -> archive -> notify
|
||||||
|
```
|
||||||
|
|
||||||
|
If `polish` is forced and promoted, then the following downstream stages should be invalidated unless rerun successfully:
|
||||||
|
|
||||||
|
```text
|
||||||
|
normalize
|
||||||
|
trim
|
||||||
|
analyze
|
||||||
|
archive
|
||||||
|
notify
|
||||||
|
```
|
||||||
|
|
||||||
|
A stale stage is not equivalent to a failed stage. It means its current durable outputs may no longer correspond to current upstream inputs or configuration.
|
||||||
|
|
||||||
|
Minimum manifest state model:
|
||||||
|
|
||||||
|
```text
|
||||||
|
pending
|
||||||
|
running
|
||||||
|
succeeded
|
||||||
|
failed
|
||||||
|
skipped
|
||||||
|
stale
|
||||||
|
```
|
||||||
|
|
||||||
|
If adding a new `stale` state is too invasive for v1.0, the implementation should at least record stale metadata or clear downstream success markers in a way that prevents accidental idempotent skips based on obsolete outputs.
|
||||||
|
|
||||||
|
## 7. Manifest Responsibilities
|
||||||
|
|
||||||
|
Narratio should distinguish between session manifests and run manifests.
|
||||||
|
|
||||||
|
The same underlying Go types may be reused where practical, but the concepts should remain separate.
|
||||||
|
|
||||||
|
### 7.1 Session Manifest
|
||||||
|
|
||||||
|
Path:
|
||||||
|
|
||||||
|
```text
|
||||||
|
{workspace.root}/work/{campaign_id}/{session_id}/manifest.json
|
||||||
|
```
|
||||||
|
|
||||||
|
The session manifest answers:
|
||||||
|
|
||||||
|
```text
|
||||||
|
What is the current durable state of this session?
|
||||||
|
```
|
||||||
|
|
||||||
|
It should record:
|
||||||
|
|
||||||
|
* campaign ID
|
||||||
|
* session ID
|
||||||
|
* current or latest run ID
|
||||||
|
* current stage states
|
||||||
|
* canonical durable output refs
|
||||||
|
* artifact IDs and paths
|
||||||
|
* producing run ID for each current stage output
|
||||||
|
* relevant input/config checksums when available
|
||||||
|
* stale/invalidated stage information
|
||||||
|
* archive/current publication metadata
|
||||||
|
|
||||||
|
A session's durable state may be a composite of multiple runs.
|
||||||
|
|
||||||
|
For example:
|
||||||
|
|
||||||
|
```text
|
||||||
|
transcripts/merged.json produced by run A
|
||||||
|
transcripts/processed.json produced by run B
|
||||||
|
transcripts/normalized.json produced by run B
|
||||||
|
transcripts/trimmed.json produced by run B
|
||||||
|
artifacts/session_recap.md produced by run C
|
||||||
|
```
|
||||||
|
|
||||||
|
This is valid and expected.
|
||||||
|
|
||||||
|
### 7.2 Run Manifest
|
||||||
|
|
||||||
|
Path:
|
||||||
|
|
||||||
|
```text
|
||||||
|
{workspace.root}/work/{campaign_id}/{session_id}/runs/{run_id}/manifest.json
|
||||||
|
```
|
||||||
|
|
||||||
|
The run manifest answers:
|
||||||
|
|
||||||
|
```text
|
||||||
|
What happened during this specific execution attempt?
|
||||||
|
```
|
||||||
|
|
||||||
|
It should record:
|
||||||
|
|
||||||
|
* run ID
|
||||||
|
* campaign ID
|
||||||
|
* session ID
|
||||||
|
* command mode and selected stages
|
||||||
|
* force flags or stage selection flags
|
||||||
|
* stages considered during this run
|
||||||
|
* stages executed during this run
|
||||||
|
* stages skipped during this run and reasons
|
||||||
|
* run-local output paths
|
||||||
|
* promoted output paths
|
||||||
|
* logs
|
||||||
|
* reports
|
||||||
|
* generated configs
|
||||||
|
* timings
|
||||||
|
* errors
|
||||||
|
* non-secret subprocess invocation metadata
|
||||||
|
|
||||||
|
Run manifests are primarily for debugging, auditability, and archive history.
|
||||||
|
|
||||||
|
## 8. Idempotency and Resume Behavior
|
||||||
|
|
||||||
|
The idempotency boundary is the session, not the run.
|
||||||
|
|
||||||
|
By default:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio run --session-id 2026-05-07
|
||||||
|
```
|
||||||
|
|
||||||
|
should consult the session manifest and skip stages that are already succeeded and not stale.
|
||||||
|
|
||||||
|
If all stages are already complete, the command should execute zero stages and report that the session is already complete.
|
||||||
|
|
||||||
|
Forced execution creates a new run record but updates session-level durable state only for stages that actually succeed and promote outputs.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio run --session-id 2026-05-07 --force
|
||||||
|
```
|
||||||
|
|
||||||
|
Creates a new run and attempts to re-execute the selected/default stage set.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio run-stage --session-id 2026-05-07 analyze --force
|
||||||
|
```
|
||||||
|
|
||||||
|
Creates a sparse run that executes only `analyze`, then promotes updated analysis artifacts if successful.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio resume --session-id 2026-05-07
|
||||||
|
```
|
||||||
|
|
||||||
|
Uses the session manifest to determine what remains incomplete or stale. Resume does not need to resume the same `run_id` unless the implementation explicitly supports resuming an interrupted active run.
|
||||||
|
|
||||||
|
## 9. Artifact Resolution Contract
|
||||||
|
|
||||||
|
Narratio should provide a first-class artifact registry and resolver.
|
||||||
|
|
||||||
|
The resolver maps symbolic artifact source names to canonical session-level paths and manifest output kinds.
|
||||||
|
|
||||||
|
Stages and adapters should not hardcode path fragments when resolving cross-stage inputs. They should ask the artifact resolver for the current durable artifact by ID.
|
||||||
|
|
||||||
|
### 9.1 Canonical Artifact IDs
|
||||||
|
|
||||||
|
Preferred artifact IDs should be namespaced:
|
||||||
|
|
||||||
|
```text
|
||||||
|
narratio.transcript.merged
|
||||||
|
narratio.transcript.polished
|
||||||
|
narratio.transcript.full
|
||||||
|
narratio.transcript.trimmed
|
||||||
|
narratio.bounds.session
|
||||||
|
narratio.artifact.session_recap
|
||||||
|
```
|
||||||
|
|
||||||
|
Recommended initial registry:
|
||||||
|
|
||||||
|
| Artifact ID | Canonical Path | Producer Stage | Output Kind | Meaning |
|
||||||
|
| --------------------------------- | ------------------------------- | -------------- | ------------------------ | ------------------------------------- |
|
||||||
|
| `narratio.transcript.merged` | `transcripts/merged.json` | `merge` | `transcript_merged` | Deterministic Seriatim merge. |
|
||||||
|
| `narratio.transcript.polished` | `transcripts/processed.json` | `polish` | `transcript_processed` | Full Audita-polished transcript. |
|
||||||
|
| `narratio.transcript.full` | `transcripts/normalized.json` | `normalize` | `transcript_normalized` | Preferred full normalized transcript. |
|
||||||
|
| `narratio.transcript.trimmed` | `transcripts/trimmed.json` | `trim` | `transcript_trimmed` | Gameplay-only transcript. |
|
||||||
|
| `narratio.bounds.session` | `artifacts/session_bounds.json` | `trim` | `session_bounds` | Trim bounds selected for the session. |
|
||||||
|
| `narratio.artifact.session_recap` | `artifacts/session_recap.md` | `analyze` | `artifact_session_recap` | Generated session recap. |
|
||||||
|
|
||||||
|
### 9.2 Backward-Compatible Aliases
|
||||||
|
|
||||||
|
Existing source names should remain supported:
|
||||||
|
|
||||||
|
| Legacy Source | Preferred Artifact ID |
|
||||||
|
| ----------------------- | ------------------------------ |
|
||||||
|
| `processed_transcript` | `narratio.transcript.polished` |
|
||||||
|
| `normalized_transcript` | `narratio.transcript.full` |
|
||||||
|
| `trimmed_transcript` | `narratio.transcript.trimmed` |
|
||||||
|
|
||||||
|
These aliases may be supported silently for v1.0. Documentation should prefer namespaced IDs.
|
||||||
|
|
||||||
|
### 9.3 Resolver Behavior
|
||||||
|
|
||||||
|
Artifact resolution should follow this order:
|
||||||
|
|
||||||
|
1. Normalize aliases to canonical artifact IDs.
|
||||||
|
2. Look for a current output reference in the session manifest.
|
||||||
|
3. Fall back to the canonical session-level path.
|
||||||
|
4. If the artifact is required, fail clearly if missing.
|
||||||
|
5. If the artifact is optional and missing, omit it from the downstream invocation.
|
||||||
|
6. Validate the artifact using the expected content validator.
|
||||||
|
7. Return a resolved artifact record containing ID, path, producer stage, output kind, and provenance.
|
||||||
|
|
||||||
|
Example conceptual result:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"id": "narratio.transcript.trimmed",
|
||||||
|
"path": "/var/lib/narratio/work/dilfs/2026-05-07/transcripts/trimmed.json",
|
||||||
|
"producer_stage": "trim",
|
||||||
|
"producer_run_id": "20260517T174748Z-abcd1234",
|
||||||
|
"output_kind": "transcript_trimmed",
|
||||||
|
"content_type": "application/json"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 9.4 Artifact Validation
|
||||||
|
|
||||||
|
Transcript artifacts must be valid JSON with a top-level `segments` array.
|
||||||
|
|
||||||
|
Markdown/text artifacts must exist and be non-empty when required.
|
||||||
|
|
||||||
|
Bounds artifacts must match the expected bounds schema and refer to segment IDs in the same transcript ID space used by the trim stage.
|
||||||
|
|
||||||
|
Validation should happen before a resolved artifact is passed to another stage or external subprocess.
|
||||||
|
|
||||||
|
## 10. Analyze Stage Implications
|
||||||
|
|
||||||
|
The analyze stage should consume artifacts through the artifact resolver.
|
||||||
|
|
||||||
|
Preferred Scriptorium config shape:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
scriptorium:
|
||||||
|
artifacts:
|
||||||
|
session_recap:
|
||||||
|
enabled: true
|
||||||
|
prompt_id: "dnd.session_recap"
|
||||||
|
output_path: "artifacts/session_recap.md"
|
||||||
|
inputs:
|
||||||
|
transcript:
|
||||||
|
source: "narratio.transcript.trimmed"
|
||||||
|
required: true
|
||||||
|
```
|
||||||
|
|
||||||
|
Additional artifacts can choose different transcript tiers:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
scriptorium:
|
||||||
|
artifacts:
|
||||||
|
table_summary:
|
||||||
|
enabled: true
|
||||||
|
prompt_id: "dnd.table_summary"
|
||||||
|
output_path: "artifacts/table_summary.md"
|
||||||
|
inputs:
|
||||||
|
transcript:
|
||||||
|
source: "narratio.transcript.full"
|
||||||
|
required: true
|
||||||
|
```
|
||||||
|
|
||||||
|
For v1.0, Narratio does not need a generic DAG engine. It may execute configured analyze artifacts in deterministic order and allow later artifacts to consume earlier artifacts only when that relationship is explicit and unambiguous.
|
||||||
|
|
||||||
|
Rules:
|
||||||
|
|
||||||
|
* Artifact inputs resolve from current session-level durable state.
|
||||||
|
* Outputs are first written run-locally.
|
||||||
|
* Successful analyze outputs are promoted to session-level `artifacts/` paths.
|
||||||
|
* Manifest output refs record the producing run ID.
|
||||||
|
* Optional inputs are omitted when unavailable.
|
||||||
|
* Required missing inputs fail before invoking Scriptorium.
|
||||||
|
|
||||||
|
## 11. Archive Alignment
|
||||||
|
|
||||||
|
Local workspace semantics should mirror archive semantics.
|
||||||
|
|
||||||
|
Local session-level durable paths:
|
||||||
|
|
||||||
|
```text
|
||||||
|
work/{campaign}/{session}/transcripts/trimmed.json
|
||||||
|
work/{campaign}/{session}/artifacts/session_recap.md
|
||||||
|
work/{campaign}/{session}/current/manifest.json
|
||||||
|
work/{campaign}/{session}/current/run_id.txt
|
||||||
|
work/{campaign}/{session}/runs/{run_id}/...
|
||||||
|
```
|
||||||
|
|
||||||
|
should map naturally to remote archive paths:
|
||||||
|
|
||||||
|
```text
|
||||||
|
{root_prefix}/campaigns/{campaign}/sessions/{session}/transcripts/trimmed.json
|
||||||
|
{root_prefix}/campaigns/{campaign}/sessions/{session}/artifacts/session_recap.md
|
||||||
|
{root_prefix}/campaigns/{campaign}/sessions/{session}/current/manifest.json
|
||||||
|
{root_prefix}/campaigns/{campaign}/sessions/{session}/current/run_id.txt
|
||||||
|
{root_prefix}/campaigns/{campaign}/sessions/{session}/runs/{run_id}/...
|
||||||
|
```
|
||||||
|
|
||||||
|
The archive stage should publish run records and promoted current artifacts consistently with the local model.
|
||||||
|
|
||||||
|
`current/run_id.txt` remains the effective commit marker for the archived current session state.
|
||||||
|
|
||||||
|
## 12. Path Helper Requirements
|
||||||
|
|
||||||
|
All code should use centralized path helpers for workspace paths.
|
||||||
|
|
||||||
|
Stage code should not manually assemble durable cross-stage paths using raw string joins except through the path model.
|
||||||
|
|
||||||
|
Recommended helper surface:
|
||||||
|
|
||||||
|
```text
|
||||||
|
SessionRoot(campaignID, sessionID)
|
||||||
|
SessionManifestPath(campaignID, sessionID)
|
||||||
|
SessionCurrentDir(campaignID, sessionID)
|
||||||
|
SessionTranscriptsDir(campaignID, sessionID)
|
||||||
|
SessionArtifactsDir(campaignID, sessionID)
|
||||||
|
SessionReportsDir(campaignID, sessionID)
|
||||||
|
SessionLogsDir(campaignID, sessionID)
|
||||||
|
SessionConfigDir(campaignID, sessionID)
|
||||||
|
RunsDir(campaignID, sessionID)
|
||||||
|
RunRoot(campaignID, sessionID, runID)
|
||||||
|
RunManifestPath(campaignID, sessionID, runID)
|
||||||
|
RunStageDir(campaignID, sessionID, runID, stage)
|
||||||
|
RunStageOutputsDir(campaignID, sessionID, runID, stage)
|
||||||
|
RunStageLogsDir(campaignID, sessionID, runID, stage)
|
||||||
|
RunStageReportsDir(campaignID, sessionID, runID, stage)
|
||||||
|
RunStageConfigDir(campaignID, sessionID, runID, stage)
|
||||||
|
CanonicalArtifactPath(campaignID, sessionID, artifactID)
|
||||||
|
```
|
||||||
|
|
||||||
|
Path helpers should enforce safe relative paths for configured output paths:
|
||||||
|
|
||||||
|
* reject absolute paths unless explicitly allowed for a particular config field
|
||||||
|
* reject `..` traversal
|
||||||
|
* normalize separators
|
||||||
|
* preserve deterministic output paths
|
||||||
|
|
||||||
|
## 13. Directory Creation Policy
|
||||||
|
|
||||||
|
Directory creation should be centralized and idempotent.
|
||||||
|
|
||||||
|
Recommended policy:
|
||||||
|
|
||||||
|
* `prepare` ensures the baseline session directory structure exists.
|
||||||
|
* Every stage also calls shared layout helpers to ensure its required run-local directories exist before writing.
|
||||||
|
* `run-stage` should not depend on a prior `prepare` invocation merely to create folders.
|
||||||
|
* Missing directories should be created with appropriate permissions.
|
||||||
|
* Directory creation should not imply stage success.
|
||||||
|
|
||||||
|
This provides consistent layout while keeping direct stage execution robust.
|
||||||
|
|
||||||
|
## 14. Cleanup and Retention
|
||||||
|
|
||||||
|
Cleanup must preserve the distinction between durable session state and run history.
|
||||||
|
|
||||||
|
Workspace cleanup after successful archive may remove selected local directories only according to explicit configuration.
|
||||||
|
|
||||||
|
Potential retention policies:
|
||||||
|
|
||||||
|
```text
|
||||||
|
keep_all_runs
|
||||||
|
keep_failed_runs
|
||||||
|
keep_last_n_runs
|
||||||
|
delete_run_after_success
|
||||||
|
```
|
||||||
|
|
||||||
|
For v1.0, conservative retention is preferred:
|
||||||
|
|
||||||
|
* Do not delete durable session-level outputs unless explicitly requested.
|
||||||
|
* Do not delete failed run directories by default.
|
||||||
|
* If cleanup is enabled, remove only documented run-scoped or spool-scoped paths.
|
||||||
|
* Local development audio inputs must never be deleted by workspace cleanup.
|
||||||
|
|
||||||
|
## 15. Canonical-Only Layout Policy
|
||||||
|
|
||||||
|
Narratio now supports only the canonical campaign-aware layout:
|
||||||
|
|
||||||
|
```text
|
||||||
|
{workspace.root}/work/{campaign_id}/{session_id}/manifest.json
|
||||||
|
{workspace.root}/work/{campaign_id}/{session_id}/runs/{run_id}/...
|
||||||
|
```
|
||||||
|
|
||||||
|
Legacy session-only layout compatibility is intentionally not implemented.
|
||||||
|
|
||||||
|
If legacy workspace data exists, operators should recreate or manually migrate that data outside Narratio before running v1.0 commands.
|
||||||
|
|
||||||
|
## 16. Documentation Updates Required
|
||||||
|
|
||||||
|
The following documentation should be updated to reflect this architecture:
|
||||||
|
|
||||||
|
* `README.md`
|
||||||
|
* `docs/architecture.md`
|
||||||
|
* a dedicated workspace/run-history document, such as this file
|
||||||
|
* S3/archive documentation
|
||||||
|
* analyze/artifact configuration documentation
|
||||||
|
* example pipeline files
|
||||||
|
|
||||||
|
Documentation should consistently use the following terms:
|
||||||
|
|
||||||
|
| Term | Meaning |
|
||||||
|
| ---------------- | ---------------------------------------------------------------------- |
|
||||||
|
| Session | Durable domain object and idempotency boundary. |
|
||||||
|
| Run | Execution attempt that may update session state. |
|
||||||
|
| Durable output | Canonical current session-level output. |
|
||||||
|
| Run-local output | Output produced inside a specific run directory before promotion. |
|
||||||
|
| Promotion | Validated copy/rename from run-local output to durable session output. |
|
||||||
|
| Session manifest | Current durable state of the session. |
|
||||||
|
| Run manifest | Execution record for a particular run. |
|
||||||
|
| Artifact ID | Symbolic source name resolved by the artifact registry. |
|
||||||
|
|
||||||
|
## 17. Architectural Invariants
|
||||||
|
|
||||||
|
The following invariants should hold after implementation:
|
||||||
|
|
||||||
|
1. `session_id` remains the idempotency boundary for normal operator commands.
|
||||||
|
2. `run_id` identifies an execution attempt, not the primary durable workspace.
|
||||||
|
3. Session-level canonical artifacts are the default inputs for downstream stages.
|
||||||
|
4. Run-local outputs are promoted only after validation.
|
||||||
|
5. A session's current durable state may be composed of outputs from multiple runs.
|
||||||
|
6. Sparse run directories are valid and expected.
|
||||||
|
7. The session manifest records current stage/artifact state and producer run IDs.
|
||||||
|
8. The run manifest records what happened during one invocation.
|
||||||
|
9. Artifact consumers resolve symbolic artifact IDs through a registry/resolver.
|
||||||
|
10. Local workspace semantics mirror S3 archive semantics.
|
||||||
|
11. Directory creation is centralized and idempotent.
|
||||||
|
12. Stage code uses path helpers rather than ad hoc path construction.
|
||||||
|
13. Forced upstream reruns invalidate downstream stage success unless downstream stages are rerun successfully.
|
||||||
|
14. Cleanup never removes durable session outputs or local development inputs unless explicitly configured to do so.
|
||||||
|
|
||||||
|
## 18. Implementation Guidance
|
||||||
|
|
||||||
|
A practical implementation sequence is:
|
||||||
|
|
||||||
|
1. Add this architecture document.
|
||||||
|
2. Add or revise path model helpers for session roots, run roots, stage directories, and canonical artifact paths.
|
||||||
|
3. Introduce session manifest versus run manifest concepts.
|
||||||
|
4. Route stage outputs through run-local directories.
|
||||||
|
5. Add promotion helpers with validation and atomic writes.
|
||||||
|
6. Update existing stages to promote durable outputs to session-level canonical paths.
|
||||||
|
7. Add artifact registry and resolver.
|
||||||
|
8. Update analyze to use artifact IDs and aliases.
|
||||||
|
9. Add simple downstream stale invalidation for forced upstream reruns.
|
||||||
|
10. Align archive/local path behavior and documentation.
|
||||||
|
11. Update examples and README.
|
||||||
|
12. Add tests for idempotency, sparse forced runs, promotion, manifest provenance, and artifact resolution.
|
||||||
|
|
||||||
|
This sequence intentionally avoids introducing a generic DAG engine. The v1.0 goal is a clear, deterministic, stage-oriented orchestrator with stable session-level outputs and inspectable run history.
|
||||||
@@ -1,147 +1,96 @@
|
|||||||
# Audita
|
# Audita Subprocess Operations
|
||||||
|
|
||||||
Audita is a framework-first transcript correction application. The public `audita` package provides:
|
This document describes how parent processes should invoke `audita process` safely in production orchestration.
|
||||||
|
|
||||||
- deterministic transcript normalization
|
## Recommended command form
|
||||||
- token-batched module orchestration
|
|
||||||
- concrete `glossary`, `homophones`, `spoken_word`, and `grammar` modules built on reusable proposal / validator contracts
|
|
||||||
- structured run reporting and work-dir diagnostics
|
|
||||||
|
|
||||||
The previous working implementation has been preserved as `audita_prototype` inside this repository. Its full regression suite lives under `tests/audita_prototype`.
|
Use explicit file outputs for orchestrated runs:
|
||||||
|
|
||||||
## Development
|
|
||||||
|
|
||||||
This project is set up for `uv`.
|
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
uv sync --extra dev
|
audita process <transcript.json> \
|
||||||
uv run pytest
|
--transcript-description "Brief context that may help resolve ambiguous terms." \
|
||||||
|
--glossary <glossary.yaml> \
|
||||||
|
--output <output-transcript.json> \
|
||||||
|
--report-json <report.json>
|
||||||
```
|
```
|
||||||
|
|
||||||
## Usage
|
Additional flags that may be situationally appropriate:
|
||||||
|
- `--config <path>` to select an explicit versioned config file.
|
||||||
|
- `--output-schema <bare-segments|audita-v1>` to select transcript output shape.
|
||||||
|
- `--work-dir <dir>` to control diagnostics location.
|
||||||
|
- `--work-dir-retention <always|auto|never>` to control retained run directories.
|
||||||
|
- `--total-llm-concurrency`, `--proposal-llm-concurrency`, and `--validation-llm-concurrency` when orchestration needs to set explicit LLM throughput controls.
|
||||||
|
- `--modules ...` only when intentionally overriding the default sequence.
|
||||||
|
|
||||||
Process a transcript with the current framework implementation:
|
For config-driven orchestration, validate config files in CI/preflight:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
uv run audita process transcript.json --glossary glossary.yaml --output corrected.json
|
audita config validate --config <path>
|
||||||
```
|
```
|
||||||
|
|
||||||
The framework currently runs this default module sequence:
|
## Stdout behavior
|
||||||
|
|
||||||
1. `glossary`
|
- With `--output`: stdout is expected to be empty on success.
|
||||||
2. `homophones`
|
- Without `--output`: stdout contains transcript JSON only on success.
|
||||||
3. `glossary`
|
- Report JSON is never written to stdout.
|
||||||
4. `spoken_word`
|
|
||||||
5. `grammar`
|
|
||||||
|
|
||||||
Resolved run instance names are auto-numbered for repeats, so the default report pipeline is:
|
## Stderr behavior
|
||||||
|
|
||||||
1. `glossary_1`
|
- Success path should be quiet or minimal human-readable logs.
|
||||||
2. `homophones`
|
- Failure path writes concise human-readable errors.
|
||||||
3. `glossary_2`
|
- When a diagnostics run directory exists, failure stderr includes its path.
|
||||||
4. `spoken_word`
|
- Prompt/response diagnostic payloads are not streamed to stderr.
|
||||||
5. `grammar`
|
|
||||||
|
|
||||||
The default module sequence is fully implemented today:
|
## Output file behavior
|
||||||
|
|
||||||
- `glossary` proposes glossary-supported acoustic corrections
|
- `--output` writes transcript JSON in the selected output schema to the provided path.
|
||||||
- `homophones` proposes conservative homophone and mistranscription corrections
|
- Output write failures return nonzero and surface actionable errors.
|
||||||
- `spoken_word` proposes conservative dysfluency cleanup
|
- The command does not silently ignore output write errors.
|
||||||
- `grammar` proposes punctuation, capitalization, and spacing cleanup only
|
|
||||||
|
|
||||||
To run a custom module sequence, pass `--modules`:
|
## Report JSON behavior
|
||||||
|
|
||||||
```sh
|
- `--report-json` writes a machine-readable process report to the requested path.
|
||||||
uv run audita process transcript.json --glossary glossary.yaml --modules grammar --output corrected.json
|
- Run-directory `report.json` is written independently under diagnostics.
|
||||||
```
|
- Best-effort failure reports are emitted when possible without masking the primary failure.
|
||||||
|
- Report write failures return nonzero with clear stderr messaging.
|
||||||
|
- Report diagnostics metadata references run-directory artifacts including utilization diagnostics and correction ledger paths when available.
|
||||||
|
|
||||||
To also write a structured JSON report:
|
## Diagnostics directory behavior
|
||||||
|
|
||||||
```sh
|
- Each run creates (when possible) a per-run diagnostics directory.
|
||||||
uv run audita process transcript.json --glossary glossary.yaml --output corrected.json --report-json report.json
|
- Typical artifacts include transcript, normalization, chunking, invocation, effective config, LLM diagnostics, `utilization-diagnostics.json`, `correction-ledger.json`, `report.json`, and `error.log` on failure.
|
||||||
```
|
- Failed runs retain diagnostics.
|
||||||
|
- Under `auto` retention, successful runs with skipped/rejected corrections are retained; clean successful runs may be removed.
|
||||||
|
|
||||||
From a checked-out repository, you can also use the root launcher:
|
## Exit codes
|
||||||
|
|
||||||
```sh
|
- `0`: success.
|
||||||
./audita process transcript.json --glossary glossary.yaml --output corrected.json
|
- Nonzero: failure (input/schema/config/module/LLM/runtime/output/report/diagnostics errors).
|
||||||
```
|
|
||||||
|
|
||||||
For a system-wide command, install the source tree under `/usr/local/src/audita`, sync dependencies there, and symlink the root launcher into your `PATH`:
|
Treat any nonzero as a failed subprocess invocation.
|
||||||
|
|
||||||
```sh
|
## Timeout and cancellation
|
||||||
cd /usr/local/src/audita
|
|
||||||
uv sync --extra dev
|
|
||||||
ln -s /usr/local/src/audita/audita /usr/local/bin/audita
|
|
||||||
audita process transcript.json --glossary glossary.yaml --output corrected.json
|
|
||||||
```
|
|
||||||
|
|
||||||
Without `--output`, Audita writes the corrected transcript JSON to stdout and progress logs to stderr.
|
- Runtime operations propagate context cancellation and request timeouts through LLM/scheduler paths.
|
||||||
`--report-json` writes a separate machine-readable run report and never mixes report data into stdout.
|
- On cancellation or timeout, the process exits nonzero and should not hang.
|
||||||
|
- If diagnostics were initialized before failure, failure artifacts remain available for debugging.
|
||||||
|
|
||||||
Useful configuration can be supplied by CLI flag or environment variable. CLI flags take precedence over environment variables. Normal runs now require LLM API credentials, because the `glossary`, `homophones`, `spoken_word`, and `grammar` modules make real LLM calls. `AUDITA_LLM_API_KEY` and `--llm-api-key` are the preferred provider-neutral credential surfaces, while `OPENROUTER_API_KEY` remains supported as a backward-compatible fallback.
|
## Secret redaction expectations
|
||||||
|
|
||||||
| Environment variable | CLI flag | Default | Purpose |
|
API keys and configured secret values are redacted from:
|
||||||
| --- | --- | --- | --- |
|
- reports (`--report-json` and run-dir `report.json`);
|
||||||
| `AUDITA_MODULES` | `--modules` | `glossary,homophones,glossary,spoken_word,grammar` | Comma-separated logical module keys to run; CLI overrides the environment value |
|
- diagnostics artifacts (including effective config and LLM interaction artifacts);
|
||||||
| `AUDITA_LLM_API_KEY` | `--llm-api-key` | unset | Preferred provider-neutral LLM API credential; CLI overrides both environment-key variants |
|
- surfaced adapter/runtime errors;
|
||||||
| `AUDITA_VALIDATION_LLM_API_KEY` | `--validation-llm-api-key` | unset | Validation-phase LLM API credential; defaults to the primary LLM API key |
|
- test fixtures and regression outputs.
|
||||||
| `AUDITA_MODEL` | `--model` | `openrouter/google/gemma-4-31b-it` | LLM model name sent to the configured OpenAI-compatible endpoint |
|
|
||||||
| `AUDITA_VALIDATION_MODEL` | `--validation-model` | unset | Validation-phase LLM model; defaults to `AUDITA_MODEL` |
|
|
||||||
| `AUDITA_BASE_URL` | `--base-url` | `https://openrouter.ai/api/v1` | OpenAI-compatible API base URL |
|
|
||||||
| `AUDITA_VALIDATION_BASE_URL` | `--validation-base-url` | unset | Validation-phase OpenAI-compatible API base URL; defaults to `AUDITA_BASE_URL` |
|
|
||||||
| `AUDITA_LLM_TIMEOUT_SECONDS` | `--llm-timeout-seconds` | `600` | Per-request timeout in seconds for LLM calls to the configured OpenAI-compatible endpoint |
|
|
||||||
| `AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS` | `--validation-llm-timeout-seconds` | unset | Validation-phase per-request timeout in seconds; defaults to `AUDITA_LLM_TIMEOUT_SECONDS` |
|
|
||||||
| `AUDITA_VALIDATION_MAX_PROMPT_TOKENS` | `--validation-max-prompt-tokens` | `2048` | Maximum estimated tokens per validation-phase LLM prompt batch |
|
|
||||||
| `AUDITA_TARGET_SECTIONS` | `--target-sections` | unset | Exact number of contiguous proposal-stage transcript sections; errors if min/max token bounds cannot be satisfied |
|
|
||||||
| `AUDITA_MAX_RETRIES` | `--max-retries` | `3` | Maximum Instructor retries for structured responses |
|
|
||||||
| `AUDITA_VALIDATION_MAX_RETRIES` | `--validation-max-retries` | unset | Validation-phase structured-output retries; defaults to `AUDITA_MAX_RETRIES` |
|
|
||||||
| `AUDITA_VALIDATION_LLM_CONCURRENCY` | `--validation-llm-concurrency` | unset | Validation-phase LLM concurrency; defaults to `AUDITA_LLM_CONCURRENCY` |
|
|
||||||
| `AUDITA_MAX_SECTION_TOKENS` | `--max-section-tokens` | `8192` | Maximum estimated tokens per proposal-stage transcript section |
|
|
||||||
| `AUDITA_MIN_SECTION_TOKENS` | `--min-section-tokens` | `2048` | Minimum estimated tokens per proposal-stage transcript section when balancing for concurrency |
|
|
||||||
| `AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD` | `--glossary-confidence-threshold` | `0.8` | Minimum confidence required for glossary proposals to survive validation |
|
|
||||||
| `AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD` | `--grammar-confidence-threshold` | `0.8` | Minimum confidence required for grammar proposals to survive validation |
|
|
||||||
| `AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD` | `--homophones-confidence-threshold` | `0.8` | Minimum confidence required for homophone proposals to survive validation |
|
|
||||||
| `AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD` | `--spoken-word-confidence-threshold` | `0.8` | Minimum confidence required for spoken-word proposals to survive validation |
|
|
||||||
| `AUDITA_NORMALIZE_MAX_SEGMENT_GAP` | `--normalize-max-segment-gap` | `4.0` | Same-speaker gaps eligible for deterministic merging |
|
|
||||||
| `AUDITA_NORMALIZE_ELLIPSIS_GAP` | `--normalize-ellipsis-gap` | `3.5` | Same-speaker gaps above this value are joined with ` ... ` |
|
|
||||||
| `AUDITA_NORMALIZE_MAX_SEGMENT_DURATION` | `--normalize-max-segment-duration` | `60.0` | Maximum merged segment duration |
|
|
||||||
| `AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS` | `--normalize-max-segment-tokens` | `2048` | Maximum merged segment prompt payload size |
|
|
||||||
| `AUDITA_WORK_DIR` | `--work-dir` | `/tmp/audita` | Per-run scratch diagnostics directory |
|
|
||||||
| `AUDITA_WORK_DIR_RETENTION` | `--work-dir-retention` | `auto` | Whether to retain the per-run work directory: `auto`, `always`, or `never` |
|
|
||||||
|
|
||||||
Set `AUDITA_MODULES=grammar` to run only the grammar module by default, or override it per command with `--modules`.
|
Parent-process logs should still avoid printing raw environment variables.
|
||||||
|
|
||||||
Validation-phase LLM settings inherit from the primary `AUDITA_*` LLM settings by default. Set any of the `AUDITA_VALIDATION_*` values only when you want LLM-backed validators to use a different model, endpoint, credential, timeout, retry budget, or concurrency level.
|
## Parent-process pipe guidance
|
||||||
|
|
||||||
OpenRouter remains the default out of the box:
|
To avoid deadlocks in orchestrators:
|
||||||
|
- always read both stdout and stderr concurrently when invoking as a subprocess;
|
||||||
|
- prefer file outputs (`--output`, `--report-json`) for machine workflows;
|
||||||
|
- treat stderr as human-readable diagnostics, not structured data;
|
||||||
|
- parse structured results from output/report files.
|
||||||
|
|
||||||
```sh
|
For Go callers, prefer `exec.CommandContext` with explicit timeout/cancellation and buffered/streamed readers for both pipes.
|
||||||
export AUDITA_LLM_API_KEY=your-openrouter-key
|
|
||||||
audita process transcript.json --glossary glossary.yaml --output corrected.json
|
|
||||||
```
|
|
||||||
|
|
||||||
You can point Audita at any OpenAI-compatible endpoint by changing `AUDITA_BASE_URL` and, if needed, `AUDITA_MODEL`. For example, a local vLLM server:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
export AUDITA_LLM_API_KEY=local-dev-key
|
|
||||||
export AUDITA_BASE_URL=http://localhost:8000/v1
|
|
||||||
export AUDITA_MODEL=meta-llama/Llama-3.1-8B-Instruct
|
|
||||||
audita process transcript.json --glossary glossary.yaml --output corrected.json
|
|
||||||
```
|
|
||||||
|
|
||||||
Or the actual OpenAI API:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
export AUDITA_LLM_API_KEY=your-openai-key
|
|
||||||
export AUDITA_BASE_URL=https://api.openai.com/v1
|
|
||||||
export AUDITA_MODEL=gpt-4.1-mini
|
|
||||||
audita process transcript.json --glossary glossary.yaml --output corrected.json
|
|
||||||
```
|
|
||||||
|
|
||||||
`AUDITA_WORK_DIR` stores per-run diagnostics while processing. Under the default `AUDITA_WORK_DIR_RETENTION=auto`, clean successful runs are removed, while failed runs and successful runs with final skipped corrections are preserved. Use `always` to keep every run directory and `never` to remove successful run directories even when skips remain.
|
|
||||||
Failed runs always preserve the run directory and include an authoritative `report.json` alongside normalization and prompt/response diagnostics.
|
|
||||||
|
|
||||||
## Prototype Archive
|
|
||||||
|
|
||||||
The archived prototype remains importable as `audita_prototype` and is still covered by its original regression suite. This is intentional: the new `audita` package is a framework-oriented rewrite, not a thin wrapper around the old code.
|
|
||||||
|
|||||||
@@ -283,9 +283,9 @@ Session recap:
|
|||||||
```bash
|
```bash
|
||||||
scriptorium run \
|
scriptorium run \
|
||||||
--prompt dnd.session_recap \
|
--prompt dnd.session_recap \
|
||||||
--input transcript=/work/session-42/transcript.polished.md \
|
--input transcript=/work/campaign-7/session-42/transcript.polished.md \
|
||||||
--input glossary=/work/session-42/glossary.yml \
|
--input glossary=/work/campaign-7/session-42/glossary.yml \
|
||||||
--out /work/session-42/artifacts/session_recap.md
|
--out /work/campaign-7/session-42/artifacts/session_recap.md
|
||||||
```
|
```
|
||||||
|
|
||||||
Structured events:
|
Structured events:
|
||||||
@@ -293,8 +293,8 @@ Structured events:
|
|||||||
```bash
|
```bash
|
||||||
scriptorium run \
|
scriptorium run \
|
||||||
--prompt dnd.structured_events \
|
--prompt dnd.structured_events \
|
||||||
--input transcript=/work/session-42/transcript.polished.md \
|
--input transcript=/work/campaign-7/session-42/transcript.polished.md \
|
||||||
--out /work/session-42/artifacts/structured_events.json
|
--out /work/campaign-7/session-42/artifacts/structured_events.json
|
||||||
```
|
```
|
||||||
|
|
||||||
Glossary suggestions:
|
Glossary suggestions:
|
||||||
@@ -302,9 +302,9 @@ Glossary suggestions:
|
|||||||
```bash
|
```bash
|
||||||
scriptorium run \
|
scriptorium run \
|
||||||
--prompt dnd.glossary_suggestions \
|
--prompt dnd.glossary_suggestions \
|
||||||
--input transcript=/work/session-42/transcript.polished.md \
|
--input transcript=/work/campaign-7/session-42/transcript.polished.md \
|
||||||
--input previous_recap=/work/session-41/artifacts/session_recap.md \
|
--input previous_recap=/work/campaign-7/session-41/artifacts/session_recap.md \
|
||||||
--out /work/session-42/artifacts/glossary_suggestions.md
|
--out /work/campaign-7/session-42/artifacts/glossary_suggestions.md
|
||||||
```
|
```
|
||||||
|
|
||||||
Player-facing summary:
|
Player-facing summary:
|
||||||
@@ -312,9 +312,9 @@ Player-facing summary:
|
|||||||
```bash
|
```bash
|
||||||
scriptorium run \
|
scriptorium run \
|
||||||
--prompt dnd.player_summary \
|
--prompt dnd.player_summary \
|
||||||
--input transcript=/work/session-42/transcript.polished.md \
|
--input transcript=/work/campaign-7/session-42/transcript.polished.md \
|
||||||
--input structured_events=/work/session-42/artifacts/structured_events.json \
|
--input structured_events=/work/campaign-7/session-42/artifacts/structured_events.json \
|
||||||
--out /work/session-42/artifacts/player_summary.md
|
--out /work/campaign-7/session-42/artifacts/player_summary.md
|
||||||
```
|
```
|
||||||
|
|
||||||
## 21. Non-Goals
|
## 21. Non-Goals
|
||||||
|
|||||||
119
docs/stages/archive.md
Normal file
119
docs/stages/archive.md
Normal file
@@ -0,0 +1,119 @@
|
|||||||
|
# Archive Storage
|
||||||
|
|
||||||
|
This document describes implemented archive-stage publish behavior.
|
||||||
|
|
||||||
|
## S3 Paths
|
||||||
|
|
||||||
|
Session root:
|
||||||
|
|
||||||
|
`{root_prefix}/campaigns/{campaign}/sessions/{session_id}/`
|
||||||
|
|
||||||
|
Run prefix:
|
||||||
|
|
||||||
|
`{root_prefix}/campaigns/{campaign}/sessions/{session_id}/runs/{run_id}/`
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
Implemented:
|
||||||
|
|
||||||
|
- archive uploads successful run records to remote object storage through the storage backend abstraction.
|
||||||
|
- archive uploads configured promoted outputs to session-level keys.
|
||||||
|
- archive uploads `current/manifest.json`.
|
||||||
|
- archive uploads `current/run_id.txt` last as the effective commit marker.
|
||||||
|
- optional post-archive local cleanup:
|
||||||
|
- `pipeline.spool.delete_audio_after_archive: true` removes only the run-scoped spool audio directory
|
||||||
|
- `pipeline.workspace.cleanup_after_archive: true` removes only the run-scoped local workdir
|
||||||
|
- tests use fake storage and do not require live S3.
|
||||||
|
|
||||||
|
Future work:
|
||||||
|
|
||||||
|
- `notify` stage behavior
|
||||||
|
- stale detection
|
||||||
|
- optional future source-audio upload mode
|
||||||
|
- additional artifact generation beyond current implemented set
|
||||||
|
|
||||||
|
## Prerequisites
|
||||||
|
|
||||||
|
Archive verifies these stages succeeded before upload:
|
||||||
|
|
||||||
|
- `prepare`
|
||||||
|
- `transcribe`
|
||||||
|
- `merge`
|
||||||
|
- `polish`
|
||||||
|
- `normalize`
|
||||||
|
- `trim`
|
||||||
|
- `analyze`
|
||||||
|
|
||||||
|
If any prerequisite is missing or not succeeded, archive fails and does not upload.
|
||||||
|
Failed or incomplete runs remain local only.
|
||||||
|
|
||||||
|
## Run Upload
|
||||||
|
|
||||||
|
Archive uploads existing files from the run workdir when present:
|
||||||
|
|
||||||
|
- `inputs/`
|
||||||
|
- `transcripts/`
|
||||||
|
- `artifacts/`
|
||||||
|
- `reports/` (optional)
|
||||||
|
- `config/`
|
||||||
|
- `logs/`
|
||||||
|
- `manifest.json`
|
||||||
|
|
||||||
|
Relative paths are preserved under `runs/{run_id}/`.
|
||||||
|
|
||||||
|
## Promotion Rules
|
||||||
|
|
||||||
|
Archive applies `archive.promote_artifacts` in config order.
|
||||||
|
|
||||||
|
Rule behavior:
|
||||||
|
|
||||||
|
- `from`: local workdir-relative source path
|
||||||
|
- `to`: session-root-relative destination key
|
||||||
|
- `required: true`: missing source fails archive
|
||||||
|
- `required: false`: missing source is skipped and recorded
|
||||||
|
|
||||||
|
Default promoted outputs:
|
||||||
|
|
||||||
|
- `transcripts/trimmed.json`
|
||||||
|
- `artifacts/session_recap.md`
|
||||||
|
|
||||||
|
## Current Pointers
|
||||||
|
|
||||||
|
Archive writes:
|
||||||
|
|
||||||
|
1. `current/manifest.json` (after run upload + promotions)
|
||||||
|
2. `current/run_id.txt` last
|
||||||
|
|
||||||
|
`current/run_id.txt` contains exactly:
|
||||||
|
|
||||||
|
- `{run_id}` plus trailing newline
|
||||||
|
|
||||||
|
Writing `current/run_id.txt` last makes it the effective commit marker for published session state.
|
||||||
|
|
||||||
|
If any required run upload, promotion upload, or current-manifest upload fails, archive returns failure and does not write `current/run_id.txt`.
|
||||||
|
Cleanup runs only after this commit-marker write has succeeded.
|
||||||
|
|
||||||
|
## Audio Upload Policy
|
||||||
|
|
||||||
|
Archive does not upload local `audio/` by default.
|
||||||
|
Original audio is expected at the session-level audio prefix and is not duplicated under `runs/{run_id}/`.
|
||||||
|
|
||||||
|
## Config Controls
|
||||||
|
|
||||||
|
- `archive.enabled: false` skips archive cleanly.
|
||||||
|
- `archive.upload_run: false` skips run upload cleanly.
|
||||||
|
- both skip cases also skip post-archive local cleanup.
|
||||||
|
|
||||||
|
## Metadata
|
||||||
|
|
||||||
|
Archive stage metadata includes non-secret upload context (for example):
|
||||||
|
|
||||||
|
- `s3_bucket`
|
||||||
|
- `s3_run_prefix`
|
||||||
|
- run upload counts/paths
|
||||||
|
- promoted upload counts/paths
|
||||||
|
- skipped optional promotions
|
||||||
|
- `current_manifest_key`
|
||||||
|
- `current_run_id_key`
|
||||||
|
- `current_pointer_written`
|
||||||
|
- `audio_upload_skipped`
|
||||||
84
docs/stages/prepare.md
Normal file
84
docs/stages/prepare.md
Normal file
@@ -0,0 +1,84 @@
|
|||||||
|
# S3 Audio Input
|
||||||
|
|
||||||
|
This document describes implemented S3 audio input behavior in `prepare`.
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
Implemented:
|
||||||
|
|
||||||
|
- `prepare` can acquire source audio from S3 when `session.inputs.audio_s3.prefix` is configured.
|
||||||
|
- object listing and download go through the storage backend abstraction.
|
||||||
|
- tests use fake storage; no live S3 service is required for test runs.
|
||||||
|
|
||||||
|
Not implemented:
|
||||||
|
|
||||||
|
- uploads of failed runs
|
||||||
|
|
||||||
|
## Required Configuration
|
||||||
|
|
||||||
|
`pipeline.yml`:
|
||||||
|
|
||||||
|
- `storage.s3.bucket` must be set when S3 audio input is used.
|
||||||
|
- `storage.s3.root_prefix` defaults to `dnd`.
|
||||||
|
- `storage.s3.access_key_id_env` defaults to `OBJECT_STORAGE_KEY_ID`.
|
||||||
|
- `storage.s3.secret_access_key_env` defaults to `OBJECT_STORAGE_KEY`.
|
||||||
|
- `spool.root` defaults to `/var/spool/narratio`.
|
||||||
|
|
||||||
|
`session.yml`:
|
||||||
|
|
||||||
|
- configure `session.campaign` and `session.session_id`.
|
||||||
|
- configure `session.inputs.audio_s3.prefix` for S3 audio input.
|
||||||
|
- do not configure `inputs.audio_dir` or `inputs.audio_files` at the same time as `inputs.audio_s3`.
|
||||||
|
|
||||||
|
## Prefix Shape
|
||||||
|
|
||||||
|
Session S3 root:
|
||||||
|
|
||||||
|
`{root_prefix}/campaigns/{campaign}/sessions/{session_id}/`
|
||||||
|
|
||||||
|
Audio prefix:
|
||||||
|
|
||||||
|
`{session_root}/{audio_s3.prefix}`
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
`dnd/campaigns/forsaken/sessions/2026-04-19/audio/`
|
||||||
|
|
||||||
|
Audio files must already exist in S3 before running Narratio.
|
||||||
|
|
||||||
|
## Prepare Behavior
|
||||||
|
|
||||||
|
When `inputs.audio_s3.prefix` is configured, `prepare`:
|
||||||
|
|
||||||
|
1. lists objects under the computed S3 audio prefix
|
||||||
|
2. filters to `.flac` objects
|
||||||
|
3. fails when no `.flac` objects are found
|
||||||
|
4. downloads selected objects to spool audio:
|
||||||
|
- `{spool.root}/{campaign}/{session_id}/{run_id}/audio/`
|
||||||
|
5. materializes audio into workdir audio:
|
||||||
|
- `{workspace.root}/work/{campaign}/{session_id}/runs/{run_id}/audio/`
|
||||||
|
6. records input provenance in the manifest (bucket, key, metadata, local paths, checksum)
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
|
||||||
|
- `.flac` filtering is case-insensitive.
|
||||||
|
- ETag is recorded as provider metadata only and is not treated as a checksum.
|
||||||
|
|
||||||
|
## Local Audio Development
|
||||||
|
|
||||||
|
Local audio workflows remain supported:
|
||||||
|
|
||||||
|
- `inputs.audio_dir`
|
||||||
|
- `inputs.audio_files`
|
||||||
|
|
||||||
|
These options are mutually exclusive with `inputs.audio_s3`.
|
||||||
|
|
||||||
|
## Archive Boundary
|
||||||
|
|
||||||
|
Current archive behavior relevant to S3 audio input:
|
||||||
|
|
||||||
|
- successful runs are uploaded by archive under `runs/{run_id}/`
|
||||||
|
- configured promotions are uploaded to session-level destinations
|
||||||
|
- `current/manifest.json` and `current/run_id.txt` are published
|
||||||
|
- local source audio is not re-uploaded by default
|
||||||
|
- failed or incomplete runs are not uploaded
|
||||||
26
examples/pipeline.audita-overrides.yml
Normal file
26
examples/pipeline.audita-overrides.yml
Normal file
@@ -0,0 +1,26 @@
|
|||||||
|
workspace:
|
||||||
|
root: ./tmp/narratio-workspace
|
||||||
|
|
||||||
|
whisperx:
|
||||||
|
transcribe_url: "https://transcription.example.com/transcribe"
|
||||||
|
|
||||||
|
seriatim:
|
||||||
|
binary: "seriatim"
|
||||||
|
timeout: "10m"
|
||||||
|
output_schema: "seriatim-intermediate"
|
||||||
|
coalesce_gap: 3.0
|
||||||
|
|
||||||
|
audita:
|
||||||
|
binary: "audita"
|
||||||
|
timeout: "3h"
|
||||||
|
base_url: "https://openrouter.ai/api/v1"
|
||||||
|
model: "openrouter/google/gemma-4-31b-it"
|
||||||
|
llm_api_key_env: "AUDITA_LLM_API_KEY"
|
||||||
|
modules: ["glossary", "homophones", "spoken_word", "grammar"]
|
||||||
|
output_schema: "audita-v1"
|
||||||
|
work_dir_retention: "auto"
|
||||||
|
total_llm_concurrency: 2
|
||||||
|
proposal_llm_concurrency: 1
|
||||||
|
validation_model: "openrouter/google/gemma-4-31b-it"
|
||||||
|
validation_llm_concurrency: 1
|
||||||
|
report: true
|
||||||
@@ -1,97 +1,51 @@
|
|||||||
workspace:
|
workspace:
|
||||||
root: ./tmp/narratio-workspace
|
root: ./tmp/narratio-workspace
|
||||||
|
cleanup_after_archive: false
|
||||||
|
|
||||||
storage:
|
storage:
|
||||||
backend: local
|
backend: s3
|
||||||
|
s3:
|
||||||
|
bucket: "my-dnd-archive"
|
||||||
|
root_prefix: "dnd"
|
||||||
|
region: "us-east-1"
|
||||||
|
# Optional credential env-var names (defaulted when omitted):
|
||||||
|
# access_key_id_env: "OBJECT_STORAGE_KEY_ID"
|
||||||
|
# secret_access_key_env: "OBJECT_STORAGE_KEY"
|
||||||
|
|
||||||
|
spool:
|
||||||
|
root: "/var/spool/narratio"
|
||||||
|
delete_audio_after_archive: false
|
||||||
|
|
||||||
|
archive:
|
||||||
|
enabled: true
|
||||||
|
upload_run: true
|
||||||
|
|
||||||
whisperx:
|
whisperx:
|
||||||
transcribe_url: "https://transcription.example.com/transcribe"
|
transcribe_url: "https://transcription.example.com/transcribe"
|
||||||
language: "en"
|
|
||||||
timeout: "30m"
|
|
||||||
retries: 3
|
|
||||||
retry_delay: "2s"
|
|
||||||
concurrency: 2
|
|
||||||
|
|
||||||
seriatim:
|
# Optional. When omitted entirely, Narratio defaults to seriatim binary + runtime defaults.
|
||||||
binary: "seriatim"
|
seriatim: {}
|
||||||
timeout: "10m"
|
|
||||||
output_schema: "seriatim-intermediate"
|
|
||||||
coalesce_gap: 3.0
|
|
||||||
report: true
|
|
||||||
env:
|
|
||||||
overlap_word_run_gap: 1.0
|
|
||||||
overlap_word_run_reorder_window: 1.0
|
|
||||||
backchannel_max_duration: 2.0
|
|
||||||
filler_max_duration: 1.25
|
|
||||||
|
|
||||||
|
# Optional runtime overrides. Model/provider can be owned by Audita runtime config.
|
||||||
audita:
|
audita:
|
||||||
binary: "audita"
|
config_path: "/usr/local/etc/audita/config.yml"
|
||||||
timeout: "3h"
|
|
||||||
llm_api_key_env: "AUDITA_LLM_API_KEY"
|
llm_api_key_env: "AUDITA_LLM_API_KEY"
|
||||||
modules:
|
|
||||||
- glossary
|
|
||||||
- homophones
|
|
||||||
- glossary
|
|
||||||
- spoken_word
|
|
||||||
- grammar
|
|
||||||
- homophones
|
|
||||||
- glossary
|
|
||||||
base_url: "https://openrouter.ai/api/v1"
|
|
||||||
model: "openrouter/google/gemma-4-31b-it"
|
|
||||||
llm_concurrency: 1
|
|
||||||
validation_model: ""
|
|
||||||
validation_llm_concurrency: 1
|
|
||||||
report: true
|
|
||||||
|
|
||||||
normalize:
|
|
||||||
# Session-workdir-relative when not absolute.
|
|
||||||
output_path: "transcripts/normalized.json"
|
|
||||||
output_schema: "seriatim-intermediate"
|
|
||||||
report: true
|
|
||||||
|
|
||||||
trim:
|
|
||||||
enabled: true
|
|
||||||
# Session-workdir-relative when not absolute.
|
|
||||||
output_path: "transcripts/trimmed.json"
|
|
||||||
bounds:
|
|
||||||
prompt_id: "dnd_session.bounds"
|
|
||||||
# Empty means use prompt default profile.
|
|
||||||
profile_id: ""
|
|
||||||
transcript_input_name: "transcript"
|
|
||||||
output_path: "artifacts/session_bounds.json"
|
|
||||||
timeout: "10m"
|
|
||||||
render_debug: false
|
|
||||||
render_output_path: "artifacts/session_bounds.render.json"
|
|
||||||
seriatim:
|
|
||||||
report: false
|
|
||||||
|
|
||||||
|
# Optional Scriptorium integration for analyze artifacts.
|
||||||
scriptorium:
|
scriptorium:
|
||||||
binary: "scriptorium"
|
config_path: "/usr/local/etc/scriptorium/config.yml"
|
||||||
config_path: "/etc/scriptorium/config.yml"
|
|
||||||
timeout: "10m"
|
|
||||||
render_debug: false
|
|
||||||
artifacts:
|
artifacts:
|
||||||
session_recap:
|
session_recap:
|
||||||
enabled: true
|
enabled: true
|
||||||
prompt_id: "dnd.session_recap"
|
prompt_id: "dnd.session_recap"
|
||||||
profile_id: "local-quality"
|
|
||||||
output_path: "artifacts/session_recap.md"
|
output_path: "artifacts/session_recap.md"
|
||||||
timeout: "10m"
|
|
||||||
# Optional per-artifact override of global scriptorium.render_debug.
|
|
||||||
# render_debug: true
|
|
||||||
inputs:
|
inputs:
|
||||||
transcript:
|
transcript:
|
||||||
# Available transcript sources:
|
|
||||||
# - trimmed_transcript (recommended for session_recap)
|
|
||||||
# - normalized_transcript (recommended for future full-session analysis)
|
|
||||||
# - processed_transcript (raw Audita-polished output)
|
|
||||||
source: "trimmed_transcript"
|
source: "trimmed_transcript"
|
||||||
required: true
|
required: true
|
||||||
previous_recap:
|
previous_recap:
|
||||||
source: "previous_session_artifact"
|
source: "previous_session_artifact"
|
||||||
artifact: "session_recap"
|
artifact: "session_recap"
|
||||||
# Optional: set when previous recap is available.
|
|
||||||
path: ""
|
|
||||||
required: false
|
required: false
|
||||||
vars:
|
vars:
|
||||||
session_id: true
|
session_id: true
|
||||||
@@ -99,11 +53,3 @@ scriptorium:
|
|||||||
campaign_name: true
|
campaign_name: true
|
||||||
previous_session_id: true
|
previous_session_id: true
|
||||||
output_kind: "session_recap"
|
output_kind: "session_recap"
|
||||||
|
|
||||||
analyzer:
|
|
||||||
timeout: 20m
|
|
||||||
artifacts:
|
|
||||||
output_dir: artifacts
|
|
||||||
|
|
||||||
notification:
|
|
||||||
timeout: 10s
|
|
||||||
|
|||||||
@@ -4,7 +4,10 @@ date: 2026-05-03
|
|||||||
title: Sample Session
|
title: Sample Session
|
||||||
inputs:
|
inputs:
|
||||||
audio_dir: ./audio
|
audio_dir: ./audio
|
||||||
|
# Optional S3 input alternative. Do not configure with audio_dir/audio_files.
|
||||||
|
# Narratio prepare lists this prefix and downloads .flac files.
|
||||||
|
# audio_s3:
|
||||||
|
# prefix: "audio/"
|
||||||
speakers_file: ./speakers.yml
|
speakers_file: ./speakers.yml
|
||||||
autocorrect_file: ./autocorrect.yml
|
autocorrect_file: ./autocorrect.yml
|
||||||
glossary_file: ./glossary.yml
|
glossary_file: ./glossary.yml
|
||||||
|
|
||||||
|
|||||||
12
examples/session.template.yml
Normal file
12
examples/session.template.yml
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
session_id: "{{ session_id }}"
|
||||||
|
campaign: sample-campaign
|
||||||
|
date: ""
|
||||||
|
title: ""
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
|
# Optional S3 input alternative. Do not configure with audio_dir/audio_files.
|
||||||
|
# audio_s3:
|
||||||
|
# prefix: "audio/{{ session_id }}/"
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
25
go.mod
25
go.mod
@@ -2,4 +2,27 @@ module gitea.maximumdirect.net/eric/narratio
|
|||||||
|
|
||||||
go 1.25.0
|
go 1.25.0
|
||||||
|
|
||||||
require gopkg.in/yaml.v3 v3.0.1
|
require (
|
||||||
|
github.com/aws/aws-sdk-go-v2/config v1.32.17
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
|
||||||
|
github.com/aws/smithy-go v1.25.1
|
||||||
|
gopkg.in/yaml.v3 v3.0.1
|
||||||
|
)
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/aws/aws-sdk-go-v2 v1.41.7 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/credentials v1.19.16 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect
|
||||||
|
)
|
||||||
|
|||||||
36
go.sum
36
go.sum
@@ -1,3 +1,39 @@
|
|||||||
|
github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=
|
||||||
|
github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=
|
||||||
|
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho=
|
||||||
|
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY=
|
||||||
|
github.com/aws/aws-sdk-go-v2/config v1.32.17 h1:FpL4/758/diKwqbytU0prpuiu60fgXKUWCpDJtApclU=
|
||||||
|
github.com/aws/aws-sdk-go-v2/config v1.32.17/go.mod h1:OXqUMzgXytfoF9JaKkhrOYsyh72t9G+MJH8mMRaexOE=
|
||||||
|
github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU=
|
||||||
|
github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug=
|
||||||
|
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 h1:UuSfcORqNSz/ey3VPRS8TcVH2Ikf0/sC+Hdj400QI6U=
|
||||||
|
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23/go.mod h1:+G/OSGiOFnSOkYloKj/9M35s74LgVAdJBSD5lsFfqKg=
|
||||||
|
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0=
|
||||||
|
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA=
|
||||||
|
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo=
|
||||||
|
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk=
|
||||||
|
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA=
|
||||||
|
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 h1:etqBTKY581iwLL/H/S2sVgk3C9lAsTJFeXWFDsDcWOU=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 h1:TdJ+HdzOBhU8+iVAOGUTU63VXopcumCOF1paFulHWZc=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11/go.mod h1:R82ZRExE/nheo0N+T8zHPcLRTcH8MGsnR3BiVGX0TwI=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 h1:7byT8HUWrgoRp6sXjxtZwgOKfhss5fW6SkLBtqzgRoE=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17/go.mod h1:xNWknVi4Ezm1vg1QsB/5EWpAJURq22uqd38U8qKvOJc=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 h1:+1Kl1zx6bWi4X7cKi3VYh29h8BvsCoHQEQ6ST9X8w7w=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21/go.mod h1:4vIRDq+CJB2xFAXZ+YgGUTiEft7oAQlhIs71xcSeuVg=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOItExNM9L1euNuh/fk=
|
||||||
|
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio=
|
||||||
|
github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
|
||||||
|
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||||
|
|||||||
@@ -24,6 +24,12 @@ type PolishRequest struct {
|
|||||||
Modules []string
|
Modules []string
|
||||||
BaseURL string
|
BaseURL string
|
||||||
Model string
|
Model string
|
||||||
|
TranscriptDescription string
|
||||||
|
ConfigPath string
|
||||||
|
OutputSchema string
|
||||||
|
WorkDirRetention string
|
||||||
|
TotalLLMConcurrency *int
|
||||||
|
ProposalLLMConcurrency *int
|
||||||
ValidationModel string
|
ValidationModel string
|
||||||
ValidationLLMConcurrency *int
|
ValidationLLMConcurrency *int
|
||||||
StdoutLogPath string
|
StdoutLogPath string
|
||||||
|
|||||||
@@ -21,7 +21,12 @@ type SubprocessRunnerConfig struct {
|
|||||||
Modules []string
|
Modules []string
|
||||||
BaseURL string
|
BaseURL string
|
||||||
Model string
|
Model string
|
||||||
LLMConcurrency *int
|
TranscriptDescription string
|
||||||
|
ConfigPath string
|
||||||
|
OutputSchema string
|
||||||
|
WorkDirRetention string
|
||||||
|
TotalLLMConcurrency *int
|
||||||
|
ProposalLLMConcurrency *int
|
||||||
ValidationModel string
|
ValidationModel string
|
||||||
ValidationLLMConcurrency *int
|
ValidationLLMConcurrency *int
|
||||||
Report bool
|
Report bool
|
||||||
@@ -35,7 +40,12 @@ type SubprocessRunner struct {
|
|||||||
modules []string
|
modules []string
|
||||||
baseURL string
|
baseURL string
|
||||||
model string
|
model string
|
||||||
llmConcurrency *int
|
transcriptDescription string
|
||||||
|
configPath string
|
||||||
|
outputSchema string
|
||||||
|
workDirRetention string
|
||||||
|
totalLLMConcurrency *int
|
||||||
|
proposalLLMConcurrency *int
|
||||||
validationModel string
|
validationModel string
|
||||||
validationLLMConcurrency *int
|
validationLLMConcurrency *int
|
||||||
report bool
|
report bool
|
||||||
@@ -49,7 +59,12 @@ func NewSubprocessRunnerFromConfigValues(
|
|||||||
modules []string,
|
modules []string,
|
||||||
baseURL string,
|
baseURL string,
|
||||||
model string,
|
model string,
|
||||||
llmConcurrency *int,
|
transcriptDescription string,
|
||||||
|
configPath string,
|
||||||
|
outputSchema string,
|
||||||
|
workDirRetention string,
|
||||||
|
totalLLMConcurrency *int,
|
||||||
|
proposalLLMConcurrency *int,
|
||||||
validationModel string,
|
validationModel string,
|
||||||
validationLLMConcurrency *int,
|
validationLLMConcurrency *int,
|
||||||
report bool,
|
report bool,
|
||||||
@@ -68,7 +83,12 @@ func NewSubprocessRunnerFromConfigValues(
|
|||||||
Modules: modules,
|
Modules: modules,
|
||||||
BaseURL: baseURL,
|
BaseURL: baseURL,
|
||||||
Model: model,
|
Model: model,
|
||||||
LLMConcurrency: llmConcurrency,
|
TranscriptDescription: transcriptDescription,
|
||||||
|
ConfigPath: configPath,
|
||||||
|
OutputSchema: outputSchema,
|
||||||
|
WorkDirRetention: workDirRetention,
|
||||||
|
TotalLLMConcurrency: totalLLMConcurrency,
|
||||||
|
ProposalLLMConcurrency: proposalLLMConcurrency,
|
||||||
ValidationModel: validationModel,
|
ValidationModel: validationModel,
|
||||||
ValidationLLMConcurrency: validationLLMConcurrency,
|
ValidationLLMConcurrency: validationLLMConcurrency,
|
||||||
Report: report,
|
Report: report,
|
||||||
@@ -83,33 +103,39 @@ func NewSubprocessRunner(cfg SubprocessRunnerConfig) (*SubprocessRunner, error)
|
|||||||
if cfg.Timeout <= 0 {
|
if cfg.Timeout <= 0 {
|
||||||
return nil, fmt.Errorf("audita timeout must be > 0")
|
return nil, fmt.Errorf("audita timeout must be > 0")
|
||||||
}
|
}
|
||||||
if len(cfg.Modules) == 0 {
|
|
||||||
return nil, fmt.Errorf("audita modules must include at least one module")
|
|
||||||
}
|
|
||||||
for i, module := range cfg.Modules {
|
for i, module := range cfg.Modules {
|
||||||
if strings.TrimSpace(module) == "" {
|
if strings.TrimSpace(module) == "" {
|
||||||
return nil, fmt.Errorf("audita module at index %d is empty", i)
|
return nil, fmt.Errorf("audita module at index %d is empty", i)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(cfg.BaseURL) == "" {
|
if strings.TrimSpace(cfg.BaseURL) != "" {
|
||||||
return nil, fmt.Errorf("audita base url is required")
|
u, err := url.Parse(cfg.BaseURL)
|
||||||
}
|
if err != nil || u.Scheme == "" || u.Host == "" {
|
||||||
u, err := url.Parse(cfg.BaseURL)
|
if err != nil {
|
||||||
if err != nil || u.Scheme == "" || u.Host == "" {
|
return nil, fmt.Errorf("audita base url %q is invalid: %w", cfg.BaseURL, err)
|
||||||
if err != nil {
|
}
|
||||||
return nil, fmt.Errorf("audita base url %q is invalid: %w", cfg.BaseURL, err)
|
return nil, fmt.Errorf("audita base url %q is invalid", cfg.BaseURL)
|
||||||
}
|
}
|
||||||
return nil, fmt.Errorf("audita base url %q is invalid", cfg.BaseURL)
|
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(cfg.Model) == "" {
|
if cfg.TotalLLMConcurrency != nil && *cfg.TotalLLMConcurrency <= 0 {
|
||||||
return nil, fmt.Errorf("audita model is required")
|
return nil, fmt.Errorf("audita total llm concurrency must be > 0 when provided")
|
||||||
}
|
}
|
||||||
if cfg.LLMConcurrency != nil && *cfg.LLMConcurrency <= 0 {
|
if cfg.ProposalLLMConcurrency != nil && *cfg.ProposalLLMConcurrency <= 0 {
|
||||||
return nil, fmt.Errorf("audita llm concurrency must be > 0 when provided")
|
return nil, fmt.Errorf("audita proposal llm concurrency must be > 0 when provided")
|
||||||
}
|
}
|
||||||
if cfg.ValidationLLMConcurrency != nil && *cfg.ValidationLLMConcurrency <= 0 {
|
if cfg.ValidationLLMConcurrency != nil && *cfg.ValidationLLMConcurrency <= 0 {
|
||||||
return nil, fmt.Errorf("audita validation llm concurrency must be > 0 when provided")
|
return nil, fmt.Errorf("audita validation llm concurrency must be > 0 when provided")
|
||||||
}
|
}
|
||||||
|
switch strings.TrimSpace(cfg.OutputSchema) {
|
||||||
|
case "", "bare-segments", "audita-v1":
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("audita output schema must be one of: bare-segments, audita-v1")
|
||||||
|
}
|
||||||
|
switch strings.TrimSpace(cfg.WorkDirRetention) {
|
||||||
|
case "", "always", "auto", "never":
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("audita work dir retention must be one of: always, auto, never")
|
||||||
|
}
|
||||||
|
|
||||||
modules := make([]string, len(cfg.Modules))
|
modules := make([]string, len(cfg.Modules))
|
||||||
for i, m := range cfg.Modules {
|
for i, m := range cfg.Modules {
|
||||||
@@ -123,7 +149,12 @@ func NewSubprocessRunner(cfg SubprocessRunnerConfig) (*SubprocessRunner, error)
|
|||||||
modules: modules,
|
modules: modules,
|
||||||
baseURL: strings.TrimSpace(cfg.BaseURL),
|
baseURL: strings.TrimSpace(cfg.BaseURL),
|
||||||
model: strings.TrimSpace(cfg.Model),
|
model: strings.TrimSpace(cfg.Model),
|
||||||
llmConcurrency: cfg.LLMConcurrency,
|
transcriptDescription: strings.TrimSpace(cfg.TranscriptDescription),
|
||||||
|
configPath: strings.TrimSpace(cfg.ConfigPath),
|
||||||
|
outputSchema: strings.TrimSpace(cfg.OutputSchema),
|
||||||
|
workDirRetention: strings.TrimSpace(cfg.WorkDirRetention),
|
||||||
|
totalLLMConcurrency: cfg.TotalLLMConcurrency,
|
||||||
|
proposalLLMConcurrency: cfg.ProposalLLMConcurrency,
|
||||||
validationModel: strings.TrimSpace(cfg.ValidationModel),
|
validationModel: strings.TrimSpace(cfg.ValidationModel),
|
||||||
validationLLMConcurrency: cfg.ValidationLLMConcurrency,
|
validationLLMConcurrency: cfg.ValidationLLMConcurrency,
|
||||||
report: cfg.Report,
|
report: cfg.Report,
|
||||||
@@ -152,7 +183,7 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
|
|||||||
}
|
}
|
||||||
|
|
||||||
reqModules := req.Modules
|
reqModules := req.Modules
|
||||||
if len(reqModules) == 0 {
|
if reqModules == nil {
|
||||||
reqModules = append([]string(nil), r.modules...)
|
reqModules = append([]string(nil), r.modules...)
|
||||||
}
|
}
|
||||||
args := r.buildArgs(req, reqModules)
|
args := r.buildArgs(req, reqModules)
|
||||||
@@ -168,14 +199,9 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
|
|||||||
env["AUDITA_LLM_API_KEY"] = credential
|
env["AUDITA_LLM_API_KEY"] = credential
|
||||||
credentialPresent = true
|
credentialPresent = true
|
||||||
}
|
}
|
||||||
primaryConcurrencyViaEnv := false
|
|
||||||
if r.llmConcurrency != nil {
|
|
||||||
env["AUDITA_LLM_CONCURRENCY"] = strconv.Itoa(*r.llmConcurrency)
|
|
||||||
primaryConcurrencyViaEnv = true
|
|
||||||
}
|
|
||||||
|
|
||||||
if req.GeneratedConfigPath != "" {
|
if req.GeneratedConfigPath != "" {
|
||||||
if err := r.writeInvocationConfig(req, args, reqModules, credentialPresent, primaryConcurrencyViaEnv); err != nil {
|
if err := r.writeInvocationConfig(req, args, reqModules, credentialPresent); err != nil {
|
||||||
return PolishResult{}, fmt.Errorf("write audita invocation config %q: %w", req.GeneratedConfigPath, err)
|
return PolishResult{}, fmt.Errorf("write audita invocation config %q: %w", req.GeneratedConfigPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -196,7 +222,7 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
|
|||||||
req.StderrLogPath,
|
req.StderrLogPath,
|
||||||
)
|
)
|
||||||
wrappedMessage = addSubprocessStreamHint(wrappedMessage, err)
|
wrappedMessage = addSubprocessStreamHint(wrappedMessage, err)
|
||||||
return r.failureResult(req, reqModules, runRes, credentialPresent, primaryConcurrencyViaEnv), fmt.Errorf(
|
return r.failureResult(req, reqModules, runRes, credentialPresent), fmt.Errorf(
|
||||||
"%s: %w",
|
"%s: %w",
|
||||||
wrappedMessage,
|
wrappedMessage,
|
||||||
err,
|
err,
|
||||||
@@ -204,11 +230,11 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
|
|||||||
}
|
}
|
||||||
|
|
||||||
if err := validateProcessedOutput(req.OutputProcessedPath); err != nil {
|
if err := validateProcessedOutput(req.OutputProcessedPath); err != nil {
|
||||||
return r.failureResult(req, reqModules, runRes, credentialPresent, primaryConcurrencyViaEnv), fmt.Errorf("validate audita processed output %q: %w", req.OutputProcessedPath, err)
|
return r.failureResult(req, reqModules, runRes, credentialPresent), fmt.Errorf("validate audita processed output %q: %w", req.OutputProcessedPath, err)
|
||||||
}
|
}
|
||||||
if r.report {
|
if r.report {
|
||||||
if err := validateJSONFile(req.ReportPath); err != nil {
|
if err := validateJSONFile(req.ReportPath); err != nil {
|
||||||
return r.failureResult(req, reqModules, runRes, credentialPresent, primaryConcurrencyViaEnv), fmt.Errorf("validate audita report output %q: %w", req.ReportPath, err)
|
return r.failureResult(req, reqModules, runRes, credentialPresent), fmt.Errorf("validate audita report output %q: %w", req.ReportPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -223,21 +249,25 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
|
|||||||
Duration: runRes.Duration,
|
Duration: runRes.Duration,
|
||||||
InvokedBinary: r.binary,
|
InvokedBinary: r.binary,
|
||||||
Metadata: map[string]any{
|
Metadata: map[string]any{
|
||||||
"adapter": "audita_subprocess",
|
"adapter": "audita_subprocess",
|
||||||
"modules": reqModules,
|
"modules": reqModules,
|
||||||
"base_url": r.baseURL,
|
"base_url": r.baseURL,
|
||||||
"model": r.model,
|
"model": r.model,
|
||||||
"validation_model": r.validationModel,
|
"transcript_description": r.transcriptDescription,
|
||||||
"validation_llm_concurrency": r.validationLLMConcurrency,
|
"config_path": r.configPath,
|
||||||
"credential_env_var": r.llmAPIKeyEnv,
|
"output_schema": r.outputSchema,
|
||||||
"credential_present": credentialPresent,
|
"work_dir_retention": r.workDirRetention,
|
||||||
"primary_llm_concurrency_via_env": primaryConcurrencyViaEnv,
|
"validation_model": r.validationModel,
|
||||||
"primary_llm_concurrency_env_name": "AUDITA_LLM_CONCURRENCY",
|
"total_llm_concurrency": r.totalLLMConcurrency,
|
||||||
|
"proposal_llm_concurrency": r.proposalLLMConcurrency,
|
||||||
|
"validation_llm_concurrency": r.validationLLMConcurrency,
|
||||||
|
"credential_env_var": r.llmAPIKeyEnv,
|
||||||
|
"credential_present": credentialPresent,
|
||||||
},
|
},
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *SubprocessRunner) failureResult(req PolishRequest, modules []string, runRes subprocess.RunResult, credentialPresent bool, primaryConcurrencyViaEnv bool) PolishResult {
|
func (r *SubprocessRunner) failureResult(req PolishRequest, modules []string, runRes subprocess.RunResult, credentialPresent bool) PolishResult {
|
||||||
return PolishResult{
|
return PolishResult{
|
||||||
ProcessedTranscriptPath: req.OutputProcessedPath,
|
ProcessedTranscriptPath: req.OutputProcessedPath,
|
||||||
ReportPath: req.ReportPath,
|
ReportPath: req.ReportPath,
|
||||||
@@ -249,16 +279,20 @@ func (r *SubprocessRunner) failureResult(req PolishRequest, modules []string, ru
|
|||||||
Duration: runRes.Duration,
|
Duration: runRes.Duration,
|
||||||
InvokedBinary: r.binary,
|
InvokedBinary: r.binary,
|
||||||
Metadata: map[string]any{
|
Metadata: map[string]any{
|
||||||
"adapter": "audita_subprocess",
|
"adapter": "audita_subprocess",
|
||||||
"modules": modules,
|
"modules": modules,
|
||||||
"base_url": r.baseURL,
|
"base_url": r.baseURL,
|
||||||
"model": r.model,
|
"model": r.model,
|
||||||
"validation_model": r.validationModel,
|
"transcript_description": r.transcriptDescription,
|
||||||
"validation_llm_concurrency": r.validationLLMConcurrency,
|
"config_path": r.configPath,
|
||||||
"credential_env_var": r.llmAPIKeyEnv,
|
"output_schema": r.outputSchema,
|
||||||
"credential_present": credentialPresent,
|
"work_dir_retention": r.workDirRetention,
|
||||||
"primary_llm_concurrency_via_env": primaryConcurrencyViaEnv,
|
"validation_model": r.validationModel,
|
||||||
"primary_llm_concurrency_env_name": "AUDITA_LLM_CONCURRENCY",
|
"total_llm_concurrency": r.totalLLMConcurrency,
|
||||||
|
"proposal_llm_concurrency": r.proposalLLMConcurrency,
|
||||||
|
"validation_llm_concurrency": r.validationLLMConcurrency,
|
||||||
|
"credential_env_var": r.llmAPIKeyEnv,
|
||||||
|
"credential_present": credentialPresent,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -269,14 +303,38 @@ func (r *SubprocessRunner) buildArgs(req PolishRequest, modules []string) []stri
|
|||||||
req.MergedTranscriptPath,
|
req.MergedTranscriptPath,
|
||||||
"--glossary", req.GlossaryPath,
|
"--glossary", req.GlossaryPath,
|
||||||
"--output", req.OutputProcessedPath,
|
"--output", req.OutputProcessedPath,
|
||||||
"--modules", strings.Join(modules, ","),
|
|
||||||
"--base-url", r.baseURL,
|
|
||||||
"--model", r.model,
|
|
||||||
"--work-dir", req.WorkDir,
|
"--work-dir", req.WorkDir,
|
||||||
}
|
}
|
||||||
|
if r.baseURL != "" {
|
||||||
|
args = append(args, "--base-url", r.baseURL)
|
||||||
|
}
|
||||||
|
if r.model != "" {
|
||||||
|
args = append(args, "--model", r.model)
|
||||||
|
}
|
||||||
|
if len(modules) > 0 {
|
||||||
|
args = append(args, "--modules", strings.Join(modules, ","))
|
||||||
|
}
|
||||||
if r.report {
|
if r.report {
|
||||||
args = append(args, "--report-json", req.ReportPath)
|
args = append(args, "--report-json", req.ReportPath)
|
||||||
}
|
}
|
||||||
|
if r.transcriptDescription != "" {
|
||||||
|
args = append(args, "--transcript-description", r.transcriptDescription)
|
||||||
|
}
|
||||||
|
if r.configPath != "" {
|
||||||
|
args = append(args, "--config", r.configPath)
|
||||||
|
}
|
||||||
|
if r.outputSchema != "" {
|
||||||
|
args = append(args, "--output-schema", r.outputSchema)
|
||||||
|
}
|
||||||
|
if r.workDirRetention != "" {
|
||||||
|
args = append(args, "--work-dir-retention", r.workDirRetention)
|
||||||
|
}
|
||||||
|
if r.totalLLMConcurrency != nil {
|
||||||
|
args = append(args, "--total-llm-concurrency", strconv.Itoa(*r.totalLLMConcurrency))
|
||||||
|
}
|
||||||
|
if r.proposalLLMConcurrency != nil {
|
||||||
|
args = append(args, "--proposal-llm-concurrency", strconv.Itoa(*r.proposalLLMConcurrency))
|
||||||
|
}
|
||||||
if r.validationModel != "" {
|
if r.validationModel != "" {
|
||||||
args = append(args, "--validation-model", r.validationModel)
|
args = append(args, "--validation-model", r.validationModel)
|
||||||
}
|
}
|
||||||
@@ -286,29 +344,31 @@ func (r *SubprocessRunner) buildArgs(req PolishRequest, modules []string) []stri
|
|||||||
return args
|
return args
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *SubprocessRunner) writeInvocationConfig(req PolishRequest, args []string, modules []string, credentialPresent bool, primaryConcurrencyViaEnv bool) error {
|
func (r *SubprocessRunner) writeInvocationConfig(req PolishRequest, args []string, modules []string, credentialPresent bool) error {
|
||||||
payload := map[string]any{
|
payload := map[string]any{
|
||||||
"schema": "audita.generated.v1",
|
"schema": "audita.generated.v1",
|
||||||
"binary": r.binary,
|
"binary": r.binary,
|
||||||
"args": args,
|
"args": args,
|
||||||
"timeout": r.timeout.String(),
|
"timeout": r.timeout.String(),
|
||||||
"modules": modules,
|
"modules": modules,
|
||||||
"base_url": r.baseURL,
|
"base_url": r.baseURL,
|
||||||
"model": r.model,
|
"model": r.model,
|
||||||
"validation_model": r.validationModel,
|
"transcript_description": r.transcriptDescription,
|
||||||
"validation_llm_concurrency": r.validationLLMConcurrency,
|
"config_path": r.configPath,
|
||||||
"report_enabled": r.report,
|
"output_schema": r.outputSchema,
|
||||||
"merged_transcript_path": req.MergedTranscriptPath,
|
"work_dir_retention": r.workDirRetention,
|
||||||
"glossary_path": req.GlossaryPath,
|
"validation_model": r.validationModel,
|
||||||
"output_path": req.OutputProcessedPath,
|
"total_llm_concurrency": r.totalLLMConcurrency,
|
||||||
"report_path": req.ReportPath,
|
"proposal_llm_concurrency": r.proposalLLMConcurrency,
|
||||||
"work_dir": req.WorkDir,
|
"validation_llm_concurrency": r.validationLLMConcurrency,
|
||||||
"credential_env_var": r.llmAPIKeyEnv,
|
"report_enabled": r.report,
|
||||||
"credential_present": credentialPresent,
|
"merged_transcript_path": req.MergedTranscriptPath,
|
||||||
"primary_llm_concurrency_via_env": primaryConcurrencyViaEnv,
|
"glossary_path": req.GlossaryPath,
|
||||||
}
|
"output_path": req.OutputProcessedPath,
|
||||||
if r.llmConcurrency != nil {
|
"report_path": req.ReportPath,
|
||||||
payload["llm_concurrency"] = *r.llmConcurrency
|
"work_dir": req.WorkDir,
|
||||||
|
"credential_env_var": r.llmAPIKeyEnv,
|
||||||
|
"credential_present": credentialPresent,
|
||||||
}
|
}
|
||||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644)
|
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -25,7 +25,8 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
|||||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||||
|
|
||||||
wrapper := writeAuditaHelperWrapper(t)
|
wrapper := writeAuditaHelperWrapper(t)
|
||||||
llmConcurrency := 1
|
totalLLMConcurrency := 3
|
||||||
|
proposalLLMConcurrency := 2
|
||||||
validationLLMConcurrency := 2
|
validationLLMConcurrency := 2
|
||||||
runner, err := NewSubprocessRunner(SubprocessRunnerConfig{
|
runner, err := NewSubprocessRunner(SubprocessRunnerConfig{
|
||||||
Binary: wrapper,
|
Binary: wrapper,
|
||||||
@@ -34,7 +35,12 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
|||||||
Modules: []string{"glossary", "homophones", "glossary"},
|
Modules: []string{"glossary", "homophones", "glossary"},
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
LLMConcurrency: &llmConcurrency,
|
TranscriptDescription: "Campaign Session 42",
|
||||||
|
ConfigPath: "/etc/audita/config.yml",
|
||||||
|
OutputSchema: "audita-v1",
|
||||||
|
WorkDirRetention: "auto",
|
||||||
|
TotalLLMConcurrency: &totalLLMConcurrency,
|
||||||
|
ProposalLLMConcurrency: &proposalLLMConcurrency,
|
||||||
ValidationModel: "openrouter/google/gemma-4-31b-it",
|
ValidationModel: "openrouter/google/gemma-4-31b-it",
|
||||||
ValidationLLMConcurrency: &validationLLMConcurrency,
|
ValidationLLMConcurrency: &validationLLMConcurrency,
|
||||||
Report: true,
|
Report: true,
|
||||||
@@ -93,11 +99,17 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
|||||||
"process", req.MergedTranscriptPath,
|
"process", req.MergedTranscriptPath,
|
||||||
"--glossary", req.GlossaryPath,
|
"--glossary", req.GlossaryPath,
|
||||||
"--output", req.OutputProcessedPath,
|
"--output", req.OutputProcessedPath,
|
||||||
"--modules", "glossary,homophones,glossary",
|
"--work-dir", req.WorkDir,
|
||||||
"--base-url", "https://openrouter.ai/api/v1",
|
"--base-url", "https://openrouter.ai/api/v1",
|
||||||
"--model", "openrouter/google/gemma-4-31b-it",
|
"--model", "openrouter/google/gemma-4-31b-it",
|
||||||
"--work-dir", req.WorkDir,
|
"--modules", "glossary,homophones,glossary",
|
||||||
"--report-json", req.ReportPath,
|
"--report-json", req.ReportPath,
|
||||||
|
"--transcript-description", "Campaign Session 42",
|
||||||
|
"--config", "/etc/audita/config.yml",
|
||||||
|
"--output-schema", "audita-v1",
|
||||||
|
"--work-dir-retention", "auto",
|
||||||
|
"--total-llm-concurrency", "3",
|
||||||
|
"--proposal-llm-concurrency", "2",
|
||||||
"--validation-model", "openrouter/google/gemma-4-31b-it",
|
"--validation-model", "openrouter/google/gemma-4-31b-it",
|
||||||
"--validation-llm-concurrency", "2",
|
"--validation-llm-concurrency", "2",
|
||||||
}
|
}
|
||||||
@@ -107,8 +119,8 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
|||||||
if rec.Env["AUDITA_LLM_API_KEY"] != "super-secret" {
|
if rec.Env["AUDITA_LLM_API_KEY"] != "super-secret" {
|
||||||
t.Fatalf("AUDITA_LLM_API_KEY = %q, want propagated secret", rec.Env["AUDITA_LLM_API_KEY"])
|
t.Fatalf("AUDITA_LLM_API_KEY = %q, want propagated secret", rec.Env["AUDITA_LLM_API_KEY"])
|
||||||
}
|
}
|
||||||
if rec.Env["AUDITA_LLM_CONCURRENCY"] != "1" {
|
if rec.Env["AUDITA_LLM_CONCURRENCY"] != "" {
|
||||||
t.Fatalf("AUDITA_LLM_CONCURRENCY = %q, want 1", rec.Env["AUDITA_LLM_CONCURRENCY"])
|
t.Fatalf("AUDITA_LLM_CONCURRENCY = %q, want empty/omitted", rec.Env["AUDITA_LLM_CONCURRENCY"])
|
||||||
}
|
}
|
||||||
|
|
||||||
cfgData, err := os.ReadFile(req.GeneratedConfigPath)
|
cfgData, err := os.ReadFile(req.GeneratedConfigPath)
|
||||||
@@ -124,16 +136,14 @@ func TestSubprocessRunnerMissingConfiguredCredentialFails(t *testing.T) {
|
|||||||
if runtime.GOOS == "windows" {
|
if runtime.GOOS == "windows" {
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
t.Skip("helper wrapper script uses /bin/sh")
|
||||||
}
|
}
|
||||||
llmConcurrency := 1
|
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
LLMAPIKeyEnv: "MISSING_AUDITA_KEY",
|
LLMAPIKeyEnv: "MISSING_AUDITA_KEY",
|
||||||
Modules: []string{"glossary"},
|
Modules: []string{"glossary"},
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
LLMConcurrency: &llmConcurrency,
|
Report: false,
|
||||||
Report: false,
|
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, false)
|
req := auditaReqForTest(t, false)
|
||||||
|
|
||||||
@@ -155,16 +165,14 @@ func TestSubprocessRunnerUnconfiguredCredentialEnvOmitsCredential(t *testing.T)
|
|||||||
recordPath := filepath.Join(t.TempDir(), "record.json")
|
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||||
|
|
||||||
llmConcurrency := 1
|
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
LLMAPIKeyEnv: "",
|
LLMAPIKeyEnv: "",
|
||||||
Modules: []string{"glossary"},
|
Modules: []string{"glossary"},
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
LLMConcurrency: &llmConcurrency,
|
Report: false,
|
||||||
Report: false,
|
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, false)
|
req := auditaReqForTest(t, false)
|
||||||
|
|
||||||
@@ -211,6 +219,65 @@ func TestSubprocessRunnerInheritsParentEnvironment(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSubprocessRunnerOmitsModulesFlagWhenNotConfigured(t *testing.T) {
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
t.Skip("helper wrapper script uses /bin/sh")
|
||||||
|
}
|
||||||
|
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||||
|
t.Setenv("AUDITA_HELPER_MODE", "success")
|
||||||
|
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||||
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||||
|
|
||||||
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
|
LLMAPIKeyEnv: "",
|
||||||
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
|
Report: false,
|
||||||
|
})
|
||||||
|
req := auditaReqForTest(t, false)
|
||||||
|
if _, err := runner.Run(context.Background(), req); err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
rec := readAuditaHelperRecord(t, recordPath)
|
||||||
|
for i := 0; i < len(rec.Args); i++ {
|
||||||
|
if rec.Args[i] == "--modules" {
|
||||||
|
t.Fatalf("args contained --modules unexpectedly: %#v", rec.Args)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessRunnerOmitsBaseURLAndModelFlagsWhenNotConfigured(t *testing.T) {
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
t.Skip("helper wrapper script uses /bin/sh")
|
||||||
|
}
|
||||||
|
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||||
|
t.Setenv("AUDITA_HELPER_MODE", "success")
|
||||||
|
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||||
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||||
|
|
||||||
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
|
LLMAPIKeyEnv: "",
|
||||||
|
Report: false,
|
||||||
|
})
|
||||||
|
req := auditaReqForTest(t, false)
|
||||||
|
if _, err := runner.Run(context.Background(), req); err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
rec := readAuditaHelperRecord(t, recordPath)
|
||||||
|
for i := 0; i < len(rec.Args); i++ {
|
||||||
|
if rec.Args[i] == "--base-url" {
|
||||||
|
t.Fatalf("args contained --base-url unexpectedly: %#v", rec.Args)
|
||||||
|
}
|
||||||
|
if rec.Args[i] == "--model" {
|
||||||
|
t.Fatalf("args contained --model unexpectedly: %#v", rec.Args)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
|
func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
if runtime.GOOS == "windows" {
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
t.Skip("helper wrapper script uses /bin/sh")
|
||||||
@@ -220,16 +287,14 @@ func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
|
|||||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
|
|
||||||
llmConcurrency := 1
|
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||||
Modules: []string{"glossary"},
|
Modules: []string{"glossary"},
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
LLMConcurrency: &llmConcurrency,
|
Report: true,
|
||||||
Report: true,
|
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, true)
|
req := auditaReqForTest(t, true)
|
||||||
_, err := runner.Run(context.Background(), req)
|
_, err := runner.Run(context.Background(), req)
|
||||||
@@ -256,16 +321,14 @@ func TestSubprocessRunnerSubprocessFailureAddsStderrDescriptorHint(t *testing.T)
|
|||||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
|
|
||||||
llmConcurrency := 1
|
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||||
Modules: []string{"glossary"},
|
Modules: []string{"glossary"},
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
LLMConcurrency: &llmConcurrency,
|
Report: true,
|
||||||
Report: true,
|
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, true)
|
req := auditaReqForTest(t, true)
|
||||||
_, err := runner.Run(context.Background(), req)
|
_, err := runner.Run(context.Background(), req)
|
||||||
@@ -286,16 +349,14 @@ func TestSubprocessRunnerMissingOutputFails(t *testing.T) {
|
|||||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
|
|
||||||
llmConcurrency := 1
|
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||||
Modules: []string{"glossary"},
|
Modules: []string{"glossary"},
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
LLMConcurrency: &llmConcurrency,
|
Report: false,
|
||||||
Report: false,
|
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, false)
|
req := auditaReqForTest(t, false)
|
||||||
_, err := runner.Run(context.Background(), req)
|
_, err := runner.Run(context.Background(), req)
|
||||||
@@ -316,16 +377,14 @@ func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) {
|
|||||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
|
|
||||||
llmConcurrency := 1
|
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||||
Modules: []string{"glossary"},
|
Modules: []string{"glossary"},
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
LLMConcurrency: &llmConcurrency,
|
Report: false,
|
||||||
Report: false,
|
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, false)
|
req := auditaReqForTest(t, false)
|
||||||
_, err := runner.Run(context.Background(), req)
|
_, err := runner.Run(context.Background(), req)
|
||||||
@@ -346,16 +405,14 @@ func TestSubprocessRunnerSegmentsMissingFails(t *testing.T) {
|
|||||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
|
|
||||||
llmConcurrency := 1
|
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||||
Modules: []string{"glossary"},
|
Modules: []string{"glossary"},
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
LLMConcurrency: &llmConcurrency,
|
Report: false,
|
||||||
Report: false,
|
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, false)
|
req := auditaReqForTest(t, false)
|
||||||
_, err := runner.Run(context.Background(), req)
|
_, err := runner.Run(context.Background(), req)
|
||||||
@@ -376,16 +433,14 @@ func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
|
|||||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
|
|
||||||
llmConcurrency := 1
|
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||||
Modules: []string{"glossary"},
|
Modules: []string{"glossary"},
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
LLMConcurrency: &llmConcurrency,
|
Report: true,
|
||||||
Report: true,
|
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, true)
|
req := auditaReqForTest(t, true)
|
||||||
_, err := runner.Run(context.Background(), req)
|
_, err := runner.Run(context.Background(), req)
|
||||||
@@ -398,11 +453,11 @@ func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerConstructorValidation(t *testing.T) {
|
func TestSubprocessRunnerConstructorValidation(t *testing.T) {
|
||||||
_, err := NewSubprocessRunnerFromConfigValues("", "3h", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", nil, "", nil, true)
|
_, err := NewSubprocessRunnerFromConfigValues("", "3h", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", "", "", "", "", nil, nil, "", nil, true)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected binary validation error")
|
t.Fatal("expected binary validation error")
|
||||||
}
|
}
|
||||||
_, err = NewSubprocessRunnerFromConfigValues("audita", "bad", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", nil, "", nil, true)
|
_, err = NewSubprocessRunnerFromConfigValues("audita", "bad", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", "", "", "", "", nil, nil, "", nil, true)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected timeout parse error")
|
t.Fatal("expected timeout parse error")
|
||||||
}
|
}
|
||||||
|
|||||||
29
internal/adapters/storage/factory.go
Normal file
29
internal/adapters/storage/factory.go
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
// NewObjectStoreFromConfig constructs a remote object store from resolved config.
|
||||||
|
func NewObjectStoreFromConfig(ctx context.Context, cfg *config.Config) (ObjectStore, error) {
|
||||||
|
if cfg == nil || cfg.Pipeline == nil {
|
||||||
|
return nil, fmt.Errorf("pipeline config is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
if strings.EqualFold(strings.TrimSpace(cfg.Pipeline.Storage.Backend), "s3") {
|
||||||
|
if cfg.Pipeline.Storage.S3 == nil {
|
||||||
|
return nil, fmt.Errorf("pipeline.storage.s3 is required when pipeline.storage.backend is s3")
|
||||||
|
}
|
||||||
|
return NewS3BackendFromConfig(ctx, *cfg.Pipeline.Storage.S3)
|
||||||
|
}
|
||||||
|
|
||||||
|
if cfg.Pipeline.Storage.S3 != nil && strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket) != "" {
|
||||||
|
return NewS3BackendFromConfig(ctx, *cfg.Pipeline.Storage.S3)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil, fmt.Errorf("no remote object store backend is configured")
|
||||||
|
}
|
||||||
56
internal/adapters/storage/factory_test.go
Normal file
56
internal/adapters/storage/factory_test.go
Normal file
@@ -0,0 +1,56 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNewObjectStoreFromConfigBuildsS3WhenBackendIsS3(t *testing.T) {
|
||||||
|
original := newS3Client
|
||||||
|
t.Cleanup(func() { newS3Client = original })
|
||||||
|
newS3Client = func(_ context.Context, _ s3ClientOptions) (s3API, error) {
|
||||||
|
return &fakeS3API{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
store, err := NewObjectStoreFromConfig(context.Background(), &config.Config{
|
||||||
|
Pipeline: &config.PipelineConfig{
|
||||||
|
Storage: config.StorageConfig{
|
||||||
|
Backend: "s3",
|
||||||
|
S3: &config.StorageS3Config{
|
||||||
|
Bucket: "my-archive",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewObjectStoreFromConfig() error = %v", err)
|
||||||
|
}
|
||||||
|
if _, ok := store.(*S3Backend); !ok {
|
||||||
|
t.Fatalf("store type = %T, want *S3Backend", store)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewObjectStoreFromConfigRequiresS3ConfigWhenBackendIsS3(t *testing.T) {
|
||||||
|
_, err := NewObjectStoreFromConfig(context.Background(), &config.Config{
|
||||||
|
Pipeline: &config.PipelineConfig{
|
||||||
|
Storage: config.StorageConfig{Backend: "s3"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "pipeline.storage.s3 is required") {
|
||||||
|
t.Fatalf("NewObjectStoreFromConfig() error = %v, want missing storage.s3 error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewObjectStoreFromConfigNoRemoteBackendConfigured(t *testing.T) {
|
||||||
|
_, err := NewObjectStoreFromConfig(context.Background(), &config.Config{
|
||||||
|
Pipeline: &config.PipelineConfig{
|
||||||
|
Storage: config.StorageConfig{Backend: "local"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "no remote object store backend is configured") {
|
||||||
|
t.Fatalf("NewObjectStoreFromConfig() error = %v, want no-backend error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,6 +1,14 @@
|
|||||||
package storage
|
package storage
|
||||||
|
|
||||||
import "context"
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
// NoopBackend is a deterministic no-op archive/storage adapter.
|
// NoopBackend is a deterministic no-op archive/storage adapter.
|
||||||
type NoopBackend struct{}
|
type NoopBackend struct{}
|
||||||
@@ -18,6 +26,21 @@ type FakeBackend struct {
|
|||||||
Requests []ArchiveRequest
|
Requests []ArchiveRequest
|
||||||
Err error
|
Err error
|
||||||
Result ArchiveResult
|
Result ArchiveResult
|
||||||
|
|
||||||
|
Objects map[string]FakeObject
|
||||||
|
Uploads []FakeUploadCall
|
||||||
|
|
||||||
|
ListErr error
|
||||||
|
DownloadErr error
|
||||||
|
UploadErr error
|
||||||
|
ExistsErr error
|
||||||
|
}
|
||||||
|
|
||||||
|
// FakeUploadCall captures one upload invocation in call order.
|
||||||
|
type FakeUploadCall struct {
|
||||||
|
LocalPath string
|
||||||
|
Key string
|
||||||
|
Options UploadOptions
|
||||||
}
|
}
|
||||||
|
|
||||||
// Archive records request and returns configured response.
|
// Archive records request and returns configured response.
|
||||||
@@ -38,3 +61,148 @@ func (f *FakeBackend) Archive(ctx context.Context, req ArchiveRequest) (ArchiveR
|
|||||||
}
|
}
|
||||||
return res, nil
|
return res, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// FakeObject is a deterministic fake object-store record.
|
||||||
|
type FakeObject struct {
|
||||||
|
Key string
|
||||||
|
Data []byte
|
||||||
|
Metadata map[string]string
|
||||||
|
ETag string
|
||||||
|
LastModified *time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// SeedObject inserts or replaces an object in the fake object store.
|
||||||
|
func (f *FakeBackend) SeedObject(obj FakeObject) {
|
||||||
|
if f.Objects == nil {
|
||||||
|
f.Objects = map[string]FakeObject{}
|
||||||
|
}
|
||||||
|
key := normalizeObjectKey(obj.Key)
|
||||||
|
obj.Key = key
|
||||||
|
obj.Data = append([]byte(nil), obj.Data...)
|
||||||
|
obj.Metadata = copyMetadata(obj.Metadata)
|
||||||
|
f.Objects[key] = obj
|
||||||
|
}
|
||||||
|
|
||||||
|
// List returns deterministic prefix-filtered objects.
|
||||||
|
func (f *FakeBackend) List(ctx context.Context, prefix string) ([]ObjectInfo, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if f.ListErr != nil {
|
||||||
|
return nil, f.ListErr
|
||||||
|
}
|
||||||
|
|
||||||
|
normalizedPrefix := normalizeObjectKey(prefix)
|
||||||
|
keys := make([]string, 0, len(f.Objects))
|
||||||
|
for key := range f.Objects {
|
||||||
|
if strings.HasPrefix(key, normalizedPrefix) {
|
||||||
|
keys = append(keys, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
|
||||||
|
out := make([]ObjectInfo, 0, len(keys))
|
||||||
|
for _, key := range keys {
|
||||||
|
obj := f.Objects[key]
|
||||||
|
out = append(out, ObjectInfo{
|
||||||
|
Key: obj.Key,
|
||||||
|
Size: int64(len(obj.Data)),
|
||||||
|
ETag: obj.ETag,
|
||||||
|
LastModified: obj.LastModified,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Download writes one object to a local path.
|
||||||
|
func (f *FakeBackend) Download(ctx context.Context, key, localPath string) error {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if f.DownloadErr != nil {
|
||||||
|
return f.DownloadErr
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(localPath) == "" {
|
||||||
|
return fmt.Errorf("download object: local path is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
obj, ok := f.Objects[normalizeObjectKey(key)]
|
||||||
|
if !ok {
|
||||||
|
return fmt.Errorf("download object %q: %w", key, os.ErrNotExist)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
|
||||||
|
return fmt.Errorf("download object %q: create parent directory: %w", key, err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(localPath, obj.Data, 0o644); err != nil {
|
||||||
|
return fmt.Errorf("download object %q: write local file: %w", key, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Upload reads a local file and stores it under key.
|
||||||
|
func (f *FakeBackend) Upload(ctx context.Context, localPath, key string, opts UploadOptions) (ObjectInfo, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return ObjectInfo{}, err
|
||||||
|
}
|
||||||
|
if f.UploadErr != nil {
|
||||||
|
return ObjectInfo{}, f.UploadErr
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(localPath) == "" {
|
||||||
|
return ObjectInfo{}, fmt.Errorf("upload object: local path is required")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(key) == "" {
|
||||||
|
return ObjectInfo{}, fmt.Errorf("upload object: key is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := os.ReadFile(localPath)
|
||||||
|
if err != nil {
|
||||||
|
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: %w", key, localPath, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
normalizedKey := normalizeObjectKey(key)
|
||||||
|
f.Uploads = append(f.Uploads, FakeUploadCall{
|
||||||
|
LocalPath: localPath,
|
||||||
|
Key: normalizedKey,
|
||||||
|
Options: UploadOptions{
|
||||||
|
Metadata: copyMetadata(opts.Metadata),
|
||||||
|
ContentType: opts.ContentType,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
now := time.Now().UTC()
|
||||||
|
obj := FakeObject{
|
||||||
|
Key: normalizedKey,
|
||||||
|
Data: data,
|
||||||
|
Metadata: copyMetadata(opts.Metadata),
|
||||||
|
LastModified: &now,
|
||||||
|
}
|
||||||
|
f.SeedObject(obj)
|
||||||
|
return ObjectInfo{
|
||||||
|
Key: normalizedKey,
|
||||||
|
Size: int64(len(data)),
|
||||||
|
LastModified: &now,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Exists checks object presence.
|
||||||
|
func (f *FakeBackend) Exists(ctx context.Context, key string) (bool, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
if f.ExistsErr != nil {
|
||||||
|
return false, f.ExistsErr
|
||||||
|
}
|
||||||
|
_, ok := f.Objects[normalizeObjectKey(key)]
|
||||||
|
return ok, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyMetadata(in map[string]string) map[string]string {
|
||||||
|
if len(in) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make(map[string]string, len(in))
|
||||||
|
for k, v := range in {
|
||||||
|
out[k] = v
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,6 +3,9 @@ package storage
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -29,3 +32,84 @@ func TestFakeBackendError(t *testing.T) {
|
|||||||
t.Fatal("expected error, got nil")
|
t.Fatal("expected error, got nil")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFakeBackendListPrefixFiltering(t *testing.T) {
|
||||||
|
fake := &FakeBackend{}
|
||||||
|
fake.SeedObject(FakeObject{Key: "dnd/campaigns/forsaken/audio/a.flac", Data: []byte("a")})
|
||||||
|
fake.SeedObject(FakeObject{Key: "dnd/campaigns/forsaken/audio/b.flac", Data: []byte("b")})
|
||||||
|
fake.SeedObject(FakeObject{Key: "dnd/campaigns/other/audio/c.flac", Data: []byte("c")})
|
||||||
|
|
||||||
|
items, err := fake.List(context.Background(), "dnd/campaigns/forsaken/audio/")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("List() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(items) != 2 {
|
||||||
|
t.Fatalf("List() len = %d, want 2", len(items))
|
||||||
|
}
|
||||||
|
if items[0].Key != "dnd/campaigns/forsaken/audio/a.flac" || items[1].Key != "dnd/campaigns/forsaken/audio/b.flac" {
|
||||||
|
t.Fatalf("List() keys = %#v", items)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFakeBackendDownload(t *testing.T) {
|
||||||
|
fake := &FakeBackend{}
|
||||||
|
fake.SeedObject(FakeObject{Key: "audio/a.flac", Data: []byte("audio-a")})
|
||||||
|
|
||||||
|
dst := filepath.Join(t.TempDir(), "nested", "a.flac")
|
||||||
|
if err := fake.Download(context.Background(), "audio/a.flac", dst); err != nil {
|
||||||
|
t.Fatalf("Download() error = %v", err)
|
||||||
|
}
|
||||||
|
data, err := os.ReadFile(dst)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadFile() error = %v", err)
|
||||||
|
}
|
||||||
|
if string(data) != "audio-a" {
|
||||||
|
t.Fatalf("downloaded content = %q, want %q", string(data), "audio-a")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFakeBackendUploadAndExists(t *testing.T) {
|
||||||
|
fake := &FakeBackend{}
|
||||||
|
|
||||||
|
local := filepath.Join(t.TempDir(), "upload.txt")
|
||||||
|
if err := os.WriteFile(local, []byte("payload"), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
info, err := fake.Upload(context.Background(), local, `runs\id\artifact.txt`, UploadOptions{
|
||||||
|
Metadata: map[string]string{"kind": "artifact"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Upload() error = %v", err)
|
||||||
|
}
|
||||||
|
if info.Key != "runs/id/artifact.txt" {
|
||||||
|
t.Fatalf("Upload() key = %q, want normalized key", info.Key)
|
||||||
|
}
|
||||||
|
|
||||||
|
ok, err := fake.Exists(context.Background(), "runs/id/artifact.txt")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Exists() error = %v", err)
|
||||||
|
}
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("Exists() = false, want true")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFakeBackendObjectErrors(t *testing.T) {
|
||||||
|
fake := &FakeBackend{DownloadErr: errors.New("download fail"), UploadErr: errors.New("upload fail"), ListErr: errors.New("list fail"), ExistsErr: errors.New("exists fail")}
|
||||||
|
|
||||||
|
if _, err := fake.List(context.Background(), "x"); err == nil || !strings.Contains(err.Error(), "list fail") {
|
||||||
|
t.Fatalf("List() error = %v, want list fail", err)
|
||||||
|
}
|
||||||
|
if err := fake.Download(context.Background(), "x", filepath.Join(t.TempDir(), "x")); err == nil || !strings.Contains(err.Error(), "download fail") {
|
||||||
|
t.Fatalf("Download() error = %v, want download fail", err)
|
||||||
|
}
|
||||||
|
local := filepath.Join(t.TempDir(), "x.txt")
|
||||||
|
_ = os.WriteFile(local, []byte("x"), 0o644)
|
||||||
|
if _, err := fake.Upload(context.Background(), local, "x", UploadOptions{}); err == nil || !strings.Contains(err.Error(), "upload fail") {
|
||||||
|
t.Fatalf("Upload() error = %v, want upload fail", err)
|
||||||
|
}
|
||||||
|
if _, err := fake.Exists(context.Background(), "x"); err == nil || !strings.Contains(err.Error(), "exists fail") {
|
||||||
|
t.Fatalf("Exists() error = %v, want exists fail", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
8
internal/adapters/storage/keys.go
Normal file
8
internal/adapters/storage/keys.go
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import "strings"
|
||||||
|
|
||||||
|
func normalizeObjectKey(key string) string {
|
||||||
|
normalized := strings.ReplaceAll(strings.TrimSpace(key), "\\", "/")
|
||||||
|
return strings.TrimLeft(normalized, "/")
|
||||||
|
}
|
||||||
21
internal/adapters/storage/keys_test.go
Normal file
21
internal/adapters/storage/keys_test.go
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestNormalizeObjectKey(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
in string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{in: `dnd\campaigns\forsaken\a.flac`, want: "dnd/campaigns/forsaken/a.flac"},
|
||||||
|
{in: " /dnd/campaigns/forsaken/a.flac ", want: "dnd/campaigns/forsaken/a.flac"},
|
||||||
|
{in: "//dnd/campaigns/forsaken/a.flac", want: "dnd/campaigns/forsaken/a.flac"},
|
||||||
|
{in: "", want: ""},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
if got := normalizeObjectKey(tt.in); got != tt.want {
|
||||||
|
t.Fatalf("normalizeObjectKey(%q) = %q, want %q", tt.in, got, tt.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
32
internal/adapters/storage/object_store.go
Normal file
32
internal/adapters/storage/object_store.go
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ObjectStore is a remote object storage boundary used by future prepare/archive work.
|
||||||
|
//
|
||||||
|
// Key invariant:
|
||||||
|
// callers pass full bucket-relative object keys. Backend implementations do not
|
||||||
|
// infer Narratio session semantics and do not prepend root prefixes.
|
||||||
|
type ObjectStore interface {
|
||||||
|
List(ctx context.Context, prefix string) ([]ObjectInfo, error)
|
||||||
|
Download(ctx context.Context, key, localPath string) error
|
||||||
|
Upload(ctx context.Context, localPath, key string, opts UploadOptions) (ObjectInfo, error)
|
||||||
|
Exists(ctx context.Context, key string) (bool, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ObjectInfo describes one object in remote storage.
|
||||||
|
type ObjectInfo struct {
|
||||||
|
Key string
|
||||||
|
Size int64
|
||||||
|
ETag string
|
||||||
|
LastModified *time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// UploadOptions configures optional object upload metadata.
|
||||||
|
type UploadOptions struct {
|
||||||
|
Metadata map[string]string
|
||||||
|
ContentType string
|
||||||
|
}
|
||||||
275
internal/adapters/storage/s3_backend.go
Normal file
275
internal/adapters/storage/s3_backend.go
Normal file
@@ -0,0 +1,275 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
awsconfig "github.com/aws/aws-sdk-go-v2/config"
|
||||||
|
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||||
|
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||||
|
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||||
|
"github.com/aws/smithy-go"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
type s3API interface {
|
||||||
|
ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error)
|
||||||
|
GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error)
|
||||||
|
PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error)
|
||||||
|
HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// S3Backend is an ObjectStore implementation backed by S3-compatible APIs.
|
||||||
|
type S3Backend struct {
|
||||||
|
bucket string
|
||||||
|
client s3API
|
||||||
|
}
|
||||||
|
|
||||||
|
type s3ClientOptions struct {
|
||||||
|
Region string
|
||||||
|
Endpoint string
|
||||||
|
ForcePathStyle bool
|
||||||
|
AccessKeyID string
|
||||||
|
SecretKey string
|
||||||
|
}
|
||||||
|
|
||||||
|
var newS3Client = func(ctx context.Context, opts s3ClientOptions) (s3API, error) {
|
||||||
|
loadOpts := make([]func(*awsconfig.LoadOptions) error, 0, 1)
|
||||||
|
if strings.TrimSpace(opts.Region) != "" {
|
||||||
|
loadOpts = append(loadOpts, awsconfig.WithRegion(strings.TrimSpace(opts.Region)))
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(opts.AccessKeyID) != "" && strings.TrimSpace(opts.SecretKey) != "" {
|
||||||
|
loadOpts = append(loadOpts, awsconfig.WithCredentialsProvider(
|
||||||
|
credentials.NewStaticCredentialsProvider(
|
||||||
|
strings.TrimSpace(opts.AccessKeyID),
|
||||||
|
strings.TrimSpace(opts.SecretKey),
|
||||||
|
"",
|
||||||
|
),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
awsCfg, err := awsconfig.LoadDefaultConfig(ctx, loadOpts...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("load aws config: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return s3.NewFromConfig(awsCfg, func(o *s3.Options) {
|
||||||
|
if strings.TrimSpace(opts.Endpoint) != "" {
|
||||||
|
endpoint := strings.TrimSpace(opts.Endpoint)
|
||||||
|
o.BaseEndpoint = &endpoint
|
||||||
|
}
|
||||||
|
o.UsePathStyle = opts.ForcePathStyle
|
||||||
|
}), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewS3BackendFromConfig builds an S3 backend from resolved config.
|
||||||
|
func NewS3BackendFromConfig(ctx context.Context, cfg config.StorageS3Config) (*S3Backend, error) {
|
||||||
|
bucket := strings.TrimSpace(cfg.Bucket)
|
||||||
|
if bucket == "" {
|
||||||
|
return nil, fmt.Errorf("storage.s3.bucket is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
client, err := newS3Client(ctx, s3ClientOptions{
|
||||||
|
Region: cfg.Region,
|
||||||
|
Endpoint: cfg.Endpoint,
|
||||||
|
ForcePathStyle: cfg.ForcePathStyle,
|
||||||
|
AccessKeyID: s3CredentialFromEnv(orDefaultEnvName(cfg.AccessKeyIDEnv, config.DefaultS3AccessKeyIDEnv)),
|
||||||
|
SecretKey: s3CredentialFromEnv(orDefaultEnvName(cfg.SecretKeyEnv, config.DefaultS3SecretAccessKeyEnv)),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("build s3 client: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &S3Backend{
|
||||||
|
bucket: bucket,
|
||||||
|
client: client,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func s3CredentialFromEnv(envVarName string) string {
|
||||||
|
name := strings.TrimSpace(envVarName)
|
||||||
|
if name == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
value, ok := os.LookupEnv(name)
|
||||||
|
if !ok {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(value)
|
||||||
|
}
|
||||||
|
|
||||||
|
func orDefaultEnvName(name, fallback string) string {
|
||||||
|
trimmed := strings.TrimSpace(name)
|
||||||
|
if trimmed == "" {
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
return trimmed
|
||||||
|
}
|
||||||
|
|
||||||
|
// List returns objects under prefix.
|
||||||
|
func (b *S3Backend) List(ctx context.Context, prefix string) ([]ObjectInfo, error) {
|
||||||
|
normalizedPrefix := normalizeObjectKey(prefix)
|
||||||
|
out := make([]ObjectInfo, 0)
|
||||||
|
var token *string
|
||||||
|
|
||||||
|
for {
|
||||||
|
resp, err := b.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
||||||
|
Bucket: &b.bucket,
|
||||||
|
Prefix: &normalizedPrefix,
|
||||||
|
ContinuationToken: token,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("list objects under %q: %w", normalizedPrefix, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, item := range resp.Contents {
|
||||||
|
var lastModified *time.Time
|
||||||
|
if item.LastModified != nil {
|
||||||
|
t := *item.LastModified
|
||||||
|
lastModified = &t
|
||||||
|
}
|
||||||
|
out = append(out, ObjectInfo{
|
||||||
|
Key: normalizeObjectKey(valueOrEmpty(item.Key)),
|
||||||
|
Size: valueOrZeroInt64(item.Size),
|
||||||
|
ETag: strings.Trim(valueOrEmpty(item.ETag), "\""),
|
||||||
|
LastModified: lastModified,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
if !valueOrFalseBool(resp.IsTruncated) || resp.NextContinuationToken == nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
token = resp.NextContinuationToken
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Download retrieves one object to localPath, creating parent directories as needed.
|
||||||
|
func (b *S3Backend) Download(ctx context.Context, key, localPath string) error {
|
||||||
|
normalizedKey := normalizeObjectKey(key)
|
||||||
|
if strings.TrimSpace(localPath) == "" {
|
||||||
|
return fmt.Errorf("download object: local path is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := b.client.GetObject(ctx, &s3.GetObjectInput{
|
||||||
|
Bucket: &b.bucket,
|
||||||
|
Key: &normalizedKey,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("download object %q: %w", normalizedKey, err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
|
||||||
|
return fmt.Errorf("download object %q: create parent directory: %w", normalizedKey, err)
|
||||||
|
}
|
||||||
|
dst, err := os.Create(localPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("download object %q: create local file: %w", normalizedKey, err)
|
||||||
|
}
|
||||||
|
defer dst.Close()
|
||||||
|
|
||||||
|
if _, err := io.Copy(dst, resp.Body); err != nil {
|
||||||
|
return fmt.Errorf("download object %q: copy body: %w", normalizedKey, err)
|
||||||
|
}
|
||||||
|
if err := dst.Sync(); err != nil {
|
||||||
|
return fmt.Errorf("download object %q: sync local file: %w", normalizedKey, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Upload sends a local file to key.
|
||||||
|
func (b *S3Backend) Upload(ctx context.Context, localPath, key string, opts UploadOptions) (ObjectInfo, error) {
|
||||||
|
normalizedKey := normalizeObjectKey(key)
|
||||||
|
if strings.TrimSpace(localPath) == "" {
|
||||||
|
return ObjectInfo{}, fmt.Errorf("upload object: local path is required")
|
||||||
|
}
|
||||||
|
if normalizedKey == "" {
|
||||||
|
return ObjectInfo{}, fmt.Errorf("upload object: key is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
file, err := os.Open(localPath)
|
||||||
|
if err != nil {
|
||||||
|
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: %w", normalizedKey, localPath, err)
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
stat, err := file.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: stat local file: %w", normalizedKey, localPath, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
input := &s3.PutObjectInput{
|
||||||
|
Bucket: &b.bucket,
|
||||||
|
Key: &normalizedKey,
|
||||||
|
Body: file,
|
||||||
|
Metadata: copyMetadata(opts.Metadata),
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(opts.ContentType) != "" {
|
||||||
|
ct := strings.TrimSpace(opts.ContentType)
|
||||||
|
input.ContentType = &ct
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := b.client.PutObject(ctx, input)
|
||||||
|
if err != nil {
|
||||||
|
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: %w", normalizedKey, localPath, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return ObjectInfo{
|
||||||
|
Key: normalizedKey,
|
||||||
|
Size: stat.Size(),
|
||||||
|
ETag: strings.Trim(valueOrEmpty(resp.ETag), "\""),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Exists checks whether one object key exists.
|
||||||
|
func (b *S3Backend) Exists(ctx context.Context, key string) (bool, error) {
|
||||||
|
normalizedKey := normalizeObjectKey(key)
|
||||||
|
_, err := b.client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||||
|
Bucket: &b.bucket,
|
||||||
|
Key: &normalizedKey,
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var notFound *types.NotFound
|
||||||
|
if errors.As(err, ¬Found) {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
var apiErr smithy.APIError
|
||||||
|
if errors.As(err, &apiErr) {
|
||||||
|
switch apiErr.ErrorCode() {
|
||||||
|
case "NotFound", "NoSuchKey", "404":
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false, fmt.Errorf("head object %q: %w", normalizedKey, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func valueOrEmpty(v *string) string {
|
||||||
|
if v == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return *v
|
||||||
|
}
|
||||||
|
|
||||||
|
func valueOrZeroInt64(v *int64) int64 {
|
||||||
|
if v == nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return *v
|
||||||
|
}
|
||||||
|
|
||||||
|
func valueOrFalseBool(v *bool) bool {
|
||||||
|
if v == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return *v
|
||||||
|
}
|
||||||
253
internal/adapters/storage/s3_backend_test.go
Normal file
253
internal/adapters/storage/s3_backend_test.go
Normal file
@@ -0,0 +1,253 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||||
|
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||||
|
"github.com/aws/smithy-go"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
type fakeS3API struct {
|
||||||
|
listOut *s3.ListObjectsV2Output
|
||||||
|
listErr error
|
||||||
|
|
||||||
|
getBody io.ReadCloser
|
||||||
|
getErr error
|
||||||
|
|
||||||
|
putOut *s3.PutObjectOutput
|
||||||
|
putErr error
|
||||||
|
|
||||||
|
headErr error
|
||||||
|
|
||||||
|
lastList *s3.ListObjectsV2Input
|
||||||
|
lastGet *s3.GetObjectInput
|
||||||
|
lastPut *s3.PutObjectInput
|
||||||
|
lastHead *s3.HeadObjectInput
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeS3API) ListObjectsV2(_ context.Context, params *s3.ListObjectsV2Input, _ ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) {
|
||||||
|
f.lastList = params
|
||||||
|
if f.listErr != nil {
|
||||||
|
return nil, f.listErr
|
||||||
|
}
|
||||||
|
if f.listOut == nil {
|
||||||
|
return &s3.ListObjectsV2Output{}, nil
|
||||||
|
}
|
||||||
|
return f.listOut, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeS3API) GetObject(_ context.Context, params *s3.GetObjectInput, _ ...func(*s3.Options)) (*s3.GetObjectOutput, error) {
|
||||||
|
f.lastGet = params
|
||||||
|
if f.getErr != nil {
|
||||||
|
return nil, f.getErr
|
||||||
|
}
|
||||||
|
body := f.getBody
|
||||||
|
if body == nil {
|
||||||
|
body = io.NopCloser(strings.NewReader(""))
|
||||||
|
}
|
||||||
|
return &s3.GetObjectOutput{Body: body}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeS3API) PutObject(_ context.Context, params *s3.PutObjectInput, _ ...func(*s3.Options)) (*s3.PutObjectOutput, error) {
|
||||||
|
f.lastPut = params
|
||||||
|
if f.putErr != nil {
|
||||||
|
return nil, f.putErr
|
||||||
|
}
|
||||||
|
if f.putOut == nil {
|
||||||
|
return &s3.PutObjectOutput{}, nil
|
||||||
|
}
|
||||||
|
return f.putOut, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeS3API) HeadObject(_ context.Context, params *s3.HeadObjectInput, _ ...func(*s3.Options)) (*s3.HeadObjectOutput, error) {
|
||||||
|
f.lastHead = params
|
||||||
|
if f.headErr != nil {
|
||||||
|
return nil, f.headErr
|
||||||
|
}
|
||||||
|
return &s3.HeadObjectOutput{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestS3BackendListAndKeyNormalization(t *testing.T) {
|
||||||
|
lastModified := time.Date(2026, 5, 16, 12, 0, 0, 0, time.UTC)
|
||||||
|
client := &fakeS3API{
|
||||||
|
listOut: &s3.ListObjectsV2Output{
|
||||||
|
Contents: []types.Object{
|
||||||
|
{Key: strPtr(`dnd\campaigns\forsaken\a.flac`), Size: int64Ptr(7), ETag: strPtr(`"abc"`), LastModified: &lastModified},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
backend := &S3Backend{bucket: "bucket-1", client: client}
|
||||||
|
|
||||||
|
items, err := backend.List(context.Background(), `dnd\campaigns\`)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("List() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(items) != 1 {
|
||||||
|
t.Fatalf("List() len = %d, want 1", len(items))
|
||||||
|
}
|
||||||
|
if items[0].Key != "dnd/campaigns/forsaken/a.flac" {
|
||||||
|
t.Fatalf("List() key = %q, want normalized slash key", items[0].Key)
|
||||||
|
}
|
||||||
|
if items[0].ETag != "abc" {
|
||||||
|
t.Fatalf("List() ETag = %q, want %q", items[0].ETag, "abc")
|
||||||
|
}
|
||||||
|
if client.lastList == nil || *client.lastList.Prefix != "dnd/campaigns/" {
|
||||||
|
t.Fatalf("List() prefix = %#v, want normalized prefix", client.lastList)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestS3BackendDownloadCreatesParentDirectory(t *testing.T) {
|
||||||
|
client := &fakeS3API{getBody: io.NopCloser(strings.NewReader("audio"))}
|
||||||
|
backend := &S3Backend{bucket: "bucket-1", client: client}
|
||||||
|
|
||||||
|
dst := filepath.Join(t.TempDir(), "nested", "clip.flac")
|
||||||
|
if err := backend.Download(context.Background(), `audio\clip.flac`, dst); err != nil {
|
||||||
|
t.Fatalf("Download() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := os.ReadFile(dst)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadFile() error = %v", err)
|
||||||
|
}
|
||||||
|
if string(data) != "audio" {
|
||||||
|
t.Fatalf("downloaded content = %q, want %q", string(data), "audio")
|
||||||
|
}
|
||||||
|
if client.lastGet == nil || *client.lastGet.Key != "audio/clip.flac" {
|
||||||
|
t.Fatalf("GetObject key = %#v, want normalized key", client.lastGet)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestS3BackendUploadAndExists(t *testing.T) {
|
||||||
|
client := &fakeS3API{putOut: &s3.PutObjectOutput{ETag: strPtr(`"etag123"`)}}
|
||||||
|
backend := &S3Backend{bucket: "bucket-1", client: client}
|
||||||
|
|
||||||
|
local := filepath.Join(t.TempDir(), "artifact.txt")
|
||||||
|
if err := os.WriteFile(local, []byte("artifact"), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
info, err := backend.Upload(context.Background(), local, `runs\id\artifact.txt`, UploadOptions{
|
||||||
|
Metadata: map[string]string{"kind": "artifact"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Upload() error = %v", err)
|
||||||
|
}
|
||||||
|
if info.Key != "runs/id/artifact.txt" {
|
||||||
|
t.Fatalf("Upload key = %q, want normalized key", info.Key)
|
||||||
|
}
|
||||||
|
if info.ETag != "etag123" {
|
||||||
|
t.Fatalf("Upload ETag = %q, want %q", info.ETag, "etag123")
|
||||||
|
}
|
||||||
|
if client.lastPut == nil || *client.lastPut.Key != "runs/id/artifact.txt" {
|
||||||
|
t.Fatalf("PutObject key = %#v, want normalized key", client.lastPut)
|
||||||
|
}
|
||||||
|
|
||||||
|
ok, err := backend.Exists(context.Background(), "runs/id/artifact.txt")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Exists() error = %v", err)
|
||||||
|
}
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("Exists() = false, want true")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestS3BackendUploadMissingLocalFile(t *testing.T) {
|
||||||
|
backend := &S3Backend{bucket: "bucket-1", client: &fakeS3API{}}
|
||||||
|
_, err := backend.Upload(context.Background(), filepath.Join(t.TempDir(), "missing.txt"), "key.txt", UploadOptions{})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "no such file") {
|
||||||
|
t.Fatalf("Upload() error = %v, want missing local file error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestS3BackendExistsNotFound(t *testing.T) {
|
||||||
|
backend := &S3Backend{
|
||||||
|
bucket: "bucket-1",
|
||||||
|
client: &fakeS3API{
|
||||||
|
headErr: &smithy.GenericAPIError{Code: "NotFound", Message: "missing"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
ok, err := backend.Exists(context.Background(), "missing-key")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Exists() error = %v", err)
|
||||||
|
}
|
||||||
|
if ok {
|
||||||
|
t.Fatal("Exists() = true, want false")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewS3BackendFromConfigUsesClientOptions(t *testing.T) {
|
||||||
|
original := newS3Client
|
||||||
|
t.Cleanup(func() { newS3Client = original })
|
||||||
|
t.Setenv("OBJECT_STORAGE_KEY_ID", "id-123")
|
||||||
|
t.Setenv("OBJECT_STORAGE_KEY", "secret-abc")
|
||||||
|
|
||||||
|
var got s3ClientOptions
|
||||||
|
newS3Client = func(_ context.Context, opts s3ClientOptions) (s3API, error) {
|
||||||
|
got = opts
|
||||||
|
return &fakeS3API{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
backend, err := NewS3BackendFromConfig(context.Background(), config.StorageS3Config{
|
||||||
|
Bucket: "my-archive",
|
||||||
|
Region: "us-east-1",
|
||||||
|
Endpoint: "http://localhost:9000",
|
||||||
|
ForcePathStyle: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewS3BackendFromConfig() error = %v", err)
|
||||||
|
}
|
||||||
|
if backend.bucket != "my-archive" {
|
||||||
|
t.Fatalf("backend.bucket = %q, want %q", backend.bucket, "my-archive")
|
||||||
|
}
|
||||||
|
if got.Region != "us-east-1" || got.Endpoint != "http://localhost:9000" || !got.ForcePathStyle {
|
||||||
|
t.Fatalf("client options = %#v, want region/endpoint/path-style values", got)
|
||||||
|
}
|
||||||
|
if got.AccessKeyID != "id-123" || got.SecretKey != "secret-abc" {
|
||||||
|
t.Fatalf("client options credentials = %#v, want env-resolved static credentials", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewS3BackendFromConfigRequiresBucket(t *testing.T) {
|
||||||
|
_, err := NewS3BackendFromConfig(context.Background(), config.StorageS3Config{})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "bucket is required") {
|
||||||
|
t.Fatalf("NewS3BackendFromConfig() error = %v, want bucket validation", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewS3BackendFromConfigFallsBackWhenCredentialEnvMissing(t *testing.T) {
|
||||||
|
original := newS3Client
|
||||||
|
t.Cleanup(func() { newS3Client = original })
|
||||||
|
|
||||||
|
var got s3ClientOptions
|
||||||
|
newS3Client = func(_ context.Context, opts s3ClientOptions) (s3API, error) {
|
||||||
|
got = opts
|
||||||
|
return &fakeS3API{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := NewS3BackendFromConfig(context.Background(), config.StorageS3Config{
|
||||||
|
Bucket: "my-archive",
|
||||||
|
Region: "us-east-1",
|
||||||
|
AccessKeyIDEnv: "MISSING_ACCESS_KEY_ID",
|
||||||
|
SecretKeyEnv: "MISSING_SECRET_KEY",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewS3BackendFromConfig() error = %v", err)
|
||||||
|
}
|
||||||
|
if got.AccessKeyID != "" || got.SecretKey != "" {
|
||||||
|
t.Fatalf("client options credentials = %#v, want empty fallback values", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func strPtr(v string) *string { return &v }
|
||||||
|
func int64Ptr(v int64) *int64 { return &v }
|
||||||
|
|
||||||
|
var _ s3API = (*fakeS3API)(nil)
|
||||||
@@ -12,6 +12,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -63,12 +64,13 @@ func TestExecuteMissingRequiredFlags(t *testing.T) {
|
|||||||
args []string
|
args []string
|
||||||
want string
|
want string
|
||||||
}{
|
}{
|
||||||
{name: "run missing flags", args: []string{"run"}, want: "run: --config and --session are required"},
|
{name: "run missing flags", args: []string{"run"}, want: "run: no pipeline config path provided and no default pipeline config found; searched:"},
|
||||||
{name: "plan missing flags", args: []string{"plan"}, want: "plan: --config and --session are required"},
|
{name: "plan missing flags", args: []string{"plan"}, want: "plan: no pipeline config path provided and no default pipeline config found; searched:"},
|
||||||
{name: "status missing flags", args: []string{"status"}, want: "status: --manifest is required"},
|
{name: "status missing flags", args: []string{"status"}, want: "status: --manifest is required"},
|
||||||
{name: "resume missing flags", args: []string{"resume"}, want: "resume: --config and --session are required"},
|
{name: "resume missing flags", args: []string{"resume"}, want: "resume: no pipeline config path provided and no default pipeline config found; searched:"},
|
||||||
{name: "run-stage missing name", args: []string{"run-stage", "--config", "a", "--session", "b"}, want: "run-stage: expected exactly one stage name"},
|
{name: "run-stage missing name", args: []string{"run-stage", "--config", "a", "--session", "b"}, want: "run-stage: expected exactly one stage name"},
|
||||||
{name: "run-stage missing config flags", args: []string{"run-stage", "polish"}, want: "run-stage: --config and --session are required"},
|
{name: "run-stage missing config flags", args: []string{"run-stage", "polish"}, want: "run-stage: no pipeline config path provided and no default pipeline config found; searched:"},
|
||||||
|
{name: "run missing config uses defaults", args: []string{"run", "--session", "session.yml"}, want: "run: no pipeline config path provided and no default pipeline config found; searched:"},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range cases {
|
for _, tc := range cases {
|
||||||
@@ -109,7 +111,7 @@ func TestExecuteRunStageUnknownFails(t *testing.T) {
|
|||||||
func TestExecuteRunStageNormalizeIsAccepted(t *testing.T) {
|
func TestExecuteRunStageNormalizeIsAccepted(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
|
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
|
||||||
workRoot := filepath.Join(workspaceRoot, "work", "2026-05-03")
|
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "processed.json"), `{"segments":[{"id":1}]}`)
|
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "processed.json"), `{"segments":[{"id":1}]}`)
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
@@ -154,7 +156,7 @@ func TestExecuteRunStageTranscribeUsesConfiguredWhisperXServer(t *testing.T) {
|
|||||||
t.Fatal("expected whisperx server to be called at least once")
|
t.Fatal("expected whisperx server to be called at least once")
|
||||||
}
|
}
|
||||||
|
|
||||||
outPath := filepath.Join(workspaceRoot, "work", "2026-05-03", "transcripts", "raw", "alice.json")
|
outPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "transcripts", "raw", "alice.json")
|
||||||
data, err := os.ReadFile(outPath)
|
data, err := os.ReadFile(outPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("ReadFile(%q): %v", outPath, err)
|
t.Fatalf("ReadFile(%q): %v", outPath, err)
|
||||||
@@ -168,6 +170,159 @@ func TestExecuteRunStageTranscribeUsesConfiguredWhisperXServer(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExecuteRunStagePolishLoadsCredentialFromSecretsDir(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
configDir := t.TempDir()
|
||||||
|
sessionID := "2026-05-03"
|
||||||
|
secretsDir := filepath.Join(configDir, "secrets")
|
||||||
|
if err := os.MkdirAll(secretsDir, 0o755); err != nil {
|
||||||
|
t.Fatalf("MkdirAll(%q): %v", secretsDir, err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(secretsDir, "OPENROUTER_API_KEY"), []byte("from-secret-file\n"), 0o600); err != nil {
|
||||||
|
t.Fatalf("write OPENROUTER_API_KEY secret file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
seriatimBinary := writeSeriatimAppTestWrapper(t)
|
||||||
|
auditaBinary := writeAuditaAppTestWrapper(t)
|
||||||
|
t.Setenv("GO_WANT_APP_SERIATIM_HELPER", "1")
|
||||||
|
t.Setenv("GO_WANT_APP_AUDITA_HELPER", "1")
|
||||||
|
|
||||||
|
pipelinePath := filepath.Join(configDir, "pipeline.yml")
|
||||||
|
sessionPath := filepath.Join(configDir, "session.yml")
|
||||||
|
pipelineYAML := `workspace:
|
||||||
|
root: ` + workspaceRoot + `
|
||||||
|
storage:
|
||||||
|
backend: local
|
||||||
|
secrets:
|
||||||
|
env_dir: ./secrets
|
||||||
|
whisperx:
|
||||||
|
transcribe_url: https://example.com/transcribe
|
||||||
|
seriatim:
|
||||||
|
binary: ` + seriatimBinary + `
|
||||||
|
audita:
|
||||||
|
binary: ` + auditaBinary + `
|
||||||
|
llm_api_key_env: OPENROUTER_API_KEY
|
||||||
|
analyzer:
|
||||||
|
timeout: 20m
|
||||||
|
notification:
|
||||||
|
timeout: 10s
|
||||||
|
`
|
||||||
|
sessionYAML := `session_id: ` + sessionID + `
|
||||||
|
campaign: sample-campaign
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write pipeline.yml: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write session.yml: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
originalWD, err := os.Getwd()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Getwd(): %v", err)
|
||||||
|
}
|
||||||
|
if err := os.Chdir(configDir); err != nil {
|
||||||
|
t.Fatalf("Chdir(%q): %v", configDir, err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
_ = os.Chdir(originalWD)
|
||||||
|
})
|
||||||
|
|
||||||
|
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", sessionID)
|
||||||
|
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"schema":"seriatim-intermediate","segments":[]}`)
|
||||||
|
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "[]\n")
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"run-stage", "--config", pipelinePath, "--session", sessionPath, "--force", "polish"}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout.String(), "stage=polish executed=1 skipped=0") {
|
||||||
|
t.Fatalf("stdout = %q, want polish execution", stdout.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteRunFailsWhenConfiguredSecretsDirMissing(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
configDir := t.TempDir()
|
||||||
|
pipelinePath := filepath.Join(configDir, "pipeline.yml")
|
||||||
|
sessionPath := filepath.Join(configDir, "session.yml")
|
||||||
|
|
||||||
|
pipelineYAML := `workspace:
|
||||||
|
root: ` + workspaceRoot + `
|
||||||
|
storage:
|
||||||
|
backend: local
|
||||||
|
secrets:
|
||||||
|
env_dir: ./missing-secrets
|
||||||
|
whisperx:
|
||||||
|
transcribe_url: https://example.com/transcribe
|
||||||
|
seriatim:
|
||||||
|
binary: seriatim
|
||||||
|
audita:
|
||||||
|
binary: audita
|
||||||
|
analyzer:
|
||||||
|
timeout: 20m
|
||||||
|
notification:
|
||||||
|
timeout: 10s
|
||||||
|
`
|
||||||
|
sessionYAML := `session_id: 2026-05-03
|
||||||
|
campaign: sample-campaign
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write pipeline.yml: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write session.yml: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"run", "--config", pipelinePath, "--session", sessionPath}, &stdout, &stderr)
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "read secrets env_dir") {
|
||||||
|
t.Fatalf("stderr = %q, want secrets read-dir error context", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteUsesDefaultPipelineConfigPathWhenConfigFlagOmitted(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_, _ = w.Write([]byte(`{"source":"default-config-test","segments":[{"speaker":"alice"}]}`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
||||||
|
originalDefaults := append([]string(nil), config.DefaultPipelineConfigSearchPaths...)
|
||||||
|
config.DefaultPipelineConfigSearchPaths = []string{pipelinePath}
|
||||||
|
defer func() {
|
||||||
|
config.DefaultPipelineConfigSearchPaths = originalDefaults
|
||||||
|
}()
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"run", "--session", sessionPath}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout.String(), "narratio run: session 2026-05-03; executed=9 skipped=0; manifest=") {
|
||||||
|
t.Fatalf("stdout = %q, want successful run output", stdout.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExecuteInvalidCommand(t *testing.T) {
|
func TestExecuteInvalidCommand(t *testing.T) {
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
@@ -224,6 +379,11 @@ func writeValidConfigFiles(t *testing.T, workspaceRoot string, transcribeURL ...
|
|||||||
root: ` + workspaceRoot + `
|
root: ` + workspaceRoot + `
|
||||||
storage:
|
storage:
|
||||||
backend: s3
|
backend: s3
|
||||||
|
s3:
|
||||||
|
bucket: test-bucket
|
||||||
|
archive:
|
||||||
|
enabled: true
|
||||||
|
upload_run: false
|
||||||
whisperx:
|
whisperx:
|
||||||
transcribe_url: ` + url + `
|
transcribe_url: ` + url + `
|
||||||
timeout: 2s
|
timeout: 2s
|
||||||
@@ -247,6 +407,7 @@ notification:
|
|||||||
`
|
`
|
||||||
|
|
||||||
sessionYAML := `session_id: 2026-05-03
|
sessionYAML := `session_id: 2026-05-03
|
||||||
|
campaign: sample-campaign
|
||||||
inputs:
|
inputs:
|
||||||
audio_dir: ./audio
|
audio_dir: ./audio
|
||||||
speakers_file: ./speakers.yml
|
speakers_file: ./speakers.yml
|
||||||
|
|||||||
49
internal/app/pipeline_config_path.go
Normal file
49
internal/app/pipeline_config_path.go
Normal file
@@ -0,0 +1,49 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func resolvePipelineConfigPath(flagValue string) (string, error) {
|
||||||
|
return resolvePipelineConfigPathWithCandidates(flagValue, config.DefaultPipelineConfigSearchPaths)
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolvePipelineConfigPathWithCandidates(flagValue string, candidates []string) (string, error) {
|
||||||
|
if explicit := strings.TrimSpace(flagValue); explicit != "" {
|
||||||
|
return explicit, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
ordered := make([]string, 0, len(candidates))
|
||||||
|
for _, raw := range candidates {
|
||||||
|
path := strings.TrimSpace(raw)
|
||||||
|
if path == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
ordered = append(ordered, path)
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
if err == nil {
|
||||||
|
if info.IsDir() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return filepath.Clean(path), nil
|
||||||
|
}
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return "", fmt.Errorf("check default pipeline config %q: %w", path, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(ordered) == 0 {
|
||||||
|
return "", fmt.Errorf("no pipeline config path provided and no default locations configured")
|
||||||
|
}
|
||||||
|
return "", fmt.Errorf(
|
||||||
|
"no pipeline config path provided and no default pipeline config found; searched: %s",
|
||||||
|
strings.Join(ordered, ", "),
|
||||||
|
)
|
||||||
|
}
|
||||||
65
internal/app/pipeline_config_path_test.go
Normal file
65
internal/app/pipeline_config_path_test.go
Normal file
@@ -0,0 +1,65 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestResolvePipelineConfigPathWithCandidatesExplicitWins(t *testing.T) {
|
||||||
|
got, err := resolvePipelineConfigPathWithCandidates(" ./custom/pipeline.yml ", []string{"/a", "/b"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolvePipelineConfigPathWithCandidates() error = %v", err)
|
||||||
|
}
|
||||||
|
if got != "./custom/pipeline.yml" {
|
||||||
|
t.Fatalf("resolved path = %q, want explicit path", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePipelineConfigPathWithCandidatesUsesFirstExisting(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
first := filepath.Join(dir, "first.yml")
|
||||||
|
second := filepath.Join(dir, "second.yml")
|
||||||
|
if err := os.WriteFile(second, []byte("workspace:\n root: ./tmp\n"), 0o644); err != nil {
|
||||||
|
t.Fatalf("write second default: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := resolvePipelineConfigPathWithCandidates("", []string{first, second})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolvePipelineConfigPathWithCandidates() error = %v", err)
|
||||||
|
}
|
||||||
|
if got != filepath.Clean(second) {
|
||||||
|
t.Fatalf("resolved path = %q, want %q", got, filepath.Clean(second))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePipelineConfigPathWithCandidatesPrecedence(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
first := filepath.Join(dir, "first.yml")
|
||||||
|
second := filepath.Join(dir, "second.yml")
|
||||||
|
if err := os.WriteFile(first, []byte("workspace:\n root: ./tmp\n"), 0o644); err != nil {
|
||||||
|
t.Fatalf("write first default: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(second, []byte("workspace:\n root: ./tmp\n"), 0o644); err != nil {
|
||||||
|
t.Fatalf("write second default: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := resolvePipelineConfigPathWithCandidates("", []string{first, second})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolvePipelineConfigPathWithCandidates() error = %v", err)
|
||||||
|
}
|
||||||
|
if got != filepath.Clean(first) {
|
||||||
|
t.Fatalf("resolved path = %q, want first candidate %q", got, filepath.Clean(first))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePipelineConfigPathWithCandidatesMissing(t *testing.T) {
|
||||||
|
_, err := resolvePipelineConfigPathWithCandidates("", []string{"/does/not/exist/one.yml", "/does/not/exist/two.yml"})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "no default pipeline config found") {
|
||||||
|
t.Fatalf("error = %q, want missing-defaults context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,9 +5,12 @@ import (
|
|||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/logging"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -18,9 +21,11 @@ func Plan(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
|
|
||||||
var pipelinePath string
|
var pipelinePath string
|
||||||
var sessionPath string
|
var sessionPath string
|
||||||
|
var sessionID string
|
||||||
var force bool
|
var force bool
|
||||||
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml")
|
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
||||||
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
||||||
|
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
|
||||||
fs.BoolVar(&force, "force", false, "force stage execution (reserved for future behavior)")
|
fs.BoolVar(&force, "force", false, "force stage execution (reserved for future behavior)")
|
||||||
|
|
||||||
if err := fs.Parse(args); err != nil {
|
if err := fs.Parse(args); err != nil {
|
||||||
@@ -29,20 +34,30 @@ func Plan(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
if fs.NArg() != 0 {
|
if fs.NArg() != 0 {
|
||||||
return fmt.Errorf("plan: unexpected positional arguments")
|
return fmt.Errorf("plan: unexpected positional arguments")
|
||||||
}
|
}
|
||||||
if pipelinePath == "" || sessionPath == "" {
|
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath)
|
||||||
return fmt.Errorf("plan: --config and --session are required")
|
if err != nil {
|
||||||
|
return fmt.Errorf("plan: %w", err)
|
||||||
|
}
|
||||||
|
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("plan: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg, err := config.Load(pipelinePath, sessionPath)
|
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
|
||||||
|
SessionID: sessionID,
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("plan: %w", err)
|
return fmt.Errorf("plan: %w", err)
|
||||||
}
|
}
|
||||||
if err := config.Validate(cfg); err != nil {
|
if err := config.Validate(cfg); err != nil {
|
||||||
return fmt.Errorf("plan: %w", err)
|
return fmt.Errorf("plan: %w", err)
|
||||||
}
|
}
|
||||||
|
if _, err := loadSecretsFromConfig(cfg, logging.NewLogger(os.Stderr, slog.LevelInfo)); err != nil {
|
||||||
|
return fmt.Errorf("plan: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
store := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root)
|
store := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root)
|
||||||
paths, err := store.EnsureLayout(cfg.Session.SessionID)
|
paths, err := store.EnsureLayoutFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("plan: prepare workdir: %w", err)
|
return fmt.Errorf("plan: prepare workdir: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ func TestPlanCreatesAndReusesWorkdir(t *testing.T) {
|
|||||||
t.Fatalf("first output = %q, want totals", got)
|
t.Fatalf("first output = %q, want totals", got)
|
||||||
}
|
}
|
||||||
|
|
||||||
sessionWorkdir := artifacts.SessionWorkDir(workspaceRoot, "2026-05-03")
|
sessionWorkdir := artifacts.SessionWorkDirForCampaign(workspaceRoot, "sample-campaign", "2026-05-03")
|
||||||
expectedDirs := []string{
|
expectedDirs := []string{
|
||||||
sessionWorkdir,
|
sessionWorkdir,
|
||||||
filepath.Join(sessionWorkdir, "inputs"),
|
filepath.Join(sessionWorkdir, "inputs"),
|
||||||
@@ -63,7 +63,7 @@ func TestPlanCreatesAndReusesWorkdir(t *testing.T) {
|
|||||||
func TestPlanShowsRunAndSkipFromManifest(t *testing.T) {
|
func TestPlanShowsRunAndSkipFromManifest(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
manifestPath := filepath.Join(workspaceRoot, "work", "2026-05-03", "manifest.json")
|
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||||
|
|
||||||
store := &manifest.LocalStore{}
|
store := &manifest.LocalStore{}
|
||||||
m := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
m := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||||
@@ -89,6 +89,54 @@ func TestPlanShowsRunAndSkipFromManifest(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPlanFailsWhenConfiguredSecretsDirMissing(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
configDir := t.TempDir()
|
||||||
|
pipelinePath := filepath.Join(configDir, "pipeline.yml")
|
||||||
|
sessionPath := filepath.Join(configDir, "session.yml")
|
||||||
|
|
||||||
|
pipelineYAML := `workspace:
|
||||||
|
root: ` + workspaceRoot + `
|
||||||
|
storage:
|
||||||
|
backend: local
|
||||||
|
secrets:
|
||||||
|
env_dir: ./missing-secrets
|
||||||
|
whisperx:
|
||||||
|
transcribe_url: https://example.com/transcribe
|
||||||
|
seriatim:
|
||||||
|
binary: seriatim
|
||||||
|
audita:
|
||||||
|
binary: audita
|
||||||
|
analyzer:
|
||||||
|
timeout: 20m
|
||||||
|
notification:
|
||||||
|
timeout: 10s
|
||||||
|
`
|
||||||
|
sessionYAML := `session_id: 2026-05-03
|
||||||
|
campaign: sample-campaign
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write pipeline.yml: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write session.yml: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
err := Plan(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath}, &out)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "read secrets env_dir") {
|
||||||
|
t.Fatalf("error = %q, want secrets read error context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func assertDir(t *testing.T, path string) {
|
func assertDir(t *testing.T, path string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
info, err := os.Stat(path)
|
info, err := os.Stat(path)
|
||||||
|
|||||||
208
internal/app/post_archive_cleanup.go
Normal file
208
internal/app/post_archive_cleanup.go
Normal file
@@ -0,0 +1,208 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
|
)
|
||||||
|
|
||||||
|
func runPostArchiveCleanup(ctx context.Context, env *Env, manifestPath string, m *manifest.Manifest, executed []string) error {
|
||||||
|
if env == nil || env.Config == nil || env.Config.Pipeline == nil || m == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
spoolRequested := env.Config.Pipeline.Spool.DeleteAudioAfterArchive
|
||||||
|
workRequested := env.Config.Pipeline.Workspace.CleanupAfterArchive
|
||||||
|
if !spoolRequested && !workRequested {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
sr := archiveStageRecordForCleanup(m, executed)
|
||||||
|
if sr == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if sr.Metadata == nil {
|
||||||
|
sr.Metadata = map[string]any{}
|
||||||
|
}
|
||||||
|
sr.Metadata["spool_cleanup_requested"] = spoolRequested
|
||||||
|
sr.Metadata["workdir_cleanup_requested"] = workRequested
|
||||||
|
|
||||||
|
eligible, reason := archiveCleanupEligible(env.Config, sr)
|
||||||
|
if !eligible {
|
||||||
|
sr.Metadata["cleanup_skipped"] = true
|
||||||
|
sr.Metadata["cleanup_skipped_reason"] = reason
|
||||||
|
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
|
||||||
|
return fmt.Errorf("save manifest cleanup skip metadata %q: %w", manifestPath, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
spoolDir := strings.TrimSpace(m.LocalSpoolDir)
|
||||||
|
if spoolDir == "" {
|
||||||
|
spoolDir = artifacts.SessionSpoolAudioDir(
|
||||||
|
env.Config.Pipeline.Spool.Root,
|
||||||
|
strings.TrimSpace(env.Config.Session.Campaign),
|
||||||
|
strings.TrimSpace(env.Config.Session.SessionID),
|
||||||
|
strings.TrimSpace(m.RunID),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
workDir := strings.TrimSpace(m.LocalWorkDir)
|
||||||
|
if workDir == "" {
|
||||||
|
workDir = artifacts.SessionRunRootForCampaign(
|
||||||
|
env.Config.Pipeline.Workspace.Root,
|
||||||
|
strings.TrimSpace(env.Config.Session.Campaign),
|
||||||
|
strings.TrimSpace(env.Config.Session.SessionID),
|
||||||
|
strings.TrimSpace(m.RunID),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
if spoolRequested {
|
||||||
|
if err := removeRunScopedDir(strings.TrimSpace(env.Config.Pipeline.Spool.Root), spoolDir, "pipeline.spool.delete_audio_after_archive"); err != nil {
|
||||||
|
sr.Metadata["cleanup_failed"] = true
|
||||||
|
sr.Metadata["cleanup_failed_policy"] = "pipeline.spool.delete_audio_after_archive"
|
||||||
|
sr.Metadata["cleanup_failed_path"] = spoolDir
|
||||||
|
_ = env.ManifestStore.Save(ctx, manifestPath, m)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
sr.Metadata["spool_cleanup_deleted"] = filepath.Clean(spoolDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !workRequested {
|
||||||
|
sr.Metadata["cleanup_completed"] = true
|
||||||
|
sr.Metadata["cleanup_skipped"] = false
|
||||||
|
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
|
||||||
|
return fmt.Errorf("save manifest cleanup metadata %q: %w", manifestPath, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := removeRunScopedDir(strings.TrimSpace(env.Config.Pipeline.Workspace.Root), workDir, "pipeline.workspace.cleanup_after_archive"); err != nil {
|
||||||
|
sr.Metadata["cleanup_failed"] = true
|
||||||
|
sr.Metadata["cleanup_failed_policy"] = "pipeline.workspace.cleanup_after_archive"
|
||||||
|
sr.Metadata["cleanup_failed_path"] = workDir
|
||||||
|
_ = env.ManifestStore.Save(ctx, manifestPath, m)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
sr.Metadata["workdir_cleanup_deleted"] = filepath.Clean(workDir)
|
||||||
|
sr.Metadata["cleanup_completed"] = true
|
||||||
|
sr.Metadata["cleanup_skipped"] = false
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func archiveStageRecordForCleanup(m *manifest.Manifest, executed []string) *manifest.StageRecord {
|
||||||
|
if m == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
archiveRan := false
|
||||||
|
for _, name := range executed {
|
||||||
|
if name == "archive" {
|
||||||
|
archiveRan = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !archiveRan {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
sr := m.Stages["archive"]
|
||||||
|
if sr == nil || sr.Status != manifest.StatusSucceeded {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return sr
|
||||||
|
}
|
||||||
|
|
||||||
|
func archiveCleanupEligible(cfg *config.Config, sr *manifest.StageRecord) (bool, string) {
|
||||||
|
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Archive == nil {
|
||||||
|
return false, "archive configuration is missing"
|
||||||
|
}
|
||||||
|
enabled := true
|
||||||
|
if cfg.Pipeline.Archive.Enabled != nil {
|
||||||
|
enabled = *cfg.Pipeline.Archive.Enabled
|
||||||
|
}
|
||||||
|
if !enabled {
|
||||||
|
return false, "archive.enabled is false"
|
||||||
|
}
|
||||||
|
uploadRun := true
|
||||||
|
if cfg.Pipeline.Archive.UploadRun != nil {
|
||||||
|
uploadRun = *cfg.Pipeline.Archive.UploadRun
|
||||||
|
}
|
||||||
|
if !uploadRun {
|
||||||
|
return false, "archive.upload_run is false"
|
||||||
|
}
|
||||||
|
if sr == nil || sr.Metadata == nil {
|
||||||
|
return false, "archive metadata is missing"
|
||||||
|
}
|
||||||
|
if skipped, _ := sr.Metadata["skipped"].(bool); skipped {
|
||||||
|
return false, "archive stage was skipped"
|
||||||
|
}
|
||||||
|
if uploaded, _ := sr.Metadata["uploaded"].(bool); !uploaded {
|
||||||
|
return false, "archive did not upload run record"
|
||||||
|
}
|
||||||
|
if pointer, _ := sr.Metadata["current_pointer_written"].(bool); !pointer {
|
||||||
|
return false, "archive did not write current pointer"
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(asString(sr.Metadata["current_run_id_key"])) == "" {
|
||||||
|
return false, "archive current run pointer key is missing"
|
||||||
|
}
|
||||||
|
return true, ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func removeRunScopedDir(root, target, policy string) error {
|
||||||
|
cleanRoot := strings.TrimSpace(root)
|
||||||
|
cleanTarget := strings.TrimSpace(target)
|
||||||
|
if cleanRoot == "" {
|
||||||
|
return fmt.Errorf("cleanup policy %s: root path is required", policy)
|
||||||
|
}
|
||||||
|
if cleanTarget == "" {
|
||||||
|
return fmt.Errorf("cleanup policy %s: target path is required", policy)
|
||||||
|
}
|
||||||
|
|
||||||
|
rootAbs, err := filepath.Abs(cleanRoot)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cleanup policy %s: resolve root %q: %w", policy, cleanRoot, err)
|
||||||
|
}
|
||||||
|
targetAbs, err := filepath.Abs(cleanTarget)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cleanup policy %s: resolve target %q: %w", policy, cleanTarget, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
rel, err := filepath.Rel(rootAbs, targetAbs)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cleanup policy %s: relative path from %q to %q: %w", policy, rootAbs, targetAbs, err)
|
||||||
|
}
|
||||||
|
if rel == "." {
|
||||||
|
return fmt.Errorf("cleanup policy %s: refusing to delete root directory %q", policy, rootAbs)
|
||||||
|
}
|
||||||
|
if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
|
||||||
|
return fmt.Errorf("cleanup policy %s: refusing to delete path outside root: root=%q target=%q", policy, rootAbs, targetAbs)
|
||||||
|
}
|
||||||
|
|
||||||
|
info, err := os.Lstat(targetAbs)
|
||||||
|
if err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return fmt.Errorf("cleanup policy %s: stat target %q: %w", policy, targetAbs, err)
|
||||||
|
}
|
||||||
|
if info.Mode()&os.ModeSymlink != 0 {
|
||||||
|
return fmt.Errorf("cleanup policy %s: refusing to delete symlink path %q", policy, targetAbs)
|
||||||
|
}
|
||||||
|
if !info.IsDir() {
|
||||||
|
return fmt.Errorf("cleanup policy %s: target %q is not a directory", policy, targetAbs)
|
||||||
|
}
|
||||||
|
if err := os.RemoveAll(targetAbs); err != nil {
|
||||||
|
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, targetAbs, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func asString(v any) string {
|
||||||
|
s, _ := v.(string)
|
||||||
|
return s
|
||||||
|
}
|
||||||
403
internal/app/post_archive_cleanup_test.go
Normal file
403
internal/app/post_archive_cleanup_test.go
Normal file
@@ -0,0 +1,403 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/stage"
|
||||||
|
)
|
||||||
|
|
||||||
|
type archiveSuccessStage struct {
|
||||||
|
metadata map[string]any
|
||||||
|
}
|
||||||
|
|
||||||
|
func (archiveSuccessStage) Name() string { return "archive" }
|
||||||
|
func (archiveSuccessStage) Declares() stage.IODecl { return stage.IODecl{} }
|
||||||
|
func (s archiveSuccessStage) Run(_ context.Context, _ *stage.Env, _ *manifest.Manifest) (*stage.StageResult, error) {
|
||||||
|
md := map[string]any{
|
||||||
|
"stage": "archive",
|
||||||
|
"uploaded": true,
|
||||||
|
"current_pointer_written": true,
|
||||||
|
"current_run_id_key": "dnd/campaigns/sample-campaign/sessions/2026-05-03/current/run_id.txt",
|
||||||
|
}
|
||||||
|
for k, v := range s.metadata {
|
||||||
|
md[k] = v
|
||||||
|
}
|
||||||
|
return &stage.StageResult{Metadata: md}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type notifyFailStage struct{}
|
||||||
|
|
||||||
|
func (notifyFailStage) Name() string { return "notify" }
|
||||||
|
func (notifyFailStage) Declares() stage.IODecl { return stage.IODecl{} }
|
||||||
|
func (notifyFailStage) Run(_ context.Context, _ *stage.Env, _ *manifest.Manifest) (*stage.StageResult, error) {
|
||||||
|
return nil, errors.New("notify failed")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPostArchiveCleanupDisabledKeepsLocalDirs(t *testing.T) {
|
||||||
|
cfg, seed := cleanupFixtureConfig(t)
|
||||||
|
cfg.Pipeline.Spool.DeleteAudioAfterArchive = false
|
||||||
|
cfg.Pipeline.Workspace.CleanupAfterArchive = false
|
||||||
|
|
||||||
|
if _, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||||
|
t.Fatalf("executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertExists(t, seed.spoolAudioDir)
|
||||||
|
assertExists(t, seed.runWorkDir)
|
||||||
|
assertExists(t, seed.localSourceAudio)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPostArchiveCleanupSpoolOnly(t *testing.T) {
|
||||||
|
cfg, seed := cleanupFixtureConfig(t)
|
||||||
|
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
|
||||||
|
cfg.Pipeline.Workspace.CleanupAfterArchive = false
|
||||||
|
|
||||||
|
if _, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||||
|
t.Fatalf("executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertMissing(t, seed.spoolAudioDir)
|
||||||
|
assertExists(t, seed.runWorkDir)
|
||||||
|
assertExists(t, seed.localSourceAudio)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPostArchiveCleanupWorkdirOnly(t *testing.T) {
|
||||||
|
cfg, seed := cleanupFixtureConfig(t)
|
||||||
|
cfg.Pipeline.Spool.DeleteAudioAfterArchive = false
|
||||||
|
cfg.Pipeline.Workspace.CleanupAfterArchive = true
|
||||||
|
|
||||||
|
if _, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||||
|
t.Fatalf("executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertExists(t, cfg.Pipeline.Workspace.Root)
|
||||||
|
assertExists(t, seed.otherRunDir)
|
||||||
|
assertMissing(t, seed.runWorkDir)
|
||||||
|
assertExists(t, seed.spoolAudioDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPostArchiveCleanupBothPolicies(t *testing.T) {
|
||||||
|
cfg, seed := cleanupFixtureConfig(t)
|
||||||
|
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
|
||||||
|
cfg.Pipeline.Workspace.CleanupAfterArchive = true
|
||||||
|
|
||||||
|
if _, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||||
|
t.Fatalf("executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertMissing(t, seed.spoolAudioDir)
|
||||||
|
assertMissing(t, seed.runWorkDir)
|
||||||
|
assertExists(t, seed.otherRunDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPostArchiveCleanupNotRunWhenArchiveFails(t *testing.T) {
|
||||||
|
cfg, seed := cleanupFixtureConfig(t)
|
||||||
|
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
|
||||||
|
cfg.Pipeline.Workspace.CleanupAfterArchive = true
|
||||||
|
|
||||||
|
_, err := executeStages(context.Background(), cfg, []stage.Stage{failingStage{name: "archive", err: errors.New("archive failed")}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "stage \"archive\" failed") {
|
||||||
|
t.Fatalf("executeStages() error = %v, want archive failure", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertExists(t, seed.spoolAudioDir)
|
||||||
|
assertExists(t, seed.runWorkDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPostArchiveCleanupNotRunWhenArchiveSkipped(t *testing.T) {
|
||||||
|
cfg, seed := cleanupFixtureConfig(t)
|
||||||
|
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
|
||||||
|
cfg.Pipeline.Workspace.CleanupAfterArchive = true
|
||||||
|
|
||||||
|
if _, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{metadata: map[string]any{"skipped": true}}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||||
|
t.Fatalf("executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertExists(t, seed.spoolAudioDir)
|
||||||
|
assertExists(t, seed.runWorkDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPostArchiveCleanupNotRunWhenCurrentPointerMissing(t *testing.T) {
|
||||||
|
cfg, seed := cleanupFixtureConfig(t)
|
||||||
|
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
|
||||||
|
cfg.Pipeline.Workspace.CleanupAfterArchive = true
|
||||||
|
|
||||||
|
if _, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{metadata: map[string]any{"current_pointer_written": false}}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||||
|
t.Fatalf("executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertExists(t, seed.spoolAudioDir)
|
||||||
|
assertExists(t, seed.runWorkDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPostArchiveCleanupNotRunWhenArchiveUploadDisabled(t *testing.T) {
|
||||||
|
cfg, seed := cleanupFixtureConfig(t)
|
||||||
|
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
|
||||||
|
cfg.Pipeline.Workspace.CleanupAfterArchive = true
|
||||||
|
cfg.Pipeline.Archive.UploadRun = boolPtr(false)
|
||||||
|
|
||||||
|
if _, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||||
|
t.Fatalf("executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertExists(t, seed.spoolAudioDir)
|
||||||
|
assertExists(t, seed.runWorkDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPostArchiveCleanupWaitsUntilAllStagesSucceed(t *testing.T) {
|
||||||
|
cfg, seed := cleanupFixtureConfig(t)
|
||||||
|
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
|
||||||
|
cfg.Pipeline.Workspace.CleanupAfterArchive = true
|
||||||
|
|
||||||
|
_, err := executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{}, notifyFailStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "stage \"notify\" failed") {
|
||||||
|
t.Fatalf("executeStages() error = %v, want notify failure", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertExists(t, seed.spoolAudioDir)
|
||||||
|
assertExists(t, seed.runWorkDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPostArchiveCleanupFailsOnUnsafePath(t *testing.T) {
|
||||||
|
cfg, _ := cleanupFixtureConfig(t)
|
||||||
|
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
|
||||||
|
cfg.Pipeline.Workspace.CleanupAfterArchive = false
|
||||||
|
|
||||||
|
manifestPath := manifestPathFor(cfg)
|
||||||
|
store := &manifest.LocalStore{}
|
||||||
|
m, err := store.Load(context.Background(), manifestPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() error = %v", err)
|
||||||
|
}
|
||||||
|
m.LocalSpoolDir = filepath.Join(filepath.Dir(cfg.Pipeline.Spool.Root), "outside-spool")
|
||||||
|
if err := store.Save(context.Background(), manifestPath, m); err != nil {
|
||||||
|
t.Fatalf("Save() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = executeStages(context.Background(), cfg, []stage.Stage{archiveSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "refusing to delete path outside root") {
|
||||||
|
t.Fatalf("executeStages() error = %v, want safe-path failure", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPostArchiveCleanupNotRunWhenPromotionIsMissing(t *testing.T) {
|
||||||
|
cfg, seed, runID := archiveStageCleanupFixture(t)
|
||||||
|
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
|
||||||
|
cfg.Pipeline.Workspace.CleanupAfterArchive = true
|
||||||
|
cfg.Pipeline.Archive.PromoteArtifacts = []config.ArchivePromotionRule{
|
||||||
|
{From: "artifacts/missing.md", To: "artifacts/missing.md", Required: boolPtr(true)},
|
||||||
|
}
|
||||||
|
|
||||||
|
archiveStageImpl, err := stage.Select("archive")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Select(archive) error = %v", err)
|
||||||
|
}
|
||||||
|
_, err = executeStages(context.Background(), cfg, []stage.Stage{archiveStageImpl}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "required promotion source missing") {
|
||||||
|
t.Fatalf("executeStages() error = %v, want promotion-missing failure", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertExists(t, seed.spoolAudioDir)
|
||||||
|
assertExists(t, seed.runWorkDir)
|
||||||
|
assertExists(t, filepath.Join(seed.runWorkDir, "manifest.json"))
|
||||||
|
assertExists(t, artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, runID))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPostArchiveCleanupNotRunWhenCurrentManifestUploadFails(t *testing.T) {
|
||||||
|
cfg, seed, _ := archiveStageCleanupFixture(t)
|
||||||
|
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
|
||||||
|
cfg.Pipeline.Workspace.CleanupAfterArchive = true
|
||||||
|
failKey := seed.sessionPrefix + "current/manifest.json"
|
||||||
|
|
||||||
|
archiveStageImpl, err := stage.Select("archive")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Select(archive) error = %v", err)
|
||||||
|
}
|
||||||
|
_, err = executeStages(context.Background(), cfg, []stage.Stage{archiveStageImpl}, RunOptions{
|
||||||
|
Env: &Env{ObjectStore: &failKeyStore{delegate: &storage.FakeBackend{}, failKey: failKey}},
|
||||||
|
})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "current manifest") {
|
||||||
|
t.Fatalf("executeStages() error = %v, want current-manifest failure", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertExists(t, seed.spoolAudioDir)
|
||||||
|
assertExists(t, seed.runWorkDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPostArchiveCleanupNotRunWhenCurrentPointerUploadFails(t *testing.T) {
|
||||||
|
cfg, seed, _ := archiveStageCleanupFixture(t)
|
||||||
|
cfg.Pipeline.Spool.DeleteAudioAfterArchive = true
|
||||||
|
cfg.Pipeline.Workspace.CleanupAfterArchive = true
|
||||||
|
failKey := seed.sessionPrefix + "current/run_id.txt"
|
||||||
|
|
||||||
|
archiveStageImpl, err := stage.Select("archive")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Select(archive) error = %v", err)
|
||||||
|
}
|
||||||
|
_, err = executeStages(context.Background(), cfg, []stage.Stage{archiveStageImpl}, RunOptions{
|
||||||
|
Env: &Env{ObjectStore: &failKeyStore{delegate: &storage.FakeBackend{}, failKey: failKey}},
|
||||||
|
})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "current run pointer") {
|
||||||
|
t.Fatalf("executeStages() error = %v, want current-run-pointer failure", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertExists(t, seed.spoolAudioDir)
|
||||||
|
assertExists(t, seed.runWorkDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
type cleanupSeed struct {
|
||||||
|
runWorkDir string
|
||||||
|
otherRunDir string
|
||||||
|
spoolAudioDir string
|
||||||
|
localSourceAudio string
|
||||||
|
sessionPrefix string
|
||||||
|
}
|
||||||
|
|
||||||
|
func cleanupFixtureConfig(t *testing.T) (*config.Config, cleanupSeed) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
cfg := testConfig(t)
|
||||||
|
cfg.Pipeline.Archive = &config.ArchiveConfig{Enabled: boolPtr(true), UploadRun: boolPtr(true)}
|
||||||
|
cfg.Pipeline.Spool.Root = filepath.Join(t.TempDir(), "spool")
|
||||||
|
|
||||||
|
runID := "20260516T010203Z-1a2b3c4d"
|
||||||
|
runWorkDir := artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, runID)
|
||||||
|
otherRunDir := artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, "20260516T010204Z-5e6f7a8b")
|
||||||
|
spoolAudioDir := artifacts.SessionSpoolAudioDir(cfg.Pipeline.Spool.Root, cfg.Session.Campaign, cfg.Session.SessionID, runID)
|
||||||
|
|
||||||
|
mustWriteFile(t, filepath.Join(runWorkDir, "manifest.json"), "{}\n")
|
||||||
|
mustWriteFile(t, filepath.Join(runWorkDir, "logs", "stage.log"), "log\n")
|
||||||
|
mustWriteFile(t, filepath.Join(otherRunDir, "logs", "stage.log"), "other\n")
|
||||||
|
mustWriteFile(t, filepath.Join(spoolAudioDir, "speaker.flac"), "flac\n")
|
||||||
|
|
||||||
|
localSourceAudio := filepath.Join(filepath.Dir(cfg.SessionPath), "audio", "alice.flac")
|
||||||
|
mustWriteFile(t, localSourceAudio, "source\n")
|
||||||
|
|
||||||
|
seed := manifest.New(cfg.Session.SessionID, time.Now().UTC())
|
||||||
|
seed.Campaign = cfg.Session.Campaign
|
||||||
|
seed.RunID = runID
|
||||||
|
seed.LocalWorkDir = runWorkDir
|
||||||
|
seed.LocalSpoolDir = spoolAudioDir
|
||||||
|
seed.S3Bucket = "my-dnd-archive"
|
||||||
|
seed.S3SessionPrefix = "dnd/campaigns/sample-campaign/sessions/2026-05-03/"
|
||||||
|
seed.S3RunPrefix = seed.S3SessionPrefix + "runs/" + runID + "/"
|
||||||
|
|
||||||
|
store := &manifest.LocalStore{}
|
||||||
|
if err := os.MkdirAll(filepath.Dir(manifestPathFor(cfg)), 0o755); err != nil {
|
||||||
|
t.Fatalf("MkdirAll() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := store.Save(context.Background(), manifestPathFor(cfg), seed); err != nil {
|
||||||
|
t.Fatalf("seed manifest save error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return cfg, cleanupSeed{
|
||||||
|
runWorkDir: runWorkDir,
|
||||||
|
otherRunDir: otherRunDir,
|
||||||
|
spoolAudioDir: spoolAudioDir,
|
||||||
|
localSourceAudio: localSourceAudio,
|
||||||
|
sessionPrefix: seed.S3SessionPrefix,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func archiveStageCleanupFixture(t *testing.T) (*config.Config, cleanupSeed, string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
cfg, seed := cleanupFixtureConfig(t)
|
||||||
|
runID := "20260516T010203Z-1a2b3c4d"
|
||||||
|
cfg.Pipeline.Storage.S3 = &config.StorageS3Config{
|
||||||
|
Bucket: "my-dnd-archive",
|
||||||
|
RootPrefix: "dnd",
|
||||||
|
}
|
||||||
|
cfg.Pipeline.Archive = &config.ArchiveConfig{
|
||||||
|
Enabled: boolPtr(true),
|
||||||
|
UploadRun: boolPtr(true),
|
||||||
|
PromoteArtifacts: []config.ArchivePromotionRule{
|
||||||
|
{From: "transcripts/trimmed.json", To: "transcripts/trimmed.json", Required: boolPtr(true)},
|
||||||
|
{From: "artifacts/session_recap.md", To: "artifacts/session_recap.md", Required: boolPtr(true)},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
writeArchiveFixtureRunFiles(
|
||||||
|
t,
|
||||||
|
seed.runWorkDir,
|
||||||
|
artifacts.SessionWorkDirForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID),
|
||||||
|
)
|
||||||
|
|
||||||
|
store := &manifest.LocalStore{}
|
||||||
|
seedManifest, err := store.Load(context.Background(), manifestPathFor(cfg))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() error = %v", err)
|
||||||
|
}
|
||||||
|
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze"} {
|
||||||
|
seedManifest.MarkStageSucceeded(name, time.Now().UTC(), nil)
|
||||||
|
}
|
||||||
|
seedManifest.S3SessionPrefix = artifacts.S3SessionPrefix("dnd", cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
seedManifest.S3RunPrefix = artifacts.S3RunPrefix(seedManifest.S3SessionPrefix, runID)
|
||||||
|
if err := store.Save(context.Background(), manifestPathFor(cfg), seedManifest); err != nil {
|
||||||
|
t.Fatalf("Save() error = %v", err)
|
||||||
|
}
|
||||||
|
seed.sessionPrefix = seedManifest.S3SessionPrefix
|
||||||
|
return cfg, seed, runID
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeArchiveFixtureRunFiles(t *testing.T, runWorkDir, sessionRoot string) {
|
||||||
|
t.Helper()
|
||||||
|
mustWriteFile(t, filepath.Join(runWorkDir, "prepare", "inputs", "session.yml"), "session_id: 2026-05-03\n")
|
||||||
|
mustWriteFile(t, filepath.Join(runWorkDir, "transcribe", "outputs", "transcripts", "raw", "speaker.json"), "{}\n")
|
||||||
|
mustWriteFile(t, filepath.Join(runWorkDir, "trim", "outputs", "transcripts", "trimmed.json"), "{}\n")
|
||||||
|
mustWriteFile(t, filepath.Join(runWorkDir, "analyze", "outputs", "artifacts", "session_recap.md"), "# recap\n")
|
||||||
|
mustWriteFile(t, filepath.Join(runWorkDir, "polish", "reports", "audita.report.json"), "{}\n")
|
||||||
|
mustWriteFile(t, filepath.Join(runWorkDir, "merge", "config", "seriatim.generated.yml"), "key: value\n")
|
||||||
|
mustWriteFile(t, filepath.Join(runWorkDir, "logs", "audita.stderr.log"), "stderr\n")
|
||||||
|
mustWriteFile(t, filepath.Join(runWorkDir, "manifest.json"), "{}\n")
|
||||||
|
|
||||||
|
mustWriteFile(t, filepath.Join(sessionRoot, "transcripts", "trimmed.json"), "{}\n")
|
||||||
|
mustWriteFile(t, filepath.Join(sessionRoot, "artifacts", "session_recap.md"), "# recap\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
type failKeyStore struct {
|
||||||
|
delegate *storage.FakeBackend
|
||||||
|
failKey string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *failKeyStore) List(ctx context.Context, prefix string) ([]storage.ObjectInfo, error) {
|
||||||
|
return s.delegate.List(ctx, prefix)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *failKeyStore) Download(ctx context.Context, key, localPath string) error {
|
||||||
|
return s.delegate.Download(ctx, key, localPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *failKeyStore) Upload(ctx context.Context, localPath, key string, opts storage.UploadOptions) (storage.ObjectInfo, error) {
|
||||||
|
if strings.TrimSpace(key) == strings.TrimSpace(s.failKey) {
|
||||||
|
return storage.ObjectInfo{}, errors.New("forced upload failure")
|
||||||
|
}
|
||||||
|
return s.delegate.Upload(ctx, localPath, key, opts)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *failKeyStore) Exists(ctx context.Context, key string) (bool, error) {
|
||||||
|
return s.delegate.Exists(ctx, key)
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertExists(t *testing.T, path string) {
|
||||||
|
t.Helper()
|
||||||
|
if _, err := os.Stat(path); err != nil {
|
||||||
|
t.Fatalf("expected path to exist %q: %v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertMissing(t *testing.T, path string) {
|
||||||
|
t.Helper()
|
||||||
|
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("expected path to be removed %q, stat err=%v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
)
|
)
|
||||||
@@ -17,9 +18,11 @@ func Resume(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
|
|
||||||
var pipelinePath string
|
var pipelinePath string
|
||||||
var sessionPath string
|
var sessionPath string
|
||||||
|
var sessionID string
|
||||||
var force bool
|
var force bool
|
||||||
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml")
|
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
||||||
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
||||||
|
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
|
||||||
fs.BoolVar(&force, "force", false, "force stage execution")
|
fs.BoolVar(&force, "force", false, "force stage execution")
|
||||||
|
|
||||||
if err := fs.Parse(args); err != nil {
|
if err := fs.Parse(args); err != nil {
|
||||||
@@ -28,11 +31,18 @@ func Resume(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
if fs.NArg() != 0 {
|
if fs.NArg() != 0 {
|
||||||
return fmt.Errorf("resume: unexpected positional arguments")
|
return fmt.Errorf("resume: unexpected positional arguments")
|
||||||
}
|
}
|
||||||
if pipelinePath == "" || sessionPath == "" {
|
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath)
|
||||||
return fmt.Errorf("resume: --config and --session are required")
|
if err != nil {
|
||||||
|
return fmt.Errorf("resume: %w", err)
|
||||||
|
}
|
||||||
|
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("resume: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg, err := config.Load(pipelinePath, sessionPath)
|
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
|
||||||
|
SessionID: sessionID,
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("resume: %w", err)
|
return fmt.Errorf("resume: %w", err)
|
||||||
}
|
}
|
||||||
@@ -74,7 +84,11 @@ func Resume(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func loadManifestIfPresent(ctx context.Context, cfg *config.Config) (*manifest.Manifest, error) {
|
func loadManifestIfPresent(ctx context.Context, cfg *config.Config) (*manifest.Manifest, error) {
|
||||||
path := manifestPathFor(cfg)
|
path := artifacts.SessionManifestPathForCampaign(
|
||||||
|
cfg.Pipeline.Workspace.Root,
|
||||||
|
cfg.Session.Campaign,
|
||||||
|
cfg.Session.SessionID,
|
||||||
|
)
|
||||||
exists, err := fileExists(path)
|
exists, err := fileExists(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("check manifest %q: %w", path, err)
|
return nil, fmt.Errorf("check manifest %q: %w", path, err)
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ import (
|
|||||||
func TestResumeStartsAfterCompletedStages(t *testing.T) {
|
func TestResumeStartsAfterCompletedStages(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
manifestPath := filepath.Join(workspaceRoot, "work", "2026-05-03", "manifest.json")
|
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||||
|
|
||||||
store := &manifest.LocalStore{}
|
store := &manifest.LocalStore{}
|
||||||
m := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
m := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||||
@@ -25,7 +25,7 @@ func TestResumeStartsAfterCompletedStages(t *testing.T) {
|
|||||||
if err := store.Save(context.Background(), manifestPath, m); err != nil {
|
if err := store.Save(context.Background(), manifestPath, m); err != nil {
|
||||||
t.Fatalf("save manifest: %v", err)
|
t.Fatalf("save manifest: %v", err)
|
||||||
}
|
}
|
||||||
workRoot := filepath.Join(workspaceRoot, "work", "2026-05-03")
|
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "raw", "alice.json"), `{"segments":[]}`)
|
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "raw", "alice.json"), `{"segments":[]}`)
|
||||||
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "speakers.yml"), "match:\n - speaker: Alice\n match: [\"alice\"]\n")
|
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "speakers.yml"), "match:\n - speaker: Alice\n match: [\"alice\"]\n")
|
||||||
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "autocorrect.yml"), "[]\n")
|
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "autocorrect.yml"), "[]\n")
|
||||||
@@ -52,7 +52,7 @@ func TestResumeStartsAfterCompletedStages(t *testing.T) {
|
|||||||
func TestResumeNoRemainingStages(t *testing.T) {
|
func TestResumeNoRemainingStages(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
manifestPath := filepath.Join(workspaceRoot, "work", "2026-05-03", "manifest.json")
|
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||||
|
|
||||||
store := &manifest.LocalStore{}
|
store := &manifest.LocalStore{}
|
||||||
m := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
m := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||||
@@ -81,7 +81,7 @@ func TestResumeForceRerunsSucceeded(t *testing.T) {
|
|||||||
}))
|
}))
|
||||||
defer srv.Close()
|
defer srv.Close()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot, srv.URL)
|
||||||
manifestPath := filepath.Join(workspaceRoot, "work", "2026-05-03", "manifest.json")
|
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||||
|
|
||||||
store := &manifest.LocalStore{}
|
store := &manifest.LocalStore{}
|
||||||
m := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
m := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||||
@@ -105,8 +105,8 @@ func TestResumeForceRerunsSucceeded(t *testing.T) {
|
|||||||
func TestRunStageExecutesOnlySelectedStage(t *testing.T) {
|
func TestRunStageExecutesOnlySelectedStage(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
manifestPath := filepath.Join(workspaceRoot, "work", "2026-05-03", "manifest.json")
|
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||||
workRoot := filepath.Join(workspaceRoot, "work", "2026-05-03")
|
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"segments":[]}`)
|
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"segments":[]}`)
|
||||||
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "terms: []\n")
|
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "terms: []\n")
|
||||||
|
|
||||||
@@ -135,8 +135,8 @@ func TestRunStageExecutesOnlySelectedStage(t *testing.T) {
|
|||||||
func TestRunStageSkipAndForce(t *testing.T) {
|
func TestRunStageSkipAndForce(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
manifestPath := filepath.Join(workspaceRoot, "work", "2026-05-03", "manifest.json")
|
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||||
workRoot := filepath.Join(workspaceRoot, "work", "2026-05-03")
|
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"segments":[]}`)
|
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"segments":[]}`)
|
||||||
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "terms: []\n")
|
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "terms: []\n")
|
||||||
|
|
||||||
@@ -166,11 +166,57 @@ func TestRunStageSkipAndForce(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunStageForceMarksDownstreamStaleAndResumeContinuesFromStale(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||||
|
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||||
|
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "merged.json"), `{"segments":[]}`)
|
||||||
|
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "terms: []\n")
|
||||||
|
|
||||||
|
store := &manifest.LocalStore{}
|
||||||
|
seed := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||||
|
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze", "archive", "notify"} {
|
||||||
|
seed.MarkStageSucceeded(name, time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||||
|
}
|
||||||
|
if err := store.Save(context.Background(), manifestPath, seed); err != nil {
|
||||||
|
t.Fatalf("save manifest: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "--force", "polish"}, &out)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("RunStage(force) error = %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out.String(), "stage=polish executed=1 skipped=0 force=true") {
|
||||||
|
t.Fatalf("output = %q, want forced polish rerun", out.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
afterForce, err := store.Load(context.Background(), manifestPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load manifest after force: %v", err)
|
||||||
|
}
|
||||||
|
for _, name := range []string{"normalize", "trim", "analyze", "archive", "notify"} {
|
||||||
|
if afterForce.Stages[name] == nil || afterForce.Stages[name].Status != manifest.StatusStale {
|
||||||
|
t.Fatalf("stage %q = %#v, want stale", name, afterForce.Stages[name])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
out.Reset()
|
||||||
|
err = Resume(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath}, &out)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Resume() error = %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out.String(), "executed=5 skipped=0") {
|
||||||
|
t.Fatalf("output = %q, want resume to execute normalize..notify", out.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunStageTrimExecutes(t *testing.T) {
|
func TestRunStageTrimExecutes(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
manifestPath := filepath.Join(workspaceRoot, "work", "2026-05-03", "manifest.json")
|
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||||
workRoot := filepath.Join(workspaceRoot, "work", "2026-05-03")
|
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "normalized.json"), `{"segments":[{"id":1},{"id":2}]}`)
|
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "normalized.json"), `{"segments":[{"id":1},{"id":2}]}`)
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
@@ -198,8 +244,8 @@ func TestRunStageTrimExecutes(t *testing.T) {
|
|||||||
func TestRunStageNormalizeExecutes(t *testing.T) {
|
func TestRunStageNormalizeExecutes(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
manifestPath := filepath.Join(workspaceRoot, "work", "2026-05-03", "manifest.json")
|
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||||
workRoot := filepath.Join(workspaceRoot, "work", "2026-05-03")
|
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "processed.json"), `{"segments":[{"id":1},{"id":2}]}`)
|
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "processed.json"), `{"segments":[{"id":1},{"id":2}]}`)
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
|
|||||||
@@ -16,9 +16,11 @@ func Run(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
|
|
||||||
var pipelinePath string
|
var pipelinePath string
|
||||||
var sessionPath string
|
var sessionPath string
|
||||||
|
var sessionID string
|
||||||
var force bool
|
var force bool
|
||||||
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml")
|
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
||||||
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
||||||
|
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
|
||||||
fs.BoolVar(&force, "force", false, "force stage execution (reserved for future behavior)")
|
fs.BoolVar(&force, "force", false, "force stage execution (reserved for future behavior)")
|
||||||
|
|
||||||
if err := fs.Parse(args); err != nil {
|
if err := fs.Parse(args); err != nil {
|
||||||
@@ -27,11 +29,18 @@ func Run(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
if fs.NArg() != 0 {
|
if fs.NArg() != 0 {
|
||||||
return fmt.Errorf("run: unexpected positional arguments")
|
return fmt.Errorf("run: unexpected positional arguments")
|
||||||
}
|
}
|
||||||
if pipelinePath == "" || sessionPath == "" {
|
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath)
|
||||||
return fmt.Errorf("run: --config and --session are required")
|
if err != nil {
|
||||||
|
return fmt.Errorf("run: %w", err)
|
||||||
|
}
|
||||||
|
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("run: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg, err := config.Load(pipelinePath, sessionPath)
|
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
|
||||||
|
SessionID: sessionID,
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("run: %w", err)
|
return fmt.Errorf("run: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
package app
|
package app
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/stage"
|
"gitea.maximumdirect.net/eric/narratio/internal/stage"
|
||||||
)
|
)
|
||||||
@@ -49,3 +51,43 @@ func firstNonSucceededIndex(stages []stage.Stage, m *manifest.Manifest) int {
|
|||||||
}
|
}
|
||||||
return len(stages)
|
return len(stages)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func canonicalStageNames() []string {
|
||||||
|
all := stage.All()
|
||||||
|
out := make([]string, 0, len(all))
|
||||||
|
for _, s := range all {
|
||||||
|
if s == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, s.Name())
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func downstreamStageNames(stageName string) []string {
|
||||||
|
names := canonicalStageNames()
|
||||||
|
for i, name := range names {
|
||||||
|
if name != stageName {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return append([]string(nil), names[i+1:]...)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func invalidateDownstreamSucceededStages(m *manifest.Manifest, upstreamStage string, at time.Time) []string {
|
||||||
|
if m == nil || m.Stages == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
invalidated := make([]string, 0)
|
||||||
|
for _, downstream := range downstreamStageNames(upstreamStage) {
|
||||||
|
sr := m.Stages[downstream]
|
||||||
|
if sr == nil || sr.Status != manifest.StatusSucceeded {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
m.MarkStageStale(downstream, at, "upstream stage rerun with force")
|
||||||
|
invalidated = append(invalidated, downstream)
|
||||||
|
}
|
||||||
|
return invalidated
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package app
|
package app
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"reflect"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -40,3 +41,48 @@ func TestDecideStageActions(t *testing.T) {
|
|||||||
t.Fatalf("forced prepare action = %q, want %q", forced[0].Action, stageActionRun)
|
t.Fatalf("forced prepare action = %q, want %q", forced[0].Action, stageActionRun)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestDownstreamStageNames(t *testing.T) {
|
||||||
|
got := downstreamStageNames("polish")
|
||||||
|
want := []string{"normalize", "trim", "analyze", "archive", "notify"}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("downstreamStageNames(polish) = %#v, want %#v", got, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
missing := downstreamStageNames("unknown")
|
||||||
|
if len(missing) != 0 {
|
||||||
|
t.Fatalf("downstreamStageNames(unknown) = %#v, want empty", missing)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInvalidateDownstreamSucceededStages(t *testing.T) {
|
||||||
|
now := time.Now().UTC()
|
||||||
|
m := manifest.New("2026-05-03", now)
|
||||||
|
m.MarkStageSucceeded("prepare", now, nil)
|
||||||
|
m.MarkStageSucceeded("transcribe", now, nil)
|
||||||
|
m.MarkStageSucceeded("merge", now, nil)
|
||||||
|
m.MarkStageSucceeded("polish", now, nil)
|
||||||
|
m.MarkStageSucceeded("normalize", now, nil)
|
||||||
|
m.MarkStageSucceeded("trim", now, nil)
|
||||||
|
m.MarkStageFailed("analyze", now, "analysis failed")
|
||||||
|
m.MarkStageSucceeded("archive", now, nil)
|
||||||
|
m.MarkStageSucceeded("notify", now, nil)
|
||||||
|
|
||||||
|
got := invalidateDownstreamSucceededStages(m, "polish", now.Add(1*time.Second))
|
||||||
|
want := []string{"normalize", "trim", "archive", "notify"}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("invalidateDownstreamSucceededStages() = %#v, want %#v", got, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, stageName := range want {
|
||||||
|
if m.Stages[stageName].Status != manifest.StatusStale {
|
||||||
|
t.Fatalf("%s status = %q, want stale", stageName, m.Stages[stageName].Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if m.Stages["analyze"].Status != manifest.StatusFailed {
|
||||||
|
t.Fatalf("analyze status = %q, want failed", m.Stages["analyze"].Status)
|
||||||
|
}
|
||||||
|
if m.Stages["prepare"].Status != manifest.StatusSucceeded {
|
||||||
|
t.Fatalf("prepare status = %q, want succeeded", m.Stages["prepare"].Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -16,9 +16,11 @@ func RunStage(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
|
|
||||||
var pipelinePath string
|
var pipelinePath string
|
||||||
var sessionPath string
|
var sessionPath string
|
||||||
|
var sessionID string
|
||||||
var force bool
|
var force bool
|
||||||
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml")
|
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
||||||
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
fs.StringVar(&sessionPath, "session", "", "path to session.yml")
|
||||||
|
fs.StringVar(&sessionID, "session-id", "", "session identifier for session.yml templates")
|
||||||
fs.BoolVar(&force, "force", false, "force stage execution (reserved for future behavior)")
|
fs.BoolVar(&force, "force", false, "force stage execution (reserved for future behavior)")
|
||||||
|
|
||||||
if err := fs.Parse(args); err != nil {
|
if err := fs.Parse(args); err != nil {
|
||||||
@@ -27,17 +29,24 @@ func RunStage(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
if fs.NArg() != 1 {
|
if fs.NArg() != 1 {
|
||||||
return fmt.Errorf("run-stage: expected exactly one stage name")
|
return fmt.Errorf("run-stage: expected exactly one stage name")
|
||||||
}
|
}
|
||||||
if pipelinePath == "" || sessionPath == "" {
|
|
||||||
return fmt.Errorf("run-stage: --config and --session are required")
|
|
||||||
}
|
|
||||||
|
|
||||||
stageName := fs.Arg(0)
|
stageName := fs.Arg(0)
|
||||||
stages, err := BuildSingleStagePlan(stageName)
|
stages, err := BuildSingleStagePlan(stageName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("run-stage: %w", err)
|
return fmt.Errorf("run-stage: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg, err := config.Load(pipelinePath, sessionPath)
|
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelinePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("run-stage: %w", err)
|
||||||
|
}
|
||||||
|
resolvedSessionPath, err := resolveSessionConfigPath(sessionPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("run-stage: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg, err := config.LoadWithSessionOptions(resolvedPipelinePath, resolvedSessionPath, config.SessionLoadOptions{
|
||||||
|
SessionID: sessionID,
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("run-stage: %w", err)
|
return fmt.Errorf("run-stage: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/audita"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/audita"
|
||||||
@@ -27,11 +26,13 @@ type RunOptions struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type RunSummary struct {
|
type RunSummary struct {
|
||||||
SessionID string
|
SessionID string
|
||||||
ManifestPath string
|
RunID string
|
||||||
StageNames []string
|
ManifestPath string
|
||||||
Executed []string
|
RunManifestPath string
|
||||||
Skipped []string
|
StageNames []string
|
||||||
|
Executed []string
|
||||||
|
Skipped []string
|
||||||
}
|
}
|
||||||
|
|
||||||
func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage, opts RunOptions) (*RunSummary, error) {
|
func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage, opts RunOptions) (*RunSummary, error) {
|
||||||
@@ -51,6 +52,9 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
|
|||||||
if env.Logger == nil {
|
if env.Logger == nil {
|
||||||
env.Logger = logging.NewLogger(os.Stderr, slog.LevelInfo)
|
env.Logger = logging.NewLogger(os.Stderr, slog.LevelInfo)
|
||||||
}
|
}
|
||||||
|
if _, err := loadSecretsFromConfig(env.Config, env.Logger); err != nil {
|
||||||
|
return nil, fmt.Errorf("load secrets from files: %w", err)
|
||||||
|
}
|
||||||
if env.WhisperX == nil {
|
if env.WhisperX == nil {
|
||||||
client, err := buildDefaultWhisperXClient(env.Config)
|
client, err := buildDefaultWhisperXClient(env.Config)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -78,17 +82,24 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
|
|||||||
if env.Storage == nil {
|
if env.Storage == nil {
|
||||||
env.Storage = &storage.NoopBackend{}
|
env.Storage = &storage.NoopBackend{}
|
||||||
}
|
}
|
||||||
|
if env.ObjectStore == nil && needsObjectStoreForRun(env.Config, stages) {
|
||||||
|
objectStore, err := storage.NewObjectStoreFromConfig(ctx, env.Config)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("initialize object store backend: %w", err)
|
||||||
|
}
|
||||||
|
env.ObjectStore = objectStore
|
||||||
|
}
|
||||||
if env.Notifier == nil {
|
if env.Notifier == nil {
|
||||||
env.Notifier = ¬ify.NoopSender{}
|
env.Notifier = ¬ify.NoopSender{}
|
||||||
}
|
}
|
||||||
|
|
||||||
artifactStore := env.ArtifactStore
|
artifactStore := env.ArtifactStore
|
||||||
paths, err := artifactStore.EnsureLayout(cfg.Session.SessionID)
|
paths, err := artifactStore.EnsureLayoutFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("prepare workdir: %w", err)
|
return nil, fmt.Errorf("prepare workdir: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
lock, err := artifactStore.AcquireSessionLock(cfg.Session.SessionID)
|
lock, err := artifactStore.AcquireSessionLockFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("acquire session lock: %w", err)
|
return nil, fmt.Errorf("acquire session lock: %w", err)
|
||||||
}
|
}
|
||||||
@@ -101,6 +112,42 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
runID, err := artifacts.NewRunID()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("generate run id: %w", err)
|
||||||
|
}
|
||||||
|
identityChanged, err := ensureManifestIdentity(cfg, m, runID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("initialize manifest identity: %w", err)
|
||||||
|
}
|
||||||
|
if identityChanged {
|
||||||
|
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
|
||||||
|
return nil, fmt.Errorf("save manifest identity %q: %w", manifestPath, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
runManifestPath := artifacts.SessionRunManifestPathForCampaign(
|
||||||
|
cfg.Pipeline.Workspace.Root,
|
||||||
|
cfg.Session.Campaign,
|
||||||
|
cfg.Session.SessionID,
|
||||||
|
runID,
|
||||||
|
)
|
||||||
|
runManifestStore := &manifest.LocalStore{}
|
||||||
|
runManifest, err := runManifestStore.CreateRun(
|
||||||
|
ctx,
|
||||||
|
cfg.Session.SessionID,
|
||||||
|
cfg.Session.Campaign,
|
||||||
|
runID,
|
||||||
|
opts.Force,
|
||||||
|
requestedStageNames(stages),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("create run manifest: %w", err)
|
||||||
|
}
|
||||||
|
runManifest.SessionManifestPath = manifestPath
|
||||||
|
syncRunManifestIdentityFromSession(m, runManifest)
|
||||||
|
if err := runManifestStore.SaveRun(ctx, runManifestPath, runManifest); err != nil {
|
||||||
|
return nil, fmt.Errorf("save initial run manifest %q: %w", runManifestPath, err)
|
||||||
|
}
|
||||||
|
|
||||||
stageEnv := env
|
stageEnv := env
|
||||||
|
|
||||||
@@ -115,12 +162,23 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
|
|||||||
|
|
||||||
if d.Action == stageActionSkip {
|
if d.Action == stageActionSkip {
|
||||||
skipped = append(skipped, s.Name())
|
skipped = append(skipped, s.Name())
|
||||||
|
skipAt := nowUTC()
|
||||||
|
runManifest.SetStageAction(s.Name(), manifest.RunStageActionSkip, skipAt)
|
||||||
|
runManifest.MarkStageSkipped(s.Name(), skipAt, "already_succeeded")
|
||||||
|
if err := runManifestStore.SaveRun(ctx, runManifestPath, runManifest); err != nil {
|
||||||
|
return nil, fmt.Errorf("save run manifest after skip %q: %w", s.Name(), err)
|
||||||
|
}
|
||||||
env.Logger.Info("skipping stage", "stage", s.Name(), "reason", "already_succeeded", "force", opts.Force)
|
env.Logger.Info("skipping stage", "stage", s.Name(), "reason", "already_succeeded", "force", opts.Force)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
executed = append(executed, s.Name())
|
executed = append(executed, s.Name())
|
||||||
|
|
||||||
now := nowUTC()
|
now := nowUTC()
|
||||||
|
runManifest.SetStageAction(s.Name(), manifest.RunStageActionRun, now)
|
||||||
|
runManifest.MarkStageRunning(s.Name(), now)
|
||||||
|
if err := runManifestStore.SaveRun(ctx, runManifestPath, runManifest); err != nil {
|
||||||
|
return nil, fmt.Errorf("save run manifest before stage %q: %w", s.Name(), err)
|
||||||
|
}
|
||||||
m.MarkStageRunning(s.Name(), now)
|
m.MarkStageRunning(s.Name(), now)
|
||||||
env.Logger.Info("starting stage", "stage", s.Name())
|
env.Logger.Info("starting stage", "stage", s.Name())
|
||||||
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
|
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
|
||||||
@@ -130,31 +188,65 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
|
|||||||
|
|
||||||
result, err := s.Run(ctx, stageEnv, m)
|
result, err := s.Run(ctx, stageEnv, m)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
m.MarkStageFailed(s.Name(), nowUTC(), err.Error())
|
failedAt := nowUTC()
|
||||||
|
m.MarkStageFailed(s.Name(), failedAt, err.Error())
|
||||||
if saveErr := env.ManifestStore.Save(ctx, manifestPath, m); saveErr != nil {
|
if saveErr := env.ManifestStore.Save(ctx, manifestPath, m); saveErr != nil {
|
||||||
return nil, fmt.Errorf("stage %q failed (%v) and manifest save failed (%v)", s.Name(), err, saveErr)
|
return nil, fmt.Errorf("stage %q failed (%v) and manifest save failed (%v)", s.Name(), err, saveErr)
|
||||||
}
|
}
|
||||||
|
runManifest.MarkStageFailed(s.Name(), failedAt, err.Error())
|
||||||
|
syncRunManifestIdentityFromSession(m, runManifest)
|
||||||
|
if saveErr := runManifestStore.SaveRun(ctx, runManifestPath, runManifest); saveErr != nil {
|
||||||
|
return nil, fmt.Errorf("stage %q failed (%v) and run-manifest save failed (%v)", s.Name(), err, saveErr)
|
||||||
|
}
|
||||||
env.Logger.Info("stage failed", "stage", s.Name(), "error", err)
|
env.Logger.Info("stage failed", "stage", s.Name(), "error", err)
|
||||||
return nil, fmt.Errorf("stage %q failed: %w", s.Name(), err)
|
return nil, fmt.Errorf("stage %q failed: %w", s.Name(), err)
|
||||||
}
|
}
|
||||||
|
|
||||||
outputs := mapResultOutputs(result)
|
outputs := mapResultOutputs(result, runID)
|
||||||
m.MarkStageSucceeded(s.Name(), nowUTC(), outputs)
|
succeededAt := nowUTC()
|
||||||
|
m.MarkStageSucceeded(s.Name(), succeededAt, outputs)
|
||||||
applyStageResultToManifest(m, s.Name(), result)
|
applyStageResultToManifest(m, s.Name(), result)
|
||||||
|
if opts.Force {
|
||||||
|
invalidateDownstreamSucceededStages(m, s.Name(), succeededAt)
|
||||||
|
}
|
||||||
|
|
||||||
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
|
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
|
||||||
return nil, fmt.Errorf("save manifest after stage %q: %w", s.Name(), err)
|
return nil, fmt.Errorf("save manifest after stage %q: %w", s.Name(), err)
|
||||||
}
|
}
|
||||||
|
runManifest.MarkStageSucceeded(s.Name(), succeededAt, outputs)
|
||||||
|
applyStageResultToRunManifest(runManifest, s.Name(), result)
|
||||||
|
syncRunManifestIdentityFromSession(m, runManifest)
|
||||||
|
if err := runManifestStore.SaveRun(ctx, runManifestPath, runManifest); err != nil {
|
||||||
|
return nil, fmt.Errorf("save run manifest after stage %q: %w", s.Name(), err)
|
||||||
|
}
|
||||||
env.Logger.Debug("manifest saved", "stage", s.Name(), "transition", "succeeded", "path", manifestPath)
|
env.Logger.Debug("manifest saved", "stage", s.Name(), "transition", "succeeded", "path", manifestPath)
|
||||||
env.Logger.Info("stage succeeded", "stage", s.Name())
|
env.Logger.Info("stage succeeded", "stage", s.Name())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if err := runPostArchiveCleanup(ctx, env, manifestPath, m, executed); err != nil {
|
||||||
|
failedAt := nowUTC()
|
||||||
|
runManifest.MarkFailed(failedAt, err.Error())
|
||||||
|
syncRunManifestIdentityFromSession(m, runManifest)
|
||||||
|
if saveErr := runManifestStore.SaveRun(ctx, runManifestPath, runManifest); saveErr != nil {
|
||||||
|
return nil, fmt.Errorf("post-archive cleanup failed (%v) and run-manifest save failed (%v)", err, saveErr)
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("post-archive cleanup: %w", err)
|
||||||
|
}
|
||||||
|
completedAt := nowUTC()
|
||||||
|
runManifest.MarkSucceeded(completedAt)
|
||||||
|
syncRunManifestIdentityFromSession(m, runManifest)
|
||||||
|
if err := runManifestStore.SaveRun(ctx, runManifestPath, runManifest); err != nil {
|
||||||
|
return nil, fmt.Errorf("save final run manifest %q: %w", runManifestPath, err)
|
||||||
|
}
|
||||||
|
|
||||||
return &RunSummary{
|
return &RunSummary{
|
||||||
SessionID: cfg.Session.SessionID,
|
SessionID: cfg.Session.SessionID,
|
||||||
ManifestPath: manifestPath,
|
RunID: runID,
|
||||||
StageNames: runNames,
|
ManifestPath: manifestPath,
|
||||||
Executed: executed,
|
RunManifestPath: runManifestPath,
|
||||||
Skipped: skipped,
|
StageNames: runNames,
|
||||||
|
Executed: executed,
|
||||||
|
Skipped: skipped,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -227,7 +319,7 @@ func buildDefaultAuditaRunner(cfg *config.Config) (audita.Runner, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
a := cfg.Pipeline.Audita
|
a := cfg.Pipeline.Audita
|
||||||
if strings.TrimSpace(a.Binary) == "" || strings.TrimSpace(a.Timeout) == "" || len(a.Modules) == 0 || strings.TrimSpace(a.BaseURL) == "" || strings.TrimSpace(a.Model) == "" {
|
if strings.TrimSpace(a.Binary) == "" || strings.TrimSpace(a.Timeout) == "" {
|
||||||
// Compatibility fallback for tests or internal call paths that bypass config validation/defaults.
|
// Compatibility fallback for tests or internal call paths that bypass config validation/defaults.
|
||||||
return &audita.NoopRunner{}, nil
|
return &audita.NoopRunner{}, nil
|
||||||
}
|
}
|
||||||
@@ -244,7 +336,12 @@ func buildDefaultAuditaRunner(cfg *config.Config) (audita.Runner, error) {
|
|||||||
append([]string(nil), a.Modules...),
|
append([]string(nil), a.Modules...),
|
||||||
a.BaseURL,
|
a.BaseURL,
|
||||||
a.Model,
|
a.Model,
|
||||||
a.LLMConcurrency,
|
a.TranscriptDescription,
|
||||||
|
a.ConfigPath,
|
||||||
|
a.OutputSchema,
|
||||||
|
a.WorkDirRetention,
|
||||||
|
a.TotalLLMConcurrency,
|
||||||
|
a.ProposalLLMConcurrency,
|
||||||
a.ValidationModel,
|
a.ValidationModel,
|
||||||
a.ValidationLLMConcurrency,
|
a.ValidationLLMConcurrency,
|
||||||
report,
|
report,
|
||||||
@@ -289,11 +386,12 @@ func fileExists(path string) (bool, error) {
|
|||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func mapResultOutputs(result *stage.StageResult) []manifest.ArtifactRecord {
|
func mapResultOutputs(result *stage.StageResult, runID string) []manifest.ArtifactRecord {
|
||||||
if result == nil || len(result.Outputs) == 0 {
|
if result == nil || len(result.Outputs) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
runID = strings.TrimSpace(runID)
|
||||||
out := make([]manifest.ArtifactRecord, 0, len(result.Outputs))
|
out := make([]manifest.ArtifactRecord, 0, len(result.Outputs))
|
||||||
for _, ref := range result.Outputs {
|
for _, ref := range result.Outputs {
|
||||||
localPath := ref.AbsolutePath
|
localPath := ref.AbsolutePath
|
||||||
@@ -301,10 +399,11 @@ func mapResultOutputs(result *stage.StageResult) []manifest.ArtifactRecord {
|
|||||||
localPath = ref.RelativePath
|
localPath = ref.RelativePath
|
||||||
}
|
}
|
||||||
out = append(out, manifest.ArtifactRecord{
|
out = append(out, manifest.ArtifactRecord{
|
||||||
Kind: ref.Kind,
|
Kind: ref.Kind,
|
||||||
LocalPath: localPath,
|
LocalPath: localPath,
|
||||||
RemoteKey: ref.RemoteKey,
|
ProducerRunID: runID,
|
||||||
Checksum: ref.Checksum,
|
RemoteKey: ref.RemoteKey,
|
||||||
|
Checksum: ref.Checksum,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -330,6 +429,132 @@ func applyStageResultToManifest(m *manifest.Manifest, stageName string, result *
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func manifestPathFor(cfg *config.Config) string {
|
func ensureManifestIdentity(cfg *config.Config, m *manifest.Manifest, runID string) (bool, error) {
|
||||||
return filepath.Join(cfg.Pipeline.Workspace.Root, "work", cfg.Session.SessionID, "manifest.json")
|
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil || m == nil {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
changed := false
|
||||||
|
campaign := strings.TrimSpace(cfg.Session.Campaign)
|
||||||
|
sessionID := strings.TrimSpace(cfg.Session.SessionID)
|
||||||
|
if sessionID == "" {
|
||||||
|
sessionID = strings.TrimSpace(m.SessionID)
|
||||||
|
}
|
||||||
|
|
||||||
|
if m.Campaign == "" && campaign != "" {
|
||||||
|
m.Campaign = campaign
|
||||||
|
changed = true
|
||||||
|
}
|
||||||
|
runID = strings.TrimSpace(runID)
|
||||||
|
if runID != "" && m.RunID != runID {
|
||||||
|
m.RunID = runID
|
||||||
|
changed = true
|
||||||
|
}
|
||||||
|
if m.LocalWorkDir == "" && campaign != "" && sessionID != "" && m.RunID != "" {
|
||||||
|
m.LocalWorkDir = artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, campaign, sessionID, m.RunID)
|
||||||
|
changed = true
|
||||||
|
}
|
||||||
|
if m.LocalSpoolDir == "" && campaign != "" && sessionID != "" && m.RunID != "" && strings.TrimSpace(cfg.Pipeline.Spool.Root) != "" {
|
||||||
|
m.LocalSpoolDir = artifacts.SessionSpoolAudioDir(cfg.Pipeline.Spool.Root, campaign, sessionID, m.RunID)
|
||||||
|
changed = true
|
||||||
|
}
|
||||||
|
if cfg.Pipeline.Storage.S3 != nil {
|
||||||
|
if m.S3Bucket == "" && strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket) != "" {
|
||||||
|
m.S3Bucket = strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket)
|
||||||
|
changed = true
|
||||||
|
}
|
||||||
|
sessionPrefix := artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, campaign, sessionID)
|
||||||
|
if m.S3SessionPrefix == "" && sessionPrefix != "" {
|
||||||
|
m.S3SessionPrefix = sessionPrefix
|
||||||
|
changed = true
|
||||||
|
}
|
||||||
|
runPrefix := artifacts.S3RunPrefix(sessionPrefix, m.RunID)
|
||||||
|
if m.S3RunPrefix == "" && runPrefix != "" {
|
||||||
|
m.S3RunPrefix = runPrefix
|
||||||
|
changed = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return changed, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func requestedStageNames(stages []stage.Stage) []string {
|
||||||
|
out := make([]string, 0, len(stages))
|
||||||
|
for _, s := range stages {
|
||||||
|
if s == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, s.Name())
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func applyStageResultToRunManifest(m *manifest.RunManifest, stageName string, result *stage.StageResult) {
|
||||||
|
if m == nil || result == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
sr := m.Stages[stageName]
|
||||||
|
if sr == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if len(result.Logs) > 0 {
|
||||||
|
sr.Logs = append([]string(nil), result.Logs...)
|
||||||
|
}
|
||||||
|
if len(result.GeneratedConfigs) > 0 {
|
||||||
|
sr.GeneratedConfigs = append([]string(nil), result.GeneratedConfigs...)
|
||||||
|
}
|
||||||
|
if len(result.Metadata) > 0 {
|
||||||
|
sr.Metadata = result.Metadata
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func syncRunManifestIdentityFromSession(session *manifest.Manifest, run *manifest.RunManifest) {
|
||||||
|
if session == nil || run == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
run.Campaign = session.Campaign
|
||||||
|
run.LocalWorkDir = session.LocalWorkDir
|
||||||
|
run.LocalSpoolDir = session.LocalSpoolDir
|
||||||
|
run.S3Bucket = session.S3Bucket
|
||||||
|
run.S3SessionPrefix = session.S3SessionPrefix
|
||||||
|
run.S3RunPrefix = session.S3RunPrefix
|
||||||
|
}
|
||||||
|
|
||||||
|
func manifestPathFor(cfg *config.Config) string {
|
||||||
|
return artifacts.SessionManifestPathForCampaign(
|
||||||
|
cfg.Pipeline.Workspace.Root,
|
||||||
|
cfg.Session.Campaign,
|
||||||
|
cfg.Session.SessionID,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func needsObjectStoreForRun(cfg *config.Config, stages []stage.Stage) bool {
|
||||||
|
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
stageRequested := func(name string) bool {
|
||||||
|
for _, s := range stages {
|
||||||
|
if s != nil && s.Name() == name {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
if cfg.Session.Inputs.AudioS3 != nil && stageRequested("prepare") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if !stageRequested("archive") {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if cfg.Pipeline.Archive == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if cfg.Pipeline.Archive.Enabled != nil && !*cfg.Pipeline.Archive.Enabled {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if cfg.Pipeline.Archive.UploadRun != nil && !*cfg.Pipeline.Archive.UploadRun {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -144,6 +144,15 @@ func TestExecuteStagesPlaceholderSuccessUpdatesManifest(t *testing.T) {
|
|||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
if name == "archive" {
|
||||||
|
if sr.Metadata == nil || sr.Metadata["stage"] != "archive" {
|
||||||
|
t.Fatalf("archive metadata missing stage=archive: %#v", sr.Metadata)
|
||||||
|
}
|
||||||
|
if sr.Metadata["skipped"] != true {
|
||||||
|
t.Fatalf("archive metadata missing skipped=true for test config without archive section: %#v", sr.Metadata)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
if sr.Metadata == nil || sr.Metadata["placeholder"] != true {
|
if sr.Metadata == nil || sr.Metadata["placeholder"] != true {
|
||||||
t.Fatalf("stage %q missing placeholder metadata", name)
|
t.Fatalf("stage %q missing placeholder metadata", name)
|
||||||
}
|
}
|
||||||
@@ -226,6 +235,56 @@ func TestExecuteStagesForceRerunsSucceeded(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExecuteStagesForceSuccessInvalidatesDownstreamSucceededStages(t *testing.T) {
|
||||||
|
cfg := testConfig(t)
|
||||||
|
manifestPath := manifestPathFor(cfg)
|
||||||
|
store := &manifest.LocalStore{}
|
||||||
|
|
||||||
|
existing := manifest.New(cfg.Session.SessionID, time.Date(2026, 5, 3, 1, 0, 0, 0, time.UTC))
|
||||||
|
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "archive", "notify"} {
|
||||||
|
existing.MarkStageSucceeded(stageName, time.Date(2026, 5, 3, 1, 1, 0, 0, time.UTC), nil)
|
||||||
|
}
|
||||||
|
existing.MarkStageFailed("analyze", time.Date(2026, 5, 3, 1, 1, 0, 0, time.UTC), "previous analyze failure")
|
||||||
|
if err := os.MkdirAll(filepath.Dir(manifestPath), 0o755); err != nil {
|
||||||
|
t.Fatalf("MkdirAll() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := store.Save(context.Background(), manifestPath, existing); err != nil {
|
||||||
|
t.Fatalf("Save manifest error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
runs := 0
|
||||||
|
stageToRun := countingStage{name: "polish", runs: &runs}
|
||||||
|
summary, err := executeStages(context.Background(), cfg, []stage.Stage{stageToRun}, RunOptions{Force: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
if runs != 1 {
|
||||||
|
t.Fatalf("runs = %d, want 1 with force", runs)
|
||||||
|
}
|
||||||
|
if len(summary.Executed) != 1 || summary.Executed[0] != "polish" || len(summary.Skipped) != 0 {
|
||||||
|
t.Fatalf("summary = %#v, want executed polish", summary)
|
||||||
|
}
|
||||||
|
|
||||||
|
loaded, err := store.Load(context.Background(), manifestPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load manifest error = %v", err)
|
||||||
|
}
|
||||||
|
if loaded.Stages["polish"] == nil || loaded.Stages["polish"].Status != manifest.StatusSucceeded {
|
||||||
|
t.Fatalf("polish status = %#v, want succeeded", loaded.Stages["polish"])
|
||||||
|
}
|
||||||
|
for _, stageName := range []string{"normalize", "trim", "archive", "notify"} {
|
||||||
|
if loaded.Stages[stageName] == nil || loaded.Stages[stageName].Status != manifest.StatusStale {
|
||||||
|
t.Fatalf("%s status = %#v, want stale", stageName, loaded.Stages[stageName])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if loaded.Stages["analyze"] == nil || loaded.Stages["analyze"].Status != manifest.StatusFailed {
|
||||||
|
t.Fatalf("analyze status = %#v, want preserved failed", loaded.Stages["analyze"])
|
||||||
|
}
|
||||||
|
if loaded.Stages["transcribe"] == nil || loaded.Stages["transcribe"].Status != manifest.StatusSucceeded {
|
||||||
|
t.Fatalf("transcribe status = %#v, want preserved succeeded", loaded.Stages["transcribe"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExecuteStagesFailureUpdatesManifest(t *testing.T) {
|
func TestExecuteStagesFailureUpdatesManifest(t *testing.T) {
|
||||||
cfg := testConfig(t)
|
cfg := testConfig(t)
|
||||||
|
|
||||||
@@ -271,7 +330,14 @@ func TestExecuteStagesLoadsExistingManifest(t *testing.T) {
|
|||||||
|
|
||||||
existing := manifest.New(cfg.Session.SessionID, time.Date(2026, 5, 3, 1, 0, 0, 0, time.UTC))
|
existing := manifest.New(cfg.Session.SessionID, time.Date(2026, 5, 3, 1, 0, 0, 0, time.UTC))
|
||||||
existing.MarkStageSucceeded("prepare", time.Date(2026, 5, 3, 1, 1, 0, 0, time.UTC), nil)
|
existing.MarkStageSucceeded("prepare", time.Date(2026, 5, 3, 1, 1, 0, 0, time.UTC), nil)
|
||||||
audioPath := filepath.Join(cfg.Pipeline.Workspace.Root, "work", cfg.Session.SessionID, "audio", "alice.flac")
|
audioPath := filepath.Join(
|
||||||
|
cfg.Pipeline.Workspace.Root,
|
||||||
|
"work",
|
||||||
|
cfg.Session.Campaign,
|
||||||
|
cfg.Session.SessionID,
|
||||||
|
"audio",
|
||||||
|
"alice.flac",
|
||||||
|
)
|
||||||
if err := os.MkdirAll(filepath.Dir(audioPath), 0o755); err != nil {
|
if err := os.MkdirAll(filepath.Dir(audioPath), 0o755); err != nil {
|
||||||
t.Fatalf("MkdirAll() error = %v", err)
|
t.Fatalf("MkdirAll() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -306,6 +372,208 @@ func TestExecuteStagesLoadsExistingManifest(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExecuteStagesCreatesRunManifestPerInvocation(t *testing.T) {
|
||||||
|
cfg := testConfig(t)
|
||||||
|
run1, err := executeStages(context.Background(), cfg, []stage.Stage{BuildFullPlan()[0]}, RunOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("first executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
run2, err := executeStages(context.Background(), cfg, []stage.Stage{BuildFullPlan()[0]}, RunOptions{Force: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("second executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if run1.RunID == "" || run2.RunID == "" {
|
||||||
|
t.Fatalf("run ids must be set, got %q and %q", run1.RunID, run2.RunID)
|
||||||
|
}
|
||||||
|
if run1.RunID == run2.RunID {
|
||||||
|
t.Fatalf("expected distinct run ids, got %q", run1.RunID)
|
||||||
|
}
|
||||||
|
if run1.RunManifestPath == "" || run2.RunManifestPath == "" {
|
||||||
|
t.Fatalf("run manifest paths must be set, got %q and %q", run1.RunManifestPath, run2.RunManifestPath)
|
||||||
|
}
|
||||||
|
if run1.RunManifestPath == run2.RunManifestPath {
|
||||||
|
t.Fatalf("expected distinct run manifest paths, got %q", run1.RunManifestPath)
|
||||||
|
}
|
||||||
|
for _, path := range []string{run1.RunManifestPath, run2.RunManifestPath} {
|
||||||
|
if _, statErr := os.Stat(path); statErr != nil {
|
||||||
|
t.Fatalf("run manifest missing at %q: %v", path, statErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
store := &manifest.LocalStore{}
|
||||||
|
sessionManifest, err := store.Load(context.Background(), run2.ManifestPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load session manifest error = %v", err)
|
||||||
|
}
|
||||||
|
if sessionManifest.RunID != run2.RunID {
|
||||||
|
t.Fatalf("session manifest run_id = %q, want latest run id %q", sessionManifest.RunID, run2.RunID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteStagesRunManifestRecordsSkippedStage(t *testing.T) {
|
||||||
|
cfg := testConfig(t)
|
||||||
|
store := &manifest.LocalStore{}
|
||||||
|
manifestPath := manifestPathFor(cfg)
|
||||||
|
existing := manifest.New(cfg.Session.SessionID, time.Date(2026, 5, 3, 1, 0, 0, 0, time.UTC))
|
||||||
|
existing.MarkStageSucceeded("transcribe", time.Date(2026, 5, 3, 1, 1, 0, 0, time.UTC), nil)
|
||||||
|
if err := os.MkdirAll(filepath.Dir(manifestPath), 0o755); err != nil {
|
||||||
|
t.Fatalf("MkdirAll() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := store.Save(context.Background(), manifestPath, existing); err != nil {
|
||||||
|
t.Fatalf("Save manifest error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
summary, err := executeStages(context.Background(), cfg, []stage.Stage{BuildFullPlan()[1]}, RunOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(summary.Skipped) != 1 || summary.Skipped[0] != "transcribe" {
|
||||||
|
t.Fatalf("summary = %#v, want skipped transcribe", summary)
|
||||||
|
}
|
||||||
|
|
||||||
|
runManifest, err := store.LoadRun(context.Background(), summary.RunManifestPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadRun() error = %v", err)
|
||||||
|
}
|
||||||
|
sr := runManifest.Stages["transcribe"]
|
||||||
|
if sr == nil {
|
||||||
|
t.Fatal("run manifest transcribe stage missing")
|
||||||
|
}
|
||||||
|
if sr.Action != manifest.RunStageActionSkip {
|
||||||
|
t.Fatalf("action = %q, want %q", sr.Action, manifest.RunStageActionSkip)
|
||||||
|
}
|
||||||
|
if sr.Status != manifest.StatusSkipped {
|
||||||
|
t.Fatalf("status = %q, want %q", sr.Status, manifest.StatusSkipped)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteStagesRunLocalArtifactsAndCanonicalPromotion(t *testing.T) {
|
||||||
|
cfg := testConfig(t)
|
||||||
|
stages := []stage.Stage{
|
||||||
|
BuildFullPlan()[0], // prepare
|
||||||
|
BuildFullPlan()[1], // transcribe
|
||||||
|
BuildFullPlan()[2], // merge
|
||||||
|
BuildFullPlan()[3], // polish
|
||||||
|
BuildFullPlan()[4], // normalize
|
||||||
|
BuildFullPlan()[5], // trim
|
||||||
|
}
|
||||||
|
|
||||||
|
summary, err := executeStages(context.Background(), cfg, stages, RunOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
if summary.RunID == "" {
|
||||||
|
t.Fatal("run id must be set")
|
||||||
|
}
|
||||||
|
|
||||||
|
runRoot := artifacts.SessionRunRootForCampaign(
|
||||||
|
cfg.Pipeline.Workspace.Root,
|
||||||
|
cfg.Session.Campaign,
|
||||||
|
cfg.Session.SessionID,
|
||||||
|
summary.RunID,
|
||||||
|
)
|
||||||
|
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
|
||||||
|
runLocalChecks := []string{
|
||||||
|
filepath.Join(runRoot, "transcribe", "outputs", "transcripts", "raw", "alice.json"),
|
||||||
|
filepath.Join(runRoot, "merge", "logs", "seriatim.stdout.log"),
|
||||||
|
filepath.Join(runRoot, "polish", "config", "audita.generated.yml"),
|
||||||
|
filepath.Join(runRoot, "normalize", "logs", "seriatim.normalize.stdout.log"),
|
||||||
|
filepath.Join(runRoot, "trim", "outputs", "transcripts", "trimmed.json"),
|
||||||
|
}
|
||||||
|
for _, p := range runLocalChecks {
|
||||||
|
if _, statErr := os.Stat(p); statErr != nil {
|
||||||
|
t.Fatalf("run-local artifact missing at %q: %v", p, statErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
canonicalChecks := []string{
|
||||||
|
filepath.Join(paths.TranscriptsRawDir, "alice.json"),
|
||||||
|
filepath.Join(paths.TranscriptsDir, "merged.json"),
|
||||||
|
filepath.Join(paths.TranscriptsDir, "processed.json"),
|
||||||
|
filepath.Join(paths.TranscriptsDir, "normalized.json"),
|
||||||
|
filepath.Join(paths.TranscriptsDir, "trimmed.json"),
|
||||||
|
}
|
||||||
|
for _, p := range canonicalChecks {
|
||||||
|
if _, statErr := os.Stat(p); statErr != nil {
|
||||||
|
t.Fatalf("canonical promoted artifact missing at %q: %v", p, statErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
store := &manifest.LocalStore{}
|
||||||
|
sessionManifest, err := store.Load(context.Background(), summary.ManifestPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load manifest error = %v", err)
|
||||||
|
}
|
||||||
|
if got := sessionManifest.Stages["trim"]; got == nil || len(got.Outputs) == 0 {
|
||||||
|
t.Fatalf("trim stage outputs missing in session manifest: %#v", got)
|
||||||
|
}
|
||||||
|
for _, out := range sessionManifest.Stages["trim"].Outputs {
|
||||||
|
if strings.Contains(out.LocalPath, string(filepath.Separator)+"runs"+string(filepath.Separator)) {
|
||||||
|
t.Fatalf("session manifest output should be canonical, got run-local path %q", out.LocalPath)
|
||||||
|
}
|
||||||
|
if out.ProducerRunID != summary.RunID {
|
||||||
|
t.Fatalf("producer_run_id = %q, want %q", out.ProducerRunID, summary.RunID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
runManifest, err := store.LoadRun(context.Background(), summary.RunManifestPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadRun() error = %v", err)
|
||||||
|
}
|
||||||
|
mergeStage := runManifest.Stages["merge"]
|
||||||
|
if mergeStage == nil || len(mergeStage.Logs) == 0 {
|
||||||
|
t.Fatalf("merge logs missing in run manifest: %#v", mergeStage)
|
||||||
|
}
|
||||||
|
for _, logPath := range mergeStage.Logs {
|
||||||
|
if !strings.Contains(logPath, filepath.Join("runs", summary.RunID, "merge", "logs")) {
|
||||||
|
t.Fatalf("run manifest merge log path = %q, want run-local merge logs path", logPath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteStagesSkippedStagePreservesExistingOutputsProvenance(t *testing.T) {
|
||||||
|
cfg := testConfig(t)
|
||||||
|
store := &manifest.LocalStore{}
|
||||||
|
manifestPath := manifestPathFor(cfg)
|
||||||
|
|
||||||
|
existing := manifest.New(cfg.Session.SessionID, time.Date(2026, 5, 3, 1, 0, 0, 0, time.UTC))
|
||||||
|
existing.MarkStageSucceeded("transcribe", time.Date(2026, 5, 3, 1, 1, 0, 0, time.UTC), []manifest.ArtifactRecord{
|
||||||
|
{
|
||||||
|
Kind: "transcript_raw",
|
||||||
|
LocalPath: "transcripts/raw/alice.json",
|
||||||
|
ProducerRunID: "20260501T000000Z-deadbeef",
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err := os.MkdirAll(filepath.Dir(manifestPath), 0o755); err != nil {
|
||||||
|
t.Fatalf("MkdirAll() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := store.Save(context.Background(), manifestPath, existing); err != nil {
|
||||||
|
t.Fatalf("Save manifest error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
summary, err := executeStages(context.Background(), cfg, []stage.Stage{BuildFullPlan()[1]}, RunOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(summary.Skipped) != 1 || summary.Skipped[0] != "transcribe" {
|
||||||
|
t.Fatalf("summary = %#v, want skipped transcribe", summary)
|
||||||
|
}
|
||||||
|
|
||||||
|
loaded, err := store.Load(context.Background(), manifestPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load manifest error = %v", err)
|
||||||
|
}
|
||||||
|
got := loaded.Stages["transcribe"]
|
||||||
|
if got == nil || len(got.Outputs) != 1 {
|
||||||
|
t.Fatalf("transcribe outputs = %#v, want one preserved output", got)
|
||||||
|
}
|
||||||
|
if got.Outputs[0].ProducerRunID != "20260501T000000Z-deadbeef" {
|
||||||
|
t.Fatalf("producer_run_id = %q, want preserved value", got.Outputs[0].ProducerRunID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestAdapterBackedStageFailureMarksManifestFailed(t *testing.T) {
|
func TestAdapterBackedStageFailureMarksManifestFailed(t *testing.T) {
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
name string
|
name string
|
||||||
@@ -315,7 +583,7 @@ func TestAdapterBackedStageFailureMarksManifestFailed(t *testing.T) {
|
|||||||
{name: "merge", env: &Env{Seriatim: &seriatim.FakeRunner{Err: errors.New("merge fail")}}},
|
{name: "merge", env: &Env{Seriatim: &seriatim.FakeRunner{Err: errors.New("merge fail")}}},
|
||||||
{name: "polish", env: &Env{Audita: &audita.FakeRunner{Err: errors.New("polish fail")}}},
|
{name: "polish", env: &Env{Audita: &audita.FakeRunner{Err: errors.New("polish fail")}}},
|
||||||
{name: "analyze", env: &Env{Scriptorium: &scriptorium.FakeRunner{RunErr: errors.New("analyze fail")}}},
|
{name: "analyze", env: &Env{Scriptorium: &scriptorium.FakeRunner{RunErr: errors.New("analyze fail")}}},
|
||||||
{name: "archive", env: &Env{Storage: &storage.FakeBackend{Err: errors.New("archive fail")}}},
|
{name: "archive", env: &Env{ObjectStore: &storage.FakeBackend{UploadErr: errors.New("archive fail")}}},
|
||||||
{name: "notify", env: &Env{Notifier: ¬ify.FakeSender{Err: errors.New("notify fail")}}},
|
{name: "notify", env: &Env{Notifier: ¬ify.FakeSender{Err: errors.New("notify fail")}}},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -332,7 +600,7 @@ func TestAdapterBackedStageFailureMarksManifestFailed(t *testing.T) {
|
|||||||
tc.env.ArtifactStore = artifactStore
|
tc.env.ArtifactStore = artifactStore
|
||||||
tc.env.ManifestStore = &manifest.LocalStore{}
|
tc.env.ManifestStore = &manifest.LocalStore{}
|
||||||
if tc.name == "transcribe" {
|
if tc.name == "transcribe" {
|
||||||
paths, ensureErr := artifactStore.EnsureLayout(cfg.Session.SessionID)
|
paths, ensureErr := artifactStore.EnsureLayoutFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
if ensureErr != nil {
|
if ensureErr != nil {
|
||||||
t.Fatalf("EnsureLayout() error = %v", ensureErr)
|
t.Fatalf("EnsureLayout() error = %v", ensureErr)
|
||||||
}
|
}
|
||||||
@@ -347,7 +615,7 @@ func TestAdapterBackedStageFailureMarksManifestFailed(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if tc.name == "merge" {
|
if tc.name == "merge" {
|
||||||
paths, ensureErr := artifactStore.EnsureLayout(cfg.Session.SessionID)
|
paths, ensureErr := artifactStore.EnsureLayoutFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
if ensureErr != nil {
|
if ensureErr != nil {
|
||||||
t.Fatalf("EnsureLayout() error = %v", ensureErr)
|
t.Fatalf("EnsureLayout() error = %v", ensureErr)
|
||||||
}
|
}
|
||||||
@@ -366,7 +634,7 @@ func TestAdapterBackedStageFailureMarksManifestFailed(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if tc.name == "polish" {
|
if tc.name == "polish" {
|
||||||
paths, ensureErr := artifactStore.EnsureLayout(cfg.Session.SessionID)
|
paths, ensureErr := artifactStore.EnsureLayoutFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
if ensureErr != nil {
|
if ensureErr != nil {
|
||||||
t.Fatalf("EnsureLayout() error = %v", ensureErr)
|
t.Fatalf("EnsureLayout() error = %v", ensureErr)
|
||||||
}
|
}
|
||||||
@@ -378,7 +646,7 @@ func TestAdapterBackedStageFailureMarksManifestFailed(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if tc.name == "analyze" {
|
if tc.name == "analyze" {
|
||||||
paths, ensureErr := artifactStore.EnsureLayout(cfg.Session.SessionID)
|
paths, ensureErr := artifactStore.EnsureLayoutFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
if ensureErr != nil {
|
if ensureErr != nil {
|
||||||
t.Fatalf("EnsureLayout() error = %v", ensureErr)
|
t.Fatalf("EnsureLayout() error = %v", ensureErr)
|
||||||
}
|
}
|
||||||
@@ -400,6 +668,41 @@ func TestAdapterBackedStageFailureMarksManifestFailed(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if tc.name == "archive" {
|
||||||
|
cfg.Pipeline.Archive = &config.ArchiveConfig{
|
||||||
|
Enabled: boolPtr(true),
|
||||||
|
UploadRun: boolPtr(true),
|
||||||
|
}
|
||||||
|
cfg.Pipeline.Storage.S3 = &config.StorageS3Config{
|
||||||
|
Bucket: "my-dnd-archive",
|
||||||
|
RootPrefix: "dnd",
|
||||||
|
}
|
||||||
|
runID := "20260516T010203Z-0a1b2c3d"
|
||||||
|
runWorkDir := filepath.Join(cfg.Pipeline.Workspace.Root, "work", cfg.Session.Campaign, cfg.Session.SessionID, runID)
|
||||||
|
if err := os.MkdirAll(filepath.Join(runWorkDir, "inputs"), 0o755); err != nil {
|
||||||
|
t.Fatalf("mkdir archive inputs dir: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(runWorkDir, "inputs", "session.yml"), []byte("session_id: 2026-05-03\n"), 0o644); err != nil {
|
||||||
|
t.Fatalf("write archive fixture session.yml: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(runWorkDir, "manifest.json"), []byte("{}\n"), 0o644); err != nil {
|
||||||
|
t.Fatalf("write archive fixture manifest.json: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
seed := manifest.New(cfg.Session.SessionID, time.Now().UTC())
|
||||||
|
seed.Campaign = cfg.Session.Campaign
|
||||||
|
seed.RunID = runID
|
||||||
|
seed.LocalWorkDir = runWorkDir
|
||||||
|
seed.S3Bucket = "my-dnd-archive"
|
||||||
|
seed.S3SessionPrefix = "dnd/campaigns/" + cfg.Session.Campaign + "/sessions/" + cfg.Session.SessionID + "/"
|
||||||
|
seed.S3RunPrefix = seed.S3SessionPrefix + "runs/" + runID + "/"
|
||||||
|
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze"} {
|
||||||
|
seed.MarkStageSucceeded(name, time.Now().UTC(), nil)
|
||||||
|
}
|
||||||
|
if err := tc.env.ManifestStore.Save(context.Background(), manifestPathFor(cfg), seed); err != nil {
|
||||||
|
t.Fatalf("seed archive manifest: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
_, runErr := executeStages(context.Background(), cfg, []stage.Stage{selected}, RunOptions{Env: tc.env})
|
_, runErr := executeStages(context.Background(), cfg, []stage.Stage{selected}, RunOptions{Env: tc.env})
|
||||||
if runErr == nil {
|
if runErr == nil {
|
||||||
@@ -430,7 +733,7 @@ func testConfig(t *testing.T) *config.Config {
|
|||||||
pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
|
pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
|
||||||
|
|
||||||
mustWriteFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n")
|
mustWriteFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n")
|
||||||
mustWriteFile(t, sessionPath, "session_id: 2026-05-03\n")
|
mustWriteFile(t, sessionPath, "session_id: 2026-05-03\ncampaign: sample-campaign\n")
|
||||||
mustWriteFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n")
|
mustWriteFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n")
|
||||||
mustWriteFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n")
|
mustWriteFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n")
|
||||||
mustWriteFile(t, filepath.Join(cfgDir, "glossary.yml"), "[]\n")
|
mustWriteFile(t, filepath.Join(cfgDir, "glossary.yml"), "[]\n")
|
||||||
@@ -442,6 +745,7 @@ func testConfig(t *testing.T) *config.Config {
|
|||||||
SessionPath: sessionPath,
|
SessionPath: sessionPath,
|
||||||
Session: &config.SessionConfig{
|
Session: &config.SessionConfig{
|
||||||
SessionID: "2026-05-03",
|
SessionID: "2026-05-03",
|
||||||
|
Campaign: "sample-campaign",
|
||||||
Inputs: config.SessionInputsConfig{
|
Inputs: config.SessionInputsConfig{
|
||||||
AudioDir: "./audio",
|
AudioDir: "./audio",
|
||||||
SpeakersFile: "./speakers.yml",
|
SpeakersFile: "./speakers.yml",
|
||||||
@@ -452,6 +756,55 @@ func testConfig(t *testing.T) *config.Config {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestBuildDefaultRunnersWithOmittedToolSections(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||||
|
sessionPath := filepath.Join(dir, "session.yml")
|
||||||
|
pipelineYAML := `workspace:
|
||||||
|
root: ` + t.TempDir() + `
|
||||||
|
whisperx:
|
||||||
|
transcribe_url: https://example.com/transcribe
|
||||||
|
analyzer:
|
||||||
|
timeout: 20m
|
||||||
|
notification:
|
||||||
|
timeout: 10s
|
||||||
|
`
|
||||||
|
sessionYAML := `session_id: 2026-05-03
|
||||||
|
campaign: sample-campaign
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
`
|
||||||
|
mustWriteFile(t, pipelinePath, pipelineYAML)
|
||||||
|
mustWriteFile(t, sessionPath, sessionYAML)
|
||||||
|
|
||||||
|
cfg, err := config.Load(pipelinePath, sessionPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := config.Validate(cfg); err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
serRunner, err := buildDefaultSeriatimRunner(cfg)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("buildDefaultSeriatimRunner() error = %v", err)
|
||||||
|
}
|
||||||
|
if _, ok := serRunner.(*seriatim.SubprocessRunner); !ok {
|
||||||
|
t.Fatalf("seriatim runner type = %T, want *seriatim.SubprocessRunner", serRunner)
|
||||||
|
}
|
||||||
|
|
||||||
|
audRunner, err := buildDefaultAuditaRunner(cfg)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("buildDefaultAuditaRunner() error = %v", err)
|
||||||
|
}
|
||||||
|
if _, ok := audRunner.(*audita.SubprocessRunner); !ok {
|
||||||
|
t.Fatalf("audita runner type = %T, want *audita.SubprocessRunner", audRunner)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func mustWriteFile(t *testing.T, path, contents string) {
|
func mustWriteFile(t *testing.T, path, contents string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||||
@@ -461,3 +814,8 @@ func mustWriteFile(t *testing.T, path, contents string) {
|
|||||||
t.Fatalf("WriteFile(%q): %v", path, err)
|
t.Fatalf("WriteFile(%q): %v", path, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func boolPtr(v bool) *bool {
|
||||||
|
p := v
|
||||||
|
return &p
|
||||||
|
}
|
||||||
|
|||||||
88
internal/app/secrets_env.go
Normal file
88
internal/app/secrets_env.go
Normal file
@@ -0,0 +1,88 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
var envVarNamePattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||||
|
|
||||||
|
type secretsLoadStats struct {
|
||||||
|
Dir string
|
||||||
|
Loaded int
|
||||||
|
PreservedExisting int
|
||||||
|
Skipped int
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadSecretsFromConfig(cfg *config.Config, logger *slog.Logger) (*secretsLoadStats, error) {
|
||||||
|
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Secrets == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
rawDir := strings.TrimSpace(cfg.Pipeline.Secrets.EnvDir)
|
||||||
|
if rawDir == "" {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
resolvedDir := rawDir
|
||||||
|
if !filepath.IsAbs(resolvedDir) {
|
||||||
|
cwd, err := os.Getwd()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("resolve secrets env_dir %q from current working directory: %w", rawDir, err)
|
||||||
|
}
|
||||||
|
resolvedDir = filepath.Join(cwd, resolvedDir)
|
||||||
|
}
|
||||||
|
resolvedDir = filepath.Clean(resolvedDir)
|
||||||
|
|
||||||
|
entries, err := os.ReadDir(resolvedDir)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("read secrets env_dir %q: %w", resolvedDir, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
stats := &secretsLoadStats{Dir: resolvedDir}
|
||||||
|
for _, entry := range entries {
|
||||||
|
name := entry.Name()
|
||||||
|
if !envVarNamePattern.MatchString(name) {
|
||||||
|
stats.Skipped++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if entry.IsDir() {
|
||||||
|
stats.Skipped++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
secretPath := filepath.Join(resolvedDir, name)
|
||||||
|
bytes, err := os.ReadFile(secretPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("read secret file %q: %w", secretPath, err)
|
||||||
|
}
|
||||||
|
value := strings.TrimRight(string(bytes), "\r\n")
|
||||||
|
|
||||||
|
if _, exists := os.LookupEnv(name); exists {
|
||||||
|
stats.PreservedExisting++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := os.Setenv(name, value); err != nil {
|
||||||
|
return nil, fmt.Errorf("set environment variable %q from %q: %w", name, secretPath, err)
|
||||||
|
}
|
||||||
|
stats.Loaded++
|
||||||
|
}
|
||||||
|
|
||||||
|
if logger != nil {
|
||||||
|
logger.Info(
|
||||||
|
"loaded secret environment variables from filesystem",
|
||||||
|
"secrets_env_dir", stats.Dir,
|
||||||
|
"loaded", stats.Loaded,
|
||||||
|
"preserved_existing", stats.PreservedExisting,
|
||||||
|
"skipped", stats.Skipped,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return stats, nil
|
||||||
|
}
|
||||||
155
internal/app/secrets_env_test.go
Normal file
155
internal/app/secrets_env_test.go
Normal file
@@ -0,0 +1,155 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestLoadSecretsFromConfigLoadsValidFiles(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
mustWriteSecretFile(t, filepath.Join(dir, "NARRATIO_TEST_SECRET_A"), "value-1\n")
|
||||||
|
mustWriteSecretFile(t, filepath.Join(dir, "NARRATIO_TEST_SECRET_B"), "value-2\r\n")
|
||||||
|
mustWriteSecretFile(t, filepath.Join(dir, "not-valid-name.txt"), "ignored")
|
||||||
|
|
||||||
|
cfg := &config.Config{
|
||||||
|
Pipeline: &config.PipelineConfig{
|
||||||
|
Secrets: &config.SecretsConfig{EnvDir: dir},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
stats, err := loadSecretsFromConfig(cfg, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("loadSecretsFromConfig() error = %v", err)
|
||||||
|
}
|
||||||
|
if stats == nil {
|
||||||
|
t.Fatal("stats = nil, want non-nil")
|
||||||
|
}
|
||||||
|
if stats.Loaded != 2 {
|
||||||
|
t.Fatalf("Loaded = %d, want 2", stats.Loaded)
|
||||||
|
}
|
||||||
|
if stats.PreservedExisting != 0 {
|
||||||
|
t.Fatalf("PreservedExisting = %d, want 0", stats.PreservedExisting)
|
||||||
|
}
|
||||||
|
if stats.Skipped == 0 {
|
||||||
|
t.Fatalf("Skipped = %d, want > 0 for invalid filename", stats.Skipped)
|
||||||
|
}
|
||||||
|
if got := os.Getenv("NARRATIO_TEST_SECRET_A"); got != "value-1" {
|
||||||
|
t.Fatalf("NARRATIO_TEST_SECRET_A = %q, want value-1", got)
|
||||||
|
}
|
||||||
|
if got := os.Getenv("NARRATIO_TEST_SECRET_B"); got != "value-2" {
|
||||||
|
t.Fatalf("NARRATIO_TEST_SECRET_B = %q, want value-2", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadSecretsFromConfigPreservesExistingEnv(t *testing.T) {
|
||||||
|
t.Setenv("OBJECT_STORAGE_KEY", "existing")
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
mustWriteSecretFile(t, filepath.Join(dir, "OBJECT_STORAGE_KEY"), "from-file\n")
|
||||||
|
|
||||||
|
cfg := &config.Config{
|
||||||
|
Pipeline: &config.PipelineConfig{
|
||||||
|
Secrets: &config.SecretsConfig{EnvDir: dir},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
stats, err := loadSecretsFromConfig(cfg, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("loadSecretsFromConfig() error = %v", err)
|
||||||
|
}
|
||||||
|
if stats.PreservedExisting != 1 {
|
||||||
|
t.Fatalf("PreservedExisting = %d, want 1", stats.PreservedExisting)
|
||||||
|
}
|
||||||
|
if got := os.Getenv("OBJECT_STORAGE_KEY"); got != "existing" {
|
||||||
|
t.Fatalf("OBJECT_STORAGE_KEY = %q, want existing", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadSecretsFromConfigRelativeDirUsesCWD(t *testing.T) {
|
||||||
|
cwd := t.TempDir()
|
||||||
|
secretsDir := filepath.Join(cwd, "secrets")
|
||||||
|
if err := os.MkdirAll(secretsDir, 0o755); err != nil {
|
||||||
|
t.Fatalf("MkdirAll(%q): %v", secretsDir, err)
|
||||||
|
}
|
||||||
|
mustWriteSecretFile(t, filepath.Join(secretsDir, "OBJECT_STORAGE_KEY_ID"), "id-123\n")
|
||||||
|
|
||||||
|
originalWD, err := os.Getwd()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Getwd(): %v", err)
|
||||||
|
}
|
||||||
|
if err := os.Chdir(cwd); err != nil {
|
||||||
|
t.Fatalf("Chdir(%q): %v", cwd, err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
_ = os.Chdir(originalWD)
|
||||||
|
})
|
||||||
|
|
||||||
|
cfg := &config.Config{
|
||||||
|
Pipeline: &config.PipelineConfig{
|
||||||
|
Secrets: &config.SecretsConfig{EnvDir: "./secrets"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if _, err := loadSecretsFromConfig(cfg, nil); err != nil {
|
||||||
|
t.Fatalf("loadSecretsFromConfig() error = %v", err)
|
||||||
|
}
|
||||||
|
if got := os.Getenv("OBJECT_STORAGE_KEY_ID"); got != "id-123" {
|
||||||
|
t.Fatalf("OBJECT_STORAGE_KEY_ID = %q, want id-123", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadSecretsFromConfigMissingDirFails(t *testing.T) {
|
||||||
|
cfg := &config.Config{
|
||||||
|
Pipeline: &config.PipelineConfig{
|
||||||
|
Secrets: &config.SecretsConfig{EnvDir: filepath.Join(t.TempDir(), "missing")},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := loadSecretsFromConfig(cfg, nil)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "read secrets env_dir") {
|
||||||
|
t.Fatalf("error = %q, want read-dir context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadSecretsFromConfigUnreadableValidEntryFails(t *testing.T) {
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
t.Skip("symlink behavior differs on windows")
|
||||||
|
}
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
broken := filepath.Join(dir, "OPENROUTER_API_KEY")
|
||||||
|
if err := os.Symlink(filepath.Join(dir, "does-not-exist"), broken); err != nil {
|
||||||
|
t.Fatalf("Symlink(%q): %v", broken, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg := &config.Config{
|
||||||
|
Pipeline: &config.PipelineConfig{
|
||||||
|
Secrets: &config.SecretsConfig{EnvDir: dir},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := loadSecretsFromConfig(cfg, nil)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "read secret file") {
|
||||||
|
t.Fatalf("error = %q, want read secret file context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func mustWriteSecretFile(t *testing.T, path, contents string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||||
|
t.Fatalf("MkdirAll(%q): %v", path, err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path, []byte(contents), 0o600); err != nil {
|
||||||
|
t.Fatalf("WriteFile(%q): %v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
86
internal/app/session_cli_test.go
Normal file
86
internal/app/session_cli_test.go
Normal file
@@ -0,0 +1,86 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPlanUsesDiscoveredSessionTemplateWithSessionID(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
|
sessionTemplate := `session_id: "{{ session_id }}"
|
||||||
|
campaign: sample-campaign
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(sessionPath, []byte(sessionTemplate), 0o644); err != nil {
|
||||||
|
t.Fatalf("write session template: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cwd := filepath.Dir(sessionPath)
|
||||||
|
originalWD, err := os.Getwd()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Getwd(): %v", err)
|
||||||
|
}
|
||||||
|
if err := os.Chdir(cwd); err != nil {
|
||||||
|
t.Fatalf("Chdir(%q): %v", cwd, err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = os.Chdir(originalWD) })
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
if err := Plan(context.Background(), []string{"--config", pipelinePath, "--session-id", "2026-04-04"}, &out); err != nil {
|
||||||
|
t.Fatalf("Plan() error = %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out.String(), "narratio plan: workdir prepared") {
|
||||||
|
t.Fatalf("output = %q, want plan output", out.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPlanFailsWhenSessionIDMismatchesConcreteSession(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
err := Plan(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "--session-id", "2026-04-04"}, &out)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "session_id mismatch") {
|
||||||
|
t.Fatalf("error = %q, want mismatch context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunStageAcceptsSessionIDFlagAndParsesStageName(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
err := RunStage(context.Background(), []string{"--config", pipelinePath, "--session", sessionPath, "--session-id", "2026-05-03", "prepare"}, &out)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("RunStage() error = %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out.String(), "stage=prepare") {
|
||||||
|
t.Fatalf("output = %q, want stage output", out.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveSessionConfigPathErrorIncludesSearchedPaths(t *testing.T) {
|
||||||
|
_, err := resolveSessionConfigPathWithCandidates("", []string{"./session.yml", "/usr/local/etc/narratio/session.yml", "/etc/narratio/session.yml"})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "searched") {
|
||||||
|
t.Fatalf("error = %q, want searched paths", err.Error())
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "pass --session") {
|
||||||
|
t.Fatalf("error = %q, want explicit-session guidance", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
49
internal/app/session_config_path.go
Normal file
49
internal/app/session_config_path.go
Normal file
@@ -0,0 +1,49 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func resolveSessionConfigPath(flagValue string) (string, error) {
|
||||||
|
return resolveSessionConfigPathWithCandidates(flagValue, config.DefaultSessionConfigSearchPaths)
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveSessionConfigPathWithCandidates(flagValue string, candidates []string) (string, error) {
|
||||||
|
if explicit := strings.TrimSpace(flagValue); explicit != "" {
|
||||||
|
return explicit, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
ordered := make([]string, 0, len(candidates))
|
||||||
|
for _, raw := range candidates {
|
||||||
|
path := strings.TrimSpace(raw)
|
||||||
|
if path == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
ordered = append(ordered, path)
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
if err == nil {
|
||||||
|
if info.IsDir() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return filepath.Clean(path), nil
|
||||||
|
}
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return "", fmt.Errorf("check default session config %q: %w", path, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(ordered) == 0 {
|
||||||
|
return "", fmt.Errorf("no session config path provided and no default locations configured")
|
||||||
|
}
|
||||||
|
return "", fmt.Errorf(
|
||||||
|
"no session config path provided and no default session config found; searched: %s; pass --session to use an explicit path",
|
||||||
|
strings.Join(ordered, ", "),
|
||||||
|
)
|
||||||
|
}
|
||||||
68
internal/app/session_config_path_test.go
Normal file
68
internal/app/session_config_path_test.go
Normal file
@@ -0,0 +1,68 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestResolveSessionConfigPathWithCandidatesExplicitWins(t *testing.T) {
|
||||||
|
got, err := resolveSessionConfigPathWithCandidates(" ./custom/session.yml ", []string{"./session.yml", "/a", "/b"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolveSessionConfigPathWithCandidates() error = %v", err)
|
||||||
|
}
|
||||||
|
if got != "./custom/session.yml" {
|
||||||
|
t.Fatalf("resolved path = %q, want explicit path", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveSessionConfigPathWithCandidatesUsesFirstExisting(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
first := filepath.Join(dir, "first.yml")
|
||||||
|
second := filepath.Join(dir, "second.yml")
|
||||||
|
if err := os.WriteFile(second, []byte("session_id: 2026-05-03\n"), 0o644); err != nil {
|
||||||
|
t.Fatalf("write second default: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := resolveSessionConfigPathWithCandidates("", []string{first, second})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolveSessionConfigPathWithCandidates() error = %v", err)
|
||||||
|
}
|
||||||
|
if got != filepath.Clean(second) {
|
||||||
|
t.Fatalf("resolved path = %q, want %q", got, filepath.Clean(second))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveSessionConfigPathWithCandidatesPrecedence(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
first := filepath.Join(dir, "first.yml")
|
||||||
|
second := filepath.Join(dir, "second.yml")
|
||||||
|
if err := os.WriteFile(first, []byte("session_id: 2026-05-03\n"), 0o644); err != nil {
|
||||||
|
t.Fatalf("write first default: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(second, []byte("session_id: 2026-05-03\n"), 0o644); err != nil {
|
||||||
|
t.Fatalf("write second default: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := resolveSessionConfigPathWithCandidates("", []string{first, second})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolveSessionConfigPathWithCandidates() error = %v", err)
|
||||||
|
}
|
||||||
|
if got != filepath.Clean(first) {
|
||||||
|
t.Fatalf("resolved path = %q, want first candidate %q", got, filepath.Clean(first))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveSessionConfigPathWithCandidatesMissing(t *testing.T) {
|
||||||
|
_, err := resolveSessionConfigPathWithCandidates("", []string{"/does/not/exist/one.yml", "/does/not/exist/two.yml"})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "no default session config found") {
|
||||||
|
t.Fatalf("error = %q, want missing-defaults context", err.Error())
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "pass --session") {
|
||||||
|
t.Fatalf("error = %q, want explicit-path guidance", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -63,7 +63,7 @@ func TestExecuteStagesDefaultWiringUsesWhisperXHTTPClient(t *testing.T) {
|
|||||||
t.Fatal("audio file payload was empty")
|
t.Fatal("audio file payload was empty")
|
||||||
}
|
}
|
||||||
|
|
||||||
outPath := filepath.Join(cfg.Pipeline.Workspace.Root, "work", cfg.Session.SessionID, "transcripts", "raw", "alice.json")
|
outPath := filepath.Join(cfg.Pipeline.Workspace.Root, "work", cfg.Session.Campaign, cfg.Session.SessionID, "transcripts", "raw", "alice.json")
|
||||||
data, err := os.ReadFile(outPath)
|
data, err := os.ReadFile(outPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("ReadFile(%q) error = %v", outPath, err)
|
t.Fatalf("ReadFile(%q) error = %v", outPath, err)
|
||||||
|
|||||||
294
internal/artifacts/artifact_resolver.go
Normal file
294
internal/artifacts/artifact_resolver.go
Normal file
@@ -0,0 +1,294 @@
|
|||||||
|
package artifacts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
ArtifactTranscriptMerged = "narratio.transcript.merged"
|
||||||
|
ArtifactTranscriptPolished = "narratio.transcript.polished"
|
||||||
|
ArtifactTranscriptFull = "narratio.transcript.full"
|
||||||
|
ArtifactTranscriptTrimmed = "narratio.transcript.trimmed"
|
||||||
|
ArtifactBoundsSession = "narratio.bounds.session"
|
||||||
|
ArtifactSessionRecap = "narratio.artifact.session_recap"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ErrSessionArtifactNotFound is returned when no readable artifact exists for a known ID.
|
||||||
|
var ErrSessionArtifactNotFound = errors.New("session artifact not found")
|
||||||
|
|
||||||
|
type artifactContentKind string
|
||||||
|
|
||||||
|
const (
|
||||||
|
contentTranscriptJSON artifactContentKind = "transcript_json"
|
||||||
|
contentJSON artifactContentKind = "json"
|
||||||
|
contentText artifactContentKind = "text"
|
||||||
|
)
|
||||||
|
|
||||||
|
type artifactSpec struct {
|
||||||
|
ID string
|
||||||
|
CanonicalRelPath string
|
||||||
|
ProducerStage string
|
||||||
|
OutputKind string
|
||||||
|
ContentKind artifactContentKind
|
||||||
|
}
|
||||||
|
|
||||||
|
var artifactRegistry = map[string]artifactSpec{
|
||||||
|
ArtifactTranscriptMerged: {
|
||||||
|
ID: ArtifactTranscriptMerged,
|
||||||
|
CanonicalRelPath: "transcripts/merged.json",
|
||||||
|
ProducerStage: "merge",
|
||||||
|
OutputKind: "transcript_merged",
|
||||||
|
ContentKind: contentTranscriptJSON,
|
||||||
|
},
|
||||||
|
ArtifactTranscriptPolished: {
|
||||||
|
ID: ArtifactTranscriptPolished,
|
||||||
|
CanonicalRelPath: "transcripts/processed.json",
|
||||||
|
ProducerStage: "polish",
|
||||||
|
OutputKind: "transcript_processed",
|
||||||
|
ContentKind: contentTranscriptJSON,
|
||||||
|
},
|
||||||
|
ArtifactTranscriptFull: {
|
||||||
|
ID: ArtifactTranscriptFull,
|
||||||
|
CanonicalRelPath: "transcripts/normalized.json",
|
||||||
|
ProducerStage: "normalize",
|
||||||
|
OutputKind: "transcript_normalized",
|
||||||
|
ContentKind: contentTranscriptJSON,
|
||||||
|
},
|
||||||
|
ArtifactTranscriptTrimmed: {
|
||||||
|
ID: ArtifactTranscriptTrimmed,
|
||||||
|
CanonicalRelPath: "transcripts/trimmed.json",
|
||||||
|
ProducerStage: "trim",
|
||||||
|
OutputKind: "transcript_trimmed",
|
||||||
|
ContentKind: contentTranscriptJSON,
|
||||||
|
},
|
||||||
|
ArtifactBoundsSession: {
|
||||||
|
ID: ArtifactBoundsSession,
|
||||||
|
CanonicalRelPath: "artifacts/session_bounds.json",
|
||||||
|
ProducerStage: "trim",
|
||||||
|
OutputKind: "session_bounds",
|
||||||
|
ContentKind: contentJSON,
|
||||||
|
},
|
||||||
|
ArtifactSessionRecap: {
|
||||||
|
ID: ArtifactSessionRecap,
|
||||||
|
CanonicalRelPath: "artifacts/session_recap.md",
|
||||||
|
ProducerStage: "analyze",
|
||||||
|
OutputKind: "session_recap",
|
||||||
|
ContentKind: contentText,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
var artifactAliases = map[string]string{
|
||||||
|
"processed_transcript": ArtifactTranscriptPolished,
|
||||||
|
"normalized_transcript": ArtifactTranscriptFull,
|
||||||
|
"trimmed_transcript": ArtifactTranscriptTrimmed,
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResolvedSessionArtifact describes one session-level artifact lookup result.
|
||||||
|
type ResolvedSessionArtifact struct {
|
||||||
|
ID string
|
||||||
|
Path string
|
||||||
|
ProducerStage string
|
||||||
|
OutputKind string
|
||||||
|
ProducerRunID string
|
||||||
|
Provenance string
|
||||||
|
}
|
||||||
|
|
||||||
|
// SessionArtifactNotFoundError includes context when a known artifact cannot be read.
|
||||||
|
type SessionArtifactNotFoundError struct {
|
||||||
|
ArtifactID string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *SessionArtifactNotFoundError) Error() string {
|
||||||
|
return fmt.Sprintf("%s: %q", ErrSessionArtifactNotFound, e.ArtifactID)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *SessionArtifactNotFoundError) Unwrap() error {
|
||||||
|
return ErrSessionArtifactNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
// NormalizeSessionArtifactSource maps legacy aliases to canonical IDs and validates IDs.
|
||||||
|
func NormalizeSessionArtifactSource(source string) (string, error) {
|
||||||
|
normalized := strings.TrimSpace(source)
|
||||||
|
if normalized == "" {
|
||||||
|
return "", fmt.Errorf("artifact source is required")
|
||||||
|
}
|
||||||
|
if alias, ok := artifactAliases[normalized]; ok {
|
||||||
|
normalized = alias
|
||||||
|
}
|
||||||
|
if _, ok := artifactRegistry[normalized]; !ok {
|
||||||
|
return "", fmt.Errorf("unsupported artifact source %q", source)
|
||||||
|
}
|
||||||
|
return normalized, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResolveSessionArtifact resolves a symbolic source to a readable local session artifact path.
|
||||||
|
// Resolution order is manifest producer outputs first, then canonical session path fallback.
|
||||||
|
func ResolveSessionArtifact(paths SessionPaths, m *manifest.Manifest, source string) (ResolvedSessionArtifact, error) {
|
||||||
|
id, err := NormalizeSessionArtifactSource(source)
|
||||||
|
if err != nil {
|
||||||
|
return ResolvedSessionArtifact{}, err
|
||||||
|
}
|
||||||
|
spec := artifactRegistry[id]
|
||||||
|
|
||||||
|
for _, candidate := range manifestArtifactCandidates(paths, m, spec) {
|
||||||
|
exists, isDir, statErr := pathExists(candidate.Path)
|
||||||
|
if statErr != nil {
|
||||||
|
return ResolvedSessionArtifact{}, fmt.Errorf("stat %q: %w", candidate.Path, statErr)
|
||||||
|
}
|
||||||
|
if !exists || isDir {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
resolved := candidate
|
||||||
|
resolved.ID = spec.ID
|
||||||
|
resolved.ProducerStage = spec.ProducerStage
|
||||||
|
resolved.OutputKind = spec.OutputKind
|
||||||
|
if err := validateResolvedContent(resolved.Path, spec.ContentKind); err != nil {
|
||||||
|
return ResolvedSessionArtifact{}, fmt.Errorf("validate %q: %w", resolved.ID, err)
|
||||||
|
}
|
||||||
|
return resolved, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
fallbackPath := filepath.Join(paths.Root, filepath.FromSlash(spec.CanonicalRelPath))
|
||||||
|
exists, isDir, statErr := pathExists(fallbackPath)
|
||||||
|
if statErr != nil {
|
||||||
|
return ResolvedSessionArtifact{}, fmt.Errorf("stat %q: %w", fallbackPath, statErr)
|
||||||
|
}
|
||||||
|
if exists && !isDir {
|
||||||
|
if err := validateResolvedContent(fallbackPath, spec.ContentKind); err != nil {
|
||||||
|
return ResolvedSessionArtifact{}, fmt.Errorf("validate %q: %w", spec.ID, err)
|
||||||
|
}
|
||||||
|
return ResolvedSessionArtifact{
|
||||||
|
ID: spec.ID,
|
||||||
|
Path: filepath.Clean(fallbackPath),
|
||||||
|
ProducerStage: spec.ProducerStage,
|
||||||
|
OutputKind: spec.OutputKind,
|
||||||
|
Provenance: "fallback.canonical_path",
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return ResolvedSessionArtifact{}, &SessionArtifactNotFoundError{ArtifactID: spec.ID}
|
||||||
|
}
|
||||||
|
|
||||||
|
func manifestArtifactCandidates(paths SessionPaths, m *manifest.Manifest, spec artifactSpec) []ResolvedSessionArtifact {
|
||||||
|
if m == nil || len(m.Stages) == 0 || spec.ProducerStage == "" || spec.OutputKind == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
sr := m.Stages[spec.ProducerStage]
|
||||||
|
if sr == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
candidates := make([]ResolvedSessionArtifact, 0, len(sr.Outputs))
|
||||||
|
for _, out := range sr.Outputs {
|
||||||
|
if strings.TrimSpace(out.Kind) != spec.OutputKind {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(out.LocalPath) == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
resolved := filepath.Clean(ResolveSessionLocalPathForRead(paths, out.LocalPath))
|
||||||
|
if resolved == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
candidates = append(candidates, ResolvedSessionArtifact{
|
||||||
|
Path: resolved,
|
||||||
|
ProducerRunID: strings.TrimSpace(out.ProducerRunID),
|
||||||
|
Provenance: "manifest." + spec.ProducerStage + ".outputs",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return dedupeResolvedArtifacts(candidates)
|
||||||
|
}
|
||||||
|
|
||||||
|
func dedupeResolvedArtifacts(values []ResolvedSessionArtifact) []ResolvedSessionArtifact {
|
||||||
|
seen := map[string]struct{}{}
|
||||||
|
out := make([]ResolvedSessionArtifact, 0, len(values))
|
||||||
|
for _, value := range values {
|
||||||
|
key := filepath.Clean(strings.TrimSpace(value.Path))
|
||||||
|
if key == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, ok := seen[key]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[key] = struct{}{}
|
||||||
|
value.Path = key
|
||||||
|
out = append(out, value)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func pathExists(path string) (exists bool, isDir bool, err error) {
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
if err == nil {
|
||||||
|
return true, info.IsDir(), nil
|
||||||
|
}
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
return false, false, nil
|
||||||
|
}
|
||||||
|
return false, false, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateResolvedContent(path string, kind artifactContentKind) error {
|
||||||
|
switch kind {
|
||||||
|
case contentTranscriptJSON:
|
||||||
|
return validateTranscriptSegmentsJSON(path)
|
||||||
|
case contentJSON:
|
||||||
|
return validateJSONContent(path)
|
||||||
|
case contentText:
|
||||||
|
return validateNonEmptyContent(path)
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("unsupported content kind %q", kind)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateTranscriptSegmentsJSON(path string) error {
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("read file: %w", err)
|
||||||
|
}
|
||||||
|
var payload map[string]any
|
||||||
|
if err := json.Unmarshal(data, &payload); err != nil {
|
||||||
|
return fmt.Errorf("decode json: %w", err)
|
||||||
|
}
|
||||||
|
segments, ok := payload["segments"]
|
||||||
|
if !ok {
|
||||||
|
return fmt.Errorf("top-level segments is required")
|
||||||
|
}
|
||||||
|
if _, ok := segments.([]any); !ok {
|
||||||
|
return fmt.Errorf("top-level segments must be an array")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateJSONContent(path string) error {
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("read file: %w", err)
|
||||||
|
}
|
||||||
|
var payload any
|
||||||
|
if err := json.Unmarshal(data, &payload); err != nil {
|
||||||
|
return fmt.Errorf("decode json: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateNonEmptyContent(path string) error {
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("stat file: %w", err)
|
||||||
|
}
|
||||||
|
if info.IsDir() {
|
||||||
|
return fmt.Errorf("path is a directory")
|
||||||
|
}
|
||||||
|
if info.Size() <= 0 {
|
||||||
|
return fmt.Errorf("file is empty")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
139
internal/artifacts/artifact_resolver_test.go
Normal file
139
internal/artifacts/artifact_resolver_test.go
Normal file
@@ -0,0 +1,139 @@
|
|||||||
|
package artifacts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNormalizeSessionArtifactSource(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
source string
|
||||||
|
wantID string
|
||||||
|
wantErr string
|
||||||
|
}{
|
||||||
|
{name: "legacy alias processed", source: "processed_transcript", wantID: ArtifactTranscriptPolished},
|
||||||
|
{name: "legacy alias normalized", source: "normalized_transcript", wantID: ArtifactTranscriptFull},
|
||||||
|
{name: "legacy alias trimmed", source: "trimmed_transcript", wantID: ArtifactTranscriptTrimmed},
|
||||||
|
{name: "canonical", source: ArtifactTranscriptTrimmed, wantID: ArtifactTranscriptTrimmed},
|
||||||
|
{name: "unsupported", source: "narratio.unknown", wantErr: "unsupported artifact source"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
got, err := NormalizeSessionArtifactSource(tt.source)
|
||||||
|
if tt.wantErr != "" {
|
||||||
|
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
|
||||||
|
t.Fatalf("NormalizeSessionArtifactSource() error = %v, want contains %q", err, tt.wantErr)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NormalizeSessionArtifactSource() error = %v", err)
|
||||||
|
}
|
||||||
|
if got != tt.wantID {
|
||||||
|
t.Fatalf("NormalizeSessionArtifactSource() = %q, want %q", got, tt.wantID)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveSessionArtifactPrefersManifestOutput(t *testing.T) {
|
||||||
|
workspace := t.TempDir()
|
||||||
|
paths := buildSessionPaths(workspace, "campaign", "session")
|
||||||
|
if err := os.MkdirAll(paths.ArtifactsDir, 0o755); err != nil {
|
||||||
|
t.Fatalf("MkdirAll() error = %v", err)
|
||||||
|
}
|
||||||
|
manifestPath := filepath.Join(paths.ArtifactsDir, "normalized.from-manifest.json")
|
||||||
|
if err := os.WriteFile(manifestPath, []byte(`{"segments":[]}`), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile() error = %v", err)
|
||||||
|
}
|
||||||
|
canonicalPath := filepath.Join(paths.TranscriptsDir, "normalized.json")
|
||||||
|
if err := os.MkdirAll(filepath.Dir(canonicalPath), 0o755); err != nil {
|
||||||
|
t.Fatalf("MkdirAll() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(canonicalPath, []byte(`{"segments":[{"id":123}]}`), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
m := manifest.New("session", time.Now().UTC())
|
||||||
|
m.MarkStageSucceeded("normalize", time.Now().UTC(), []manifest.ArtifactRecord{
|
||||||
|
{Kind: "transcript_normalized", LocalPath: manifestPath, ProducerRunID: "run-123"},
|
||||||
|
})
|
||||||
|
|
||||||
|
resolved, err := ResolveSessionArtifact(paths, m, "normalized_transcript")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ResolveSessionArtifact() error = %v", err)
|
||||||
|
}
|
||||||
|
if resolved.Path != manifestPath {
|
||||||
|
t.Fatalf("resolved path = %q, want %q", resolved.Path, manifestPath)
|
||||||
|
}
|
||||||
|
if resolved.Provenance != "manifest.normalize.outputs" {
|
||||||
|
t.Fatalf("provenance = %q, want manifest.normalize.outputs", resolved.Provenance)
|
||||||
|
}
|
||||||
|
if resolved.ProducerRunID != "run-123" {
|
||||||
|
t.Fatalf("producer run id = %q, want run-123", resolved.ProducerRunID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveSessionArtifactFallsBackToCanonicalPath(t *testing.T) {
|
||||||
|
workspace := t.TempDir()
|
||||||
|
paths := buildSessionPaths(workspace, "campaign", "session")
|
||||||
|
canonicalPath := filepath.Join(paths.TranscriptsDir, "trimmed.json")
|
||||||
|
if err := os.MkdirAll(filepath.Dir(canonicalPath), 0o755); err != nil {
|
||||||
|
t.Fatalf("MkdirAll() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(canonicalPath, []byte(`{"segments":[]}`), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
resolved, err := ResolveSessionArtifact(paths, nil, ArtifactTranscriptTrimmed)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ResolveSessionArtifact() error = %v", err)
|
||||||
|
}
|
||||||
|
if resolved.Path != canonicalPath {
|
||||||
|
t.Fatalf("resolved path = %q, want %q", resolved.Path, canonicalPath)
|
||||||
|
}
|
||||||
|
if resolved.Provenance != "fallback.canonical_path" {
|
||||||
|
t.Fatalf("provenance = %q, want fallback.canonical_path", resolved.Provenance)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveSessionArtifactMissingReturnsTypedError(t *testing.T) {
|
||||||
|
workspace := t.TempDir()
|
||||||
|
paths := buildSessionPaths(workspace, "campaign", "session")
|
||||||
|
|
||||||
|
_, err := ResolveSessionArtifact(paths, nil, ArtifactTranscriptTrimmed)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error, got nil")
|
||||||
|
}
|
||||||
|
if !errors.Is(err, ErrSessionArtifactNotFound) {
|
||||||
|
t.Fatalf("errors.Is(err, ErrSessionArtifactNotFound) = false; err=%v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveSessionArtifactValidatesTranscriptShape(t *testing.T) {
|
||||||
|
workspace := t.TempDir()
|
||||||
|
paths := buildSessionPaths(workspace, "campaign", "session")
|
||||||
|
canonicalPath := filepath.Join(paths.TranscriptsDir, "processed.json")
|
||||||
|
if err := os.MkdirAll(filepath.Dir(canonicalPath), 0o755); err != nil {
|
||||||
|
t.Fatalf("MkdirAll() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(canonicalPath, []byte(`{"not_segments":[]}`), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := ResolveSessionArtifact(paths, nil, ArtifactTranscriptPolished)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "top-level segments is required") {
|
||||||
|
t.Fatalf("error = %q, want segments validation error", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -30,13 +30,13 @@ func NewLocalStore(workspaceRoot string) *LocalStore {
|
|||||||
return &LocalStore{WorkspaceRoot: workspaceRoot}
|
return &LocalStore{WorkspaceRoot: workspaceRoot}
|
||||||
}
|
}
|
||||||
|
|
||||||
// SessionPaths resolves canonical paths for a session workdir.
|
// SessionPathsFor resolves canonical campaign-aware paths for a session workdir.
|
||||||
func (s *LocalStore) SessionPaths(sessionID string) SessionPaths {
|
func (s *LocalStore) SessionPathsFor(campaign, sessionID string) SessionPaths {
|
||||||
return buildSessionPaths(s.WorkspaceRoot, sessionID)
|
return buildSessionPaths(s.WorkspaceRoot, campaign, sessionID)
|
||||||
}
|
}
|
||||||
|
|
||||||
// EnsureLayout creates and verifies the canonical session workdir directory layout.
|
// EnsureLayoutFor creates and verifies campaign-aware session layout.
|
||||||
func (s *LocalStore) EnsureLayout(sessionID string) (SessionPaths, error) {
|
func (s *LocalStore) EnsureLayoutFor(campaign, sessionID string) (SessionPaths, error) {
|
||||||
if strings.TrimSpace(s.WorkspaceRoot) == "" {
|
if strings.TrimSpace(s.WorkspaceRoot) == "" {
|
||||||
return SessionPaths{}, fmt.Errorf("workspace root is required")
|
return SessionPaths{}, fmt.Errorf("workspace root is required")
|
||||||
}
|
}
|
||||||
@@ -44,7 +44,22 @@ func (s *LocalStore) EnsureLayout(sessionID string) (SessionPaths, error) {
|
|||||||
return SessionPaths{}, fmt.Errorf("sessionID is required")
|
return SessionPaths{}, fmt.Errorf("sessionID is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
paths := s.SessionPaths(sessionID)
|
campaign = strings.TrimSpace(campaign)
|
||||||
|
if campaign == "" {
|
||||||
|
return SessionPaths{}, fmt.Errorf("campaign is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
return s.ensureLayout(s.SessionPathsFor(campaign, sessionID))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *LocalStore) ensureLayout(paths SessionPaths) (SessionPaths, error) {
|
||||||
|
if strings.TrimSpace(s.WorkspaceRoot) == "" {
|
||||||
|
return SessionPaths{}, fmt.Errorf("workspace root is required")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(paths.SessionID) == "" {
|
||||||
|
return SessionPaths{}, fmt.Errorf("sessionID is required")
|
||||||
|
}
|
||||||
|
|
||||||
dirs := []string{
|
dirs := []string{
|
||||||
paths.Root,
|
paths.Root,
|
||||||
paths.InputsDir,
|
paths.InputsDir,
|
||||||
@@ -53,8 +68,11 @@ func (s *LocalStore) EnsureLayout(sessionID string) (SessionPaths, error) {
|
|||||||
paths.TranscriptsRawDir,
|
paths.TranscriptsRawDir,
|
||||||
paths.TranscriptsTrimmedDir,
|
paths.TranscriptsTrimmedDir,
|
||||||
paths.ArtifactsDir,
|
paths.ArtifactsDir,
|
||||||
|
paths.ReportsDir,
|
||||||
paths.ConfigDir,
|
paths.ConfigDir,
|
||||||
paths.LogsDir,
|
paths.LogsDir,
|
||||||
|
paths.CurrentDir,
|
||||||
|
paths.RunsDir,
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, dir := range dirs {
|
for _, dir := range dirs {
|
||||||
@@ -66,13 +84,16 @@ func (s *LocalStore) EnsureLayout(sessionID string) (SessionPaths, error) {
|
|||||||
return paths, nil
|
return paths, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// CopyInput copies an input file into the session workdir under destRelativePath.
|
// CopyInputFor copies an input file into the campaign-aware session workdir under destRelativePath.
|
||||||
func (s *LocalStore) CopyInput(sessionID, srcPath, destRelativePath string) (Ref, error) {
|
func (s *LocalStore) CopyInputFor(campaign, sessionID, srcPath, destRelativePath string) (Ref, error) {
|
||||||
paths, err := s.EnsureLayout(sessionID)
|
paths, err := s.EnsureLayoutFor(campaign, sessionID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Ref{}, err
|
return Ref{}, err
|
||||||
}
|
}
|
||||||
|
return s.copyInputWithPaths(paths, sessionID, srcPath, destRelativePath)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *LocalStore) copyInputWithPaths(paths SessionPaths, sessionID, srcPath, destRelativePath string) (Ref, error) {
|
||||||
destAbs, err := resolveInRoot(paths.Root, destRelativePath)
|
destAbs, err := resolveInRoot(paths.Root, destRelativePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Ref{}, fmt.Errorf("copy input: %w", err)
|
return Ref{}, fmt.Errorf("copy input: %w", err)
|
||||||
@@ -173,13 +194,16 @@ func (s *LocalStore) Checksum(path string) (string, error) {
|
|||||||
return digest, nil
|
return digest, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// AcquireSessionLock acquires an exclusive lock file for a session workdir.
|
// AcquireSessionLockFor acquires an exclusive lock file for a campaign/session workdir.
|
||||||
func (s *LocalStore) AcquireSessionLock(sessionID string) (*LockHandle, error) {
|
func (s *LocalStore) AcquireSessionLockFor(campaign, sessionID string) (*LockHandle, error) {
|
||||||
paths, err := s.EnsureLayout(sessionID)
|
paths, err := s.EnsureLayoutFor(campaign, sessionID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
return s.acquireSessionLockForPaths(paths)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *LocalStore) acquireSessionLockForPaths(paths SessionPaths) (*LockHandle, error) {
|
||||||
f, err := os.OpenFile(paths.LockPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o644)
|
f, err := os.OpenFile(paths.LockPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o644)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, os.ErrExist) {
|
if errors.Is(err, os.ErrExist) {
|
||||||
|
|||||||
@@ -10,9 +10,9 @@ import (
|
|||||||
|
|
||||||
func TestEnsureLayoutCreatesExpectedDirectories(t *testing.T) {
|
func TestEnsureLayoutCreatesExpectedDirectories(t *testing.T) {
|
||||||
store := NewLocalStore(t.TempDir())
|
store := NewLocalStore(t.TempDir())
|
||||||
paths, err := store.EnsureLayout("session-1")
|
paths, err := store.EnsureLayoutFor("sample-campaign", "session-1")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("EnsureLayout() error = %v", err)
|
t.Fatalf("EnsureLayoutFor() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
checkDirExists(t, paths.Root)
|
checkDirExists(t, paths.Root)
|
||||||
@@ -22,8 +22,11 @@ func TestEnsureLayoutCreatesExpectedDirectories(t *testing.T) {
|
|||||||
checkDirExists(t, paths.TranscriptsRawDir)
|
checkDirExists(t, paths.TranscriptsRawDir)
|
||||||
checkDirExists(t, paths.TranscriptsTrimmedDir)
|
checkDirExists(t, paths.TranscriptsTrimmedDir)
|
||||||
checkDirExists(t, paths.ArtifactsDir)
|
checkDirExists(t, paths.ArtifactsDir)
|
||||||
|
checkDirExists(t, paths.ReportsDir)
|
||||||
checkDirExists(t, paths.ConfigDir)
|
checkDirExists(t, paths.ConfigDir)
|
||||||
checkDirExists(t, paths.LogsDir)
|
checkDirExists(t, paths.LogsDir)
|
||||||
|
checkDirExists(t, paths.CurrentDir)
|
||||||
|
checkDirExists(t, paths.RunsDir)
|
||||||
|
|
||||||
if filepath.Base(paths.ManifestPath) != "manifest.json" {
|
if filepath.Base(paths.ManifestPath) != "manifest.json" {
|
||||||
t.Fatalf("ManifestPath = %q, want basename manifest.json", paths.ManifestPath)
|
t.Fatalf("ManifestPath = %q, want basename manifest.json", paths.ManifestPath)
|
||||||
@@ -33,6 +36,17 @@ func TestEnsureLayoutCreatesExpectedDirectories(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestEnsureLayoutForRequiresCampaign(t *testing.T) {
|
||||||
|
store := NewLocalStore(t.TempDir())
|
||||||
|
_, err := store.EnsureLayoutFor("", "session-1")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected campaign-required error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "campaign is required") {
|
||||||
|
t.Fatalf("error = %v, want campaign-required error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestChecksumCalculation(t *testing.T) {
|
func TestChecksumCalculation(t *testing.T) {
|
||||||
store := NewLocalStore(t.TempDir())
|
store := NewLocalStore(t.TempDir())
|
||||||
path := filepath.Join(t.TempDir(), "sample.txt")
|
path := filepath.Join(t.TempDir(), "sample.txt")
|
||||||
@@ -53,9 +67,9 @@ func TestChecksumCalculation(t *testing.T) {
|
|||||||
func TestLockAcquireRelease(t *testing.T) {
|
func TestLockAcquireRelease(t *testing.T) {
|
||||||
store := NewLocalStore(t.TempDir())
|
store := NewLocalStore(t.TempDir())
|
||||||
|
|
||||||
lock, err := store.AcquireSessionLock("session-1")
|
lock, err := store.AcquireSessionLockFor("sample-campaign", "session-1")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("AcquireSessionLock() error = %v", err)
|
t.Fatalf("AcquireSessionLockFor() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
exists, err := store.Exists(lock.path)
|
exists, err := store.Exists(lock.path)
|
||||||
@@ -81,15 +95,15 @@ func TestLockAcquireRelease(t *testing.T) {
|
|||||||
|
|
||||||
func TestLockConflict(t *testing.T) {
|
func TestLockConflict(t *testing.T) {
|
||||||
store := NewLocalStore(t.TempDir())
|
store := NewLocalStore(t.TempDir())
|
||||||
lock1, err := store.AcquireSessionLock("session-1")
|
lock1, err := store.AcquireSessionLockFor("sample-campaign", "session-1")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("first AcquireSessionLock() error = %v", err)
|
t.Fatalf("first AcquireSessionLockFor() error = %v", err)
|
||||||
}
|
}
|
||||||
defer func() {
|
defer func() {
|
||||||
_ = store.ReleaseSessionLock(lock1)
|
_ = store.ReleaseSessionLock(lock1)
|
||||||
}()
|
}()
|
||||||
|
|
||||||
_, err = store.AcquireSessionLock("session-1")
|
_, err = store.AcquireSessionLockFor("sample-campaign", "session-1")
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected lock conflict error, got nil")
|
t.Fatal("expected lock conflict error, got nil")
|
||||||
}
|
}
|
||||||
@@ -138,9 +152,9 @@ func TestCopyInput(t *testing.T) {
|
|||||||
t.Fatalf("WriteFile() error = %v", err)
|
t.Fatalf("WriteFile() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
ref, err := store.CopyInput("session-1", srcPath, "inputs/speakers.yml")
|
ref, err := store.CopyInputFor("sample-campaign", "session-1", srcPath, "inputs/speakers.yml")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("CopyInput() error = %v", err)
|
t.Fatalf("CopyInputFor() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if ref.Kind != "input" {
|
if ref.Kind != "input" {
|
||||||
|
|||||||
@@ -1,10 +1,16 @@
|
|||||||
package artifacts
|
package artifacts
|
||||||
|
|
||||||
import "path/filepath"
|
import (
|
||||||
|
"path/filepath"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
// SessionPaths contains canonical local paths for one session work directory.
|
// SessionPaths contains canonical local paths for one session work directory.
|
||||||
type SessionPaths struct {
|
type SessionPaths struct {
|
||||||
WorkspaceRoot string
|
WorkspaceRoot string
|
||||||
|
CampaignID string
|
||||||
|
SessionID string
|
||||||
Root string
|
Root string
|
||||||
InputsDir string
|
InputsDir string
|
||||||
AudioDir string
|
AudioDir string
|
||||||
@@ -12,32 +18,73 @@ type SessionPaths struct {
|
|||||||
TranscriptsRawDir string
|
TranscriptsRawDir string
|
||||||
TranscriptsTrimmedDir string
|
TranscriptsTrimmedDir string
|
||||||
ArtifactsDir string
|
ArtifactsDir string
|
||||||
|
ReportsDir string
|
||||||
ConfigDir string
|
ConfigDir string
|
||||||
LogsDir string
|
LogsDir string
|
||||||
|
CurrentDir string
|
||||||
|
RunsDir string
|
||||||
ManifestPath string
|
ManifestPath string
|
||||||
LockPath string
|
LockPath string
|
||||||
}
|
}
|
||||||
|
|
||||||
// SessionWorkDir returns the work directory for one session.
|
// SessionWorkDirForCampaign returns the canonical campaign-aware work directory for one session.
|
||||||
func SessionWorkDir(rootDir, sessionID string) string {
|
func SessionWorkDirForCampaign(rootDir, campaign, sessionID string) string {
|
||||||
return filepath.Join(rootDir, "work", sessionID)
|
return filepath.Join(rootDir, config.PathWorkDirSegment, campaign, sessionID)
|
||||||
}
|
}
|
||||||
|
|
||||||
func buildSessionPaths(workspaceRoot, sessionID string) SessionPaths {
|
// SessionManifestPathForCampaign returns the canonical session manifest path.
|
||||||
root := SessionWorkDir(workspaceRoot, sessionID)
|
func SessionManifestPathForCampaign(rootDir, campaign, sessionID string) string {
|
||||||
transcripts := filepath.Join(root, "transcripts")
|
return filepath.Join(SessionWorkDirForCampaign(rootDir, campaign, sessionID), config.PathManifestFile)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SessionRunsDirForCampaign returns the canonical runs directory for one session.
|
||||||
|
func SessionRunsDirForCampaign(rootDir, campaign, sessionID string) string {
|
||||||
|
return filepath.Join(SessionWorkDirForCampaign(rootDir, campaign, sessionID), config.PathRunsDirSegment)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SessionRunRootForCampaign returns the canonical run root under runs/{run_id}.
|
||||||
|
func SessionRunRootForCampaign(rootDir, campaign, sessionID, runID string) string {
|
||||||
|
return filepath.Join(SessionRunsDirForCampaign(rootDir, campaign, sessionID), runID)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SessionRunManifestPathForCampaign returns the canonical run manifest path under runs/{run_id}/manifest.json.
|
||||||
|
func SessionRunManifestPathForCampaign(rootDir, campaign, sessionID, runID string) string {
|
||||||
|
return filepath.Join(SessionRunRootForCampaign(rootDir, campaign, sessionID, runID), config.PathManifestFile)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SessionRunStageDirForCampaign returns the canonical stage directory under runs/{run_id}/{stage}.
|
||||||
|
func SessionRunStageDirForCampaign(rootDir, campaign, sessionID, runID, stageName string) string {
|
||||||
|
return filepath.Join(SessionRunRootForCampaign(rootDir, campaign, sessionID, runID), stageName)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SessionSpoolAudioDir returns the campaign/session/run scoped local spool audio path.
|
||||||
|
func SessionSpoolAudioDir(spoolRoot, campaign, sessionID, runID string) string {
|
||||||
|
return filepath.Join(spoolRoot, campaign, sessionID, runID, config.PathAudioDirSegment)
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildSessionPaths(workspaceRoot, campaign, sessionID string) SessionPaths {
|
||||||
|
root := SessionWorkDirForCampaign(workspaceRoot, campaign, sessionID)
|
||||||
|
return buildSessionPathsFromRoot(workspaceRoot, campaign, sessionID, root)
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildSessionPathsFromRoot(workspaceRoot, campaign, sessionID, root string) SessionPaths {
|
||||||
return SessionPaths{
|
return SessionPaths{
|
||||||
WorkspaceRoot: workspaceRoot,
|
WorkspaceRoot: workspaceRoot,
|
||||||
|
CampaignID: campaign,
|
||||||
|
SessionID: sessionID,
|
||||||
Root: root,
|
Root: root,
|
||||||
InputsDir: filepath.Join(root, "inputs"),
|
InputsDir: filepath.Join(root, config.PathInputsDirSegment),
|
||||||
AudioDir: filepath.Join(root, "audio"),
|
AudioDir: filepath.Join(root, config.PathAudioDirSegment),
|
||||||
TranscriptsDir: transcripts,
|
TranscriptsDir: filepath.Join(root, config.PathTranscriptsSegment),
|
||||||
TranscriptsRawDir: filepath.Join(transcripts, "raw"),
|
TranscriptsRawDir: filepath.Join(root, filepath.FromSlash(config.PathTranscriptsRaw)),
|
||||||
TranscriptsTrimmedDir: filepath.Join(transcripts, "trimmed"),
|
TranscriptsTrimmedDir: filepath.Join(root, filepath.FromSlash(config.PathTranscriptsTrimmed)),
|
||||||
ArtifactsDir: filepath.Join(root, "artifacts"),
|
ArtifactsDir: filepath.Join(root, config.PathArtifactsDirSegment),
|
||||||
ConfigDir: filepath.Join(root, "config"),
|
ReportsDir: filepath.Join(root, config.PathReportsDirSegment),
|
||||||
LogsDir: filepath.Join(root, "logs"),
|
ConfigDir: filepath.Join(root, config.PathConfigDirSegment),
|
||||||
ManifestPath: filepath.Join(root, "manifest.json"),
|
LogsDir: filepath.Join(root, config.PathLogsDirSegment),
|
||||||
LockPath: filepath.Join(root, ".lock"),
|
CurrentDir: filepath.Join(root, config.PathCurrentDirSegment),
|
||||||
|
RunsDir: filepath.Join(root, config.PathRunsDirSegment),
|
||||||
|
ManifestPath: filepath.Join(root, config.PathManifestFile),
|
||||||
|
LockPath: filepath.Join(root, config.PathLockFile),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
59
internal/artifacts/paths_model_test.go
Normal file
59
internal/artifacts/paths_model_test.go
Normal file
@@ -0,0 +1,59 @@
|
|||||||
|
package artifacts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSessionWorkDirForCampaign(t *testing.T) {
|
||||||
|
root := "/tmp/workspace"
|
||||||
|
got := SessionWorkDirForCampaign(root, "forsaken", "2026-04-19")
|
||||||
|
want := filepath.Join(root, "work", "forsaken", "2026-04-19")
|
||||||
|
if got != want {
|
||||||
|
t.Fatalf("SessionWorkDirForCampaign() = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSessionManifestPathForCampaign(t *testing.T) {
|
||||||
|
root := "/tmp/workspace"
|
||||||
|
got := SessionManifestPathForCampaign(root, "forsaken", "2026-04-19")
|
||||||
|
want := filepath.Join(root, "work", "forsaken", "2026-04-19", "manifest.json")
|
||||||
|
if got != want {
|
||||||
|
t.Fatalf("SessionManifestPathForCampaign() = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSessionRunRootAndStageDirForCampaign(t *testing.T) {
|
||||||
|
root := "/tmp/workspace"
|
||||||
|
runID := "20260515T031522Z-a1b2c3d4"
|
||||||
|
runRoot := SessionRunRootForCampaign(root, "forsaken", "2026-04-19", runID)
|
||||||
|
wantRoot := filepath.Join(root, "work", "forsaken", "2026-04-19", "runs", runID)
|
||||||
|
if runRoot != wantRoot {
|
||||||
|
t.Fatalf("SessionRunRootForCampaign() = %q, want %q", runRoot, wantRoot)
|
||||||
|
}
|
||||||
|
|
||||||
|
stageDir := SessionRunStageDirForCampaign(root, "forsaken", "2026-04-19", runID, "transcribe")
|
||||||
|
wantStage := filepath.Join(root, "work", "forsaken", "2026-04-19", "runs", runID, "transcribe")
|
||||||
|
if stageDir != wantStage {
|
||||||
|
t.Fatalf("SessionRunStageDirForCampaign() = %q, want %q", stageDir, wantStage)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSessionRunManifestPathForCampaign(t *testing.T) {
|
||||||
|
root := "/tmp/workspace"
|
||||||
|
runID := "20260515T031522Z-a1b2c3d4"
|
||||||
|
got := SessionRunManifestPathForCampaign(root, "forsaken", "2026-04-19", runID)
|
||||||
|
want := filepath.Join(root, "work", "forsaken", "2026-04-19", "runs", runID, "manifest.json")
|
||||||
|
if got != want {
|
||||||
|
t.Fatalf("SessionRunManifestPathForCampaign() = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSessionSpoolAudioDir(t *testing.T) {
|
||||||
|
root := "/var/spool/narratio"
|
||||||
|
got := SessionSpoolAudioDir(root, "forsaken", "2026-04-19", "20260515T031522Z-a1b2c3d4")
|
||||||
|
want := filepath.Join(root, "forsaken", "2026-04-19", "20260515T031522Z-a1b2c3d4", "audio")
|
||||||
|
if got != want {
|
||||||
|
t.Fatalf("SessionSpoolAudioDir() = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -8,7 +8,7 @@ import (
|
|||||||
|
|
||||||
func TestResolveSessionLocalPathForRead(t *testing.T) {
|
func TestResolveSessionLocalPathForRead(t *testing.T) {
|
||||||
workspace := t.TempDir()
|
workspace := t.TempDir()
|
||||||
paths := buildSessionPaths(workspace, "s-1")
|
paths := buildSessionPaths(workspace, "sample-campaign", "s-1")
|
||||||
if err := os.MkdirAll(paths.TranscriptsRawDir, 0o755); err != nil {
|
if err := os.MkdirAll(paths.TranscriptsRawDir, 0o755); err != nil {
|
||||||
t.Fatalf("MkdirAll() error = %v", err)
|
t.Fatalf("MkdirAll() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -41,7 +41,7 @@ func TestResolveSessionLocalPathForReadRelativeWorkspaceRootQualifiedPath(t *tes
|
|||||||
t.Fatalf("Rel() error = %v", err)
|
t.Fatalf("Rel() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
paths := buildSessionPaths(workspaceRel, "s-1")
|
paths := buildSessionPaths(workspaceRel, "sample-campaign", "s-1")
|
||||||
target := filepath.Join(paths.TranscriptsRawDir, "alice.json")
|
target := filepath.Join(paths.TranscriptsRawDir, "alice.json")
|
||||||
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
|
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
|
||||||
t.Fatalf("MkdirAll() error = %v", err)
|
t.Fatalf("MkdirAll() error = %v", err)
|
||||||
@@ -50,7 +50,7 @@ func TestResolveSessionLocalPathForReadRelativeWorkspaceRootQualifiedPath(t *tes
|
|||||||
t.Fatalf("WriteFile() error = %v", err)
|
t.Fatalf("WriteFile() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
manifestPath := filepath.Join(workspaceRel, "work", "s-1", "transcripts", "raw", "alice.json")
|
manifestPath := filepath.Join(workspaceRel, "work", "sample-campaign", "s-1", "transcripts", "raw", "alice.json")
|
||||||
got := ResolveSessionLocalPathForRead(paths, manifestPath)
|
got := ResolveSessionLocalPathForRead(paths, manifestPath)
|
||||||
if got != filepath.Clean(manifestPath) {
|
if got != filepath.Clean(manifestPath) {
|
||||||
t.Fatalf("resolution = %q, want %q", got, filepath.Clean(manifestPath))
|
t.Fatalf("resolution = %q, want %q", got, filepath.Clean(manifestPath))
|
||||||
|
|||||||
30
internal/artifacts/run_id.go
Normal file
30
internal/artifacts/run_id.go
Normal file
@@ -0,0 +1,30 @@
|
|||||||
|
package artifacts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/rand"
|
||||||
|
"encoding/hex"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// NewRunID returns a run ID in format: YYYYMMDDTHHMMSSZ-xxxxxxxx.
|
||||||
|
func NewRunID() (string, error) {
|
||||||
|
return NewRunIDWith(time.Now().UTC(), rand.Reader)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewRunIDWith returns a run ID in format: YYYYMMDDTHHMMSSZ-xxxxxxxx
|
||||||
|
// using an injected timestamp and randomness source.
|
||||||
|
func NewRunIDWith(now time.Time, random io.Reader) (string, error) {
|
||||||
|
if random == nil {
|
||||||
|
random = rand.Reader
|
||||||
|
}
|
||||||
|
|
||||||
|
var suffix [4]byte
|
||||||
|
if _, err := io.ReadFull(random, suffix[:]); err != nil {
|
||||||
|
return "", fmt.Errorf("generate run id random suffix: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ts := now.UTC().Format("20060102T150405Z")
|
||||||
|
return ts + "-" + hex.EncodeToString(suffix[:]), nil
|
||||||
|
}
|
||||||
37
internal/artifacts/run_id_test.go
Normal file
37
internal/artifacts/run_id_test.go
Normal file
@@ -0,0 +1,37 @@
|
|||||||
|
package artifacts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNewRunIDWithFormat(t *testing.T) {
|
||||||
|
now := time.Date(2026, 5, 15, 3, 15, 22, 0, time.UTC)
|
||||||
|
random := bytes.NewReader([]byte{0xa1, 0xb2, 0xc3, 0xd4})
|
||||||
|
|
||||||
|
runID, err := NewRunIDWith(now, random)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewRunIDWith() error = %v", err)
|
||||||
|
}
|
||||||
|
if runID != "20260515T031522Z-a1b2c3d4" {
|
||||||
|
t.Fatalf("runID = %q, want %q", runID, "20260515T031522Z-a1b2c3d4")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewRunIDWithShape(t *testing.T) {
|
||||||
|
runID, err := NewRunIDWith(time.Now().UTC(), bytes.NewReader([]byte{0x01, 0x02, 0x03, 0x04}))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewRunIDWith() error = %v", err)
|
||||||
|
}
|
||||||
|
pattern := regexp.MustCompile(`^\d{8}T\d{6}Z-[0-9a-f]{8}$`)
|
||||||
|
if !pattern.MatchString(runID) {
|
||||||
|
t.Fatalf("runID = %q, want pattern %q", runID, pattern.String())
|
||||||
|
}
|
||||||
|
suffix := runID[len(runID)-8:]
|
||||||
|
if strings.ToLower(suffix) != suffix {
|
||||||
|
t.Fatalf("runID suffix = %q, want lowercase", suffix)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -11,19 +11,19 @@ type S3Store struct {
|
|||||||
Prefix string
|
Prefix string
|
||||||
}
|
}
|
||||||
|
|
||||||
// SessionPaths is not implemented for S3-backed storage.
|
// SessionPathsFor is not implemented for S3-backed storage.
|
||||||
func (s *S3Store) SessionPaths(_ string) SessionPaths {
|
func (s *S3Store) SessionPathsFor(_, _ string) SessionPaths {
|
||||||
return SessionPaths{}
|
return SessionPaths{}
|
||||||
}
|
}
|
||||||
|
|
||||||
// EnsureLayout returns a not-yet-implemented error in the scaffold.
|
// EnsureLayoutFor returns a not-yet-implemented error in the scaffold.
|
||||||
func (s *S3Store) EnsureLayout(_ string) (SessionPaths, error) {
|
func (s *S3Store) EnsureLayoutFor(_, _ string) (SessionPaths, error) {
|
||||||
return SessionPaths{}, fmt.Errorf("artifacts s3 ensure layout: not yet implemented")
|
return SessionPaths{}, fmt.Errorf("artifacts s3 ensure layout for campaign/session: not yet implemented")
|
||||||
}
|
}
|
||||||
|
|
||||||
// CopyInput returns a not-yet-implemented error in the scaffold.
|
// CopyInputFor returns a not-yet-implemented error in the scaffold.
|
||||||
func (s *S3Store) CopyInput(_, _, _ string) (Ref, error) {
|
func (s *S3Store) CopyInputFor(_, _, _, _ string) (Ref, error) {
|
||||||
return Ref{}, fmt.Errorf("artifacts s3 copy input: not yet implemented")
|
return Ref{}, fmt.Errorf("artifacts s3 copy input for campaign/session: not yet implemented")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Exists returns a not-yet-implemented error in the scaffold.
|
// Exists returns a not-yet-implemented error in the scaffold.
|
||||||
@@ -46,9 +46,9 @@ func (s *S3Store) Checksum(_ string) (string, error) {
|
|||||||
return "", fmt.Errorf("artifacts s3 checksum: not yet implemented")
|
return "", fmt.Errorf("artifacts s3 checksum: not yet implemented")
|
||||||
}
|
}
|
||||||
|
|
||||||
// AcquireSessionLock returns a not-yet-implemented error in the scaffold.
|
// AcquireSessionLockFor returns a not-yet-implemented error in the scaffold.
|
||||||
func (s *S3Store) AcquireSessionLock(_ string) (*LockHandle, error) {
|
func (s *S3Store) AcquireSessionLockFor(_, _ string) (*LockHandle, error) {
|
||||||
return nil, fmt.Errorf("artifacts s3 acquire lock: not yet implemented")
|
return nil, fmt.Errorf("artifacts s3 acquire lock for campaign/session: not yet implemented")
|
||||||
}
|
}
|
||||||
|
|
||||||
// ReleaseSessionLock returns a not-yet-implemented error in the scaffold.
|
// ReleaseSessionLock returns a not-yet-implemented error in the scaffold.
|
||||||
|
|||||||
75
internal/artifacts/s3_keys.go
Normal file
75
internal/artifacts/s3_keys.go
Normal file
@@ -0,0 +1,75 @@
|
|||||||
|
package artifacts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"path"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
// S3SessionPrefix builds the canonical S3 session prefix.
|
||||||
|
// Format: {root_prefix}/campaigns/{campaign}/sessions/{session_id}/
|
||||||
|
func S3SessionPrefix(rootPrefix, campaign, sessionID string) string {
|
||||||
|
prefix := path.Join(
|
||||||
|
cleanS3PathPart(rootPrefix),
|
||||||
|
config.S3CampaignsSegment,
|
||||||
|
cleanS3PathPart(campaign),
|
||||||
|
config.S3SessionsSegment,
|
||||||
|
cleanS3PathPart(sessionID),
|
||||||
|
)
|
||||||
|
return ensureS3TrailingSlash(prefix)
|
||||||
|
}
|
||||||
|
|
||||||
|
// S3RunPrefix builds the canonical S3 run prefix.
|
||||||
|
// Format: {session_prefix}/runs/{run_id}/
|
||||||
|
func S3RunPrefix(sessionPrefix, runID string) string {
|
||||||
|
prefix := path.Join(strings.TrimSuffix(cleanS3Key(sessionPrefix), "/"), config.S3RunsSegment, cleanS3PathPart(runID))
|
||||||
|
return ensureS3TrailingSlash(prefix)
|
||||||
|
}
|
||||||
|
|
||||||
|
// S3AudioPrefix builds the session audio prefix from configured audio_s3.prefix.
|
||||||
|
// Format: {session_prefix}/{audio_s3.prefix}
|
||||||
|
func S3AudioPrefix(sessionPrefix, audioPrefix string) string {
|
||||||
|
key := path.Join(strings.TrimSuffix(cleanS3Key(sessionPrefix), "/"), cleanS3Key(audioPrefix))
|
||||||
|
return ensureS3TrailingSlash(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
// S3CurrentManifestKey returns the current manifest pointer key.
|
||||||
|
// Format: {session_prefix}/current/manifest.json
|
||||||
|
func S3CurrentManifestKey(sessionPrefix string) string {
|
||||||
|
return path.Join(strings.TrimSuffix(cleanS3Key(sessionPrefix), "/"), config.S3CurrentSegment, config.S3ManifestFile)
|
||||||
|
}
|
||||||
|
|
||||||
|
// S3CurrentRunPointerKey returns the current run pointer key.
|
||||||
|
// Format: {session_prefix}/current/run_id.txt
|
||||||
|
func S3CurrentRunPointerKey(sessionPrefix string) string {
|
||||||
|
return path.Join(strings.TrimSuffix(cleanS3Key(sessionPrefix), "/"), config.S3CurrentSegment, config.S3RunIDFile)
|
||||||
|
}
|
||||||
|
|
||||||
|
// S3PromotedArtifactKey returns the destination key for one promoted artifact.
|
||||||
|
// Format: {session_prefix}/{promotion.to}
|
||||||
|
func S3PromotedArtifactKey(sessionPrefix, to string) string {
|
||||||
|
return path.Join(strings.TrimSuffix(cleanS3Key(sessionPrefix), "/"), cleanS3Key(to))
|
||||||
|
}
|
||||||
|
|
||||||
|
// S3RunRelativeDestinationKey returns a run-scoped key for a workdir-relative path.
|
||||||
|
// Format: {run_prefix}/{relative_workdir_path}
|
||||||
|
func S3RunRelativeDestinationKey(runPrefix, relativeWorkdirPath string) string {
|
||||||
|
return path.Join(strings.TrimSuffix(cleanS3Key(runPrefix), "/"), cleanS3Key(relativeWorkdirPath))
|
||||||
|
}
|
||||||
|
|
||||||
|
func ensureS3TrailingSlash(v string) string {
|
||||||
|
key := cleanS3Key(v)
|
||||||
|
if key == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return strings.TrimSuffix(key, "/") + "/"
|
||||||
|
}
|
||||||
|
|
||||||
|
func cleanS3PathPart(v string) string {
|
||||||
|
return strings.Trim(strings.ReplaceAll(strings.TrimSpace(v), "\\", "/"), "/")
|
||||||
|
}
|
||||||
|
|
||||||
|
func cleanS3Key(v string) string {
|
||||||
|
return strings.ReplaceAll(strings.TrimSpace(v), "\\", "/")
|
||||||
|
}
|
||||||
45
internal/artifacts/s3_keys_test.go
Normal file
45
internal/artifacts/s3_keys_test.go
Normal file
@@ -0,0 +1,45 @@
|
|||||||
|
package artifacts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestS3KeyConstruction(t *testing.T) {
|
||||||
|
runID := "20260515T031522Z-a1b2c3d4"
|
||||||
|
sessionPrefix := S3SessionPrefix("dnd", "forsaken", "2026-04-19")
|
||||||
|
if sessionPrefix != "dnd/campaigns/forsaken/sessions/2026-04-19/" {
|
||||||
|
t.Fatalf("sessionPrefix = %q", sessionPrefix)
|
||||||
|
}
|
||||||
|
|
||||||
|
audioPrefix := S3AudioPrefix(sessionPrefix, "audio/")
|
||||||
|
if audioPrefix != "dnd/campaigns/forsaken/sessions/2026-04-19/audio/" {
|
||||||
|
t.Fatalf("audioPrefix = %q", audioPrefix)
|
||||||
|
}
|
||||||
|
|
||||||
|
runPrefix := S3RunPrefix(sessionPrefix, runID)
|
||||||
|
wantRunPrefix := "dnd/campaigns/forsaken/sessions/2026-04-19/runs/" + runID + "/"
|
||||||
|
if runPrefix != wantRunPrefix {
|
||||||
|
t.Fatalf("runPrefix = %q, want %q", runPrefix, wantRunPrefix)
|
||||||
|
}
|
||||||
|
|
||||||
|
runPointer := S3CurrentRunPointerKey(sessionPrefix)
|
||||||
|
if runPointer != "dnd/campaigns/forsaken/sessions/2026-04-19/current/run_id.txt" {
|
||||||
|
t.Fatalf("run pointer key = %q", runPointer)
|
||||||
|
}
|
||||||
|
|
||||||
|
manifestKey := S3CurrentManifestKey(sessionPrefix)
|
||||||
|
if manifestKey != "dnd/campaigns/forsaken/sessions/2026-04-19/current/manifest.json" {
|
||||||
|
t.Fatalf("manifest key = %q", manifestKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
promoted := S3PromotedArtifactKey(sessionPrefix, "transcripts/trimmed.json")
|
||||||
|
if promoted != "dnd/campaigns/forsaken/sessions/2026-04-19/transcripts/trimmed.json" {
|
||||||
|
t.Fatalf("promoted key = %q", promoted)
|
||||||
|
}
|
||||||
|
|
||||||
|
runRelative := S3RunRelativeDestinationKey(runPrefix, `logs\whisperx.stdout.log`)
|
||||||
|
if !strings.HasSuffix(runRelative, "/logs/whisperx.stdout.log") {
|
||||||
|
t.Fatalf("runRelative key = %q, want normalized forward slashes", runRelative)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -15,13 +15,13 @@ type Ref struct {
|
|||||||
|
|
||||||
// Store is the local artifact/workdir abstraction used by orchestration code.
|
// Store is the local artifact/workdir abstraction used by orchestration code.
|
||||||
type Store interface {
|
type Store interface {
|
||||||
SessionPaths(sessionID string) SessionPaths
|
SessionPathsFor(campaign, sessionID string) SessionPaths
|
||||||
EnsureLayout(sessionID string) (SessionPaths, error)
|
EnsureLayoutFor(campaign, sessionID string) (SessionPaths, error)
|
||||||
CopyInput(sessionID, srcPath, destRelativePath string) (Ref, error)
|
CopyInputFor(campaign, sessionID, srcPath, destRelativePath string) (Ref, error)
|
||||||
Exists(path string) (bool, error)
|
Exists(path string) (bool, error)
|
||||||
ExistsRef(ref Ref) (bool, error)
|
ExistsRef(ref Ref) (bool, error)
|
||||||
WriteFileAtomic(path string, data []byte, perm os.FileMode) error
|
WriteFileAtomic(path string, data []byte, perm os.FileMode) error
|
||||||
Checksum(path string) (string, error)
|
Checksum(path string) (string, error)
|
||||||
AcquireSessionLock(sessionID string) (*LockHandle, error)
|
AcquireSessionLockFor(campaign, sessionID string) (*LockHandle, error)
|
||||||
ReleaseSessionLock(lock *LockHandle) error
|
ReleaseSessionLock(lock *LockHandle) error
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,6 +12,9 @@ type Config struct {
|
|||||||
type PipelineConfig struct {
|
type PipelineConfig struct {
|
||||||
Workspace WorkspaceConfig `yaml:"workspace"`
|
Workspace WorkspaceConfig `yaml:"workspace"`
|
||||||
Storage StorageConfig `yaml:"storage"`
|
Storage StorageConfig `yaml:"storage"`
|
||||||
|
Spool SpoolConfig `yaml:"spool"`
|
||||||
|
Archive *ArchiveConfig `yaml:"archive"`
|
||||||
|
Secrets *SecretsConfig `yaml:"secrets"`
|
||||||
WhisperX WhisperXConfig `yaml:"whisperx"`
|
WhisperX WhisperXConfig `yaml:"whisperx"`
|
||||||
Seriatim SeriatimConfig `yaml:"seriatim"`
|
Seriatim SeriatimConfig `yaml:"seriatim"`
|
||||||
Audita AuditaConfig `yaml:"audita"`
|
Audita AuditaConfig `yaml:"audita"`
|
||||||
@@ -33,14 +36,52 @@ type SessionConfig struct {
|
|||||||
|
|
||||||
// WorkspaceConfig configures local workspace behavior.
|
// WorkspaceConfig configures local workspace behavior.
|
||||||
type WorkspaceConfig struct {
|
type WorkspaceConfig struct {
|
||||||
Root string `yaml:"root"`
|
Root string `yaml:"root"`
|
||||||
|
CleanupAfterArchive bool `yaml:"cleanup_after_archive"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// SecretsConfig configures optional local filesystem secret loading.
|
||||||
|
type SecretsConfig struct {
|
||||||
|
EnvDir string `yaml:"env_dir"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// StorageConfig configures storage backends and related parameters.
|
// StorageConfig configures storage backends and related parameters.
|
||||||
type StorageConfig struct {
|
type StorageConfig struct {
|
||||||
Backend string `yaml:"backend"`
|
Backend string `yaml:"backend"`
|
||||||
Bucket string `yaml:"bucket"`
|
Bucket string `yaml:"bucket"`
|
||||||
Prefix string `yaml:"prefix"`
|
Prefix string `yaml:"prefix"`
|
||||||
|
S3 *StorageS3Config `yaml:"s3"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// StorageS3Config configures S3 storage coordinates.
|
||||||
|
type StorageS3Config struct {
|
||||||
|
Bucket string `yaml:"bucket"`
|
||||||
|
RootPrefix string `yaml:"root_prefix"`
|
||||||
|
Region string `yaml:"region"`
|
||||||
|
Endpoint string `yaml:"endpoint"`
|
||||||
|
ForcePathStyle bool `yaml:"force_path_style"`
|
||||||
|
AccessKeyIDEnv string `yaml:"access_key_id_env"`
|
||||||
|
SecretKeyEnv string `yaml:"secret_access_key_env"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// SpoolConfig configures local spool storage for staged data.
|
||||||
|
type SpoolConfig struct {
|
||||||
|
Root string `yaml:"root"`
|
||||||
|
DeleteAudioAfterArchive bool `yaml:"delete_audio_after_archive"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ArchiveConfig configures archive behavior and artifact promotions.
|
||||||
|
type ArchiveConfig struct {
|
||||||
|
Enabled *bool `yaml:"enabled"`
|
||||||
|
UploadRun *bool `yaml:"upload_run"`
|
||||||
|
PromoteArtifacts []ArchivePromotionRule `yaml:"promote_artifacts"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ArchivePromotionRule configures one artifact promotion mapping.
|
||||||
|
type ArchivePromotionRule struct {
|
||||||
|
From string `yaml:"from"`
|
||||||
|
To string `yaml:"to"`
|
||||||
|
Required *bool `yaml:"required"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// WhisperXConfig configures WhisperX adapter settings.
|
// WhisperXConfig configures WhisperX adapter settings.
|
||||||
@@ -79,9 +120,14 @@ type AuditaConfig struct {
|
|||||||
Modules []string `yaml:"modules"`
|
Modules []string `yaml:"modules"`
|
||||||
BaseURL string `yaml:"base_url"`
|
BaseURL string `yaml:"base_url"`
|
||||||
Model string `yaml:"model"`
|
Model string `yaml:"model"`
|
||||||
LLMConcurrency *int `yaml:"llm_concurrency"`
|
TotalLLMConcurrency *int `yaml:"total_llm_concurrency"`
|
||||||
|
ProposalLLMConcurrency *int `yaml:"proposal_llm_concurrency"`
|
||||||
ValidationModel string `yaml:"validation_model"`
|
ValidationModel string `yaml:"validation_model"`
|
||||||
ValidationLLMConcurrency *int `yaml:"validation_llm_concurrency"`
|
ValidationLLMConcurrency *int `yaml:"validation_llm_concurrency"`
|
||||||
|
TranscriptDescription string `yaml:"transcript_description"`
|
||||||
|
ConfigPath string `yaml:"config_path"`
|
||||||
|
OutputSchema string `yaml:"output_schema"`
|
||||||
|
WorkDirRetention string `yaml:"work_dir_retention"`
|
||||||
Report *bool `yaml:"report"`
|
Report *bool `yaml:"report"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -169,9 +215,15 @@ type ArtifactSettings struct {
|
|||||||
|
|
||||||
// SessionInputsConfig contains per-session input references.
|
// SessionInputsConfig contains per-session input references.
|
||||||
type SessionInputsConfig struct {
|
type SessionInputsConfig struct {
|
||||||
AudioDir string `yaml:"audio_dir"`
|
AudioDir string `yaml:"audio_dir"`
|
||||||
AudioFiles []string `yaml:"audio_files"`
|
AudioFiles []string `yaml:"audio_files"`
|
||||||
SpeakersFile string `yaml:"speakers_file"`
|
AudioS3 *SessionAudioS3Input `yaml:"audio_s3"`
|
||||||
AutocorrectFile string `yaml:"autocorrect_file"`
|
SpeakersFile string `yaml:"speakers_file"`
|
||||||
GlossaryFile string `yaml:"glossary_file"`
|
AutocorrectFile string `yaml:"autocorrect_file"`
|
||||||
|
GlossaryFile string `yaml:"glossary_file"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// SessionAudioS3Input configures S3 session-audio input discovery.
|
||||||
|
type SessionAudioS3Input struct {
|
||||||
|
Prefix string `yaml:"prefix"`
|
||||||
}
|
}
|
||||||
|
|||||||
98
internal/config/defaults.go
Normal file
98
internal/config/defaults.go
Normal file
@@ -0,0 +1,98 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
// Default filesystem locations for pipeline configuration lookup when --config
|
||||||
|
// is omitted. Order is highest to lowest precedence.
|
||||||
|
const (
|
||||||
|
DefaultPipelineConfigPathUsrLocal = "/usr/local/etc/narratio/pipeline.yml"
|
||||||
|
DefaultPipelineConfigPathEtc = "/etc/narratio/pipeline.yml"
|
||||||
|
DefaultSessionConfigPathLocal = "./session.yml"
|
||||||
|
DefaultSessionConfigPathUsrLocal = "/usr/local/etc/narratio/session.yml"
|
||||||
|
DefaultSessionConfigPathEtc = "/etc/narratio/session.yml"
|
||||||
|
DefaultS3AccessKeyIDEnv = "OBJECT_STORAGE_KEY_ID"
|
||||||
|
DefaultS3SecretAccessKeyEnv = "OBJECT_STORAGE_KEY"
|
||||||
|
DefaultStorageS3RootPrefix = "dnd"
|
||||||
|
DefaultSpoolRoot = "/var/spool/narratio"
|
||||||
|
|
||||||
|
DefaultWhisperXLanguage = "en"
|
||||||
|
DefaultWhisperXTimeout = "30m"
|
||||||
|
DefaultWhisperXRetryDelay = "2s"
|
||||||
|
DefaultWhisperXConcurrency = 2
|
||||||
|
DefaultWhisperXRetries = 3
|
||||||
|
|
||||||
|
DefaultSeriatimBinary = "seriatim"
|
||||||
|
DefaultSeriatimTimeout = "10m"
|
||||||
|
DefaultSeriatimOutputSchema = "seriatim-intermediate"
|
||||||
|
DefaultSeriatimCoalesceGap = 3.0
|
||||||
|
DefaultSeriatimReport = true
|
||||||
|
|
||||||
|
DefaultAuditaBinary = "audita"
|
||||||
|
DefaultAuditaTimeout = "3h"
|
||||||
|
DefaultAuditaReport = true
|
||||||
|
|
||||||
|
DefaultScriptoriumBinary = "scriptorium"
|
||||||
|
DefaultScriptoriumTimeout = "10m"
|
||||||
|
|
||||||
|
DefaultTrimBoundsTimeout = "10m"
|
||||||
|
DefaultTrimSeriatimReport = false
|
||||||
|
|
||||||
|
DefaultNormalizeOutputPath = "transcripts/normalized.json"
|
||||||
|
DefaultNormalizeOutputSchema = "seriatim-intermediate"
|
||||||
|
DefaultNormalizeReport = true
|
||||||
|
|
||||||
|
DefaultArchiveEnabled = true
|
||||||
|
DefaultArchiveUploadRun = true
|
||||||
|
|
||||||
|
PathWorkDirSegment = "work"
|
||||||
|
PathInputsDirSegment = "inputs"
|
||||||
|
PathAudioDirSegment = "audio"
|
||||||
|
PathTranscriptsSegment = "transcripts"
|
||||||
|
PathTranscriptsRaw = "transcripts/raw"
|
||||||
|
PathTranscriptsTrimmed = "transcripts/trimmed"
|
||||||
|
PathArtifactsDirSegment = "artifacts"
|
||||||
|
PathReportsDirSegment = "reports"
|
||||||
|
PathConfigDirSegment = "config"
|
||||||
|
PathLogsDirSegment = "logs"
|
||||||
|
PathCurrentDirSegment = "current"
|
||||||
|
PathRunsDirSegment = "runs"
|
||||||
|
PathManifestFile = "manifest.json"
|
||||||
|
PathLockFile = ".lock"
|
||||||
|
PathTranscriptMerged = "transcripts/merged.json"
|
||||||
|
PathTranscriptProcessed = "transcripts/processed.json"
|
||||||
|
PathTranscriptNormalized = "transcripts/normalized.json"
|
||||||
|
PathTranscriptTrimmed = "transcripts/trimmed.json"
|
||||||
|
|
||||||
|
S3CampaignsSegment = "campaigns"
|
||||||
|
S3SessionsSegment = "sessions"
|
||||||
|
S3RunsSegment = "runs"
|
||||||
|
S3CurrentSegment = "current"
|
||||||
|
S3ManifestFile = "manifest.json"
|
||||||
|
S3RunIDFile = "run_id.txt"
|
||||||
|
)
|
||||||
|
|
||||||
|
// DefaultArchivePromoteArtifacts defines the default archive promotion rules.
|
||||||
|
// Callers should copy this slice before mutating.
|
||||||
|
var DefaultArchivePromoteArtifacts = []ArchivePromotionRule{
|
||||||
|
{From: PathTranscriptTrimmed, To: PathTranscriptTrimmed},
|
||||||
|
{From: "artifacts/session_recap.md", To: "artifacts/session_recap.md"},
|
||||||
|
}
|
||||||
|
|
||||||
|
// DefaultPipelineConfigSearchPaths defines the default search order for
|
||||||
|
// pipeline.yml when callers do not provide an explicit path.
|
||||||
|
//
|
||||||
|
// Keep this in a variable so future defaults can be extended without changing
|
||||||
|
// call sites.
|
||||||
|
var DefaultPipelineConfigSearchPaths = []string{
|
||||||
|
DefaultPipelineConfigPathUsrLocal,
|
||||||
|
DefaultPipelineConfigPathEtc,
|
||||||
|
}
|
||||||
|
|
||||||
|
// DefaultSessionConfigSearchPaths defines the default search order for
|
||||||
|
// session.yml when callers do not provide an explicit path.
|
||||||
|
//
|
||||||
|
// Keep this in a variable so future defaults can be extended without changing
|
||||||
|
// call sites.
|
||||||
|
var DefaultSessionConfigSearchPaths = []string{
|
||||||
|
DefaultSessionConfigPathLocal,
|
||||||
|
DefaultSessionConfigPathUsrLocal,
|
||||||
|
DefaultSessionConfigPathEtc,
|
||||||
|
}
|
||||||
@@ -5,6 +5,8 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
|
||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v3"
|
||||||
)
|
)
|
||||||
@@ -21,21 +23,56 @@ func LoadPipeline(path string) (*PipelineConfig, error) {
|
|||||||
|
|
||||||
// LoadSession loads session configuration from a YAML file with strict field checking.
|
// LoadSession loads session configuration from a YAML file with strict field checking.
|
||||||
func LoadSession(path string) (*SessionConfig, error) {
|
func LoadSession(path string) (*SessionConfig, error) {
|
||||||
var cfg SessionConfig
|
return LoadSessionWithOptions(path, SessionLoadOptions{})
|
||||||
if err := decodeStrictYAML("session", path, &cfg); err != nil {
|
}
|
||||||
|
|
||||||
|
// SessionLoadOptions configures session template rendering behavior.
|
||||||
|
type SessionLoadOptions struct {
|
||||||
|
SessionID string
|
||||||
|
}
|
||||||
|
|
||||||
|
// LoadSessionWithOptions loads session configuration from a YAML file with
|
||||||
|
// strict field checking after template rendering.
|
||||||
|
func LoadSessionWithOptions(path string, opts SessionLoadOptions) (*SessionConfig, error) {
|
||||||
|
sessionBytes, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("load session config: session file %q: open: %w", path, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
rendered, err := renderSessionTemplate(string(sessionBytes), opts)
|
||||||
|
if err != nil {
|
||||||
return nil, fmt.Errorf("load session config: %w", err)
|
return nil, fmt.Errorf("load session config: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var cfg SessionConfig
|
||||||
|
if err := decodeStrictYAMLFromReader("session", path, strings.NewReader(rendered), &cfg); err != nil {
|
||||||
|
return nil, fmt.Errorf("load session config: %w", err)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(opts.SessionID) != "" && strings.TrimSpace(cfg.SessionID) != "" && strings.TrimSpace(cfg.SessionID) != strings.TrimSpace(opts.SessionID) {
|
||||||
|
return nil, fmt.Errorf(
|
||||||
|
"load session config: session file %q: session_id mismatch: --session-id %q does not match rendered session_id %q",
|
||||||
|
path,
|
||||||
|
strings.TrimSpace(opts.SessionID),
|
||||||
|
strings.TrimSpace(cfg.SessionID),
|
||||||
|
)
|
||||||
|
}
|
||||||
return &cfg, nil
|
return &cfg, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Load loads and resolves combined pipeline and session configuration.
|
// Load loads and resolves combined pipeline and session configuration.
|
||||||
func Load(pipelinePath, sessionPath string) (*Config, error) {
|
func Load(pipelinePath, sessionPath string) (*Config, error) {
|
||||||
|
return LoadWithSessionOptions(pipelinePath, sessionPath, SessionLoadOptions{})
|
||||||
|
}
|
||||||
|
|
||||||
|
// LoadWithSessionOptions loads and resolves combined pipeline and session
|
||||||
|
// configuration with session template options.
|
||||||
|
func LoadWithSessionOptions(pipelinePath, sessionPath string, sessionOpts SessionLoadOptions) (*Config, error) {
|
||||||
pipelineCfg, err := LoadPipeline(pipelinePath)
|
pipelineCfg, err := LoadPipeline(pipelinePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
sessionCfg, err := LoadSession(sessionPath)
|
sessionCfg, err := LoadSessionWithOptions(sessionPath, sessionOpts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -55,7 +92,11 @@ func decodeStrictYAML(kind, path string, out any) error {
|
|||||||
}
|
}
|
||||||
defer f.Close()
|
defer f.Close()
|
||||||
|
|
||||||
dec := yaml.NewDecoder(f)
|
return decodeStrictYAMLFromReader(kind, path, f, out)
|
||||||
|
}
|
||||||
|
|
||||||
|
func decodeStrictYAMLFromReader(kind, path string, r io.Reader, out any) error {
|
||||||
|
dec := yaml.NewDecoder(r)
|
||||||
dec.KnownFields(true)
|
dec.KnownFields(true)
|
||||||
if err := dec.Decode(out); err != nil {
|
if err := dec.Decode(out); err != nil {
|
||||||
return fmt.Errorf("%s file %q: strict decode failed: %w", kind, path, err)
|
return fmt.Errorf("%s file %q: strict decode failed: %w", kind, path, err)
|
||||||
@@ -69,6 +110,36 @@ func decodeStrictYAML(kind, path string, out any) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var sessionTemplatePattern = regexp.MustCompile(`\{\{\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\}\}`)
|
||||||
|
|
||||||
|
func renderSessionTemplate(content string, opts SessionLoadOptions) (string, error) {
|
||||||
|
sessionID := strings.TrimSpace(opts.SessionID)
|
||||||
|
rendered := content
|
||||||
|
if sessionID != "" {
|
||||||
|
rendered = strings.ReplaceAll(rendered, "{{session_id}}", sessionID)
|
||||||
|
rendered = strings.ReplaceAll(rendered, "{{ session_id }}", sessionID)
|
||||||
|
}
|
||||||
|
|
||||||
|
unresolved := sessionTemplatePattern.FindAllStringSubmatch(rendered, -1)
|
||||||
|
if len(unresolved) > 0 {
|
||||||
|
vars := make([]string, 0, len(unresolved))
|
||||||
|
for _, m := range unresolved {
|
||||||
|
if len(m) > 1 {
|
||||||
|
vars = append(vars, m[1])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(vars) > 0 {
|
||||||
|
return "", fmt.Errorf(
|
||||||
|
"session file template rendering failed: unresolved template variable(s): %s; pass --session-id when using {{ session_id }}",
|
||||||
|
strings.Join(vars, ", "),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
return "", fmt.Errorf("session file template rendering failed: unresolved template placeholders remain")
|
||||||
|
}
|
||||||
|
|
||||||
|
return rendered, nil
|
||||||
|
}
|
||||||
|
|
||||||
func shortName(path, fallback string) string {
|
func shortName(path, fallback string) string {
|
||||||
base := filepath.Base(path)
|
base := filepath.Base(path)
|
||||||
if base == "." || base == string(filepath.Separator) {
|
if base == "." || base == string(filepath.Separator) {
|
||||||
@@ -81,6 +152,9 @@ func applyPipelineDefaults(cfg *PipelineConfig) {
|
|||||||
if cfg == nil {
|
if cfg == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
applyStorageDefaults(&cfg.Storage)
|
||||||
|
applySpoolDefaults(&cfg.Spool)
|
||||||
|
applyArchiveDefaults(&cfg.Archive)
|
||||||
applyWhisperXDefaults(&cfg.WhisperX)
|
applyWhisperXDefaults(&cfg.WhisperX)
|
||||||
applySeriatimDefaults(&cfg.Seriatim)
|
applySeriatimDefaults(&cfg.Seriatim)
|
||||||
applyAuditaDefaults(&cfg.Audita)
|
applyAuditaDefaults(&cfg.Audita)
|
||||||
@@ -92,24 +166,75 @@ func applyPipelineDefaults(cfg *PipelineConfig) {
|
|||||||
applyScriptoriumDefaults(cfg.Scriptorium)
|
applyScriptoriumDefaults(cfg.Scriptorium)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func applyStorageDefaults(cfg *StorageConfig) {
|
||||||
|
if cfg == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if cfg.S3 == nil {
|
||||||
|
cfg.S3 = &StorageS3Config{}
|
||||||
|
}
|
||||||
|
if cfg.S3.RootPrefix == "" {
|
||||||
|
cfg.S3.RootPrefix = DefaultStorageS3RootPrefix
|
||||||
|
}
|
||||||
|
if cfg.S3.AccessKeyIDEnv == "" {
|
||||||
|
cfg.S3.AccessKeyIDEnv = DefaultS3AccessKeyIDEnv
|
||||||
|
}
|
||||||
|
if cfg.S3.SecretKeyEnv == "" {
|
||||||
|
cfg.S3.SecretKeyEnv = DefaultS3SecretAccessKeyEnv
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func applySpoolDefaults(cfg *SpoolConfig) {
|
||||||
|
if cfg == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if cfg.Root == "" {
|
||||||
|
cfg.Root = DefaultSpoolRoot
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func applyArchiveDefaults(cfg **ArchiveConfig) {
|
||||||
|
if cfg == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if *cfg == nil {
|
||||||
|
*cfg = &ArchiveConfig{}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (*cfg).Enabled == nil {
|
||||||
|
(*cfg).Enabled = boolPtr(DefaultArchiveEnabled)
|
||||||
|
}
|
||||||
|
if (*cfg).UploadRun == nil {
|
||||||
|
(*cfg).UploadRun = boolPtr(DefaultArchiveUploadRun)
|
||||||
|
}
|
||||||
|
if len((*cfg).PromoteArtifacts) == 0 {
|
||||||
|
(*cfg).PromoteArtifacts = append([]ArchivePromotionRule(nil), DefaultArchivePromoteArtifacts...)
|
||||||
|
}
|
||||||
|
for i := range (*cfg).PromoteArtifacts {
|
||||||
|
if (*cfg).PromoteArtifacts[i].Required == nil {
|
||||||
|
(*cfg).PromoteArtifacts[i].Required = boolPtr(true)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func applyWhisperXDefaults(cfg *WhisperXConfig) {
|
func applyWhisperXDefaults(cfg *WhisperXConfig) {
|
||||||
if cfg == nil {
|
if cfg == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if cfg.Language == "" {
|
if cfg.Language == "" {
|
||||||
cfg.Language = "en"
|
cfg.Language = DefaultWhisperXLanguage
|
||||||
}
|
}
|
||||||
if cfg.Timeout == "" {
|
if cfg.Timeout == "" {
|
||||||
cfg.Timeout = "30m"
|
cfg.Timeout = DefaultWhisperXTimeout
|
||||||
}
|
}
|
||||||
if cfg.RetryDelay == "" {
|
if cfg.RetryDelay == "" {
|
||||||
cfg.RetryDelay = "2s"
|
cfg.RetryDelay = DefaultWhisperXRetryDelay
|
||||||
}
|
}
|
||||||
if cfg.Concurrency == nil {
|
if cfg.Concurrency == nil {
|
||||||
cfg.Concurrency = intPtr(2)
|
cfg.Concurrency = intPtr(DefaultWhisperXConcurrency)
|
||||||
}
|
}
|
||||||
if cfg.Retries == nil {
|
if cfg.Retries == nil {
|
||||||
cfg.Retries = intPtr(3)
|
cfg.Retries = intPtr(DefaultWhisperXRetries)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -122,17 +247,20 @@ func applySeriatimDefaults(cfg *SeriatimConfig) {
|
|||||||
if cfg == nil {
|
if cfg == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if cfg.Binary == "" {
|
||||||
|
cfg.Binary = DefaultSeriatimBinary
|
||||||
|
}
|
||||||
if cfg.Timeout == "" {
|
if cfg.Timeout == "" {
|
||||||
cfg.Timeout = "10m"
|
cfg.Timeout = DefaultSeriatimTimeout
|
||||||
}
|
}
|
||||||
if cfg.OutputSchema == "" {
|
if cfg.OutputSchema == "" {
|
||||||
cfg.OutputSchema = "seriatim-intermediate"
|
cfg.OutputSchema = DefaultSeriatimOutputSchema
|
||||||
}
|
}
|
||||||
if cfg.CoalesceGap == nil {
|
if cfg.CoalesceGap == nil {
|
||||||
cfg.CoalesceGap = float64Ptr(3.0)
|
cfg.CoalesceGap = float64Ptr(DefaultSeriatimCoalesceGap)
|
||||||
}
|
}
|
||||||
if cfg.Report == nil {
|
if cfg.Report == nil {
|
||||||
cfg.Report = boolPtr(true)
|
cfg.Report = boolPtr(DefaultSeriatimReport)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -140,34 +268,14 @@ func applyAuditaDefaults(cfg *AuditaConfig) {
|
|||||||
if cfg == nil {
|
if cfg == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if cfg.Binary == "" {
|
||||||
|
cfg.Binary = DefaultAuditaBinary
|
||||||
|
}
|
||||||
if cfg.Timeout == "" {
|
if cfg.Timeout == "" {
|
||||||
cfg.Timeout = "3h"
|
cfg.Timeout = DefaultAuditaTimeout
|
||||||
}
|
|
||||||
if cfg.Modules == nil {
|
|
||||||
cfg.Modules = []string{
|
|
||||||
"glossary",
|
|
||||||
"homophones",
|
|
||||||
"glossary",
|
|
||||||
"spoken_word",
|
|
||||||
"grammar",
|
|
||||||
"homophones",
|
|
||||||
"glossary",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if cfg.BaseURL == "" {
|
|
||||||
cfg.BaseURL = "https://openrouter.ai/api/v1"
|
|
||||||
}
|
|
||||||
if cfg.Model == "" {
|
|
||||||
cfg.Model = "openrouter/google/gemma-4-31b-it"
|
|
||||||
}
|
|
||||||
if cfg.LLMConcurrency == nil {
|
|
||||||
cfg.LLMConcurrency = intPtr(1)
|
|
||||||
}
|
|
||||||
if cfg.ValidationLLMConcurrency == nil {
|
|
||||||
cfg.ValidationLLMConcurrency = intPtr(1)
|
|
||||||
}
|
}
|
||||||
if cfg.Report == nil {
|
if cfg.Report == nil {
|
||||||
cfg.Report = boolPtr(true)
|
cfg.Report = boolPtr(DefaultAuditaReport)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -175,8 +283,11 @@ func applyScriptoriumDefaults(cfg *ScriptoriumConfig) {
|
|||||||
if cfg == nil {
|
if cfg == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if cfg.Binary == "" {
|
||||||
|
cfg.Binary = DefaultScriptoriumBinary
|
||||||
|
}
|
||||||
if cfg.Timeout == "" {
|
if cfg.Timeout == "" {
|
||||||
cfg.Timeout = "10m"
|
cfg.Timeout = DefaultScriptoriumTimeout
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -185,10 +296,10 @@ func applyTrimDefaults(cfg *TrimConfig) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
if cfg.Bounds.Timeout == "" {
|
if cfg.Bounds.Timeout == "" {
|
||||||
cfg.Bounds.Timeout = "10m"
|
cfg.Bounds.Timeout = DefaultTrimBoundsTimeout
|
||||||
}
|
}
|
||||||
if cfg.Seriatim.Report == nil {
|
if cfg.Seriatim.Report == nil {
|
||||||
cfg.Seriatim.Report = boolPtr(false)
|
cfg.Seriatim.Report = boolPtr(DefaultTrimSeriatimReport)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -197,13 +308,13 @@ func applyNormalizeDefaults(cfg *NormalizeConfig) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
if cfg.OutputSchema == "" {
|
if cfg.OutputSchema == "" {
|
||||||
cfg.OutputSchema = defaultNormalizeOutputSchema
|
cfg.OutputSchema = DefaultNormalizeOutputSchema
|
||||||
}
|
}
|
||||||
if cfg.OutputPath == "" && !cfg.outputPathWasSet() {
|
if cfg.OutputPath == "" && !cfg.outputPathWasSet() {
|
||||||
cfg.OutputPath = defaultNormalizeOutputPath
|
cfg.OutputPath = DefaultNormalizeOutputPath
|
||||||
}
|
}
|
||||||
if cfg.Report == nil {
|
if cfg.Report == nil {
|
||||||
cfg.Report = boolPtr(true)
|
cfg.Report = boolPtr(DefaultNormalizeReport)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -37,6 +37,26 @@ inputs:
|
|||||||
speakers_file: ./speakers.yml
|
speakers_file: ./speakers.yml
|
||||||
autocorrect_file: ./autocorrect.yml
|
autocorrect_file: ./autocorrect.yml
|
||||||
glossary_file: ./glossary.yml
|
glossary_file: ./glossary.yml
|
||||||
|
`,
|
||||||
|
checkDefault: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "seriatim and audita sections can be omitted",
|
||||||
|
pipelineYAML: `workspace:
|
||||||
|
root: /tmp/narratio
|
||||||
|
whisperx:
|
||||||
|
transcribe_url: https://transcription.ai.rakestrawhome.com/transcribe
|
||||||
|
analyzer:
|
||||||
|
timeout: 20m
|
||||||
|
notification:
|
||||||
|
timeout: 15s
|
||||||
|
`,
|
||||||
|
sessionYAML: `session_id: 2026-05-03
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
`,
|
`,
|
||||||
checkDefault: true,
|
checkDefault: true,
|
||||||
},
|
},
|
||||||
@@ -72,6 +92,46 @@ inputs:
|
|||||||
`,
|
`,
|
||||||
wantLoadErr: "strict decode failed",
|
wantLoadErr: "strict decode failed",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "unknown secrets field fails",
|
||||||
|
pipelineYAML: `workspace:
|
||||||
|
root: /tmp/narratio
|
||||||
|
whisperx:
|
||||||
|
transcribe_url: https://transcription.ai.rakestrawhome.com/transcribe
|
||||||
|
seriatim:
|
||||||
|
binary: seriatim
|
||||||
|
secrets:
|
||||||
|
bogus: true
|
||||||
|
`,
|
||||||
|
sessionYAML: `session_id: 2026-05-03
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
`,
|
||||||
|
wantLoadErr: "strict decode failed",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty secrets env_dir fails",
|
||||||
|
pipelineYAML: `workspace:
|
||||||
|
root: /tmp/narratio
|
||||||
|
whisperx:
|
||||||
|
transcribe_url: https://transcription.ai.rakestrawhome.com/transcribe
|
||||||
|
seriatim:
|
||||||
|
binary: seriatim
|
||||||
|
secrets:
|
||||||
|
env_dir: " "
|
||||||
|
`,
|
||||||
|
sessionYAML: `session_id: 2026-05-03
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
`,
|
||||||
|
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.secrets.env_dir must be non-empty when pipeline.secrets is configured",
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: "unknown session field fails",
|
name: "unknown session field fails",
|
||||||
pipelineYAML: `workspace:
|
pipelineYAML: `workspace:
|
||||||
@@ -256,7 +316,7 @@ inputs:
|
|||||||
wantLoadErr: "strict decode failed",
|
wantLoadErr: "strict decode failed",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "missing seriatim binary fails",
|
name: "missing seriatim binary uses default",
|
||||||
pipelineYAML: `workspace:
|
pipelineYAML: `workspace:
|
||||||
root: /tmp/narratio
|
root: /tmp/narratio
|
||||||
whisperx:
|
whisperx:
|
||||||
@@ -271,7 +331,6 @@ inputs:
|
|||||||
autocorrect_file: ./autocorrect.yml
|
autocorrect_file: ./autocorrect.yml
|
||||||
glossary_file: ./glossary.yml
|
glossary_file: ./glossary.yml
|
||||||
`,
|
`,
|
||||||
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.seriatim.binary is required",
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "invalid seriatim timeout fails",
|
name: "invalid seriatim timeout fails",
|
||||||
@@ -372,7 +431,7 @@ inputs:
|
|||||||
wantLoadErr: "strict decode failed",
|
wantLoadErr: "strict decode failed",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "missing audita binary fails",
|
name: "missing audita binary uses default",
|
||||||
pipelineYAML: `workspace:
|
pipelineYAML: `workspace:
|
||||||
root: /tmp/narratio
|
root: /tmp/narratio
|
||||||
whisperx:
|
whisperx:
|
||||||
@@ -389,7 +448,6 @@ inputs:
|
|||||||
autocorrect_file: ./autocorrect.yml
|
autocorrect_file: ./autocorrect.yml
|
||||||
glossary_file: ./glossary.yml
|
glossary_file: ./glossary.yml
|
||||||
`,
|
`,
|
||||||
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.audita.binary is required",
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "invalid audita timeout fails",
|
name: "invalid audita timeout fails",
|
||||||
@@ -413,7 +471,7 @@ inputs:
|
|||||||
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.audita.timeout must be a valid duration",
|
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.audita.timeout must be a valid duration",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "empty audita modules fails",
|
name: "empty audita modules is valid override",
|
||||||
pipelineYAML: `workspace:
|
pipelineYAML: `workspace:
|
||||||
root: /tmp/narratio
|
root: /tmp/narratio
|
||||||
whisperx:
|
whisperx:
|
||||||
@@ -431,7 +489,6 @@ inputs:
|
|||||||
autocorrect_file: ./autocorrect.yml
|
autocorrect_file: ./autocorrect.yml
|
||||||
glossary_file: ./glossary.yml
|
glossary_file: ./glossary.yml
|
||||||
`,
|
`,
|
||||||
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.audita.modules must include at least one module",
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "empty audita module item fails",
|
name: "empty audita module item fails",
|
||||||
@@ -501,7 +558,7 @@ inputs:
|
|||||||
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.audita.base_url must be a valid URL",
|
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.audita.base_url must be a valid URL",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "invalid audita llm_concurrency fails",
|
name: "legacy audita llm_concurrency field fails strict decode",
|
||||||
pipelineYAML: `workspace:
|
pipelineYAML: `workspace:
|
||||||
root: /tmp/narratio
|
root: /tmp/narratio
|
||||||
whisperx:
|
whisperx:
|
||||||
@@ -510,7 +567,7 @@ seriatim:
|
|||||||
binary: seriatim
|
binary: seriatim
|
||||||
audita:
|
audita:
|
||||||
binary: audita
|
binary: audita
|
||||||
llm_concurrency: 0
|
llm_concurrency: 1
|
||||||
`,
|
`,
|
||||||
sessionYAML: `session_id: 2026-05-03
|
sessionYAML: `session_id: 2026-05-03
|
||||||
inputs:
|
inputs:
|
||||||
@@ -519,7 +576,49 @@ inputs:
|
|||||||
autocorrect_file: ./autocorrect.yml
|
autocorrect_file: ./autocorrect.yml
|
||||||
glossary_file: ./glossary.yml
|
glossary_file: ./glossary.yml
|
||||||
`,
|
`,
|
||||||
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.audita.llm_concurrency must be > 0",
|
wantLoadErr: "strict decode failed",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid audita total_llm_concurrency fails",
|
||||||
|
pipelineYAML: `workspace:
|
||||||
|
root: /tmp/narratio
|
||||||
|
whisperx:
|
||||||
|
transcribe_url: https://transcription.ai.rakestrawhome.com/transcribe
|
||||||
|
seriatim:
|
||||||
|
binary: seriatim
|
||||||
|
audita:
|
||||||
|
binary: audita
|
||||||
|
total_llm_concurrency: 0
|
||||||
|
`,
|
||||||
|
sessionYAML: `session_id: 2026-05-03
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
`,
|
||||||
|
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.audita.total_llm_concurrency must be > 0",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid audita proposal_llm_concurrency fails",
|
||||||
|
pipelineYAML: `workspace:
|
||||||
|
root: /tmp/narratio
|
||||||
|
whisperx:
|
||||||
|
transcribe_url: https://transcription.ai.rakestrawhome.com/transcribe
|
||||||
|
seriatim:
|
||||||
|
binary: seriatim
|
||||||
|
audita:
|
||||||
|
binary: audita
|
||||||
|
proposal_llm_concurrency: 0
|
||||||
|
`,
|
||||||
|
sessionYAML: `session_id: 2026-05-03
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
`,
|
||||||
|
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.audita.proposal_llm_concurrency must be > 0",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "invalid audita validation_llm_concurrency fails",
|
name: "invalid audita validation_llm_concurrency fails",
|
||||||
@@ -542,6 +641,48 @@ inputs:
|
|||||||
`,
|
`,
|
||||||
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.audita.validation_llm_concurrency must be > 0",
|
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.audita.validation_llm_concurrency must be > 0",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "invalid audita output_schema fails",
|
||||||
|
pipelineYAML: `workspace:
|
||||||
|
root: /tmp/narratio
|
||||||
|
whisperx:
|
||||||
|
transcribe_url: https://transcription.ai.rakestrawhome.com/transcribe
|
||||||
|
seriatim:
|
||||||
|
binary: seriatim
|
||||||
|
audita:
|
||||||
|
binary: audita
|
||||||
|
output_schema: bad
|
||||||
|
`,
|
||||||
|
sessionYAML: `session_id: 2026-05-03
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
`,
|
||||||
|
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.audita.output_schema must be one of: bare-segments, audita-v1",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid audita work_dir_retention fails",
|
||||||
|
pipelineYAML: `workspace:
|
||||||
|
root: /tmp/narratio
|
||||||
|
whisperx:
|
||||||
|
transcribe_url: https://transcription.ai.rakestrawhome.com/transcribe
|
||||||
|
seriatim:
|
||||||
|
binary: seriatim
|
||||||
|
audita:
|
||||||
|
binary: audita
|
||||||
|
work_dir_retention: sometimes
|
||||||
|
`,
|
||||||
|
sessionYAML: `session_id: 2026-05-03
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
`,
|
||||||
|
wantValidate: "pipeline config \"pipeline.yml\" invalid: pipeline.audita.work_dir_retention must be one of: always, auto, never",
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
@@ -590,6 +731,9 @@ inputs:
|
|||||||
if cfg.Pipeline.Seriatim.Timeout != "10m" {
|
if cfg.Pipeline.Seriatim.Timeout != "10m" {
|
||||||
t.Fatalf("seriatim.timeout = %q, want %q", cfg.Pipeline.Seriatim.Timeout, "10m")
|
t.Fatalf("seriatim.timeout = %q, want %q", cfg.Pipeline.Seriatim.Timeout, "10m")
|
||||||
}
|
}
|
||||||
|
if cfg.Pipeline.Seriatim.Binary != "seriatim" {
|
||||||
|
t.Fatalf("seriatim.binary = %q, want %q", cfg.Pipeline.Seriatim.Binary, "seriatim")
|
||||||
|
}
|
||||||
if cfg.Pipeline.Seriatim.OutputSchema != "seriatim-intermediate" {
|
if cfg.Pipeline.Seriatim.OutputSchema != "seriatim-intermediate" {
|
||||||
t.Fatalf("seriatim.output_schema = %q, want %q", cfg.Pipeline.Seriatim.OutputSchema, "seriatim-intermediate")
|
t.Fatalf("seriatim.output_schema = %q, want %q", cfg.Pipeline.Seriatim.OutputSchema, "seriatim-intermediate")
|
||||||
}
|
}
|
||||||
@@ -602,26 +746,32 @@ inputs:
|
|||||||
if cfg.Pipeline.Audita.Timeout != "3h" {
|
if cfg.Pipeline.Audita.Timeout != "3h" {
|
||||||
t.Fatalf("audita.timeout = %q, want %q", cfg.Pipeline.Audita.Timeout, "3h")
|
t.Fatalf("audita.timeout = %q, want %q", cfg.Pipeline.Audita.Timeout, "3h")
|
||||||
}
|
}
|
||||||
|
if cfg.Pipeline.Audita.Binary != "audita" {
|
||||||
|
t.Fatalf("audita.binary = %q, want %q", cfg.Pipeline.Audita.Binary, "audita")
|
||||||
|
}
|
||||||
if cfg.Pipeline.Audita.LLMAPIKeyEnv != "" {
|
if cfg.Pipeline.Audita.LLMAPIKeyEnv != "" {
|
||||||
t.Fatalf("audita.llm_api_key_env = %q, want empty by default", cfg.Pipeline.Audita.LLMAPIKeyEnv)
|
t.Fatalf("audita.llm_api_key_env = %q, want empty by default", cfg.Pipeline.Audita.LLMAPIKeyEnv)
|
||||||
}
|
}
|
||||||
if got := strings.Join(cfg.Pipeline.Audita.Modules, ","); got != "glossary,homophones,glossary,spoken_word,grammar,homophones,glossary" {
|
if cfg.Pipeline.Audita.Modules != nil {
|
||||||
t.Fatalf("audita.modules = %q, want default sequence", got)
|
t.Fatalf("audita.modules = %#v, want nil default (optional override)", cfg.Pipeline.Audita.Modules)
|
||||||
}
|
}
|
||||||
if cfg.Pipeline.Audita.BaseURL != "https://openrouter.ai/api/v1" {
|
if cfg.Pipeline.Audita.BaseURL != "" {
|
||||||
t.Fatalf("audita.base_url = %q, want %q", cfg.Pipeline.Audita.BaseURL, "https://openrouter.ai/api/v1")
|
t.Fatalf("audita.base_url = %q, want empty default", cfg.Pipeline.Audita.BaseURL)
|
||||||
}
|
}
|
||||||
if cfg.Pipeline.Audita.Model != "openrouter/google/gemma-4-31b-it" {
|
if cfg.Pipeline.Audita.Model != "" {
|
||||||
t.Fatalf("audita.model = %q, want %q", cfg.Pipeline.Audita.Model, "openrouter/google/gemma-4-31b-it")
|
t.Fatalf("audita.model = %q, want empty default", cfg.Pipeline.Audita.Model)
|
||||||
}
|
|
||||||
if cfg.Pipeline.Audita.LLMConcurrency == nil || *cfg.Pipeline.Audita.LLMConcurrency != 1 {
|
|
||||||
t.Fatalf("audita.llm_concurrency = %v, want 1", cfg.Pipeline.Audita.LLMConcurrency)
|
|
||||||
}
|
}
|
||||||
if cfg.Pipeline.Audita.ValidationModel != "" {
|
if cfg.Pipeline.Audita.ValidationModel != "" {
|
||||||
t.Fatalf("audita.validation_model = %q, want empty default", cfg.Pipeline.Audita.ValidationModel)
|
t.Fatalf("audita.validation_model = %q, want empty default", cfg.Pipeline.Audita.ValidationModel)
|
||||||
}
|
}
|
||||||
if cfg.Pipeline.Audita.ValidationLLMConcurrency == nil || *cfg.Pipeline.Audita.ValidationLLMConcurrency != 1 {
|
if cfg.Pipeline.Audita.TotalLLMConcurrency != nil {
|
||||||
t.Fatalf("audita.validation_llm_concurrency = %v, want 1", cfg.Pipeline.Audita.ValidationLLMConcurrency)
|
t.Fatalf("audita.total_llm_concurrency = %v, want nil default", cfg.Pipeline.Audita.TotalLLMConcurrency)
|
||||||
|
}
|
||||||
|
if cfg.Pipeline.Audita.ProposalLLMConcurrency != nil {
|
||||||
|
t.Fatalf("audita.proposal_llm_concurrency = %v, want nil default", cfg.Pipeline.Audita.ProposalLLMConcurrency)
|
||||||
|
}
|
||||||
|
if cfg.Pipeline.Audita.ValidationLLMConcurrency != nil {
|
||||||
|
t.Fatalf("audita.validation_llm_concurrency = %v, want nil default", cfg.Pipeline.Audita.ValidationLLMConcurrency)
|
||||||
}
|
}
|
||||||
if cfg.Pipeline.Audita.Report == nil || *cfg.Pipeline.Audita.Report != true {
|
if cfg.Pipeline.Audita.Report == nil || *cfg.Pipeline.Audita.Report != true {
|
||||||
t.Fatalf("audita.report = %v, want true", cfg.Pipeline.Audita.Report)
|
t.Fatalf("audita.report = %v, want true", cfg.Pipeline.Audita.Report)
|
||||||
@@ -685,7 +835,8 @@ func TestValidateMissingAudioSource(t *testing.T) {
|
|||||||
Modules: []string{"glossary", "homophones"},
|
Modules: []string{"glossary", "homophones"},
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
LLMConcurrency: intPtr(1),
|
TotalLLMConcurrency: intPtr(1),
|
||||||
|
ProposalLLMConcurrency: intPtr(1),
|
||||||
ValidationModel: "",
|
ValidationModel: "",
|
||||||
ValidationLLMConcurrency: intPtr(1),
|
ValidationLLMConcurrency: intPtr(1),
|
||||||
Report: boolPtr(true),
|
Report: boolPtr(true),
|
||||||
@@ -693,6 +844,7 @@ func TestValidateMissingAudioSource(t *testing.T) {
|
|||||||
},
|
},
|
||||||
Session: &SessionConfig{
|
Session: &SessionConfig{
|
||||||
SessionID: "2026-05-03",
|
SessionID: "2026-05-03",
|
||||||
|
Campaign: "sample-campaign",
|
||||||
Inputs: SessionInputsConfig{
|
Inputs: SessionInputsConfig{
|
||||||
SpeakersFile: "speakers.yml",
|
SpeakersFile: "speakers.yml",
|
||||||
AutocorrectFile: "autocorrect.yml",
|
AutocorrectFile: "autocorrect.yml",
|
||||||
@@ -705,7 +857,7 @@ func TestValidateMissingAudioSource(t *testing.T) {
|
|||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected validation error, got nil")
|
t.Fatal("expected validation error, got nil")
|
||||||
}
|
}
|
||||||
if !strings.Contains(err.Error(), "audio_dir or at least one audio_files") {
|
if !strings.Contains(err.Error(), "audio_dir, at least one audio_files entry, or audio_s3") {
|
||||||
t.Fatalf("error = %q, want audio source guidance", err.Error())
|
t.Fatalf("error = %q, want audio source guidance", err.Error())
|
||||||
}
|
}
|
||||||
if !strings.Contains(err.Error(), "session config") {
|
if !strings.Contains(err.Error(), "session config") {
|
||||||
@@ -734,6 +886,12 @@ func writeConfigFiles(t *testing.T, pipelineYAML, sessionYAML string) (string, s
|
|||||||
}
|
}
|
||||||
pipelineYAML += "audita:\n binary: audita\n"
|
pipelineYAML += "audita:\n binary: audita\n"
|
||||||
}
|
}
|
||||||
|
if !strings.Contains(sessionYAML, "\ncampaign:") && !strings.HasPrefix(sessionYAML, "campaign:") {
|
||||||
|
if !strings.HasSuffix(sessionYAML, "\n") {
|
||||||
|
sessionYAML += "\n"
|
||||||
|
}
|
||||||
|
sessionYAML += "campaign: sample-campaign\n"
|
||||||
|
}
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||||
|
|||||||
@@ -6,11 +6,6 @@ import (
|
|||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
|
||||||
defaultNormalizeOutputPath = "transcripts/normalized.json"
|
|
||||||
defaultNormalizeOutputSchema = "seriatim-intermediate"
|
|
||||||
)
|
|
||||||
|
|
||||||
// UnmarshalYAML tracks explicit normalize.output_path presence so validation can
|
// UnmarshalYAML tracks explicit normalize.output_path presence so validation can
|
||||||
// distinguish omitted vs explicitly empty values.
|
// distinguish omitted vs explicitly empty values.
|
||||||
func (cfg *NormalizeConfig) UnmarshalYAML(node *yaml.Node) error {
|
func (cfg *NormalizeConfig) UnmarshalYAML(node *yaml.Node) error {
|
||||||
|
|||||||
@@ -49,11 +49,19 @@ func TestScriptoriumLoadAndValidate(t *testing.T) {
|
|||||||
wantLoadErr: "strict decode failed",
|
wantLoadErr: "strict decode failed",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "missing binary fails when section present",
|
name: "missing binary defaults when section present",
|
||||||
scriptoriumYAML: `scriptorium:
|
scriptoriumYAML: `scriptorium:
|
||||||
timeout: 10m
|
timeout: 10m
|
||||||
`,
|
`,
|
||||||
wantValidateErr: "pipeline.scriptorium.binary is required",
|
assert: func(t *testing.T, cfg *Config) {
|
||||||
|
t.Helper()
|
||||||
|
if cfg.Pipeline.Scriptorium == nil {
|
||||||
|
t.Fatal("scriptorium config should be present")
|
||||||
|
}
|
||||||
|
if cfg.Pipeline.Scriptorium.Binary != "scriptorium" {
|
||||||
|
t.Fatalf("scriptorium.binary = %q, want scriptorium", cfg.Pipeline.Scriptorium.Binary)
|
||||||
|
}
|
||||||
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "enabled artifact missing prompt id fails",
|
name: "enabled artifact missing prompt id fails",
|
||||||
@@ -108,6 +116,37 @@ func TestScriptoriumLoadAndValidate(t *testing.T) {
|
|||||||
output_kind: session_recap
|
output_kind: session_recap
|
||||||
`,
|
`,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "canonical artifact source is accepted",
|
||||||
|
scriptoriumYAML: `scriptorium:
|
||||||
|
binary: scriptorium
|
||||||
|
artifacts:
|
||||||
|
session_recap:
|
||||||
|
enabled: true
|
||||||
|
prompt_id: dnd.session_recap
|
||||||
|
output_path: artifacts/session_recap.md
|
||||||
|
inputs:
|
||||||
|
transcript:
|
||||||
|
source: narratio.transcript.trimmed
|
||||||
|
required: true
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "unknown artifact source fails validation",
|
||||||
|
scriptoriumYAML: `scriptorium:
|
||||||
|
binary: scriptorium
|
||||||
|
artifacts:
|
||||||
|
session_recap:
|
||||||
|
enabled: true
|
||||||
|
prompt_id: dnd.session_recap
|
||||||
|
output_path: artifacts/session_recap.md
|
||||||
|
inputs:
|
||||||
|
transcript:
|
||||||
|
source: narratio.unknown
|
||||||
|
required: true
|
||||||
|
`,
|
||||||
|
wantValidateErr: `pipeline.scriptorium.artifacts.session_recap.inputs.transcript.source "narratio.unknown" is unsupported`,
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: "artifact render_debug override is accepted",
|
name: "artifact render_debug override is accepted",
|
||||||
scriptoriumYAML: `scriptorium:
|
scriptoriumYAML: `scriptorium:
|
||||||
|
|||||||
156
internal/config/session_template_test.go
Normal file
156
internal/config/session_template_test.go
Normal file
@@ -0,0 +1,156 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestLoadSessionWithOptionsRendersCompactPlaceholder(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
sessionPath := filepath.Join(dir, "session.yml")
|
||||||
|
sessionYAML := `session_id: "{{session_id}}"
|
||||||
|
campaign: sample-campaign
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write session.yml: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg, err := LoadSessionWithOptions(sessionPath, SessionLoadOptions{SessionID: "2026-04-04"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadSessionWithOptions() error = %v", err)
|
||||||
|
}
|
||||||
|
if cfg.SessionID != "2026-04-04" {
|
||||||
|
t.Fatalf("SessionID = %q, want 2026-04-04", cfg.SessionID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadSessionWithOptionsRendersSpacedPlaceholder(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
sessionPath := filepath.Join(dir, "session.yml")
|
||||||
|
sessionYAML := `session_id: "{{ session_id }}"
|
||||||
|
campaign: sample-campaign
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write session.yml: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg, err := LoadSessionWithOptions(sessionPath, SessionLoadOptions{SessionID: "2026-04-04"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadSessionWithOptions() error = %v", err)
|
||||||
|
}
|
||||||
|
if cfg.SessionID != "2026-04-04" {
|
||||||
|
t.Fatalf("SessionID = %q, want 2026-04-04", cfg.SessionID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadSessionWithOptionsUnresolvedPlaceholderFails(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
sessionPath := filepath.Join(dir, "session.yml")
|
||||||
|
sessionYAML := `session_id: "{{ session_id }}"
|
||||||
|
campaign: sample-campaign
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write session.yml: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := LoadSessionWithOptions(sessionPath, SessionLoadOptions{})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "unresolved template variable") {
|
||||||
|
t.Fatalf("error = %q, want unresolved-variable context", err.Error())
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "session_id") {
|
||||||
|
t.Fatalf("error = %q, want session_id variable", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadSessionWithOptionsMismatchFails(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
sessionPath := filepath.Join(dir, "session.yml")
|
||||||
|
sessionYAML := `session_id: 2026-05-03
|
||||||
|
campaign: sample-campaign
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write session.yml: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := LoadSessionWithOptions(sessionPath, SessionLoadOptions{SessionID: "2026-04-04"})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "session_id mismatch") {
|
||||||
|
t.Fatalf("error = %q, want mismatch context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadSessionWithOptionsUnknownFieldStillRejectedAfterRendering(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
sessionPath := filepath.Join(dir, "session.yml")
|
||||||
|
sessionYAML := `session_id: "{{ session_id }}"
|
||||||
|
campaign: sample-campaign
|
||||||
|
unknown_field: true
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write session.yml: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := LoadSessionWithOptions(sessionPath, SessionLoadOptions{SessionID: "2026-04-04"})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "strict decode failed") {
|
||||||
|
t.Fatalf("error = %q, want strict-decode context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadSessionWithOptionsConcreteSessionStillLoads(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
sessionPath := filepath.Join(dir, "session.yml")
|
||||||
|
sessionYAML := `session_id: 2026-05-03
|
||||||
|
campaign: sample-campaign
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(sessionPath, []byte(sessionYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write session.yml: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg, err := LoadSessionWithOptions(sessionPath, SessionLoadOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadSessionWithOptions() error = %v", err)
|
||||||
|
}
|
||||||
|
if cfg.SessionID != "2026-05-03" {
|
||||||
|
t.Fatalf("SessionID = %q, want 2026-05-03", cfg.SessionID)
|
||||||
|
}
|
||||||
|
}
|
||||||
315
internal/config/storage_archive_test.go
Normal file
315
internal/config/storage_archive_test.go
Normal file
@@ -0,0 +1,315 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestStorageS3DefaultsAndValidation(t *testing.T) {
|
||||||
|
pipelineYAML := testPipelineBaseYAML + `
|
||||||
|
storage:
|
||||||
|
backend: s3
|
||||||
|
s3:
|
||||||
|
bucket: my-dnd-archive
|
||||||
|
`
|
||||||
|
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
|
||||||
|
|
||||||
|
cfg, err := Load(pipelinePath, sessionPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() error = %v", err)
|
||||||
|
}
|
||||||
|
if cfg.Pipeline.Storage.S3 == nil {
|
||||||
|
t.Fatal("storage.s3 should be initialized")
|
||||||
|
}
|
||||||
|
if cfg.Pipeline.Storage.S3.RootPrefix != "dnd" {
|
||||||
|
t.Fatalf("storage.s3.root_prefix = %q, want dnd", cfg.Pipeline.Storage.S3.RootPrefix)
|
||||||
|
}
|
||||||
|
if cfg.Pipeline.Storage.S3.AccessKeyIDEnv != DefaultS3AccessKeyIDEnv {
|
||||||
|
t.Fatalf("storage.s3.access_key_id_env = %q, want %q", cfg.Pipeline.Storage.S3.AccessKeyIDEnv, DefaultS3AccessKeyIDEnv)
|
||||||
|
}
|
||||||
|
if cfg.Pipeline.Storage.S3.SecretKeyEnv != DefaultS3SecretAccessKeyEnv {
|
||||||
|
t.Fatalf("storage.s3.secret_access_key_env = %q, want %q", cfg.Pipeline.Storage.S3.SecretKeyEnv, DefaultS3SecretAccessKeyEnv)
|
||||||
|
}
|
||||||
|
if cfg.Pipeline.Storage.S3.ForcePathStyle {
|
||||||
|
t.Fatalf("storage.s3.force_path_style = true, want false default")
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := Validate(cfg); err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStorageS3CredentialEnvNamesLoadAndValidate(t *testing.T) {
|
||||||
|
pipelineYAML := testPipelineBaseYAML + `
|
||||||
|
storage:
|
||||||
|
backend: s3
|
||||||
|
s3:
|
||||||
|
bucket: my-dnd-archive
|
||||||
|
access_key_id_env: CUSTOM_KEY_ID
|
||||||
|
secret_access_key_env: CUSTOM_SECRET
|
||||||
|
`
|
||||||
|
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
|
||||||
|
|
||||||
|
cfg, err := Load(pipelinePath, sessionPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() error = %v", err)
|
||||||
|
}
|
||||||
|
if cfg.Pipeline.Storage.S3.AccessKeyIDEnv != "CUSTOM_KEY_ID" {
|
||||||
|
t.Fatalf("storage.s3.access_key_id_env = %q, want CUSTOM_KEY_ID", cfg.Pipeline.Storage.S3.AccessKeyIDEnv)
|
||||||
|
}
|
||||||
|
if cfg.Pipeline.Storage.S3.SecretKeyEnv != "CUSTOM_SECRET" {
|
||||||
|
t.Fatalf("storage.s3.secret_access_key_env = %q, want CUSTOM_SECRET", cfg.Pipeline.Storage.S3.SecretKeyEnv)
|
||||||
|
}
|
||||||
|
if err := Validate(cfg); err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStorageS3CredentialEnvValidation(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
pipelineYML string
|
||||||
|
wantErr string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "invalid access key env name",
|
||||||
|
pipelineYML: testPipelineBaseYAML + `
|
||||||
|
storage:
|
||||||
|
backend: s3
|
||||||
|
s3:
|
||||||
|
bucket: my-dnd-archive
|
||||||
|
access_key_id_env: "123BAD"
|
||||||
|
`,
|
||||||
|
wantErr: "pipeline.storage.s3.access_key_id_env must be a valid environment variable name",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid secret key env name",
|
||||||
|
pipelineYML: testPipelineBaseYAML + `
|
||||||
|
storage:
|
||||||
|
backend: s3
|
||||||
|
s3:
|
||||||
|
bucket: my-dnd-archive
|
||||||
|
secret_access_key_env: "bad-name"
|
||||||
|
`,
|
||||||
|
wantErr: "pipeline.storage.s3.secret_access_key_env must be a valid environment variable name",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
pipelinePath, sessionPath := writeConfigFiles(t, tt.pipelineYML, testSessionBaseYAML)
|
||||||
|
cfg, err := Load(pipelinePath, sessionPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() error = %v", err)
|
||||||
|
}
|
||||||
|
err = Validate(cfg)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
|
||||||
|
t.Fatalf("Validate() error = %v, want to contain %q", err, tt.wantErr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSpoolAndArchiveDefaults(t *testing.T) {
|
||||||
|
pipelinePath, sessionPath := writeConfigFiles(t, testPipelineBaseYAML, testSessionBaseYAML)
|
||||||
|
|
||||||
|
cfg, err := Load(pipelinePath, sessionPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if cfg.Pipeline.Spool.Root != "/var/spool/narratio" {
|
||||||
|
t.Fatalf("spool.root = %q, want /var/spool/narratio", cfg.Pipeline.Spool.Root)
|
||||||
|
}
|
||||||
|
if cfg.Pipeline.Spool.DeleteAudioAfterArchive {
|
||||||
|
t.Fatalf("spool.delete_audio_after_archive = true, want false")
|
||||||
|
}
|
||||||
|
if cfg.Pipeline.Workspace.CleanupAfterArchive {
|
||||||
|
t.Fatalf("workspace.cleanup_after_archive = true, want false")
|
||||||
|
}
|
||||||
|
if cfg.Pipeline.Archive == nil {
|
||||||
|
t.Fatal("archive should be initialized by defaults")
|
||||||
|
}
|
||||||
|
if cfg.Pipeline.Archive.Enabled == nil || !*cfg.Pipeline.Archive.Enabled {
|
||||||
|
t.Fatalf("archive.enabled = %#v, want true", cfg.Pipeline.Archive.Enabled)
|
||||||
|
}
|
||||||
|
if cfg.Pipeline.Archive.UploadRun == nil || !*cfg.Pipeline.Archive.UploadRun {
|
||||||
|
t.Fatalf("archive.upload_run = %#v, want true", cfg.Pipeline.Archive.UploadRun)
|
||||||
|
}
|
||||||
|
if len(cfg.Pipeline.Archive.PromoteArtifacts) != 2 {
|
||||||
|
t.Fatalf("archive.promote_artifacts len = %d, want 2 defaults", len(cfg.Pipeline.Archive.PromoteArtifacts))
|
||||||
|
}
|
||||||
|
for i, item := range cfg.Pipeline.Archive.PromoteArtifacts {
|
||||||
|
if item.Required == nil || !*item.Required {
|
||||||
|
t.Fatalf("archive.promote_artifacts[%d].required = %#v, want true", i, item.Required)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArchivePromotionPathValidation(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
ruleYML string
|
||||||
|
wantErr string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "absolute from path rejected",
|
||||||
|
ruleYML: `archive:
|
||||||
|
promote_artifacts:
|
||||||
|
- from: "/transcripts/trimmed.json"
|
||||||
|
to: "transcripts/trimmed.json"
|
||||||
|
`,
|
||||||
|
wantErr: "must be a relative path",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "traversal to path rejected",
|
||||||
|
ruleYML: `archive:
|
||||||
|
promote_artifacts:
|
||||||
|
- from: "transcripts/trimmed.json"
|
||||||
|
to: "../trimmed.json"
|
||||||
|
`,
|
||||||
|
wantErr: "must not contain path traversal",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
pipelineYAML := testPipelineBaseYAML + "\n" + tt.ruleYML
|
||||||
|
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
|
||||||
|
|
||||||
|
cfg, err := Load(pipelinePath, sessionPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() error = %v", err)
|
||||||
|
}
|
||||||
|
err = Validate(cfg)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
|
||||||
|
t.Fatalf("Validate() error = %v, want to contain %q", err, tt.wantErr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSessionAudioS3Validation(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
sessionYAML string
|
||||||
|
wantErr string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "valid audio_s3 prefix",
|
||||||
|
sessionYAML: `session_id: 2026-05-03
|
||||||
|
campaign: forsaken
|
||||||
|
inputs:
|
||||||
|
audio_s3:
|
||||||
|
prefix: audio/
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid audio_s3 absolute prefix",
|
||||||
|
sessionYAML: `session_id: 2026-05-03
|
||||||
|
campaign: forsaken
|
||||||
|
inputs:
|
||||||
|
audio_s3:
|
||||||
|
prefix: /audio/
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
`,
|
||||||
|
wantErr: "session.inputs.audio_s3.prefix must be a relative path",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid audio_s3 traversal prefix",
|
||||||
|
sessionYAML: `session_id: 2026-05-03
|
||||||
|
campaign: forsaken
|
||||||
|
inputs:
|
||||||
|
audio_s3:
|
||||||
|
prefix: ../audio/
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
`,
|
||||||
|
wantErr: "session.inputs.audio_s3.prefix must not contain path traversal",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "local and s3 audio conflict",
|
||||||
|
sessionYAML: `session_id: 2026-05-03
|
||||||
|
campaign: forsaken
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
|
audio_s3:
|
||||||
|
prefix: audio/
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
`,
|
||||||
|
wantErr: "mutually exclusive",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
pipelineYAML := testPipelineBaseYAML + `
|
||||||
|
storage:
|
||||||
|
backend: s3
|
||||||
|
s3:
|
||||||
|
bucket: my-dnd-archive
|
||||||
|
`
|
||||||
|
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, tt.sessionYAML)
|
||||||
|
|
||||||
|
cfg, err := Load(pipelinePath, sessionPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() error = %v", err)
|
||||||
|
}
|
||||||
|
err = Validate(cfg)
|
||||||
|
if tt.wantErr != "" {
|
||||||
|
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
|
||||||
|
t.Fatalf("Validate() error = %v, want to contain %q", err, tt.wantErr)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStorageS3BucketRequiredWhenS3DependentFeatureEnabled(t *testing.T) {
|
||||||
|
pipelineYAML := testPipelineBaseYAML + `
|
||||||
|
storage:
|
||||||
|
backend: s3
|
||||||
|
`
|
||||||
|
sessionYAML := `session_id: 2026-05-03
|
||||||
|
campaign: forsaken
|
||||||
|
inputs:
|
||||||
|
audio_s3:
|
||||||
|
prefix: audio/
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
`
|
||||||
|
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, sessionYAML)
|
||||||
|
cfg, err := Load(pipelinePath, sessionPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = Validate(cfg)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "pipeline.storage.s3.bucket is required") {
|
||||||
|
t.Fatalf("Validate() error = %v, want bucket requirement", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLocalAudioConfigStillValid(t *testing.T) {
|
||||||
|
pipelinePath, sessionPath := writeConfigFiles(t, testPipelineBaseYAML, testSessionBaseYAML)
|
||||||
|
cfg, err := Load(pipelinePath, sessionPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := Validate(cfg); err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -3,6 +3,8 @@ package config
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/url"
|
"net/url"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
@@ -25,6 +27,9 @@ func Validate(cfg *Config) error {
|
|||||||
if err := validateSession(cfg.Session); err != nil {
|
if err := validateSession(cfg.Session); err != nil {
|
||||||
return fmt.Errorf("session config %q invalid: %w", shortName(cfg.SessionPath, "session.yml"), err)
|
return fmt.Errorf("session config %q invalid: %w", shortName(cfg.SessionPath, "session.yml"), err)
|
||||||
}
|
}
|
||||||
|
if err := validateCrossConfig(cfg.Pipeline, cfg.Session); err != nil {
|
||||||
|
return fmt.Errorf("pipeline/session config invalid: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -33,6 +38,18 @@ func validatePipeline(cfg *PipelineConfig) error {
|
|||||||
if strings.TrimSpace(cfg.Workspace.Root) == "" {
|
if strings.TrimSpace(cfg.Workspace.Root) == "" {
|
||||||
return fmt.Errorf("pipeline.workspace.root is required")
|
return fmt.Errorf("pipeline.workspace.root is required")
|
||||||
}
|
}
|
||||||
|
if err := validateSecrets(cfg.Secrets); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := validateStorage(cfg.Storage); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := validateSpool(cfg.Spool); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := validateArchive(cfg.Archive); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
if err := validateWhisperX(cfg.WhisperX); err != nil {
|
if err := validateWhisperX(cfg.WhisperX); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -61,6 +78,64 @@ func validatePipeline(cfg *PipelineConfig) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validateStorage(cfg StorageConfig) error {
|
||||||
|
if cfg.S3 == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(cfg.S3.RootPrefix) == "" {
|
||||||
|
return fmt.Errorf("pipeline.storage.s3.root_prefix must be non-empty")
|
||||||
|
}
|
||||||
|
if err := validateRelativeSafePath("pipeline.storage.s3.root_prefix", cfg.S3.RootPrefix); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if cfg.S3.Endpoint != "" && strings.TrimSpace(cfg.S3.Endpoint) == "" {
|
||||||
|
return fmt.Errorf("pipeline.storage.s3.endpoint must be non-empty when provided")
|
||||||
|
}
|
||||||
|
if err := validateEnvVarNameField("pipeline.storage.s3.access_key_id_env", cfg.S3.AccessKeyIDEnv); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := validateEnvVarNameField("pipeline.storage.s3.secret_access_key_env", cfg.S3.SecretKeyEnv); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateSpool(cfg SpoolConfig) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateArchive(cfg *ArchiveConfig) error {
|
||||||
|
if cfg == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for i, item := range cfg.PromoteArtifacts {
|
||||||
|
prefix := fmt.Sprintf("pipeline.archive.promote_artifacts[%d]", i)
|
||||||
|
if strings.TrimSpace(item.From) == "" {
|
||||||
|
return fmt.Errorf("%s.from is required", prefix)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(item.To) == "" {
|
||||||
|
return fmt.Errorf("%s.to is required", prefix)
|
||||||
|
}
|
||||||
|
if err := validateRelativeSafePath(prefix+".from", item.From); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := validateRelativeSafePath(prefix+".to", item.To); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateSecrets(cfg *SecretsConfig) error {
|
||||||
|
if cfg == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(cfg.EnvDir) == "" {
|
||||||
|
return fmt.Errorf("pipeline.secrets.env_dir must be non-empty when pipeline.secrets is configured")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func validateNormalize(cfg *NormalizeConfig) error {
|
func validateNormalize(cfg *NormalizeConfig) error {
|
||||||
if cfg == nil {
|
if cfg == nil {
|
||||||
return nil
|
return nil
|
||||||
@@ -187,15 +262,12 @@ func validateAudita(cfg AuditaConfig) error {
|
|||||||
if err := validateDuration("pipeline.audita.timeout", cfg.Timeout); err != nil {
|
if err := validateDuration("pipeline.audita.timeout", cfg.Timeout); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if len(cfg.Modules) == 0 {
|
for i, m := range cfg.Modules {
|
||||||
return fmt.Errorf("pipeline.audita.modules must include at least one module")
|
module := strings.TrimSpace(m)
|
||||||
}
|
if module == "" {
|
||||||
for i, mod := range cfg.Modules {
|
|
||||||
m := strings.TrimSpace(mod)
|
|
||||||
if m == "" {
|
|
||||||
return fmt.Errorf("pipeline.audita.modules[%d] must be non-empty", i)
|
return fmt.Errorf("pipeline.audita.modules[%d] must be non-empty", i)
|
||||||
}
|
}
|
||||||
switch m {
|
switch module {
|
||||||
case "glossary", "homophones", "spoken_word", "grammar":
|
case "glossary", "homophones", "spoken_word", "grammar":
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("pipeline.audita.modules[%d] must be one of: glossary, homophones, spoken_word, grammar", i)
|
return fmt.Errorf("pipeline.audita.modules[%d] must be one of: glossary, homophones, spoken_word, grammar", i)
|
||||||
@@ -210,21 +282,31 @@ func validateAudita(cfg AuditaConfig) error {
|
|||||||
return fmt.Errorf("pipeline.audita.base_url must be a valid URL")
|
return fmt.Errorf("pipeline.audita.base_url must be a valid URL")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(cfg.Model) == "" {
|
if cfg.TotalLLMConcurrency != nil && *cfg.TotalLLMConcurrency <= 0 {
|
||||||
return fmt.Errorf("pipeline.audita.model is required")
|
return fmt.Errorf("pipeline.audita.total_llm_concurrency must be > 0")
|
||||||
}
|
}
|
||||||
if cfg.LLMConcurrency == nil {
|
if cfg.ProposalLLMConcurrency != nil && *cfg.ProposalLLMConcurrency <= 0 {
|
||||||
return fmt.Errorf("pipeline.audita.llm_concurrency must be set (defaults should populate this)")
|
return fmt.Errorf("pipeline.audita.proposal_llm_concurrency must be > 0")
|
||||||
}
|
}
|
||||||
if *cfg.LLMConcurrency <= 0 {
|
if cfg.ValidationLLMConcurrency != nil && *cfg.ValidationLLMConcurrency <= 0 {
|
||||||
return fmt.Errorf("pipeline.audita.llm_concurrency must be > 0")
|
|
||||||
}
|
|
||||||
if cfg.ValidationLLMConcurrency == nil {
|
|
||||||
return fmt.Errorf("pipeline.audita.validation_llm_concurrency must be set (defaults should populate this)")
|
|
||||||
}
|
|
||||||
if *cfg.ValidationLLMConcurrency <= 0 {
|
|
||||||
return fmt.Errorf("pipeline.audita.validation_llm_concurrency must be > 0")
|
return fmt.Errorf("pipeline.audita.validation_llm_concurrency must be > 0")
|
||||||
}
|
}
|
||||||
|
if strings.TrimSpace(cfg.TranscriptDescription) == "" && cfg.TranscriptDescription != "" {
|
||||||
|
return fmt.Errorf("pipeline.audita.transcript_description must be non-empty when provided")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(cfg.ConfigPath) == "" && cfg.ConfigPath != "" {
|
||||||
|
return fmt.Errorf("pipeline.audita.config_path must be non-empty when provided")
|
||||||
|
}
|
||||||
|
switch strings.TrimSpace(cfg.OutputSchema) {
|
||||||
|
case "", "bare-segments", "audita-v1":
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("pipeline.audita.output_schema must be one of: bare-segments, audita-v1")
|
||||||
|
}
|
||||||
|
switch strings.TrimSpace(cfg.WorkDirRetention) {
|
||||||
|
case "", "always", "auto", "never":
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("pipeline.audita.work_dir_retention must be one of: always, auto, never")
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -261,9 +343,13 @@ func validateScriptorium(cfg *ScriptoriumConfig) error {
|
|||||||
if trimmedInputName == "" {
|
if trimmedInputName == "" {
|
||||||
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.inputs keys must be non-empty", artifactName)
|
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.inputs keys must be non-empty", artifactName)
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(inputCfg.Source) == "" {
|
source := strings.TrimSpace(inputCfg.Source)
|
||||||
|
if source == "" {
|
||||||
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.inputs.%s.source is required", artifactName, inputName)
|
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.inputs.%s.source is required", artifactName, inputName)
|
||||||
}
|
}
|
||||||
|
if !isSupportedScriptoriumInputSource(source) {
|
||||||
|
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.inputs.%s.source %q is unsupported", artifactName, inputName, inputCfg.Source)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
for varName, varValue := range artifactCfg.Vars {
|
for varName, varValue := range artifactCfg.Vars {
|
||||||
if strings.TrimSpace(varName) == "" {
|
if strings.TrimSpace(varName) == "" {
|
||||||
@@ -284,6 +370,9 @@ func validateSession(cfg *SessionConfig) error {
|
|||||||
if strings.TrimSpace(cfg.SessionID) == "" {
|
if strings.TrimSpace(cfg.SessionID) == "" {
|
||||||
return fmt.Errorf("session.session_id is required")
|
return fmt.Errorf("session.session_id is required")
|
||||||
}
|
}
|
||||||
|
if strings.TrimSpace(cfg.Campaign) == "" {
|
||||||
|
return fmt.Errorf("session.campaign is required")
|
||||||
|
}
|
||||||
|
|
||||||
if strings.TrimSpace(cfg.Inputs.SpeakersFile) == "" {
|
if strings.TrimSpace(cfg.Inputs.SpeakersFile) == "" {
|
||||||
return fmt.Errorf("session.inputs.speakers_file is required")
|
return fmt.Errorf("session.inputs.speakers_file is required")
|
||||||
@@ -297,13 +386,118 @@ func validateSession(cfg *SessionConfig) error {
|
|||||||
|
|
||||||
hasAudioDir := strings.TrimSpace(cfg.Inputs.AudioDir) != ""
|
hasAudioDir := strings.TrimSpace(cfg.Inputs.AudioDir) != ""
|
||||||
hasAudioFiles := len(cfg.Inputs.AudioFiles) > 0
|
hasAudioFiles := len(cfg.Inputs.AudioFiles) > 0
|
||||||
if !hasAudioDir && !hasAudioFiles {
|
hasAudioS3 := cfg.Inputs.AudioS3 != nil
|
||||||
return fmt.Errorf("session.inputs requires audio_dir or at least one audio_files entry")
|
if hasAudioS3 {
|
||||||
|
if strings.TrimSpace(cfg.Inputs.AudioS3.Prefix) == "" {
|
||||||
|
return fmt.Errorf("session.inputs.audio_s3.prefix is required when session.inputs.audio_s3 is configured")
|
||||||
|
}
|
||||||
|
if err := validateRelativeSafePath("session.inputs.audio_s3.prefix", cfg.Inputs.AudioS3.Prefix); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if hasAudioS3 && (hasAudioDir || hasAudioFiles) {
|
||||||
|
return fmt.Errorf("session.inputs.audio_dir/audio_files and session.inputs.audio_s3 are mutually exclusive")
|
||||||
|
}
|
||||||
|
if !hasAudioDir && !hasAudioFiles && !hasAudioS3 {
|
||||||
|
return fmt.Errorf("session.inputs requires audio_dir, at least one audio_files entry, or audio_s3")
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validateCrossConfig(pipeline *PipelineConfig, session *SessionConfig) error {
|
||||||
|
if pipeline == nil || session == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if pipeline.Storage.S3 == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
audioS3Enabled := session.Inputs.AudioS3 != nil
|
||||||
|
archiveUploadEnabled := archiveUploadConfiguredForS3(pipeline)
|
||||||
|
if (audioS3Enabled || archiveUploadEnabled) && strings.TrimSpace(pipeline.Storage.S3.Bucket) == "" {
|
||||||
|
return fmt.Errorf("pipeline.storage.s3.bucket is required when S3 session audio or archive upload is enabled")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func archiveUploadConfiguredForS3(pipeline *PipelineConfig) bool {
|
||||||
|
if pipeline == nil || pipeline.Archive == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if !strings.EqualFold(strings.TrimSpace(pipeline.Storage.Backend), "s3") {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
enabled := true
|
||||||
|
if pipeline.Archive.Enabled != nil {
|
||||||
|
enabled = *pipeline.Archive.Enabled
|
||||||
|
}
|
||||||
|
upload := true
|
||||||
|
if pipeline.Archive.UploadRun != nil {
|
||||||
|
upload = *pipeline.Archive.UploadRun
|
||||||
|
}
|
||||||
|
return enabled && upload
|
||||||
|
}
|
||||||
|
|
||||||
|
func isSupportedScriptoriumInputSource(source string) bool {
|
||||||
|
switch strings.TrimSpace(source) {
|
||||||
|
case "previous_session_artifact":
|
||||||
|
return true
|
||||||
|
case "processed_transcript":
|
||||||
|
return true
|
||||||
|
case "normalized_transcript":
|
||||||
|
return true
|
||||||
|
case "trimmed_transcript":
|
||||||
|
return true
|
||||||
|
case "narratio.transcript.merged":
|
||||||
|
return true
|
||||||
|
case "narratio.transcript.polished":
|
||||||
|
return true
|
||||||
|
case "narratio.transcript.full":
|
||||||
|
return true
|
||||||
|
case "narratio.transcript.trimmed":
|
||||||
|
return true
|
||||||
|
case "narratio.bounds.session":
|
||||||
|
return true
|
||||||
|
case "narratio.artifact.session_recap":
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var windowsAbsPathRE = regexp.MustCompile(`^[A-Za-z]:[\\/].*`)
|
||||||
|
var envVarNameRE = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||||
|
|
||||||
|
func validateEnvVarNameField(fieldName, value string) error {
|
||||||
|
trimmed := strings.TrimSpace(value)
|
||||||
|
if trimmed == "" {
|
||||||
|
return fmt.Errorf("%s must be non-empty", fieldName)
|
||||||
|
}
|
||||||
|
if !envVarNameRE.MatchString(trimmed) {
|
||||||
|
return fmt.Errorf("%s must be a valid environment variable name", fieldName)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateRelativeSafePath(fieldName, value string) error {
|
||||||
|
trimmed := strings.TrimSpace(value)
|
||||||
|
if trimmed == "" {
|
||||||
|
return fmt.Errorf("%s must be non-empty", fieldName)
|
||||||
|
}
|
||||||
|
if filepath.IsAbs(trimmed) || strings.HasPrefix(trimmed, "/") || strings.HasPrefix(trimmed, "\\") || windowsAbsPathRE.MatchString(trimmed) {
|
||||||
|
return fmt.Errorf("%s must be a relative path", fieldName)
|
||||||
|
}
|
||||||
|
|
||||||
|
normalized := strings.ReplaceAll(trimmed, "\\", "/")
|
||||||
|
for _, segment := range strings.Split(normalized, "/") {
|
||||||
|
if segment == ".." {
|
||||||
|
return fmt.Errorf("%s must not contain path traversal", fieldName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func validateDuration(fieldName, value string) error {
|
func validateDuration(fieldName, value string) error {
|
||||||
trimmed := strings.TrimSpace(value)
|
trimmed := strings.TrimSpace(value)
|
||||||
if trimmed == "" {
|
if trimmed == "" {
|
||||||
|
|||||||
@@ -14,17 +14,25 @@ type ErrorRecord struct {
|
|||||||
|
|
||||||
// InputRecord captures one resolved input and optional checksum.
|
// InputRecord captures one resolved input and optional checksum.
|
||||||
type InputRecord struct {
|
type InputRecord struct {
|
||||||
Kind string `json:"kind"`
|
Kind string `json:"kind"`
|
||||||
Path string `json:"path"`
|
Path string `json:"path"`
|
||||||
Checksum string `json:"checksum,omitempty"`
|
Checksum string `json:"checksum,omitempty"`
|
||||||
|
Source string `json:"source,omitempty"`
|
||||||
|
S3Bucket string `json:"s3_bucket,omitempty"`
|
||||||
|
S3Key string `json:"s3_key,omitempty"`
|
||||||
|
S3Size int64 `json:"s3_size,omitempty"`
|
||||||
|
S3ETag string `json:"s3_etag,omitempty"`
|
||||||
|
SpoolPath string `json:"spool_path,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ArtifactRecord captures one produced artifact and optional remote metadata.
|
// ArtifactRecord captures one produced artifact and optional remote metadata.
|
||||||
type ArtifactRecord struct {
|
type ArtifactRecord struct {
|
||||||
Kind string `json:"kind"`
|
Kind string `json:"kind"`
|
||||||
LocalPath string `json:"local_path"`
|
LocalPath string `json:"local_path"`
|
||||||
RemoteKey string `json:"remote_key,omitempty"`
|
// ProducerRunID identifies the run that produced this durable artifact.
|
||||||
Checksum string `json:"checksum,omitempty"`
|
ProducerRunID string `json:"producer_run_id,omitempty"`
|
||||||
|
RemoteKey string `json:"remote_key,omitempty"`
|
||||||
|
Checksum string `json:"checksum,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// StageRecord tracks lifecycle and provenance for one pipeline stage.
|
// StageRecord tracks lifecycle and provenance for one pipeline stage.
|
||||||
@@ -45,6 +53,13 @@ type StageRecord struct {
|
|||||||
// Manifest is the durable run-state record for a session execution.
|
// Manifest is the durable run-state record for a session execution.
|
||||||
type Manifest struct {
|
type Manifest struct {
|
||||||
SessionID string `json:"session_id"`
|
SessionID string `json:"session_id"`
|
||||||
|
Campaign string `json:"campaign,omitempty"`
|
||||||
|
RunID string `json:"run_id,omitempty"`
|
||||||
|
LocalWorkDir string `json:"local_workdir,omitempty"`
|
||||||
|
LocalSpoolDir string `json:"local_spool_dir,omitempty"`
|
||||||
|
S3Bucket string `json:"s3_bucket,omitempty"`
|
||||||
|
S3SessionPrefix string `json:"s3_session_prefix,omitempty"`
|
||||||
|
S3RunPrefix string `json:"s3_run_prefix,omitempty"`
|
||||||
PipelineVersion string `json:"pipeline_version,omitempty"`
|
PipelineVersion string `json:"pipeline_version,omitempty"`
|
||||||
CreatedAt time.Time `json:"created_at"`
|
CreatedAt time.Time `json:"created_at"`
|
||||||
UpdatedAt time.Time `json:"updated_at"`
|
UpdatedAt time.Time `json:"updated_at"`
|
||||||
@@ -107,6 +122,15 @@ func (m *Manifest) MarkStageSkipped(name string, at time.Time, reason string) {
|
|||||||
m.UpdatedAt = at
|
m.UpdatedAt = at
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// MarkStageStale marks a stage as stale so it is not skipped as idempotently complete.
|
||||||
|
func (m *Manifest) MarkStageStale(name string, at time.Time, reason string) {
|
||||||
|
s := m.ensureStage(name, at)
|
||||||
|
s.Status = StatusStale
|
||||||
|
s.Error = &ErrorRecord{Message: strings.TrimSpace(reason), Code: "stale", At: timePtr(at)}
|
||||||
|
s.UpdatedAt = at
|
||||||
|
m.UpdatedAt = at
|
||||||
|
}
|
||||||
|
|
||||||
func (m *Manifest) ensureStage(name string, at time.Time) *StageRecord {
|
func (m *Manifest) ensureStage(name string, at time.Time) *StageRecord {
|
||||||
if m.Stages == nil {
|
if m.Stages == nil {
|
||||||
m.Stages = map[string]*StageRecord{}
|
m.Stages = map[string]*StageRecord{}
|
||||||
|
|||||||
164
internal/manifest/run_manifest.go
Normal file
164
internal/manifest/run_manifest.go
Normal file
@@ -0,0 +1,164 @@
|
|||||||
|
package manifest
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type RunManifestStatus string
|
||||||
|
|
||||||
|
const (
|
||||||
|
RunManifestStatusRunning RunManifestStatus = "running"
|
||||||
|
RunManifestStatusSucceeded RunManifestStatus = "succeeded"
|
||||||
|
RunManifestStatusFailed RunManifestStatus = "failed"
|
||||||
|
)
|
||||||
|
|
||||||
|
type RunStageAction string
|
||||||
|
|
||||||
|
const (
|
||||||
|
RunStageActionRun RunStageAction = "run"
|
||||||
|
RunStageActionSkip RunStageAction = "skip"
|
||||||
|
)
|
||||||
|
|
||||||
|
// RunStageRecord tracks lifecycle and provenance for one stage within a single invocation.
|
||||||
|
type RunStageRecord struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Action RunStageAction `json:"action"`
|
||||||
|
Status StageStatus `json:"status"`
|
||||||
|
CreatedAt time.Time `json:"created_at"`
|
||||||
|
UpdatedAt time.Time `json:"updated_at"`
|
||||||
|
StartedAt *time.Time `json:"started_at,omitempty"`
|
||||||
|
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
||||||
|
Outputs []ArtifactRecord `json:"outputs,omitempty"`
|
||||||
|
Logs []string `json:"logs,omitempty"`
|
||||||
|
GeneratedConfigs []string `json:"generated_configs,omitempty"`
|
||||||
|
Error *ErrorRecord `json:"error,omitempty"`
|
||||||
|
Metadata map[string]any `json:"metadata,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// RunManifest is the invocation-scoped execution record under runs/{run_id}/manifest.json.
|
||||||
|
type RunManifest struct {
|
||||||
|
SessionID string `json:"session_id"`
|
||||||
|
Campaign string `json:"campaign,omitempty"`
|
||||||
|
RunID string `json:"run_id"`
|
||||||
|
Force bool `json:"force"`
|
||||||
|
RequestedStages []string `json:"requested_stages,omitempty"`
|
||||||
|
SessionManifestPath string `json:"session_manifest_path,omitempty"`
|
||||||
|
LocalWorkDir string `json:"local_workdir,omitempty"`
|
||||||
|
LocalSpoolDir string `json:"local_spool_dir,omitempty"`
|
||||||
|
S3Bucket string `json:"s3_bucket,omitempty"`
|
||||||
|
S3SessionPrefix string `json:"s3_session_prefix,omitempty"`
|
||||||
|
S3RunPrefix string `json:"s3_run_prefix,omitempty"`
|
||||||
|
CreatedAt time.Time `json:"created_at"`
|
||||||
|
UpdatedAt time.Time `json:"updated_at"`
|
||||||
|
StartedAt *time.Time `json:"started_at,omitempty"`
|
||||||
|
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
||||||
|
Status RunManifestStatus `json:"status"`
|
||||||
|
LastError *ErrorRecord `json:"last_error,omitempty"`
|
||||||
|
Stages map[string]*RunStageRecord `json:"stages"`
|
||||||
|
Metadata map[string]any `json:"metadata,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewRun constructs a new run manifest with deterministic timestamps.
|
||||||
|
func NewRun(sessionID, campaign, runID string, force bool, requestedStages []string, now time.Time) *RunManifest {
|
||||||
|
return &RunManifest{
|
||||||
|
SessionID: strings.TrimSpace(sessionID),
|
||||||
|
Campaign: strings.TrimSpace(campaign),
|
||||||
|
RunID: strings.TrimSpace(runID),
|
||||||
|
Force: force,
|
||||||
|
RequestedStages: append([]string(nil), requestedStages...),
|
||||||
|
CreatedAt: now,
|
||||||
|
UpdatedAt: now,
|
||||||
|
StartedAt: timePtr(now),
|
||||||
|
Status: RunManifestStatusRunning,
|
||||||
|
Stages: map[string]*RunStageRecord{},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *RunManifest) SetStageAction(name string, action RunStageAction, at time.Time) {
|
||||||
|
s := m.ensureStage(name, at)
|
||||||
|
s.Action = action
|
||||||
|
s.UpdatedAt = at
|
||||||
|
m.UpdatedAt = at
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *RunManifest) MarkStageRunning(name string, at time.Time) {
|
||||||
|
s := m.ensureStage(name, at)
|
||||||
|
s.Status = StatusRunning
|
||||||
|
s.StartedAt = timePtr(at)
|
||||||
|
s.CompletedAt = nil
|
||||||
|
s.Error = nil
|
||||||
|
s.UpdatedAt = at
|
||||||
|
m.UpdatedAt = at
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *RunManifest) MarkStageSucceeded(name string, at time.Time, outputs []ArtifactRecord) {
|
||||||
|
s := m.ensureStage(name, at)
|
||||||
|
s.Status = StatusSucceeded
|
||||||
|
s.CompletedAt = timePtr(at)
|
||||||
|
s.Error = nil
|
||||||
|
s.Outputs = append([]ArtifactRecord(nil), outputs...)
|
||||||
|
s.UpdatedAt = at
|
||||||
|
m.UpdatedAt = at
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *RunManifest) MarkStageFailed(name string, at time.Time, message string) {
|
||||||
|
s := m.ensureStage(name, at)
|
||||||
|
s.Status = StatusFailed
|
||||||
|
s.CompletedAt = timePtr(at)
|
||||||
|
s.Error = &ErrorRecord{Message: strings.TrimSpace(message), At: timePtr(at)}
|
||||||
|
s.UpdatedAt = at
|
||||||
|
m.LastError = &ErrorRecord{Message: strings.TrimSpace(message), At: timePtr(at)}
|
||||||
|
m.UpdatedAt = at
|
||||||
|
m.Status = RunManifestStatusFailed
|
||||||
|
m.CompletedAt = timePtr(at)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *RunManifest) MarkStageSkipped(name string, at time.Time, reason string) {
|
||||||
|
s := m.ensureStage(name, at)
|
||||||
|
s.Status = StatusSkipped
|
||||||
|
s.CompletedAt = timePtr(at)
|
||||||
|
s.Error = &ErrorRecord{Message: strings.TrimSpace(reason), Code: "skipped", At: timePtr(at)}
|
||||||
|
s.UpdatedAt = at
|
||||||
|
m.UpdatedAt = at
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *RunManifest) MarkSucceeded(at time.Time) {
|
||||||
|
m.Status = RunManifestStatusSucceeded
|
||||||
|
m.CompletedAt = timePtr(at)
|
||||||
|
m.UpdatedAt = at
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *RunManifest) MarkFailed(at time.Time, message string) {
|
||||||
|
m.Status = RunManifestStatusFailed
|
||||||
|
m.CompletedAt = timePtr(at)
|
||||||
|
m.LastError = &ErrorRecord{Message: strings.TrimSpace(message), At: timePtr(at)}
|
||||||
|
m.UpdatedAt = at
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *RunManifest) ensureStage(name string, at time.Time) *RunStageRecord {
|
||||||
|
if m.Stages == nil {
|
||||||
|
m.Stages = map[string]*RunStageRecord{}
|
||||||
|
}
|
||||||
|
|
||||||
|
stageName := strings.TrimSpace(name)
|
||||||
|
s, ok := m.Stages[stageName]
|
||||||
|
if !ok || s == nil {
|
||||||
|
s = &RunStageRecord{
|
||||||
|
Name: stageName,
|
||||||
|
Action: RunStageActionRun,
|
||||||
|
Status: StatusPending,
|
||||||
|
CreatedAt: at,
|
||||||
|
UpdatedAt: at,
|
||||||
|
}
|
||||||
|
m.Stages[stageName] = s
|
||||||
|
}
|
||||||
|
if s.Name == "" {
|
||||||
|
s.Name = stageName
|
||||||
|
}
|
||||||
|
if s.CreatedAt.IsZero() {
|
||||||
|
s.CreatedAt = at
|
||||||
|
}
|
||||||
|
|
||||||
|
return s
|
||||||
|
}
|
||||||
50
internal/manifest/run_manifest_test.go
Normal file
50
internal/manifest/run_manifest_test.go
Normal file
@@ -0,0 +1,50 @@
|
|||||||
|
package manifest
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestRunManifestStageMarkHelpers(t *testing.T) {
|
||||||
|
rm := NewRun("2026-05-03", "forsaken", "20260517T000000Z-abcdef12", false, []string{"prepare"}, time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||||
|
if rm.Status != RunManifestStatusRunning {
|
||||||
|
t.Fatalf("status = %q, want %q", rm.Status, RunManifestStatusRunning)
|
||||||
|
}
|
||||||
|
|
||||||
|
runningAt := time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC)
|
||||||
|
rm.SetStageAction("prepare", RunStageActionRun, runningAt)
|
||||||
|
rm.MarkStageRunning("prepare", runningAt)
|
||||||
|
rm.MarkStageSucceeded("prepare", runningAt.Add(30*time.Second), []ArtifactRecord{
|
||||||
|
{Kind: "input", LocalPath: "inputs/session.yml"},
|
||||||
|
})
|
||||||
|
|
||||||
|
stage := rm.Stages["prepare"]
|
||||||
|
if stage == nil {
|
||||||
|
t.Fatal("prepare stage missing")
|
||||||
|
}
|
||||||
|
if stage.Action != RunStageActionRun {
|
||||||
|
t.Fatalf("action = %q, want %q", stage.Action, RunStageActionRun)
|
||||||
|
}
|
||||||
|
if stage.Status != StatusSucceeded {
|
||||||
|
t.Fatalf("status = %q, want %q", stage.Status, StatusSucceeded)
|
||||||
|
}
|
||||||
|
|
||||||
|
skippedAt := runningAt.Add(1 * time.Minute)
|
||||||
|
rm.SetStageAction("notify", RunStageActionSkip, skippedAt)
|
||||||
|
rm.MarkStageSkipped("notify", skippedAt, "already_succeeded")
|
||||||
|
skipped := rm.Stages["notify"]
|
||||||
|
if skipped == nil {
|
||||||
|
t.Fatal("notify stage missing")
|
||||||
|
}
|
||||||
|
if skipped.Action != RunStageActionSkip {
|
||||||
|
t.Fatalf("action = %q, want %q", skipped.Action, RunStageActionSkip)
|
||||||
|
}
|
||||||
|
if skipped.Status != StatusSkipped {
|
||||||
|
t.Fatalf("status = %q, want %q", skipped.Status, StatusSkipped)
|
||||||
|
}
|
||||||
|
|
||||||
|
rm.MarkSucceeded(skippedAt.Add(10 * time.Second))
|
||||||
|
if rm.Status != RunManifestStatusSucceeded {
|
||||||
|
t.Fatalf("status = %q, want %q", rm.Status, RunManifestStatusSucceeded)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -129,6 +129,89 @@ func (s *LocalStore) Save(ctx context.Context, path string, m *Manifest) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// CreateRun returns a new in-memory run manifest for one invocation.
|
||||||
|
func (s *LocalStore) CreateRun(
|
||||||
|
ctx context.Context,
|
||||||
|
sessionID, campaign, runID string,
|
||||||
|
force bool,
|
||||||
|
requestedStages []string,
|
||||||
|
) (*RunManifest, error) {
|
||||||
|
if err := checkContext(ctx); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(sessionID) == "" {
|
||||||
|
return nil, fmt.Errorf("create run manifest: session_id is required")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(runID) == "" {
|
||||||
|
return nil, fmt.Errorf("create run manifest: run_id is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
now := time.Now().UTC()
|
||||||
|
return NewRun(sessionID, campaign, runID, force, requestedStages, now), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// LoadRun reads and validates a local JSON run manifest from path.
|
||||||
|
func (s *LocalStore) LoadRun(ctx context.Context, path string) (*RunManifest, error) {
|
||||||
|
if err := checkContext(ctx); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(path) == "" {
|
||||||
|
return nil, fmt.Errorf("load run manifest: path is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("load run manifest %q: %w", path, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var m RunManifest
|
||||||
|
if err := json.Unmarshal(data, &m); err != nil {
|
||||||
|
return nil, fmt.Errorf("decode run manifest %q: %w", path, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := validateLoadedRunManifest(&m); err != nil {
|
||||||
|
return nil, fmt.Errorf("run manifest %q invalid: %w", path, err)
|
||||||
|
}
|
||||||
|
normalizeRunManifest(&m)
|
||||||
|
|
||||||
|
return &m, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SaveRun writes the run manifest to path atomically via temp file + rename.
|
||||||
|
func (s *LocalStore) SaveRun(ctx context.Context, path string, m *RunManifest) error {
|
||||||
|
if err := checkContext(ctx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(path) == "" {
|
||||||
|
return fmt.Errorf("save run manifest: path is required")
|
||||||
|
}
|
||||||
|
if m == nil {
|
||||||
|
return fmt.Errorf("save run manifest: manifest is nil")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(m.SessionID) == "" {
|
||||||
|
return fmt.Errorf("save run manifest: session_id is required")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(m.RunID) == "" {
|
||||||
|
return fmt.Errorf("save run manifest: run_id is required")
|
||||||
|
}
|
||||||
|
if m.CreatedAt.IsZero() {
|
||||||
|
return fmt.Errorf("save run manifest: created_at is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
m.UpdatedAt = time.Now().UTC()
|
||||||
|
if m.Stages == nil {
|
||||||
|
m.Stages = map[string]*RunStageRecord{}
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := json.MarshalIndent(m, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("save run manifest: marshal: %w", err)
|
||||||
|
}
|
||||||
|
data = append(data, '\n')
|
||||||
|
|
||||||
|
return writeJSONAtomically(ctx, path, ".run-manifest.json.tmp-*", data)
|
||||||
|
}
|
||||||
|
|
||||||
func validateLoadedManifest(m *Manifest) error {
|
func validateLoadedManifest(m *Manifest) error {
|
||||||
if m == nil {
|
if m == nil {
|
||||||
return fmt.Errorf("manifest is nil")
|
return fmt.Errorf("manifest is nil")
|
||||||
@@ -161,6 +244,88 @@ func normalizeManifest(m *Manifest) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validateLoadedRunManifest(m *RunManifest) error {
|
||||||
|
if m == nil {
|
||||||
|
return fmt.Errorf("manifest is nil")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(m.SessionID) == "" {
|
||||||
|
return fmt.Errorf("session_id is required")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(m.RunID) == "" {
|
||||||
|
return fmt.Errorf("run_id is required")
|
||||||
|
}
|
||||||
|
if m.CreatedAt.IsZero() {
|
||||||
|
return fmt.Errorf("created_at is required")
|
||||||
|
}
|
||||||
|
if m.UpdatedAt.IsZero() {
|
||||||
|
return fmt.Errorf("updated_at is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeRunManifest(m *RunManifest) {
|
||||||
|
if m.Stages == nil {
|
||||||
|
m.Stages = map[string]*RunStageRecord{}
|
||||||
|
}
|
||||||
|
for name, stage := range m.Stages {
|
||||||
|
if stage == nil {
|
||||||
|
stage = &RunStageRecord{
|
||||||
|
Name: name,
|
||||||
|
Action: RunStageActionRun,
|
||||||
|
Status: StatusPending,
|
||||||
|
CreatedAt: m.CreatedAt,
|
||||||
|
UpdatedAt: m.UpdatedAt,
|
||||||
|
}
|
||||||
|
m.Stages[name] = stage
|
||||||
|
}
|
||||||
|
if stage.Name == "" {
|
||||||
|
stage.Name = name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeJSONAtomically(ctx context.Context, path, tempPattern string, data []byte) error {
|
||||||
|
dir := filepath.Dir(path)
|
||||||
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||||
|
return fmt.Errorf("create directory %q: %w", dir, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
tmp, err := os.CreateTemp(dir, tempPattern)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("create temp file: %w", err)
|
||||||
|
}
|
||||||
|
tmpName := tmp.Name()
|
||||||
|
removeTmp := true
|
||||||
|
defer func() {
|
||||||
|
if removeTmp {
|
||||||
|
_ = os.Remove(tmpName)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
if _, err := tmp.Write(data); err != nil {
|
||||||
|
_ = tmp.Close()
|
||||||
|
return fmt.Errorf("write temp file: %w", err)
|
||||||
|
}
|
||||||
|
if err := tmp.Sync(); err != nil {
|
||||||
|
_ = tmp.Close()
|
||||||
|
return fmt.Errorf("sync temp file: %w", err)
|
||||||
|
}
|
||||||
|
if err := tmp.Close(); err != nil {
|
||||||
|
return fmt.Errorf("close temp file: %w", err)
|
||||||
|
}
|
||||||
|
if err := checkContext(ctx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.Rename(tmpName, path); err != nil {
|
||||||
|
return fmt.Errorf("rename temp file: %w", err)
|
||||||
|
}
|
||||||
|
removeTmp = false
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func checkContext(ctx context.Context) error {
|
func checkContext(ctx context.Context) error {
|
||||||
if ctx == nil {
|
if ctx == nil {
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -22,6 +22,13 @@ func TestLocalStoreCreateSaveLoadRoundTrip(t *testing.T) {
|
|||||||
now := time.Date(2026, 5, 3, 12, 0, 0, 0, time.UTC)
|
now := time.Date(2026, 5, 3, 12, 0, 0, 0, time.UTC)
|
||||||
m.MarkStageRunning("prepare", now)
|
m.MarkStageRunning("prepare", now)
|
||||||
m.MarkStageSucceeded("prepare", now.Add(2*time.Second), []ArtifactRecord{{Kind: "transcript", LocalPath: "transcripts/merged.json"}})
|
m.MarkStageSucceeded("prepare", now.Add(2*time.Second), []ArtifactRecord{{Kind: "transcript", LocalPath: "transcripts/merged.json"}})
|
||||||
|
m.Campaign = "forsaken"
|
||||||
|
m.RunID = "20260515T031522Z-a1b2c3d4"
|
||||||
|
m.LocalWorkDir = "/var/lib/narratio/work/forsaken/2026-05-03/20260515T031522Z-a1b2c3d4"
|
||||||
|
m.LocalSpoolDir = "/var/spool/narratio/forsaken/2026-05-03/20260515T031522Z-a1b2c3d4/audio"
|
||||||
|
m.S3Bucket = "my-dnd-archive"
|
||||||
|
m.S3SessionPrefix = "dnd/campaigns/forsaken/sessions/2026-05-03/"
|
||||||
|
m.S3RunPrefix = "dnd/campaigns/forsaken/sessions/2026-05-03/runs/20260515T031522Z-a1b2c3d4/"
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "manifest.json")
|
path := filepath.Join(t.TempDir(), "manifest.json")
|
||||||
if err := store.Save(ctx, path, m); err != nil {
|
if err := store.Save(ctx, path, m); err != nil {
|
||||||
@@ -36,6 +43,12 @@ func TestLocalStoreCreateSaveLoadRoundTrip(t *testing.T) {
|
|||||||
if loaded.SessionID != "2026-05-03" {
|
if loaded.SessionID != "2026-05-03" {
|
||||||
t.Fatalf("SessionID = %q, want %q", loaded.SessionID, "2026-05-03")
|
t.Fatalf("SessionID = %q, want %q", loaded.SessionID, "2026-05-03")
|
||||||
}
|
}
|
||||||
|
if loaded.Campaign != "forsaken" {
|
||||||
|
t.Fatalf("Campaign = %q, want %q", loaded.Campaign, "forsaken")
|
||||||
|
}
|
||||||
|
if loaded.RunID != "20260515T031522Z-a1b2c3d4" {
|
||||||
|
t.Fatalf("RunID = %q, want run id", loaded.RunID)
|
||||||
|
}
|
||||||
stage, ok := loaded.Stages["prepare"]
|
stage, ok := loaded.Stages["prepare"]
|
||||||
if !ok {
|
if !ok {
|
||||||
t.Fatalf("stage prepare not found")
|
t.Fatalf("stage prepare not found")
|
||||||
@@ -137,3 +150,73 @@ func TestLoadRejectsInvalidManifest(t *testing.T) {
|
|||||||
t.Fatalf("error = %q, want session_id validation", err.Error())
|
t.Fatalf("error = %q, want session_id validation", err.Error())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLocalStoreCreateSaveLoadRunManifestRoundTrip(t *testing.T) {
|
||||||
|
store := &LocalStore{}
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
run, err := store.CreateRun(
|
||||||
|
ctx,
|
||||||
|
"2026-05-03",
|
||||||
|
"forsaken",
|
||||||
|
"20260517T000000Z-abcdef12",
|
||||||
|
true,
|
||||||
|
[]string{"prepare", "transcribe"},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("CreateRun() error = %v", err)
|
||||||
|
}
|
||||||
|
run.SessionManifestPath = "/var/lib/narratio/work/forsaken/2026-05-03/manifest.json"
|
||||||
|
run.MarkStageRunning("prepare", time.Date(2026, 5, 3, 12, 1, 0, 0, time.UTC))
|
||||||
|
run.MarkStageSucceeded("prepare", time.Date(2026, 5, 3, 12, 2, 0, 0, time.UTC), []ArtifactRecord{
|
||||||
|
{Kind: "input", LocalPath: "inputs/session.yml"},
|
||||||
|
})
|
||||||
|
run.MarkSucceeded(time.Date(2026, 5, 3, 12, 3, 0, 0, time.UTC))
|
||||||
|
|
||||||
|
path := filepath.Join(t.TempDir(), "run-manifest.json")
|
||||||
|
if err := store.SaveRun(ctx, path, run); err != nil {
|
||||||
|
t.Fatalf("SaveRun() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
loaded, err := store.LoadRun(ctx, path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadRun() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if loaded.SessionID != "2026-05-03" {
|
||||||
|
t.Fatalf("SessionID = %q, want %q", loaded.SessionID, "2026-05-03")
|
||||||
|
}
|
||||||
|
if loaded.Campaign != "forsaken" {
|
||||||
|
t.Fatalf("Campaign = %q, want %q", loaded.Campaign, "forsaken")
|
||||||
|
}
|
||||||
|
if loaded.RunID != "20260517T000000Z-abcdef12" {
|
||||||
|
t.Fatalf("RunID = %q, want %q", loaded.RunID, "20260517T000000Z-abcdef12")
|
||||||
|
}
|
||||||
|
if loaded.Status != RunManifestStatusSucceeded {
|
||||||
|
t.Fatalf("Status = %q, want %q", loaded.Status, RunManifestStatusSucceeded)
|
||||||
|
}
|
||||||
|
if loaded.Stages["prepare"] == nil || loaded.Stages["prepare"].Status != StatusSucceeded {
|
||||||
|
t.Fatalf("prepare stage = %#v, want succeeded", loaded.Stages["prepare"])
|
||||||
|
}
|
||||||
|
if loaded.Stages["prepare"].Action != RunStageActionRun {
|
||||||
|
t.Fatalf("prepare action = %q, want %q", loaded.Stages["prepare"].Action, RunStageActionRun)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadRunRejectsInvalidManifest(t *testing.T) {
|
||||||
|
store := &LocalStore{}
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
path := filepath.Join(t.TempDir(), "run-manifest.json")
|
||||||
|
if err := os.WriteFile(path, []byte(`{"session_id":"2026-05-03"}`), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := store.LoadRun(ctx, path)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "run_id is required") {
|
||||||
|
t.Fatalf("error = %q, want run_id validation", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package stage
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -58,7 +59,11 @@ func (analyzeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
|||||||
return nil, fmt.Errorf("analyze: session id is required")
|
return nil, fmt.Errorf("analyze: session id is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
paths := env.ArtifactStore.SessionPaths(sessionID)
|
paths := sessionPathsForEnv(env, sessionID)
|
||||||
|
runLayout, err := resolveRunStageLayout(env, m, paths, sessionID, "analyze")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("analyze: resolve run-stage layout: %w", err)
|
||||||
|
}
|
||||||
if env.Config.Pipeline.Scriptorium == nil {
|
if env.Config.Pipeline.Scriptorium == nil {
|
||||||
return &StageResult{
|
return &StageResult{
|
||||||
Metadata: map[string]any{
|
Metadata: map[string]any{
|
||||||
@@ -83,27 +88,7 @@ func (analyzeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
processedTranscriptPath, processedSource, err := discoverProcessedTranscript(m, paths)
|
transcriptRefs := discoverAnalyzeTranscriptRefs(m, paths)
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("analyze: resolve processed transcript: %w", err)
|
|
||||||
}
|
|
||||||
normalizedTranscriptPath, normalizedSource, err := discoverNormalizedTranscript(m, paths)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("analyze: resolve normalized transcript: %w", err)
|
|
||||||
}
|
|
||||||
trimmedTranscriptPath, trimmedSource, err := discoverTrimmedTranscript(m, paths)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("analyze: resolve trimmed transcript: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
transcriptInputs := analyzeTranscriptInputs{
|
|
||||||
ProcessedPath: processedTranscriptPath,
|
|
||||||
ProcessedSource: processedSource,
|
|
||||||
NormalizedPath: normalizedTranscriptPath,
|
|
||||||
NormalizedSource: normalizedSource,
|
|
||||||
TrimmedPath: trimmedTranscriptPath,
|
|
||||||
TrimmedSource: trimmedSource,
|
|
||||||
}
|
|
||||||
|
|
||||||
inputPaths := map[string]string{}
|
inputPaths := map[string]string{}
|
||||||
omittedOptionalInputs := []string{}
|
omittedOptionalInputs := []string{}
|
||||||
@@ -111,7 +96,7 @@ func (analyzeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
|||||||
inputNames := sortedScriptoriumInputNames(artifactCfg.Inputs)
|
inputNames := sortedScriptoriumInputNames(artifactCfg.Inputs)
|
||||||
for _, inputName := range inputNames {
|
for _, inputName := range inputNames {
|
||||||
inputCfg := artifactCfg.Inputs[inputName]
|
inputCfg := artifactCfg.Inputs[inputName]
|
||||||
resolvedPath, resolved, resolveErr := resolveScriptoriumInput(inputName, inputCfg, transcriptInputs, paths, sessionDir)
|
resolvedPath, resolved, resolveErr := resolveScriptoriumInput(inputName, inputCfg, m, paths, sessionDir)
|
||||||
if resolveErr != nil {
|
if resolveErr != nil {
|
||||||
return nil, fmt.Errorf("analyze: resolve input %q: %w", inputName, resolveErr)
|
return nil, fmt.Errorf("analyze: resolve input %q: %w", inputName, resolveErr)
|
||||||
}
|
}
|
||||||
@@ -130,13 +115,22 @@ func (analyzeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
|||||||
return nil, fmt.Errorf("analyze: resolve vars: %w", err)
|
return nil, fmt.Errorf("analyze: resolve vars: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
outputPath, err := resolveScriptoriumOutputPath(paths, artifactCfg.OutputPath)
|
canonicalOutputPath, err := resolveScriptoriumOutputPath(paths, artifactCfg.OutputPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("analyze: resolve output path: %w", err)
|
return nil, fmt.Errorf("analyze: resolve output path: %w", err)
|
||||||
}
|
}
|
||||||
|
outputPath, err := runLocalPathForCanonical(runLayout, paths, canonicalOutputPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("analyze: resolve run-local output path: %w", err)
|
||||||
|
}
|
||||||
stdoutLogPath := filepath.Join(paths.LogsDir, "scriptorium."+artifactName+".stdout.log")
|
stdoutLogPath := filepath.Join(paths.LogsDir, "scriptorium."+artifactName+".stdout.log")
|
||||||
stderrLogPath := filepath.Join(paths.LogsDir, "scriptorium."+artifactName+".stderr.log")
|
stderrLogPath := filepath.Join(paths.LogsDir, "scriptorium."+artifactName+".stderr.log")
|
||||||
generatedConfigPath := filepath.Join(paths.ConfigDir, "scriptorium."+artifactName+".generated.yml")
|
generatedConfigPath := filepath.Join(paths.ConfigDir, "scriptorium."+artifactName+".generated.yml")
|
||||||
|
if runLayout.Enabled {
|
||||||
|
stdoutLogPath = filepath.Join(runLayout.LogsDir, "scriptorium."+artifactName+".stdout.log")
|
||||||
|
stderrLogPath = filepath.Join(runLayout.LogsDir, "scriptorium."+artifactName+".stderr.log")
|
||||||
|
generatedConfigPath = filepath.Join(runLayout.ConfigDir, "scriptorium."+artifactName+".generated.yml")
|
||||||
|
}
|
||||||
|
|
||||||
timeout, err := resolveScriptoriumTimeout(env.Config.Pipeline.Scriptorium.Timeout, artifactCfg.Timeout)
|
timeout, err := resolveScriptoriumTimeout(env.Config.Pipeline.Scriptorium.Timeout, artifactCfg.Timeout)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -157,20 +151,28 @@ func (analyzeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
|||||||
"omitted_optional_inputs": omittedOptionalInputs,
|
"omitted_optional_inputs": omittedOptionalInputs,
|
||||||
"vars": vars,
|
"vars": vars,
|
||||||
"timeout": timeout.String(),
|
"timeout": timeout.String(),
|
||||||
"processed_transcript_path": processedTranscriptPath,
|
"processed_transcript_path": transcriptRefs.ProcessedPath,
|
||||||
"processed_transcript_source": processedSource,
|
"processed_transcript_source": transcriptRefs.ProcessedSource,
|
||||||
"normalized_transcript_path": normalizedTranscriptPath,
|
"normalized_transcript_path": transcriptRefs.NormalizedPath,
|
||||||
"normalized_transcript_source": normalizedSource,
|
"normalized_transcript_source": transcriptRefs.NormalizedSource,
|
||||||
"trimmed_transcript_path": trimmedTranscriptPath,
|
"trimmed_transcript_path": transcriptRefs.TrimmedPath,
|
||||||
"trimmed_transcript_source": trimmedSource,
|
"trimmed_transcript_source": transcriptRefs.TrimmedSource,
|
||||||
"render_debug_enabled": resolveRenderDebugEnabled(env.Config.Pipeline.Scriptorium.RenderDebug, artifactCfg.RenderDebug),
|
"render_debug_enabled": resolveRenderDebugEnabled(env.Config.Pipeline.Scriptorium.RenderDebug, artifactCfg.RenderDebug),
|
||||||
}
|
}
|
||||||
|
|
||||||
if meta["render_debug_enabled"] == true {
|
if meta["render_debug_enabled"] == true {
|
||||||
renderOutputPath := filepath.Join(paths.ArtifactsDir, artifactName+".render.json")
|
renderOutputPath := filepath.Join(paths.ArtifactsDir, artifactName+".render.json")
|
||||||
|
if runLayout.Enabled {
|
||||||
|
renderOutputPath = filepath.Join(runLayout.ReportsDir, artifactName+".render.json")
|
||||||
|
}
|
||||||
renderStdoutPath := filepath.Join(paths.LogsDir, "scriptorium."+artifactName+".render.stdout.log")
|
renderStdoutPath := filepath.Join(paths.LogsDir, "scriptorium."+artifactName+".render.stdout.log")
|
||||||
renderStderrPath := filepath.Join(paths.LogsDir, "scriptorium."+artifactName+".render.stderr.log")
|
renderStderrPath := filepath.Join(paths.LogsDir, "scriptorium."+artifactName+".render.stderr.log")
|
||||||
renderGeneratedConfigPath := filepath.Join(paths.ConfigDir, "scriptorium."+artifactName+".render.generated.yml")
|
renderGeneratedConfigPath := filepath.Join(paths.ConfigDir, "scriptorium."+artifactName+".render.generated.yml")
|
||||||
|
if runLayout.Enabled {
|
||||||
|
renderStdoutPath = filepath.Join(runLayout.LogsDir, "scriptorium."+artifactName+".render.stdout.log")
|
||||||
|
renderStderrPath = filepath.Join(runLayout.LogsDir, "scriptorium."+artifactName+".render.stderr.log")
|
||||||
|
renderGeneratedConfigPath = filepath.Join(runLayout.ConfigDir, "scriptorium."+artifactName+".render.generated.yml")
|
||||||
|
}
|
||||||
|
|
||||||
renderReq := scriptorium.RenderArtifactRequest{
|
renderReq := scriptorium.RenderArtifactRequest{
|
||||||
Binary: env.Config.Pipeline.Scriptorium.Binary,
|
Binary: env.Config.Pipeline.Scriptorium.Binary,
|
||||||
@@ -257,17 +259,19 @@ func (analyzeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
|||||||
if err := requireNonEmptyFile(finalOutputPath, artifactName+" output"); err != nil {
|
if err := requireNonEmptyFile(finalOutputPath, artifactName+" output"); err != nil {
|
||||||
return nil, fmt.Errorf("analyze: %w", err)
|
return nil, fmt.Errorf("analyze: %w", err)
|
||||||
}
|
}
|
||||||
|
promotedArtifact, err := promoteRunLocalOutput(env.ArtifactStore, finalOutputPath, canonicalOutputPath, artifacts.Ref{
|
||||||
artifactRef := artifacts.Ref{
|
Kind: artifactName,
|
||||||
Kind: artifactName,
|
Category: "artifacts",
|
||||||
Category: "artifacts",
|
SessionID: sessionID,
|
||||||
SessionID: sessionID,
|
})
|
||||||
AbsolutePath: finalOutputPath,
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("analyze: promote artifact output: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
logPaths = append(logPaths, stdoutLogPath, stderrLogPath)
|
logPaths = append(logPaths, stdoutLogPath, stderrLogPath)
|
||||||
generatedConfigs = append(generatedConfigs, generatedConfigPath)
|
generatedConfigs = append(generatedConfigs, generatedConfigPath)
|
||||||
meta["output_path"] = finalOutputPath
|
meta["run_output_path"] = finalOutputPath
|
||||||
|
meta["output_path"] = canonicalOutputPath
|
||||||
meta["generated_config_path"] = generatedConfigPath
|
meta["generated_config_path"] = generatedConfigPath
|
||||||
meta["stdout_log_path"] = stdoutLogPath
|
meta["stdout_log_path"] = stdoutLogPath
|
||||||
meta["stderr_log_path"] = stderrLogPath
|
meta["stderr_log_path"] = stderrLogPath
|
||||||
@@ -286,7 +290,7 @@ func (analyzeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
|||||||
}
|
}
|
||||||
|
|
||||||
return &StageResult{
|
return &StageResult{
|
||||||
Outputs: []artifacts.Ref{artifactRef},
|
Outputs: []artifacts.Ref{promotedArtifact},
|
||||||
Logs: logPaths,
|
Logs: logPaths,
|
||||||
GeneratedConfigs: generatedConfigs,
|
GeneratedConfigs: generatedConfigs,
|
||||||
Metadata: meta,
|
Metadata: meta,
|
||||||
@@ -350,40 +354,6 @@ func discoverProcessedTranscript(m *manifest.Manifest, paths artifacts.SessionPa
|
|||||||
return "", "", nil
|
return "", "", nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func discoverTrimmedTranscript(m *manifest.Manifest, paths artifacts.SessionPaths) (string, string, error) {
|
|
||||||
candidates := []string{}
|
|
||||||
if m != nil && m.Stages != nil {
|
|
||||||
if sr := m.Stages["trim"]; sr != nil {
|
|
||||||
for _, out := range sr.Outputs {
|
|
||||||
if out.Kind != "transcript_trimmed" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
p := strings.TrimSpace(out.LocalPath)
|
|
||||||
if p == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
resolved := artifacts.ResolveSessionLocalPathForRead(paths, p)
|
|
||||||
candidates = append(candidates, filepath.Clean(resolved))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
deduped := dedupeAndSortPaths(candidates)
|
|
||||||
for _, p := range deduped {
|
|
||||||
if info, err := os.Stat(p); err == nil && !info.IsDir() {
|
|
||||||
return p, "manifest.trim.outputs", nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fallback := filepath.Join(paths.TranscriptsDir, "trimmed.json")
|
|
||||||
if info, err := os.Stat(fallback); err == nil && !info.IsDir() {
|
|
||||||
return filepath.Clean(fallback), "fallback.transcripts_dir", nil
|
|
||||||
}
|
|
||||||
if len(deduped) > 0 {
|
|
||||||
return deduped[0], "manifest.trim.outputs", nil
|
|
||||||
}
|
|
||||||
return "", "", nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type analyzeTranscriptInputs struct {
|
type analyzeTranscriptInputs struct {
|
||||||
ProcessedPath string
|
ProcessedPath string
|
||||||
ProcessedSource string
|
ProcessedSource string
|
||||||
@@ -393,38 +363,36 @@ type analyzeTranscriptInputs struct {
|
|||||||
TrimmedSource string
|
TrimmedSource string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func discoverAnalyzeTranscriptRefs(m *manifest.Manifest, paths artifacts.SessionPaths) analyzeTranscriptInputs {
|
||||||
|
processedPath, processedSource := discoverAnalyzeArtifactRef(m, paths, artifacts.ArtifactTranscriptPolished)
|
||||||
|
normalizedPath, normalizedSource := discoverAnalyzeArtifactRef(m, paths, artifacts.ArtifactTranscriptFull)
|
||||||
|
trimmedPath, trimmedSource := discoverAnalyzeArtifactRef(m, paths, artifacts.ArtifactTranscriptTrimmed)
|
||||||
|
return analyzeTranscriptInputs{
|
||||||
|
ProcessedPath: processedPath,
|
||||||
|
ProcessedSource: processedSource,
|
||||||
|
NormalizedPath: normalizedPath,
|
||||||
|
NormalizedSource: normalizedSource,
|
||||||
|
TrimmedPath: trimmedPath,
|
||||||
|
TrimmedSource: trimmedSource,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func discoverAnalyzeArtifactRef(m *manifest.Manifest, paths artifacts.SessionPaths, source string) (string, string) {
|
||||||
|
resolved, err := artifacts.ResolveSessionArtifact(paths, m, source)
|
||||||
|
if err != nil {
|
||||||
|
return "", ""
|
||||||
|
}
|
||||||
|
return resolved.Path, resolved.Provenance
|
||||||
|
}
|
||||||
|
|
||||||
func resolveScriptoriumInput(
|
func resolveScriptoriumInput(
|
||||||
inputName string,
|
inputName string,
|
||||||
inputCfg config.ScriptoriumInputConfig,
|
inputCfg config.ScriptoriumInputConfig,
|
||||||
transcriptInputs analyzeTranscriptInputs,
|
m *manifest.Manifest,
|
||||||
paths artifacts.SessionPaths,
|
paths artifacts.SessionPaths,
|
||||||
sessionDir string,
|
sessionDir string,
|
||||||
) (string, bool, error) {
|
) (string, bool, error) {
|
||||||
switch strings.TrimSpace(inputCfg.Source) {
|
switch strings.TrimSpace(inputCfg.Source) {
|
||||||
case "processed_transcript":
|
|
||||||
if strings.TrimSpace(transcriptInputs.ProcessedPath) == "" {
|
|
||||||
return "", false, nil
|
|
||||||
}
|
|
||||||
if err := validateProcessedTranscriptOutput(transcriptInputs.ProcessedPath); err != nil {
|
|
||||||
return "", false, fmt.Errorf("processed transcript %q invalid: %w", transcriptInputs.ProcessedPath, err)
|
|
||||||
}
|
|
||||||
return transcriptInputs.ProcessedPath, true, nil
|
|
||||||
case "normalized_transcript":
|
|
||||||
if strings.TrimSpace(transcriptInputs.NormalizedPath) == "" {
|
|
||||||
return "", false, fmt.Errorf("normalized transcript input is unavailable; run normalize stage first")
|
|
||||||
}
|
|
||||||
if err := validateProcessedTranscriptOutput(transcriptInputs.NormalizedPath); err != nil {
|
|
||||||
return "", false, fmt.Errorf("normalized transcript %q invalid: %w", transcriptInputs.NormalizedPath, err)
|
|
||||||
}
|
|
||||||
return transcriptInputs.NormalizedPath, true, nil
|
|
||||||
case "trimmed_transcript":
|
|
||||||
if strings.TrimSpace(transcriptInputs.TrimmedPath) == "" {
|
|
||||||
return "", false, fmt.Errorf("trimmed transcript input is unavailable; run trim stage first")
|
|
||||||
}
|
|
||||||
if err := validateProcessedTranscriptOutput(transcriptInputs.TrimmedPath); err != nil {
|
|
||||||
return "", false, fmt.Errorf("trimmed transcript %q invalid: %w", transcriptInputs.TrimmedPath, err)
|
|
||||||
}
|
|
||||||
return transcriptInputs.TrimmedPath, true, nil
|
|
||||||
case "previous_session_artifact":
|
case "previous_session_artifact":
|
||||||
if strings.TrimSpace(inputCfg.Path) == "" {
|
if strings.TrimSpace(inputCfg.Path) == "" {
|
||||||
return "", false, nil
|
return "", false, nil
|
||||||
@@ -435,7 +403,27 @@ func resolveScriptoriumInput(
|
|||||||
}
|
}
|
||||||
return resolved, true, nil
|
return resolved, true, nil
|
||||||
default:
|
default:
|
||||||
return "", false, fmt.Errorf("unsupported source %q", inputCfg.Source)
|
resolved, err := artifacts.ResolveSessionArtifact(paths, m, inputCfg.Source)
|
||||||
|
if err == nil {
|
||||||
|
return resolved.Path, true, nil
|
||||||
|
}
|
||||||
|
if errors.Is(err, artifacts.ErrSessionArtifactNotFound) {
|
||||||
|
normalized, normalizeErr := artifacts.NormalizeSessionArtifactSource(inputCfg.Source)
|
||||||
|
if normalizeErr != nil {
|
||||||
|
return "", false, normalizeErr
|
||||||
|
}
|
||||||
|
switch normalized {
|
||||||
|
case artifacts.ArtifactTranscriptPolished:
|
||||||
|
return "", false, nil
|
||||||
|
case artifacts.ArtifactTranscriptFull:
|
||||||
|
return "", false, fmt.Errorf("normalized transcript input is unavailable; run normalize stage first")
|
||||||
|
case artifacts.ArtifactTranscriptTrimmed:
|
||||||
|
return "", false, fmt.Errorf("trimmed transcript input is unavailable; run trim stage first")
|
||||||
|
default:
|
||||||
|
return "", false, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "", false, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ import (
|
|||||||
|
|
||||||
func TestAnalyzeGeneratesSessionRecapFromTrimmedTranscript(t *testing.T) {
|
func TestAnalyzeGeneratesSessionRecapFromTrimmedTranscript(t *testing.T) {
|
||||||
env, m, fake := setupAnalyzeEnv(t)
|
env, m, fake := setupAnalyzeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
|
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
|
||||||
|
|
||||||
result, err := (analyzeStage{}).Run(context.Background(), env, m)
|
result, err := (analyzeStage{}).Run(context.Background(), env, m)
|
||||||
@@ -70,7 +70,7 @@ func TestAnalyzeGeneratesSessionRecapFromTrimmedTranscript(t *testing.T) {
|
|||||||
|
|
||||||
func TestAnalyzeRenderDebugFalseDoesNotCallRenderArtifact(t *testing.T) {
|
func TestAnalyzeRenderDebugFalseDoesNotCallRenderArtifact(t *testing.T) {
|
||||||
env, m, fake := setupAnalyzeEnv(t)
|
env, m, fake := setupAnalyzeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
|
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
|
||||||
|
|
||||||
env.Config.Pipeline.Scriptorium.RenderDebug = false
|
env.Config.Pipeline.Scriptorium.RenderDebug = false
|
||||||
@@ -86,7 +86,7 @@ func TestAnalyzeRenderDebugFalseDoesNotCallRenderArtifact(t *testing.T) {
|
|||||||
|
|
||||||
func TestAnalyzeRenderDebugArtifactOverrideFalseWinsOverGlobalTrue(t *testing.T) {
|
func TestAnalyzeRenderDebugArtifactOverrideFalseWinsOverGlobalTrue(t *testing.T) {
|
||||||
env, m, fake := setupAnalyzeEnv(t)
|
env, m, fake := setupAnalyzeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
|
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
|
||||||
|
|
||||||
env.Config.Pipeline.Scriptorium.RenderDebug = true
|
env.Config.Pipeline.Scriptorium.RenderDebug = true
|
||||||
@@ -109,7 +109,7 @@ func TestAnalyzeRenderDebugArtifactOverrideFalseWinsOverGlobalTrue(t *testing.T)
|
|||||||
|
|
||||||
func TestAnalyzeRenderDebugTrueCallsRenderBeforeRun(t *testing.T) {
|
func TestAnalyzeRenderDebugTrueCallsRenderBeforeRun(t *testing.T) {
|
||||||
env, m, _ := setupAnalyzeEnv(t)
|
env, m, _ := setupAnalyzeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
|
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
|
||||||
|
|
||||||
env.Config.Pipeline.Scriptorium.RenderDebug = true
|
env.Config.Pipeline.Scriptorium.RenderDebug = true
|
||||||
@@ -130,7 +130,7 @@ func TestAnalyzeRenderDebugTrueCallsRenderBeforeRun(t *testing.T) {
|
|||||||
|
|
||||||
func TestAnalyzeRenderOutputPathIsRecorded(t *testing.T) {
|
func TestAnalyzeRenderOutputPathIsRecorded(t *testing.T) {
|
||||||
env, m, fake := setupAnalyzeEnv(t)
|
env, m, fake := setupAnalyzeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
|
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
|
||||||
env.Config.Pipeline.Scriptorium.RenderDebug = true
|
env.Config.Pipeline.Scriptorium.RenderDebug = true
|
||||||
|
|
||||||
@@ -154,7 +154,7 @@ func TestAnalyzeRenderOutputPathIsRecorded(t *testing.T) {
|
|||||||
|
|
||||||
func TestAnalyzeRenderFailurePreventsRun(t *testing.T) {
|
func TestAnalyzeRenderFailurePreventsRun(t *testing.T) {
|
||||||
env, m, fake := setupAnalyzeEnv(t)
|
env, m, fake := setupAnalyzeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
|
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
|
||||||
env.Config.Pipeline.Scriptorium.RenderDebug = true
|
env.Config.Pipeline.Scriptorium.RenderDebug = true
|
||||||
fake.RenderErr = errors.New("render boom")
|
fake.RenderErr = errors.New("render boom")
|
||||||
@@ -176,7 +176,7 @@ func TestAnalyzeRenderFailurePreventsRun(t *testing.T) {
|
|||||||
|
|
||||||
func TestAnalyzeRenderInvalidJSONFailsClearly(t *testing.T) {
|
func TestAnalyzeRenderInvalidJSONFailsClearly(t *testing.T) {
|
||||||
env, m, _ := setupAnalyzeEnv(t)
|
env, m, _ := setupAnalyzeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
|
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
|
||||||
env.Config.Pipeline.Scriptorium.RenderDebug = true
|
env.Config.Pipeline.Scriptorium.RenderDebug = true
|
||||||
runner := &orderedScriptoriumRunner{
|
runner := &orderedScriptoriumRunner{
|
||||||
@@ -199,7 +199,7 @@ func TestAnalyzeRenderInvalidJSONFailsClearly(t *testing.T) {
|
|||||||
|
|
||||||
func TestAnalyzeRunStillSucceedsWhenRenderSucceeds(t *testing.T) {
|
func TestAnalyzeRunStillSucceedsWhenRenderSucceeds(t *testing.T) {
|
||||||
env, m, fake := setupAnalyzeEnv(t)
|
env, m, fake := setupAnalyzeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
|
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
|
||||||
env.Config.Pipeline.Scriptorium.RenderDebug = true
|
env.Config.Pipeline.Scriptorium.RenderDebug = true
|
||||||
|
|
||||||
@@ -220,7 +220,7 @@ func TestAnalyzeRunStillSucceedsWhenRenderSucceeds(t *testing.T) {
|
|||||||
|
|
||||||
func TestAnalyzeOmitsOptionalPreviousRecapWhenUnavailable(t *testing.T) {
|
func TestAnalyzeOmitsOptionalPreviousRecapWhenUnavailable(t *testing.T) {
|
||||||
env, m, fake := setupAnalyzeEnv(t)
|
env, m, fake := setupAnalyzeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[]}`)
|
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[]}`)
|
||||||
|
|
||||||
env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] = config.ScriptoriumArtifactConfig{
|
env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] = config.ScriptoriumArtifactConfig{
|
||||||
@@ -256,6 +256,32 @@ func TestAnalyzeOmitsOptionalPreviousRecapWhenUnavailable(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestAnalyzeUsesRunLocalPathsAndPromotesCanonical(t *testing.T) {
|
||||||
|
env, m, fake := setupAnalyzeEnv(t)
|
||||||
|
env.Config.Session.Campaign = "sample-campaign"
|
||||||
|
m.Campaign = "sample-campaign"
|
||||||
|
m.RunID = "20260518T010203Z-abcdef12"
|
||||||
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
|
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
|
||||||
|
|
||||||
|
result, err := (analyzeStage{}).Run(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(fake.RunRequests) != 1 {
|
||||||
|
t.Fatalf("run requests = %d, want 1", len(fake.RunRequests))
|
||||||
|
}
|
||||||
|
if !strings.Contains(fake.RunRequests[0].OutputPath, filepath.Join("runs", m.RunID, "analyze", "outputs")) {
|
||||||
|
t.Fatalf("run output path = %q, want run-local path", fake.RunRequests[0].OutputPath)
|
||||||
|
}
|
||||||
|
if len(result.Outputs) != 1 {
|
||||||
|
t.Fatalf("outputs len = %d, want 1", len(result.Outputs))
|
||||||
|
}
|
||||||
|
if strings.Contains(result.Outputs[0].AbsolutePath, string(filepath.Separator)+"runs"+string(filepath.Separator)) {
|
||||||
|
t.Fatalf("promoted output path = %q, want canonical session path", result.Outputs[0].AbsolutePath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type orderedScriptoriumRunner struct {
|
type orderedScriptoriumRunner struct {
|
||||||
Calls []string
|
Calls []string
|
||||||
RenderErr error
|
RenderErr error
|
||||||
@@ -318,7 +344,7 @@ func (r *orderedScriptoriumRunner) RunArtifact(_ context.Context, req scriptoriu
|
|||||||
|
|
||||||
func TestAnalyzeIncludesPreviousRecapWhenConfiguredAndAvailable(t *testing.T) {
|
func TestAnalyzeIncludesPreviousRecapWhenConfiguredAndAvailable(t *testing.T) {
|
||||||
env, m, fake := setupAnalyzeEnv(t)
|
env, m, fake := setupAnalyzeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[]}`)
|
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[]}`)
|
||||||
|
|
||||||
previousRecapPath := filepath.Join(filepath.Dir(env.Config.SessionPath), "previous", "session_recap.md")
|
previousRecapPath := filepath.Join(filepath.Dir(env.Config.SessionPath), "previous", "session_recap.md")
|
||||||
@@ -359,7 +385,7 @@ func TestAnalyzeIncludesPreviousRecapWhenConfiguredAndAvailable(t *testing.T) {
|
|||||||
|
|
||||||
func TestAnalyzeFailsWhenRequiredPreviousRecapMissing(t *testing.T) {
|
func TestAnalyzeFailsWhenRequiredPreviousRecapMissing(t *testing.T) {
|
||||||
env, m, _ := setupAnalyzeEnv(t)
|
env, m, _ := setupAnalyzeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[]}`)
|
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[]}`)
|
||||||
|
|
||||||
env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] = config.ScriptoriumArtifactConfig{
|
env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] = config.ScriptoriumArtifactConfig{
|
||||||
@@ -390,7 +416,7 @@ func TestAnalyzeFailsWhenRequiredPreviousRecapMissing(t *testing.T) {
|
|||||||
|
|
||||||
func TestAnalyzeFailsWhenOutputPathMissing(t *testing.T) {
|
func TestAnalyzeFailsWhenOutputPathMissing(t *testing.T) {
|
||||||
env, m, fake := setupAnalyzeEnv(t)
|
env, m, fake := setupAnalyzeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
|
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
|
||||||
|
|
||||||
artifact := env.Config.Pipeline.Scriptorium.Artifacts["session_recap"]
|
artifact := env.Config.Pipeline.Scriptorium.Artifacts["session_recap"]
|
||||||
@@ -425,7 +451,7 @@ func TestAnalyzeFailsWhenTrimmedTranscriptMissing(t *testing.T) {
|
|||||||
|
|
||||||
func TestAnalyzeSupportsProcessedTranscriptSourceWhenConfigured(t *testing.T) {
|
func TestAnalyzeSupportsProcessedTranscriptSourceWhenConfigured(t *testing.T) {
|
||||||
env, m, fake := setupAnalyzeEnv(t)
|
env, m, fake := setupAnalyzeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[]}`)
|
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[]}`)
|
||||||
|
|
||||||
artifact := env.Config.Pipeline.Scriptorium.Artifacts["session_recap"]
|
artifact := env.Config.Pipeline.Scriptorium.Artifacts["session_recap"]
|
||||||
@@ -447,9 +473,33 @@ func TestAnalyzeSupportsProcessedTranscriptSourceWhenConfigured(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestAnalyzeSupportsCanonicalTrimmedTranscriptSourceWhenConfigured(t *testing.T) {
|
||||||
|
env, m, fake := setupAnalyzeEnv(t)
|
||||||
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
|
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
|
||||||
|
|
||||||
|
artifact := env.Config.Pipeline.Scriptorium.Artifacts["session_recap"]
|
||||||
|
artifact.Inputs["transcript"] = config.ScriptoriumInputConfig{
|
||||||
|
Source: "narratio.transcript.trimmed",
|
||||||
|
Required: true,
|
||||||
|
}
|
||||||
|
env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] = artifact
|
||||||
|
|
||||||
|
_, err := (analyzeStage{}).Run(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(fake.RunRequests) != 1 {
|
||||||
|
t.Fatalf("run requests = %d, want 1", len(fake.RunRequests))
|
||||||
|
}
|
||||||
|
if fake.RunRequests[0].InputPaths["transcript"] != filepath.Join(paths.TranscriptsDir, "trimmed.json") {
|
||||||
|
t.Fatalf("transcript input = %q, want trimmed transcript path", fake.RunRequests[0].InputPaths["transcript"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestAnalyzeSupportsNormalizedTranscriptSourceWhenConfigured(t *testing.T) {
|
func TestAnalyzeSupportsNormalizedTranscriptSourceWhenConfigured(t *testing.T) {
|
||||||
env, m, fake := setupAnalyzeEnv(t)
|
env, m, fake := setupAnalyzeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
normalizedPath := filepath.Join(paths.TranscriptsDir, "normalized.json")
|
normalizedPath := filepath.Join(paths.TranscriptsDir, "normalized.json")
|
||||||
writeAnalyzeFile(t, normalizedPath, `{"segments":[{"id":1}]}`)
|
writeAnalyzeFile(t, normalizedPath, `{"segments":[{"id":1}]}`)
|
||||||
|
|
||||||
@@ -472,9 +522,39 @@ func TestAnalyzeSupportsNormalizedTranscriptSourceWhenConfigured(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestAnalyzeSupportsCanonicalNormalizedTranscriptSourceFromManifestOutput(t *testing.T) {
|
||||||
|
env, m, fake := setupAnalyzeEnv(t)
|
||||||
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
|
fallbackPath := filepath.Join(paths.TranscriptsDir, "normalized.json")
|
||||||
|
manifestPath := filepath.Join(paths.ArtifactsDir, "normalized.from-manifest.json")
|
||||||
|
writeAnalyzeFile(t, fallbackPath, `{"segments":[{"id":999}]}`)
|
||||||
|
writeAnalyzeFile(t, manifestPath, `{"segments":[{"id":10}]}`)
|
||||||
|
m.MarkStageSucceeded("normalize", time.Now().UTC(), []manifest.ArtifactRecord{
|
||||||
|
{Kind: "transcript_normalized", LocalPath: manifestPath},
|
||||||
|
})
|
||||||
|
|
||||||
|
artifact := env.Config.Pipeline.Scriptorium.Artifacts["session_recap"]
|
||||||
|
artifact.Inputs["transcript"] = config.ScriptoriumInputConfig{
|
||||||
|
Source: "narratio.transcript.full",
|
||||||
|
Required: true,
|
||||||
|
}
|
||||||
|
env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] = artifact
|
||||||
|
|
||||||
|
_, err := (analyzeStage{}).Run(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(fake.RunRequests) != 1 {
|
||||||
|
t.Fatalf("run requests = %d, want 1", len(fake.RunRequests))
|
||||||
|
}
|
||||||
|
if fake.RunRequests[0].InputPaths["transcript"] != manifestPath {
|
||||||
|
t.Fatalf("transcript input = %q, want manifest normalized transcript path", fake.RunRequests[0].InputPaths["transcript"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestAnalyzeSupportsNormalizedTranscriptSourceFromManifestOutput(t *testing.T) {
|
func TestAnalyzeSupportsNormalizedTranscriptSourceFromManifestOutput(t *testing.T) {
|
||||||
env, m, fake := setupAnalyzeEnv(t)
|
env, m, fake := setupAnalyzeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
fallbackPath := filepath.Join(paths.TranscriptsDir, "normalized.json")
|
fallbackPath := filepath.Join(paths.TranscriptsDir, "normalized.json")
|
||||||
manifestPath := filepath.Join(paths.ArtifactsDir, "normalized.from-manifest.json")
|
manifestPath := filepath.Join(paths.ArtifactsDir, "normalized.from-manifest.json")
|
||||||
writeAnalyzeFile(t, fallbackPath, `{"segments":[{"id":999}]}`)
|
writeAnalyzeFile(t, fallbackPath, `{"segments":[{"id":999}]}`)
|
||||||
@@ -528,7 +608,7 @@ func TestAnalyzeFailsWhenNormalizedTranscriptMissing(t *testing.T) {
|
|||||||
|
|
||||||
func TestAnalyzeFailsWhenProcessedTranscriptInvalidJSON(t *testing.T) {
|
func TestAnalyzeFailsWhenProcessedTranscriptInvalidJSON(t *testing.T) {
|
||||||
env, m, _ := setupAnalyzeEnv(t)
|
env, m, _ := setupAnalyzeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{not-json`)
|
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{not-json`)
|
||||||
|
|
||||||
_, err := (analyzeStage{}).Run(context.Background(), env, m)
|
_, err := (analyzeStage{}).Run(context.Background(), env, m)
|
||||||
@@ -542,7 +622,7 @@ func TestAnalyzeFailsWhenProcessedTranscriptInvalidJSON(t *testing.T) {
|
|||||||
|
|
||||||
func TestAnalyzeFailsWhenProcessedTranscriptMissingSegmentsArray(t *testing.T) {
|
func TestAnalyzeFailsWhenProcessedTranscriptMissingSegmentsArray(t *testing.T) {
|
||||||
env, m, _ := setupAnalyzeEnv(t)
|
env, m, _ := setupAnalyzeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"not_segments":[]}`)
|
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"not_segments":[]}`)
|
||||||
|
|
||||||
_, err := (analyzeStage{}).Run(context.Background(), env, m)
|
_, err := (analyzeStage{}).Run(context.Background(), env, m)
|
||||||
@@ -556,7 +636,7 @@ func TestAnalyzeFailsWhenProcessedTranscriptMissingSegmentsArray(t *testing.T) {
|
|||||||
|
|
||||||
func TestAnalyzeRecordsRefsAndMetadata(t *testing.T) {
|
func TestAnalyzeRecordsRefsAndMetadata(t *testing.T) {
|
||||||
env, m, _ := setupAnalyzeEnv(t)
|
env, m, _ := setupAnalyzeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
|
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
|
||||||
|
|
||||||
result, err := (analyzeStage{}).Run(context.Background(), env, m)
|
result, err := (analyzeStage{}).Run(context.Background(), env, m)
|
||||||
@@ -585,7 +665,7 @@ func TestAnalyzeRecordsRefsAndMetadata(t *testing.T) {
|
|||||||
|
|
||||||
func TestAnalyzeHandlesAdapterError(t *testing.T) {
|
func TestAnalyzeHandlesAdapterError(t *testing.T) {
|
||||||
env, m, fake := setupAnalyzeEnv(t)
|
env, m, fake := setupAnalyzeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
|
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
|
||||||
fake.RunErr = errors.New("adapter boom")
|
fake.RunErr = errors.New("adapter boom")
|
||||||
|
|
||||||
@@ -600,7 +680,7 @@ func TestAnalyzeHandlesAdapterError(t *testing.T) {
|
|||||||
|
|
||||||
func TestAnalyzeHandlesValidationFailedResultAsError(t *testing.T) {
|
func TestAnalyzeHandlesValidationFailedResultAsError(t *testing.T) {
|
||||||
env, m, fake := setupAnalyzeEnv(t)
|
env, m, fake := setupAnalyzeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
|
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
|
||||||
fake.RunResult = scriptorium.ArtifactResult{
|
fake.RunResult = scriptorium.ArtifactResult{
|
||||||
ValidationFailed: true,
|
ValidationFailed: true,
|
||||||
@@ -619,7 +699,7 @@ func TestAnalyzeHandlesValidationFailedResultAsError(t *testing.T) {
|
|||||||
|
|
||||||
func TestAnalyzeSkipsWhenNoEnabledScriptoriumArtifactsConfigured(t *testing.T) {
|
func TestAnalyzeSkipsWhenNoEnabledScriptoriumArtifactsConfigured(t *testing.T) {
|
||||||
env, m, _ := setupAnalyzeEnv(t)
|
env, m, _ := setupAnalyzeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
|
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`)
|
||||||
|
|
||||||
env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] = config.ScriptoriumArtifactConfig{
|
env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] = config.ScriptoriumArtifactConfig{
|
||||||
@@ -692,7 +772,7 @@ func setupAnalyzeEnv(t *testing.T) (*Env, *manifest.Manifest, *scriptorium.FakeR
|
|||||||
}
|
}
|
||||||
|
|
||||||
store := artifacts.NewLocalStore(workspace)
|
store := artifacts.NewLocalStore(workspace)
|
||||||
if _, err := store.EnsureLayout("2026-05-03"); err != nil {
|
if _, err := store.EnsureLayoutFor("sample-campaign", "2026-05-03"); err != nil {
|
||||||
t.Fatalf("EnsureLayout() error = %v", err)
|
t.Fatalf("EnsureLayout() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
582
internal/stage/archive.go
Normal file
582
internal/stage/archive.go
Normal file
@@ -0,0 +1,582 @@
|
|||||||
|
package stage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
|
)
|
||||||
|
|
||||||
|
type archiveStage struct{}
|
||||||
|
type archiveUploadFile struct {
|
||||||
|
RelativePath string
|
||||||
|
LocalPath string
|
||||||
|
}
|
||||||
|
|
||||||
|
var archivePrerequisiteStages = []string{
|
||||||
|
"prepare",
|
||||||
|
"transcribe",
|
||||||
|
"merge",
|
||||||
|
"polish",
|
||||||
|
"normalize",
|
||||||
|
"trim",
|
||||||
|
"analyze",
|
||||||
|
}
|
||||||
|
|
||||||
|
func (archiveStage) Name() string { return "archive" }
|
||||||
|
|
||||||
|
func (archiveStage) Declares() IODecl {
|
||||||
|
return IODecl{
|
||||||
|
Inputs: []artifacts.Ref{
|
||||||
|
{Kind: "manifest", Category: "input", RelativePath: "manifest.json"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (archiveStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*StageResult, error) {
|
||||||
|
if env == nil || env.Config == nil || env.Config.Pipeline == nil || env.Config.Session == nil {
|
||||||
|
return nil, fmt.Errorf("archive: resolved config must include pipeline and session")
|
||||||
|
}
|
||||||
|
|
||||||
|
if archiveDisabled(env) {
|
||||||
|
return &StageResult{
|
||||||
|
Metadata: map[string]any{
|
||||||
|
"stage": "archive",
|
||||||
|
"skipped": true,
|
||||||
|
"archive_enabled": false,
|
||||||
|
"audio_upload_skipped": true,
|
||||||
|
"current_pointer_written": false,
|
||||||
|
},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
if archiveRunUploadDisabled(env) {
|
||||||
|
return &StageResult{
|
||||||
|
Metadata: map[string]any{
|
||||||
|
"stage": "archive",
|
||||||
|
"skipped": true,
|
||||||
|
"upload_run_enabled": false,
|
||||||
|
"audio_upload_skipped": true,
|
||||||
|
"current_pointer_written": false,
|
||||||
|
},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := validateArchivePrerequisites(m); err != nil {
|
||||||
|
return nil, fmt.Errorf("archive: %w", err)
|
||||||
|
}
|
||||||
|
if env.ObjectStore == nil {
|
||||||
|
return nil, fmt.Errorf("archive: remote object store backend is required when archive run upload is enabled")
|
||||||
|
}
|
||||||
|
|
||||||
|
runRoot, err := resolveArchiveRunRoot(env, m)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("archive: resolve run root: %w", err)
|
||||||
|
}
|
||||||
|
runRootInfo, err := os.Stat(runRoot)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("archive: run root %q: %w", runRoot, err)
|
||||||
|
}
|
||||||
|
if !runRootInfo.IsDir() {
|
||||||
|
return nil, fmt.Errorf("archive: run root %q is not a directory", runRoot)
|
||||||
|
}
|
||||||
|
|
||||||
|
runPrefix, err := archiveRunPrefix(env, m)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("archive: resolve s3 run prefix: %w", err)
|
||||||
|
}
|
||||||
|
sessionPrefix, err := archiveSessionPrefix(env, m)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("archive: resolve s3 session prefix: %w", err)
|
||||||
|
}
|
||||||
|
bucket := archiveBucket(env, m)
|
||||||
|
if bucket == "" {
|
||||||
|
return nil, fmt.Errorf("archive: resolve s3 bucket: bucket is required")
|
||||||
|
}
|
||||||
|
runID := strings.TrimSpace(m.RunID)
|
||||||
|
if runID == "" {
|
||||||
|
return nil, fmt.Errorf("archive: run id is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
manifestSource, err := resolveArchiveRunManifestSource(runRoot)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("archive: resolve run manifest source: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
runFiles, err := collectArchiveRunFiles(runRoot, manifestSource)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("archive: collect run files: %w", err)
|
||||||
|
}
|
||||||
|
sessionRoot, err := resolveArchiveSessionRoot(env, m)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("archive: resolve session root for promotions: %w", err)
|
||||||
|
}
|
||||||
|
promotions, err := resolveArchivePromotions(sessionRoot, env.Config.Pipeline.Archive.PromoteArtifacts)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("archive: resolve promotion rules: %w", err)
|
||||||
|
}
|
||||||
|
runUploaded := make([]string, 0, len(runFiles))
|
||||||
|
for _, file := range runFiles {
|
||||||
|
key := artifacts.S3RunRelativeDestinationKey(runPrefix, file.RelativePath)
|
||||||
|
if _, err := env.ObjectStore.Upload(ctx, file.LocalPath, key, storage.UploadOptions{}); err != nil {
|
||||||
|
return nil, fmt.Errorf("archive: upload run file %q to %q: %w", file.RelativePath, key, err)
|
||||||
|
}
|
||||||
|
runUploaded = append(runUploaded, file.RelativePath)
|
||||||
|
}
|
||||||
|
|
||||||
|
promotedUploaded := make([]string, 0, len(promotions))
|
||||||
|
skippedOptional := make([]string, 0)
|
||||||
|
for _, promotion := range promotions {
|
||||||
|
if !promotion.Exists {
|
||||||
|
if promotion.Required {
|
||||||
|
return nil, fmt.Errorf("archive: required promotion source missing: %q", promotion.From)
|
||||||
|
}
|
||||||
|
skippedOptional = append(skippedOptional, promotion.To)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
key := artifacts.S3PromotedArtifactKey(sessionPrefix, promotion.To)
|
||||||
|
if _, err := env.ObjectStore.Upload(ctx, promotion.LocalPath, key, storage.UploadOptions{}); err != nil {
|
||||||
|
return nil, fmt.Errorf("archive: upload promoted output %q to %q: %w", promotion.From, key, err)
|
||||||
|
}
|
||||||
|
promotedUploaded = append(promotedUploaded, promotion.To)
|
||||||
|
}
|
||||||
|
|
||||||
|
currentManifestKey := artifacts.S3CurrentManifestKey(sessionPrefix)
|
||||||
|
manifestTempPath, err := writeCurrentManifestSnapshot(m, archiveMetadataPreview(
|
||||||
|
bucket,
|
||||||
|
runPrefix,
|
||||||
|
sessionPrefix,
|
||||||
|
runUploaded,
|
||||||
|
promotedUploaded,
|
||||||
|
skippedOptional,
|
||||||
|
currentManifestKey,
|
||||||
|
))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("archive: build current manifest snapshot: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = os.Remove(manifestTempPath) }()
|
||||||
|
|
||||||
|
if _, err := env.ObjectStore.Upload(ctx, manifestTempPath, currentManifestKey, storage.UploadOptions{
|
||||||
|
ContentType: "application/json",
|
||||||
|
}); err != nil {
|
||||||
|
return nil, fmt.Errorf("archive: upload current manifest to %q: %w", currentManifestKey, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
currentRunPointerKey := artifacts.S3CurrentRunPointerKey(sessionPrefix)
|
||||||
|
runIDTempPath, err := writeCurrentRunIDPointer(runID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("archive: build current run id pointer: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = os.Remove(runIDTempPath) }()
|
||||||
|
|
||||||
|
if _, err := env.ObjectStore.Upload(ctx, runIDTempPath, currentRunPointerKey, storage.UploadOptions{
|
||||||
|
ContentType: "text/plain; charset=utf-8",
|
||||||
|
}); err != nil {
|
||||||
|
return nil, fmt.Errorf("archive: upload current run pointer to %q: %w", currentRunPointerKey, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &StageResult{
|
||||||
|
Metadata: map[string]any{
|
||||||
|
"stage": "archive",
|
||||||
|
"uploaded": true,
|
||||||
|
"s3_bucket": bucket,
|
||||||
|
"s3_run_prefix": runPrefix,
|
||||||
|
"run_files_uploaded": len(runUploaded),
|
||||||
|
"run_uploaded_paths": runUploaded,
|
||||||
|
"promoted_files_uploaded": len(promotedUploaded),
|
||||||
|
"promoted_paths": promotedUploaded,
|
||||||
|
"skipped_optional_promotions": skippedOptional,
|
||||||
|
"current_manifest_key": currentManifestKey,
|
||||||
|
"current_run_id_key": currentRunPointerKey,
|
||||||
|
"current_pointer_written": true,
|
||||||
|
"audio_upload_skipped": true,
|
||||||
|
},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type archivePromotion struct {
|
||||||
|
From string
|
||||||
|
To string
|
||||||
|
Required bool
|
||||||
|
LocalPath string
|
||||||
|
Exists bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func archiveDisabled(env *Env) bool {
|
||||||
|
cfg := env.Config.Pipeline.Archive
|
||||||
|
if cfg == nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return cfg.Enabled != nil && !*cfg.Enabled
|
||||||
|
}
|
||||||
|
|
||||||
|
func archiveRunUploadDisabled(env *Env) bool {
|
||||||
|
cfg := env.Config.Pipeline.Archive
|
||||||
|
if cfg == nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return cfg.UploadRun != nil && !*cfg.UploadRun
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateArchivePrerequisites(m *manifest.Manifest) error {
|
||||||
|
if m == nil {
|
||||||
|
return fmt.Errorf("manifest is required")
|
||||||
|
}
|
||||||
|
for _, stageName := range archivePrerequisiteStages {
|
||||||
|
sr := m.Stages[stageName]
|
||||||
|
if sr == nil {
|
||||||
|
return fmt.Errorf("prerequisite stage %q has not succeeded", stageName)
|
||||||
|
}
|
||||||
|
if sr.Status != manifest.StatusSucceeded {
|
||||||
|
return fmt.Errorf("prerequisite stage %q status is %q (want %q)", stageName, sr.Status, manifest.StatusSucceeded)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveArchiveRunRoot(env *Env, m *manifest.Manifest) (string, error) {
|
||||||
|
sessionID := strings.TrimSpace(env.Config.Session.SessionID)
|
||||||
|
if sessionID == "" && m != nil {
|
||||||
|
sessionID = strings.TrimSpace(m.SessionID)
|
||||||
|
}
|
||||||
|
campaign := strings.TrimSpace(env.Config.Session.Campaign)
|
||||||
|
if campaign == "" && m != nil {
|
||||||
|
campaign = strings.TrimSpace(m.Campaign)
|
||||||
|
}
|
||||||
|
runID := ""
|
||||||
|
if m != nil {
|
||||||
|
runID = strings.TrimSpace(m.RunID)
|
||||||
|
}
|
||||||
|
if sessionID == "" || campaign == "" {
|
||||||
|
return "", fmt.Errorf("campaign and session id are required")
|
||||||
|
}
|
||||||
|
if runID == "" {
|
||||||
|
return "", fmt.Errorf("run id is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
canonical := filepath.Clean(artifacts.SessionRunRootForCampaign(env.Config.Pipeline.Workspace.Root, campaign, sessionID, runID))
|
||||||
|
canonicalExists, err := directoryExists(canonical)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("check canonical run root %q: %w", canonical, err)
|
||||||
|
}
|
||||||
|
if !canonicalExists {
|
||||||
|
return "", fmt.Errorf("run root not found for campaign %q session %q run %q at canonical path %q", campaign, sessionID, runID, canonical)
|
||||||
|
}
|
||||||
|
return canonical, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveArchiveSessionRoot(env *Env, m *manifest.Manifest) (string, error) {
|
||||||
|
sessionID := strings.TrimSpace(env.Config.Session.SessionID)
|
||||||
|
if sessionID == "" && m != nil {
|
||||||
|
sessionID = strings.TrimSpace(m.SessionID)
|
||||||
|
}
|
||||||
|
campaign := strings.TrimSpace(env.Config.Session.Campaign)
|
||||||
|
if campaign == "" && m != nil {
|
||||||
|
campaign = strings.TrimSpace(m.Campaign)
|
||||||
|
}
|
||||||
|
if sessionID == "" {
|
||||||
|
return "", fmt.Errorf("session id is required")
|
||||||
|
}
|
||||||
|
if campaign == "" {
|
||||||
|
return "", fmt.Errorf("campaign is required")
|
||||||
|
}
|
||||||
|
return filepath.Clean(artifacts.SessionWorkDirForCampaign(env.Config.Pipeline.Workspace.Root, campaign, sessionID)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func archiveRunPrefix(env *Env, m *manifest.Manifest) (string, error) {
|
||||||
|
runPrefix := strings.TrimSpace(m.S3RunPrefix)
|
||||||
|
if runPrefix != "" {
|
||||||
|
return runPrefix, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
sessionPrefix, err := archiveSessionPrefix(env, m)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
runID := strings.TrimSpace(m.RunID)
|
||||||
|
if runID == "" {
|
||||||
|
return "", fmt.Errorf("run id is required")
|
||||||
|
}
|
||||||
|
return artifacts.S3RunPrefix(sessionPrefix, runID), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func archiveSessionPrefix(env *Env, m *manifest.Manifest) (string, error) {
|
||||||
|
if m != nil && strings.TrimSpace(m.S3SessionPrefix) != "" {
|
||||||
|
return strings.TrimSpace(m.S3SessionPrefix), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
sessionID := strings.TrimSpace(env.Config.Session.SessionID)
|
||||||
|
if sessionID == "" {
|
||||||
|
sessionID = strings.TrimSpace(m.SessionID)
|
||||||
|
}
|
||||||
|
campaign := strings.TrimSpace(env.Config.Session.Campaign)
|
||||||
|
if campaign == "" {
|
||||||
|
campaign = strings.TrimSpace(m.Campaign)
|
||||||
|
}
|
||||||
|
if env.Config.Pipeline.Storage.S3 == nil {
|
||||||
|
return "", fmt.Errorf("pipeline.storage.s3 configuration is required")
|
||||||
|
}
|
||||||
|
sessionPrefix := artifacts.S3SessionPrefix(env.Config.Pipeline.Storage.S3.RootPrefix, campaign, sessionID)
|
||||||
|
if strings.TrimSpace(sessionPrefix) == "" {
|
||||||
|
return "", fmt.Errorf("session prefix is required")
|
||||||
|
}
|
||||||
|
return sessionPrefix, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func archiveBucket(env *Env, m *manifest.Manifest) string {
|
||||||
|
if m != nil && strings.TrimSpace(m.S3Bucket) != "" {
|
||||||
|
return strings.TrimSpace(m.S3Bucket)
|
||||||
|
}
|
||||||
|
if env == nil || env.Config == nil || env.Config.Pipeline == nil || env.Config.Pipeline.Storage.S3 == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(env.Config.Pipeline.Storage.S3.Bucket)
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveArchivePromotions(sessionRoot string, rules []config.ArchivePromotionRule) ([]archivePromotion, error) {
|
||||||
|
sessionRoot = filepath.Clean(strings.TrimSpace(sessionRoot))
|
||||||
|
if sessionRoot == "" {
|
||||||
|
return nil, fmt.Errorf("session root is required")
|
||||||
|
}
|
||||||
|
out := make([]archivePromotion, 0, len(rules))
|
||||||
|
for _, rule := range rules {
|
||||||
|
from := strings.TrimSpace(rule.From)
|
||||||
|
to := strings.TrimSpace(rule.To)
|
||||||
|
required := rule.Required == nil || *rule.Required
|
||||||
|
|
||||||
|
resolvedPath, err := resolveWorkDirRelativePath(sessionRoot, from)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("promotion from %q: %w", from, err)
|
||||||
|
}
|
||||||
|
info, err := os.Stat(resolvedPath)
|
||||||
|
exists := err == nil && !info.IsDir()
|
||||||
|
if err != nil && !os.IsNotExist(err) {
|
||||||
|
return nil, fmt.Errorf("promotion source %q: %w", from, err)
|
||||||
|
}
|
||||||
|
localPath := resolvedPath
|
||||||
|
|
||||||
|
out = append(out, archivePromotion{
|
||||||
|
From: from,
|
||||||
|
To: to,
|
||||||
|
Required: required,
|
||||||
|
LocalPath: localPath,
|
||||||
|
Exists: exists,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveWorkDirRelativePath(workDir, rel string) (string, error) {
|
||||||
|
rel = filepath.Clean(filepath.FromSlash(strings.TrimSpace(rel)))
|
||||||
|
if rel == "." || rel == "" {
|
||||||
|
return "", fmt.Errorf("relative path is required")
|
||||||
|
}
|
||||||
|
full := filepath.Join(workDir, rel)
|
||||||
|
cleanedWork := filepath.Clean(workDir)
|
||||||
|
cleanedFull := filepath.Clean(full)
|
||||||
|
relative, err := filepath.Rel(cleanedWork, cleanedFull)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("compute relative path: %w", err)
|
||||||
|
}
|
||||||
|
if relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
|
||||||
|
return "", fmt.Errorf("path escapes workdir")
|
||||||
|
}
|
||||||
|
return cleanedFull, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func collectArchiveRunFiles(runRoot, manifestPath string) ([]archiveUploadFile, error) {
|
||||||
|
files := make([]archiveUploadFile, 0, 64)
|
||||||
|
err := filepath.WalkDir(runRoot, func(path string, d fs.DirEntry, walkErr error) error {
|
||||||
|
if walkErr != nil {
|
||||||
|
return walkErr
|
||||||
|
}
|
||||||
|
if d.IsDir() {
|
||||||
|
if path == runRoot {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
relDir, err := filepath.Rel(runRoot, path)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("relative dir from %q to %q: %w", runRoot, path, err)
|
||||||
|
}
|
||||||
|
relDir = filepath.ToSlash(relDir)
|
||||||
|
// Preserve existing behavior: audio is not uploaded in archive run record.
|
||||||
|
if relDir == "audio" || strings.HasPrefix(relDir, "audio/") {
|
||||||
|
return filepath.SkipDir
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
rel, err := filepath.Rel(runRoot, path)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("relative path from %q to %q: %w", runRoot, path, err)
|
||||||
|
}
|
||||||
|
rel = filepath.ToSlash(rel)
|
||||||
|
files = append(files, archiveUploadFile{
|
||||||
|
RelativePath: rel,
|
||||||
|
LocalPath: path,
|
||||||
|
})
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("walk %q: %w", runRoot, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
manifestInfo, err := os.Stat(manifestPath)
|
||||||
|
if err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return nil, fmt.Errorf("manifest.json not found (checked path %q)", manifestPath)
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("stat %q: %w", manifestPath, err)
|
||||||
|
}
|
||||||
|
if manifestInfo.IsDir() {
|
||||||
|
return nil, fmt.Errorf("manifest path %q is a directory", manifestPath)
|
||||||
|
}
|
||||||
|
files = append(files, archiveUploadFile{
|
||||||
|
RelativePath: "manifest.json",
|
||||||
|
LocalPath: manifestPath,
|
||||||
|
})
|
||||||
|
seen := map[string]archiveUploadFile{}
|
||||||
|
for _, file := range files {
|
||||||
|
seen[file.RelativePath] = file
|
||||||
|
}
|
||||||
|
files = files[:0]
|
||||||
|
for _, file := range seen {
|
||||||
|
files = append(files, file)
|
||||||
|
}
|
||||||
|
|
||||||
|
sort.Slice(files, func(i, j int) bool {
|
||||||
|
return files[i].RelativePath < files[j].RelativePath
|
||||||
|
})
|
||||||
|
return files, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveArchiveRunManifestSource(runRoot string) (string, error) {
|
||||||
|
path := filepath.Join(filepath.Clean(runRoot), "manifest.json")
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
if err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return "", fmt.Errorf("manifest.json not found in run root %q", runRoot)
|
||||||
|
}
|
||||||
|
return "", fmt.Errorf("stat %q: %w", path, err)
|
||||||
|
}
|
||||||
|
if info.IsDir() {
|
||||||
|
return "", fmt.Errorf("manifest path %q is a directory", path)
|
||||||
|
}
|
||||||
|
return path, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func directoryExists(path string) (bool, error) {
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
if err == nil {
|
||||||
|
return info.IsDir(), nil
|
||||||
|
}
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeCurrentManifestSnapshot(m *manifest.Manifest, archiveMetadata map[string]any) (string, error) {
|
||||||
|
if m == nil {
|
||||||
|
return "", fmt.Errorf("manifest is required")
|
||||||
|
}
|
||||||
|
clone := *m
|
||||||
|
clone.Stages = make(map[string]*manifest.StageRecord, len(m.Stages))
|
||||||
|
for name, sr := range m.Stages {
|
||||||
|
if sr == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
stageCopy := *sr
|
||||||
|
if sr.Outputs != nil {
|
||||||
|
stageCopy.Outputs = append([]manifest.ArtifactRecord(nil), sr.Outputs...)
|
||||||
|
}
|
||||||
|
if sr.Logs != nil {
|
||||||
|
stageCopy.Logs = append([]string(nil), sr.Logs...)
|
||||||
|
}
|
||||||
|
if sr.GeneratedConfigs != nil {
|
||||||
|
stageCopy.GeneratedConfigs = append([]string(nil), sr.GeneratedConfigs...)
|
||||||
|
}
|
||||||
|
if sr.Metadata != nil {
|
||||||
|
metaCopy := make(map[string]any, len(sr.Metadata))
|
||||||
|
for k, v := range sr.Metadata {
|
||||||
|
metaCopy[k] = v
|
||||||
|
}
|
||||||
|
stageCopy.Metadata = metaCopy
|
||||||
|
}
|
||||||
|
clone.Stages[name] = &stageCopy
|
||||||
|
}
|
||||||
|
|
||||||
|
now := time.Now().UTC()
|
||||||
|
clone.MarkStageSucceeded("archive", now, nil)
|
||||||
|
if sr := clone.Stages["archive"]; sr != nil {
|
||||||
|
sr.Metadata = archiveMetadata
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := json.MarshalIndent(&clone, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("marshal manifest: %w", err)
|
||||||
|
}
|
||||||
|
data = append(data, '\n')
|
||||||
|
|
||||||
|
tmp, err := os.CreateTemp("", "narratio-current-manifest-*.json")
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("create temp file: %w", err)
|
||||||
|
}
|
||||||
|
path := tmp.Name()
|
||||||
|
if _, err := tmp.Write(data); err != nil {
|
||||||
|
_ = tmp.Close()
|
||||||
|
return "", fmt.Errorf("write temp manifest: %w", err)
|
||||||
|
}
|
||||||
|
if err := tmp.Close(); err != nil {
|
||||||
|
return "", fmt.Errorf("close temp manifest: %w", err)
|
||||||
|
}
|
||||||
|
return path, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeCurrentRunIDPointer(runID string) (string, error) {
|
||||||
|
tmp, err := os.CreateTemp("", "narratio-current-run-id-*.txt")
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("create temp file: %w", err)
|
||||||
|
}
|
||||||
|
path := tmp.Name()
|
||||||
|
if _, err := tmp.WriteString(runID + "\n"); err != nil {
|
||||||
|
_ = tmp.Close()
|
||||||
|
return "", fmt.Errorf("write temp run id pointer: %w", err)
|
||||||
|
}
|
||||||
|
if err := tmp.Close(); err != nil {
|
||||||
|
return "", fmt.Errorf("close temp run id pointer: %w", err)
|
||||||
|
}
|
||||||
|
return path, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func archiveMetadataPreview(
|
||||||
|
bucket, runPrefix, sessionPrefix string,
|
||||||
|
runUploaded []string,
|
||||||
|
promotedUploaded []string,
|
||||||
|
skippedOptional []string,
|
||||||
|
currentManifestKey string,
|
||||||
|
) map[string]any {
|
||||||
|
return map[string]any{
|
||||||
|
"stage": "archive",
|
||||||
|
"uploaded": true,
|
||||||
|
"s3_bucket": bucket,
|
||||||
|
"s3_run_prefix": runPrefix,
|
||||||
|
"run_files_uploaded": len(runUploaded),
|
||||||
|
"run_uploaded_paths": append([]string(nil), runUploaded...),
|
||||||
|
"promoted_files_uploaded": len(promotedUploaded),
|
||||||
|
"promoted_paths": append([]string(nil), promotedUploaded...),
|
||||||
|
"skipped_optional_promotions": append([]string(nil), skippedOptional...),
|
||||||
|
"current_manifest_key": currentManifestKey,
|
||||||
|
"current_run_id_key": artifacts.S3CurrentRunPointerKey(sessionPrefix),
|
||||||
|
"current_pointer_written": false,
|
||||||
|
"audio_upload_skipped": true,
|
||||||
|
}
|
||||||
|
}
|
||||||
343
internal/stage/archive_test.go
Normal file
343
internal/stage/archive_test.go
Normal file
@@ -0,0 +1,343 @@
|
|||||||
|
package stage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestArchiveSkipsWhenDisabled(t *testing.T) {
|
||||||
|
env, m, _ := archiveFixture(t)
|
||||||
|
env.Config.Pipeline.Archive.Enabled = boolPtr(false)
|
||||||
|
|
||||||
|
result, err := archiveStage{}.Run(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if result.Metadata["skipped"] != true {
|
||||||
|
t.Fatalf("metadata = %#v, want skipped=true", result.Metadata)
|
||||||
|
}
|
||||||
|
if len(env.ObjectStore.(*storage.FakeBackend).Uploads) != 0 {
|
||||||
|
t.Fatalf("unexpected uploads when archive disabled")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArchiveSkipsRunUploadWhenDisabled(t *testing.T) {
|
||||||
|
env, m, _ := archiveFixture(t)
|
||||||
|
env.Config.Pipeline.Archive.UploadRun = boolPtr(false)
|
||||||
|
|
||||||
|
result, err := archiveStage{}.Run(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if result.Metadata["skipped"] != true {
|
||||||
|
t.Fatalf("metadata = %#v, want skipped=true", result.Metadata)
|
||||||
|
}
|
||||||
|
if len(env.ObjectStore.(*storage.FakeBackend).Uploads) != 0 {
|
||||||
|
t.Fatalf("unexpected uploads when upload_run disabled")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArchiveFailsWhenPrerequisiteNotSucceeded(t *testing.T) {
|
||||||
|
env, m, _ := archiveFixture(t)
|
||||||
|
m.Stages["trim"].Status = manifest.StatusFailed
|
||||||
|
|
||||||
|
_, err := archiveStage{}.Run(context.Background(), env, m)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), `prerequisite stage "trim"`) {
|
||||||
|
t.Fatalf("Run() error = %v, want prerequisite failure", err)
|
||||||
|
}
|
||||||
|
if len(env.ObjectStore.(*storage.FakeBackend).Uploads) != 0 {
|
||||||
|
t.Fatalf("unexpected uploads on prerequisite failure")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArchiveUploadsRunRecordPromotionsAndCurrentPointer(t *testing.T) {
|
||||||
|
env, m, _ := archiveFixture(t)
|
||||||
|
fake := env.ObjectStore.(*storage.FakeBackend)
|
||||||
|
|
||||||
|
result, err := archiveStage{}.Run(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
runPrefix := m.S3RunPrefix
|
||||||
|
sessionPrefix := m.S3SessionPrefix
|
||||||
|
wantRunUploads := []string{
|
||||||
|
"analyze/outputs/artifacts/session_recap.md",
|
||||||
|
"merge/config/seriatim.generated.yml",
|
||||||
|
"prepare/inputs/session.yml",
|
||||||
|
"prepare/outputs/audio/speaker.flac",
|
||||||
|
"logs/audita.stderr.log",
|
||||||
|
"manifest.json",
|
||||||
|
"polish/reports/audita.report.json",
|
||||||
|
"transcribe/outputs/transcripts/raw/speaker.json",
|
||||||
|
"trim/outputs/transcripts/trimmed.json",
|
||||||
|
}
|
||||||
|
for _, rel := range wantRunUploads {
|
||||||
|
key := runPrefix + rel
|
||||||
|
if _, ok := fake.Objects[key]; !ok {
|
||||||
|
t.Fatalf("missing run upload key %q", key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
trimmedKey := sessionPrefix + "transcripts/trimmed.json"
|
||||||
|
recapKey := sessionPrefix + "artifacts/session_recap.md"
|
||||||
|
if _, ok := fake.Objects[trimmedKey]; !ok {
|
||||||
|
t.Fatalf("missing promoted key %q", trimmedKey)
|
||||||
|
}
|
||||||
|
if _, ok := fake.Objects[recapKey]; !ok {
|
||||||
|
t.Fatalf("missing promoted key %q", recapKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
currentManifestKey := sessionPrefix + "current/manifest.json"
|
||||||
|
currentRunIDKey := sessionPrefix + "current/run_id.txt"
|
||||||
|
if _, ok := fake.Objects[currentManifestKey]; !ok {
|
||||||
|
t.Fatalf("missing current manifest key %q", currentManifestKey)
|
||||||
|
}
|
||||||
|
if _, ok := fake.Objects[currentRunIDKey]; !ok {
|
||||||
|
t.Fatalf("missing current run pointer key %q", currentRunIDKey)
|
||||||
|
}
|
||||||
|
if got := string(fake.Objects[currentRunIDKey].Data); got != m.RunID+"\n" {
|
||||||
|
t.Fatalf("run pointer contents = %q, want %q", got, m.RunID+"\\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
audioKey := runPrefix + "audio/speaker.flac"
|
||||||
|
if _, ok := fake.Objects[audioKey]; ok {
|
||||||
|
t.Fatalf("audio key %q should not be uploaded", audioKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
uploads := fake.Uploads
|
||||||
|
if len(uploads) == 0 {
|
||||||
|
t.Fatal("expected uploads")
|
||||||
|
}
|
||||||
|
if uploads[len(uploads)-1].Key != currentRunIDKey {
|
||||||
|
t.Fatalf("last upload key = %q, want current run pointer key %q", uploads[len(uploads)-1].Key, currentRunIDKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
if result.Metadata["current_pointer_written"] != true {
|
||||||
|
t.Fatalf("metadata = %#v, want current_pointer_written=true", result.Metadata)
|
||||||
|
}
|
||||||
|
if result.Metadata["promoted_files_uploaded"] != 2 {
|
||||||
|
t.Fatalf("metadata promoted_files_uploaded = %#v, want 2", result.Metadata["promoted_files_uploaded"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArchiveUsesCustomPromotionRules(t *testing.T) {
|
||||||
|
env, m, _ := archiveFixture(t)
|
||||||
|
env.Config.Pipeline.Archive.PromoteArtifacts = []config.ArchivePromotionRule{
|
||||||
|
{From: "transcripts/trimmed.json", To: "published/trimmed.json", Required: boolPtr(true)},
|
||||||
|
{From: "artifacts/session_recap.md", To: "published/recap.md", Required: boolPtr(true)},
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := archiveStage{}.Run(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
fake := env.ObjectStore.(*storage.FakeBackend)
|
||||||
|
if _, ok := fake.Objects[m.S3SessionPrefix+"published/trimmed.json"]; !ok {
|
||||||
|
t.Fatalf("missing custom promoted trimmed key")
|
||||||
|
}
|
||||||
|
if _, ok := fake.Objects[m.S3SessionPrefix+"published/recap.md"]; !ok {
|
||||||
|
t.Fatalf("missing custom promoted recap key")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArchiveSkipsOptionalMissingPromotion(t *testing.T) {
|
||||||
|
env, m, _ := archiveFixture(t)
|
||||||
|
env.Config.Pipeline.Archive.PromoteArtifacts = []config.ArchivePromotionRule{
|
||||||
|
{From: "transcripts/trimmed.json", To: "transcripts/trimmed.json", Required: boolPtr(true)},
|
||||||
|
{From: "artifacts/optional.md", To: "artifacts/optional.md", Required: boolPtr(false)},
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := archiveStage{}.Run(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
got, _ := result.Metadata["skipped_optional_promotions"].([]string)
|
||||||
|
want := []string{"artifacts/optional.md"}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("skipped_optional_promotions = %#v, want %#v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArchiveFailsWhenRequiredPromotionMissing(t *testing.T) {
|
||||||
|
env, m, _ := archiveFixture(t)
|
||||||
|
env.Config.Pipeline.Archive.PromoteArtifacts = []config.ArchivePromotionRule{
|
||||||
|
{From: "artifacts/missing.md", To: "artifacts/missing.md", Required: boolPtr(true)},
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := archiveStage{}.Run(context.Background(), env, m)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "required promotion source missing") {
|
||||||
|
t.Fatalf("Run() error = %v, want required promotion missing failure", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArchiveFailsWhenCanonicalRunRootMissing(t *testing.T) {
|
||||||
|
env, m, runRoot := archiveFixture(t)
|
||||||
|
if err := os.RemoveAll(runRoot); err != nil {
|
||||||
|
t.Fatalf("remove run root: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := archiveStage{}.Run(context.Background(), env, m)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected missing run-root error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "run root not found") {
|
||||||
|
t.Fatalf("error = %v, want missing run-root error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArchiveDoesNotWriteCurrentPointerWhenPromotionUploadFails(t *testing.T) {
|
||||||
|
env, m, _ := archiveFixture(t)
|
||||||
|
fake := env.ObjectStore.(*storage.FakeBackend)
|
||||||
|
trimmedKey := m.S3SessionPrefix + "transcripts/trimmed.json"
|
||||||
|
|
||||||
|
origUploadErr := fake.UploadErr
|
||||||
|
fake.UploadErr = nil
|
||||||
|
failingKey := trimmedKey
|
||||||
|
fake.Uploads = nil
|
||||||
|
|
||||||
|
originalUpload := fake.Upload
|
||||||
|
_ = originalUpload
|
||||||
|
// Use UploadErr toggle by checking call sequence in postcondition.
|
||||||
|
// First failure point is promotion upload; simulate by setting error immediately before promotion key write.
|
||||||
|
// We cannot hook FakeBackend per-key without changing public behavior; use dedicated backend wrapper instead.
|
||||||
|
env.ObjectStore = &promotionFailingStore{delegate: fake, failKey: failingKey}
|
||||||
|
|
||||||
|
_, err := archiveStage{}.Run(context.Background(), env, m)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "promoted output") {
|
||||||
|
t.Fatalf("Run() error = %v, want promotion upload failure", err)
|
||||||
|
}
|
||||||
|
if _, ok := fake.Objects[m.S3SessionPrefix+"current/run_id.txt"]; ok {
|
||||||
|
t.Fatalf("unexpected current pointer write on promotion failure")
|
||||||
|
}
|
||||||
|
fake.UploadErr = origUploadErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArchiveDoesNotWriteCurrentPointerWhenCurrentManifestUploadFails(t *testing.T) {
|
||||||
|
env, m, _ := archiveFixture(t)
|
||||||
|
fake := env.ObjectStore.(*storage.FakeBackend)
|
||||||
|
env.ObjectStore = &promotionFailingStore{delegate: fake, failKey: m.S3SessionPrefix + "current/manifest.json"}
|
||||||
|
|
||||||
|
_, err := archiveStage{}.Run(context.Background(), env, m)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "current manifest") {
|
||||||
|
t.Fatalf("Run() error = %v, want current manifest upload failure", err)
|
||||||
|
}
|
||||||
|
if _, ok := fake.Objects[m.S3SessionPrefix+"current/run_id.txt"]; ok {
|
||||||
|
t.Fatalf("unexpected current pointer write when current manifest upload fails")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArchiveFailsWithoutObjectStore(t *testing.T) {
|
||||||
|
env, m, _ := archiveFixture(t)
|
||||||
|
env.ObjectStore = nil
|
||||||
|
|
||||||
|
_, err := archiveStage{}.Run(context.Background(), env, m)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "object store") {
|
||||||
|
t.Fatalf("Run() error = %v, want object store backend failure", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func archiveFixture(t *testing.T) (*Env, *manifest.Manifest, string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
root := t.TempDir()
|
||||||
|
runID := "20260516T010203Z-1a2b3c4d"
|
||||||
|
campaign := "forsaken"
|
||||||
|
sessionID := "2026-04-19"
|
||||||
|
sessionRoot := artifacts.SessionWorkDirForCampaign(root, campaign, sessionID)
|
||||||
|
runRoot := artifacts.SessionRunRootForCampaign(root, campaign, sessionID, runID)
|
||||||
|
|
||||||
|
writeStageTestFile(t, filepath.Join(sessionRoot, "transcripts", "trimmed.json"), "{}\n")
|
||||||
|
writeStageTestFile(t, filepath.Join(sessionRoot, "artifacts", "session_recap.md"), "# recap\n")
|
||||||
|
|
||||||
|
writeStageTestFile(t, filepath.Join(runRoot, "prepare", "inputs", "session.yml"), "session_id: 2026-04-19\n")
|
||||||
|
writeStageTestFile(t, filepath.Join(runRoot, "prepare", "outputs", "audio", "speaker.flac"), "flac\n")
|
||||||
|
writeStageTestFile(t, filepath.Join(runRoot, "transcribe", "outputs", "transcripts", "raw", "speaker.json"), "{}\n")
|
||||||
|
writeStageTestFile(t, filepath.Join(runRoot, "trim", "outputs", "transcripts", "trimmed.json"), "{}\n")
|
||||||
|
writeStageTestFile(t, filepath.Join(runRoot, "analyze", "outputs", "artifacts", "session_recap.md"), "# recap\n")
|
||||||
|
writeStageTestFile(t, filepath.Join(runRoot, "polish", "reports", "audita.report.json"), "{}\n")
|
||||||
|
writeStageTestFile(t, filepath.Join(runRoot, "merge", "config", "seriatim.generated.yml"), "key: value\n")
|
||||||
|
writeStageTestFile(t, filepath.Join(runRoot, "logs", "audita.stderr.log"), "stderr\n")
|
||||||
|
writeStageTestFile(t, filepath.Join(runRoot, "audio", "speaker.flac"), "flac")
|
||||||
|
writeStageTestFile(t, filepath.Join(runRoot, "manifest.json"), "{}\n")
|
||||||
|
|
||||||
|
m := manifest.New(sessionID, time.Date(2026, 5, 16, 1, 2, 3, 0, time.UTC))
|
||||||
|
m.Campaign = campaign
|
||||||
|
m.RunID = runID
|
||||||
|
m.LocalWorkDir = runRoot
|
||||||
|
m.S3Bucket = "my-dnd-archive"
|
||||||
|
m.S3SessionPrefix = artifacts.S3SessionPrefix("dnd", campaign, sessionID)
|
||||||
|
m.S3RunPrefix = artifacts.S3RunPrefix(m.S3SessionPrefix, runID)
|
||||||
|
for _, name := range archivePrerequisiteStages {
|
||||||
|
m.MarkStageSucceeded(name, time.Date(2026, 5, 16, 1, 2, 3, 0, time.UTC), nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
env := &Env{
|
||||||
|
Config: &config.Config{
|
||||||
|
Pipeline: &config.PipelineConfig{
|
||||||
|
Workspace: config.WorkspaceConfig{Root: root},
|
||||||
|
Storage: config.StorageConfig{
|
||||||
|
S3: &config.StorageS3Config{
|
||||||
|
Bucket: "my-dnd-archive",
|
||||||
|
RootPrefix: "dnd",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Archive: &config.ArchiveConfig{
|
||||||
|
Enabled: boolPtr(true),
|
||||||
|
UploadRun: boolPtr(true),
|
||||||
|
PromoteArtifacts: []config.ArchivePromotionRule{
|
||||||
|
{From: "transcripts/trimmed.json", To: "transcripts/trimmed.json", Required: boolPtr(true)},
|
||||||
|
{From: "artifacts/session_recap.md", To: "artifacts/session_recap.md", Required: boolPtr(true)},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Session: &config.SessionConfig{
|
||||||
|
SessionID: sessionID,
|
||||||
|
Campaign: campaign,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
ObjectStore: &storage.FakeBackend{},
|
||||||
|
}
|
||||||
|
return env, m, runRoot
|
||||||
|
}
|
||||||
|
|
||||||
|
type promotionFailingStore struct {
|
||||||
|
delegate *storage.FakeBackend
|
||||||
|
failKey string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *promotionFailingStore) List(ctx context.Context, prefix string) ([]storage.ObjectInfo, error) {
|
||||||
|
return s.delegate.List(ctx, prefix)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *promotionFailingStore) Download(ctx context.Context, key, localPath string) error {
|
||||||
|
return s.delegate.Download(ctx, key, localPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *promotionFailingStore) Upload(ctx context.Context, localPath, key string, opts storage.UploadOptions) (storage.ObjectInfo, error) {
|
||||||
|
if strings.TrimSpace(key) == strings.TrimSpace(s.failKey) {
|
||||||
|
return storage.ObjectInfo{}, errors.New("forced upload failure")
|
||||||
|
}
|
||||||
|
return s.delegate.Upload(ctx, localPath, key, opts)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *promotionFailingStore) Exists(ctx context.Context, key string) (bool, error) {
|
||||||
|
return s.delegate.Exists(ctx, key)
|
||||||
|
}
|
||||||
|
|
||||||
|
func boolPtr(v bool) *bool {
|
||||||
|
p := v
|
||||||
|
return &p
|
||||||
|
}
|
||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/seriatim"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/seriatim"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
@@ -56,7 +57,11 @@ func (mergeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Sta
|
|||||||
return nil, fmt.Errorf("merge: session id is required")
|
return nil, fmt.Errorf("merge: session id is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
paths := env.ArtifactStore.SessionPaths(sessionID)
|
paths := sessionPathsForEnv(env, sessionID)
|
||||||
|
runLayout, err := resolveRunStageLayout(env, m, paths, sessionID, "merge")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("merge: resolve run-stage layout: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
inputs, err := discoverRawTranscripts(m, paths)
|
inputs, err := discoverRawTranscripts(m, paths)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -80,16 +85,37 @@ func (mergeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Sta
|
|||||||
return nil, fmt.Errorf("merge: %w", err)
|
return nil, fmt.Errorf("merge: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
mergedPath := filepath.Join(paths.TranscriptsDir, "merged.json")
|
canonicalMergedPath := filepath.Join(paths.TranscriptsDir, "merged.json")
|
||||||
reportPath := filepath.Join(paths.ArtifactsDir, "seriatim.report.json")
|
mergedPath, err := runLocalPathForCanonical(runLayout, paths, canonicalMergedPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("merge: resolve run-local merged transcript path: %w", err)
|
||||||
|
}
|
||||||
|
canonicalReportPath := filepath.Join(paths.ArtifactsDir, "seriatim.report.json")
|
||||||
|
reportPath := canonicalReportPath
|
||||||
|
if runLayout.Enabled {
|
||||||
|
reportPath, err = runLocalPathForCanonical(runLayout, paths, canonicalReportPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("merge: resolve run-local report path: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
stdoutPath := filepath.Join(paths.LogsDir, "seriatim.stdout.log")
|
stdoutPath := filepath.Join(paths.LogsDir, "seriatim.stdout.log")
|
||||||
stderrPath := filepath.Join(paths.LogsDir, "seriatim.stderr.log")
|
stderrPath := filepath.Join(paths.LogsDir, "seriatim.stderr.log")
|
||||||
genCfgPath := filepath.Join(paths.ConfigDir, "seriatim.generated.yml")
|
genCfgPath := filepath.Join(paths.ConfigDir, "seriatim.generated.yml")
|
||||||
|
if runLayout.Enabled {
|
||||||
|
stdoutPath = filepath.Join(runLayout.LogsDir, "seriatim.stdout.log")
|
||||||
|
stderrPath = filepath.Join(runLayout.LogsDir, "seriatim.stderr.log")
|
||||||
|
genCfgPath = filepath.Join(runLayout.ConfigDir, "seriatim.generated.yml")
|
||||||
|
}
|
||||||
|
|
||||||
|
normalizedInputs, normalizeLogs, normalizeConfigs, normalizeMeta, err := normalizeMergeInputs(ctx, env, inputs, paths, runLayout)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
reportEnabled := env.Config.Pipeline.Seriatim.Report != nil && *env.Config.Pipeline.Seriatim.Report
|
reportEnabled := env.Config.Pipeline.Seriatim.Report != nil && *env.Config.Pipeline.Seriatim.Report
|
||||||
req := seriatim.MergeRequest{
|
req := seriatim.MergeRequest{
|
||||||
GeneratedConfigPath: genCfgPath,
|
GeneratedConfigPath: genCfgPath,
|
||||||
InputTranscriptPaths: inputs,
|
InputTranscriptPaths: normalizedInputs,
|
||||||
OutputMergedTranscriptPath: mergedPath,
|
OutputMergedTranscriptPath: mergedPath,
|
||||||
ReportPath: "",
|
ReportPath: "",
|
||||||
SpeakersPath: speakersPath,
|
SpeakersPath: speakersPath,
|
||||||
@@ -124,35 +150,50 @@ func (mergeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Sta
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
outputs := []artifacts.Ref{{
|
promotedMerged, err := promoteRunLocalOutput(env.ArtifactStore, finalMergedPath, canonicalMergedPath, artifacts.Ref{
|
||||||
Kind: "transcript_merged",
|
Kind: "transcript_merged",
|
||||||
Category: "transcripts",
|
Category: "transcripts",
|
||||||
SessionID: sessionID,
|
SessionID: sessionID,
|
||||||
AbsolutePath: finalMergedPath,
|
})
|
||||||
}}
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("merge: promote merged transcript: %w", err)
|
||||||
|
}
|
||||||
|
outputs := []artifacts.Ref{promotedMerged}
|
||||||
if reportEnabled {
|
if reportEnabled {
|
||||||
outputs = append(outputs, artifacts.Ref{
|
promotedReport, err := promoteRunLocalOutput(env.ArtifactStore, finalReportPath, canonicalReportPath, artifacts.Ref{
|
||||||
Kind: "seriatim_report",
|
Kind: "seriatim_report",
|
||||||
Category: "artifacts",
|
Category: "artifacts",
|
||||||
SessionID: sessionID,
|
SessionID: sessionID,
|
||||||
AbsolutePath: finalReportPath,
|
|
||||||
})
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("merge: promote report: %w", err)
|
||||||
|
}
|
||||||
|
outputs = append(outputs, promotedReport)
|
||||||
}
|
}
|
||||||
|
|
||||||
coalesceGap := any(nil)
|
coalesceGap := any(nil)
|
||||||
if env.Config.Pipeline.Seriatim.CoalesceGap != nil {
|
if env.Config.Pipeline.Seriatim.CoalesceGap != nil {
|
||||||
coalesceGap = *env.Config.Pipeline.Seriatim.CoalesceGap
|
coalesceGap = *env.Config.Pipeline.Seriatim.CoalesceGap
|
||||||
}
|
}
|
||||||
|
reportCanonicalPath := ""
|
||||||
|
if reportEnabled {
|
||||||
|
reportCanonicalPath = canonicalReportPath
|
||||||
|
}
|
||||||
|
|
||||||
meta := map[string]any{
|
meta := map[string]any{
|
||||||
"stage": "merge",
|
"stage": "merge",
|
||||||
"input_transcripts_count": len(inputs),
|
"input_transcripts_count": len(normalizedInputs),
|
||||||
"input_transcript_paths": inputs,
|
"input_transcript_paths": inputs,
|
||||||
|
"normalized_inputs_count": len(normalizedInputs),
|
||||||
|
"normalized_input_paths": normalizedInputs,
|
||||||
|
"normalize_inputs": normalizeMeta,
|
||||||
"output_schema": env.Config.Pipeline.Seriatim.OutputSchema,
|
"output_schema": env.Config.Pipeline.Seriatim.OutputSchema,
|
||||||
"coalesce_gap": coalesceGap,
|
"coalesce_gap": coalesceGap,
|
||||||
"report_enabled": reportEnabled,
|
"report_enabled": reportEnabled,
|
||||||
"output_path": finalMergedPath,
|
"run_output_path": finalMergedPath,
|
||||||
"report_path": finalReportPath,
|
"output_path": canonicalMergedPath,
|
||||||
|
"run_report_path": finalReportPath,
|
||||||
|
"report_path": reportCanonicalPath,
|
||||||
"timeout": env.Config.Pipeline.Seriatim.Timeout,
|
"timeout": env.Config.Pipeline.Seriatim.Timeout,
|
||||||
"binary": env.Config.Pipeline.Seriatim.Binary,
|
"binary": env.Config.Pipeline.Seriatim.Binary,
|
||||||
"adapter_duration_ms": res.Duration.Milliseconds(),
|
"adapter_duration_ms": res.Duration.Milliseconds(),
|
||||||
@@ -172,12 +213,111 @@ func (mergeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Sta
|
|||||||
|
|
||||||
return &StageResult{
|
return &StageResult{
|
||||||
Outputs: outputs,
|
Outputs: outputs,
|
||||||
Logs: []string{stdoutPath, stderrPath},
|
Logs: append(normalizeLogs, stdoutPath, stderrPath),
|
||||||
GeneratedConfigs: []string{genCfgPath},
|
GeneratedConfigs: append(normalizeConfigs, genCfgPath),
|
||||||
Metadata: meta,
|
Metadata: meta,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type normalizeMergeInputMeta struct {
|
||||||
|
InputPath string `json:"input_path"`
|
||||||
|
OutputPath string `json:"output_path"`
|
||||||
|
StdoutLogPath string `json:"stdout_log_path"`
|
||||||
|
StderrLogPath string `json:"stderr_log_path"`
|
||||||
|
GeneratedConfig string `json:"generated_config_path"`
|
||||||
|
DurationMs int64 `json:"duration_ms"`
|
||||||
|
ExitCode int `json:"exit_code"`
|
||||||
|
InvokedBinary string `json:"invoked_binary"`
|
||||||
|
OutputSchema string `json:"output_schema"`
|
||||||
|
AdapterReportPath string `json:"adapter_report_path,omitempty"`
|
||||||
|
AdapterOutputPath string `json:"adapter_output_path,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeMergeInputs(
|
||||||
|
ctx context.Context,
|
||||||
|
env *Env,
|
||||||
|
rawInputs []string,
|
||||||
|
paths artifacts.SessionPaths,
|
||||||
|
runLayout runStageLayout,
|
||||||
|
) ([]string, []string, []string, []normalizeMergeInputMeta, error) {
|
||||||
|
normalizedInputs := make([]string, 0, len(rawInputs))
|
||||||
|
logs := make([]string, 0, len(rawInputs)*2)
|
||||||
|
configs := make([]string, 0, len(rawInputs))
|
||||||
|
meta := make([]normalizeMergeInputMeta, 0, len(rawInputs))
|
||||||
|
normalizedDir := filepath.Join(paths.TranscriptsRawDir, "normalized")
|
||||||
|
if runLayout.Enabled {
|
||||||
|
normalizedDir = filepath.Join(runLayout.ScratchDir, "normalized")
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(normalizedDir, 0o755); err != nil {
|
||||||
|
return nil, nil, nil, nil, fmt.Errorf("merge: ensure normalized transcripts directory %q: %w", normalizedDir, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var timeout time.Duration
|
||||||
|
timeoutRaw := strings.TrimSpace(env.Config.Pipeline.Seriatim.Timeout)
|
||||||
|
if timeoutRaw != "" {
|
||||||
|
parsed, err := time.ParseDuration(timeoutRaw)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, nil, nil, fmt.Errorf("merge: parse seriatim timeout %q: %w", env.Config.Pipeline.Seriatim.Timeout, err)
|
||||||
|
}
|
||||||
|
timeout = parsed
|
||||||
|
}
|
||||||
|
for _, input := range rawInputs {
|
||||||
|
base := strings.TrimSuffix(filepath.Base(input), filepath.Ext(input))
|
||||||
|
outPath := filepath.Join(normalizedDir, base+".normalized.json")
|
||||||
|
stdoutPath := filepath.Join(paths.LogsDir, "seriatim.normalize."+base+".stdout.log")
|
||||||
|
stderrPath := filepath.Join(paths.LogsDir, "seriatim.normalize."+base+".stderr.log")
|
||||||
|
cfgPath := filepath.Join(paths.ConfigDir, "seriatim.normalize."+base+".generated.yml")
|
||||||
|
if runLayout.Enabled {
|
||||||
|
stdoutPath = filepath.Join(runLayout.LogsDir, "seriatim.normalize."+base+".stdout.log")
|
||||||
|
stderrPath = filepath.Join(runLayout.LogsDir, "seriatim.normalize."+base+".stderr.log")
|
||||||
|
cfgPath = filepath.Join(runLayout.ConfigDir, "seriatim.normalize."+base+".generated.yml")
|
||||||
|
}
|
||||||
|
|
||||||
|
req := seriatim.NormalizeRequest{
|
||||||
|
Binary: env.Config.Pipeline.Seriatim.Binary,
|
||||||
|
InputTranscriptPath: input,
|
||||||
|
OutputNormalizedPath: outPath,
|
||||||
|
OutputSchema: env.Config.Pipeline.Seriatim.OutputSchema,
|
||||||
|
ReportPath: "",
|
||||||
|
StdoutLogPath: stdoutPath,
|
||||||
|
StderrLogPath: stderrPath,
|
||||||
|
GeneratedConfigPath: cfgPath,
|
||||||
|
Timeout: timeout,
|
||||||
|
}
|
||||||
|
res, err := env.Seriatim.Normalize(ctx, req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, nil, nil, fmt.Errorf("merge: normalize input %q failed: %w", input, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
finalOutputPath := outPath
|
||||||
|
if strings.TrimSpace(res.OutputNormalizedPath) != "" {
|
||||||
|
finalOutputPath = strings.TrimSpace(res.OutputNormalizedPath)
|
||||||
|
}
|
||||||
|
if err := validateTranscriptJSONFile(finalOutputPath); err != nil {
|
||||||
|
return nil, nil, nil, nil, fmt.Errorf("merge: normalized transcript %q invalid (input %q): %w", finalOutputPath, input, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
normalizedInputs = append(normalizedInputs, finalOutputPath)
|
||||||
|
logs = append(logs, stdoutPath, stderrPath)
|
||||||
|
configs = append(configs, cfgPath)
|
||||||
|
meta = append(meta, normalizeMergeInputMeta{
|
||||||
|
InputPath: input,
|
||||||
|
OutputPath: finalOutputPath,
|
||||||
|
StdoutLogPath: stdoutPath,
|
||||||
|
StderrLogPath: stderrPath,
|
||||||
|
GeneratedConfig: cfgPath,
|
||||||
|
DurationMs: res.Duration.Milliseconds(),
|
||||||
|
ExitCode: res.ExitCode,
|
||||||
|
InvokedBinary: res.InvokedBinary,
|
||||||
|
OutputSchema: res.OutputSchema,
|
||||||
|
AdapterReportPath: res.ReportPath,
|
||||||
|
AdapterOutputPath: res.OutputNormalizedPath,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return normalizedInputs, logs, configs, meta, nil
|
||||||
|
}
|
||||||
|
|
||||||
func discoverRawTranscripts(m *manifest.Manifest, paths artifacts.SessionPaths) ([]string, error) {
|
func discoverRawTranscripts(m *manifest.Manifest, paths artifacts.SessionPaths) ([]string, error) {
|
||||||
fromManifest := make([]string, 0)
|
fromManifest := make([]string, 0)
|
||||||
if m != nil && m.Stages != nil {
|
if m != nil && m.Stages != nil {
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package stage
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -14,9 +15,25 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type normalizeDirAssertingRunner struct {
|
||||||
|
*seriatim.FakeRunner
|
||||||
|
ExpectedDir string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *normalizeDirAssertingRunner) Normalize(ctx context.Context, req seriatim.NormalizeRequest) (seriatim.NormalizeResult, error) {
|
||||||
|
info, err := os.Stat(r.ExpectedDir)
|
||||||
|
if err != nil {
|
||||||
|
return seriatim.NormalizeResult{}, fmt.Errorf("normalized directory check failed for %q: %w", r.ExpectedDir, err)
|
||||||
|
}
|
||||||
|
if !info.IsDir() {
|
||||||
|
return seriatim.NormalizeResult{}, fmt.Errorf("normalized path %q exists but is not a directory", r.ExpectedDir)
|
||||||
|
}
|
||||||
|
return r.FakeRunner.Normalize(ctx, req)
|
||||||
|
}
|
||||||
|
|
||||||
func TestMergeStageMergesRawTranscriptsAndRecordsMetadata(t *testing.T) {
|
func TestMergeStageMergesRawTranscriptsAndRecordsMetadata(t *testing.T) {
|
||||||
env, m := setupMergeEnv(t)
|
env, m := setupMergeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
|
|
||||||
inA := filepath.Join(paths.TranscriptsRawDir, "alice.json")
|
inA := filepath.Join(paths.TranscriptsRawDir, "alice.json")
|
||||||
inB := filepath.Join(paths.TranscriptsRawDir, "bob.json")
|
inB := filepath.Join(paths.TranscriptsRawDir, "bob.json")
|
||||||
@@ -54,6 +71,17 @@ func TestMergeStageMergesRawTranscriptsAndRecordsMetadata(t *testing.T) {
|
|||||||
if len(req.InputTranscriptPaths) != 2 {
|
if len(req.InputTranscriptPaths) != 2 {
|
||||||
t.Fatalf("input transcripts = %#v, want 2", req.InputTranscriptPaths)
|
t.Fatalf("input transcripts = %#v, want 2", req.InputTranscriptPaths)
|
||||||
}
|
}
|
||||||
|
if len(fake.NormalizeRequests) != 2 {
|
||||||
|
t.Fatalf("normalize requests = %#v, want 2", fake.NormalizeRequests)
|
||||||
|
}
|
||||||
|
if fake.NormalizeRequests[0].InputTranscriptPath != inA || fake.NormalizeRequests[1].InputTranscriptPath != inB {
|
||||||
|
t.Fatalf("normalize request inputs = %#v", fake.NormalizeRequests)
|
||||||
|
}
|
||||||
|
for _, mergeIn := range req.InputTranscriptPaths {
|
||||||
|
if !strings.Contains(mergeIn, filepath.Join("transcripts", "raw", "normalized")) {
|
||||||
|
t.Fatalf("merge input path = %q, want normalized input path", mergeIn)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if len(result.Outputs) != 2 {
|
if len(result.Outputs) != 2 {
|
||||||
t.Fatalf("outputs len = %d, want 2", len(result.Outputs))
|
t.Fatalf("outputs len = %d, want 2", len(result.Outputs))
|
||||||
@@ -64,11 +92,11 @@ func TestMergeStageMergesRawTranscriptsAndRecordsMetadata(t *testing.T) {
|
|||||||
if result.Outputs[1].Kind != "seriatim_report" {
|
if result.Outputs[1].Kind != "seriatim_report" {
|
||||||
t.Fatalf("output[1] kind = %q, want seriatim_report", result.Outputs[1].Kind)
|
t.Fatalf("output[1] kind = %q, want seriatim_report", result.Outputs[1].Kind)
|
||||||
}
|
}
|
||||||
if len(result.Logs) != 2 {
|
if len(result.Logs) != 6 {
|
||||||
t.Fatalf("logs = %#v, want 2 paths", result.Logs)
|
t.Fatalf("logs = %#v, want 6 paths (4 normalize + 2 merge)", result.Logs)
|
||||||
}
|
}
|
||||||
if len(result.GeneratedConfigs) != 1 {
|
if len(result.GeneratedConfigs) != 3 {
|
||||||
t.Fatalf("generated configs = %#v, want 1 path", result.GeneratedConfigs)
|
t.Fatalf("generated configs = %#v, want 3 paths (2 normalize + 1 merge)", result.GeneratedConfigs)
|
||||||
}
|
}
|
||||||
|
|
||||||
meta := result.Metadata
|
meta := result.Metadata
|
||||||
@@ -84,11 +112,17 @@ func TestMergeStageMergesRawTranscriptsAndRecordsMetadata(t *testing.T) {
|
|||||||
if meta["input_transcripts_count"] != 2 {
|
if meta["input_transcripts_count"] != 2 {
|
||||||
t.Fatalf("metadata input_transcripts_count = %#v, want 2", meta["input_transcripts_count"])
|
t.Fatalf("metadata input_transcripts_count = %#v, want 2", meta["input_transcripts_count"])
|
||||||
}
|
}
|
||||||
|
if meta["normalized_inputs_count"] != 2 {
|
||||||
|
t.Fatalf("metadata normalized_inputs_count = %#v, want 2", meta["normalized_inputs_count"])
|
||||||
|
}
|
||||||
|
if _, ok := meta["normalize_inputs"]; !ok {
|
||||||
|
t.Fatalf("metadata normalize_inputs missing: %#v", meta)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMergeStageFailsWhenNoRawTranscripts(t *testing.T) {
|
func TestMergeStageFailsWhenNoRawTranscripts(t *testing.T) {
|
||||||
env, m := setupMergeEnv(t)
|
env, m := setupMergeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeFile(t, filepath.Join(paths.InputsDir, "speakers.yml"), "match: []\n")
|
writeFile(t, filepath.Join(paths.InputsDir, "speakers.yml"), "match: []\n")
|
||||||
writeFile(t, filepath.Join(paths.InputsDir, "autocorrect.yml"), "rules: []\n")
|
writeFile(t, filepath.Join(paths.InputsDir, "autocorrect.yml"), "rules: []\n")
|
||||||
env.Seriatim = &seriatim.FakeRunner{}
|
env.Seriatim = &seriatim.FakeRunner{}
|
||||||
@@ -104,7 +138,7 @@ func TestMergeStageFailsWhenNoRawTranscripts(t *testing.T) {
|
|||||||
|
|
||||||
func TestMergeStageFailsOnInvalidInputJSON(t *testing.T) {
|
func TestMergeStageFailsOnInvalidInputJSON(t *testing.T) {
|
||||||
env, m := setupMergeEnv(t)
|
env, m := setupMergeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeFile(t, filepath.Join(paths.TranscriptsRawDir, "alice.json"), "not-json")
|
writeFile(t, filepath.Join(paths.TranscriptsRawDir, "alice.json"), "not-json")
|
||||||
writeFile(t, filepath.Join(paths.InputsDir, "speakers.yml"), "match: []\n")
|
writeFile(t, filepath.Join(paths.InputsDir, "speakers.yml"), "match: []\n")
|
||||||
writeFile(t, filepath.Join(paths.InputsDir, "autocorrect.yml"), "rules: []\n")
|
writeFile(t, filepath.Join(paths.InputsDir, "autocorrect.yml"), "rules: []\n")
|
||||||
@@ -121,7 +155,7 @@ func TestMergeStageFailsOnInvalidInputJSON(t *testing.T) {
|
|||||||
|
|
||||||
func TestMergeStageFailsWhenAdapterFails(t *testing.T) {
|
func TestMergeStageFailsWhenAdapterFails(t *testing.T) {
|
||||||
env, m := setupMergeEnv(t)
|
env, m := setupMergeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeFile(t, filepath.Join(paths.TranscriptsRawDir, "alice.json"), `{"segments":[]}`)
|
writeFile(t, filepath.Join(paths.TranscriptsRawDir, "alice.json"), `{"segments":[]}`)
|
||||||
writeFile(t, filepath.Join(paths.InputsDir, "speakers.yml"), "match: []\n")
|
writeFile(t, filepath.Join(paths.InputsDir, "speakers.yml"), "match: []\n")
|
||||||
writeFile(t, filepath.Join(paths.InputsDir, "autocorrect.yml"), "rules: []\n")
|
writeFile(t, filepath.Join(paths.InputsDir, "autocorrect.yml"), "rules: []\n")
|
||||||
@@ -138,7 +172,7 @@ func TestMergeStageFailsWhenAdapterFails(t *testing.T) {
|
|||||||
|
|
||||||
func TestMergeStageFallsBackToRawDirectoryWhenTranscribeOutputsMissing(t *testing.T) {
|
func TestMergeStageFallsBackToRawDirectoryWhenTranscribeOutputsMissing(t *testing.T) {
|
||||||
env, m := setupMergeEnv(t)
|
env, m := setupMergeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeFile(t, filepath.Join(paths.TranscriptsRawDir, "alice.json"), `{"segments":[]}`)
|
writeFile(t, filepath.Join(paths.TranscriptsRawDir, "alice.json"), `{"segments":[]}`)
|
||||||
writeFile(t, filepath.Join(paths.InputsDir, "speakers.yml"), "match: []\n")
|
writeFile(t, filepath.Join(paths.InputsDir, "speakers.yml"), "match: []\n")
|
||||||
writeFile(t, filepath.Join(paths.InputsDir, "autocorrect.yml"), "rules: []\n")
|
writeFile(t, filepath.Join(paths.InputsDir, "autocorrect.yml"), "rules: []\n")
|
||||||
@@ -152,11 +186,14 @@ func TestMergeStageFallsBackToRawDirectoryWhenTranscribeOutputsMissing(t *testin
|
|||||||
if len(fake.Requests) != 1 || len(fake.Requests[0].InputTranscriptPaths) != 1 {
|
if len(fake.Requests) != 1 || len(fake.Requests[0].InputTranscriptPaths) != 1 {
|
||||||
t.Fatalf("fallback inputs = %#v", fake.Requests)
|
t.Fatalf("fallback inputs = %#v", fake.Requests)
|
||||||
}
|
}
|
||||||
|
if len(fake.NormalizeRequests) != 1 {
|
||||||
|
t.Fatalf("normalize requests = %#v, want 1", fake.NormalizeRequests)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMergeStageResolvesWorkspaceQualifiedManifestOutputsWithoutDuplication(t *testing.T) {
|
func TestMergeStageResolvesWorkspaceQualifiedManifestOutputsWithoutDuplication(t *testing.T) {
|
||||||
env, m := setupMergeEnvWithRelativeWorkspaceRoot(t)
|
env, m := setupMergeEnvWithRelativeWorkspaceRoot(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
|
|
||||||
rawPath := filepath.Join(paths.TranscriptsRawDir, "alice.json")
|
rawPath := filepath.Join(paths.TranscriptsRawDir, "alice.json")
|
||||||
writeFile(t, rawPath, `{"segments":[]}`)
|
writeFile(t, rawPath, `{"segments":[]}`)
|
||||||
@@ -180,14 +217,114 @@ func TestMergeStageResolvesWorkspaceQualifiedManifestOutputsWithoutDuplication(t
|
|||||||
if len(got) != 1 {
|
if len(got) != 1 {
|
||||||
t.Fatalf("input transcript paths = %#v, want len 1", got)
|
t.Fatalf("input transcript paths = %#v, want len 1", got)
|
||||||
}
|
}
|
||||||
if got[0] != filepath.Clean(rawPath) {
|
if !strings.Contains(got[0], filepath.Join("transcripts", "raw", "normalized")) {
|
||||||
t.Fatalf("resolved transcript path = %q, want %q", got[0], filepath.Clean(rawPath))
|
t.Fatalf("resolved transcript path = %q, want normalized path under transcripts/raw/normalized", got[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMergeStageCreatesNormalizedRawDirectoryBeforeNormalize(t *testing.T) {
|
||||||
|
env, m := setupMergeEnv(t)
|
||||||
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
|
|
||||||
|
rawPath := filepath.Join(paths.TranscriptsRawDir, "alice.json")
|
||||||
|
writeFile(t, rawPath, `{"segments":[]}`)
|
||||||
|
writeFile(t, filepath.Join(paths.InputsDir, "speakers.yml"), "match: []\n")
|
||||||
|
writeFile(t, filepath.Join(paths.InputsDir, "autocorrect.yml"), "rules: []\n")
|
||||||
|
|
||||||
|
normalizedDir := filepath.Join(paths.TranscriptsRawDir, "normalized")
|
||||||
|
if err := os.RemoveAll(normalizedDir); err != nil {
|
||||||
|
t.Fatalf("remove normalized dir: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
env.Seriatim = &normalizeDirAssertingRunner{
|
||||||
|
FakeRunner: &seriatim.FakeRunner{},
|
||||||
|
ExpectedDir: normalizedDir,
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := (mergeStage{}).Run(context.Background(), env, m); err != nil {
|
||||||
|
t.Fatalf("merge.Run() error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMergeStageFailsWhenNormalizeAdapterFails(t *testing.T) {
|
||||||
|
env, m := setupMergeEnv(t)
|
||||||
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
|
in := filepath.Join(paths.TranscriptsRawDir, "alice.json")
|
||||||
|
writeFile(t, in, `{"segments":[]}`)
|
||||||
|
writeFile(t, filepath.Join(paths.InputsDir, "speakers.yml"), "match: []\n")
|
||||||
|
writeFile(t, filepath.Join(paths.InputsDir, "autocorrect.yml"), "rules: []\n")
|
||||||
|
env.Seriatim = &seriatim.FakeRunner{NormalizeErr: context.DeadlineExceeded}
|
||||||
|
|
||||||
|
_, err := (mergeStage{}).Run(context.Background(), env, m)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "normalize input") {
|
||||||
|
t.Fatalf("error = %q", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMergeStageFailsWhenNormalizedOutputInvalid(t *testing.T) {
|
||||||
|
env, m := setupMergeEnv(t)
|
||||||
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
|
in := filepath.Join(paths.TranscriptsRawDir, "alice.json")
|
||||||
|
writeFile(t, in, `{"segments":[]}`)
|
||||||
|
writeFile(t, filepath.Join(paths.InputsDir, "speakers.yml"), "match: []\n")
|
||||||
|
writeFile(t, filepath.Join(paths.InputsDir, "autocorrect.yml"), "rules: []\n")
|
||||||
|
|
||||||
|
badNormalized := filepath.Join(paths.ArtifactsDir, "bad.normalized.json")
|
||||||
|
writeFile(t, badNormalized, "not-json")
|
||||||
|
env.Seriatim = &seriatim.FakeRunner{
|
||||||
|
NormalizeResult: seriatim.NormalizeResult{
|
||||||
|
OutputNormalizedPath: badNormalized,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := (mergeStage{}).Run(context.Background(), env, m)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "normalized transcript") {
|
||||||
|
t.Fatalf("error = %q", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMergeStageUsesRunLocalPathsAndPromotesCanonical(t *testing.T) {
|
||||||
|
env, m := setupMergeEnv(t)
|
||||||
|
env.Config.Session.Campaign = "sample-campaign"
|
||||||
|
m.Campaign = "sample-campaign"
|
||||||
|
m.RunID = "20260518T010203Z-abcdef12"
|
||||||
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
|
|
||||||
|
in := filepath.Join(paths.TranscriptsRawDir, "alice.json")
|
||||||
|
writeFile(t, in, `{"segments":[]}`)
|
||||||
|
writeFile(t, filepath.Join(paths.InputsDir, "speakers.yml"), "match: []\n")
|
||||||
|
writeFile(t, filepath.Join(paths.InputsDir, "autocorrect.yml"), "rules: []\n")
|
||||||
|
|
||||||
|
fake := &seriatim.FakeRunner{}
|
||||||
|
env.Seriatim = fake
|
||||||
|
result, err := (mergeStage{}).Run(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("merge.Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(fake.Requests) != 1 {
|
||||||
|
t.Fatalf("requests = %d, want 1", len(fake.Requests))
|
||||||
|
}
|
||||||
|
req := fake.Requests[0]
|
||||||
|
if !strings.Contains(req.OutputMergedTranscriptPath, filepath.Join("runs", m.RunID, "merge", "outputs")) {
|
||||||
|
t.Fatalf("run output path = %q, want run-local path", req.OutputMergedTranscriptPath)
|
||||||
|
}
|
||||||
|
if len(result.Outputs) == 0 {
|
||||||
|
t.Fatalf("outputs = %#v, want promoted outputs", result.Outputs)
|
||||||
|
}
|
||||||
|
if strings.Contains(result.Outputs[0].AbsolutePath, string(filepath.Separator)+"runs"+string(filepath.Separator)) {
|
||||||
|
t.Fatalf("promoted output path = %q, want canonical session path", result.Outputs[0].AbsolutePath)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMergeStageResolvesSessionRelativeManifestOutputs(t *testing.T) {
|
func TestMergeStageResolvesSessionRelativeManifestOutputs(t *testing.T) {
|
||||||
env, m := setupMergeEnv(t)
|
env, m := setupMergeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
|
|
||||||
rawPath := filepath.Join(paths.TranscriptsRawDir, "alice.json")
|
rawPath := filepath.Join(paths.TranscriptsRawDir, "alice.json")
|
||||||
writeFile(t, rawPath, `{"segments":[]}`)
|
writeFile(t, rawPath, `{"segments":[]}`)
|
||||||
@@ -211,8 +348,8 @@ func TestMergeStageResolvesSessionRelativeManifestOutputs(t *testing.T) {
|
|||||||
if len(got) != 1 {
|
if len(got) != 1 {
|
||||||
t.Fatalf("input transcript paths = %#v, want len 1", got)
|
t.Fatalf("input transcript paths = %#v, want len 1", got)
|
||||||
}
|
}
|
||||||
if got[0] != filepath.Clean(rawPath) {
|
if !strings.Contains(got[0], filepath.Join("transcripts", "raw", "normalized")) {
|
||||||
t.Fatalf("resolved transcript path = %q, want %q", got[0], filepath.Clean(rawPath))
|
t.Fatalf("resolved transcript path = %q, want normalized path under transcripts/raw/normalized", got[0])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -269,7 +406,7 @@ func setupMergeEnvWithWorkspace(t *testing.T, workspace string) (*Env, *manifest
|
|||||||
}
|
}
|
||||||
|
|
||||||
store := artifacts.NewLocalStore(workspace)
|
store := artifacts.NewLocalStore(workspace)
|
||||||
if _, err := store.EnsureLayout("2026-05-03"); err != nil {
|
if _, err := store.EnsureLayoutFor("sample-campaign", "2026-05-03"); err != nil {
|
||||||
t.Fatalf("EnsureLayout() error = %v", err)
|
t.Fatalf("EnsureLayout() error = %v", err)
|
||||||
}
|
}
|
||||||
return &Env{
|
return &Env{
|
||||||
|
|||||||
@@ -54,7 +54,11 @@ func (normalizeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (
|
|||||||
return nil, fmt.Errorf("normalize: session id is required")
|
return nil, fmt.Errorf("normalize: session id is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
paths := env.ArtifactStore.SessionPaths(sessionID)
|
paths := sessionPathsForEnv(env, sessionID)
|
||||||
|
runLayout, err := resolveRunStageLayout(env, m, paths, sessionID, "normalize")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("normalize: resolve run-stage layout: %w", err)
|
||||||
|
}
|
||||||
processedPath, processedSource, err := discoverProcessedTranscript(m, paths)
|
processedPath, processedSource, err := discoverProcessedTranscript(m, paths)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("normalize: resolve processed transcript: %w", err)
|
return nil, fmt.Errorf("normalize: resolve processed transcript: %w", err)
|
||||||
@@ -67,18 +71,34 @@ func (normalizeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (
|
|||||||
}
|
}
|
||||||
|
|
||||||
normalizeCfg := normalizeConfigOrDefault(env.Config.Pipeline.Normalize)
|
normalizeCfg := normalizeConfigOrDefault(env.Config.Pipeline.Normalize)
|
||||||
normalizedPath, err := resolveScriptoriumOutputPath(paths, normalizeCfg.OutputPath)
|
canonicalNormalizedPath, err := resolveScriptoriumOutputPath(paths, normalizeCfg.OutputPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("normalize: resolve normalized output path: %w", err)
|
return nil, fmt.Errorf("normalize: resolve normalized output path: %w", err)
|
||||||
}
|
}
|
||||||
|
normalizedPath, err := runLocalPathForCanonical(runLayout, paths, canonicalNormalizedPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("normalize: resolve run-local normalized output path: %w", err)
|
||||||
|
}
|
||||||
reportEnabled := normalizeCfg.Report != nil && *normalizeCfg.Report
|
reportEnabled := normalizeCfg.Report != nil && *normalizeCfg.Report
|
||||||
reportPath := ""
|
reportPath := ""
|
||||||
|
canonicalReportPath := filepath.Join(paths.ArtifactsDir, "seriatim.normalize.report.json")
|
||||||
if reportEnabled {
|
if reportEnabled {
|
||||||
reportPath = filepath.Join(paths.ArtifactsDir, "seriatim.normalize.report.json")
|
reportPath = canonicalReportPath
|
||||||
|
if runLayout.Enabled {
|
||||||
|
reportPath, err = runLocalPathForCanonical(runLayout, paths, canonicalReportPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("normalize: resolve run-local report path: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
stdoutPath := filepath.Join(paths.LogsDir, "seriatim.normalize.stdout.log")
|
stdoutPath := filepath.Join(paths.LogsDir, "seriatim.normalize.stdout.log")
|
||||||
stderrPath := filepath.Join(paths.LogsDir, "seriatim.normalize.stderr.log")
|
stderrPath := filepath.Join(paths.LogsDir, "seriatim.normalize.stderr.log")
|
||||||
generatedConfigPath := filepath.Join(paths.ConfigDir, "seriatim.normalize.generated.yml")
|
generatedConfigPath := filepath.Join(paths.ConfigDir, "seriatim.normalize.generated.yml")
|
||||||
|
if runLayout.Enabled {
|
||||||
|
stdoutPath = filepath.Join(runLayout.LogsDir, "seriatim.normalize.stdout.log")
|
||||||
|
stderrPath = filepath.Join(runLayout.LogsDir, "seriatim.normalize.stderr.log")
|
||||||
|
generatedConfigPath = filepath.Join(runLayout.ConfigDir, "seriatim.normalize.generated.yml")
|
||||||
|
}
|
||||||
timeout, err := resolveTrimSeriatimTimeout(env.Config.Pipeline.Seriatim.Timeout)
|
timeout, err := resolveTrimSeriatimTimeout(env.Config.Pipeline.Seriatim.Timeout)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("normalize: resolve seriatim timeout: %w", err)
|
return nil, fmt.Errorf("normalize: resolve seriatim timeout: %w", err)
|
||||||
@@ -113,44 +133,56 @@ func (normalizeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
outputs := []artifacts.Ref{{
|
promotedNormalized, err := promoteRunLocalOutput(env.ArtifactStore, finalNormalizedPath, canonicalNormalizedPath, artifacts.Ref{
|
||||||
Kind: "transcript_normalized",
|
Kind: "transcript_normalized",
|
||||||
Category: "transcripts",
|
Category: "transcripts",
|
||||||
SessionID: sessionID,
|
SessionID: sessionID,
|
||||||
AbsolutePath: finalNormalizedPath,
|
})
|
||||||
}}
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("normalize: promote normalized transcript: %w", err)
|
||||||
|
}
|
||||||
|
outputs := []artifacts.Ref{promotedNormalized}
|
||||||
if reportEnabled {
|
if reportEnabled {
|
||||||
outputs = append(outputs, artifacts.Ref{
|
promotedReport, err := promoteRunLocalOutput(env.ArtifactStore, finalReportPath, canonicalReportPath, artifacts.Ref{
|
||||||
Kind: "seriatim_normalize_report",
|
Kind: "seriatim_normalize_report",
|
||||||
Category: "artifacts",
|
Category: "artifacts",
|
||||||
SessionID: sessionID,
|
SessionID: sessionID,
|
||||||
AbsolutePath: finalReportPath,
|
|
||||||
})
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("normalize: promote report: %w", err)
|
||||||
|
}
|
||||||
|
outputs = append(outputs, promotedReport)
|
||||||
|
}
|
||||||
|
reportCanonicalPath := ""
|
||||||
|
if reportEnabled {
|
||||||
|
reportCanonicalPath = canonicalReportPath
|
||||||
}
|
}
|
||||||
|
|
||||||
meta := map[string]any{
|
meta := map[string]any{
|
||||||
"stage": "normalize",
|
"stage": "normalize",
|
||||||
"processed_transcript_path": processedPath,
|
"processed_transcript_path": processedPath,
|
||||||
"processed_transcript_source": processedSource,
|
"processed_transcript_source": processedSource,
|
||||||
"normalized_transcript_path": finalNormalizedPath,
|
"run_normalized_transcript_path": finalNormalizedPath,
|
||||||
"normalized_transcript_source": "stage.normalize.output",
|
"normalized_transcript_path": canonicalNormalizedPath,
|
||||||
"output_schema": normalizeCfg.OutputSchema,
|
"normalized_transcript_source": "stage.normalize.output",
|
||||||
"report_enabled": reportEnabled,
|
"output_schema": normalizeCfg.OutputSchema,
|
||||||
"report_path": finalReportPath,
|
"report_enabled": reportEnabled,
|
||||||
"timeout": env.Config.Pipeline.Seriatim.Timeout,
|
"run_report_path": finalReportPath,
|
||||||
"binary": env.Config.Pipeline.Seriatim.Binary,
|
"report_path": reportCanonicalPath,
|
||||||
"stdout_log_path": stdoutPath,
|
"timeout": env.Config.Pipeline.Seriatim.Timeout,
|
||||||
"stderr_log_path": stderrPath,
|
"binary": env.Config.Pipeline.Seriatim.Binary,
|
||||||
"generated_config_path": generatedConfigPath,
|
"stdout_log_path": stdoutPath,
|
||||||
"adapter_duration_ms": res.Duration.Milliseconds(),
|
"stderr_log_path": stderrPath,
|
||||||
"adapter_exit_code": res.ExitCode,
|
"generated_config_path": generatedConfigPath,
|
||||||
"adapter_invoked_binary": res.InvokedBinary,
|
"adapter_duration_ms": res.Duration.Milliseconds(),
|
||||||
"adapter_output_schema": res.OutputSchema,
|
"adapter_exit_code": res.ExitCode,
|
||||||
"adapter_output_path": res.OutputNormalizedPath,
|
"adapter_invoked_binary": res.InvokedBinary,
|
||||||
"adapter_report_path": res.ReportPath,
|
"adapter_output_schema": res.OutputSchema,
|
||||||
"adapter_generated_config": res.GeneratedConfigPath,
|
"adapter_output_path": res.OutputNormalizedPath,
|
||||||
"adapter_stdout_log_path": res.StdoutLogPath,
|
"adapter_report_path": res.ReportPath,
|
||||||
"adapter_stderr_log_path": res.StderrLogPath,
|
"adapter_generated_config": res.GeneratedConfigPath,
|
||||||
|
"adapter_stdout_log_path": res.StdoutLogPath,
|
||||||
|
"adapter_stderr_log_path": res.StderrLogPath,
|
||||||
}
|
}
|
||||||
if res.Metadata != nil {
|
if res.Metadata != nil {
|
||||||
meta["adapter_metadata"] = res.Metadata
|
meta["adapter_metadata"] = res.Metadata
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ import (
|
|||||||
|
|
||||||
func TestNormalizeStageConsumesProcessedTranscriptFromManifest(t *testing.T) {
|
func TestNormalizeStageConsumesProcessedTranscriptFromManifest(t *testing.T) {
|
||||||
env, m, ser := setupNormalizeEnv(t)
|
env, m, ser := setupNormalizeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
manifestProcessed := filepath.Join(paths.ArtifactsDir, "processed.from-manifest.json")
|
manifestProcessed := filepath.Join(paths.ArtifactsDir, "processed.from-manifest.json")
|
||||||
writeFile(t, manifestProcessed, `{"segments":[{"id":10}]}`)
|
writeFile(t, manifestProcessed, `{"segments":[{"id":10}]}`)
|
||||||
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[{"id":99}]}`)
|
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[{"id":99}]}`)
|
||||||
@@ -43,7 +43,7 @@ func TestNormalizeStageConsumesProcessedTranscriptFromManifest(t *testing.T) {
|
|||||||
|
|
||||||
func TestNormalizeStageFallsBackToProcessedTranscriptPath(t *testing.T) {
|
func TestNormalizeStageFallsBackToProcessedTranscriptPath(t *testing.T) {
|
||||||
env, m, ser := setupNormalizeEnv(t)
|
env, m, ser := setupNormalizeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
fallback := filepath.Join(paths.TranscriptsDir, "processed.json")
|
fallback := filepath.Join(paths.TranscriptsDir, "processed.json")
|
||||||
writeFile(t, fallback, `{"segments":[{"id":1}]}`)
|
writeFile(t, fallback, `{"segments":[{"id":1}]}`)
|
||||||
|
|
||||||
@@ -72,7 +72,7 @@ func TestNormalizeStageFailsWhenProcessedTranscriptMissing(t *testing.T) {
|
|||||||
|
|
||||||
func TestNormalizeStageFailsWhenProcessedTranscriptInvalidJSON(t *testing.T) {
|
func TestNormalizeStageFailsWhenProcessedTranscriptInvalidJSON(t *testing.T) {
|
||||||
env, m, _ := setupNormalizeEnv(t)
|
env, m, _ := setupNormalizeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), "not-json")
|
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), "not-json")
|
||||||
|
|
||||||
_, err := (normalizeStage{}).Run(context.Background(), env, m)
|
_, err := (normalizeStage{}).Run(context.Background(), env, m)
|
||||||
@@ -86,7 +86,7 @@ func TestNormalizeStageFailsWhenProcessedTranscriptInvalidJSON(t *testing.T) {
|
|||||||
|
|
||||||
func TestNormalizeStageFailsWhenProcessedTranscriptMissingSegments(t *testing.T) {
|
func TestNormalizeStageFailsWhenProcessedTranscriptMissingSegments(t *testing.T) {
|
||||||
env, m, _ := setupNormalizeEnv(t)
|
env, m, _ := setupNormalizeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"schema":"audita.processed.v1"}`)
|
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"schema":"audita.processed.v1"}`)
|
||||||
|
|
||||||
_, err := (normalizeStage{}).Run(context.Background(), env, m)
|
_, err := (normalizeStage{}).Run(context.Background(), env, m)
|
||||||
@@ -100,7 +100,7 @@ func TestNormalizeStageFailsWhenProcessedTranscriptMissingSegments(t *testing.T)
|
|||||||
|
|
||||||
func TestNormalizeStagePassesConfiguredOutputSchemaToAdapter(t *testing.T) {
|
func TestNormalizeStagePassesConfiguredOutputSchemaToAdapter(t *testing.T) {
|
||||||
env, m, ser := setupNormalizeEnv(t)
|
env, m, ser := setupNormalizeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[{"id":1}]}`)
|
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[{"id":1}]}`)
|
||||||
env.Config.Pipeline.Normalize.OutputSchema = "seriatim-full"
|
env.Config.Pipeline.Normalize.OutputSchema = "seriatim-full"
|
||||||
|
|
||||||
@@ -121,7 +121,7 @@ func TestNormalizeStagePassesConfiguredOutputSchemaToAdapter(t *testing.T) {
|
|||||||
|
|
||||||
func TestNormalizeStageRecordsNormalizedTranscriptOutputKind(t *testing.T) {
|
func TestNormalizeStageRecordsNormalizedTranscriptOutputKind(t *testing.T) {
|
||||||
env, m, _ := setupNormalizeEnv(t)
|
env, m, _ := setupNormalizeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[{"id":1}]}`)
|
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[{"id":1}]}`)
|
||||||
|
|
||||||
result, err := (normalizeStage{}).Run(context.Background(), env, m)
|
result, err := (normalizeStage{}).Run(context.Background(), env, m)
|
||||||
@@ -138,7 +138,7 @@ func TestNormalizeStageRecordsNormalizedTranscriptOutputKind(t *testing.T) {
|
|||||||
|
|
||||||
func TestNormalizeStageRecordsReportLogAndGeneratedConfigRefs(t *testing.T) {
|
func TestNormalizeStageRecordsReportLogAndGeneratedConfigRefs(t *testing.T) {
|
||||||
env, m, _ := setupNormalizeEnv(t)
|
env, m, _ := setupNormalizeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[{"id":1}]}`)
|
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[{"id":1}]}`)
|
||||||
report := true
|
report := true
|
||||||
env.Config.Pipeline.Normalize.Report = &report
|
env.Config.Pipeline.Normalize.Report = &report
|
||||||
@@ -164,7 +164,7 @@ func TestNormalizeStageRecordsReportLogAndGeneratedConfigRefs(t *testing.T) {
|
|||||||
|
|
||||||
func TestNormalizeStageFailsWhenAdapterReturnsError(t *testing.T) {
|
func TestNormalizeStageFailsWhenAdapterReturnsError(t *testing.T) {
|
||||||
env, m, ser := setupNormalizeEnv(t)
|
env, m, ser := setupNormalizeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[{"id":1}]}`)
|
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[{"id":1}]}`)
|
||||||
ser.NormalizeErr = errors.New("normalize failed")
|
ser.NormalizeErr = errors.New("normalize failed")
|
||||||
|
|
||||||
@@ -179,7 +179,7 @@ func TestNormalizeStageFailsWhenAdapterReturnsError(t *testing.T) {
|
|||||||
|
|
||||||
func TestNormalizeStageFailsWhenNormalizedOutputInvalid(t *testing.T) {
|
func TestNormalizeStageFailsWhenNormalizedOutputInvalid(t *testing.T) {
|
||||||
env, m, ser := setupNormalizeEnv(t)
|
env, m, ser := setupNormalizeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[{"id":1}]}`)
|
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[{"id":1}]}`)
|
||||||
badOutput := filepath.Join(paths.TranscriptsDir, "normalized.bad.json")
|
badOutput := filepath.Join(paths.TranscriptsDir, "normalized.bad.json")
|
||||||
writeFile(t, badOutput, "not-json")
|
writeFile(t, badOutput, "not-json")
|
||||||
@@ -196,7 +196,7 @@ func TestNormalizeStageFailsWhenNormalizedOutputInvalid(t *testing.T) {
|
|||||||
|
|
||||||
func TestNormalizeStageReportEnabledFailsWhenReportMissing(t *testing.T) {
|
func TestNormalizeStageReportEnabledFailsWhenReportMissing(t *testing.T) {
|
||||||
env, m, ser := setupNormalizeEnv(t)
|
env, m, ser := setupNormalizeEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[{"id":1}]}`)
|
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[{"id":1}]}`)
|
||||||
report := true
|
report := true
|
||||||
env.Config.Pipeline.Normalize.Report = &report
|
env.Config.Pipeline.Normalize.Report = &report
|
||||||
@@ -211,6 +211,33 @@ func TestNormalizeStageReportEnabledFailsWhenReportMissing(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestNormalizeStageUsesRunLocalPathsAndPromotesCanonical(t *testing.T) {
|
||||||
|
env, m, ser := setupNormalizeEnv(t)
|
||||||
|
env.Config.Session.Campaign = "sample-campaign"
|
||||||
|
m.Campaign = "sample-campaign"
|
||||||
|
m.RunID = "20260518T010203Z-abcdef12"
|
||||||
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
|
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[{"id":1}]}`)
|
||||||
|
|
||||||
|
result, err := (normalizeStage{}).Run(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("normalize.Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(ser.NormalizeRequests) != 1 {
|
||||||
|
t.Fatalf("normalize requests = %d, want 1", len(ser.NormalizeRequests))
|
||||||
|
}
|
||||||
|
req := ser.NormalizeRequests[0]
|
||||||
|
if !strings.Contains(req.OutputNormalizedPath, filepath.Join("runs", m.RunID, "normalize", "outputs")) {
|
||||||
|
t.Fatalf("run output path = %q, want run-local path", req.OutputNormalizedPath)
|
||||||
|
}
|
||||||
|
if len(result.Outputs) == 0 {
|
||||||
|
t.Fatalf("outputs = %#v, want promoted outputs", result.Outputs)
|
||||||
|
}
|
||||||
|
if strings.Contains(result.Outputs[0].AbsolutePath, string(filepath.Separator)+"runs"+string(filepath.Separator)) {
|
||||||
|
t.Fatalf("promoted output path = %q, want canonical session path", result.Outputs[0].AbsolutePath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func setupNormalizeEnv(t *testing.T) (*Env, *manifest.Manifest, *seriatim.FakeRunner) {
|
func setupNormalizeEnv(t *testing.T) (*Env, *manifest.Manifest, *seriatim.FakeRunner) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
workspace := t.TempDir()
|
workspace := t.TempDir()
|
||||||
@@ -243,7 +270,7 @@ func setupNormalizeEnv(t *testing.T) (*Env, *manifest.Manifest, *seriatim.FakeRu
|
|||||||
}
|
}
|
||||||
|
|
||||||
store := artifacts.NewLocalStore(workspace)
|
store := artifacts.NewLocalStore(workspace)
|
||||||
if _, err := store.EnsureLayout("2026-05-03"); err != nil {
|
if _, err := store.EnsureLayoutFor("sample-campaign", "2026-05-03"); err != nil {
|
||||||
t.Fatalf("EnsureLayout() error = %v", err)
|
t.Fatalf("EnsureLayout() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -3,10 +3,8 @@ package stage
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"path/filepath"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/notify"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/notify"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
)
|
)
|
||||||
@@ -47,25 +45,7 @@ func (s placeholderStage) Run(ctx context.Context, env *Env, m *manifest.Manifes
|
|||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
paths := env.ArtifactStore.SessionPaths(sessionID)
|
|
||||||
|
|
||||||
switch s.name {
|
switch s.name {
|
||||||
case "archive":
|
|
||||||
if env.Storage != nil {
|
|
||||||
req := storage.ArchiveRequest{
|
|
||||||
SessionID: sessionID,
|
|
||||||
ManifestPath: paths.ManifestPath,
|
|
||||||
Items: []storage.ArchiveItem{{
|
|
||||||
Kind: "artifact",
|
|
||||||
LocalPath: filepath.Join(paths.ArtifactsDir, "session-log.md"),
|
|
||||||
RemoteKey: "sessions/" + sessionID + "/artifacts/session-log.md",
|
|
||||||
}},
|
|
||||||
}
|
|
||||||
_, err := env.Storage.Archive(ctx, req)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("placeholder archive adapter call failed: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
case "notify":
|
case "notify":
|
||||||
if env.Notifier != nil {
|
if env.Notifier != nil {
|
||||||
req := notify.SendRequest{
|
req := notify.SendRequest{
|
||||||
@@ -95,7 +75,7 @@ func All() []Stage {
|
|||||||
normalizeStage{},
|
normalizeStage{},
|
||||||
trimStage{},
|
trimStage{},
|
||||||
analyzeStage{},
|
analyzeStage{},
|
||||||
placeholderStage{name: "archive"},
|
archiveStage{},
|
||||||
placeholderStage{name: "notify"},
|
placeholderStage{name: "notify"},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -49,9 +49,22 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
|
|||||||
Config: &config.Config{
|
Config: &config.Config{
|
||||||
SessionPath: sessionPath,
|
SessionPath: sessionPath,
|
||||||
PipelinePath: pipelinePath,
|
PipelinePath: pipelinePath,
|
||||||
Pipeline: &config.PipelineConfig{Workspace: config.WorkspaceConfig{Root: root}},
|
Pipeline: &config.PipelineConfig{
|
||||||
|
Workspace: config.WorkspaceConfig{Root: root},
|
||||||
|
Storage: config.StorageConfig{
|
||||||
|
S3: &config.StorageS3Config{
|
||||||
|
Bucket: "my-dnd-archive",
|
||||||
|
RootPrefix: "dnd",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Archive: &config.ArchiveConfig{
|
||||||
|
Enabled: boolPtr(true),
|
||||||
|
UploadRun: boolPtr(true),
|
||||||
|
},
|
||||||
|
},
|
||||||
Session: &config.SessionConfig{
|
Session: &config.SessionConfig{
|
||||||
SessionID: "2026-05-03",
|
SessionID: "2026-05-03",
|
||||||
|
Campaign: "sample-campaign",
|
||||||
Inputs: config.SessionInputsConfig{
|
Inputs: config.SessionInputsConfig{
|
||||||
AudioDir: "./audio",
|
AudioDir: "./audio",
|
||||||
SpeakersFile: "./speakers.yml",
|
SpeakersFile: "./speakers.yml",
|
||||||
@@ -66,10 +79,24 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
|
|||||||
Audita: af,
|
Audita: af,
|
||||||
Scriptorium: sc,
|
Scriptorium: sc,
|
||||||
Storage: st,
|
Storage: st,
|
||||||
|
ObjectStore: st,
|
||||||
Notifier: nf,
|
Notifier: nf,
|
||||||
}
|
}
|
||||||
|
|
||||||
m := manifest.New("2026-05-03", time.Now().UTC())
|
m := manifest.New("2026-05-03", time.Now().UTC())
|
||||||
|
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze"} {
|
||||||
|
m.MarkStageSucceeded(name, time.Now().UTC(), nil)
|
||||||
|
}
|
||||||
|
m.RunID = "20260516T000000Z-abcdef12"
|
||||||
|
m.Campaign = "sample-campaign"
|
||||||
|
m.LocalWorkDir = artifacts.SessionRunRootForCampaign(root, "sample-campaign", "2026-05-03", m.RunID)
|
||||||
|
m.S3RunPrefix = "dnd/campaigns/sample-campaign/sessions/2026-05-03/runs/" + m.RunID + "/"
|
||||||
|
m.S3Bucket = "my-dnd-archive"
|
||||||
|
if err := os.MkdirAll(filepath.Join(m.LocalWorkDir, "inputs"), 0o755); err != nil {
|
||||||
|
t.Fatalf("mkdir workdir inputs: %v", err)
|
||||||
|
}
|
||||||
|
writeStageTestFile(t, filepath.Join(m.LocalWorkDir, "inputs", "session.yml"), "session_id: 2026-05-03\n")
|
||||||
|
writeStageTestFile(t, filepath.Join(m.LocalWorkDir, "manifest.json"), "{}\n")
|
||||||
for _, s := range stages {
|
for _, s := range stages {
|
||||||
result, err := s.Run(context.Background(), env, m)
|
result, err := s.Run(context.Background(), env, m)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -150,6 +177,15 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
|
|||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
if s.Name() == "archive" {
|
||||||
|
if result.Metadata["stage"] != "archive" {
|
||||||
|
t.Fatalf("archive metadata = %#v, want stage=archive", result.Metadata)
|
||||||
|
}
|
||||||
|
if result.Metadata["uploaded"] != true {
|
||||||
|
t.Fatalf("archive metadata = %#v, want uploaded=true", result.Metadata)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
if result.Metadata["placeholder"] != true {
|
if result.Metadata["placeholder"] != true {
|
||||||
t.Fatalf("stage %q missing placeholder metadata", s.Name())
|
t.Fatalf("stage %q missing placeholder metadata", s.Name())
|
||||||
}
|
}
|
||||||
@@ -167,8 +203,11 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
|
|||||||
if len(sc.RunRequests) != 0 {
|
if len(sc.RunRequests) != 0 {
|
||||||
t.Fatalf("scriptorium run calls = %d, want 0 when scriptorium config is absent", len(sc.RunRequests))
|
t.Fatalf("scriptorium run calls = %d, want 0 when scriptorium config is absent", len(sc.RunRequests))
|
||||||
}
|
}
|
||||||
if len(st.Requests) != 1 {
|
if len(st.Requests) != 0 {
|
||||||
t.Fatalf("storage calls = %d, want 1", len(st.Requests))
|
t.Fatalf("storage archive calls = %d, want 0", len(st.Requests))
|
||||||
|
}
|
||||||
|
if _, ok := st.Objects["dnd/campaigns/sample-campaign/sessions/2026-05-03/runs/"+m.RunID+"/manifest.json"]; !ok {
|
||||||
|
t.Fatalf("archive upload missing manifest key in fake object store")
|
||||||
}
|
}
|
||||||
if len(nf.Requests) != 1 {
|
if len(nf.Requests) != 1 {
|
||||||
t.Fatalf("notify calls = %d, want 1", len(nf.Requests))
|
t.Fatalf("notify calls = %d, want 1", len(nf.Requests))
|
||||||
@@ -181,13 +220,12 @@ func TestPlaceholderAdapterErrorPropagation(t *testing.T) {
|
|||||||
env *Env
|
env *Env
|
||||||
wantErr string
|
wantErr string
|
||||||
}{
|
}{
|
||||||
{stageName: "archive", env: &Env{Storage: &storage.FakeBackend{Err: errors.New("sterr")}}, wantErr: "archive"},
|
|
||||||
{stageName: "notify", env: &Env{Notifier: ¬ify.FakeSender{Err: errors.New("nerr")}}, wantErr: "notify"},
|
{stageName: "notify", env: &Env{Notifier: ¬ify.FakeSender{Err: errors.New("nerr")}}, wantErr: "notify"},
|
||||||
}
|
}
|
||||||
|
|
||||||
root := t.TempDir()
|
root := t.TempDir()
|
||||||
store := artifacts.NewLocalStore(root)
|
store := artifacts.NewLocalStore(root)
|
||||||
_, err := store.EnsureLayout("2026-05-03")
|
_, err := store.EnsureLayoutFor("sample-campaign", "2026-05-03")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("EnsureLayout() error = %v", err)
|
t.Fatalf("EnsureLayout() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -55,7 +55,11 @@ func (polishStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*St
|
|||||||
return nil, fmt.Errorf("polish: session id is required")
|
return nil, fmt.Errorf("polish: session id is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
paths := env.ArtifactStore.SessionPaths(sessionID)
|
paths := sessionPathsForEnv(env, sessionID)
|
||||||
|
runLayout, err := resolveRunStageLayout(env, m, paths, sessionID, "polish")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("polish: resolve run-stage layout: %w", err)
|
||||||
|
}
|
||||||
mergedPath, source, err := discoverMergedTranscript(m, paths)
|
mergedPath, source, err := discoverMergedTranscript(m, paths)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("polish: resolve merged transcript: %w", err)
|
return nil, fmt.Errorf("polish: resolve merged transcript: %w", err)
|
||||||
@@ -72,12 +76,29 @@ func (polishStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*St
|
|||||||
return nil, fmt.Errorf("polish: %w", err)
|
return nil, fmt.Errorf("polish: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
processedPath := filepath.Join(paths.TranscriptsDir, "processed.json")
|
canonicalProcessedPath := filepath.Join(paths.TranscriptsDir, "processed.json")
|
||||||
reportPath := filepath.Join(paths.ArtifactsDir, "audita.report.json")
|
processedPath, err := runLocalPathForCanonical(runLayout, paths, canonicalProcessedPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("polish: resolve run-local processed transcript path: %w", err)
|
||||||
|
}
|
||||||
|
canonicalReportPath := filepath.Join(paths.ArtifactsDir, "audita.report.json")
|
||||||
|
reportPath := canonicalReportPath
|
||||||
|
if runLayout.Enabled {
|
||||||
|
reportPath, err = runLocalPathForCanonical(runLayout, paths, canonicalReportPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("polish: resolve run-local report path: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
workDir := filepath.Join(paths.ArtifactsDir, "audita-work")
|
workDir := filepath.Join(paths.ArtifactsDir, "audita-work")
|
||||||
stdoutPath := filepath.Join(paths.LogsDir, "audita.stdout.log")
|
stdoutPath := filepath.Join(paths.LogsDir, "audita.stdout.log")
|
||||||
stderrPath := filepath.Join(paths.LogsDir, "audita.stderr.log")
|
stderrPath := filepath.Join(paths.LogsDir, "audita.stderr.log")
|
||||||
generatedConfigPath := filepath.Join(paths.ConfigDir, "audita.generated.yml")
|
generatedConfigPath := filepath.Join(paths.ConfigDir, "audita.generated.yml")
|
||||||
|
if runLayout.Enabled {
|
||||||
|
workDir = filepath.Join(runLayout.ScratchDir, "audita-work")
|
||||||
|
stdoutPath = filepath.Join(runLayout.LogsDir, "audita.stdout.log")
|
||||||
|
stderrPath = filepath.Join(runLayout.LogsDir, "audita.stderr.log")
|
||||||
|
generatedConfigPath = filepath.Join(runLayout.ConfigDir, "audita.generated.yml")
|
||||||
|
}
|
||||||
|
|
||||||
reportEnabled := env.Config.Pipeline.Audita.Report != nil && *env.Config.Pipeline.Audita.Report
|
reportEnabled := env.Config.Pipeline.Audita.Report != nil && *env.Config.Pipeline.Audita.Report
|
||||||
req := audita.PolishRequest{
|
req := audita.PolishRequest{
|
||||||
@@ -90,6 +111,12 @@ func (polishStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*St
|
|||||||
Modules: append([]string(nil), env.Config.Pipeline.Audita.Modules...),
|
Modules: append([]string(nil), env.Config.Pipeline.Audita.Modules...),
|
||||||
BaseURL: env.Config.Pipeline.Audita.BaseURL,
|
BaseURL: env.Config.Pipeline.Audita.BaseURL,
|
||||||
Model: env.Config.Pipeline.Audita.Model,
|
Model: env.Config.Pipeline.Audita.Model,
|
||||||
|
TranscriptDescription: env.Config.Pipeline.Audita.TranscriptDescription,
|
||||||
|
ConfigPath: env.Config.Pipeline.Audita.ConfigPath,
|
||||||
|
OutputSchema: env.Config.Pipeline.Audita.OutputSchema,
|
||||||
|
WorkDirRetention: env.Config.Pipeline.Audita.WorkDirRetention,
|
||||||
|
TotalLLMConcurrency: env.Config.Pipeline.Audita.TotalLLMConcurrency,
|
||||||
|
ProposalLLMConcurrency: env.Config.Pipeline.Audita.ProposalLLMConcurrency,
|
||||||
ValidationModel: env.Config.Pipeline.Audita.ValidationModel,
|
ValidationModel: env.Config.Pipeline.Audita.ValidationModel,
|
||||||
ValidationLLMConcurrency: env.Config.Pipeline.Audita.ValidationLLMConcurrency,
|
ValidationLLMConcurrency: env.Config.Pipeline.Audita.ValidationLLMConcurrency,
|
||||||
StdoutLogPath: stdoutPath,
|
StdoutLogPath: stdoutPath,
|
||||||
@@ -122,63 +149,83 @@ func (polishStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*St
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
outputs := []artifacts.Ref{{
|
promotedProcessed, err := promoteRunLocalOutput(env.ArtifactStore, finalProcessedPath, canonicalProcessedPath, artifacts.Ref{
|
||||||
Kind: "transcript_processed",
|
Kind: "transcript_processed",
|
||||||
Category: "transcripts",
|
Category: "transcripts",
|
||||||
SessionID: sessionID,
|
SessionID: sessionID,
|
||||||
AbsolutePath: finalProcessedPath,
|
})
|
||||||
}}
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("polish: promote processed transcript: %w", err)
|
||||||
|
}
|
||||||
|
outputs := []artifacts.Ref{promotedProcessed}
|
||||||
if reportEnabled {
|
if reportEnabled {
|
||||||
outputs = append(outputs, artifacts.Ref{
|
promotedReport, err := promoteRunLocalOutput(env.ArtifactStore, finalReportPath, canonicalReportPath, artifacts.Ref{
|
||||||
Kind: "audita_report",
|
Kind: "audita_report",
|
||||||
Category: "artifacts",
|
Category: "artifacts",
|
||||||
SessionID: sessionID,
|
SessionID: sessionID,
|
||||||
AbsolutePath: finalReportPath,
|
|
||||||
})
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("polish: promote report: %w", err)
|
||||||
|
}
|
||||||
|
outputs = append(outputs, promotedReport)
|
||||||
}
|
}
|
||||||
|
|
||||||
var validationConcurrency any
|
var validationConcurrency any
|
||||||
if env.Config.Pipeline.Audita.ValidationLLMConcurrency != nil {
|
if env.Config.Pipeline.Audita.ValidationLLMConcurrency != nil {
|
||||||
validationConcurrency = *env.Config.Pipeline.Audita.ValidationLLMConcurrency
|
validationConcurrency = *env.Config.Pipeline.Audita.ValidationLLMConcurrency
|
||||||
}
|
}
|
||||||
var llmConcurrency any
|
var totalLLMConcurrency any
|
||||||
if env.Config.Pipeline.Audita.LLMConcurrency != nil {
|
if env.Config.Pipeline.Audita.TotalLLMConcurrency != nil {
|
||||||
llmConcurrency = *env.Config.Pipeline.Audita.LLMConcurrency
|
totalLLMConcurrency = *env.Config.Pipeline.Audita.TotalLLMConcurrency
|
||||||
|
}
|
||||||
|
var proposalLLMConcurrency any
|
||||||
|
if env.Config.Pipeline.Audita.ProposalLLMConcurrency != nil {
|
||||||
|
proposalLLMConcurrency = *env.Config.Pipeline.Audita.ProposalLLMConcurrency
|
||||||
|
}
|
||||||
|
reportCanonicalPath := ""
|
||||||
|
if reportEnabled {
|
||||||
|
reportCanonicalPath = canonicalReportPath
|
||||||
}
|
}
|
||||||
|
|
||||||
meta := map[string]any{
|
meta := map[string]any{
|
||||||
"stage": "polish",
|
"stage": "polish",
|
||||||
"merged_transcript_path": mergedPath,
|
"merged_transcript_path": mergedPath,
|
||||||
"merged_transcript_source": source,
|
"merged_transcript_source": source,
|
||||||
"glossary_path": glossaryPath,
|
"glossary_path": glossaryPath,
|
||||||
"output_path": finalProcessedPath,
|
"run_output_path": finalProcessedPath,
|
||||||
"report_path": finalReportPath,
|
"output_path": canonicalProcessedPath,
|
||||||
"audita_work_dir": workDir,
|
"run_report_path": finalReportPath,
|
||||||
"report_enabled": reportEnabled,
|
"report_path": reportCanonicalPath,
|
||||||
"modules": append([]string(nil), req.Modules...),
|
"audita_work_dir": workDir,
|
||||||
"base_url": req.BaseURL,
|
"report_enabled": reportEnabled,
|
||||||
"model": req.Model,
|
"modules": append([]string(nil), req.Modules...),
|
||||||
"validation_model": req.ValidationModel,
|
"base_url": req.BaseURL,
|
||||||
"llm_concurrency": llmConcurrency,
|
"model": req.Model,
|
||||||
"validation_llm_concurrency": validationConcurrency,
|
"transcript_description": req.TranscriptDescription,
|
||||||
"llm_api_key_env": env.Config.Pipeline.Audita.LLMAPIKeyEnv,
|
"config_path": req.ConfigPath,
|
||||||
"timeout": env.Config.Pipeline.Audita.Timeout,
|
"output_schema": req.OutputSchema,
|
||||||
"binary": env.Config.Pipeline.Audita.Binary,
|
"work_dir_retention": req.WorkDirRetention,
|
||||||
"generated_config_path": generatedConfigPath,
|
"validation_model": req.ValidationModel,
|
||||||
"stdout_log_path": stdoutPath,
|
"total_llm_concurrency": totalLLMConcurrency,
|
||||||
"stderr_log_path": stderrPath,
|
"proposal_llm_concurrency": proposalLLMConcurrency,
|
||||||
"adapter_duration_ms": res.Duration.Milliseconds(),
|
"validation_llm_concurrency": validationConcurrency,
|
||||||
"adapter_exit_code": res.ExitCode,
|
"llm_api_key_env": env.Config.Pipeline.Audita.LLMAPIKeyEnv,
|
||||||
"adapter_invoked_binary": res.InvokedBinary,
|
"timeout": env.Config.Pipeline.Audita.Timeout,
|
||||||
"adapter_processed_output_path": res.ProcessedTranscriptPath,
|
"binary": env.Config.Pipeline.Audita.Binary,
|
||||||
"adapter_report_path": res.ReportPath,
|
"generated_config_path": generatedConfigPath,
|
||||||
"adapter_generated_config_path": res.GeneratedConfigPath,
|
"stdout_log_path": stdoutPath,
|
||||||
"adapter_work_dir": res.WorkDir,
|
"stderr_log_path": stderrPath,
|
||||||
"adapter_stdout_log_path": res.StdoutLogPath,
|
"adapter_duration_ms": res.Duration.Milliseconds(),
|
||||||
"adapter_stderr_log_path": res.StderrLogPath,
|
"adapter_exit_code": res.ExitCode,
|
||||||
"credential_env_var": env.Config.Pipeline.Audita.LLMAPIKeyEnv,
|
"adapter_invoked_binary": res.InvokedBinary,
|
||||||
"credential_present": false,
|
"adapter_processed_output_path": res.ProcessedTranscriptPath,
|
||||||
"primary_llm_concurrency_via_env": false,
|
"adapter_report_path": res.ReportPath,
|
||||||
|
"adapter_generated_config_path": res.GeneratedConfigPath,
|
||||||
|
"adapter_work_dir": res.WorkDir,
|
||||||
|
"adapter_stdout_log_path": res.StdoutLogPath,
|
||||||
|
"adapter_stderr_log_path": res.StderrLogPath,
|
||||||
|
"credential_env_var": env.Config.Pipeline.Audita.LLMAPIKeyEnv,
|
||||||
|
"credential_present": false,
|
||||||
}
|
}
|
||||||
if res.Metadata != nil {
|
if res.Metadata != nil {
|
||||||
meta["adapter_metadata"] = res.Metadata
|
meta["adapter_metadata"] = res.Metadata
|
||||||
@@ -188,9 +235,6 @@ func (polishStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*St
|
|||||||
if value, ok := res.Metadata["credential_env_var"]; ok {
|
if value, ok := res.Metadata["credential_env_var"]; ok {
|
||||||
meta["credential_env_var"] = value
|
meta["credential_env_var"] = value
|
||||||
}
|
}
|
||||||
if value, ok := res.Metadata["primary_llm_concurrency_via_env"]; ok {
|
|
||||||
meta["primary_llm_concurrency_via_env"] = value
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return &StageResult{
|
return &StageResult{
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ import (
|
|||||||
|
|
||||||
func TestPolishStagePolishesMergedTranscriptAndRecordsMetadata(t *testing.T) {
|
func TestPolishStagePolishesMergedTranscriptAndRecordsMetadata(t *testing.T) {
|
||||||
env, m := setupPolishEnv(t)
|
env, m := setupPolishEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
|
|
||||||
mergedPath := filepath.Join(paths.TranscriptsDir, "merged.json")
|
mergedPath := filepath.Join(paths.TranscriptsDir, "merged.json")
|
||||||
writeFile(t, mergedPath, `{"segments":[]}`)
|
writeFile(t, mergedPath, `{"segments":[]}`)
|
||||||
@@ -60,6 +60,24 @@ func TestPolishStagePolishesMergedTranscriptAndRecordsMetadata(t *testing.T) {
|
|||||||
if req.ValidationModel != "openrouter/google/gemma-4-31b-it" {
|
if req.ValidationModel != "openrouter/google/gemma-4-31b-it" {
|
||||||
t.Fatalf("validation model = %q", req.ValidationModel)
|
t.Fatalf("validation model = %q", req.ValidationModel)
|
||||||
}
|
}
|
||||||
|
if req.TranscriptDescription != "Campaign Session 42" {
|
||||||
|
t.Fatalf("transcript description = %q", req.TranscriptDescription)
|
||||||
|
}
|
||||||
|
if req.ConfigPath != "/etc/audita/config.yml" {
|
||||||
|
t.Fatalf("config path = %q", req.ConfigPath)
|
||||||
|
}
|
||||||
|
if req.OutputSchema != "audita-v1" {
|
||||||
|
t.Fatalf("output schema = %q", req.OutputSchema)
|
||||||
|
}
|
||||||
|
if req.WorkDirRetention != "auto" {
|
||||||
|
t.Fatalf("work dir retention = %q", req.WorkDirRetention)
|
||||||
|
}
|
||||||
|
if req.TotalLLMConcurrency == nil || *req.TotalLLMConcurrency != 3 {
|
||||||
|
t.Fatalf("total llm concurrency = %#v, want 3", req.TotalLLMConcurrency)
|
||||||
|
}
|
||||||
|
if req.ProposalLLMConcurrency == nil || *req.ProposalLLMConcurrency != 2 {
|
||||||
|
t.Fatalf("proposal llm concurrency = %#v, want 2", req.ProposalLLMConcurrency)
|
||||||
|
}
|
||||||
if req.ValidationLLMConcurrency == nil || *req.ValidationLLMConcurrency != 2 {
|
if req.ValidationLLMConcurrency == nil || *req.ValidationLLMConcurrency != 2 {
|
||||||
t.Fatalf("validation llm concurrency = %#v, want 2", req.ValidationLLMConcurrency)
|
t.Fatalf("validation llm concurrency = %#v, want 2", req.ValidationLLMConcurrency)
|
||||||
}
|
}
|
||||||
@@ -89,11 +107,20 @@ func TestPolishStagePolishesMergedTranscriptAndRecordsMetadata(t *testing.T) {
|
|||||||
if result.Metadata["audita_work_dir"] != filepath.Join(paths.ArtifactsDir, "audita-work") {
|
if result.Metadata["audita_work_dir"] != filepath.Join(paths.ArtifactsDir, "audita-work") {
|
||||||
t.Fatalf("metadata audita_work_dir = %#v", result.Metadata["audita_work_dir"])
|
t.Fatalf("metadata audita_work_dir = %#v", result.Metadata["audita_work_dir"])
|
||||||
}
|
}
|
||||||
|
if result.Metadata["total_llm_concurrency"] != 3 {
|
||||||
|
t.Fatalf("metadata total_llm_concurrency = %#v, want 3", result.Metadata["total_llm_concurrency"])
|
||||||
|
}
|
||||||
|
if result.Metadata["proposal_llm_concurrency"] != 2 {
|
||||||
|
t.Fatalf("metadata proposal_llm_concurrency = %#v, want 2", result.Metadata["proposal_llm_concurrency"])
|
||||||
|
}
|
||||||
|
if result.Metadata["output_schema"] != "audita-v1" {
|
||||||
|
t.Fatalf("metadata output_schema = %#v, want audita-v1", result.Metadata["output_schema"])
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPolishStageFallsBackToMergedTranscriptPath(t *testing.T) {
|
func TestPolishStageFallsBackToMergedTranscriptPath(t *testing.T) {
|
||||||
env, m := setupPolishEnv(t)
|
env, m := setupPolishEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
mergedPath := filepath.Join(paths.TranscriptsDir, "merged.json")
|
mergedPath := filepath.Join(paths.TranscriptsDir, "merged.json")
|
||||||
writeFile(t, mergedPath, `{"segments":[]}`)
|
writeFile(t, mergedPath, `{"segments":[]}`)
|
||||||
writeFile(t, filepath.Join(paths.InputsDir, "glossary.yml"), "terms: []\n")
|
writeFile(t, filepath.Join(paths.InputsDir, "glossary.yml"), "terms: []\n")
|
||||||
@@ -115,7 +142,7 @@ func TestPolishStageFallsBackToMergedTranscriptPath(t *testing.T) {
|
|||||||
|
|
||||||
func TestPolishStageFailsWhenMergedTranscriptMissing(t *testing.T) {
|
func TestPolishStageFailsWhenMergedTranscriptMissing(t *testing.T) {
|
||||||
env, m := setupPolishEnv(t)
|
env, m := setupPolishEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeFile(t, filepath.Join(paths.InputsDir, "glossary.yml"), "terms: []\n")
|
writeFile(t, filepath.Join(paths.InputsDir, "glossary.yml"), "terms: []\n")
|
||||||
env.Audita = &audita.FakeRunner{}
|
env.Audita = &audita.FakeRunner{}
|
||||||
|
|
||||||
@@ -130,7 +157,7 @@ func TestPolishStageFailsWhenMergedTranscriptMissing(t *testing.T) {
|
|||||||
|
|
||||||
func TestPolishStageFailsWhenMergedTranscriptInvalidJSON(t *testing.T) {
|
func TestPolishStageFailsWhenMergedTranscriptInvalidJSON(t *testing.T) {
|
||||||
env, m := setupPolishEnv(t)
|
env, m := setupPolishEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeFile(t, filepath.Join(paths.TranscriptsDir, "merged.json"), "not-json")
|
writeFile(t, filepath.Join(paths.TranscriptsDir, "merged.json"), "not-json")
|
||||||
writeFile(t, filepath.Join(paths.InputsDir, "glossary.yml"), "terms: []\n")
|
writeFile(t, filepath.Join(paths.InputsDir, "glossary.yml"), "terms: []\n")
|
||||||
env.Audita = &audita.FakeRunner{}
|
env.Audita = &audita.FakeRunner{}
|
||||||
@@ -146,7 +173,7 @@ func TestPolishStageFailsWhenMergedTranscriptInvalidJSON(t *testing.T) {
|
|||||||
|
|
||||||
func TestPolishStageFailsWhenGlossaryMissing(t *testing.T) {
|
func TestPolishStageFailsWhenGlossaryMissing(t *testing.T) {
|
||||||
env, m := setupPolishEnv(t)
|
env, m := setupPolishEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeFile(t, filepath.Join(paths.TranscriptsDir, "merged.json"), `{"segments":[]}`)
|
writeFile(t, filepath.Join(paths.TranscriptsDir, "merged.json"), `{"segments":[]}`)
|
||||||
env.Audita = &audita.FakeRunner{}
|
env.Audita = &audita.FakeRunner{}
|
||||||
|
|
||||||
@@ -161,7 +188,7 @@ func TestPolishStageFailsWhenGlossaryMissing(t *testing.T) {
|
|||||||
|
|
||||||
func TestPolishStageFailsWhenAdapterFails(t *testing.T) {
|
func TestPolishStageFailsWhenAdapterFails(t *testing.T) {
|
||||||
env, m := setupPolishEnv(t)
|
env, m := setupPolishEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeFile(t, filepath.Join(paths.TranscriptsDir, "merged.json"), `{"segments":[]}`)
|
writeFile(t, filepath.Join(paths.TranscriptsDir, "merged.json"), `{"segments":[]}`)
|
||||||
writeFile(t, filepath.Join(paths.InputsDir, "glossary.yml"), "terms: []\n")
|
writeFile(t, filepath.Join(paths.InputsDir, "glossary.yml"), "terms: []\n")
|
||||||
env.Audita = &audita.FakeRunner{Err: errors.New("audita failed")}
|
env.Audita = &audita.FakeRunner{Err: errors.New("audita failed")}
|
||||||
@@ -177,7 +204,7 @@ func TestPolishStageFailsWhenAdapterFails(t *testing.T) {
|
|||||||
|
|
||||||
func TestPolishStageFailsWhenProcessedOutputInvalid(t *testing.T) {
|
func TestPolishStageFailsWhenProcessedOutputInvalid(t *testing.T) {
|
||||||
env, m := setupPolishEnv(t)
|
env, m := setupPolishEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeFile(t, filepath.Join(paths.TranscriptsDir, "merged.json"), `{"segments":[]}`)
|
writeFile(t, filepath.Join(paths.TranscriptsDir, "merged.json"), `{"segments":[]}`)
|
||||||
writeFile(t, filepath.Join(paths.InputsDir, "glossary.yml"), "terms: []\n")
|
writeFile(t, filepath.Join(paths.InputsDir, "glossary.yml"), "terms: []\n")
|
||||||
badPath := filepath.Join(paths.TranscriptsDir, "processed.invalid.json")
|
badPath := filepath.Join(paths.TranscriptsDir, "processed.invalid.json")
|
||||||
@@ -195,7 +222,7 @@ func TestPolishStageFailsWhenProcessedOutputInvalid(t *testing.T) {
|
|||||||
|
|
||||||
func TestPolishStageFailsWhenReportInvalid(t *testing.T) {
|
func TestPolishStageFailsWhenReportInvalid(t *testing.T) {
|
||||||
env, m := setupPolishEnv(t)
|
env, m := setupPolishEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeFile(t, filepath.Join(paths.TranscriptsDir, "merged.json"), `{"segments":[]}`)
|
writeFile(t, filepath.Join(paths.TranscriptsDir, "merged.json"), `{"segments":[]}`)
|
||||||
writeFile(t, filepath.Join(paths.InputsDir, "glossary.yml"), "terms: []\n")
|
writeFile(t, filepath.Join(paths.InputsDir, "glossary.yml"), "terms: []\n")
|
||||||
badReport := filepath.Join(paths.ArtifactsDir, "bad.report.json")
|
badReport := filepath.Join(paths.ArtifactsDir, "bad.report.json")
|
||||||
@@ -211,6 +238,36 @@ func TestPolishStageFailsWhenReportInvalid(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPolishStageUsesRunLocalPathsAndPromotesCanonical(t *testing.T) {
|
||||||
|
env, m := setupPolishEnv(t)
|
||||||
|
env.Config.Session.Campaign = "sample-campaign"
|
||||||
|
m.Campaign = "sample-campaign"
|
||||||
|
m.RunID = "20260518T010203Z-abcdef12"
|
||||||
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
|
writeFile(t, filepath.Join(paths.TranscriptsDir, "merged.json"), `{"segments":[]}`)
|
||||||
|
writeFile(t, filepath.Join(paths.InputsDir, "glossary.yml"), "terms: []\n")
|
||||||
|
|
||||||
|
fake := &audita.FakeRunner{}
|
||||||
|
env.Audita = fake
|
||||||
|
result, err := (polishStage{}).Run(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("polish.Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(fake.Requests) != 1 {
|
||||||
|
t.Fatalf("requests = %d, want 1", len(fake.Requests))
|
||||||
|
}
|
||||||
|
req := fake.Requests[0]
|
||||||
|
if !strings.Contains(req.OutputProcessedPath, filepath.Join("runs", m.RunID, "polish", "outputs")) {
|
||||||
|
t.Fatalf("run output path = %q, want run-local path", req.OutputProcessedPath)
|
||||||
|
}
|
||||||
|
if len(result.Outputs) == 0 {
|
||||||
|
t.Fatalf("outputs = %#v, want promoted outputs", result.Outputs)
|
||||||
|
}
|
||||||
|
if strings.Contains(result.Outputs[0].AbsolutePath, string(filepath.Separator)+"runs"+string(filepath.Separator)) {
|
||||||
|
t.Fatalf("promoted output path = %q, want canonical session path", result.Outputs[0].AbsolutePath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func setupPolishEnv(t *testing.T) (*Env, *manifest.Manifest) {
|
func setupPolishEnv(t *testing.T) (*Env, *manifest.Manifest) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
workspace := t.TempDir()
|
workspace := t.TempDir()
|
||||||
@@ -221,7 +278,8 @@ func setupPolishEnv(t *testing.T) (*Env, *manifest.Manifest) {
|
|||||||
writeFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n")
|
writeFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n")
|
||||||
|
|
||||||
report := true
|
report := true
|
||||||
llmConcurrency := 1
|
totalLLMConcurrency := 3
|
||||||
|
proposalLLMConcurrency := 2
|
||||||
validationLLMConcurrency := 2
|
validationLLMConcurrency := 2
|
||||||
|
|
||||||
cfg := &config.Config{
|
cfg := &config.Config{
|
||||||
@@ -236,8 +294,13 @@ func setupPolishEnv(t *testing.T) (*Env, *manifest.Manifest) {
|
|||||||
Modules: []string{"glossary", "homophones", "grammar"},
|
Modules: []string{"glossary", "homophones", "grammar"},
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
|
TranscriptDescription: "Campaign Session 42",
|
||||||
|
ConfigPath: "/etc/audita/config.yml",
|
||||||
|
OutputSchema: "audita-v1",
|
||||||
|
WorkDirRetention: "auto",
|
||||||
ValidationModel: "openrouter/google/gemma-4-31b-it",
|
ValidationModel: "openrouter/google/gemma-4-31b-it",
|
||||||
LLMConcurrency: &llmConcurrency,
|
TotalLLMConcurrency: &totalLLMConcurrency,
|
||||||
|
ProposalLLMConcurrency: &proposalLLMConcurrency,
|
||||||
ValidationLLMConcurrency: &validationLLMConcurrency,
|
ValidationLLMConcurrency: &validationLLMConcurrency,
|
||||||
Report: &report,
|
Report: &report,
|
||||||
},
|
},
|
||||||
@@ -251,7 +314,7 @@ func setupPolishEnv(t *testing.T) (*Env, *manifest.Manifest) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
store := artifacts.NewLocalStore(workspace)
|
store := artifacts.NewLocalStore(workspace)
|
||||||
if _, err := store.EnsureLayout("2026-05-03"); err != nil {
|
if _, err := store.EnsureLayoutFor("sample-campaign", "2026-05-03"); err != nil {
|
||||||
t.Fatalf("EnsureLayout() error = %v", err)
|
t.Fatalf("EnsureLayout() error = %v", err)
|
||||||
}
|
}
|
||||||
return &Env{
|
return &Env{
|
||||||
|
|||||||
@@ -6,10 +6,12 @@ import (
|
|||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
"path"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
@@ -33,7 +35,7 @@ func (prepareStage) Declares() IODecl {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (prepareStage) Run(_ context.Context, env *Env, m *manifest.Manifest) (*StageResult, error) {
|
func (prepareStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*StageResult, error) {
|
||||||
if env == nil || env.Config == nil {
|
if env == nil || env.Config == nil {
|
||||||
return nil, fmt.Errorf("prepare: stage environment config is required")
|
return nil, fmt.Errorf("prepare: stage environment config is required")
|
||||||
}
|
}
|
||||||
@@ -52,7 +54,7 @@ func (prepareStage) Run(_ context.Context, env *Env, m *manifest.Manifest) (*Sta
|
|||||||
return nil, fmt.Errorf("prepare: session id is required")
|
return nil, fmt.Errorf("prepare: session id is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
paths, err := env.ArtifactStore.EnsureLayout(sessionID)
|
paths, err := ensureLayoutForEnv(env, sessionID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("prepare: ensure workdir layout: %w", err)
|
return nil, fmt.Errorf("prepare: ensure workdir layout: %w", err)
|
||||||
}
|
}
|
||||||
@@ -89,13 +91,12 @@ func (prepareStage) Run(_ context.Context, env *Env, m *manifest.Manifest) (*Sta
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
resolvedAudio, err := resolveAudioFiles(sessionDir, env.Config.Session.Inputs)
|
resolvedLocalAudio, useS3Audio, err := resolveAudioInputs(sessionDir, env.Config.Session.Inputs)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("prepare: resolve audio inputs: %w", err)
|
return nil, fmt.Errorf("prepare: resolve audio inputs: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
copiedByDest := map[string]string{}
|
inputs := make([]manifest.InputRecord, 0, 5+len(resolvedLocalAudio))
|
||||||
inputs := make([]manifest.InputRecord, 0, 5+len(resolvedAudio))
|
|
||||||
registerInput := func(kind, path, checksum string) {
|
registerInput := func(kind, path, checksum string) {
|
||||||
inputs = append(inputs, manifest.InputRecord{Kind: kind, Path: path, Checksum: checksum})
|
inputs = append(inputs, manifest.InputRecord{Kind: kind, Path: path, Checksum: checksum})
|
||||||
}
|
}
|
||||||
@@ -134,19 +135,14 @@ func (prepareStage) Run(_ context.Context, env *Env, m *manifest.Manifest) (*Sta
|
|||||||
registerInput(cfgFile.kind, cfgFile.dst, checksum)
|
registerInput(cfgFile.kind, cfgFile.dst, checksum)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, src := range resolvedAudio {
|
if useS3Audio {
|
||||||
base := filepath.Base(src)
|
if err := materializeS3AudioInputs(ctx, env, m, sessionID, &inputs); err != nil {
|
||||||
if prev, exists := copiedByDest[base]; exists && prev != src {
|
return nil, fmt.Errorf("prepare: materialize s3 audio: %w", err)
|
||||||
return nil, fmt.Errorf("prepare: duplicate audio basename %q from %q and %q", base, prev, src)
|
|
||||||
}
|
}
|
||||||
copiedByDest[base] = src
|
} else {
|
||||||
|
if err := materializeLocalAudioInputs(env, paths, resolvedLocalAudio, registerInput); err != nil {
|
||||||
dst := filepath.Join(paths.AudioDir, base)
|
return nil, fmt.Errorf("prepare: %w", err)
|
||||||
checksum, err := copyFileIfChanged(env.ArtifactStore, src, dst)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("prepare: materialize audio %q: %w", base, err)
|
|
||||||
}
|
}
|
||||||
registerInput("audio", dst, checksum)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
sort.Slice(inputs, func(i, j int) bool {
|
sort.Slice(inputs, func(i, j int) bool {
|
||||||
@@ -162,7 +158,7 @@ func (prepareStage) Run(_ context.Context, env *Env, m *manifest.Manifest) (*Sta
|
|||||||
"prepared": true,
|
"prepared": true,
|
||||||
"stage": "prepare",
|
"stage": "prepare",
|
||||||
"inputs_count": len(inputs),
|
"inputs_count": len(inputs),
|
||||||
"audio_files_resolved": len(resolvedAudio),
|
"audio_files_resolved": countAudioInputs(inputs),
|
||||||
},
|
},
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
@@ -171,7 +167,23 @@ func renderResolvedPipeline(cfg *config.PipelineConfig) ([]byte, error) {
|
|||||||
return yaml.Marshal(cfg)
|
return yaml.Marshal(cfg)
|
||||||
}
|
}
|
||||||
|
|
||||||
func resolveAudioFiles(sessionDir string, inputs config.SessionInputsConfig) ([]string, error) {
|
func resolveAudioInputs(sessionDir string, inputs config.SessionInputsConfig) ([]string, bool, error) {
|
||||||
|
hasLocal := strings.TrimSpace(inputs.AudioDir) != "" || len(inputs.AudioFiles) > 0
|
||||||
|
if inputs.AudioS3 != nil {
|
||||||
|
if hasLocal {
|
||||||
|
return nil, false, fmt.Errorf("audio_dir/audio_files and audio_s3 are mutually exclusive")
|
||||||
|
}
|
||||||
|
return nil, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
local, err := resolveLocalAudioFiles(sessionDir, inputs)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
return local, false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveLocalAudioFiles(sessionDir string, inputs config.SessionInputsConfig) ([]string, error) {
|
||||||
if len(inputs.AudioFiles) > 0 {
|
if len(inputs.AudioFiles) > 0 {
|
||||||
out := make([]string, 0, len(inputs.AudioFiles))
|
out := make([]string, 0, len(inputs.AudioFiles))
|
||||||
for _, p := range inputs.AudioFiles {
|
for _, p := range inputs.AudioFiles {
|
||||||
@@ -223,6 +235,146 @@ func resolveAudioFiles(sessionDir string, inputs config.SessionInputsConfig) ([]
|
|||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func materializeLocalAudioInputs(env *Env, paths artifacts.SessionPaths, resolvedAudio []string, registerInput func(kind, path, checksum string)) error {
|
||||||
|
copiedByDest := map[string]string{}
|
||||||
|
for _, src := range resolvedAudio {
|
||||||
|
base := filepath.Base(src)
|
||||||
|
if prev, exists := copiedByDest[base]; exists && prev != src {
|
||||||
|
return fmt.Errorf("duplicate audio basename %q from %q and %q", base, prev, src)
|
||||||
|
}
|
||||||
|
copiedByDest[base] = src
|
||||||
|
|
||||||
|
dst := filepath.Join(paths.AudioDir, base)
|
||||||
|
checksum, err := copyFileIfChanged(env.ArtifactStore, src, dst)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("materialize audio %q: %w", base, err)
|
||||||
|
}
|
||||||
|
registerInput("audio", dst, checksum)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func materializeS3AudioInputs(ctx context.Context, env *Env, m *manifest.Manifest, sessionID string, inputs *[]manifest.InputRecord) error {
|
||||||
|
if env.ObjectStore == nil {
|
||||||
|
return fmt.Errorf("s3 audio input requires object store backend")
|
||||||
|
}
|
||||||
|
if env.Config == nil || env.Config.Pipeline == nil || env.Config.Session == nil || env.Config.Pipeline.Storage.S3 == nil || env.Config.Session.Inputs.AudioS3 == nil {
|
||||||
|
return fmt.Errorf("s3 audio input requires pipeline.storage.s3 and session.inputs.audio_s3 configuration")
|
||||||
|
}
|
||||||
|
|
||||||
|
campaign := strings.TrimSpace(env.Config.Session.Campaign)
|
||||||
|
if campaign == "" {
|
||||||
|
return fmt.Errorf("session campaign is required for s3 audio input")
|
||||||
|
}
|
||||||
|
runID := strings.TrimSpace(m.RunID)
|
||||||
|
if runID == "" {
|
||||||
|
return fmt.Errorf("run id is required for s3 audio input")
|
||||||
|
}
|
||||||
|
|
||||||
|
sessionPrefix := artifacts.S3SessionPrefix(env.Config.Pipeline.Storage.S3.RootPrefix, campaign, sessionID)
|
||||||
|
audioPrefix := artifacts.S3AudioPrefix(sessionPrefix, env.Config.Session.Inputs.AudioS3.Prefix)
|
||||||
|
objects, err := env.ObjectStore.List(ctx, audioPrefix)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("list s3 audio objects under %q: %w", audioPrefix, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
audioObjects := make([]storage.ObjectInfo, 0, len(objects))
|
||||||
|
for _, obj := range objects {
|
||||||
|
key := strings.TrimSpace(obj.Key)
|
||||||
|
if key == "" || strings.HasSuffix(key, "/") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !isFlac(key) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
audioObjects = append(audioObjects, obj)
|
||||||
|
}
|
||||||
|
sort.Slice(audioObjects, func(i, j int) bool {
|
||||||
|
return audioObjects[i].Key < audioObjects[j].Key
|
||||||
|
})
|
||||||
|
if len(audioObjects) == 0 {
|
||||||
|
return fmt.Errorf("no .flac files found under s3 audio prefix %q", audioPrefix)
|
||||||
|
}
|
||||||
|
|
||||||
|
spoolAudioDir := strings.TrimSpace(m.LocalSpoolDir)
|
||||||
|
if spoolAudioDir == "" {
|
||||||
|
spoolAudioDir = artifacts.SessionSpoolAudioDir(env.Config.Pipeline.Spool.Root, campaign, sessionID, runID)
|
||||||
|
}
|
||||||
|
workAudioDir := filepath.Join(pathsWorkDirForManifest(env, m, sessionID), "audio")
|
||||||
|
|
||||||
|
if err := os.MkdirAll(spoolAudioDir, 0o755); err != nil {
|
||||||
|
return fmt.Errorf("create spool audio directory %q: %w", spoolAudioDir, err)
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(workAudioDir, 0o755); err != nil {
|
||||||
|
return fmt.Errorf("create work audio directory %q: %w", workAudioDir, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
seenBase := map[string]string{}
|
||||||
|
for _, obj := range audioObjects {
|
||||||
|
base := path.Base(obj.Key)
|
||||||
|
if prev, exists := seenBase[base]; exists && prev != obj.Key {
|
||||||
|
return fmt.Errorf("duplicate s3 audio basename %q from %q and %q", base, prev, obj.Key)
|
||||||
|
}
|
||||||
|
seenBase[base] = obj.Key
|
||||||
|
|
||||||
|
spoolPath := filepath.Join(spoolAudioDir, base)
|
||||||
|
if err := env.ObjectStore.Download(ctx, obj.Key, spoolPath); err != nil {
|
||||||
|
return fmt.Errorf("download s3 audio object %q: %w", obj.Key, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
workPath := filepath.Join(workAudioDir, base)
|
||||||
|
checksum, err := copyFileIfChanged(env.ArtifactStore, spoolPath, workPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("materialize downloaded audio %q: %w", base, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
*inputs = append(*inputs, manifest.InputRecord{
|
||||||
|
Kind: "audio",
|
||||||
|
Path: workPath,
|
||||||
|
Checksum: checksum,
|
||||||
|
Source: "s3",
|
||||||
|
S3Bucket: strings.TrimSpace(env.Config.Pipeline.Storage.S3.Bucket),
|
||||||
|
S3Key: obj.Key,
|
||||||
|
S3Size: obj.Size,
|
||||||
|
S3ETag: obj.ETag,
|
||||||
|
SpoolPath: spoolPath,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func countAudioInputs(inputs []manifest.InputRecord) int {
|
||||||
|
count := 0
|
||||||
|
for _, in := range inputs {
|
||||||
|
if in.Kind == "audio" {
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return count
|
||||||
|
}
|
||||||
|
|
||||||
|
func pathsWorkDirForManifest(env *Env, m *manifest.Manifest, sessionID string) string {
|
||||||
|
if env == nil || env.Config == nil || env.Config.Pipeline == nil || env.Config.Session == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
campaign := strings.TrimSpace(env.Config.Session.Campaign)
|
||||||
|
if campaign == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
if m != nil && strings.TrimSpace(m.LocalWorkDir) != "" {
|
||||||
|
return strings.TrimSpace(m.LocalWorkDir)
|
||||||
|
}
|
||||||
|
runID := ""
|
||||||
|
if m != nil {
|
||||||
|
runID = strings.TrimSpace(m.RunID)
|
||||||
|
}
|
||||||
|
if runID != "" {
|
||||||
|
return artifacts.SessionRunRootForCampaign(env.Config.Pipeline.Workspace.Root, campaign, sessionID, runID)
|
||||||
|
}
|
||||||
|
return artifacts.SessionWorkDirForCampaign(env.Config.Pipeline.Workspace.Root, campaign, sessionID)
|
||||||
|
}
|
||||||
|
|
||||||
func resolvePath(baseDir, p string) (string, error) {
|
func resolvePath(baseDir, p string) (string, error) {
|
||||||
trimmed := strings.TrimSpace(p)
|
trimmed := strings.TrimSpace(p)
|
||||||
if trimmed == "" {
|
if trimmed == "" {
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
@@ -32,7 +33,7 @@ func TestPrepareStageExplicitAudioFiles(t *testing.T) {
|
|||||||
t.Fatalf("result metadata = %#v, want prepared=true", result)
|
t.Fatalf("result metadata = %#v, want prepared=true", result)
|
||||||
}
|
}
|
||||||
|
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
for _, p := range []string{
|
for _, p := range []string{
|
||||||
filepath.Join(paths.InputsDir, "session.yml"),
|
filepath.Join(paths.InputsDir, "session.yml"),
|
||||||
filepath.Join(paths.InputsDir, "pipeline.resolved.yml"),
|
filepath.Join(paths.InputsDir, "pipeline.resolved.yml"),
|
||||||
@@ -75,7 +76,7 @@ func TestPrepareStageAudioDirEnumeration(t *testing.T) {
|
|||||||
t.Fatalf("audio_files_resolved = %#v, want 1", got)
|
t.Fatalf("audio_files_resolved = %#v, want 1", got)
|
||||||
}
|
}
|
||||||
|
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
if _, err := os.Stat(filepath.Join(paths.AudioDir, "a.flac")); err != nil {
|
if _, err := os.Stat(filepath.Join(paths.AudioDir, "a.flac")); err != nil {
|
||||||
t.Fatalf("expected copied flac: %v", err)
|
t.Fatalf("expected copied flac: %v", err)
|
||||||
}
|
}
|
||||||
@@ -150,6 +151,139 @@ func TestPrepareStageIdempotent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPrepareStageS3AudioDownloadAndMaterialization(t *testing.T) {
|
||||||
|
env, m := setupPrepareEnv(t)
|
||||||
|
env.Config.Session.Campaign = "forsaken"
|
||||||
|
env.Config.Session.Inputs.AudioDir = ""
|
||||||
|
env.Config.Session.Inputs.AudioFiles = nil
|
||||||
|
env.Config.Session.Inputs.AudioS3 = &config.SessionAudioS3Input{Prefix: "audio/"}
|
||||||
|
env.Config.Pipeline.Spool = config.SpoolConfig{Root: filepath.Join(t.TempDir(), "spool")}
|
||||||
|
env.Config.Pipeline.Storage.S3 = &config.StorageS3Config{Bucket: "my-dnd-archive", RootPrefix: "dnd"}
|
||||||
|
m.RunID = "20260515T031522Z-a1b2c3d4"
|
||||||
|
m.LocalWorkDir = artifacts.SessionRunRootForCampaign(env.Config.Pipeline.Workspace.Root, "forsaken", m.SessionID, m.RunID)
|
||||||
|
m.LocalSpoolDir = artifacts.SessionSpoolAudioDir(env.Config.Pipeline.Spool.Root, "forsaken", m.SessionID, m.RunID)
|
||||||
|
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: "dnd/campaigns/forsaken/sessions/2026-05-03/audio/alice.flac", Data: []byte("alice")})
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: "dnd/campaigns/forsaken/sessions/2026-05-03/audio/bob.FLAC", Data: []byte("bob")})
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: "dnd/campaigns/forsaken/sessions/2026-05-03/audio/ignore.txt", Data: []byte("x")})
|
||||||
|
env.ObjectStore = fake
|
||||||
|
|
||||||
|
result, err := (prepareStage{}).Run(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare.Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if result == nil || result.Metadata["audio_files_resolved"] != 2 {
|
||||||
|
t.Fatalf("result metadata = %#v, want audio_files_resolved=2", result)
|
||||||
|
}
|
||||||
|
|
||||||
|
aliceWork := filepath.Join(m.LocalWorkDir, "audio", "alice.flac")
|
||||||
|
bobWork := filepath.Join(m.LocalWorkDir, "audio", "bob.FLAC")
|
||||||
|
aliceSpool := filepath.Join(m.LocalSpoolDir, "alice.flac")
|
||||||
|
bobSpool := filepath.Join(m.LocalSpoolDir, "bob.FLAC")
|
||||||
|
for _, p := range []string{aliceWork, bobWork, aliceSpool, bobSpool} {
|
||||||
|
if _, err := os.Stat(p); err != nil {
|
||||||
|
t.Fatalf("expected file %q: %v", p, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
audioInputs := 0
|
||||||
|
for _, in := range m.Inputs {
|
||||||
|
if in.Kind != "audio" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
audioInputs++
|
||||||
|
if in.Source != "s3" {
|
||||||
|
t.Fatalf("audio input source = %q, want s3", in.Source)
|
||||||
|
}
|
||||||
|
if in.S3Bucket != "my-dnd-archive" {
|
||||||
|
t.Fatalf("audio input bucket = %q", in.S3Bucket)
|
||||||
|
}
|
||||||
|
if in.S3Key == "" || in.SpoolPath == "" || in.Checksum == "" {
|
||||||
|
t.Fatalf("audio input missing provenance: %#v", in)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if audioInputs != 2 {
|
||||||
|
t.Fatalf("audio input count = %d, want 2", audioInputs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrepareStageS3AudioFailures(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
setup func(env *Env, m *manifest.Manifest, fake *storage.FakeBackend)
|
||||||
|
wantError string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "no flac files",
|
||||||
|
setup: func(_ *Env, _ *manifest.Manifest, fake *storage.FakeBackend) {
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: "dnd/campaigns/forsaken/sessions/2026-05-03/audio/readme.txt", Data: []byte("x")})
|
||||||
|
},
|
||||||
|
wantError: "no .flac files found",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "list error",
|
||||||
|
setup: func(_ *Env, _ *manifest.Manifest, fake *storage.FakeBackend) {
|
||||||
|
fake.ListErr = os.ErrPermission
|
||||||
|
},
|
||||||
|
wantError: "list s3 audio objects",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "download error",
|
||||||
|
setup: func(_ *Env, _ *manifest.Manifest, fake *storage.FakeBackend) {
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: "dnd/campaigns/forsaken/sessions/2026-05-03/audio/alice.flac", Data: []byte("alice")})
|
||||||
|
fake.DownloadErr = os.ErrPermission
|
||||||
|
},
|
||||||
|
wantError: "download s3 audio object",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "basename collision",
|
||||||
|
setup: func(_ *Env, _ *manifest.Manifest, fake *storage.FakeBackend) {
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: "dnd/campaigns/forsaken/sessions/2026-05-03/audio/a/alice.flac", Data: []byte("a")})
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: "dnd/campaigns/forsaken/sessions/2026-05-03/audio/b/alice.flac", Data: []byte("b")})
|
||||||
|
},
|
||||||
|
wantError: "duplicate s3 audio basename",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
env, m := setupPrepareEnv(t)
|
||||||
|
env.Config.Session.Campaign = "forsaken"
|
||||||
|
env.Config.Session.Inputs.AudioDir = ""
|
||||||
|
env.Config.Session.Inputs.AudioFiles = nil
|
||||||
|
env.Config.Session.Inputs.AudioS3 = &config.SessionAudioS3Input{Prefix: "audio/"}
|
||||||
|
env.Config.Pipeline.Spool = config.SpoolConfig{Root: filepath.Join(t.TempDir(), "spool")}
|
||||||
|
env.Config.Pipeline.Storage.S3 = &config.StorageS3Config{Bucket: "my-dnd-archive", RootPrefix: "dnd"}
|
||||||
|
m.RunID = "20260515T031522Z-a1b2c3d4"
|
||||||
|
m.LocalWorkDir = artifacts.SessionRunRootForCampaign(env.Config.Pipeline.Workspace.Root, "forsaken", m.SessionID, m.RunID)
|
||||||
|
m.LocalSpoolDir = artifacts.SessionSpoolAudioDir(env.Config.Pipeline.Spool.Root, "forsaken", m.SessionID, m.RunID)
|
||||||
|
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
tt.setup(env, m, fake)
|
||||||
|
env.ObjectStore = fake
|
||||||
|
|
||||||
|
_, err := (prepareStage{}).Run(context.Background(), env, m)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), tt.wantError) {
|
||||||
|
t.Fatalf("error = %v, want %q", err, tt.wantError)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrepareStageAudioSourceConflictFails(t *testing.T) {
|
||||||
|
env, m := setupPrepareEnv(t)
|
||||||
|
root := filepath.Dir(env.Config.SessionPath)
|
||||||
|
writeFile(t, filepath.Join(root, "audio", "a.flac"), "a")
|
||||||
|
env.Config.Session.Inputs.AudioDir = "./audio"
|
||||||
|
env.Config.Session.Inputs.AudioS3 = &config.SessionAudioS3Input{Prefix: "audio/"}
|
||||||
|
|
||||||
|
_, err := (prepareStage{}).Run(context.Background(), env, m)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "mutually exclusive") {
|
||||||
|
t.Fatalf("error = %v, want mutually exclusive error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func setupPrepareEnv(t *testing.T) (*Env, *manifest.Manifest) {
|
func setupPrepareEnv(t *testing.T) (*Env, *manifest.Manifest) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
workspace := t.TempDir()
|
workspace := t.TempDir()
|
||||||
@@ -170,6 +304,7 @@ func setupPrepareEnv(t *testing.T) (*Env, *manifest.Manifest) {
|
|||||||
PipelinePath: pipelinePath,
|
PipelinePath: pipelinePath,
|
||||||
Session: &config.SessionConfig{
|
Session: &config.SessionConfig{
|
||||||
SessionID: "2026-05-03",
|
SessionID: "2026-05-03",
|
||||||
|
Campaign: "sample-campaign",
|
||||||
Inputs: config.SessionInputsConfig{
|
Inputs: config.SessionInputsConfig{
|
||||||
AudioDir: "./audio",
|
AudioDir: "./audio",
|
||||||
SpeakersFile: "./speakers.yml",
|
SpeakersFile: "./speakers.yml",
|
||||||
|
|||||||
136
internal/stage/run_local.go
Normal file
136
internal/stage/run_local.go
Normal file
@@ -0,0 +1,136 @@
|
|||||||
|
package stage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
|
)
|
||||||
|
|
||||||
|
type runStageLayout struct {
|
||||||
|
Enabled bool
|
||||||
|
Root string
|
||||||
|
OutputsDir string
|
||||||
|
LogsDir string
|
||||||
|
ReportsDir string
|
||||||
|
ConfigDir string
|
||||||
|
ScratchDir string
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveRunStageLayout(
|
||||||
|
env *Env,
|
||||||
|
m *manifest.Manifest,
|
||||||
|
sessionPaths artifacts.SessionPaths,
|
||||||
|
sessionID, stageName string,
|
||||||
|
) (runStageLayout, error) {
|
||||||
|
if env == nil || env.Config == nil || env.Config.Pipeline == nil {
|
||||||
|
return runStageLayout{}, fmt.Errorf("stage environment pipeline config is required")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(sessionID) == "" {
|
||||||
|
return runStageLayout{}, fmt.Errorf("session id is required")
|
||||||
|
}
|
||||||
|
stageName = strings.TrimSpace(stageName)
|
||||||
|
if stageName == "" {
|
||||||
|
return runStageLayout{}, fmt.Errorf("stage name is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
runID := ""
|
||||||
|
if m != nil {
|
||||||
|
runID = strings.TrimSpace(m.RunID)
|
||||||
|
}
|
||||||
|
campaign := strings.TrimSpace(env.Config.Session.Campaign)
|
||||||
|
if campaign == "" && m != nil {
|
||||||
|
campaign = strings.TrimSpace(m.Campaign)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compatibility fallback for direct stage tests and older call paths
|
||||||
|
// that execute a stage without a run id.
|
||||||
|
if runID == "" || campaign == "" {
|
||||||
|
return runStageLayout{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
root := artifacts.SessionRunStageDirForCampaign(
|
||||||
|
env.Config.Pipeline.Workspace.Root,
|
||||||
|
campaign,
|
||||||
|
sessionID,
|
||||||
|
runID,
|
||||||
|
stageName,
|
||||||
|
)
|
||||||
|
layout := runStageLayout{
|
||||||
|
Enabled: true,
|
||||||
|
Root: root,
|
||||||
|
OutputsDir: filepath.Join(root, "outputs"),
|
||||||
|
LogsDir: filepath.Join(root, "logs"),
|
||||||
|
ReportsDir: filepath.Join(root, "reports"),
|
||||||
|
ConfigDir: filepath.Join(root, "config"),
|
||||||
|
ScratchDir: filepath.Join(root, "scratch"),
|
||||||
|
}
|
||||||
|
for _, dir := range []string{
|
||||||
|
layout.Root,
|
||||||
|
layout.OutputsDir,
|
||||||
|
layout.LogsDir,
|
||||||
|
layout.ReportsDir,
|
||||||
|
layout.ConfigDir,
|
||||||
|
layout.ScratchDir,
|
||||||
|
} {
|
||||||
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||||
|
return runStageLayout{}, fmt.Errorf("create run-stage directory %q: %w", dir, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return layout, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func runLocalPathForCanonical(layout runStageLayout, sessionPaths artifacts.SessionPaths, canonicalPath string) (string, error) {
|
||||||
|
if !layout.Enabled {
|
||||||
|
return filepath.Clean(canonicalPath), nil
|
||||||
|
}
|
||||||
|
cleanCanonical := filepath.Clean(strings.TrimSpace(canonicalPath))
|
||||||
|
if cleanCanonical == "" {
|
||||||
|
return "", fmt.Errorf("canonical path is required")
|
||||||
|
}
|
||||||
|
rel, err := filepath.Rel(filepath.Clean(sessionPaths.Root), cleanCanonical)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("derive session-relative path for %q: %w", cleanCanonical, err)
|
||||||
|
}
|
||||||
|
rel = filepath.Clean(rel)
|
||||||
|
if rel == "." || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
|
||||||
|
return "", fmt.Errorf("canonical path %q is outside session root %q", cleanCanonical, sessionPaths.Root)
|
||||||
|
}
|
||||||
|
localPath := filepath.Join(layout.OutputsDir, rel)
|
||||||
|
if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
|
||||||
|
return "", fmt.Errorf("create run-local output parent for %q: %w", localPath, err)
|
||||||
|
}
|
||||||
|
return localPath, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func promoteRunLocalOutput(
|
||||||
|
store artifacts.Store,
|
||||||
|
srcPath, canonicalPath string,
|
||||||
|
ref artifacts.Ref,
|
||||||
|
) (artifacts.Ref, error) {
|
||||||
|
srcPath = filepath.Clean(strings.TrimSpace(srcPath))
|
||||||
|
canonicalPath = filepath.Clean(strings.TrimSpace(canonicalPath))
|
||||||
|
if srcPath == "" {
|
||||||
|
return artifacts.Ref{}, fmt.Errorf("source path is required")
|
||||||
|
}
|
||||||
|
if canonicalPath == "" {
|
||||||
|
return artifacts.Ref{}, fmt.Errorf("canonical destination path is required")
|
||||||
|
}
|
||||||
|
data, err := os.ReadFile(srcPath)
|
||||||
|
if err != nil {
|
||||||
|
return artifacts.Ref{}, fmt.Errorf("read run-local output %q: %w", srcPath, err)
|
||||||
|
}
|
||||||
|
if err := store.WriteFileAtomic(canonicalPath, data, 0o644); err != nil {
|
||||||
|
return artifacts.Ref{}, fmt.Errorf("promote output to %q: %w", canonicalPath, err)
|
||||||
|
}
|
||||||
|
checksum, err := store.Checksum(canonicalPath)
|
||||||
|
if err != nil {
|
||||||
|
return artifacts.Ref{}, fmt.Errorf("checksum promoted output %q: %w", canonicalPath, err)
|
||||||
|
}
|
||||||
|
ref.AbsolutePath = canonicalPath
|
||||||
|
ref.Checksum = checksum
|
||||||
|
return ref, nil
|
||||||
|
}
|
||||||
35
internal/stage/run_local_test.go
Normal file
35
internal/stage/run_local_test.go
Normal file
@@ -0,0 +1,35 @@
|
|||||||
|
package stage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestRunLocalPathForCanonicalCreatesParentDirectories(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
sessionRoot := filepath.Join(root, "work", "dilfs", "2026-05-17")
|
||||||
|
layout := runStageLayout{
|
||||||
|
Enabled: true,
|
||||||
|
OutputsDir: filepath.Join(sessionRoot, "runs", "run-1", "merge", "outputs"),
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(layout.OutputsDir, 0o755); err != nil {
|
||||||
|
t.Fatalf("mkdir outputs dir: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
canonical := filepath.Join(sessionRoot, "transcripts", "merged.json")
|
||||||
|
got, err := runLocalPathForCanonical(layout, artifacts.SessionPaths{Root: sessionRoot}, canonical)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("runLocalPathForCanonical() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
want := filepath.Join(layout.OutputsDir, "transcripts", "merged.json")
|
||||||
|
if got != want {
|
||||||
|
t.Fatalf("runLocalPathForCanonical() = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Dir(got)); err != nil {
|
||||||
|
t.Fatalf("expected run-local parent directory to exist: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
23
internal/stage/session_paths.go
Normal file
23
internal/stage/session_paths.go
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
package stage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
)
|
||||||
|
|
||||||
|
func sessionPathsForEnv(env *Env, sessionID string) artifacts.SessionPaths {
|
||||||
|
campaign := ""
|
||||||
|
if env != nil && env.Config != nil && env.Config.Session != nil {
|
||||||
|
campaign = strings.TrimSpace(env.Config.Session.Campaign)
|
||||||
|
}
|
||||||
|
return env.ArtifactStore.SessionPathsFor(campaign, sessionID)
|
||||||
|
}
|
||||||
|
|
||||||
|
func ensureLayoutForEnv(env *Env, sessionID string) (artifacts.SessionPaths, error) {
|
||||||
|
campaign := ""
|
||||||
|
if env != nil && env.Config != nil && env.Config.Session != nil {
|
||||||
|
campaign = strings.TrimSpace(env.Config.Session.Campaign)
|
||||||
|
}
|
||||||
|
return env.ArtifactStore.EnsureLayoutFor(campaign, sessionID)
|
||||||
|
}
|
||||||
@@ -29,6 +29,7 @@ type Env struct {
|
|||||||
Scriptorium scriptorium.Runner
|
Scriptorium scriptorium.Runner
|
||||||
Analyzer analyzer.Runner
|
Analyzer analyzer.Runner
|
||||||
Storage storage.Backend
|
Storage storage.Backend
|
||||||
|
ObjectStore storage.ObjectStore
|
||||||
Notifier notify.Sender
|
Notifier notify.Sender
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -55,7 +55,11 @@ func (transcribeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest)
|
|||||||
return nil, fmt.Errorf("transcribe: session id is required")
|
return nil, fmt.Errorf("transcribe: session id is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
paths := env.ArtifactStore.SessionPaths(sessionID)
|
paths := sessionPathsForEnv(env, sessionID)
|
||||||
|
runLayout, err := resolveRunStageLayout(env, m, paths, sessionID, "transcribe")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("transcribe: resolve run-stage layout: %w", err)
|
||||||
|
}
|
||||||
audioFiles, err := discoverPreparedAudio(m, paths.AudioDir)
|
audioFiles, err := discoverPreparedAudio(m, paths.AudioDir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("transcribe: resolve audio inputs: %w", err)
|
return nil, fmt.Errorf("transcribe: resolve audio inputs: %w", err)
|
||||||
@@ -88,10 +92,15 @@ func (transcribeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest)
|
|||||||
return nil, fmt.Errorf("transcribe: duplicate speaker/audio basename %q from %q and %q", base, prev, audioPath)
|
return nil, fmt.Errorf("transcribe: duplicate speaker/audio basename %q from %q and %q", base, prev, audioPath)
|
||||||
}
|
}
|
||||||
seenSpeaker[base] = audioPath
|
seenSpeaker[base] = audioPath
|
||||||
|
canonicalOut := filepath.Join(paths.TranscriptsRawDir, base+".json")
|
||||||
|
runOut, err := runLocalPathForCanonical(runLayout, paths, canonicalOut)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("transcribe: resolve run-local output path for %q: %w", base, err)
|
||||||
|
}
|
||||||
jobs = append(jobs, job{
|
jobs = append(jobs, job{
|
||||||
speakerID: base,
|
speakerID: base,
|
||||||
audioPath: audioPath,
|
audioPath: audioPath,
|
||||||
outPath: filepath.Join(paths.TranscriptsRawDir, base+".json"),
|
outPath: runOut,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -197,12 +206,19 @@ dispatch:
|
|||||||
sort.Strings(speakers)
|
sort.Strings(speakers)
|
||||||
|
|
||||||
outputs := make([]artifacts.Ref, 0, len(speakers))
|
outputs := make([]artifacts.Ref, 0, len(speakers))
|
||||||
|
runOutputPaths := make([]string, 0, len(speakers))
|
||||||
outputPaths := make([]string, 0, len(speakers))
|
outputPaths := make([]string, 0, len(speakers))
|
||||||
orderedPerFile := make(map[string]any, len(speakers))
|
orderedPerFile := make(map[string]any, len(speakers))
|
||||||
for _, speaker := range speakers {
|
for _, speaker := range speakers {
|
||||||
ref := outputRef[speaker]
|
ref := outputRef[speaker]
|
||||||
outputs = append(outputs, ref)
|
runOutputPaths = append(runOutputPaths, ref.AbsolutePath)
|
||||||
outputPaths = append(outputPaths, ref.AbsolutePath)
|
canonicalOut := filepath.Join(paths.TranscriptsRawDir, speaker+".json")
|
||||||
|
promoted, err := promoteRunLocalOutput(env.ArtifactStore, ref.AbsolutePath, canonicalOut, ref)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("transcribe: promote %q output: %w", speaker, err)
|
||||||
|
}
|
||||||
|
outputs = append(outputs, promoted)
|
||||||
|
outputPaths = append(outputPaths, canonicalOut)
|
||||||
orderedPerFile[speaker] = perFile[speaker]
|
orderedPerFile[speaker] = perFile[speaker]
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -221,6 +237,7 @@ dispatch:
|
|||||||
"retries": retries,
|
"retries": retries,
|
||||||
"retry_delay": env.Config.Pipeline.WhisperX.RetryDelay,
|
"retry_delay": env.Config.Pipeline.WhisperX.RetryDelay,
|
||||||
"timeout": env.Config.Pipeline.WhisperX.Timeout,
|
"timeout": env.Config.Pipeline.WhisperX.Timeout,
|
||||||
|
"run_output_paths": runOutputPaths,
|
||||||
"output_paths": outputPaths,
|
"output_paths": outputPaths,
|
||||||
"per_file": orderedPerFile,
|
"per_file": orderedPerFile,
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -53,8 +53,8 @@ func TestTranscribeStageTranscribesPreparedAudio(t *testing.T) {
|
|||||||
}
|
}
|
||||||
sort.Strings(gotPaths)
|
sort.Strings(gotPaths)
|
||||||
wantPaths := []string{
|
wantPaths := []string{
|
||||||
filepath.Join(env.ArtifactStore.SessionPaths(m.SessionID).TranscriptsRawDir, "alice.json"),
|
filepath.Join(sessionPathsForEnv(env, m.SessionID).TranscriptsRawDir, "alice.json"),
|
||||||
filepath.Join(env.ArtifactStore.SessionPaths(m.SessionID).TranscriptsRawDir, "bob.json"),
|
filepath.Join(sessionPathsForEnv(env, m.SessionID).TranscriptsRawDir, "bob.json"),
|
||||||
}
|
}
|
||||||
sort.Strings(wantPaths)
|
sort.Strings(wantPaths)
|
||||||
if strings.Join(gotPaths, "|") != strings.Join(wantPaths, "|") {
|
if strings.Join(gotPaths, "|") != strings.Join(wantPaths, "|") {
|
||||||
@@ -191,6 +191,36 @@ func TestTranscribeStageInvalidJSONFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestTranscribeStageUsesRunLocalOutputAndPromotesCanonical(t *testing.T) {
|
||||||
|
env, m := setupTranscribeEnv(t, []string{"alice.flac"})
|
||||||
|
env.Config.Session.Campaign = "sample-campaign"
|
||||||
|
m.Campaign = "sample-campaign"
|
||||||
|
m.RunID = "20260518T010203Z-abcdef12"
|
||||||
|
fake := &whisperx.FakeClient{}
|
||||||
|
env.WhisperX = fake
|
||||||
|
|
||||||
|
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
|
||||||
|
t.Fatalf("prepare.Run() error = %v", err)
|
||||||
|
}
|
||||||
|
result, err := (transcribeStage{}).Run(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("transcribe.Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(fake.Requests) != 1 {
|
||||||
|
t.Fatalf("requests = %d, want 1", len(fake.Requests))
|
||||||
|
}
|
||||||
|
runOut := fake.Requests[0].OutputRawTranscriptPath
|
||||||
|
if !strings.Contains(runOut, filepath.Join("runs", m.RunID, "transcribe", "outputs")) {
|
||||||
|
t.Fatalf("run-local output path = %q, want runs/{run_id}/transcribe/outputs path", runOut)
|
||||||
|
}
|
||||||
|
if len(result.Outputs) != 1 {
|
||||||
|
t.Fatalf("outputs = %#v, want one output", result.Outputs)
|
||||||
|
}
|
||||||
|
if strings.Contains(result.Outputs[0].AbsolutePath, string(filepath.Separator)+"runs"+string(filepath.Separator)) {
|
||||||
|
t.Fatalf("promoted output path = %q, want canonical session path", result.Outputs[0].AbsolutePath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func setupTranscribeEnv(t *testing.T, audioFiles []string) (*Env, *manifest.Manifest) {
|
func setupTranscribeEnv(t *testing.T, audioFiles []string) (*Env, *manifest.Manifest) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -203,7 +233,7 @@ func setupTranscribeEnv(t *testing.T, audioFiles []string) (*Env, *manifest.Mani
|
|||||||
|
|
||||||
sessionPath := filepath.Join(cfgDir, "session.yml")
|
sessionPath := filepath.Join(cfgDir, "session.yml")
|
||||||
pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
|
pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
|
||||||
writeFile(t, sessionPath, "session_id: 2026-05-03\n")
|
writeFile(t, sessionPath, "session_id: 2026-05-03\ncampaign: sample-campaign\n")
|
||||||
writeFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n")
|
writeFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n")
|
||||||
writeFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n")
|
writeFile(t, filepath.Join(cfgDir, "speakers.yml"), "alice: alice.flac\n")
|
||||||
writeFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n")
|
writeFile(t, filepath.Join(cfgDir, "autocorrect.yml"), "[]\n")
|
||||||
@@ -227,6 +257,7 @@ func setupTranscribeEnv(t *testing.T, audioFiles []string) (*Env, *manifest.Mani
|
|||||||
},
|
},
|
||||||
Session: &config.SessionConfig{
|
Session: &config.SessionConfig{
|
||||||
SessionID: "2026-05-03",
|
SessionID: "2026-05-03",
|
||||||
|
Campaign: "sample-campaign",
|
||||||
Inputs: config.SessionInputsConfig{
|
Inputs: config.SessionInputsConfig{
|
||||||
AudioDir: "./audio",
|
AudioDir: "./audio",
|
||||||
SpeakersFile: "./speakers.yml",
|
SpeakersFile: "./speakers.yml",
|
||||||
|
|||||||
@@ -54,7 +54,11 @@ func (trimStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Stag
|
|||||||
return nil, fmt.Errorf("trim: session id is required")
|
return nil, fmt.Errorf("trim: session id is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
paths := env.ArtifactStore.SessionPaths(sessionID)
|
paths := sessionPathsForEnv(env, sessionID)
|
||||||
|
runLayout, err := resolveRunStageLayout(env, m, paths, sessionID, "trim")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("trim: resolve run-stage layout: %w", err)
|
||||||
|
}
|
||||||
normalizedPath, normalizedSource, err := discoverNormalizedTranscript(m, paths)
|
normalizedPath, normalizedSource, err := discoverNormalizedTranscript(m, paths)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("trim: resolve normalized transcript: %w", err)
|
return nil, fmt.Errorf("trim: resolve normalized transcript: %w", err)
|
||||||
@@ -69,10 +73,14 @@ func (trimStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Stag
|
|||||||
trimCfg := env.Config.Pipeline.Trim
|
trimCfg := env.Config.Pipeline.Trim
|
||||||
enabled := trimCfg != nil && trimCfg.Enabled
|
enabled := trimCfg != nil && trimCfg.Enabled
|
||||||
|
|
||||||
trimmedPath, err := resolveTrimmedOutputPath(paths, trimCfg)
|
canonicalTrimmedPath, err := resolveTrimmedOutputPath(paths, trimCfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("trim: resolve trimmed output path: %w", err)
|
return nil, fmt.Errorf("trim: resolve trimmed output path: %w", err)
|
||||||
}
|
}
|
||||||
|
trimmedPath, err := runLocalPathForCanonical(runLayout, paths, canonicalTrimmedPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("trim: resolve run-local trimmed output path: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
logPaths := []string{}
|
logPaths := []string{}
|
||||||
generatedConfigs := []string{}
|
generatedConfigs := []string{}
|
||||||
@@ -81,7 +89,8 @@ func (trimStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Stag
|
|||||||
"trim_enabled": enabled,
|
"trim_enabled": enabled,
|
||||||
"normalized_transcript_path": normalizedPath,
|
"normalized_transcript_path": normalizedPath,
|
||||||
"normalized_transcript_source": normalizedSource,
|
"normalized_transcript_source": normalizedSource,
|
||||||
"trimmed_output_path": trimmedPath,
|
"run_trimmed_output_path": trimmedPath,
|
||||||
|
"trimmed_output_path": canonicalTrimmedPath,
|
||||||
}
|
}
|
||||||
|
|
||||||
if !enabled {
|
if !enabled {
|
||||||
@@ -91,14 +100,17 @@ func (trimStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Stag
|
|||||||
if err := validateProcessedTranscriptOutput(trimmedPath); err != nil {
|
if err := validateProcessedTranscriptOutput(trimmedPath); err != nil {
|
||||||
return nil, fmt.Errorf("trim: copied trimmed transcript %q invalid: %w", trimmedPath, err)
|
return nil, fmt.Errorf("trim: copied trimmed transcript %q invalid: %w", trimmedPath, err)
|
||||||
}
|
}
|
||||||
|
promotedTrimmed, err := promoteRunLocalOutput(env.ArtifactStore, trimmedPath, canonicalTrimmedPath, artifacts.Ref{
|
||||||
|
Kind: "transcript_trimmed",
|
||||||
|
Category: "transcripts",
|
||||||
|
SessionID: sessionID,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("trim: promote trimmed transcript: %w", err)
|
||||||
|
}
|
||||||
metadata["trim_action"] = "copy_disabled"
|
metadata["trim_action"] = "copy_disabled"
|
||||||
return &StageResult{
|
return &StageResult{
|
||||||
Outputs: []artifacts.Ref{{
|
Outputs: []artifacts.Ref{promotedTrimmed},
|
||||||
Kind: "transcript_trimmed",
|
|
||||||
Category: "transcripts",
|
|
||||||
SessionID: sessionID,
|
|
||||||
AbsolutePath: trimmedPath,
|
|
||||||
}},
|
|
||||||
Metadata: metadata,
|
Metadata: metadata,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
@@ -114,13 +126,22 @@ func (trimStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Stag
|
|||||||
}
|
}
|
||||||
|
|
||||||
boundsCfg := trimCfg.Bounds
|
boundsCfg := trimCfg.Bounds
|
||||||
boundsOutputPath, err := resolveScriptoriumOutputPath(paths, boundsCfg.OutputPath)
|
canonicalBoundsOutputPath, err := resolveScriptoriumOutputPath(paths, boundsCfg.OutputPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("trim: resolve bounds output path: %w", err)
|
return nil, fmt.Errorf("trim: resolve bounds output path: %w", err)
|
||||||
}
|
}
|
||||||
|
boundsOutputPath, err := runLocalPathForCanonical(runLayout, paths, canonicalBoundsOutputPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("trim: resolve run-local bounds output path: %w", err)
|
||||||
|
}
|
||||||
boundsStdoutLogPath := filepath.Join(paths.LogsDir, "scriptorium.bounds.stdout.log")
|
boundsStdoutLogPath := filepath.Join(paths.LogsDir, "scriptorium.bounds.stdout.log")
|
||||||
boundsStderrLogPath := filepath.Join(paths.LogsDir, "scriptorium.bounds.stderr.log")
|
boundsStderrLogPath := filepath.Join(paths.LogsDir, "scriptorium.bounds.stderr.log")
|
||||||
boundsGeneratedConfigPath := filepath.Join(paths.ConfigDir, "scriptorium.bounds.generated.yml")
|
boundsGeneratedConfigPath := filepath.Join(paths.ConfigDir, "scriptorium.bounds.generated.yml")
|
||||||
|
if runLayout.Enabled {
|
||||||
|
boundsStdoutLogPath = filepath.Join(runLayout.LogsDir, "scriptorium.bounds.stdout.log")
|
||||||
|
boundsStderrLogPath = filepath.Join(runLayout.LogsDir, "scriptorium.bounds.stderr.log")
|
||||||
|
boundsGeneratedConfigPath = filepath.Join(runLayout.ConfigDir, "scriptorium.bounds.generated.yml")
|
||||||
|
}
|
||||||
boundsTimeout, err := resolveScriptoriumTimeout(env.Config.Pipeline.Scriptorium.Timeout, boundsCfg.Timeout)
|
boundsTimeout, err := resolveScriptoriumTimeout(env.Config.Pipeline.Scriptorium.Timeout, boundsCfg.Timeout)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("trim: resolve bounds timeout: %w", err)
|
return nil, fmt.Errorf("trim: resolve bounds timeout: %w", err)
|
||||||
@@ -133,7 +154,8 @@ func (trimStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Stag
|
|||||||
|
|
||||||
metadata["bounds_prompt_id"] = boundsCfg.PromptID
|
metadata["bounds_prompt_id"] = boundsCfg.PromptID
|
||||||
metadata["bounds_profile_id"] = boundsCfg.ProfileID
|
metadata["bounds_profile_id"] = boundsCfg.ProfileID
|
||||||
metadata["bounds_output_path"] = boundsOutputPath
|
metadata["run_bounds_output_path"] = boundsOutputPath
|
||||||
|
metadata["bounds_output_path"] = canonicalBoundsOutputPath
|
||||||
metadata["bounds_timeout"] = boundsTimeout.String()
|
metadata["bounds_timeout"] = boundsTimeout.String()
|
||||||
metadata["bounds_input_name"] = boundsCfg.TranscriptInputName
|
metadata["bounds_input_name"] = boundsCfg.TranscriptInputName
|
||||||
metadata["bounds_input_path"] = normalizedPath
|
metadata["bounds_input_path"] = normalizedPath
|
||||||
@@ -141,13 +163,22 @@ func (trimStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Stag
|
|||||||
|
|
||||||
renderOutputPath := ""
|
renderOutputPath := ""
|
||||||
if boundsCfg.RenderDebug {
|
if boundsCfg.RenderDebug {
|
||||||
renderOutputPath, err = resolveScriptoriumOutputPath(paths, boundsCfg.RenderOutputPath)
|
canonicalRenderOutputPath, err := resolveScriptoriumOutputPath(paths, boundsCfg.RenderOutputPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("trim: resolve bounds render output path: %w", err)
|
return nil, fmt.Errorf("trim: resolve bounds render output path: %w", err)
|
||||||
}
|
}
|
||||||
|
renderOutputPath, err = runLocalPathForCanonical(runLayout, paths, canonicalRenderOutputPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("trim: resolve run-local bounds render output path: %w", err)
|
||||||
|
}
|
||||||
renderStdoutLogPath := filepath.Join(paths.LogsDir, "scriptorium.bounds.render.stdout.log")
|
renderStdoutLogPath := filepath.Join(paths.LogsDir, "scriptorium.bounds.render.stdout.log")
|
||||||
renderStderrLogPath := filepath.Join(paths.LogsDir, "scriptorium.bounds.render.stderr.log")
|
renderStderrLogPath := filepath.Join(paths.LogsDir, "scriptorium.bounds.render.stderr.log")
|
||||||
renderGeneratedConfigPath := filepath.Join(paths.ConfigDir, "scriptorium.bounds.render.generated.yml")
|
renderGeneratedConfigPath := filepath.Join(paths.ConfigDir, "scriptorium.bounds.render.generated.yml")
|
||||||
|
if runLayout.Enabled {
|
||||||
|
renderStdoutLogPath = filepath.Join(runLayout.LogsDir, "scriptorium.bounds.render.stdout.log")
|
||||||
|
renderStderrLogPath = filepath.Join(runLayout.LogsDir, "scriptorium.bounds.render.stderr.log")
|
||||||
|
renderGeneratedConfigPath = filepath.Join(runLayout.ConfigDir, "scriptorium.bounds.render.generated.yml")
|
||||||
|
}
|
||||||
|
|
||||||
renderReq := scriptorium.RenderArtifactRequest{
|
renderReq := scriptorium.RenderArtifactRequest{
|
||||||
Binary: env.Config.Pipeline.Scriptorium.Binary,
|
Binary: env.Config.Pipeline.Scriptorium.Binary,
|
||||||
@@ -254,7 +285,7 @@ func (trimStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Stag
|
|||||||
metadata["end_segment_id"] = boundsPayload.EndSegmentID
|
metadata["end_segment_id"] = boundsPayload.EndSegmentID
|
||||||
metadata["warnings"] = boundsPayload.Warnings
|
metadata["warnings"] = boundsPayload.Warnings
|
||||||
metadata["keep_selector"] = keepSelector
|
metadata["keep_selector"] = keepSelector
|
||||||
metadata["bounds_output_path"] = finalBoundsOutputPath
|
metadata["run_bounds_output_path"] = finalBoundsOutputPath
|
||||||
metadata["bounds_stdout_log_path"] = boundsStdoutLogPath
|
metadata["bounds_stdout_log_path"] = boundsStdoutLogPath
|
||||||
metadata["bounds_stderr_log_path"] = boundsStderrLogPath
|
metadata["bounds_stderr_log_path"] = boundsStderrLogPath
|
||||||
metadata["bounds_generated_config_path"] = boundsGeneratedConfigPath
|
metadata["bounds_generated_config_path"] = boundsGeneratedConfigPath
|
||||||
@@ -279,6 +310,11 @@ func (trimStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Stag
|
|||||||
trimStdoutLogPath := filepath.Join(paths.LogsDir, "seriatim.trim.stdout.log")
|
trimStdoutLogPath := filepath.Join(paths.LogsDir, "seriatim.trim.stdout.log")
|
||||||
trimStderrLogPath := filepath.Join(paths.LogsDir, "seriatim.trim.stderr.log")
|
trimStderrLogPath := filepath.Join(paths.LogsDir, "seriatim.trim.stderr.log")
|
||||||
trimGeneratedConfigPath := filepath.Join(paths.ConfigDir, "seriatim.trim.generated.yml")
|
trimGeneratedConfigPath := filepath.Join(paths.ConfigDir, "seriatim.trim.generated.yml")
|
||||||
|
if runLayout.Enabled {
|
||||||
|
trimStdoutLogPath = filepath.Join(runLayout.LogsDir, "seriatim.trim.stdout.log")
|
||||||
|
trimStderrLogPath = filepath.Join(runLayout.LogsDir, "seriatim.trim.stderr.log")
|
||||||
|
trimGeneratedConfigPath = filepath.Join(runLayout.ConfigDir, "seriatim.trim.generated.yml")
|
||||||
|
}
|
||||||
trimTimeout, err := resolveTrimSeriatimTimeout(env.Config.Pipeline.Seriatim.Timeout)
|
trimTimeout, err := resolveTrimSeriatimTimeout(env.Config.Pipeline.Seriatim.Timeout)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("trim: resolve seriatim timeout: %w", err)
|
return nil, fmt.Errorf("trim: resolve seriatim timeout: %w", err)
|
||||||
@@ -315,23 +351,25 @@ func (trimStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Stag
|
|||||||
return nil, fmt.Errorf("trim: trimmed transcript %q invalid: %w", trimmedPath, err)
|
return nil, fmt.Errorf("trim: trimmed transcript %q invalid: %w", trimmedPath, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
outputs := []artifacts.Ref{
|
promotedTrimmed, err := promoteRunLocalOutput(env.ArtifactStore, trimmedPath, canonicalTrimmedPath, artifacts.Ref{
|
||||||
{
|
Kind: "transcript_trimmed",
|
||||||
Kind: "transcript_trimmed",
|
Category: "transcripts",
|
||||||
Category: "transcripts",
|
SessionID: sessionID,
|
||||||
SessionID: sessionID,
|
})
|
||||||
AbsolutePath: trimmedPath,
|
if err != nil {
|
||||||
},
|
return nil, fmt.Errorf("trim: promote trimmed transcript: %w", err)
|
||||||
{
|
}
|
||||||
Kind: "session_bounds",
|
promotedBounds, err := promoteRunLocalOutput(env.ArtifactStore, finalBoundsOutputPath, canonicalBoundsOutputPath, artifacts.Ref{
|
||||||
Category: "artifacts",
|
Kind: "session_bounds",
|
||||||
SessionID: sessionID,
|
Category: "artifacts",
|
||||||
AbsolutePath: finalBoundsOutputPath,
|
SessionID: sessionID,
|
||||||
},
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("trim: promote session bounds: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return &StageResult{
|
return &StageResult{
|
||||||
Outputs: outputs,
|
Outputs: []artifacts.Ref{promotedTrimmed, promotedBounds},
|
||||||
Logs: logPaths,
|
Logs: logPaths,
|
||||||
GeneratedConfigs: generatedConfigs,
|
GeneratedConfigs: generatedConfigs,
|
||||||
Metadata: metadata,
|
Metadata: metadata,
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ import (
|
|||||||
|
|
||||||
func TestTrimStageConsumesNormalizedAndProducesTrimmedTranscript(t *testing.T) {
|
func TestTrimStageConsumesNormalizedAndProducesTrimmedTranscript(t *testing.T) {
|
||||||
env, m, scr, ser := setupTrimEnv(t)
|
env, m, scr, ser := setupTrimEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
normalized := filepath.Join(paths.TranscriptsDir, "normalized.json")
|
normalized := filepath.Join(paths.TranscriptsDir, "normalized.json")
|
||||||
writeFile(t, normalized, `{"segments":[{"id":10},{"id":868}]}`)
|
writeFile(t, normalized, `{"segments":[{"id":10},{"id":868}]}`)
|
||||||
m.MarkStageSucceeded("normalize", time.Now().UTC(), []manifest.ArtifactRecord{
|
m.MarkStageSucceeded("normalize", time.Now().UTC(), []manifest.ArtifactRecord{
|
||||||
@@ -70,7 +70,7 @@ func TestTrimStageConsumesNormalizedAndProducesTrimmedTranscript(t *testing.T) {
|
|||||||
|
|
||||||
func TestTrimStageUsesConfiguredScriptoriumInputName(t *testing.T) {
|
func TestTrimStageUsesConfiguredScriptoriumInputName(t *testing.T) {
|
||||||
env, m, scr, _ := setupTrimEnv(t)
|
env, m, scr, _ := setupTrimEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
normalized := filepath.Join(paths.TranscriptsDir, "normalized.json")
|
normalized := filepath.Join(paths.TranscriptsDir, "normalized.json")
|
||||||
writeFile(t, normalized, `{"segments":[{"id":10},{"id":11}]}`)
|
writeFile(t, normalized, `{"segments":[{"id":10},{"id":11}]}`)
|
||||||
scr.BoundsBody = `{"trim_action":"trim","start_segment_id":10,"end_segment_id":11}`
|
scr.BoundsBody = `{"trim_action":"trim","start_segment_id":10,"end_segment_id":11}`
|
||||||
@@ -94,7 +94,7 @@ func TestTrimStageUsesConfiguredScriptoriumInputName(t *testing.T) {
|
|||||||
|
|
||||||
func TestTrimStageRecordsLogAndGeneratedConfigRefs(t *testing.T) {
|
func TestTrimStageRecordsLogAndGeneratedConfigRefs(t *testing.T) {
|
||||||
env, m, scr, _ := setupTrimEnv(t)
|
env, m, scr, _ := setupTrimEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeFile(t, filepath.Join(paths.TranscriptsDir, "normalized.json"), `{"segments":[{"id":1},{"id":2}]}`)
|
writeFile(t, filepath.Join(paths.TranscriptsDir, "normalized.json"), `{"segments":[{"id":1},{"id":2}]}`)
|
||||||
scr.BoundsBody = `{"trim_action":"trim","start_segment_id":1,"end_segment_id":2}`
|
scr.BoundsBody = `{"trim_action":"trim","start_segment_id":1,"end_segment_id":2}`
|
||||||
|
|
||||||
@@ -114,7 +114,7 @@ func TestTrimStageRecordsLogAndGeneratedConfigRefs(t *testing.T) {
|
|||||||
|
|
||||||
func TestTrimStageRenderDebugDiagnosticsAreNotStageOutputs(t *testing.T) {
|
func TestTrimStageRenderDebugDiagnosticsAreNotStageOutputs(t *testing.T) {
|
||||||
env, m, scr, _ := setupTrimEnv(t)
|
env, m, scr, _ := setupTrimEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeFile(t, filepath.Join(paths.TranscriptsDir, "normalized.json"), `{"segments":[{"id":1},{"id":2}]}`)
|
writeFile(t, filepath.Join(paths.TranscriptsDir, "normalized.json"), `{"segments":[{"id":1},{"id":2}]}`)
|
||||||
scr.BoundsBody = `{"trim_action":"trim","start_segment_id":1,"end_segment_id":2}`
|
scr.BoundsBody = `{"trim_action":"trim","start_segment_id":1,"end_segment_id":2}`
|
||||||
|
|
||||||
@@ -152,7 +152,7 @@ func TestTrimStageFailsWhenNormalizedTranscriptMissing(t *testing.T) {
|
|||||||
|
|
||||||
func TestTrimStageDoesNotFallBackToProcessedTranscript(t *testing.T) {
|
func TestTrimStageDoesNotFallBackToProcessedTranscript(t *testing.T) {
|
||||||
env, m, scr, ser := setupTrimEnv(t)
|
env, m, scr, ser := setupTrimEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[{"id":1},{"id":2}]}`)
|
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[{"id":1},{"id":2}]}`)
|
||||||
|
|
||||||
_, err := (trimStage{}).Run(context.Background(), env, m)
|
_, err := (trimStage{}).Run(context.Background(), env, m)
|
||||||
@@ -172,7 +172,7 @@ func TestTrimStageDoesNotFallBackToProcessedTranscript(t *testing.T) {
|
|||||||
|
|
||||||
func TestTrimStageFailsWhenNormalizedTranscriptInvalidJSON(t *testing.T) {
|
func TestTrimStageFailsWhenNormalizedTranscriptInvalidJSON(t *testing.T) {
|
||||||
env, m, _, _ := setupTrimEnv(t)
|
env, m, _, _ := setupTrimEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeFile(t, filepath.Join(paths.TranscriptsDir, "normalized.json"), "not-json")
|
writeFile(t, filepath.Join(paths.TranscriptsDir, "normalized.json"), "not-json")
|
||||||
_, err := (trimStage{}).Run(context.Background(), env, m)
|
_, err := (trimStage{}).Run(context.Background(), env, m)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
@@ -185,7 +185,7 @@ func TestTrimStageFailsWhenNormalizedTranscriptInvalidJSON(t *testing.T) {
|
|||||||
|
|
||||||
func TestTrimStageFailsWhenNormalizedTranscriptMissingSegmentsArray(t *testing.T) {
|
func TestTrimStageFailsWhenNormalizedTranscriptMissingSegmentsArray(t *testing.T) {
|
||||||
env, m, _, _ := setupTrimEnv(t)
|
env, m, _, _ := setupTrimEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeFile(t, filepath.Join(paths.TranscriptsDir, "normalized.json"), `{"schema":"audita.processed.v1"}`)
|
writeFile(t, filepath.Join(paths.TranscriptsDir, "normalized.json"), `{"schema":"audita.processed.v1"}`)
|
||||||
_, err := (trimStage{}).Run(context.Background(), env, m)
|
_, err := (trimStage{}).Run(context.Background(), env, m)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
@@ -198,7 +198,7 @@ func TestTrimStageFailsWhenNormalizedTranscriptMissingSegmentsArray(t *testing.T
|
|||||||
|
|
||||||
func TestTrimStageFailsWhenBoundsOutputInvalidJSON(t *testing.T) {
|
func TestTrimStageFailsWhenBoundsOutputInvalidJSON(t *testing.T) {
|
||||||
env, m, scr, _ := setupTrimEnv(t)
|
env, m, scr, _ := setupTrimEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeFile(t, filepath.Join(paths.TranscriptsDir, "normalized.json"), `{"segments":[{"id":1},{"id":2}]}`)
|
writeFile(t, filepath.Join(paths.TranscriptsDir, "normalized.json"), `{"segments":[{"id":1},{"id":2}]}`)
|
||||||
scr.BoundsBody = "not-json"
|
scr.BoundsBody = "not-json"
|
||||||
_, err := (trimStage{}).Run(context.Background(), env, m)
|
_, err := (trimStage{}).Run(context.Background(), env, m)
|
||||||
@@ -212,7 +212,7 @@ func TestTrimStageFailsWhenBoundsOutputInvalidJSON(t *testing.T) {
|
|||||||
|
|
||||||
func TestTrimStageFailsWhenBoundsRangeIsDescending(t *testing.T) {
|
func TestTrimStageFailsWhenBoundsRangeIsDescending(t *testing.T) {
|
||||||
env, m, scr, _ := setupTrimEnv(t)
|
env, m, scr, _ := setupTrimEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeFile(t, filepath.Join(paths.TranscriptsDir, "normalized.json"), `{"segments":[{"id":1},{"id":2}]}`)
|
writeFile(t, filepath.Join(paths.TranscriptsDir, "normalized.json"), `{"segments":[{"id":1},{"id":2}]}`)
|
||||||
scr.BoundsBody = `{"trim_action":"trim","start_segment_id":2,"end_segment_id":1}`
|
scr.BoundsBody = `{"trim_action":"trim","start_segment_id":2,"end_segment_id":1}`
|
||||||
_, err := (trimStage{}).Run(context.Background(), env, m)
|
_, err := (trimStage{}).Run(context.Background(), env, m)
|
||||||
@@ -226,7 +226,7 @@ func TestTrimStageFailsWhenBoundsRangeIsDescending(t *testing.T) {
|
|||||||
|
|
||||||
func TestTrimStageFailsWhenBoundsIDsMissingFromTranscript(t *testing.T) {
|
func TestTrimStageFailsWhenBoundsIDsMissingFromTranscript(t *testing.T) {
|
||||||
env, m, scr, _ := setupTrimEnv(t)
|
env, m, scr, _ := setupTrimEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeFile(t, filepath.Join(paths.TranscriptsDir, "normalized.json"), `{"segments":[{"id":20},{"id":21}]}`)
|
writeFile(t, filepath.Join(paths.TranscriptsDir, "normalized.json"), `{"segments":[{"id":20},{"id":21}]}`)
|
||||||
scr.BoundsBody = `{"trim_action":"trim","start_segment_id":10,"end_segment_id":21}`
|
scr.BoundsBody = `{"trim_action":"trim","start_segment_id":10,"end_segment_id":21}`
|
||||||
_, err := (trimStage{}).Run(context.Background(), env, m)
|
_, err := (trimStage{}).Run(context.Background(), env, m)
|
||||||
@@ -240,7 +240,7 @@ func TestTrimStageFailsWhenBoundsIDsMissingFromTranscript(t *testing.T) {
|
|||||||
|
|
||||||
func TestTrimStageFailsWhenScriptoriumAdapterFails(t *testing.T) {
|
func TestTrimStageFailsWhenScriptoriumAdapterFails(t *testing.T) {
|
||||||
env, m, scr, _ := setupTrimEnv(t)
|
env, m, scr, _ := setupTrimEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeFile(t, filepath.Join(paths.TranscriptsDir, "normalized.json"), `{"segments":[{"id":1},{"id":2}]}`)
|
writeFile(t, filepath.Join(paths.TranscriptsDir, "normalized.json"), `{"segments":[{"id":1},{"id":2}]}`)
|
||||||
scr.RunErr = errors.New("bounds failed")
|
scr.RunErr = errors.New("bounds failed")
|
||||||
_, err := (trimStage{}).Run(context.Background(), env, m)
|
_, err := (trimStage{}).Run(context.Background(), env, m)
|
||||||
@@ -254,7 +254,7 @@ func TestTrimStageFailsWhenScriptoriumAdapterFails(t *testing.T) {
|
|||||||
|
|
||||||
func TestTrimStageFailsWhenSeriatimTrimAdapterFails(t *testing.T) {
|
func TestTrimStageFailsWhenSeriatimTrimAdapterFails(t *testing.T) {
|
||||||
env, m, scr, ser := setupTrimEnv(t)
|
env, m, scr, ser := setupTrimEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
writeFile(t, filepath.Join(paths.TranscriptsDir, "normalized.json"), `{"segments":[{"id":1},{"id":2}]}`)
|
writeFile(t, filepath.Join(paths.TranscriptsDir, "normalized.json"), `{"segments":[{"id":1},{"id":2}]}`)
|
||||||
scr.BoundsBody = `{"trim_action":"trim","start_segment_id":1,"end_segment_id":2}`
|
scr.BoundsBody = `{"trim_action":"trim","start_segment_id":1,"end_segment_id":2}`
|
||||||
ser.TrimErr = errors.New("trim failed")
|
ser.TrimErr = errors.New("trim failed")
|
||||||
@@ -269,7 +269,7 @@ func TestTrimStageFailsWhenSeriatimTrimAdapterFails(t *testing.T) {
|
|||||||
|
|
||||||
func TestTrimStageDisabledCopiesNormalizedTranscript(t *testing.T) {
|
func TestTrimStageDisabledCopiesNormalizedTranscript(t *testing.T) {
|
||||||
env, m, scr, ser := setupTrimEnv(t)
|
env, m, scr, ser := setupTrimEnv(t)
|
||||||
paths := env.ArtifactStore.SessionPaths(m.SessionID)
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
normalized := filepath.Join(paths.TranscriptsDir, "normalized.json")
|
normalized := filepath.Join(paths.TranscriptsDir, "normalized.json")
|
||||||
normalizedBody := `{"segments":[{"id":1,"text":"alpha"},{"id":2,"text":"beta"}]}`
|
normalizedBody := `{"segments":[{"id":1,"text":"alpha"},{"id":2,"text":"beta"}]}`
|
||||||
writeFile(t, normalized, normalizedBody)
|
writeFile(t, normalized, normalizedBody)
|
||||||
@@ -298,6 +298,39 @@ func TestTrimStageDisabledCopiesNormalizedTranscript(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestTrimStageUsesRunLocalPathsAndPromotesCanonical(t *testing.T) {
|
||||||
|
env, m, scr, ser := setupTrimEnv(t)
|
||||||
|
env.Config.Session.Campaign = "sample-campaign"
|
||||||
|
m.Campaign = "sample-campaign"
|
||||||
|
m.RunID = "20260518T010203Z-abcdef12"
|
||||||
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
|
normalized := filepath.Join(paths.TranscriptsDir, "normalized.json")
|
||||||
|
writeFile(t, normalized, `{"segments":[{"id":10},{"id":11}]}`)
|
||||||
|
scr.BoundsBody = `{"trim_action":"trim","start_segment_id":10,"end_segment_id":11,"warnings":[]}`
|
||||||
|
|
||||||
|
result, err := (trimStage{}).Run(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("trim.Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(scr.RunRequests) != 1 || len(ser.TrimRequests) != 1 {
|
||||||
|
t.Fatalf("scriptorium run=%d seriatim trim=%d, want 1/1", len(scr.RunRequests), len(ser.TrimRequests))
|
||||||
|
}
|
||||||
|
if !strings.Contains(scr.RunRequests[0].OutputPath, filepath.Join("runs", m.RunID, "trim", "outputs")) {
|
||||||
|
t.Fatalf("bounds run output path = %q, want run-local path", scr.RunRequests[0].OutputPath)
|
||||||
|
}
|
||||||
|
if !strings.Contains(ser.TrimRequests[0].OutputTrimmedPath, filepath.Join("runs", m.RunID, "trim", "outputs")) {
|
||||||
|
t.Fatalf("trim output path = %q, want run-local path", ser.TrimRequests[0].OutputTrimmedPath)
|
||||||
|
}
|
||||||
|
if len(result.Outputs) < 2 {
|
||||||
|
t.Fatalf("outputs = %#v, want promoted trimmed+bounds outputs", result.Outputs)
|
||||||
|
}
|
||||||
|
for _, out := range result.Outputs {
|
||||||
|
if strings.Contains(out.AbsolutePath, string(filepath.Separator)+"runs"+string(filepath.Separator)) {
|
||||||
|
t.Fatalf("promoted output path = %q, want canonical session path", out.AbsolutePath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type boundsScriptoriumRunner struct {
|
type boundsScriptoriumRunner struct {
|
||||||
RunRequests []scriptorium.RunArtifactRequest
|
RunRequests []scriptorium.RunArtifactRequest
|
||||||
RenderRequests []scriptorium.RenderArtifactRequest
|
RenderRequests []scriptorium.RenderArtifactRequest
|
||||||
@@ -430,7 +463,7 @@ func setupTrimEnv(t *testing.T) (*Env, *manifest.Manifest, *boundsScriptoriumRun
|
|||||||
}
|
}
|
||||||
|
|
||||||
store := artifacts.NewLocalStore(workspace)
|
store := artifacts.NewLocalStore(workspace)
|
||||||
if _, err := store.EnsureLayout("2026-05-03"); err != nil {
|
if _, err := store.EnsureLayoutFor("sample-campaign", "2026-05-03"); err != nil {
|
||||||
t.Fatalf("EnsureLayout() error = %v", err)
|
t.Fatalf("EnsureLayout() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user