23 Commits

Author SHA1 Message Date
70f1dbfb59 Prepare the v2.2.1 release
All checks were successful
ci/woodpecker/push/verify Pipeline was successful
ci/woodpecker/tag/release Pipeline was successful
2026-09-22 18:18:45 +00:00
f1575effc2 Document shared checkpoint persistence 2026-09-22 15:13:46 +00:00
7e39a1d51f Delegate transcription checkpoints to shared persistence 2026-09-22 15:12:11 +00:00
3722e572df Delegate analyze checkpoints to shared persistence 2026-09-22 15:09:29 +00:00
7147e95252 Add shared stage checkpoint persistence 2026-09-22 15:05:25 +00:00
9f2cfddce6 Plan shared stage checkpoint mechanics 2026-09-22 14:54:34 +00:00
792cbd0677 Prepare the v2.2.0 release
All checks were successful
ci/woodpecker/push/verify Pipeline was successful
ci/woodpecker/tag/release Pipeline was successful
2026-09-22 01:50:49 +00:00
5dca34e0e0 Allow checkpoints for empty audio accepted by WhisperX 2026-09-22 00:53:28 +00:00
54d335c820 Repair stale transcription outputs when resuming 2026-09-22 00:50:51 +00:00
96b4e73f53 Document resumable transcription and replay 2026-09-22 00:31:32 +00:00
6e6054f744 Verify WhisperX transport replay 2026-09-22 00:22:48 +00:00
dc3288d15a Make WhisperX uploads replayable 2026-09-22 00:15:48 +00:00
d872edca94 Integrate transcription checkpoints across lifecycle 2026-09-22 00:00:51 +00:00
2ae63dbc2b Checkpoint speaker transcriptions incrementally 2026-09-21 16:20:53 +00:00
26aa0cce29 Reconcile reusable speaker transcriptions 2026-09-21 16:09:45 +00:00
7b0bf99ed1 Add durable speaker transcription state 2026-09-21 16:01:19 +00:00
89ada1b269 Plan resumable transcription and replayable uploads 2026-09-21 15:50:48 +00:00
3438fe82c9 Document Notarius configuration profiles 2026-09-21 15:16:11 +00:00
a51a513804 Verify Notarius profile lifecycle behavior 2026-09-21 15:11:19 +00:00
995354f1a5 Track Notarius profiles in extraction reuse 2026-09-21 15:07:36 +00:00
931c9591cd Add Notarius configuration profile support 2026-09-21 15:03:29 +00:00
46096fc4ba Plan Notarius configuration profile support 2026-09-21 14:54:47 +00:00
d3ecd907ee Update ConfigKit and prepare Narratio v2.1.2
All checks were successful
ci/woodpecker/push/verify Pipeline was successful
ci/woodpecker/tag/release Pipeline was successful
2026-09-21 13:42:21 +00:00
60 changed files with 5894 additions and 384 deletions

View File

@@ -240,8 +240,10 @@ inputs:
and a missing or empty explicit selection is an error. and a missing or empty explicit selection is an error.
- Each immediate regular file whose name is a valid environment-variable name - Each immediate regular file whose name is a valid environment-variable name
supplies one value. ConfigKit removes trailing CR/LF, permits a symlink as the supplies one value. ConfigKit removes trailing CR/LF, permits a symlink as the
selected directory, rejects symlink entries, and limits input to 1 MiB per selected directory, rejects non-hidden symlink entries, and limits input to
file, 8 MiB total, and 1024 entries. 1 MiB per file, 8 MiB total, and 1024 entries. Dot-prefixed entries (including
files, directories, and symlinks) are ignored and omitted from bootstrap
reports, but still count toward the directory entry limit.
- An existing process environment value takes precedence over a credential file. - An existing process environment value takes precedence over a credential file.
- On POSIX, operators must provision the directory with no group/other access - On POSIX, operators must provision the directory with no group/other access
(normally `0700`) and files with no group/other access (normally `0600`); (normally `0700`) and files with no group/other access (normally `0600`);
@@ -358,6 +360,7 @@ Rules:
| `pipeline.notarius.enabled` | bool | No | `false` | | `pipeline.notarius.enabled` | bool | No | `false` |
| `pipeline.notarius.binary` | string | No | `notarius` | | `pipeline.notarius.binary` | string | No | `notarius` |
| `pipeline.notarius.llm_api_key` | secret reference | No | unset; exact environment-variable name forwarded to Notarius | | `pipeline.notarius.llm_api_key` | secret reference | No | unset; exact environment-variable name forwarded to Notarius |
| `pipeline.notarius.config_profile` | string | No | unset; exact Notarius ConfigKit profile selector |
| `pipeline.notarius.config_path` | string | Conditional | required when enabled; relative paths resolve from the pipeline file directory | | `pipeline.notarius.config_path` | string | Conditional | required when enabled; relative paths resolve from the pipeline file directory |
| `pipeline.notarius.pipeline_id` | string | Conditional | required when enabled | | `pipeline.notarius.pipeline_id` | string | Conditional | required when enabled |
| `pipeline.notarius.timeout` | duration | No | `3h`; must be positive | | `pipeline.notarius.timeout` | duration | No | `3h`; must be positive |
@@ -380,6 +383,31 @@ Rules:
| `pipeline.scriptorium.artifact_families` | map | No | empty; expands one ordinary artifact per canonical party character | | `pipeline.scriptorium.artifact_families` | map | No | empty; expands one ordinary artifact per canonical party character |
| `pipeline.notification.mode` | string | No | `noop`; the only supported notification mode until a provider is implemented | | `pipeline.notification.mode` | string | No | `noop`; the only supported notification mode until a provider is implemented |
### Notarius Configuration Profile
`pipeline.notarius.config_profile` optionally selects a Notarius ConfigKit
configuration profile when extraction executes:
```yaml
notarius:
config_profile: downstream-economy
```
When the field is omitted or `null`, Narratio supplies no downstream profile
selector and preserves Notarius's existing configuration-selection behavior.
When present, Narratio forwards the exact string to Notarius, including an
empty string or `default`, which explicitly select Notarius's composed base
configuration. Notarius owns selector interpretation and validation; Narratio
does not restrict profile names, resolve profile files, or rewrite the value.
This selection is independent of Narratio's own `--profile` selection. A
Narratio overlay may explicitly select a differently named Notarius overlay,
but Narratio never forwards its own profile name automatically. It is also
independent of `pipeline.notarius.llm_api_key`: an overlay may change either or
both fields according to the deployed Notarius configuration. See the
[Notarius integration contract](./integrations/notarius.md) for invocation and
failure behavior.
### Notarius Credential ### Notarius Credential
`pipeline.notarius.llm_api_key` optionally names the one environment variable `pipeline.notarius.llm_api_key` optionally names the one environment variable

View File

@@ -30,12 +30,24 @@ contract introduced in Notarius v0.6.0. Cache controls require support for the
selected `--resume` and `--chunk_cache` flags. Notarius v0.7.0 is the tested selected `--resume` and `--chunk_cache` flags. Notarius v0.7.0 is the tested
cache integration baseline, not an unconditional minimum version; Narratio cache integration baseline, not an unconditional minimum version; Narratio
does not impose a runtime version gate. Each reference binding is a separate does not impose a runtime version gate. Each reference binding is a separate
argument before `--json`: argument before `--json`. When `pipeline.notarius.config_profile` is present,
its exact value is another argument pair before `--json`:
```text ```text
notarius run <pipeline_id> --config <config_path> --input <trimmed_json> --output-dir <staging_dir> [--resume] [--chunk_cache <mode>] [--reference <selector>=<verified_snapshot_path>]... --json notarius run <pipeline_id> --config <config_path> --input <trimmed_json> --output-dir <staging_dir> [--resume] [--chunk_cache <mode>] [--reference <selector>=<verified_snapshot_path>]... [--config-profile <name>] --json
``` ```
Notarius v0.7.0 is a known supporting version for `--config-profile`; using the
option requires a compatible Notarius executable, but deployments that omit the
field acquire no additional version requirement. Omitted or null configuration
adds no flag. Supplied empty, `default`, and named selectors are forwarded
without normalization so Notarius can apply its own ConfigKit semantics. This
option selects a Notarius configuration overlay, not a PromptKit profile through
`--llm-profile`, and Narratio never substitutes its own selected profile name.
A missing or invalid downstream overlay is an ordinary Notarius subprocess
failure: Narratio does not retry without the selector or fall back to another
profile.
Reference paths are absolute stable snapshots streamed from manifest-verified Reference paths are absolute stable snapshots streamed from manifest-verified
canonical files prepared inside the current Narratio session workspace. They canonical files prepared inside the current Narratio session workspace. They
live at `cache/notarius/references/<sha256>/<prepared-basename>` and retain the live at `cache/notarius/references/<sha256>/<prepared-basename>` and retain the

View File

@@ -17,11 +17,20 @@ Narratio sends an HTTP `POST` to the configured transcription URL using
The server must return a `2xx` response whose body is valid JSON. Narratio does The server must return a `2xx` response whose body is valid JSON. Narratio does
not currently require a more specific response schema at this boundary. not currently require a more specific response schema at this boundary.
The transcription URL must be an absolute `http` or `https` URL. The audio body The transcription URL must be an absolute `http` or `https` URL. Each
is streamed through a fresh multipart writer for every attempt, so its memory application attempt creates one bounded multipart framing and streams the audio
use is bounded by the transport buffer rather than by the complete audio file. from an open file between that framing; it does not buffer the complete audio
or spool a multipart copy. The file field precedes the language field.
WhisperX response acquisition is capped at 10 MiB. WhisperX response acquisition is capped at 10 MiB.
The initial request body and any transport-requested replay use the same
multipart boundary and framing. Every body opens an independent audio handle
and replay starts at the beginning. Narratio closes all bodies created for the
attempt on response, failure, timeout, or cancellation, including when a
transport abandons an earlier body. Genuine source read/close failures remain
request failures; deliberate early closure of an abandoned original does not
poison a valid replay.
## Request And Result Contract ## Request And Result Contract
Each adapter request identifies a speaker, a readable audio file, and the Each adapter request identifies a speaker, a readable audio file, and the
@@ -43,6 +52,11 @@ transcript output.
- Narratio waits the configured retry delay between attempts and aborts that - Narratio waits the configured retry delay between attempts and aborts that
wait when the parent context is canceled. wait when the parent context is canceled.
Transport replay occurs inside one application attempt and does not consume an
entry from the configured retry budget or increment the reported attempt
count. The standard HTTP redirect policy remains in effect; body replay can
also support body-preserving redirects.
## Validation And Failure Semantics ## Validation And Failure Semantics
Client construction rejects a missing or non-HTTP(S) absolute transcription URL, Client construction rejects a missing or non-HTTP(S) absolute transcription URL,
@@ -55,12 +69,19 @@ failure to install the output causes the transcription to fail. Errors include
attempt context, and the result retains attempts, final status when available, attempt context, and the result retains attempts, final status when available,
and elapsed duration for diagnostics. and elapsed duration for diagnostics.
Replay makes a fresh request body available to the HTTP transport; it does not
provide an exactly-once server guarantee and does not prove recovery from every
proxy or protocol-level failure. A server may have received work before the
client learns that a request failed. WhisperX operations should therefore be
safe to retry for the same input when practical.
## Determinism And Concurrency ## Determinism And Concurrency
Each audio request has stable multipart field names, and successful bytes are Each audio request has stable multipart field names, and successful bytes are
installed atomically. The transcribe stage may process speaker files in installed atomically. The transcribe stage may process speaker files in
parallel, bounded by the configured concurrency. It records results in stable parallel, bounded by the configured concurrency. Each validated speaker is
speaker order after all work completes; any speaker failure fails the stage. checkpointed independently; any speaker failure still fails the aggregate
stage, while committed earlier speakers remain reusable on an ordinary retry.
## Related Canonical Docs ## Related Canonical Docs

View File

@@ -50,6 +50,45 @@ The model admits these stage states:
- `stale` - `stale`
- `interrupted` - `interrupted`
### Transcribe-owned speaker state
A `transcribe` stage record may carry `transcribe_state_version: 1` and a
`transcribe_speakers` map keyed by the existing derived speaker ID. The version
is the authority marker: a versioned empty map means that no earlier speaker
completion is reusable. Absence of both fields is accepted as legacy
aggregate-only state so current Narratio can read older manifests.
Each map entry represents one completed speaker only. It binds a versioned
fingerprint to the session-relative audio path, audio SHA-256 and size, one
canonical `transcript_raw` output with its path, SHA-256 and size, and the
producer run ID. Paths must be canonical and session-relative, checksums must
be lowercase SHA-256 values, and the record and output producer identities
must agree. Other stages cannot carry this transcribe-owned state.
Audio size is nonnegative; WhisperX determines whether that audio can be
transcribed. Output size remains positive and the transcript must be valid JSON.
Transcribe reconciles this collection against the current prepared audio set
and observable WhisperX semantics. Missing or corrupt regular outputs become
work; unsafe or non-regular evidence is rejected. It saves the reconciled
collection before HTTP dispatch and saves a detached candidate after each
validated completion. Only after a save succeeds is the candidate installed as
live authority. Canonical bytes produced before an unsuccessful save are not a
completion record.
Aggregate transcribe status still describes the whole invocation. A failed,
running, stale, or interrupted aggregate may therefore retain reusable speaker
records, while aggregate outputs are declared only after the current
collection is complete. Ordinary retry reconciles and reuses those records.
Force first checkpoints an empty current collection, preventing later retries
from reviving pre-force results.
Older aggregate-only transcribe success is conservatively regenerated once
when transcribe is selected because it does not bind individual outputs to the
audio bytes used. The session-manifest schema identifier remains
`narratio.session-manifest.v1`; this additive state is accepted by current
Narratio, but older strict-decoding binaries may reject manifests after these
fields have been written.
### Analyze-owned artifact state ### Analyze-owned artifact state
Every `analyze` stage record carries `analyze_state_version: 1` and an Every `analyze` stage record carries `analyze_state_version: 1` and an
@@ -154,12 +193,31 @@ establish a current immutable snapshot.
- writes through a sibling temporary file and atomic replacement visibility; - writes through a sibling temporary file and atomic replacement visibility;
- updates `updated_at` on save. - updates `updated_at` on save.
### Checkpoint persistence boundary
Analyze and transcribe share one private checkpoint operation in
`internal/stage`. Their typed callers still decide when to checkpoint, validate
their collections, populate stage-owned fields, and serialize access. The
shared operation copies the manifest and its checkpoint-relevant mutable
collections, applies the typed update to the copied target record, and calls
the existing `manifest.Store` exactly once. Only after that save succeeds does
it promote the saved target record and manifest `updated_at` into live state;
it preserves an existing target record's identity and leaves unrelated live
stage records in place.
The copy boundary includes stage records, input and artifact slices, aggregate
output/log/configuration slices, and the typed analyze and transcribe
collections that callbacks or `LocalStore.Save` may modify. Metadata and other
pointer fields remain shared read-only because neither checkpoint callback nor
the store mutates them. This is a bounded execution helper, not a general
manifest clone or a transaction across transcript/artifact files.
Manifest replacement provides atomic visibility, not a crash- or power-loss Manifest replacement provides atomic visibility, not a crash- or power-loss
durability guarantee. Callers reload after a returned save error before retrying. durability guarantee. Callers reload after a returned save error before retrying.
An analyze checkpoint error leaves the prior manifest as per-artifact authority. A transcribe or analyze checkpoint error leaves the prior manifest collection
The runner then attempts to persist the bounded failed/stale retry collection; as authority. The runner then attempts to persist the aggregate failure; if
if that terminal save also fails, its error is joined with the originating that terminal save also fails, its error is joined with the originating failure
failure and the next invocation must reconcile from the last readable manifest. and the next invocation must reconcile from the last readable manifest.
## Execution Semantics ## Execution Semantics
@@ -193,11 +251,11 @@ only fields returned by the new result. Marking a record stale does not clear
those details because resume validation and diagnosis may still require them those details because resume validation and diagnosis may still require them
before execution begins. before execution begins.
Aggregate lifecycle clearing deliberately preserves the analyze-owned Aggregate lifecycle clearing deliberately preserves both the transcribe-owned
per-artifact collection. This lets later reconciliation replace only evaluated speaker collection and the analyze-owned per-artifact collection. This lets
entries without erasing unrelated current results. Other stages retain their later reconciliation replace only non-current entries without erasing durable
existing aggregate-only lifecycle behavior and are forbidden from carrying the completions. Other stages retain aggregate-only lifecycle behavior and are
analyze-specific fields. forbidden from carrying either collection.
A stage may explicitly return a skipped disposition and stable reason. The A stage may explicitly return a skipped disposition and stable reason. The
runner persists that outcome in the session manifest, clears older outputs, runner persists that outcome in the session manifest, clears older outputs,
@@ -228,7 +286,7 @@ evidence and focused durable validators:
| Work | Reuse authority | Focused owners | | Work | Reuse authority | Focused owners |
| --- | --- | --- | | --- | --- | --- |
| prepare | aggregate semantic fingerprint | [prepare](stage-prepare.md) | | prepare | aggregate semantic fingerprint | [prepare](stage-prepare.md) |
| transcribe | aggregate semantic fingerprint | [transcribe](stage-transcribe.md), [WhisperX](../integrations/whisperx.md) | | transcribe speakers | aggregate semantic fingerprint plus per-speaker input/output reconciliation | [transcribe](stage-transcribe.md), [WhisperX](../integrations/whisperx.md) |
| merge | aggregate semantic fingerprint | [merge](stage-merge.md), [Seriatim](../integrations/seriatim.md) | | merge | aggregate semantic fingerprint | [merge](stage-merge.md), [Seriatim](../integrations/seriatim.md) |
| polish | aggregate semantic fingerprint | [polish](stage-polish.md), [Audita](../integrations/audita.md) | | polish | aggregate semantic fingerprint | [polish](stage-polish.md), [Audita](../integrations/audita.md) |
| normalize | aggregate semantic fingerprint | [normalize](stage-normalize.md), [Seriatim](../integrations/seriatim.md) | | normalize | aggregate semantic fingerprint | [normalize](stage-normalize.md), [Seriatim](../integrations/seriatim.md) |
@@ -310,6 +368,7 @@ where the last session transition can require operator interpretation.
- Remote commit model and readers: `internal/artifacts/remote_commit.go`, - Remote commit model and readers: `internal/artifacts/remote_commit.go`,
`internal/artifacts/current_state_commit.go`, `internal/artifacts/current_state_commit.go`,
- Persistence and validation: `internal/manifest/store.go` - Persistence and validation: `internal/manifest/store.go`
- Transcribe speaker state: `internal/manifest/transcribe_state.go`
- Package tests: `internal/manifest/*_test.go` - Package tests: `internal/manifest/*_test.go`
- Assembled execution behavior: `internal/app/runner_test.go`, - Assembled execution behavior: `internal/app/runner_test.go`,
`internal/app/run_stage_test.go` `internal/app/run_stage_test.go`

View File

@@ -115,7 +115,11 @@ Supported source families:
- before the first adapter call, checkpoints the reconciled starting collection - before the first adapter call, checkpoints the reconciled starting collection
when it removed configuration records or cleared obsolete authority. After when it removed configuration records or cleared obsolete authority. After
each successful artifact, checkpoints the complete collection before making each successful artifact, checkpoints the complete collection before making
that artifact available to the next scheduled dependent. that artifact available to the next scheduled dependent. Analyze supplies
its typed update to the shared
[checkpoint persistence boundary](manifest.md#checkpoint-persistence-boundary)
and refreshes its working and committed collections only after the save
succeeds.
- keeps adapter output in the invocation's run-local analyze directory until - keeps adapter output in the invocation's run-local analyze directory until
it is a safe, non-empty, bounded regular file with a calculated checksum and it is a safe, non-empty, bounded regular file with a calculated checksum and
complete output contract. Canonical replacement uses the shared atomic file complete output contract. Canonical replacement uses the shared atomic file

View File

@@ -23,11 +23,13 @@ procedures belong in [Operations](../operations.md).
reusing verified bytes or reconstructing a missing/corrupt regular entry, reusing verified bytes or reconstructing a missing/corrupt regular entry,
and rejects any source change observed while copying; and rejects any source change observed while copying;
4. fingerprints the byte- and provenance-bearing Notarius invocation evidence, 4. fingerprints the byte- and provenance-bearing Notarius invocation evidence,
including sorted reference identities; including sorted reference identities and the explicit downstream
configuration-profile selection;
5. creates a run-local staging directory and invokes the injected 5. creates a run-local staging directory and invokes the injected
`notarius.Runner`, carrying the optional already-resolved credential `notarius.Runner`, carrying the optional already-resolved credential
reference without exposing its value, plus effective resume and chunk-cache reference without exposing its value, the optional exact configuration
controls after force precedence; profile, plus effective resume and chunk-cache controls after force
precedence;
6. revalidates the reference snapshots, then validates the v2 successful 6. revalidates the reference snapshots, then validates the v2 successful
receipt, confined index, management documents, configured required lane receipt, confined index, management documents, configured required lane
descriptors, validation summaries, and regular payload files; descriptors, validation summaries, and regular payload files;
@@ -70,7 +72,9 @@ outputs is stable and does not repeatedly invalidate dependent stages.
Before the focused validator runs, the application compares extract's versioned Before the focused validator runs, the application compares extract's versioned
semantic fingerprint. It covers enablement, Notarius pipeline identity, sorted semantic fingerprint. It covers enablement, Notarius pipeline identity, sorted
reference selector/source mappings, sorted declared output contracts, and each reference selector/source mappings, sorted declared output contracts, and each
canonical `narratio.extraction.<key>` output identity. It excludes executable, canonical `narratio.extraction.<key>` output identity. It also covers the
explicit Notarius configuration-profile selection; omitted, empty, and exact
`default` selectors share the base-selection identity. It excludes executable,
timeout, working directory, config path, and private Notarius config contents. timeout, working directory, config path, and private Notarius config contents.
Credential presence, reference name, and resolved value are likewise Credential presence, reference name, and resolved value are likewise
operational-only and excluded. Ordinary command loading still resolves a operational-only and excluded. Ordinary command loading still resolves a
@@ -81,13 +85,15 @@ not rerun a valid completed extraction or stale its dependents.
stage record still matches the current byte- and provenance-bearing invocation stage record still matches the current byte- and provenance-bearing invocation
evidence. That evidence covers the current direct trimmed-transcript identity, evidence. That evidence covers the current direct trimmed-transcript identity,
sorted prepared-reference identities, pipeline identity, and configured output sorted prepared-reference identities, pipeline identity, and configured output
contracts. The same reference helper and transcript identity are resolved again contracts, plus the same downstream profile-selection identity. The same
for artifact evidence, so changing current transcript bytes, reference bytes, reference helper and transcript identity are resolved again for artifact
or producer identity makes the prior extraction obsolete. Operational runner evidence, so changing current transcript bytes, reference bytes, producer
settings do not invalidate otherwise current durable evidence. This includes identity, or a named downstream selection makes the prior extraction obsolete.
`resume` and `chunk_cache`; changing them alone cannot make a complete Operational runner settings do not invalidate otherwise current durable
extraction non-resumable. If extraction executes, ordinary runs forward the evidence. This includes `resume` and `chunk_cache`; changing them alone cannot
configured values, while force disables resume and selects `refresh`. make a complete extraction non-resumable. If extraction executes, ordinary
runs forward the configured values, while force disables resume and selects
`refresh`.
A valid prepared-reference change makes extraction non-resumable. Missing, A valid prepared-reference change makes extraction non-resumable. Missing,
unsafe, or checksum-inconsistent prepared evidence is a hard validation error unsafe, or checksum-inconsistent prepared evidence is a hard validation error
@@ -100,10 +106,12 @@ contracts and provenance, regular-file status, and stored checksums. Missing or
obsolete results are non-resumable and run again; unsafe filesystem conditions obsolete results are non-resumable and run again; unsafe filesystem conditions
return an error rather than silently accepting or replacing data. return an error rather than silently accepting or replacing data.
Neither contract can observe files imported by Notarius configuration, profile This sensitivity is limited to the explicit downstream selector. Narratio's
contents, prompt/module definitions, or other transitive inputs. Operators must own profile label and whole effective-configuration digest are not blanket
force extraction after changing any such private input behind a stable extraction invalidators. Neither contract can observe files imported by
identifier. Notarius configuration, profile contents, prompt/module definitions, or other
transitive inputs. Operators must force extraction after changing any such
private input behind a stable identifier.
## Failure Behavior ## Failure Behavior

View File

@@ -17,7 +17,11 @@ Normalize raw transcript inputs and merge into base transcript via Seriatim.
## Key Behavior ## Key Behavior
- discovers and validates raw transcript inputs. - discovers and validates raw transcript inputs. For versioned transcribe
state, discovery requires aggregate transcribe success and consumes only the
aggregate outputs that exactly match the current per-speaker collection.
Partial checkpoints and orphaned files are not merge inputs. Legacy/manual
workflows without versioned speaker state retain directory discovery.
- normalizes each raw transcript (`seriatim.Normalize`) into run-local scratch output. - normalizes each raw transcript (`seriatim.Normalize`) into run-local scratch output.
- merges normalized inputs (`seriatim.Run`) into base transcript. - merges normalized inputs (`seriatim.Run`) into base transcript.
- validates merged transcript and optional report JSON. - validates merged transcript and optional report JSON.
@@ -26,6 +30,8 @@ Normalize raw transcript inputs and merge into base transcript via Seriatim.
## Invariants ## Invariants
- merge always consumes normalized forms of raw inputs. - merge always consumes normalized forms of raw inputs.
- a versioned transcribe collection is usable only through its complete,
successful aggregate output declaration.
- base transcript must validate before stage success. - base transcript must validate before stage success.
- report output is config-gated. - report output is config-gated.

View File

@@ -14,12 +14,19 @@ Generate raw per-speaker transcripts from prepared audio using WhisperX.
## Key Behavior ## Key Behavior
- discovers prepared audio from manifest inputs or canonical audio directory. - discovers prepared audio from manifest inputs or the canonical audio
- derives the transcript identity from the prepared `.flac` filename. directory and derives each speaker identity from its `.flac` filename.
- dispatches one WhisperX request per prepared file through a bounded errgroup. - hashes the current audio and reconciles it with the versioned per-speaker
- validates each output as JSON. completion collection before starting HTTP work.
- writes run-local outputs then materializes canonical transcript outputs only - reuses only records whose audio identity, observable WhisperX semantics,
after every planned request succeeds. canonical output location, JSON, checksum, and size remain current.
- durably checkpoints the reconciled collection before dispatch, then sends
only missing or obsolete speakers through the bounded worker group.
- validates each successful run-local response, rechecks the audio bytes,
materializes the canonical raw transcript, and checkpoints that speaker
before accepting another completion.
- rebuilds aggregate stage outputs in stable speaker order from the complete
current collection, including records produced by earlier invocations.
## Invariants ## Invariants
@@ -28,21 +35,49 @@ Generate raw per-speaker transcripts from prepared audio using WhisperX.
- output path returned by adapter must match requested output path. - output path returned by adapter must match requested output path.
- an empty adapter result path means the requested path; adapters cannot select - an empty adapter result path means the requested path; adapters cannot select
an alternate destination. an alternate destination.
- each successful output is validated before stage success, and cancellation or - candidate collection construction, the shared
incomplete dispatch cannot be reported as a successful result. [checkpoint persistence boundary](manifest.md#checkpoint-persistence-boundary),
and installation into live state are serialized by transcribe's mutex.
Expensive hashing and HTTP work remain outside that critical section.
- a canonical file has no completion authority until its manifest checkpoint
succeeds. A save failure preserves the last successfully committed
collection for ordinary retry.
- each successful output is validated before its checkpoint, and cancellation,
incomplete dispatch, or a failed aggregate save cannot be reported as
aggregate stage success.
- removed audio inputs are omitted from the current collection and aggregate
output list; reconciliation does not delete arbitrary transcript files.
- merge cannot consume a partial versioned collection: it requires transcribe
aggregate success and an exact match between current speaker records and
declared aggregate outputs.
## Resume Evidence ## Resume Evidence
Transcribe records a versioned semantic-configuration fingerprint containing Each completed speaker record binds the speaker ID, session-relative audio
the Narratio-visible WhisperX service URL and recognition language. Changes to path, streamed audio checksum and size, the Narratio-visible WhisperX service
either rerun transcription and stale its fixed descendants while leaving URL and recognition language, and the canonical transcript path, checksum,
prepare reusable. Retry count/delay, concurrency, timeout, credentials, and size, and producer run. Changed audio bytes select that speaker for replacement;
diagnostic locations are operational and do not change this evidence. a changed URL or language selects every speaker. Either case stales fixed
descendants through the normal runner decision. Retry count/delay, concurrency,
timeout, credentials, and diagnostic locations are operational and do not
change this evidence.
An ordinary retry reuses every valid speaker checkpoint even when aggregate
transcribe status is failed, running, or stale. A succeeded stage is skipped only
when its stored speaker set and aggregate output evidence also match the current
collection. Otherwise, execution repairs those records without retranscribing
reusable speakers and stales dependents through the normal runner path.
`--force` first durably replaces
the collection with an empty versioned collection, so a later ordinary retry
cannot resurrect pre-force results; speakers checkpointed by that forced
attempt remain reusable.
WhisperX models or private service configuration not exposed by Narratio's WhisperX models or private service configuration not exposed by Narratio's
adapter contract cannot be fingerprinted; use `--force` after changing them. adapter contract cannot be fingerprinted; use `--force` after changing them.
An existing successful transcribe record without evidence reruns once when An older aggregate-only transcribe record has no trustworthy audio-to-output
selected. binding. When transcribe is next selected, every current speaker is therefore
generated once and recorded in the versioned collection; later invocations can
reuse it normally.
## Related Contracts And Tests ## Related Contracts And Tests
@@ -51,6 +86,9 @@ selected.
- [Configuration](../config.md#pipeline) owns concurrency and other - [Configuration](../config.md#pipeline) owns concurrency and other
operator-selected values. operator-selected values.
- Implementation and tests: `internal/stage/transcribe.go`, - Implementation and tests: `internal/stage/transcribe.go`,
`internal/stage/transcribe_reconciliation.go`,
`internal/stage/transcribe_execution_state.go`,
`internal/stage/transcribe_resume.go`,
`internal/stage/transcribe_test.go`, `internal/stage/transcribe_test.go`,
`internal/stage/semantic_contracts_initial.go`, and `internal/stage/transcribe_reconciliation_test.go`, and
`internal/stage/semantic_contracts_initial_test.go` `internal/stage/transcribe_checkpoint_test.go`

View File

@@ -116,8 +116,8 @@ Execution rules:
perpetually rerun dependent work. perpetually rerun dependent work.
Every aggregate stage except analyze currently provides semantic-configuration Every aggregate stage except analyze currently provides semantic-configuration
evidence; analyze retains its more precise per-artifact fingerprints and evidence; transcribe additionally reconciles per-speaker completion evidence,
validator. while analyze retains its more precise per-artifact fingerprints and validator.
Prepare additionally validates current stable/local/S3 source identity and the Prepare additionally validates current stable/local/S3 source identity and the
checksums of its durable prepared copies before reuse. Changed bytes, audio checksums of its durable prepared copies before reuse. Changed bytes, audio
membership, S3 object identity, or missing/tampered copies rerun prepare and membership, S3 object identity, or missing/tampered copies rerun prepare and
@@ -187,6 +187,54 @@ shared session lifecycle. In particular, render does not require Notarius or
Scriptorium, extract does not require Scriptorium, and analyze does not require Scriptorium, extract does not require Scriptorium, and analyze does not require
the transcription, Seriatim, Audita, or Notarius adapters. the transcription, Seriatim, Audita, or Notarius adapters.
### Transcription continuation and replacement
Transcribe checkpoints each validated speaker independently. If a later
speaker exhausts HTTP retries, the process is canceled, or the aggregate stage
save fails, the aggregate stage is not successful and downstream merge cannot
consume the partial set. The last successfully saved speaker records and their
canonical `transcripts/raw/<speaker>.json` files remain durable, however.
After correcting the underlying problem, retry normally:
```bash
narratio run-stage transcribe 2026-04-04
```
Narratio hashes the current prepared audio, validates recorded transcript JSON,
checksum, size, and location, and calls WhisperX only for missing or obsolete
speakers. A changed audio file replaces only its speaker; a changed WhisperX
URL or language makes every current speaker non-reusable. Retry, timeout, and
concurrency tuning do not invalidate completed speakers.
Use force only when every current transcript must be replaced, including after
a WhisperX model or private server-side configuration changes behind the same
URL and language:
```bash
narratio run-stage transcribe 2026-04-04 --force
```
Force durably clears the reusable collection before sending requests. If that
forced attempt fails, a later ordinary retry reuses only replacement speakers
checkpointed after the reset; it does not revive pre-force results. Do not edit
speaker checksums or copy older JSON into the canonical directory to influence
reuse.
The first current Narratio invocation that selects transcribe for an older
aggregate-only manifest retranscribes every current speaker once because the
old record cannot prove which audio bytes produced each file. Subsequent runs
reuse the new versioned evidence normally. The overall session-manifest schema
identifier is unchanged, but older strict-decoding Narratio binaries may reject
a manifest after current Narratio writes the additive speaker fields; retain a
backup or complete the upgrade before relying on downgrade.
Restoring a manifest does not implicitly publish or restore every raw speaker
file. If a recorded canonical transcript is absent locally, ordinary
reconciliation schedules that speaker again. Removing run directories does not
remove canonical checkpointed transcripts and does not by itself invalidate
them.
## Artifact Development Workflows ## Artifact Development Workflows
Choose the narrowest workflow that matches the intended replacement: Choose the narrowest workflow that matches the intended replacement:
@@ -233,6 +281,30 @@ earlier checkpoints are reused and execution resumes with failed, stale, or
missing work in dependency order. `--force` is unnecessary unless the operator missing work in dependency order. `--force` is unnecessary unless the operator
intends to replace otherwise-current artifacts. intends to replace otherwise-current artifacts.
### Developing With A Notarius Configuration Profile
A Narratio development overlay can explicitly select a Notarius overlay without
requiring the names to match. For example, `profiles/testing.yml` may contain:
```yaml
notarius:
config_profile: downstream-economy
```
Selecting Narratio's `testing` profile then supplies `downstream-economy` only
when extraction executes. Switching that explicit selector makes a recorded
extraction non-current and reruns it when `extract` is in the selected stage
range; the usual dependent invalidation then applies. Selecting a profile does
not create a separate workspace or retain a separate current extraction result.
Narratio cannot observe edits inside Notarius configuration behind an unchanged
selector. Force extraction after such an edit when a replacement is intended.
Use `regenerate-artifacts` when both extraction and analysis must be rebuilt;
use `analyze` when the existing extraction bundle should be preserved. Field
semantics belong in [Configuration](./config.md#notarius-configuration-profile),
and the exact downstream invocation belongs in the
[Notarius integration contract](./integrations/notarius.md#invocation).
## Artifact Selection ## Artifact Selection
For a configured artifact family, selecting its family key expands to every For a configured artifact family, selecting its family key expands to every
@@ -377,13 +449,13 @@ narratio run-stage extract 2026-04-04 --force
Narratio automatically reruns extraction when its recorded invocation contract, Narratio automatically reruns extraction when its recorded invocation contract,
prepared Narratio reference identities, or durable output validation changes. prepared Narratio reference identities, or durable output validation changes.
The semantic portion covers Notarius enablement, pipeline identity, declared The semantic portion covers Notarius enablement, pipeline identity, declared
reference mapping, and output contracts. Executable, timeout, working directory, reference mapping, output contracts, and the explicit Notarius configuration
and private config-file paths are operational and do not invalidate a current profile selector. Executable, timeout, working directory, and private
result. config-file paths are operational and do not invalidate a current result.
It cannot fingerprint configuration files, profiles, prompts, modules, or It cannot fingerprint configuration or profile contents, prompts, modules, or
other references loaded transitively by Notarius itself. Force extraction after other references loaded transitively by Notarius itself. Force extraction after
changing any of those inputs, even when the top-level Narratio and Notarius changing any of those inputs behind an unchanged selector, even when the
config paths remain the same. A forced extract top-level Narratio and Notarius config paths remain the same. A forced extract
marks successful downstream stages stale. Ordinary extraction failures or marks successful downstream stages stale. Ordinary extraction failures or
outcome changes also stale affected downstream stages, while an identical outcome changes also stale affected downstream stages, while an identical
repeated `notarius_disabled` self-skip does not repeatedly invalidate them. repeated `notarius_disabled` self-skip does not repeatedly invalidate them.

26
docs/releases/v2.1.2.md Normal file
View File

@@ -0,0 +1,26 @@
# Narratio v2.1.2
This release allows secrets directories to contain hidden metadata without
preventing credential loading.
## Summary
ConfigKit v0.3.1 ignores dot-prefixed entries such as `.git`, `.gitignore`, and
`.DS_Store` during secrets bootstrap. These entries no longer block otherwise
valid credentials from loading and are omitted from bootstrap reports.
## Compatibility
This is a backward-compatible dependency bug fix. Configuration schemas,
existing-environment precedence, and validation of non-hidden entries are
unchanged. Hidden entries still count toward the directory entry limit.
## Upgrade
No special upgrade action or configuration migration is required. See the
[credential configuration reference](../config.md) for secrets-directory usage.
## Changes
- Updated ConfigKit from v0.3.0 to v0.3.1.
- Clarified hidden-entry handling in the secrets-directory documentation.

53
docs/releases/v2.2.0.md Normal file
View File

@@ -0,0 +1,53 @@
# Narratio v2.2.0
This release adds resumable per-speaker transcription, replayable WhisperX
uploads, and explicit Notarius configuration profile selection.
## Summary
Successfully checkpointed speaker transcripts survive later transcription
failures. Ordinary retry reuses valid completed speakers and processes only the
remaining work. WhisperX uploads can now be replayed by the HTTP transport
without buffering whole audio files.
## Compatibility
Existing CLI commands, speaker identities, transcript formats, and pipeline
ordering are unchanged. Session manifests gain optional per-speaker completion
evidence. Older manifests remain readable, but older Narratio binaries may
reject manifests containing the new fields; downgrading after transcription
is not transparent.
The optional `pipeline.notarius.config_profile` field selects a downstream
Notarius configuration profile independently of Narratio's own profile. Using
it requires a Notarius version supporting `--config-profile`; v0.7.0 is a known
supporting version. Omitting the field retains downstream default selection.
## Upgrade
No configuration migration is required. When transcribe is next selected for
an older aggregate-only manifest, all current speakers are transcribed once
to establish reusable evidence. Previously unrecorded results from failed runs
are not automatically imported.
Use ordinary run/resume to retain valid checkpoints. Forcing transcribe resets
reuse of prior results; an ordinary retry after a failed forced invocation can
retain replacements already checkpointed by that attempt. See the
[operations guide](../operations.md) and
[Notarius profile configuration](../config.md#notarius-configuration-profile).
## Changes
- Persist validated speaker completions independently of aggregate stage status,
with serialized saves for concurrent work and input/output integrity checks.
- Rebuild aggregate transcript references when speakers are removed or those
references need repair, without retranscribing otherwise-current speakers.
- Keep removed and incomplete speaker results out of versioned merge inputs.
- Permit checkpoint evidence for zero-byte audio when WhisperX accepts it;
service rejection still fails normally.
- Provide fresh streaming multipart bodies for HTTP transport replay while
preserving application retries, cancellation, and resource cleanup. This
addresses the missing-body-replay limitation, not persistent upstream
protocol failures or exactly-once service execution.
- Pass explicit Notarius configuration profiles through the CLI and include
their selection in extraction reuse decisions.

35
docs/releases/v2.2.1.md Normal file
View File

@@ -0,0 +1,35 @@
# Narratio v2.2.1
This maintenance release consolidates checkpoint persistence for transcription
and analysis without changing their operator-facing behavior.
## Summary
Transcribe and analyze now share one internal operation for preparing isolated
checkpoint candidates, saving them, and promoting successfully saved progress
into live state. Each stage retains its own progress model and recovery policy.
## Compatibility
CLI commands, configuration, manifest schemas, artifact formats, and external
integration contracts are unchanged. Speaker concurrency, artifact selection,
force behavior, and ordinary resume semantics are preserved. No dependency
upgrade or new external-tool version requirement is introduced.
## Upgrade
No special upgrade action or configuration migration is required when upgrading
from v2.2.0. See the [v2.2.0 release notes](v2.2.0.md) when upgrading from an
earlier version for the original speaker-checkpoint upgrade considerations.
## Changes
- Replace duplicate analyze/transcribe save-and-promote sequences with a
shared, private checkpoint helper.
- Isolate checkpoint collections from live state, including analyze records
normalized during a transcription checkpoint save.
- Preserve transcription's caller-owned locking and failure handling, and
analysis's working/committed collections and rollback behavior.
- Add focused checkpoint integrity tests while retaining stage and application
recovery coverage.
- Document the shared [checkpoint persistence boundary](../internal/manifest.md#checkpoint-persistence-boundary).

View File

@@ -0,0 +1,102 @@
---
title: Select a Notarius configuration profile through Narratio
status: proposed
consumer: rakestraw.dnd Ansible collection and its pipeline role
use_case: >-
The Ansible pipeline role deploys Narratio and Notarius as independently
configured applications. Operators select Narratio's testing profile to
develop prompts and other pipeline assets with inexpensive models while
retaining production settings in the base configuration. That selection
currently cannot select a Notarius configuration profile, leaving extraction
on its production settings.
desired_outcome: >-
Narratio's testing overlay can explicitly select Notarius's testing overlay.
Notarius owns the resulting model choices and any other testing-specific
settings. The operator selects testing once when invoking Narratio, without
editing production files or separately invoking Notarius.
rationale: >-
Both applications already use ConfigKit configuration overlays. Selecting
the downstream overlay preserves each application's configuration ownership
and supports future Notarius testing adjustments without adding a Narratio
configuration field for every downstream setting. This is explicit profile
composition, not a requirement for arbitrary recursive orchestration.
requested_interface:
field: pipeline.notarius.config_profile
value: Optional Notarius ConfigKit configuration profile name
supplied: >-
Invoke Notarius with --config-profile and the selected name when extraction
executes. The setting is available in Narratio's base configuration and
its ordinary configuration overlays.
omitted_or_null: >-
Do not supply a profile-selection flag. Preserve existing Notarius
configuration selection behavior.
downstream_semantics: >-
Retain Notarius's existing selector semantics, including the default
selector for its base configuration. Notarius resolves profiles relative
to its own configuration root, not Narratio's root.
explicit_selection: >-
Do not automatically forward Narratio's profile name. The selected
Notarius profile may have the same name or a different name.
distinction: >-
This selects a ConfigKit configuration overlay through --config-profile,
not a PromptKit LLM profile through --llm-profile. The Notarius overlay
can itself select the desired LLM profiles using native Notarius settings.
illustrative_consumer_configuration:
note: >-
These are separate files managed by the Ansible suite, not a proposed
combined configuration document. The Notarius pipeline and LLM profile
names are illustrative; the referenced LLM profile must be available to
Notarius.
narratio_profiles_testing_yml: |
notarius:
config_profile: testing
notarius_profiles_testing_yml: |
pipelines:
dnd-session:
llm_profile: gemini-flash-latest
operator_workflow: >-
Select --profile testing on the ordinary Narratio invocation. Narratio
selects Notarius's testing configuration overlay when it runs extraction;
other Narratio testing overrides continue to apply normally.
observable_behavior:
- >-
Notarius owns overlay discovery, composition, validation, and the effective
downstream settings. A missing or invalid selected overlay must not silently
fall back to production or be reported as successful extraction.
- >-
Changing the selected Notarius configuration profile must not silently
reuse Narratio's completed extraction from the previous selection.
Existing downstream invalidation behavior applies when extraction changes.
- >-
Editing Notarius overlay contents behind an unchanged selector retains
the existing operator responsibility to force extraction when needed.
This request does not require inspecting or tracking private Notarius files.
- >-
Existing credential forwarding remains independently configurable through
notarius.llm_api_key, including in the Narratio testing overlay when the
selected Notarius settings require a different credential reference.
consumer_follow_up: >-
After Narratio releases this interface, the standalone Narratio Ansible role
can expose an optional variable and omit the field when unset. The suite role
can deploy Notarius's testing overlay and select it from Narratio's testing
overlay, then distribute the changes through the collection. These consumer
updates are separate from this upstream feature request.
scope_boundaries:
- No Notarius change is currently required; v0.7.0 supports --config-profile.
- No separate Narratio-side Notarius LLM-profile selector is requested.
- No automatic profile-name propagation or downstream configuration rewriting.
- No new credential discovery, copying, or provider-name remapping.
- This request specifies consumer behavior, not internal Narratio code changes.
references:
narratio_configuration: ../config.md
narratio_notarius_integration: ../integrations/notarius.md
notarius_cli: https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/cli.md

View File

@@ -0,0 +1,333 @@
# Implementation Plan: Shared Stage Checkpoint Mechanics
Status: Complete.
## Scope and Authority
Implement the agreed follow-up to v2.2.0: consolidate the thin checkpoint
mechanism shared by transcribe and analyze without combining their progress
models or changing operator behavior. There is no separate consolidation
roadmap in this checkout. This plan uses the accompanying discussion as its
scope: persistence is already centralized; candidate isolation and
save-before-promotion are the duplication to remove. The completed
[transcription roadmap](transcribe-checkpoints-and-replay.md) provides background,
not authorization to reimplement transcription or HTTP replay. The preceding
completed implementation plan remains in Git history.
Complete the numbered stages in order, one bounded implementation prompt per
stage for gpt-5.6-terra. Record completion and actual verification results here.
This planning pass changes documentation only; the steps below are for the
subsequent implementation sprint.
Before each stage, read `AGENTS.md`, `docs/development.md`, and all three
documents in `docs/policy/`: architecture, documentation, and testing. Use
maintained Go 1.26. Prefer graph tools for code discovery; inspect current
files if their index is stale. Read the stage-specific preparation below,
rather than unrelated historical roadmaps.
## Target and Scope Boundaries
The existing `internal/manifest.Store` and `LocalStore` remain the session
manifest persistence interface and implementation. Encoding, schemas,
validation, normalization, and atomic file replacement remain unchanged.
There is no new store, journal, file format, dependency, command, configuration,
or migration.
One private helper in `internal/stage` owns candidate preparation, calling the
existing store, and promoting the selected stage record only after successful
save. The existing typed analyze/transcribe checkpoint methods become thin
callers. This is execution plumbing: keep it beside the callers rather than
moving orchestration into the storage package or exporting a reusable API.
Keep these responsibilities stage-owned:
- Transcribe's completed-only speaker map, evidence checks, bounded concurrency,
mutex, checkpoint-failure latch, cancellation, pre-dispatch force reset,
output installation, and aggregate repair.
- Analyze's dispositions, selection, dependencies, catalog updates, working and
committed collections, failure reconciliation, invalidation, and outputs.
- Each stage's decision whether/when to checkpoint, typed validation, contextual
errors, and post-save working-state updates.
- Runner aggregate transitions, session locking, terminal failure saves, and
fixed dependent-stage invalidation.
Do not introduce a common subtask model, scheduler, transaction manager, global
mutex, automatic retry, additional checkpoint, or batching policy. Do not
migrate publish/restore, planning clones, cleanup, or ordinary runner saves:
their operations are not the demonstrated duplication. Leave HTTP replay and
all adapters unchanged.
## Selected Design
### Shared Save-and-Promote Operation
Add `internal/stage/checkpoint.go` with a private helper shaped as:
```go
func checkpointStageCollection(
ctx context.Context,
store manifest.Store,
path string,
live *manifest.Manifest,
stageName string,
update func(*manifest.StageRecord) error,
) error
```
Names are implementation choices, not public contracts. Use a function and
small copying helpers, not a factory/options/interface framework. The two
existing typed checkpoint methods are the only production callers. They pass
the already-resolved manifest path; the helper does not discover paths or
inspect configuration.
The helper performs this sequence:
1. Check required live/store inputs without mutation. Copy a checkpoint
candidate and select a copied target record. If absent, provide an empty
candidate record to the caller, without inferring its name, status, or time.
2. Apply the supplied update to that candidate record only. On update error,
return without saving or promoting. The typed caller validates and fills
its version, collection, and stage-owned initialization/timestamps.
3. Invoke `Store.Save` once with the supplied context. Preserve error identity
through `%w` and useful path context. On error, do not promote, retry,
reload, or synthesize aggregate failure.
4. After success, install the saved target record and candidate manifest
`UpdatedAt` into live state. Preserve an existing target record's pointer
identity by assigning through it. Allocate missing live maps/records only
after success. Do not replace the entire live manifest or overwrite other
stage records. Copy mutable checkpoint collections on promotion so saved
candidates and caller working collections cannot mutate live progress.
Callers serialize the whole operation. Transcribe's current mutex does so;
analyze executes artifacts sequentially. Do not add a lock to the helper or
promise support for concurrent independent manifest writers. Atomic replacement
visibility does not become power-loss durability or a transaction covering
transcript files.
### Bounded Candidate Isolation
Factor `cloneManifestForTranscribeCheckpoint` into a shared checkpoint-scoped
copy helper in the new file. Use ordinary typed copies and existing
`manifest.CloneAnalyzeArtifactCollection` and
`manifest.CloneTranscribeSpeakerCollection`. Do not introduce reflection
cloning, JSON round trips, or a public general-purpose clone API.
Copy the manifest value, stages map, stage records, and mutable collections
touched by these updates or `LocalStore.Save`. In particular, Save normalizes
analyze collections throughout the candidate, including on a transcribe
checkpoint. Clone those collections for every stage so normalization cannot
leak into live state on failure. Preserve the existing bounded copies of
input/artifact and output/log/config slices. Retain nil entries for existing
validation to reject, rather than silently repairing invalid manifests.
This is not arbitrary deep isolation of metadata. Neither the two update
callbacks nor the current Store mutates `map[string]any` metadata, so it may
remain shared read-only. Document that boundary and inspect every current
normalization write. Use existing typed clones for nested checkpoint evidence;
do not restrict legal metadata values or alter their Go/JSON representations
to simplify copying.
### Differences to Preserve
- Analyze still requires an existing stage record. Its callback changes only
analyze version/collection, not transcribe-style initialization or timestamp
policy. It refreshes working and committed collections only after success.
- Transcribe retains missing-record initialization for direct stage calls:
its typed caller supplies name, running status, and initial timestamps when
needed. Its callback retains the existing stage `UpdatedAt` update. Copying
an existing record must not clear aggregate fields or invoke lifecycle
transitions.
- Existing transcribe live-record initialization is incomplete on the unusual
missing-record path; promoting the actual successfully saved candidate makes
its identity agree with disk. This does not add a new missing-record
rejection or change ordinary runner status transitions.
- Analyze rollback/failure handling and transcribe failure latching remain in
their execution owners. The shared operation never invents failed items.
- Preserve starting-checkpoint elision, force/reset semantics, empty versioned
collections, evidence and producer identity, legacy transcription handling,
supported zero-sized audio, and aggregate repair without HTTP calls.
- Do not freeze exact error wording, wall-clock values, copy choreography, or
iteration order into new contracts. No new schema, limits, filename rules,
statuses, or compatibility guarantees are authorized.
## Stage 1: Add the Shared Checkpoint Operation
Status: Complete.
Completion marker (2026-09-22): Added the private stage-level checkpoint
save-and-promote helper and bounded candidate copying without migrating either
production caller. Focused real-store coverage proves live isolation through
updates, validation, save-time normalization and timestamps, save failures,
missing-target creation, pointer-preserving promotion, unrelated aggregate
preservation, caller-copy isolation, context/error identity, and exact one-save
behavior. Verified with `go test ./internal/manifest ./internal/stage`,
`go test -race ./internal/stage`, and `git diff --check`.
Preparation: `internal/manifest/store.go`, `manifest.go`, `analyze_state.go`,
`transcribe_state.go`; `internal/stage/analyze_execution_state.go`,
`transcribe_execution_state.go`; `docs/internal/manifest.md`.
1. Add the private helper and bounded candidate copying described above.
Leave existing production checkpoint paths unchanged until their migration
stages. Do not duplicate manifest record-validation rules.
2. Add focused `internal/stage/checkpoint_test.go` coverage with real
`manifest.LocalStore` and temporary directories for successful save/reload.
Use a small Store wrapper only to observe/fail the persistence boundary.
3. Prove live progress remains old during Save, matches saved progress after
success, and does not change on update, validation, or save failure. Include
both typed collections and a transcribe checkpoint alongside analyze
records subject to save-time normalization. A failure test must exercise
candidate timestamp/normalization mutation before returning an error, not
merely fail before any work.
4. Prove unrelated stage/aggregate data survives, existing target references
observe updates, missing-target creation is invisible on failure, and later
caller collection edits do not mutate committed records. Preserve context
and error identity. Test the supported copy/write boundary, not arbitrary
malicious Store mutation.
Verify: `go test ./internal/manifest ./internal/stage` and
`go test -race ./internal/stage`.
Done when the shared integrity boundary is tested without changing existing
stage behavior. Temporary copying overlap is removed in Stage 3.
## Stage 2: Migrate Analyze
Status: Complete.
Completion marker (2026-09-22): Replaced analyze's local candidate-copy,
save, and promotion choreography with the shared checkpoint operation while
retaining its existing-record requirement, typed validation, path resolution,
and post-save working and committed collection refresh. Existing recovery,
selection, invalidation, and rollback coverage required no duplication.
Verified with `go test ./internal/stage ./internal/app ./internal/manifest`,
`go test -race ./internal/stage ./internal/app`, and `git diff --check`.
Preparation: `internal/stage/analyze_execution_state.go`,
`analyze_checkpoint_test.go`, `analyze_failure_test.go`,
`internal/app/analyze_checkpoint_recovery_test.go`, and
`docs/internal/stage-analyze.md`.
1. Replace manual candidate copying, saving, and live promotion in analyze
with the shared helper. Retain typed validation, required-record checking,
the typed update callback, and committed/working collection refresh.
2. Preserve starting-checkpoint elision, selected-artifact behavior, catalog
sequencing, dependent invalidation on changed output, and rollback after
checkpoint failure. Do not change the runner's terminal failure save or
joined errors.
3. Run existing tests before adding coverage. Retain tests proving earlier
artifacts survive failure/cancellation, unsaved output bytes gain no
authority, selected/forced work preserves appropriate unrelated artifacts,
and interrupted runs recover across store reloads. Add only missing
integration coverage; do not duplicate Stage 1's isolation matrix.
Verify: `go test ./internal/stage ./internal/app ./internal/manifest` and
`go test -race ./internal/stage ./internal/app`.
Done when analyze delegates persistence choreography while progress, artifact
selection, and failure behavior remain unchanged.
## Stage 3: Migrate Transcribe and Remove Duplicate Machinery
Status: Complete.
Completion marker (2026-09-22): Delegated transcription's locked checkpoint
operation to the shared helper while retaining typed validation, direct-call
record initialization, stage timestamp updates, post-save working collection
refresh, and the existing caller-owned mutex and failure latch. Removed the
duplicate production manifest clone; the read-only resume assertion now uses
the shared bounded snapshot. Existing real-store, concurrency, force/retry,
interruption, aggregate-repair, and zero-sized-audio coverage passed unchanged.
Verified with `go test ./internal/stage ./internal/app ./internal/manifest`,
`go test -race ./internal/stage ./internal/app`, and `git diff --check`.
Preparation: `internal/stage/transcribe_execution_state.go`, `transcribe.go`,
`transcribe_resume.go`, `transcribe_checkpoint_test.go`,
`transcribe_reconciliation_test.go`,
`internal/app/transcribe_checkpoint_recovery_test.go`, and
`docs/internal/stage-transcribe.md`.
1. Delegate `checkpointCollectionLocked` to the shared helper, retaining
typed initialization/update and successful working-collection refresh.
Keep the existing mutex across candidate creation, save, and promotion.
Do not move HTTP/hashing work into that critical section or add a lock.
2. Preserve failure-latch checks and worker joining, the parent versus worker
context distinction, force's durable empty starting collection, and
ordinary retry's reuse of committed forced replacements.
3. Remove `cloneManifestForTranscribeCheckpoint`. Its current use by a
read-only resume test is not a reason to retain duplicate production code.
Use an independent test snapshot or the shared bounded copy where sufficient,
without exporting a helper for tests.
4. Retain existing real-store and concurrency coverage: no lost completions;
pre-dispatch save failure prevents requests/canonical writes; later failure
retains prior authority; interrupted and aggregate-save recovery; failed
force followed by retry; run-directory independence; zero-sized audio;
removed/mismatched aggregate repair without additional WhisperX requests
or removed-speaker merge input. Do not weaken these regression assertions.
Verify: `go test ./internal/stage ./internal/app ./internal/manifest` and
`go test -race ./internal/stage ./internal/app`.
Done when both stages use one save/promote implementation and one bounded
copy implementation, with their different models and synchronization intact.
## Stage 4: Document, Review Scope, and Validate
Status: Complete.
Completion marker (2026-09-22): Documented the private shared checkpoint
persistence boundary and linked the stage guides back to it while retaining
their separate analyze and transcribe progress policies. The refactor removed
the two local candidate/save/promotion sequences and the transcription-only
bounded clone; the retained thin wrappers own typed validation, updates,
working-state refresh, and their existing serialization and failure behavior.
The collective test review retained focused helper integrity coverage plus
stage/application recovery tests, without duplicating the helper's isolation
matrix. A cumulative scope audit found no schema, validation-policy,
scheduling, force, concurrency, output-selection, stage-order, adapter-call,
configuration, or public-interface change.
Verified with `go test ./...`, `go test -race ./...`, `go vet ./...`,
`go build ./...`, `go test ./internal/doccheck`,
`go test ./internal/config -run '^TestExamplesLoadAndValidate$'`, and
`git diff --check`; all passed.
Preparation: `docs/internal/manifest.md`, `docs/internal/stage-transcribe.md`,
`docs/internal/stage-analyze.md`, documentation/testing policies, and the
implementation diff against this plan's starting commit.
1. Update manifest internals to identify the shared checkpoint owner and copy/
save/promote boundary. Stage guides retain stage-specific policy and link
to that explanation instead of duplicating shared implementation details.
Do not rewrite immutable release notes or announce new operator behavior.
2. Assess tests collectively. Consolidate genuinely duplicated persistence
mechanics coverage, but retain stage/application recovery tests proving
correct integration. Avoid test-count/LOC quotas. Record a brief summary
here of duplicated mechanisms removed and thin wrappers retained.
3. Confirm no schema, validation policy, scheduling, force, concurrency, output
selection, stage order, adapter call, or configuration change slipped into
the refactor. Additional restrictions require a separate decision, not
tests that retrospectively declare new policy.
4. Run all gates below and record actual results. Mark this plan complete only
when both migrations and verification pass. Do not release, tag, push, delete
other roadmaps, or expand into a repository-wide cloning refactor.
```sh
go test ./...
go test -race ./...
go vet ./...
go build ./...
go test ./internal/doccheck
go test ./internal/config -run '^TestExamplesLoadAndValidate$'
git diff --check
```
Done when duplication is removed, recovery remains covered, documentation
matches the implementation, and all gates pass.
## Open Questions
None requiring a product decision. A private stage-level helper is selected
because persistence already has an owner and the duplication is execution
plumbing. Model unification, library extraction, generalized transactions, and
broader cloning cleanup remain out of scope.

View File

@@ -0,0 +1,151 @@
# Notarius Configuration Profile Selection
Status: Implemented.
## Goal and Authority
Implement the consumer behavior in the
[feature request](feature-request-notarius-config-profile.yml): let a Narratio
configuration overlay explicitly select a Notarius configuration overlay.
Operators can select testing once in Narratio while each application retains
ownership of its own model choices and other settings.
This roadmap describes the target state and required change areas, not an
implementation sequence; the [implementation plan](implementation.md) owns
the ordered work and implementation decisions. The feature request is the scope
authority; the mechanisms suggested below do not establish additional
user-facing contracts.
## Underlying Issue
Narratio can select its own testing profile, but extraction currently invokes
Notarius without a configuration-profile selector. Consequently, a Narratio
testing invocation cannot explicitly select the independently deployed Notarius
testing overlay through pipeline configuration.
Adding individual downstream model or tuning fields would duplicate Notarius's
configuration responsibilities. Passing its existing profile selector provides
the requested flexibility without that duplication.
## Targeted End State
### Explicit Configuration Composition
`pipeline.notarius.config_profile` is an optional Notarius configuration-profile
name, usable in Narratio's base configuration and ordinary overlays. For
example, Narratio's `profiles/testing.yml` may contain:
```yaml
notarius:
config_profile: testing
```
Selecting Narratio's `--profile testing` then selects that explicit downstream
profile whenever extraction executes. The two profile names need not match.
Narratio does not automatically forward its own profile name.
When supplied, the selection is passed through Notarius's `--config-profile`
argument. Omitted or null values add no profile-selection flag and preserve
existing downstream selection behavior. Notarius's selector semantics remain
authoritative, including empty and `default` selections for the composed base
configuration. Narratio does not introduce a stricter profile-name policy.
This field selects a ConfigKit configuration overlay, not a PromptKit profile
through `--llm-profile`. The Notarius overlay owns any resulting LLM-profile
selection. Notarius resolves overlays relative to its own configuration root;
Narratio neither resolves those files against its own root nor rewrites them.
### Execution, Reuse, and Failure
A changed downstream profile selection must not silently reuse Narratio's
completed extraction from the previous selection. Existing dependent-stage
invalidation applies when extraction changes; the feature does not change the
pipeline order or transcript-generation behavior.
Notarius continues to own downstream overlay discovery, composition, validation,
and cache compatibility. A missing or invalid selected overlay must fail
extraction through the existing subprocess error path, without silently falling
back to production or reporting successful extraction.
Existing resume and chunk-cache controls, including force precedence, remain
in place. Profile selection does not itself mean forced regeneration or cache
deletion. Narratio's decision to execute extraction and Notarius's decision to
reuse compatible work remain separate responsibilities.
Changes to private Notarius configuration behind an unchanged selector retain
the existing operator responsibility to force extraction when needed. There is
no new downstream file inspection or content-tracking requirement.
Credential forwarding remains independently configured through
`notarius.llm_api_key`. A Narratio testing overlay may select a different
credential reference alongside the downstream profile; no new credential
discovery or provider-name mapping is introduced.
## Required Change Areas
### Configuration and Inspection
Extend the typed Notarius configuration and its existing safe effective-YAML
projection to carry the optional field. Preserve the ordinary configuration
composition behavior for base values, overlay overrides, and null values.
Configuration inspection must reflect the selected field without exposing
credential values. No replacement configuration loader or downstream preflight
loader is needed.
### Stage and Adapter Boundary
Carry the selection from resolved configuration through the extraction request
to the subprocess adapter. Construct the optional CLI flag using the existing
argument-vector mechanism, alongside the current configuration path, references,
credential forwarding, and cache controls. Retain normal Notarius error handling.
### Extraction Reuse Evidence
Account for the downstream selection in the existing extraction reuse evidence.
The current semantic-configuration and invocation fingerprints are the relevant
change points. Keep their treatment of selection consistent so that a profile
change cannot bypass the required rerun and downstream invalidation.
This roadmap does not require a new manifest schema, separate profile-indexed
workspaces, historical per-profile result retention, or an upgrade-wide forced
rerun.
### Verification and Documentation
Use focused behavioral coverage for configuration composition and inspection,
explicit argument forwarding and omission, changed-selection reuse decisions,
and ordinary subprocess failure handling. Reuse existing test coverage where
it already protects unchanged credentials, force behavior, and invalidation;
do not duplicate Notarius's complete profile-validation or cache test suites.
When implemented, update the canonical
[configuration reference](../config.md),
[Notarius integration contract](../integrations/notarius.md),
[extract internals](../internal/stage-extract.md), and relevant maintained
examples. The development workflow in [Operations](../operations.md) should
explain explicit profile composition and the existing need to force extraction
after private downstream edits. Clarify that profile selection does not create
an isolated testing workspace or separate current extraction state.
## Compatibility and Scope Boundaries
Notarius v0.7.0 already documents `--config-profile`; no upstream change is
required. Document the capability needed when the option is used, without adding
an unconditional runtime version gate for existing unconfigured deployments.
This feature adds no Narratio CLI selector, automatic profile propagation,
Notarius LLM-profile selector, downstream configuration rewriting, new
credential mechanism, or generalized recursive orchestration. Existing output
and receipt contracts remain unchanged.
The consumer's Ansible role and suite updates are follow-up work outside this
repository. They can deploy downstream overlays and populate this field after
the interface is released.
## Scope Review
No stronger requirements or new invariants are proposed beyond the feature
request and existing repository contracts. In particular, minimizing one-time
upgrade invalidation and avoiding reruns between equivalent base selectors are
implementation considerations, not additional acceptance guarantees imposed
by this roadmap. No user decisions are currently required before planning.

104
docs/roadmap/remediation.md Normal file
View File

@@ -0,0 +1,104 @@
# Transcription Checkpoint Review Remediation
Status: Complete.
Reviewed commits `7b0bf99` through `96b4e73` against the speaker-checkpoint and
upload-replay roadmap and the original user request. No production changes were
made during review. The two failures below were reproduced with temporary tests
using the existing stage fixtures; those temporary tests were removed afterward.
## 1. Reconcile Aggregate State Even When No Speaker Needs HTTP Work
Priority: P2. Status: Complete.
Implemented read-only aggregate identity/evidence checks before resume skips.
Regression tests cover removed inputs, missing/duplicate/mismatched outputs,
repair without HTTP requests, downstream invalidation, and exclusion of removed
speakers from merge without deleting their files. Equivalent path forms and
output ordering remain reusable. Focused app/stage tests and documentation
checks pass.
Location: `internal/stage/transcribe_resume.go:21` and the reconciliation result
consumed there.
`ValidateResume` returns reusable whenever `reconciliation.work` is empty. That
only establishes that the remaining speakers need no transcription; it does
not establish that the stored speaker collection and aggregate outputs match
the current input set. Removing a speaker while the remaining inputs are
unchanged produces an empty work list, so a succeeded record can be skipped
without pruning the removed speaker. Its old aggregate output can then remain
available to merge. Likewise, missing/inconsistent aggregate output references
are not repaired when all individual checkpoints remain valid. A prepare rerun
may already stale transcribe in some workflows, but the resume validator cannot
rely on that to satisfy its own current-input/output checks.
Reproduction: create complete alice/bob checkpoint records and aggregate
outputs with the existing fixture, mark the stage succeeded, remove bob from
the manifest's audio inputs, and call `ValidateResume`. It returns reusable
despite still retaining bob in the stored collection and aggregate outputs.
Required change:
- Distinguish no HTTP work from no reconciliation work. Validate that the
committed speaker set and aggregate transcript references represent the
reconciled current set, including relevant identity/evidence, before skipping.
- When only collection/output repair is needed, return non-resumable and let
normal execution checkpoint the pruned collection and rebuild aggregate
outputs without retranscribing reusable speakers. Preserve existing dependent
invalidation; do not force otherwise-current speaker jobs.
- Keep resume inspection read-only. Do not fix this by deleting incidental
transcript files or changing stage selection rules.
Verification: add removed-only input coverage (not just removal combined with
an added speaker), plus incomplete/inconsistent aggregate output coverage.
Exercise the runner skip path and assert zero additional WhisperX requests,
correct rebuilt outputs, and no removed speaker reaching merge.
## 2. Remove the Unrequested Positive Audio-Size Requirement
Priority: P2. Status: Complete.
Audio evidence now permits zero size while still rejecting negative values.
Regression tests verify zero-size manifest round trips, successful service
results being checkpointed and reused, and service rejection being propagated
without creating a completion record. Nonempty valid-JSON output validation is
unchanged. Both new success regressions failed before the fix.
Location: `internal/manifest/transcribe_state.go:95`.
The new record validator requires `AudioSize > 0`. Previously Narratio accepted
a regular `.flac` file and delegated transcription acceptance to WhisperX;
neither the user request nor the roadmap introduced minimum audio size or a new
audio-content validation policy. Zero-byte content still has a valid checksum
and can be represented by the checkpoint mechanism. The current check runs
after a successful response and output materialization, causing the entire
stage to fail instead of retaining the completed work.
Reproduction: prepare one speaker, replace its prepared audio with an empty
regular file, and use the existing successful fake WhisperX client. Stage
execution fails at checkpoint validation with `audio_size must be positive`.
This demonstrates the extra Narratio restriction; it does not assert that a
particular production WhisperX deployment accepts empty audio.
Required change:
- Permit zero audio size and reject negative sizes. Keep byte hashing and
size equality checks. Let the downstream service decide whether the input
can be transcribed, as before.
- Do not remove the nonempty valid-JSON output requirement, which has a separate
existing justification. Do not add a new early empty-audio rejection instead.
- Update any new tests/documentation that promote strictly positive input size
to a contract.
Verification: cover a zero-size record round-trip and successful adapter result
being checkpointed/reused for that input; retain negative-size rejection and
ordinary downstream-failure tests.
## Verification After Remediation
Completed on 2026-09-22 with Go 1.26.8: focused stage/application tests,
changed-package race tests, `go test ./...`, `go test -race ./...`,
`go vet ./...`, `go build ./...`, `go test ./internal/doccheck`,
`go test ./internal/config -run '^TestExamplesLoadAndValidate$'`, and
`git diff --check` all passed. Existing replay and cancellation regression
tests remain intact; the HTTP adapter required no changes.

View File

@@ -0,0 +1,183 @@
# Resumable Speaker Transcription and Replayable Uploads
Status: Implemented.
## Goal and User Intent
Preserve completed speaker transcriptions when another speaker fails, so an
ordinary run or resume does not repeat expensive successful work. Separately,
allow Go's HTTP transport to replay WhisperX uploads when it encounters a
recoverable connection or stream failure.
The motivating incident exhausted four transcription attempts for one speaker
after three other speakers had completed. The final error reported an HTTP/2
peer `PROTOCOL_ERROR` and an inability to replay the request body without
`Request.GetBody`.
These are related reliability improvements, not a promise to eliminate all
service failures. This document records the target state and scope. The
completed implementation sequence is retained in Git history; current behavior
is documented in the canonical guides linked below.
## Pre-implementation Limitations
The [transcribe stage](../internal/stage-transcribe.md) writes successful
responses into the invocation's run directory. It installs canonical transcripts
and returns its per-file metadata only after every speaker succeeds. A failure
therefore leaves no durable per-speaker progress that ordinary resume can use,
even though some completed JSON files may remain on disk.
The [WhisperX adapter](../integrations/whisperx.md) already retries requests at
the application level, reopening audio for each attempt. Its multipart request
body is streamed through a pipe and has no replay callback. Transport-level
recovery within an attempt cannot reconstruct a consumed body. Providing that
capability addresses the replay limitation, not the underlying peer protocol
error; persistent failures can still require service or proxy investigation.
## Targeted End State
### Durable Per-Speaker Completion
Each successfully transcribed speaker file is validated, saved, and recorded
durably before that work is treated as reusable. The session manifest remains
the progress authority. Follow the existing analyze-stage checkpoint pattern
without turning speakers into pipeline stages or introducing a general task
engine.
Ordinary execution reconciles the current prepared audio set with recorded
speaker results. It reuses valid completed work and schedules failed,
unattempted, missing, or obsolete results. A result is not reusable merely
because a matching filename exists. Evidence binds it to the speaker/audio
identity, audio content, Narratio-visible recognition settings, and the saved
transcript's integrity.
The relevant recognition settings remain the WhisperX service URL and language.
Retry count/delay, concurrency, timeout, credentials, and diagnostic locations
remain operational settings rather than reasons to discard completed results.
Changes to private service models or configuration remain an operator reason
to force transcription; Narratio does not inspect the service's private state.
Changes to individual audio inputs require replacement of affected results,
not automatic retranscription of unrelated unchanged speakers. New speakers
need transcription; removed speakers do not remain in the current aggregate
output set. Records must describe the inputs actually used, not grant reuse to
old bytes under a changed identity.
### Aggregate Completion, Failure, and Force
The overall stage succeeds only when every currently required speaker has a
validated, durably recorded result. Incomplete per-speaker progress does not
make the aggregate stage successful or authorize downstream consumption of an
incomplete collection. Existing dependent-stage invalidation remains in effect.
Retain bounded concurrency and the current fail-fast behavior: a speaker
failure cancels outstanding work. Results already checkpointed remain reusable
after failure, cancellation, or process interruption. A later ordinary run or
resume retries the remaining work; it does not need a new CLI command.
Checkpoint writes must be serialized so concurrently finishing speakers cannot
overwrite each other's progress. A failed persistence operation must not
advertise an unrecorded result as complete. Previously committed progress remains
the recovery authority, and checkpoint errors must be surfaced rather than
reported as successful transcription.
`--force` on transcribe regenerates all currently required speakers instead of
reusing their prior completions. Successful replacements are checkpointed too,
so an ordinary retry after a failed forced invocation can retain the newly
completed work. Old results for speakers not successfully regenerated must not
satisfy that unfinished forced replacement on the next ordinary retry.
This deliberately relaxes the current all-speakers-before-canonical-installation
implementation: individual durable results may exist while the stage remains
failed or incomplete. The all-speakers-before-stage-success boundary remains.
### Replayable WhisperX Requests
Provide `Request.GetBody` so the HTTP transport can obtain a fresh body for the
same request. Replay reopens the audio and reproduces the multipart boundary,
filename, fields, and framing used by the original request. Independent bodies
must not share a consumed reader or mutable read position.
Preserve bounded-memory streaming rather than buffering entire audio files in
memory. Correctly close every original and replay body and its audio resources;
preserve source-read errors and cancellation, including partial uploads and
early server responses. Successful response handling must not accidentally wait
on, or report errors solely from, an abandoned body instead of the replay that
produced the response.
Keep the existing application-level retry count, delay, classification, and
per-attempt timeout semantics. Transport replays occur within an application
attempt and its context; they are not additional Narratio-configured attempts.
Do not add another custom retry loop, silently increase retries, or disable
HTTP/2 globally as a substitute for replay support.
The adapter still validates and installs only successful JSON responses.
Request replay does not provide exactly-once server execution: a lost response
can cause repeated transcription work, as can existing application retries.
No new service-side idempotency protocol is required by this feature.
## Required Change Areas
- **Manifest and persistence:** Add transcribe-owned per-speaker progress and
evidence, with safe checkpoint updates and preservation through runner failure
handling. Review manifest validation, copying, serialization, and existing
publish/restore handling wherever the new records pass through those paths.
- **Stage execution and reuse:** Reconcile speaker records before scheduling,
checkpoint validated completions, rebuild aggregate outputs from the current
set, and account for partial progress in ordinary and forced execution.
Review successful-stage reuse as well as failed-stage retries so stale audio
or missing/corrupt transcript evidence is not bypassed by an aggregate skip.
- **Artifact lifecycle:** Use existing path, installation, integrity, and manifest
mechanisms. Ensure checkpointed results do not depend on an ephemeral run
directory surviving cleanup. Do not introduce a second authoritative ledger
or infer completion by scanning abandoned runs.
- **HTTP adapter:** Make multipart bodies reproducible and correctly manage
every replay's lifetime, without changing the WhisperX HTTP schema or requiring
an upstream service change.
- **Tests:** Cover per-speaker recovery after a later failure, interruption and
checkpoint failure; changed/missing inputs or outputs; force followed by an
ordinary retry; concurrent successful completions; and aggregate downstream
gating. Extend existing fixtures rather than duplicating analyze's complete
test suite. Exercise actual body reconstruction after partial consumption,
multipart equivalence, cancellation/resource cleanup, and existing bounded
streaming and retry behavior—not just the presence of a callback. Default
tests remain offline and use local/fake HTTP boundaries, never paid services.
- **Documentation:** Update the canonical
[stage guide](../internal/stage-transcribe.md),
[manifest guide](../internal/manifest.md),
[WhisperX contract](../integrations/whisperx.md),
[operations guide](../operations.md), and
[troubleshooting guide](../troubleshooting.md) when implemented. Explain
partial completion, ordinary retry versus force, and the distinction between
request replay limitations and persistent peer protocol failures.
## Compatibility and Scope Boundaries
Existing speaker identities, raw transcript formats, and the fixed pipeline
order remain unchanged. Existing manifests remain readable, but must not be
treated as containing per-speaker evidence they never recorded. Recovery relies
on verified recorded evidence, not guessed associations between old transcripts
and current audio. No general legacy-migration framework is part of the feature.
Automatic adoption of unrecorded outputs from previously failed runs is outside
this feature. Such files may be recoverable after verifying their inputs, but
their presence alone does not prove completion. No recovery/import command,
speaker-selection CLI, historical invocation ledger, new service configuration
discovery, or broader orchestration refactor is included.
The feature does not diagnose the remote service/proxy's underlying HTTP/2
problem, guarantee recovery from persistent protocol errors, or change the
policy for continuing unrelated work after failure.
## Completion Criteria
For speakers A, B, C, and D: if A and B are checkpointed, C fails, and D has not
completed, ordinary resume retains A and B, retries C, and completes D. No
downstream stage receives an incomplete successful transcription collection.
Force replaces all four, while an ordinary retry can reuse checkpoints from
that forced replacement attempt.
The HTTP transport can reconstruct the upload after consuming the original
body, without whole-file memory buffering, lost source errors, or stranded
upload resources. A persistent peer error still fails normally after the
configured application attempts, without discarding checkpointed speakers.

View File

@@ -213,6 +213,100 @@ input validation.
Relevant reference: [Operations: Stage Execution and Continuation Behavior](./operations.md#stage-execution-and-continuation-behavior). Relevant reference: [Operations: Stage Execution and Continuation Behavior](./operations.md#stage-execution-and-continuation-behavior).
## Transcription retries speakers that previously finished
Symptom:
- an ordinary transcribe retry calls WhisperX for a speaker that appeared to
finish earlier; or
- an older successful session retranscribes all speakers once after upgrade.
Likely causes:
- the earlier response or canonical file was not followed by a successful
speaker manifest checkpoint;
- the prepared audio bytes/path, WhisperX URL, or language changed;
- the canonical transcript is missing, unsafe, non-regular, invalid JSON, or
no longer matches its recorded size/checksum;
- the previous invocation used `--force`, which durably cleared pre-force
speaker authority; or
- the manifest has only legacy aggregate transcribe state and cannot bind old
outputs to their source audio.
Safe fix:
- inspect the reported speaker and filesystem error, repair the prepared input
or canonical-output path when needed, then retry transcribe normally;
- allow a legacy aggregate-only record to regenerate once; and
- do not edit the manifest or copy an old JSON file into place as a substitute
for a successful checkpoint.
```bash
narratio run-stage transcribe 2026-04-04
```
Valid earlier speaker checkpoints remain reusable even while aggregate
transcribe status is failed or interrupted. Merge remains gated until
transcribe reaches aggregate success with the complete current set.
Relevant reference: [Operations: Transcription continuation and replacement](./operations.md#transcription-continuation-and-replacement).
## WhisperX model changed but transcription was reused
Symptom:
- WhisperX model weights, prompts, or private server configuration changed at
the same configured URL, but Narratio still considers speaker transcripts
current.
Likely cause:
- Narratio can fingerprint its configured URL and language, not private state
loaded behind the service boundary.
Safe fix:
- explicitly force transcribe to establish a durable reset, then use ordinary
retry if the replacement attempt is interrupted.
```bash
narratio run-stage transcribe 2026-04-04 --force
```
Transport replay and application retries do not provide an exactly-once remote
execution guarantee. Investigate server-side failures or duplicate work using
WhisperX diagnostics; do not infer completion from a server log without the
local speaker checkpoint.
Relevant references: [WhisperX integration](./integrations/whisperx.md) and
[Transcribe internals](./internal/stage-transcribe.md).
## Transcription manifest checkpoint failed
Symptom:
- transcribe reports that a speaker completed but the session manifest could
not be saved.
Likely causes:
- the workspace is unwritable or out of space;
- the manifest path or parent directory is unsafe or the wrong type; or
- a transient filesystem error interrupted atomic replacement.
Safe fix:
- correct the reported filesystem or permission problem and retry transcribe
normally. Narratio reconciles from the last readable collection and reuses
only successfully checkpointed speakers.
A canonical raw transcript left by the failed checkpoint is incidental and
does not become reusable authority. Preserve the manifest and diagnostics if
the failure repeats; do not advertise the file by editing manifest evidence.
Relevant references: [Manifest persistence](./internal/manifest.md#persistence-semantics)
and [Transcribe internals](./internal/stage-transcribe.md).
## Notarius credential reference unavailable ## Notarius credential reference unavailable
Symptom: Symptom:
@@ -580,18 +674,22 @@ Symptom:
or unsupported; or or unsupported; or
- restore rejects a committed remote session manifest for the same reason. - restore rejects a committed remote session manifest for the same reason.
Likely cause: Likely causes:
- the manifest predates the supported canonical state contract. Narratio - the manifest predates the supported canonical state contract. Narratio
rejects it at the schema boundary rather than interpreting aggregate output rejects it at the schema boundary rather than interpreting aggregate output
lists or incidental files as current authority. lists or incidental files as current authority; or
- an older strict-decoding Narratio binary is reading a current manifest that
contains additive transcribe speaker state it does not recognize.
Safe fix: Safe fix:
- for unsupported local state, move the old manifest aside and regenerate, or - for unsupported local state, move the old manifest aside and regenerate, or
restore a verified current committed snapshot when one is authoritative; restore a verified current committed snapshot when one is authoritative;
- for unsupported committed remote state, regenerate healthy local state as - for unsupported committed remote state, regenerate healthy local state as
needed and republish the session. needed and republish the session; and
- for a downgrade rejection, return to the current binary or restore a verified
pre-upgrade backup. Do not delete only the speaker fields by hand.
```bash ```bash
narratio run 2026-04-04 narratio run 2026-04-04
@@ -603,7 +701,7 @@ old files into canonical paths merely to influence selection; availability is
manifest-owned. manifest-owned.
Relevant references: [Operations: Stage Execution and Continuation Behavior](./operations.md#stage-execution-and-continuation-behavior) Relevant references: [Operations: Stage Execution and Continuation Behavior](./operations.md#stage-execution-and-continuation-behavior)
and [Manifest Internals](./internal/manifest.md#analyze-owned-artifact-state). and [Manifest Internals](./internal/manifest.md#transcribe-owned-speaker-state).
## Scriptorium private input changed without a rerun ## Scriptorium private input changed without a rerun

View File

@@ -134,6 +134,10 @@ notarius:
# Optional environment-variable reference. Match this name to api_key_env in # Optional environment-variable reference. Match this name to api_key_env in
# the selected Notarius/PromptKit profile; never put the credential here. # the selected Notarius/PromptKit profile; never put the credential here.
llm_api_key: NOTARIUS_LLM_API_KEY llm_api_key: NOTARIUS_LLM_API_KEY
# Optionally select a Notarius ConfigKit overlay when extraction runs. This
# is independent of Narratio's selected profile and may use a different name.
# See docs/config.md and docs/integrations/notarius.md for the contracts.
# config_profile: downstream-economy
config_path: /usr/local/etc/notarius/config.yml config_path: /usr/local/etc/notarius/config.yml
pipeline_id: dnd-session pipeline_id: dnd-session
timeout: 3h timeout: 3h

2
go.mod
View File

@@ -3,7 +3,7 @@ module gitea.maximumdirect.net/eric/narratio
go 1.26.0 go 1.26.0
require ( require (
gitea.maximumdirect.net/eric/ConfigKit v0.3.0 gitea.maximumdirect.net/eric/ConfigKit v0.3.1
github.com/aws/aws-sdk-go-v2/config v1.32.17 github.com/aws/aws-sdk-go-v2/config v1.32.17
github.com/aws/aws-sdk-go-v2/credentials v1.19.16 github.com/aws/aws-sdk-go-v2/credentials v1.19.16
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0

4
go.sum
View File

@@ -1,5 +1,5 @@
gitea.maximumdirect.net/eric/ConfigKit v0.3.0 h1:LgXEUjR0KsgwGBznOWa8FJTQ+iElctHtI1via5vKdu4= gitea.maximumdirect.net/eric/ConfigKit v0.3.1 h1:etfTI/dxO3NnS18La0V6IeUh/Iv+3PjVfEyaN8hTuOM=
gitea.maximumdirect.net/eric/ConfigKit v0.3.0/go.mod h1:tYWpsrbLyu05r8+1MapWp5fN6ZrX9I8hU2xEPedYmYk= gitea.maximumdirect.net/eric/ConfigKit v0.3.1/go.mod h1:tYWpsrbLyu05r8+1MapWp5fN6ZrX9I8hU2xEPedYmYk=
github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=
github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho=

View File

@@ -15,6 +15,10 @@ func (f *FakeRunner) Run(ctx context.Context, req RunRequest) (RunResult, error)
return RunResult{}, err return RunResult{}, err
} }
copyRequest := req copyRequest := req
if req.ConfigProfile != nil {
profile := *req.ConfigProfile
copyRequest.ConfigProfile = &profile
}
copyRequest.References = append([]ReferenceBinding(nil), req.References...) copyRequest.References = append([]ReferenceBinding(nil), req.References...)
f.Requests = append(f.Requests, copyRequest) f.Requests = append(f.Requests, copyRequest)
if f.Err != nil { if f.Err != nil {

View File

@@ -27,6 +27,7 @@ type RunRequest struct {
Binary string Binary string
// LLMAPIKey is an optional already-resolved credential reference. // LLMAPIKey is an optional already-resolved credential reference.
LLMAPIKey *configkit.Secret LLMAPIKey *configkit.Secret
ConfigProfile *string
ConfigPath string ConfigPath string
PipelineID string PipelineID string
InputPath string InputPath string

View File

@@ -82,6 +82,9 @@ func (r *SubprocessRunner) Run(ctx context.Context, req RunRequest) (RunResult,
for _, reference := range references { for _, reference := range references {
args = append(args, "--reference", reference.Selector+"="+reference.Path) args = append(args, "--reference", reference.Selector+"="+reference.Path)
} }
if req.ConfigProfile != nil {
args = append(args, "--config-profile", *req.ConfigProfile)
}
args = append(args, "--json") args = append(args, "--json")
processResult, err := r.run(ctx, subprocess.RunRequest{ processResult, err := r.run(ctx, subprocess.RunRequest{
Executable: req.Binary, Executable: req.Binary,

View File

@@ -118,6 +118,53 @@ func TestSubprocessRunnerMapsCacheControlsToArguments(t *testing.T) {
} }
} }
func TestSubprocessRunnerMapsConfigProfileToArgumentPair(t *testing.T) {
tests := []struct {
name string
supplied bool
value string
}{
{name: "omitted"},
{name: "empty", supplied: true},
{name: "default", supplied: true, value: "default"},
{name: "named", supplied: true, value: "inexpensive"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
req := validRunRequest(t)
if test.supplied {
req.ConfigProfile = &test.value
}
var captured sharedsubprocess.RunRequest
runner := &SubprocessRunner{run: func(_ context.Context, processReq sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
captured = processReq
writeValidBundleAndReceipt(t, req, false)
return sharedsubprocess.RunResult{ExitCode: 0}, nil
}}
if _, err := runner.Run(context.Background(), req); err != nil {
t.Fatalf("Run() error = %v", err)
}
flagIndex := -1
for index, arg := range captured.Args {
if arg == "--config-profile" {
flagIndex = index
break
}
}
if !test.supplied {
if flagIndex >= 0 {
t.Fatalf("subprocess args unexpectedly contain config profile: %#v", captured.Args)
}
return
}
if flagIndex < 0 || flagIndex+2 >= len(captured.Args) || captured.Args[flagIndex+1] != test.value || captured.Args[flagIndex+2] != "--json" {
t.Fatalf("config profile argument boundary = %#v, want flag, exact value, then --json", captured.Args)
}
})
}
}
func TestSubprocessRunnerRejectsInvalidChunkCacheBeforeLaunch(t *testing.T) { func TestSubprocessRunnerRejectsInvalidChunkCacheBeforeLaunch(t *testing.T) {
for _, mode := range []string{"AUTO", "invalid", " auto "} { for _, mode := range []string{"AUTO", "invalid", " auto "} {
t.Run(mode, func(t *testing.T) { t.Run(mode, func(t *testing.T) {
@@ -388,6 +435,10 @@ func TestSubprocessRunnerReturnsProcessFailuresWithoutParsingStdout(t *testing.T
for _, test := range tests { for _, test := range tests {
t.Run(test.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
req := validRunRequest(t) req := validRunRequest(t)
if test.name == "nonzero" {
profile := "missing-profile"
req.ConfigProfile = &profile
}
req.Binary = writeShellScript(t, "#!/bin/sh\n"+test.scriptBody) req.Binary = writeShellScript(t, "#!/bin/sh\n"+test.scriptBody)
req.Timeout = test.timeout req.Timeout = test.timeout
ctx := context.Background() ctx := context.Background()
@@ -684,7 +735,8 @@ func TestLoadDiagnosticSummariesValidateBoundsAndTolerateUnknownFields(t *testin
} }
func TestFakeRunnerCapturesRequestsAndHonorsContextAndError(t *testing.T) { func TestFakeRunnerCapturesRequestsAndHonorsContextAndError(t *testing.T) {
req := RunRequest{PipelineID: "pipeline", References: []ReferenceBinding{{Selector: "party", Path: "/references/party.json"}}} profile := "testing"
req := RunRequest{PipelineID: "pipeline", ConfigProfile: &profile, References: []ReferenceBinding{{Selector: "party", Path: "/references/party.json"}}}
want := RunResult{BundleRoot: "/bundle"} want := RunResult{BundleRoot: "/bundle"}
fake := &FakeRunner{Result: want} fake := &FakeRunner{Result: want}
got, err := fake.Run(context.Background(), req) got, err := fake.Run(context.Background(), req)
@@ -695,6 +747,10 @@ func TestFakeRunnerCapturesRequestsAndHonorsContextAndError(t *testing.T) {
if fake.Requests[0].References[0].Path != "/references/party.json" { if fake.Requests[0].References[0].Path != "/references/party.json" {
t.Fatalf("fake retained aliased request references: %#v", fake.Requests[0].References) t.Fatalf("fake retained aliased request references: %#v", fake.Requests[0].References)
} }
profile = "changed"
if *fake.Requests[0].ConfigProfile != "testing" {
t.Fatalf("fake retained aliased config profile: %q", *fake.Requests[0].ConfigProfile)
}
wantErr := errors.New("configured failure") wantErr := errors.New("configured failure")
fake.Err = wantErr fake.Err = wantErr

View File

@@ -6,14 +6,11 @@ import (
"errors" "errors"
"fmt" "fmt"
"io" "io"
"mime/multipart"
"net" "net"
"net/http" "net/http"
"net/url" "net/url"
"os" "os"
"path/filepath"
"strings" "strings"
"sync"
"time" "time"
"gitea.maximumdirect.net/eric/narratio/internal/fileops" "gitea.maximumdirect.net/eric/narratio/internal/fileops"
@@ -21,7 +18,6 @@ import (
const ( const (
defaultMaxWhisperXResponseBytes int64 = 10 * 1024 * 1024 defaultMaxWhisperXResponseBytes int64 = 10 * 1024 * 1024
whisperXUploadBufferSize = 32 * 1024
) )
// HTTPClientConfig contains parsed, deterministic WhisperX HTTP client settings. // HTTPClientConfig contains parsed, deterministic WhisperX HTTP client settings.
@@ -192,21 +188,32 @@ func (c *HTTPClient) Transcribe(ctx context.Context, req TranscribeRequest) (Tra
} }
func (c *HTTPClient) doTranscribeAttempt(ctx context.Context, audioPath string) (int, []byte, error) { func (c *HTTPClient) doTranscribeAttempt(ctx context.Context, audioPath string) (int, []byte, error) {
upload := newMultipartUpload(ctx, audioPath, c.language, c.openAudio) upload, err := newMultipartBodyFactory(ctx, audioPath, c.language, c.openAudio)
defer upload.Close() if err != nil {
return 0, nil, fmt.Errorf("prepare whisperx request body: %w", err)
}
defer func() { _ = upload.Close() }()
body, err := upload.NewBody()
if err != nil {
return 0, nil, fmt.Errorf("open whisperx request body: %w", err)
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.url.String(), upload) req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.url.String(), body)
if err != nil { if err != nil {
return 0, nil, fmt.Errorf("build whisperx request: %w", err) return 0, nil, fmt.Errorf("build whisperx request: %w", err)
} }
req.Header.Set("Content-Type", upload.contentType) req.Header.Set("Content-Type", upload.contentType)
req.Header.Set("Expect", "100-continue") req.Header.Set("Expect", "100-continue")
req.GetBody = upload.NewBody
resp, err := c.httpClient.Do(req) resp, err := c.httpClient.Do(req)
if err != nil { if err != nil {
_ = upload.Close() _ = upload.Close()
if producerErr := upload.Wait(); producerErr != nil { if sourceErr := upload.SourceError(true); sourceErr != nil {
return 0, nil, fmt.Errorf("stream whisperx request body: %w", producerErr) return 0, nil, fmt.Errorf("stream whisperx request body: %w", sourceErr)
}
if ctxErr := ctx.Err(); ctxErr != nil {
return 0, nil, fmt.Errorf("perform whisperx request: %w", ctxErr)
} }
return 0, nil, fmt.Errorf("perform whisperx request: %w", err) return 0, nil, fmt.Errorf("perform whisperx request: %w", err)
} }
@@ -214,8 +221,8 @@ func (c *HTTPClient) doTranscribeAttempt(ctx context.Context, audioPath string)
if resp.StatusCode < 200 || resp.StatusCode >= 300 { if resp.StatusCode < 200 || resp.StatusCode >= 300 {
_ = upload.Close() _ = upload.Close()
if producerErr := upload.Wait(); producerErr != nil { if sourceErr := upload.SourceError(false); sourceErr != nil {
return resp.StatusCode, nil, fmt.Errorf("stream whisperx request body: %w", producerErr) return resp.StatusCode, nil, fmt.Errorf("stream whisperx request body: %w", sourceErr)
} }
if _, err := readWhisperXResponse(resp.Body, c.maxResponseBytes); err != nil { if _, err := readWhisperXResponse(resp.Body, c.maxResponseBytes); err != nil {
return resp.StatusCode, nil, fmt.Errorf("read whisperx response body: %w", err) return resp.StatusCode, nil, fmt.Errorf("read whisperx response body: %w", err)
@@ -223,13 +230,19 @@ func (c *HTTPClient) doTranscribeAttempt(ctx context.Context, audioPath string)
return resp.StatusCode, nil, fmt.Errorf("whisperx returned status %d", resp.StatusCode) return resp.StatusCode, nil, fmt.Errorf("whisperx returned status %d", resp.StatusCode)
} }
if err := upload.Wait(); err != nil { data, responseErr := readWhisperXResponse(resp.Body, c.maxResponseBytes)
return resp.StatusCode, nil, fmt.Errorf("stream whisperx request body: %w", err) _ = upload.Close()
if sourceErr := upload.SourceError(false); sourceErr != nil {
if responseErr != nil {
return resp.StatusCode, nil, fmt.Errorf("stream whisperx request body: %w", errors.Join(sourceErr, responseErr))
}
return resp.StatusCode, nil, fmt.Errorf("stream whisperx request body: %w", sourceErr)
} }
if err := ctx.Err(); err != nil {
data, err := readWhisperXResponse(resp.Body, c.maxResponseBytes) return resp.StatusCode, nil, fmt.Errorf("complete whisperx request: %w", err)
if err != nil { }
return resp.StatusCode, nil, fmt.Errorf("read whisperx response body: %w", err) if responseErr != nil {
return resp.StatusCode, nil, fmt.Errorf("read whisperx response body: %w", responseErr)
} }
return resp.StatusCode, data, nil return resp.StatusCode, data, nil
} }
@@ -286,161 +299,6 @@ func isHTTPURLScheme(scheme string) bool {
} }
} }
type multipartUpload struct {
reader *io.PipeReader
writer *io.PipeWriter
contentType string
done chan struct{}
mu sync.Mutex
audio io.Closer
err error
aborted bool
}
func newMultipartUpload(ctx context.Context, audioPath, language string, openAudio func(string) (io.ReadCloser, error)) *multipartUpload {
reader, writer := io.Pipe()
multipartWriter := multipart.NewWriter(writer)
upload := &multipartUpload{
reader: reader,
writer: writer,
contentType: multipartWriter.FormDataContentType(),
done: make(chan struct{}),
}
go func() {
err := upload.write(ctx, multipartWriter, audioPath, language, openAudio)
if err != nil {
_ = writer.CloseWithError(err)
} else {
_ = writer.Close()
}
upload.mu.Lock()
upload.err = err
upload.audio = nil
upload.mu.Unlock()
close(upload.done)
}()
go func() {
select {
case <-ctx.Done():
upload.abort()
case <-upload.done:
}
}()
return upload
}
func (u *multipartUpload) Read(p []byte) (int, error) {
return u.reader.Read(p)
}
func (u *multipartUpload) Close() error {
u.abort()
return nil
}
func (u *multipartUpload) Wait() error {
<-u.done
u.mu.Lock()
defer u.mu.Unlock()
return u.err
}
func (u *multipartUpload) write(ctx context.Context, writer *multipart.Writer, audioPath, language string, openAudio func(string) (io.ReadCloser, error)) error {
fileWriter, err := writer.CreateFormFile("file", filepath.Base(audioPath))
if err != nil {
return u.producerError(ctx, fmt.Errorf("create multipart file field: %w", err))
}
audioFile, err := openAudio(audioPath)
if err != nil {
return u.producerError(ctx, fmt.Errorf("open audio file %q: %w", audioPath, err))
}
u.setAudio(audioFile)
_, copyErr := io.CopyBuffer(fileWriter, &contextReader{ctx: ctx, reader: audioFile}, make([]byte, whisperXUploadBufferSize))
closeErr := audioFile.Close()
u.clearAudio(audioFile)
if copyErr != nil {
return u.producerError(ctx, fmt.Errorf("copy audio file %q: %w", audioPath, copyErr))
}
if closeErr != nil {
return u.producerError(ctx, fmt.Errorf("close audio file %q: %w", audioPath, closeErr))
}
if err := writer.WriteField("language", language); err != nil {
return u.producerError(ctx, fmt.Errorf("write language form field: %w", err))
}
if err := writer.Close(); err != nil {
return u.producerError(ctx, fmt.Errorf("close multipart writer: %w", err))
}
return nil
}
func (u *multipartUpload) producerError(ctx context.Context, err error) error {
if ctx.Err() != nil {
return ctx.Err()
}
u.mu.Lock()
aborted := u.aborted
u.mu.Unlock()
if aborted {
return nil
}
return err
}
func (u *multipartUpload) setAudio(audio io.Closer) {
u.mu.Lock()
u.audio = audio
aborted := u.aborted
u.mu.Unlock()
if aborted {
_ = audio.Close()
}
}
func (u *multipartUpload) clearAudio(audio io.Closer) {
u.mu.Lock()
if u.audio == audio {
u.audio = nil
}
u.mu.Unlock()
}
func (u *multipartUpload) abort() {
u.mu.Lock()
if u.aborted {
u.mu.Unlock()
return
}
u.aborted = true
audio := u.audio
u.mu.Unlock()
_ = u.reader.Close()
if audio != nil {
_ = audio.Close()
}
}
type contextReader struct {
ctx context.Context
reader io.Reader
}
func (r *contextReader) Read(p []byte) (int, error) {
select {
case <-r.ctx.Done():
return 0, r.ctx.Err()
default:
return r.reader.Read(p)
}
}
func writeFileAtomic(path string, data []byte, perm os.FileMode) error { func writeFileAtomic(path string, data []byte, perm os.FileMode) error {
if strings.TrimSpace(path) == "" { if strings.TrimSpace(path) == "" {
return fmt.Errorf("path is required") return fmt.Errorf("path is required")

View File

@@ -0,0 +1,261 @@
package whisperx
import (
"context"
"errors"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
)
func TestHTTPClientTransportReplaysAbandonedMultipartBody(t *testing.T) {
var opens atomic.Int32
var closes atomic.Int32
transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
partial := make([]byte, 17)
if _, err := io.ReadFull(request.Body, partial); err != nil {
return nil, err
}
if err := request.Body.Close(); err != nil {
return nil, err
}
if request.GetBody == nil {
return nil, errors.New("request GetBody is nil")
}
replay, err := request.GetBody()
if err != nil {
return nil, err
}
replayedBytes, err := io.ReadAll(replay)
if err != nil {
_ = replay.Close()
return nil, err
}
if err := replay.Close(); err != nil {
return nil, err
}
parts := parseMultipartParts(t, request.Header.Get("Content-Type"), replayedBytes)
if len(parts) != 2 || parts[0].name != "file" || parts[0].filename != "audio.flac" || string(parts[0].data) != "audio-data" {
t.Fatalf("replayed file part = %#v, want exact audio.flac payload", parts)
}
if parts[1].name != "language" || string(parts[1].data) != "en" {
t.Fatalf("replayed language part = %#v, want language=en", parts[1])
}
return whisperXJSONResponse(request), nil
})
client := newReplayTestHTTPClient(t, transport)
client.openAudio = func(string) (io.ReadCloser, error) {
opens.Add(1)
return &countedReadCloser{Reader: strings.NewReader("audio-data"), closes: &closes}, nil
}
outPath := filepath.Join(t.TempDir(), "raw.json")
result, err := client.Transcribe(context.Background(), TranscribeRequest{
AudioPath: "audio.flac",
OutputRawTranscriptPath: outPath,
})
if err != nil {
t.Fatalf("Transcribe() error = %v", err)
}
if result.Attempts != 1 {
t.Fatalf("application attempts = %d, want 1", result.Attempts)
}
verifyJSONFile(t, outPath)
if got := opens.Load(); got != 2 {
t.Fatalf("audio opens = %d, want original and replay", got)
}
if got := closes.Load(); got != 2 {
t.Fatalf("audio closes = %d, want all body handles disposed", got)
}
}
func TestHTTPClientTransportReturnsReplayOpenFailure(t *testing.T) {
replayErr := errors.New("replay open failed")
var opens atomic.Int32
var closes atomic.Int32
transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
if _, err := io.ReadFull(request.Body, make([]byte, 1)); err != nil {
return nil, err
}
_ = request.Body.Close()
_, err := request.GetBody()
return nil, err
})
client := newReplayTestHTTPClient(t, transport)
client.openAudio = func(string) (io.ReadCloser, error) {
if opens.Add(1) == 2 {
return nil, replayErr
}
return &countedReadCloser{Reader: strings.NewReader("audio-data"), closes: &closes}, nil
}
outPath := filepath.Join(t.TempDir(), "raw.json")
result, err := client.Transcribe(context.Background(), TranscribeRequest{
AudioPath: "audio.flac",
OutputRawTranscriptPath: outPath,
})
if !errors.Is(err, replayErr) {
t.Fatalf("Transcribe() error = %v, want replay open failure", err)
}
if result.Attempts != 1 {
t.Fatalf("application attempts = %d, want 1", result.Attempts)
}
if got := closes.Load(); got != 1 {
t.Fatalf("audio closes = %d, want original disposed", got)
}
if _, statErr := os.Stat(outPath); !errors.Is(statErr, os.ErrNotExist) {
t.Fatalf("output should not exist, stat error = %v", statErr)
}
}
func TestHTTPClientTransportCannotSwallowReplayReadFailure(t *testing.T) {
replayErr := errors.New("replay read failed")
var opens atomic.Int32
transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
if _, err := io.ReadFull(request.Body, make([]byte, 1)); err != nil {
return nil, err
}
_ = request.Body.Close()
replay, err := request.GetBody()
if err != nil {
return nil, err
}
_, _ = io.ReadAll(replay)
_ = replay.Close()
return whisperXJSONResponse(request), nil
})
client := newReplayTestHTTPClient(t, transport)
client.openAudio = func(string) (io.ReadCloser, error) {
if opens.Add(1) == 2 {
return &failingReadCloser{first: []byte("partial"), err: replayErr}, nil
}
return io.NopCloser(strings.NewReader("audio-data")), nil
}
outPath := filepath.Join(t.TempDir(), "raw.json")
_, err := client.Transcribe(context.Background(), TranscribeRequest{
AudioPath: "audio.flac",
OutputRawTranscriptPath: outPath,
})
if !errors.Is(err, replayErr) {
t.Fatalf("Transcribe() error = %v, want replay read failure", err)
}
if _, statErr := os.Stat(outPath); !errors.Is(statErr, os.ErrNotExist) {
t.Fatalf("output should not exist, stat error = %v", statErr)
}
}
func TestHTTPClientTransportEarlyResponseDisposesOriginalBody(t *testing.T) {
var closes atomic.Int32
client := newReplayTestHTTPClient(t, roundTripFunc(func(request *http.Request) (*http.Response, error) {
return whisperXJSONResponse(request), nil
}))
client.openAudio = func(string) (io.ReadCloser, error) {
return &countedReadCloser{Reader: strings.NewReader("audio-data"), closes: &closes}, nil
}
_, err := client.Transcribe(context.Background(), TranscribeRequest{
AudioPath: "audio.flac",
OutputRawTranscriptPath: filepath.Join(t.TempDir(), "raw.json"),
})
if err != nil {
t.Fatalf("Transcribe() error = %v", err)
}
if got := closes.Load(); got != 1 {
t.Fatalf("audio closes = %d, want early-response body disposed", got)
}
}
func TestHTTPClientParentCancellationDuringReplayClosesEveryBody(t *testing.T) {
var opens atomic.Int32
var originalCloses atomic.Int32
replaySource := newSignalingBlockedReadCloser()
transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
if _, err := io.ReadFull(request.Body, make([]byte, 1)); err != nil {
return nil, err
}
_ = request.Body.Close()
replay, err := request.GetBody()
if err != nil {
return nil, err
}
_, _ = io.ReadAll(replay)
_ = replay.Close()
return nil, request.Context().Err()
})
client := newReplayTestHTTPClient(t, transport)
client.openAudio = func(string) (io.ReadCloser, error) {
if opens.Add(1) == 1 {
return &countedReadCloser{Reader: strings.NewReader("audio-data"), closes: &originalCloses}, nil
}
return replaySource, nil
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() {
_, err := client.Transcribe(ctx, TranscribeRequest{
AudioPath: "audio.flac",
OutputRawTranscriptPath: filepath.Join(t.TempDir(), "raw.json"),
})
done <- err
}()
select {
case <-replaySource.blocked:
cancel()
case <-time.After(time.Second):
cancel()
t.Fatal("replay source did not block")
}
select {
case err := <-done:
if !errors.Is(err, context.Canceled) {
t.Fatalf("Transcribe() error = %v, want context cancellation", err)
}
case <-time.After(time.Second):
t.Fatal("Transcribe() did not return after replay cancellation")
}
if got := originalCloses.Load(); got != 1 {
t.Fatalf("original audio closes = %d, want 1", got)
}
select {
case <-replaySource.closed:
case <-time.After(time.Second):
t.Fatal("replay audio source was not closed")
}
}
type roundTripFunc func(*http.Request) (*http.Response, error)
func (roundTrip roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) {
return roundTrip(request)
}
func newReplayTestHTTPClient(t *testing.T, transport http.RoundTripper) *HTTPClient {
t.Helper()
client, err := NewHTTPClient(HTTPClientConfig{
TranscribeURL: "http://whisperx.test/transcribe",
Language: "en",
Timeout: time.Second,
HTTPClient: &http.Client{Transport: transport},
})
if err != nil {
t.Fatalf("NewHTTPClient() error = %v", err)
}
return client
}
func whisperXJSONResponse(request *http.Request) *http.Response {
return &http.Response{
StatusCode: http.StatusOK,
Status: "200 OK",
Header: make(http.Header),
Body: io.NopCloser(strings.NewReader(`{"source":"transport-replay","segments":[]}`)),
Request: request,
}
}

View File

@@ -365,7 +365,7 @@ func TestHTTPClientEarlyServerResponseReturns(t *testing.T) {
} }
} }
func TestHTTPClientCancellationReleasesBlockedProducer(t *testing.T) { func TestHTTPClientCancellationReleasesBlockedSource(t *testing.T) {
release := make(chan struct{}) release := make(chan struct{})
source := newGatedReadCloser([]byte("audio-data"), release) source := newGatedReadCloser([]byte("audio-data"), release)
firstByteReceived := make(chan struct{}) firstByteReceived := make(chan struct{})

View File

@@ -0,0 +1,263 @@
package whisperx
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"mime/multipart"
"path/filepath"
"sync"
)
type multipartBodyFactory struct {
ctx context.Context
audioPath string
openAudio func(string) (io.ReadCloser, error)
prefix []byte
suffix []byte
contentType string
mu sync.Mutex
closed bool
bodies []*multipartBody
}
func newMultipartBodyFactory(
ctx context.Context,
audioPath string,
language string,
openAudio func(string) (io.ReadCloser, error),
) (*multipartBodyFactory, error) {
if ctx == nil {
ctx = context.Background()
}
if openAudio == nil {
return nil, fmt.Errorf("audio opener is required")
}
var framing bytes.Buffer
writer := multipart.NewWriter(&framing)
if _, err := writer.CreateFormFile("file", filepath.Base(audioPath)); err != nil {
return nil, fmt.Errorf("create multipart file field: %w", err)
}
prefixLength := framing.Len()
if err := writer.WriteField("language", language); err != nil {
return nil, fmt.Errorf("write language form field: %w", err)
}
if err := writer.Close(); err != nil {
return nil, fmt.Errorf("close multipart framing: %w", err)
}
framingBytes := framing.Bytes()
return &multipartBodyFactory{
ctx: ctx,
audioPath: audioPath,
openAudio: openAudio,
prefix: append([]byte(nil), framingBytes[:prefixLength]...),
suffix: append([]byte(nil), framingBytes[prefixLength:]...),
contentType: writer.FormDataContentType(),
}, nil
}
func (factory *multipartBodyFactory) NewBody() (io.ReadCloser, error) {
factory.mu.Lock()
if factory.closed {
factory.mu.Unlock()
return nil, fmt.Errorf("multipart body factory is closed")
}
if err := factory.ctx.Err(); err != nil {
factory.mu.Unlock()
return nil, err
}
audio, err := factory.openAudio(factory.audioPath)
if err != nil {
factory.mu.Unlock()
return nil, fmt.Errorf("open audio file %q: %w", factory.audioPath, err)
}
if err := factory.ctx.Err(); err != nil {
factory.mu.Unlock()
_ = audio.Close()
return nil, err
}
source := &multipartAudioSource{audio: audio, closeDone: make(chan struct{})}
body := &multipartBody{
ctx: factory.ctx,
source: source,
reader: io.MultiReader(bytes.NewReader(factory.prefix), source, bytes.NewReader(factory.suffix)),
}
factory.bodies = append(factory.bodies, body)
factory.mu.Unlock()
stopCancel := context.AfterFunc(factory.ctx, func() {
_ = body.Close()
})
body.mu.Lock()
body.stopCancel = stopCancel
closed := body.closed
body.mu.Unlock()
if closed {
stopCancel()
}
if err := factory.ctx.Err(); err != nil {
_ = body.Close()
return nil, err
}
return body, nil
}
func (factory *multipartBodyFactory) Close() error {
factory.mu.Lock()
if factory.closed {
bodies := append([]*multipartBody(nil), factory.bodies...)
factory.mu.Unlock()
return multipartBodyCloseErrors(bodies)
}
factory.closed = true
bodies := append([]*multipartBody(nil), factory.bodies...)
factory.mu.Unlock()
for _, body := range bodies {
_ = body.Close()
}
return multipartBodyCloseErrors(bodies)
}
func (factory *multipartBodyFactory) SourceError(includeAbandoned bool) error {
factory.mu.Lock()
bodies := append([]*multipartBody(nil), factory.bodies...)
factory.mu.Unlock()
var result error
for _, body := range bodies {
if err := body.sourceError(includeAbandoned); err != nil {
result = errors.Join(result, err)
}
}
return result
}
func multipartBodyCloseErrors(bodies []*multipartBody) error {
var result error
for _, body := range bodies {
if err := body.sourceError(false); err != nil {
result = errors.Join(result, err)
}
}
return result
}
type multipartBody struct {
ctx context.Context
reader io.Reader
source *multipartAudioSource
stopCancel func() bool
mu sync.Mutex
closed bool
abandoned bool
}
func (body *multipartBody) Read(buffer []byte) (int, error) {
if err := body.ctx.Err(); err != nil {
return 0, err
}
body.mu.Lock()
closed := body.closed
body.mu.Unlock()
if closed {
return 0, io.ErrClosedPipe
}
return body.reader.Read(buffer)
}
func (body *multipartBody) Close() error {
body.mu.Lock()
if body.closed {
body.mu.Unlock()
return body.source.Close()
}
body.closed = true
body.abandoned = !body.source.reachedEOF()
stopCancel := body.stopCancel
body.mu.Unlock()
if stopCancel != nil {
stopCancel()
}
return body.source.Close()
}
func (body *multipartBody) sourceError(includeAbandoned bool) error {
body.mu.Lock()
abandoned := body.abandoned
body.mu.Unlock()
err := body.source.Err()
if err != nil {
return err
}
if abandoned && !includeAbandoned {
return nil
}
return nil
}
type multipartAudioSource struct {
audio io.ReadCloser
mu sync.Mutex
closing bool
eof bool
readErr error
closeErr error
closeDone chan struct{}
closeResult error
closed bool
}
func (source *multipartAudioSource) Read(buffer []byte) (int, error) {
n, err := source.audio.Read(buffer)
source.mu.Lock()
switch {
case errors.Is(err, io.EOF):
source.eof = true
case err != nil && !source.closing:
source.readErr = errors.Join(source.readErr, err)
}
source.mu.Unlock()
return n, err
}
func (source *multipartAudioSource) Close() error {
source.mu.Lock()
if source.closed {
done := source.closeDone
source.mu.Unlock()
<-done
source.mu.Lock()
err := source.closeResult
source.mu.Unlock()
return err
}
source.closed = true
source.closing = true
reachedEOF := source.eof
source.mu.Unlock()
err := source.audio.Close()
source.mu.Lock()
source.closeResult = err
if err != nil && reachedEOF {
source.closeErr = err
}
close(source.closeDone)
source.mu.Unlock()
return err
}
func (source *multipartAudioSource) reachedEOF() bool {
source.mu.Lock()
defer source.mu.Unlock()
return source.eof
}
func (source *multipartAudioSource) Err() error {
source.mu.Lock()
defer source.mu.Unlock()
return errors.Join(source.readErr, source.closeErr)
}

View File

@@ -0,0 +1,327 @@
package whisperx
import (
"bytes"
"context"
"errors"
"io"
"mime"
"mime/multipart"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
)
func TestMultipartBodyReplayUsesIdenticalFraming(t *testing.T) {
factory := newTestMultipartBodyFactory(t, context.Background(), "audio-data")
defer factory.Close()
original, err := factory.NewBody()
if err != nil {
t.Fatalf("NewBody() original error = %v", err)
}
replay, err := factory.NewBody()
if err != nil {
t.Fatalf("NewBody() replay error = %v", err)
}
originalBytes, err := io.ReadAll(original)
if err != nil {
t.Fatalf("ReadAll(original) error = %v", err)
}
replayBytes, err := io.ReadAll(replay)
if err != nil {
t.Fatalf("ReadAll(replay) error = %v", err)
}
if !bytes.Equal(originalBytes, replayBytes) {
t.Fatal("original and replay multipart bytes differ")
}
parts := parseMultipartParts(t, factory.contentType, replayBytes)
if len(parts) != 2 {
t.Fatalf("part count = %d, want 2", len(parts))
}
if parts[0].name != "file" || parts[0].filename != "audio.flac" || string(parts[0].data) != "audio-data" {
t.Fatalf("file part = %#v, want file/audio.flac with exact audio", parts[0])
}
if parts[1].name != "language" || string(parts[1].data) != "en" {
t.Fatalf("language part = %#v, want language=en", parts[1])
}
}
func TestMultipartBodyReplayRestartsAfterPartialOriginal(t *testing.T) {
factory := newTestMultipartBodyFactory(t, context.Background(), "audio-data")
defer factory.Close()
original, err := factory.NewBody()
if err != nil {
t.Fatalf("NewBody() original error = %v", err)
}
partial := make([]byte, len(factory.prefix)+3)
if _, err := io.ReadFull(original, partial); err != nil {
t.Fatalf("ReadFull(original) error = %v", err)
}
if err := original.Close(); err != nil {
t.Fatalf("Close(original) error = %v", err)
}
replay, err := factory.NewBody()
if err != nil {
t.Fatalf("NewBody() replay error = %v", err)
}
replayBytes, err := io.ReadAll(replay)
if err != nil {
t.Fatalf("ReadAll(replay) error = %v", err)
}
parts := parseMultipartParts(t, factory.contentType, replayBytes)
if got := string(parts[0].data); got != "audio-data" {
t.Fatalf("replayed audio = %q, want complete payload", got)
}
if err := factory.SourceError(false); err != nil {
t.Fatalf("SourceError(false) = %v, want abandoned original ignored", err)
}
}
func TestMultipartBodySimultaneousBodiesUseIndependentSources(t *testing.T) {
var opens atomic.Int32
var closes atomic.Int32
factory, err := newMultipartBodyFactory(context.Background(), "audio.flac", "en", func(string) (io.ReadCloser, error) {
opens.Add(1)
return &countedReadCloser{Reader: strings.NewReader("audio-data"), closes: &closes}, nil
})
if err != nil {
t.Fatalf("newMultipartBodyFactory() error = %v", err)
}
first, err := factory.NewBody()
if err != nil {
t.Fatalf("NewBody() first error = %v", err)
}
second, err := factory.NewBody()
if err != nil {
t.Fatalf("NewBody() second error = %v", err)
}
var firstBytes, secondBytes []byte
var firstErr, secondErr error
var group sync.WaitGroup
group.Add(2)
go func() {
defer group.Done()
firstBytes, firstErr = io.ReadAll(first)
}()
go func() {
defer group.Done()
secondBytes, secondErr = io.ReadAll(second)
}()
group.Wait()
if firstErr != nil || secondErr != nil {
t.Fatalf("concurrent reads returned (%v, %v)", firstErr, secondErr)
}
if !bytes.Equal(firstBytes, secondBytes) {
t.Fatal("independent body bytes differ")
}
if got := opens.Load(); got != 2 {
t.Fatalf("audio opens = %d, want 2", got)
}
if err := factory.Close(); err != nil {
t.Fatalf("Close() error = %v", err)
}
if err := factory.Close(); err != nil {
t.Fatalf("second Close() error = %v", err)
}
if got := closes.Load(); got != 2 {
t.Fatalf("audio closes = %d, want 2", got)
}
}
func TestMultipartBodyReportsSourceFailures(t *testing.T) {
t.Run("read", func(t *testing.T) {
sourceErr := errors.New("read source")
factory, err := newMultipartBodyFactory(context.Background(), "audio.flac", "en", func(string) (io.ReadCloser, error) {
return &failingReadCloser{first: []byte("partial"), err: sourceErr}, nil
})
if err != nil {
t.Fatalf("newMultipartBodyFactory() error = %v", err)
}
body, err := factory.NewBody()
if err != nil {
t.Fatalf("NewBody() error = %v", err)
}
if _, err := io.ReadAll(body); !errors.Is(err, sourceErr) {
t.Fatalf("ReadAll() error = %v, want source failure", err)
}
_ = body.Close()
if err := factory.SourceError(false); !errors.Is(err, sourceErr) {
t.Fatalf("SourceError(false) = %v, want source failure", err)
}
})
t.Run("close after complete read", func(t *testing.T) {
closeErr := errors.New("close source")
factory, err := newMultipartBodyFactory(context.Background(), "audio.flac", "en", func(string) (io.ReadCloser, error) {
return &errorCloseReadCloser{Reader: strings.NewReader("audio-data"), err: closeErr}, nil
})
if err != nil {
t.Fatalf("newMultipartBodyFactory() error = %v", err)
}
body, err := factory.NewBody()
if err != nil {
t.Fatalf("NewBody() error = %v", err)
}
if _, err := io.ReadAll(body); err != nil {
t.Fatalf("ReadAll() error = %v", err)
}
if err := body.Close(); !errors.Is(err, closeErr) {
t.Fatalf("Close() error = %v, want source close failure", err)
}
if err := factory.SourceError(false); !errors.Is(err, closeErr) {
t.Fatalf("SourceError(false) = %v, want source close failure", err)
}
})
}
func TestMultipartBodyCloseAndCancelReleaseBlockedSource(t *testing.T) {
for _, test := range []struct {
name string
release func(context.CancelFunc, io.ReadCloser)
}{
{name: "close", release: func(_ context.CancelFunc, body io.ReadCloser) { _ = body.Close() }},
{name: "cancel", release: func(cancel context.CancelFunc, _ io.ReadCloser) { cancel() }},
} {
t.Run(test.name, func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
source := newSignalingBlockedReadCloser()
factory, err := newMultipartBodyFactory(ctx, "audio.flac", "en", func(string) (io.ReadCloser, error) {
return source, nil
})
if err != nil {
t.Fatalf("newMultipartBodyFactory() error = %v", err)
}
defer factory.Close()
body, err := factory.NewBody()
if err != nil {
t.Fatalf("NewBody() error = %v", err)
}
readDone := make(chan error, 1)
go func() {
_, err := io.ReadAll(body)
readDone <- err
}()
select {
case <-source.blocked:
case <-time.After(time.Second):
t.Fatal("audio source did not block")
}
test.release(cancel, body)
select {
case <-source.closed:
case <-time.After(time.Second):
t.Fatal("audio source was not closed")
}
select {
case <-readDone:
case <-time.After(time.Second):
t.Fatal("blocked body read did not return")
}
if test.name == "cancel" {
if _, err := factory.NewBody(); !errors.Is(err, context.Canceled) {
t.Fatalf("NewBody() after cancellation error = %v, want context.Canceled", err)
}
}
})
}
}
func newTestMultipartBodyFactory(t *testing.T, ctx context.Context, audio string) *multipartBodyFactory {
t.Helper()
factory, err := newMultipartBodyFactory(ctx, "audio.flac", "en", func(string) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader(audio)), nil
})
if err != nil {
t.Fatalf("newMultipartBodyFactory() error = %v", err)
}
return factory
}
type multipartTestPart struct {
name string
filename string
data []byte
}
func parseMultipartParts(t *testing.T, contentType string, body []byte) []multipartTestPart {
t.Helper()
mediaType, params, err := mime.ParseMediaType(contentType)
if err != nil {
t.Fatalf("ParseMediaType() error = %v", err)
}
if mediaType != "multipart/form-data" {
t.Fatalf("media type = %q, want multipart/form-data", mediaType)
}
reader := multipart.NewReader(bytes.NewReader(body), params["boundary"])
var parts []multipartTestPart
for {
part, err := reader.NextPart()
if errors.Is(err, io.EOF) {
return parts
}
if err != nil {
t.Fatalf("NextPart() error = %v", err)
}
data, err := io.ReadAll(part)
if err != nil {
t.Fatalf("ReadAll(part %q) error = %v", part.FormName(), err)
}
parts = append(parts, multipartTestPart{name: part.FormName(), filename: part.FileName(), data: data})
}
}
type countedReadCloser struct {
io.Reader
closes *atomic.Int32
once sync.Once
}
func (reader *countedReadCloser) Close() error {
reader.once.Do(func() { reader.closes.Add(1) })
return nil
}
type errorCloseReadCloser struct {
io.Reader
err error
}
func (reader *errorCloseReadCloser) Close() error { return reader.err }
type signalingBlockedReadCloser struct {
blocked chan struct{}
closed chan struct{}
once sync.Once
read bool
}
func newSignalingBlockedReadCloser() *signalingBlockedReadCloser {
return &signalingBlockedReadCloser{blocked: make(chan struct{}), closed: make(chan struct{})}
}
func (reader *signalingBlockedReadCloser) Read(buffer []byte) (int, error) {
if !reader.read {
reader.read = true
return copy(buffer, "audio-data"), nil
}
reader.once.Do(func() { close(reader.blocked) })
<-reader.closed
return 0, errors.New("audio source closed")
}
func (reader *signalingBlockedReadCloser) Close() error {
select {
case <-reader.closed:
default:
close(reader.closed)
}
return nil
}

View File

@@ -691,6 +691,187 @@ func TestExtractLifecycleCredentialChangesRemainResumable(t *testing.T) {
} }
} }
func TestExtractLifecycleConfigProfileChangesRerunExtractionAndDependents(t *testing.T) {
cfg, env, runner := extractionLifecycleFixture(t, true)
selected := "inexpensive"
cfg.Pipeline.Notarius.ConfigProfile = &selected
analyzeRuns := 0
plan := extractionLifecyclePlan(t, &analyzeRuns)
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err != nil {
t.Fatalf("initial executeStages() error = %v", err)
}
if len(runner.requests) != 1 || runner.requests[0].ConfigProfile == nil || *runner.requests[0].ConfigProfile != selected || analyzeRuns != 1 {
t.Fatalf("initial requests=%#v analyze=%d, want selected extraction and analysis", runner.requests, analyzeRuns)
}
reused, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
if err != nil {
t.Fatalf("reuse executeStages() error = %v", err)
}
if len(reused.Executed) != 0 || len(runner.requests) != 1 || analyzeRuns != 1 {
t.Fatalf("same-profile reuse summary=%#v requests=%d analyze=%d", reused, len(runner.requests), analyzeRuns)
}
markLifecycleStageSucceeded(t, cfg, "render")
markLifecycleStageSucceeded(t, cfg, "publish")
before := loadLifecycleManifest(t, cfg)
trimBefore, err := json.Marshal(before.Stages["trim"])
if err != nil {
t.Fatal(err)
}
changed := "experimental"
cfg.Pipeline.Notarius.ConfigProfile = &changed
rerun, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
if err != nil {
t.Fatalf("changed-profile executeStages() error = %v", err)
}
if len(rerun.Executed) != 2 || len(runner.requests) != 2 || runner.requests[1].ConfigProfile == nil || *runner.requests[1].ConfigProfile != changed || analyzeRuns != 2 {
t.Fatalf("changed-profile summary=%#v requests=%#v analyze=%d", rerun, runner.requests, analyzeRuns)
}
after := loadLifecycleManifest(t, cfg)
trimAfter, err := json.Marshal(after.Stages["trim"])
if err != nil {
t.Fatal(err)
}
if string(trimAfter) != string(trimBefore) {
t.Fatalf("profile change altered upstream trim record:\nbefore=%s\nafter=%s", trimBefore, trimAfter)
}
if after.Stages["render"].Status != manifest.StatusSucceeded || after.Stages["analyze"].Status != manifest.StatusSucceeded || after.Stages["publish"].Status != manifest.StatusStale {
t.Fatalf("changed-profile lifecycle render=%#v analyze=%#v publish=%#v", after.Stages["render"], after.Stages["analyze"], after.Stages["publish"])
}
cfg.Pipeline.Notarius.ConfigProfile = nil
returned, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
if err != nil {
t.Fatalf("return-to-base executeStages() error = %v", err)
}
if len(returned.Executed) != 2 || len(runner.requests) != 3 || runner.requests[2].ConfigProfile != nil || analyzeRuns != 3 {
t.Fatalf("return-to-base summary=%#v requests=%#v analyze=%d", returned, runner.requests, analyzeRuns)
}
final := loadLifecycleManifest(t, cfg)
if final.Stages["trim"].Status != manifest.StatusSucceeded || final.Stages["render"].Status != manifest.StatusSucceeded {
t.Fatalf("return-to-base invalidated upstream siblings: trim=%#v render=%#v", final.Stages["trim"], final.Stages["render"])
}
}
func TestExtractLifecycleConfigProfileFailureClearsCurrentResultAndDependents(t *testing.T) {
cfg, env, runner := extractionLifecycleFixture(t, true)
selected := "inexpensive"
cfg.Pipeline.Notarius.ConfigProfile = &selected
analyzeRuns := 0
plan := extractionLifecyclePlan(t, &analyzeRuns)
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err != nil {
t.Fatalf("initial executeStages() error = %v", err)
}
markLifecycleStageSucceeded(t, cfg, "render")
markLifecycleStageSucceeded(t, cfg, "publish")
changed := "missing-profile"
cfg.Pipeline.Notarius.ConfigProfile = &changed
runner.failuresRemaining = 1
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err == nil {
t.Fatal("changed-profile executeStages() error = nil, want downstream failure")
}
failed := loadLifecycleManifest(t, cfg)
extract := failed.Stages["extract"]
if extract.Status != manifest.StatusFailed || len(extract.Outputs) != 0 || len(extract.Logs) != 0 || len(extract.GeneratedConfigs) != 0 || len(extract.Metadata) != 0 {
t.Fatalf("failed changed-profile extraction advertised prior success: %#v", extract)
}
if failed.Stages["trim"].Status != manifest.StatusSucceeded || failed.Stages["render"].Status != manifest.StatusSucceeded ||
failed.Stages["analyze"].Status != manifest.StatusStale || failed.Stages["publish"].Status != manifest.StatusStale || analyzeRuns != 1 {
t.Fatalf("failed changed-profile lifecycle trim=%#v render=%#v analyze=%#v publish=%#v analyzeRuns=%d",
failed.Stages["trim"], failed.Stages["render"], failed.Stages["analyze"], failed.Stages["publish"], analyzeRuns)
}
if len(runner.requests) != 2 || runner.requests[1].ConfigProfile == nil || *runner.requests[1].ConfigProfile != changed {
t.Fatalf("failed request = %#v, want changed selector", runner.requests)
}
}
func TestExtractLifecycleRealConfigProfilesSelectNotariusExplicitly(t *testing.T) {
seed, env, runner := extractionLifecycleFixture(t, true)
workspace := seed.Pipeline.Workspace.Root
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspace)
pipelineFile, err := os.OpenFile(pipelinePath, os.O_APPEND|os.O_WRONLY, 0)
if err != nil {
t.Fatal(err)
}
_, writeErr := pipelineFile.WriteString(`notarius:
enabled: true
binary: ` + filepath.Join(workspace, "notarius") + `
config_path: notarius.yml
pipeline_id: dnd-session
timeout: 45m
outputs:
npc_registry:
lane_id: npc-registry
media_type: application/json
schema_id: notarius.dnd.npc_registry
schema_version: v1
module_key: dnd/npc-registry
`)
closeErr := pipelineFile.Close()
if writeErr != nil || closeErr != nil {
t.Fatalf("write Notarius configuration: %v / %v", writeErr, closeErr)
}
if err := os.WriteFile(filepath.Join(filepath.Dir(pipelinePath), "notarius.yml"), []byte("pipelines: {}\n"), 0o644); err != nil {
t.Fatal(err)
}
profilesDir := filepath.Join(filepath.Dir(pipelinePath), "profiles")
if err := os.MkdirAll(profilesDir, 0o755); err != nil {
t.Fatal(err)
}
for name, contents := range map[string]string{
"testing": "whisperx:\n language: fr\nnotarius:\n config_profile: downstream-economy\n",
"narratio-only": "whisperx:\n language: de\n",
"narratio-only-two": "whisperx:\n language: es\n",
} {
if err := os.WriteFile(filepath.Join(profilesDir, name+".yml"), []byte(contents), 0o644); err != nil {
t.Fatal(err)
}
}
load := func(profile string) *config.Config {
t.Helper()
loaded, err := config.LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{Profile: &profile})
if err != nil {
t.Fatalf("load profile %q: %v", profile, err)
}
return loaded
}
plan, err := BuildSingleStagePlan("extract")
if err != nil {
t.Fatal(err)
}
testingConfig := load("testing")
runner.cfg = testingConfig.Pipeline.Notarius
if _, err := executeStages(context.Background(), testingConfig, plan, RunOptions{Env: env}); err != nil {
t.Fatalf("execute testing profile: %v", err)
}
if len(runner.requests) != 1 || runner.requests[0].ConfigProfile == nil || *runner.requests[0].ConfigProfile != "downstream-economy" {
t.Fatalf("testing profile request = %#v, want explicit differently named downstream selector", runner.requests)
}
narratioOnly := load("narratio-only")
runner.cfg = narratioOnly.Pipeline.Notarius
if _, err := executeStages(context.Background(), narratioOnly, plan, RunOptions{Env: env}); err != nil {
t.Fatalf("execute Narratio-only profile: %v", err)
}
if len(runner.requests) != 2 || runner.requests[1].ConfigProfile != nil {
t.Fatalf("Narratio-only profile request = %#v, want no automatic downstream selector", runner.requests)
}
narratioOnlyTwo := load("narratio-only-two")
runner.cfg = narratioOnlyTwo.Pipeline.Notarius
reused, err := executeStages(context.Background(), narratioOnlyTwo, plan, RunOptions{Env: env})
if err != nil {
t.Fatalf("execute second Narratio-only profile: %v", err)
}
if len(reused.Executed) != 0 || len(runner.requests) != 2 {
t.Fatalf("Narratio profile label/effective digest broadened invalidation: summary=%#v requests=%#v", reused, runner.requests)
}
}
func TestPreparedPipelineAndManifestExcludeNotariusCredentialValue(t *testing.T) { func TestPreparedPipelineAndManifestExcludeNotariusCredentialValue(t *testing.T) {
const name = "NARRATIO_LIFECYCLE_PERSISTED_KEY" const name = "NARRATIO_LIFECYCLE_PERSISTED_KEY"
const canary = "notarius-persisted-credential-canary" const canary = "notarius-persisted-credential-canary"

View File

@@ -82,6 +82,28 @@ func TestExecuteRestoreRoundTripsPublishedExtractionAndManifestMetadata(t *testi
remoteManifest.RunID = "20260519T010203Z-a1b2c3d4" remoteManifest.RunID = "20260519T010203Z-a1b2c3d4"
producerRoot := "/prior/workspace/work/sample-campaign/2026-05-03" producerRoot := "/prior/workspace/work/sample-campaign/2026-05-03"
remoteManifest.LocalWorkDir = filepath.Join(producerRoot, "runs", remoteManifest.RunID) remoteManifest.LocalWorkDir = filepath.Join(producerRoot, "runs", remoteManifest.RunID)
rawRelative := "transcripts/raw/alice.json"
rawOutput := manifest.ArtifactRecord{
Kind: "transcript_raw", LocalPath: filepath.Join(producerRoot, filepath.FromSlash(rawRelative)),
ProducerRunID: remoteManifest.RunID, Checksum: strings.Repeat("b", 64),
}
remoteManifest.MarkStageSucceeded("transcribe", time.Now().UTC(), []manifest.ArtifactRecord{rawOutput})
remoteManifest.Stages["transcribe"].TranscribeStateVersion = manifest.TranscribeStateContractVersion
remoteManifest.Stages["transcribe"].TranscribeSpeakers = map[string]manifest.TranscribeSpeakerRecord{
"alice": {
FingerprintVersion: manifest.TranscribeFingerprintContractVersion,
Fingerprint: strings.Repeat("c", 64),
AudioPath: "audio/alice.flac",
AudioSHA256: strings.Repeat("a", 64),
AudioSize: 12,
Output: manifest.ArtifactRecord{
Kind: "transcript_raw", LocalPath: rawRelative,
ProducerRunID: remoteManifest.RunID, Checksum: strings.Repeat("b", 64),
},
OutputSize: 15,
ProducerRunID: remoteManifest.RunID,
},
}
remoteManifest.Stages["extract"] = &manifest.StageRecord{ remoteManifest.Stages["extract"] = &manifest.StageRecord{
Name: "extract", Status: manifest.StatusSucceeded, Name: "extract", Status: manifest.StatusSucceeded,
Outputs: []manifest.ArtifactRecord{{ Outputs: []manifest.ArtifactRecord{{
@@ -121,6 +143,13 @@ func TestExecuteRestoreRoundTripsPublishedExtractionAndManifestMetadata(t *testi
if want := filepath.Join(sessionRoot, "artifacts", "encounters.json"); lane.LocalPath != want { if want := filepath.Join(sessionRoot, "artifacts", "encounters.json"); lane.LocalPath != want {
t.Fatalf("rebased extraction path = %q, want %q", lane.LocalPath, want) t.Fatalf("rebased extraction path = %q, want %q", lane.LocalPath, want)
} }
speaker := restored.Stages["transcribe"].TranscribeSpeakers["alice"]
if speaker.AudioPath != "audio/alice.flac" || speaker.Output.LocalPath != rawRelative {
t.Fatalf("restored speaker paths = %#v, want session-relative paths", speaker)
}
if _, err := os.Stat(filepath.Join(sessionRoot, filepath.FromSlash(rawRelative))); !os.IsNotExist(err) {
t.Fatalf("unpublished raw transcript should not be restored; stat err=%v", err)
}
} }
func TestExecuteRestoreIncludeAudioRestoresAudio(t *testing.T) { func TestExecuteRestoreIncludeAudioRestoresAudio(t *testing.T) {

View File

@@ -1082,7 +1082,7 @@ func TestExecuteStagesReportsSkippedStage(t *testing.T) {
t.Fatalf("Save manifest error = %v", err) t.Fatalf("Save manifest error = %v", err)
} }
summary, err := executeStages(context.Background(), cfg, []stage.Stage{BuildFullPlan()[1]}, RunOptions{}) summary, err := executeStages(context.Background(), cfg, []stage.Stage{resultStage{name: "transcribe"}}, RunOptions{})
if err != nil { if err != nil {
t.Fatalf("executeStages() error = %v", err) t.Fatalf("executeStages() error = %v", err)
} }
@@ -1302,7 +1302,7 @@ func TestExecuteStagesSkippedStagePreservesExistingOutputsProvenance(t *testing.
t.Fatalf("Save manifest error = %v", err) t.Fatalf("Save manifest error = %v", err)
} }
summary, err := executeStages(context.Background(), cfg, []stage.Stage{BuildFullPlan()[1]}, RunOptions{}) summary, err := executeStages(context.Background(), cfg, []stage.Stage{resultStage{name: "transcribe"}}, RunOptions{})
if err != nil { if err != nil {
t.Fatalf("executeStages() error = %v", err) t.Fatalf("executeStages() error = %v", err)
} }

View File

@@ -2,10 +2,18 @@ package app
import ( import (
"context" "context"
"crypto/sha256"
"encoding/hex"
"os"
"path/filepath"
"slices"
"strings"
"testing" "testing"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config" "gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest" "gitea.maximumdirect.net/eric/narratio/internal/manifest"
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
"gitea.maximumdirect.net/eric/narratio/internal/stage" "gitea.maximumdirect.net/eric/narratio/internal/stage"
) )
@@ -38,6 +46,94 @@ func seedCurrentSemanticEvidence(t *testing.T, cfg *config.Config, m *manifest.M
t.Fatalf("fingerprint stage %q: %v", name, err) t.Fatalf("fingerprint stage %q: %v", name, err)
} }
record.SemanticConfig = &fingerprint record.SemanticConfig = &fingerprint
if name == "transcribe" {
seedCurrentTranscribeEvidence(t, cfg, m, fingerprint)
}
}
}
func seedCurrentTranscribeEvidence(
t *testing.T,
cfg *config.Config,
m *manifest.Manifest,
semantic manifest.SemanticConfigFingerprint,
) {
t.Helper()
store := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root)
paths := store.SessionPathsFor(cfg.Session.Campaign, m.SessionID)
if strings.TrimSpace(m.RunID) == "" {
m.RunID = "20260503T010203Z-a1b2c3d4"
}
records := make(map[string]manifest.TranscribeSpeakerRecord)
for _, input := range m.Inputs {
if input.Kind != "audio" {
continue
}
audioPath := artifacts.ResolveSessionLocalPathForRead(paths, input.Path)
audioBytes, err := os.ReadFile(audioPath)
if err != nil {
t.Fatalf("read transcribe audio fixture %q: %v", audioPath, err)
}
audioRelative, err := pathsafe.SlashRelativeFromRoot(paths.Root, audioPath)
if err != nil {
t.Fatalf("relativize transcribe audio fixture %q: %v", audioPath, err)
}
speaker := strings.TrimSuffix(filepath.Base(audioPath), filepath.Ext(audioPath))
outputRelative := filepath.ToSlash(filepath.Join("transcripts", "raw", speaker+".json"))
outputPath, err := pathsafe.JoinSlashRelativeUnderRoot(paths.Root, outputRelative)
if err != nil {
t.Fatalf("resolve transcribe output fixture: %v", err)
}
if _, err := os.Stat(outputPath); os.IsNotExist(err) {
mustWriteTestFile(t, outputPath, `{"segments":[]}`)
} else if err != nil {
t.Fatalf("inspect transcribe output fixture %q: %v", outputPath, err)
}
outputBytes, err := os.ReadFile(outputPath)
if err != nil {
t.Fatalf("read transcribe output fixture %q: %v", outputPath, err)
}
audioDigest := sha256.Sum256(audioBytes)
outputDigest := sha256.Sum256(outputBytes)
fingerprint, err := stage.FingerprintSemanticConfig(
manifest.TranscribeFingerprintContractVersion,
struct {
SpeakerID string `json:"speaker_id"`
AudioPath string `json:"audio_path"`
AudioSHA256 string `json:"audio_sha256"`
AudioSize int64 `json:"audio_size"`
SemanticConfigVersion int `json:"semantic_config_version"`
SemanticConfigDigest string `json:"semantic_config_digest"`
}{
SpeakerID: speaker, AudioPath: audioRelative,
AudioSHA256: hex.EncodeToString(audioDigest[:]), AudioSize: int64(len(audioBytes)),
SemanticConfigVersion: semantic.Version, SemanticConfigDigest: semantic.Digest,
},
)
if err != nil {
t.Fatalf("fingerprint transcribe speaker fixture %q: %v", speaker, err)
}
output := manifest.ArtifactRecord{
Kind: "transcript_raw", LocalPath: outputRelative,
ProducerRunID: m.RunID, Checksum: hex.EncodeToString(outputDigest[:]),
}
records[speaker] = manifest.TranscribeSpeakerRecord{
FingerprintVersion: manifest.TranscribeFingerprintContractVersion,
Fingerprint: fingerprint.Digest, AudioPath: audioRelative,
AudioSHA256: hex.EncodeToString(audioDigest[:]), AudioSize: int64(len(audioBytes)),
Output: output, OutputSize: int64(len(outputBytes)), ProducerRunID: m.RunID,
}
}
record := m.Stages["transcribe"]
record.TranscribeStateVersion = manifest.TranscribeStateContractVersion
record.TranscribeSpeakers = records
if len(record.Outputs) == 0 {
for _, speakerRecord := range records {
record.Outputs = append(record.Outputs, speakerRecord.Output)
}
slices.SortFunc(record.Outputs, func(a, b manifest.ArtifactRecord) int {
return strings.Compare(a.LocalPath, b.LocalPath)
})
} }
} }

View File

@@ -0,0 +1,472 @@
package app
import (
"context"
"errors"
"os"
"path/filepath"
"slices"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/seriatim"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/whisperx"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
"gitea.maximumdirect.net/eric/narratio/internal/stage"
)
func TestExecuteStagesDelegatesEmptyAudioAcceptanceToWhisperX(t *testing.T) {
for _, rejected := range []bool{false, true} {
name := "accepted"
if rejected {
name = "rejected"
}
t.Run(name, func(t *testing.T) {
cfg, store := setupTranscribeRecoveryFixture(t, []string{"alice"})
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
if err := os.WriteFile(filepath.Join(paths.AudioDir, "alice.flac"), nil, 0o644); err != nil {
t.Fatal(err)
}
fake := &whisperx.FakeClient{}
if rejected {
fake.Err = errors.New("service rejected audio")
}
transcribe := mustSelectStage(t, "transcribe")
_, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{
Env: &Env{ManifestStore: store, WhisperX: fake},
})
if !errors.Is(err, fake.Err) {
t.Fatalf("transcribe error = %v, want %v", err, fake.Err)
}
if len(fake.RequestsSnapshot()) != 1 {
t.Fatal("empty audio was not sent to service")
}
m := loadTranscribeRecoveryManifest(t, store, cfg)
if rejected {
if len(m.Stages["transcribe"].TranscribeSpeakers) != 0 {
t.Fatal("rejected audio was checkpointed")
}
return
}
record, ok := m.Stages["transcribe"].TranscribeSpeakers["alice"]
if !ok || record.AudioSize != 0 {
t.Fatalf("missing zero-size checkpoint: %#v", record)
}
retry := &whisperx.FakeClient{}
summary, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{
Env: &Env{ManifestStore: store, WhisperX: retry},
})
if err != nil {
t.Fatal(err)
}
if len(summary.Skipped) != 1 || len(retry.RequestsSnapshot()) != 0 {
t.Fatal("successful empty-audio result was not reused")
}
})
}
}
func TestExecuteStagesRetriesOnlyUncommittedTranscribeSpeakers(t *testing.T) {
cfg, baseStore := setupTranscribeRecoveryFixture(t, []string{"a", "b", "c", "d"})
concurrency := 1
cfg.Pipeline.WhisperX.Concurrency = &concurrency
cause := errors.New("speaker c failed")
failing := &whisperx.FakeClient{TranscribeFn: func(
_ context.Context,
req whisperx.TranscribeRequest,
) (whisperx.TranscribeResult, error) {
if req.SpeakerID == "c" {
return whisperx.TranscribeResult{}, cause
}
if req.SpeakerID == "d" {
t.Fatalf("speaker d was dispatched after c failed")
}
return writeTranscribeRecoveryResult(req)
}}
transcribe := mustSelectStage(t, "transcribe")
if _, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{
Env: &Env{ManifestStore: baseStore, WhisperX: failing},
}); !errors.Is(err, cause) {
t.Fatalf("failed executeStages() error = %v, want %v", err, cause)
}
failed := loadTranscribeRecoveryManifest(t, baseStore, cfg)
if failed.Stages["transcribe"].Status != manifest.StatusFailed {
t.Fatalf("transcribe status = %q, want failed", failed.Stages["transcribe"].Status)
}
if got := transcribeSpeakerKeys(failed); !slices.Equal(got, []string{"a", "b"}) {
t.Fatalf("checkpointed speakers = %#v, want a and b", got)
}
if failed.Stages["prepare"].Status != manifest.StatusSucceeded {
t.Fatalf("prepare status = %q, want unchanged success", failed.Stages["prepare"].Status)
}
if failed.Stages["merge"].Status != manifest.StatusStale {
t.Fatalf("merge status = %q, want stale after transcribe failure", failed.Stages["merge"].Status)
}
retry := &whisperx.FakeClient{}
if _, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{
Env: &Env{ManifestStore: baseStore, WhisperX: retry},
}); err != nil {
t.Fatalf("retry executeStages() error = %v", err)
}
if got := requestedTranscribeSpeakers(retry); !slices.Equal(got, []string{"c", "d"}) {
t.Fatalf("retry speakers = %#v, want c and d", got)
}
completed := loadTranscribeRecoveryManifest(t, baseStore, cfg)
if completed.Stages["transcribe"].Status != manifest.StatusSucceeded {
t.Fatalf("transcribe status = %q, want succeeded", completed.Stages["transcribe"].Status)
}
if got := transcribeSpeakerKeys(completed); !slices.Equal(got, []string{"a", "b", "c", "d"}) {
t.Fatalf("completed speakers = %#v", got)
}
}
func TestExecuteStagesRebuildsTranscribeAggregateAfterFinalSaveFailure(t *testing.T) {
cfg, baseStore := setupTranscribeRecoveryFixture(t, []string{"alice", "bob"})
store := &transcribeAggregateSaveFailingStore{
Store: baseStore,
Err: errors.New("aggregate save failed"),
}
fake := &whisperx.FakeClient{}
transcribe := mustSelectStage(t, "transcribe")
if _, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{
Env: &Env{ManifestStore: store, WhisperX: fake},
}); !errors.Is(err, store.Err) {
t.Fatalf("executeStages() error = %v, want aggregate save failure", err)
}
failed := loadTranscribeRecoveryManifest(t, baseStore, cfg)
if failed.Stages["transcribe"].Status != manifest.StatusFailed {
t.Fatalf("transcribe status = %q, want failed", failed.Stages["transcribe"].Status)
}
if got := transcribeSpeakerKeys(failed); !slices.Equal(got, []string{"alice", "bob"}) {
t.Fatalf("checkpointed speakers = %#v, want both speakers", got)
}
retry := &whisperx.FakeClient{}
if _, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{
Env: &Env{ManifestStore: baseStore, WhisperX: retry},
}); err != nil {
t.Fatalf("retry executeStages() error = %v", err)
}
if got := len(retry.RequestsSnapshot()); got != 0 {
t.Fatalf("retry adapter requests = %d, want aggregate-only rebuild", got)
}
completed := loadTranscribeRecoveryManifest(t, baseStore, cfg)
if completed.Stages["transcribe"].Status != manifest.StatusSucceeded || len(completed.Stages["transcribe"].Outputs) != 2 {
t.Fatalf("rebuilt aggregate = %#v", completed.Stages["transcribe"])
}
}
func TestExecuteStagesRepairsSucceededTranscribeAggregateWithoutRequests(t *testing.T) {
for _, test := range []struct {
name string
mutate func(*manifest.Manifest)
removed bool
}{
{name: "removed speaker", removed: true, mutate: func(m *manifest.Manifest) {
m.Inputs = slices.DeleteFunc(m.Inputs, func(in manifest.InputRecord) bool { return filepath.Base(in.Path) == "bob.flac" })
}},
{name: "missing output", mutate: func(m *manifest.Manifest) { m.Stages["transcribe"].Outputs = m.Stages["transcribe"].Outputs[:1] }},
{name: "duplicate output", mutate: func(m *manifest.Manifest) { m.Stages["transcribe"].Outputs[1] = m.Stages["transcribe"].Outputs[0] }},
{name: "wrong path", mutate: func(m *manifest.Manifest) { m.Stages["transcribe"].Outputs[0].LocalPath = "transcripts/raw/other.json" }},
{name: "wrong checksum", mutate: func(m *manifest.Manifest) { m.Stages["transcribe"].Outputs[0].Checksum = "obsolete" }},
{name: "wrong producer", mutate: func(m *manifest.Manifest) { m.Stages["transcribe"].Outputs[0].ProducerRunID = "other-run" }},
} {
t.Run(test.name, func(t *testing.T) {
cfg, store := setupTranscribeRecoveryFixture(t, []string{"alice", "bob"})
transcribe := mustSelectStage(t, "transcribe")
if _, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{
Env: &Env{ManifestStore: store, WhisperX: &whisperx.FakeClient{}},
}); err != nil {
t.Fatal(err)
}
m := loadTranscribeRecoveryManifest(t, store, cfg)
test.mutate(m)
m.MarkStageSucceeded("merge", time.Now().UTC(), nil)
if err := store.Save(context.Background(), manifestPathFor(cfg), m); err != nil {
t.Fatal(err)
}
retry := &whisperx.FakeClient{}
summary, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{
Env: &Env{ManifestStore: store, WhisperX: retry},
})
if err != nil {
t.Fatal(err)
}
if len(summary.Skipped) != 0 || len(retry.RequestsSnapshot()) != 0 {
t.Fatalf("want aggregate repair without HTTP; summary=%#v requests=%#v", summary, retry.RequestsSnapshot())
}
completed := loadTranscribeRecoveryManifest(t, store, cfg)
want := []string{"alice", "bob"}
if test.removed {
want = []string{"alice"}
}
record := completed.Stages["transcribe"]
if !slices.Equal(transcribeSpeakerKeys(completed), want) || len(record.Outputs) != len(want) {
t.Fatalf("unexpected repaired state: %#v", record)
}
for _, output := range record.Outputs {
speaker := record.TranscribeSpeakers[strings.TrimSuffix(filepath.Base(output.LocalPath), ".json")]
if output.Checksum != speaker.Output.Checksum || output.ProducerRunID != speaker.ProducerRunID {
t.Fatalf("aggregate evidence does not match checkpoint: %#v", output)
}
}
if completed.Stages["merge"].Status != manifest.StatusStale {
t.Fatal("aggregate repair did not stale merge")
}
if test.removed {
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
for _, name := range []string{"speakers.yml", "autocorrect.yml"} {
if err := os.WriteFile(filepath.Join(paths.InputsDir, name), []byte("{}"), 0o644); err != nil {
t.Fatal(err)
}
}
merge := &seriatim.FakeRunner{}
if _, err := executeStages(context.Background(), cfg, []stage.Stage{mustSelectStage(t, "merge")}, RunOptions{
Env: &Env{ManifestStore: store, Seriatim: merge},
}); err != nil {
t.Fatal(err)
}
if len(merge.NormalizeRequests) != 1 || filepath.Base(merge.NormalizeRequests[0].InputTranscriptPath) != "alice.json" {
t.Fatalf("removed speaker reached merge: %#v", merge.NormalizeRequests)
}
if _, err := os.Stat(filepath.Join(paths.TranscriptsRawDir, "bob.json")); err != nil {
t.Fatalf("incidental transcript was removed: %v", err)
}
}
})
}
}
func TestExecuteStagesResumesInterruptedRunningTranscribeState(t *testing.T) {
cfg, baseStore := setupTranscribeRecoveryFixture(t, []string{"alice", "bob"})
concurrency := 1
cfg.Pipeline.WhisperX.Concurrency = &concurrency
stageErr := errors.New("speaker b failed")
terminalErr := errors.New("terminal save interrupted")
store := &interruptAfterTranscribeCheckpointStore{Store: baseStore, Err: terminalErr}
fake := &whisperx.FakeClient{TranscribeFn: func(
_ context.Context,
req whisperx.TranscribeRequest,
) (whisperx.TranscribeResult, error) {
if req.SpeakerID == "bob" {
return whisperx.TranscribeResult{}, stageErr
}
return writeTranscribeRecoveryResult(req)
}}
transcribe := mustSelectStage(t, "transcribe")
_, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{
Env: &Env{ManifestStore: store, WhisperX: fake},
})
if !errors.Is(err, stageErr) || !errors.Is(err, terminalErr) {
t.Fatalf("executeStages() error = %v, want stage and terminal failures", err)
}
interrupted := loadTranscribeRecoveryManifest(t, baseStore, cfg)
if interrupted.Stages["transcribe"].Status != manifest.StatusRunning {
t.Fatalf("transcribe status = %q, want durable running state", interrupted.Stages["transcribe"].Status)
}
if got := transcribeSpeakerKeys(interrupted); !slices.Equal(got, []string{"alice"}) {
t.Fatalf("checkpointed speakers = %#v, want alice", got)
}
retry := &whisperx.FakeClient{}
if _, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{
Env: &Env{ManifestStore: baseStore, WhisperX: retry},
}); err != nil {
t.Fatalf("retry executeStages() error = %v", err)
}
if got := requestedTranscribeSpeakers(retry); !slices.Equal(got, []string{"bob"}) {
t.Fatalf("retry speakers = %#v, want bob", got)
}
}
func TestExecuteStagesPersistsForcedTranscribeResetAcrossRetry(t *testing.T) {
cfg, store := setupTranscribeRecoveryFixture(t, []string{"alice", "bob"})
concurrency := 1
cfg.Pipeline.WhisperX.Concurrency = &concurrency
transcribe := mustSelectStage(t, "transcribe")
if _, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{
Env: &Env{ManifestStore: store, WhisperX: &whisperx.FakeClient{}},
}); err != nil {
t.Fatalf("baseline executeStages() error = %v", err)
}
cause := errors.New("forced bob failed")
forced := &whisperx.FakeClient{TranscribeFn: func(
_ context.Context,
req whisperx.TranscribeRequest,
) (whisperx.TranscribeResult, error) {
if req.SpeakerID == "bob" {
return whisperx.TranscribeResult{}, cause
}
return writeTranscribeRecoveryResult(req)
}}
if _, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{
Force: true,
Env: &Env{ManifestStore: store, WhisperX: forced},
}); !errors.Is(err, cause) {
t.Fatalf("forced executeStages() error = %v, want %v", err, cause)
}
failed := loadTranscribeRecoveryManifest(t, store, cfg)
if got := transcribeSpeakerKeys(failed); !slices.Equal(got, []string{"alice"}) {
t.Fatalf("speakers after forced failure = %#v, want only replacement alice", got)
}
retry := &whisperx.FakeClient{}
if _, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{
Env: &Env{ManifestStore: store, WhisperX: retry},
}); err != nil {
t.Fatalf("retry executeStages() error = %v", err)
}
if got := requestedTranscribeSpeakers(retry); !slices.Equal(got, []string{"bob"}) {
t.Fatalf("retry speakers = %#v, want bob", got)
}
}
func TestExecuteStagesUpgradesLegacyTranscribeSuccessOnce(t *testing.T) {
cfg, store := setupTranscribeRecoveryFixture(t, []string{"alice"})
m := loadTranscribeRecoveryManifest(t, store, cfg)
m.MarkStageSucceeded("transcribe", time.Now().UTC(), []manifest.ArtifactRecord{{
Kind: "transcript_raw", LocalPath: "transcripts/raw/alice.json",
}})
if err := store.Save(context.Background(), manifestPathFor(cfg), m); err != nil {
t.Fatalf("save legacy transcribe success: %v", err)
}
transcribe := mustSelectStage(t, "transcribe")
upgrade := &whisperx.FakeClient{}
if _, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{
Env: &Env{ManifestStore: store, WhisperX: upgrade},
}); err != nil {
t.Fatalf("upgrade executeStages() error = %v", err)
}
if got := requestedTranscribeSpeakers(upgrade); !slices.Equal(got, []string{"alice"}) {
t.Fatalf("upgrade speakers = %#v, want alice", got)
}
upgraded := loadTranscribeRecoveryManifest(t, store, cfg)
if upgraded.Stages["transcribe"].TranscribeStateVersion != manifest.TranscribeStateContractVersion {
t.Fatalf("transcribe state version = %d, want current", upgraded.Stages["transcribe"].TranscribeStateVersion)
}
reused := &whisperx.FakeClient{}
summary, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{
Env: &Env{ManifestStore: store, WhisperX: reused},
})
if err != nil {
t.Fatalf("reuse executeStages() error = %v", err)
}
if len(summary.Skipped) != 1 || len(reused.RequestsSnapshot()) != 0 {
t.Fatalf("reuse summary = %#v, requests = %#v, want skipped without HTTP", summary, reused.RequestsSnapshot())
}
}
type transcribeAggregateSaveFailingStore struct {
manifest.Store
Err error
Failed bool
}
func (store *transcribeAggregateSaveFailingStore) Save(
ctx context.Context,
path string,
m *manifest.Manifest,
) error {
record := m.Stages["transcribe"]
if !store.Failed && record != nil && record.Status == manifest.StatusSucceeded &&
record.TranscribeStateVersion == manifest.TranscribeStateContractVersion {
store.Failed = true
return store.Err
}
return store.Store.Save(ctx, path, m)
}
type interruptAfterTranscribeCheckpointStore struct {
manifest.Store
Err error
Interrupted bool
}
func (store *interruptAfterTranscribeCheckpointStore) Save(
ctx context.Context,
path string,
m *manifest.Manifest,
) error {
if store.Interrupted {
return store.Err
}
record := m.Stages["transcribe"]
err := store.Store.Save(ctx, path, m)
if err == nil && record != nil && record.Status == manifest.StatusRunning && len(record.TranscribeSpeakers) == 1 {
store.Interrupted = true
}
return err
}
func setupTranscribeRecoveryFixture(t *testing.T, speakers []string) (*config.Config, *manifest.LocalStore) {
t.Helper()
cfg := testConfig(t)
store := &manifest.LocalStore{}
artifactStore := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root)
paths, err := artifactStore.EnsureLayoutFor(cfg.Session.Campaign, cfg.Session.SessionID)
if err != nil {
t.Fatalf("prepare session layout: %v", err)
}
m := manifest.New(cfg.Session.SessionID, time.Now().UTC())
m.Campaign = cfg.Session.Campaign
m.Inputs = make([]manifest.InputRecord, 0, len(speakers))
for _, speaker := range speakers {
relative := filepath.ToSlash(filepath.Join("audio", speaker+".flac"))
if err := os.WriteFile(filepath.Join(paths.AudioDir, speaker+".flac"), []byte("audio-"+speaker), 0o644); err != nil {
t.Fatalf("write audio fixture: %v", err)
}
m.Inputs = append(m.Inputs, manifest.InputRecord{Kind: "audio", Path: relative})
}
m.MarkStageSucceeded("prepare", time.Now().UTC(), nil)
m.MarkStageSucceeded("merge", time.Now().UTC(), nil)
if err := store.Save(context.Background(), paths.ManifestPath, m); err != nil {
t.Fatalf("save recovery fixture: %v", err)
}
return cfg, store
}
func writeTranscribeRecoveryResult(req whisperx.TranscribeRequest) (whisperx.TranscribeResult, error) {
if err := os.MkdirAll(filepath.Dir(req.OutputRawTranscriptPath), 0o755); err != nil {
return whisperx.TranscribeResult{}, err
}
if err := os.WriteFile(req.OutputRawTranscriptPath, []byte(`{"segments":[]}`), 0o644); err != nil {
return whisperx.TranscribeResult{}, err
}
return whisperx.TranscribeResult{OutputRawTranscriptPath: req.OutputRawTranscriptPath}, nil
}
func loadTranscribeRecoveryManifest(t *testing.T, store manifest.Store, cfg *config.Config) *manifest.Manifest {
t.Helper()
m, err := store.Load(context.Background(), manifestPathFor(cfg))
if err != nil {
t.Fatalf("load transcribe recovery manifest: %v", err)
}
return m
}
func transcribeSpeakerKeys(m *manifest.Manifest) []string {
keys := make([]string, 0, len(m.Stages["transcribe"].TranscribeSpeakers))
for key := range m.Stages["transcribe"].TranscribeSpeakers {
keys = append(keys, key)
}
slices.Sort(keys)
return keys
}
func requestedTranscribeSpeakers(fake *whisperx.FakeClient) []string {
requests := fake.RequestsSnapshot()
speakers := make([]string, 0, len(requests))
for _, request := range requests {
speakers = append(speakers, request.SpeakerID)
}
return speakers
}

View File

@@ -7,9 +7,12 @@ import (
"net/http/httptest" "net/http/httptest"
"os" "os"
"path/filepath" "path/filepath"
"slices"
"strings" "strings"
"sync"
"sync/atomic" "sync/atomic"
"testing" "testing"
"time"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/whisperx" "gitea.maximumdirect.net/eric/narratio/internal/adapters/whisperx"
"gitea.maximumdirect.net/eric/narratio/internal/config" "gitea.maximumdirect.net/eric/narratio/internal/config"
@@ -118,6 +121,86 @@ func TestExecuteStagesDefaultWiringWhisperXFailureMarksManifestFailed(t *testing
} }
} }
func TestExhaustedWhisperXFailurePreservesEarlierSpeakerCheckpoint(t *testing.T) {
var successful atomic.Bool
var requestMu sync.Mutex
var requested []string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
file, header, err := r.FormFile("file")
if err != nil {
t.Errorf("FormFile(file) error = %v", err)
http.Error(w, "invalid multipart", http.StatusBadRequest)
return
}
_, readErr := io.Copy(io.Discard, file)
_ = file.Close()
if readErr != nil {
t.Errorf("read uploaded file: %v", readErr)
return
}
speaker := strings.TrimSuffix(header.Filename, filepath.Ext(header.Filename))
requestMu.Lock()
requested = append(requested, speaker)
requestMu.Unlock()
if speaker == "b" && !successful.Load() {
http.Error(w, "temporary", http.StatusInternalServerError)
return
}
_, _ = w.Write([]byte(`{"segments":[]}`))
}))
defer srv.Close()
cfg, store := setupTranscribeRecoveryFixture(t, []string{"a", "b", "c"})
setWhisperXConfig(cfg, srv.URL, 1)
concurrency := 1
cfg.Pipeline.WhisperX.Concurrency = &concurrency
httpClient, err := whisperx.NewHTTPClient(whisperx.HTTPClientConfig{
TranscribeURL: srv.URL,
Language: "en",
Timeout: 2 * time.Second,
Retries: 1,
RetryDelay: time.Millisecond,
})
if err != nil {
t.Fatalf("NewHTTPClient() error = %v", err)
}
transcribe := mustSelectStage(t, "transcribe")
if _, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{
Env: &Env{ManifestStore: store, WhisperX: httpClient},
}); err == nil {
t.Fatal("executeStages() error = nil, want exhausted HTTP failure")
}
failed := loadTranscribeRecoveryManifest(t, store, cfg)
if got := transcribeSpeakerKeys(failed); !slices.Equal(got, []string{"a"}) {
t.Fatalf("checkpointed speakers = %#v, want a", got)
}
requestMu.Lock()
firstRequests := append([]string(nil), requested...)
requested = nil
requestMu.Unlock()
if !slices.Equal(firstRequests, []string{"a", "b", "b"}) {
t.Fatalf("initial HTTP speakers = %#v, want a then two attempts for b", firstRequests)
}
successful.Store(true)
if _, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{
Env: &Env{ManifestStore: store, WhisperX: httpClient},
}); err != nil {
t.Fatalf("retry executeStages() error = %v", err)
}
requestMu.Lock()
retryRequests := append([]string(nil), requested...)
requestMu.Unlock()
if !slices.Equal(retryRequests, []string{"b", "c"}) {
t.Fatalf("retry HTTP speakers = %#v, want only b and c", retryRequests)
}
completed := loadTranscribeRecoveryManifest(t, store, cfg)
if got := transcribeSpeakerKeys(completed); !slices.Equal(got, []string{"a", "b", "c"}) {
t.Fatalf("completed speakers = %#v, want all speakers", got)
}
}
func TestExecuteStagesExplicitWhisperXInjectionOverridesDefaultWiring(t *testing.T) { func TestExecuteStagesExplicitWhisperXInjectionOverridesDefaultWiring(t *testing.T) {
cfg := testConfig(t) cfg := testConfig(t)
setWhisperXConfig(cfg, "https://127.0.0.1:1/transcribe", 0) setWhisperXConfig(cfg, "https://127.0.0.1:1/transcribe", 0)

View File

@@ -310,6 +310,7 @@ type NotariusConfig struct {
Enabled bool `yaml:"enabled" json:"enabled"` Enabled bool `yaml:"enabled" json:"enabled"`
Binary string `yaml:"binary" json:"binary"` Binary string `yaml:"binary" json:"binary"`
LLMAPIKey *configkit.Secret `yaml:"llm_api_key,omitempty" json:"llm_api_key,omitempty"` LLMAPIKey *configkit.Secret `yaml:"llm_api_key,omitempty" json:"llm_api_key,omitempty"`
ConfigProfile *string `yaml:"config_profile,omitempty" json:"config_profile,omitempty"`
Resume bool `yaml:"resume,omitempty" json:"resume,omitempty"` Resume bool `yaml:"resume,omitempty" json:"resume,omitempty"`
ChunkCache string `yaml:"chunk_cache,omitempty" json:"chunk_cache,omitempty"` ChunkCache string `yaml:"chunk_cache,omitempty" json:"chunk_cache,omitempty"`
ConfigPath string `yaml:"config_path" json:"config_path"` ConfigPath string `yaml:"config_path" json:"config_path"`

View File

@@ -56,6 +56,7 @@ func (cfg NotariusConfig) MarshalYAML() (any, error) {
Enabled bool `yaml:"enabled"` Enabled bool `yaml:"enabled"`
Binary string `yaml:"binary"` Binary string `yaml:"binary"`
LLMAPIKey *string `yaml:"llm_api_key,omitempty"` LLMAPIKey *string `yaml:"llm_api_key,omitempty"`
ConfigProfile *string `yaml:"config_profile,omitempty"`
Resume bool `yaml:"resume,omitempty"` Resume bool `yaml:"resume,omitempty"`
ChunkCache string `yaml:"chunk_cache,omitempty"` ChunkCache string `yaml:"chunk_cache,omitempty"`
ConfigPath string `yaml:"config_path"` ConfigPath string `yaml:"config_path"`
@@ -67,7 +68,8 @@ func (cfg NotariusConfig) MarshalYAML() (any, error) {
} }
return safeNotarius{ return safeNotarius{
Enabled: cfg.Enabled, Binary: cfg.Binary, LLMAPIKey: secretReferenceName(cfg.LLMAPIKey), Enabled: cfg.Enabled, Binary: cfg.Binary, LLMAPIKey: secretReferenceName(cfg.LLMAPIKey),
Resume: cfg.Resume, ChunkCache: cfg.ChunkCache, ConfigProfile: cfg.ConfigProfile,
Resume: cfg.Resume, ChunkCache: cfg.ChunkCache,
ConfigPath: cfg.ConfigPath, PipelineID: cfg.PipelineID, Timeout: cfg.Timeout, ConfigPath: cfg.ConfigPath, PipelineID: cfg.PipelineID, Timeout: cfg.Timeout,
WorkingDirectory: cfg.WorkingDirectory, References: cfg.References, Outputs: cfg.Outputs, WorkingDirectory: cfg.WorkingDirectory, References: cfg.References, Outputs: cfg.Outputs,
}, nil }, nil

View File

@@ -103,6 +103,7 @@ storage:
notarius: notarius:
enabled: false enabled: false
llm_api_key: NARRATIO_TEST_NOTARIUS_KEY llm_api_key: NARRATIO_TEST_NOTARIUS_KEY
config_profile: ""
` `
pipelinePath, _ := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML) pipelinePath, _ := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
@@ -140,6 +141,24 @@ notarius:
} }
} }
} }
if !strings.Contains(string(firstYAML), "config_profile: \"\"\n") {
t.Fatalf("effective YAML omitted explicit empty Notarius config profile: %q", firstYAML)
}
var roundTrip struct {
Notarius *struct {
LLMAPIKey *string `yaml:"llm_api_key"`
ConfigProfile *string `yaml:"config_profile"`
} `yaml:"notarius"`
}
if err := yaml.Unmarshal(firstYAML, &roundTrip); err != nil {
t.Fatalf("round-trip effective YAML: %v", err)
}
if roundTrip.Notarius == nil || roundTrip.Notarius.ConfigProfile == nil || *roundTrip.Notarius.ConfigProfile != "" {
t.Fatalf("round-trip config profile = %#v, want explicit empty", roundTrip.Notarius)
}
if roundTrip.Notarius.LLMAPIKey == nil || *roundTrip.Notarius.LLMAPIKey != "NARRATIO_TEST_NOTARIUS_KEY" {
t.Fatalf("round-trip credential reference = %#v, want safe public name", roundTrip.Notarius.LLMAPIKey)
}
t.Setenv("NARRATIO_TEST_ACCESS_KEY", secondCanary) t.Setenv("NARRATIO_TEST_ACCESS_KEY", secondCanary)
t.Setenv("NARRATIO_TEST_SECRET_KEY", secondCanary) t.Setenv("NARRATIO_TEST_SECRET_KEY", secondCanary)

View File

@@ -3,6 +3,7 @@ package config
import ( import (
"os" "os"
"path/filepath" "path/filepath"
"reflect"
"strings" "strings"
"testing" "testing"
) )
@@ -161,6 +162,45 @@ func TestNotariusCredentialProfileReplacementAndProvenance(t *testing.T) {
} }
} }
func TestNotariusConfigProfileComposition(t *testing.T) {
tests := []struct {
name string
baseValue string
overlayName string
overlay string
want *string
}{
{name: "omitted"},
{name: "explicit empty", baseValue: `""`, want: stringPointer("")},
{name: "explicit default", baseValue: "default", want: stringPointer("default")},
{name: "base named selection", baseValue: "production", want: stringPointer("production")},
{name: "profile replaces named selection", baseValue: "production", overlayName: "testing", overlay: "notarius:\n config_profile: inexpensive\n", want: stringPointer("inexpensive")},
{name: "profile clears named selection", baseValue: "production", overlayName: "testing", overlay: "notarius:\n config_profile: null\n"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
dir := t.TempDir()
rootYAML := "whisperx:\n transcribe_url: https://transcription.example.test/transcribe\nnotarius:\n enabled: false\n"
if test.baseValue != "" {
rootYAML += " config_profile: " + test.baseValue + "\n"
}
root := writePipelineFixture(t, dir, "pipeline.yml", rootYAML)
options := PipelineLoadOptions{}
if test.overlayName != "" {
writePipelineFixture(t, dir, "profiles/"+test.overlayName+".yml", test.overlay)
options.Profile = &test.overlayName
}
cfg, err := LoadPipelineWithOptions(root, options)
if err != nil {
t.Fatal(err)
}
if cfg.Notarius == nil || !reflect.DeepEqual(cfg.Notarius.ConfigProfile, test.want) {
t.Fatalf("config profile = %#v, want %#v", cfg.Notarius.ConfigProfile, test.want)
}
})
}
}
func TestNotariusCredentialRootProvenance(t *testing.T) { func TestNotariusCredentialRootProvenance(t *testing.T) {
const envName = "NARRATIO_NOTARIUS_ROOT_KEY" const envName = "NARRATIO_NOTARIUS_ROOT_KEY"
t.Setenv(envName, "synthetic-notarius-root-key") t.Setenv(envName, "synthetic-notarius-root-key")

View File

@@ -50,20 +50,22 @@ type ArtifactRecord struct {
// StageRecord tracks lifecycle and provenance for one pipeline stage. // StageRecord tracks lifecycle and provenance for one pipeline stage.
type StageRecord struct { type StageRecord struct {
Name string `json:"name"` Name string `json:"name"`
Status StageStatus `json:"status"` Status StageStatus `json:"status"`
CreatedAt time.Time `json:"created_at"` CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"` UpdatedAt time.Time `json:"updated_at"`
StartedAt *time.Time `json:"started_at,omitempty"` StartedAt *time.Time `json:"started_at,omitempty"`
CompletedAt *time.Time `json:"completed_at,omitempty"` CompletedAt *time.Time `json:"completed_at,omitempty"`
Outputs []ArtifactRecord `json:"outputs,omitempty"` Outputs []ArtifactRecord `json:"outputs,omitempty"`
Logs []string `json:"logs,omitempty"` Logs []string `json:"logs,omitempty"`
GeneratedConfigs []string `json:"generated_configs,omitempty"` GeneratedConfigs []string `json:"generated_configs,omitempty"`
Error *ErrorRecord `json:"error,omitempty"` Error *ErrorRecord `json:"error,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"` Metadata map[string]any `json:"metadata,omitempty"`
SemanticConfig *SemanticConfigFingerprint `json:"semantic_config,omitempty"` SemanticConfig *SemanticConfigFingerprint `json:"semantic_config,omitempty"`
AnalyzeStateVersion int `json:"analyze_state_version,omitempty"` TranscribeStateVersion int `json:"transcribe_state_version,omitempty"`
AnalyzeArtifacts map[string]AnalyzeArtifactRecord `json:"analyze_artifacts,omitempty"` TranscribeSpeakers map[string]TranscribeSpeakerRecord `json:"transcribe_speakers,omitempty"`
AnalyzeStateVersion int `json:"analyze_state_version,omitempty"`
AnalyzeArtifacts map[string]AnalyzeArtifactRecord `json:"analyze_artifacts,omitempty"`
} }
// EffectiveConfigProvenance records bounded non-secret configuration identity // EffectiveConfigProvenance records bounded non-secret configuration identity

View File

@@ -119,6 +119,9 @@ func (s *LocalStore) Save(ctx context.Context, path string, m *Manifest) error {
} }
for name, stage := range m.Stages { for name, stage := range m.Stages {
normalizeAnalyzeArtifactCollection(stage.AnalyzeArtifacts) normalizeAnalyzeArtifactCollection(stage.AnalyzeArtifacts)
if err := validateTranscribeStageState(name, stage.TranscribeStateVersion, stage.TranscribeSpeakers); err != nil {
return fmt.Errorf("save manifest: stages.%s: %w", name, err)
}
if err := validateAnalyzeStageState(name, stage.AnalyzeStateVersion, stage.AnalyzeArtifacts); err != nil { if err := validateAnalyzeStageState(name, stage.AnalyzeStateVersion, stage.AnalyzeArtifacts); err != nil {
return fmt.Errorf("save manifest: stages.%s: %w", name, err) return fmt.Errorf("save manifest: stages.%s: %w", name, err)
} }
@@ -160,6 +163,9 @@ func validateLoadedManifest(m *Manifest) error {
return err return err
} }
for name, stage := range m.Stages { for name, stage := range m.Stages {
if err := validateTranscribeStageState(name, stage.TranscribeStateVersion, stage.TranscribeSpeakers); err != nil {
return fmt.Errorf("stages.%s: %w", name, err)
}
if err := validateAnalyzeStageState(name, stage.AnalyzeStateVersion, stage.AnalyzeArtifacts); err != nil { if err := validateAnalyzeStageState(name, stage.AnalyzeStateVersion, stage.AnalyzeArtifacts); err != nil {
return fmt.Errorf("stages.%s: %w", name, err) return fmt.Errorf("stages.%s: %w", name, err)
} }

View File

@@ -0,0 +1,128 @@
package manifest
import (
"fmt"
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
)
const (
// TranscribeStateContractVersion identifies the supported per-speaker
// completion representation owned by the transcribe stage.
TranscribeStateContractVersion = 1
// TranscribeFingerprintContractVersion identifies the fingerprint
// representation stored by the supported transcribe state contract.
TranscribeFingerprintContractVersion = 1
)
// TranscribeSpeakerRecord binds one completed raw transcript to the audio and
// observable recognition settings that produced it.
type TranscribeSpeakerRecord struct {
FingerprintVersion int `json:"fingerprint_version"`
Fingerprint string `json:"fingerprint"`
AudioPath string `json:"audio_path"`
AudioSHA256 string `json:"audio_sha256"`
AudioSize int64 `json:"audio_size"`
Output ArtifactRecord `json:"output"`
OutputSize int64 `json:"output_size"`
ProducerRunID string `json:"producer_run_id"`
}
// HasVersionedTranscribeState reports whether a transcribe stage record carries
// the supported per-speaker authority. An aggregate-only record returns false.
func (s *StageRecord) HasVersionedTranscribeState() bool {
return s != nil && s.Name == "transcribe" && s.TranscribeStateVersion == TranscribeStateContractVersion
}
// ValidateTranscribeSpeakerCollection validates one complete session
// collection independently of its containing manifest.
func ValidateTranscribeSpeakerCollection(version int, records map[string]TranscribeSpeakerRecord) error {
if version != TranscribeStateContractVersion {
return fmt.Errorf("unsupported transcribe state version %d", version)
}
for speakerID, record := range records {
if err := validateTranscribeSpeakerRecord(speakerID, record); err != nil {
return fmt.Errorf("transcribe speaker %q: %w", speakerID, err)
}
}
return nil
}
// CloneTranscribeSpeakerCollection returns a deep copy so checkpoint candidates
// and installed authority cannot share mutable artifact metadata.
func CloneTranscribeSpeakerCollection(records map[string]TranscribeSpeakerRecord) map[string]TranscribeSpeakerRecord {
if records == nil {
return nil
}
cloned := make(map[string]TranscribeSpeakerRecord, len(records))
for speakerID, record := range records {
output := cloneArtifactRecord(&record.Output)
record.Output = *output
cloned[speakerID] = record
}
return cloned
}
func validateTranscribeStageState(stageName string, version int, records map[string]TranscribeSpeakerRecord) error {
if stageName != "transcribe" {
if version != 0 || records != nil {
return fmt.Errorf("stage %q cannot contain transcribe-owned state", stageName)
}
return nil
}
if version == 0 && records == nil {
return nil
}
return ValidateTranscribeSpeakerCollection(version, records)
}
func validateTranscribeSpeakerRecord(speakerID string, record TranscribeSpeakerRecord) error {
if speakerID == "" {
return fmt.Errorf("speaker ID is required")
}
if record.FingerprintVersion != TranscribeFingerprintContractVersion {
return fmt.Errorf("unsupported fingerprint version %d", record.FingerprintVersion)
}
if err := validateSHA256("fingerprint", record.Fingerprint); err != nil {
return err
}
if err := validateCanonicalRelativePath("audio_path", record.AudioPath); err != nil {
return err
}
if err := validateSHA256("audio_sha256", record.AudioSHA256); err != nil {
return err
}
if record.AudioSize < 0 {
return fmt.Errorf("audio_size must not be negative")
}
if record.Output.Kind != "transcript_raw" {
return fmt.Errorf("output kind %q must be transcript_raw", record.Output.Kind)
}
if err := validateCanonicalRelativePath("output local_path", record.Output.LocalPath); err != nil {
return err
}
if err := validateSHA256("output checksum", record.Output.Checksum); err != nil {
return err
}
if record.OutputSize <= 0 {
return fmt.Errorf("output_size must be positive")
}
if err := pathsafe.ValidateOpaqueSegment(record.ProducerRunID); err != nil {
return fmt.Errorf("producer_run_id is invalid: %w", err)
}
if record.Output.ProducerRunID != record.ProducerRunID {
return fmt.Errorf("output producer_run_id %q does not match record", record.Output.ProducerRunID)
}
return nil
}
func validateCanonicalRelativePath(field, value string) error {
normalized, err := pathsafe.NormalizeRelativeDestination(value)
if err != nil {
return fmt.Errorf("%s is unsafe: %w", field, err)
}
if normalized != value {
return fmt.Errorf("%s %q is not canonical %q", field, value, normalized)
}
return nil
}

View File

@@ -0,0 +1,254 @@
package manifest
import (
"context"
"encoding/json"
"path/filepath"
"strings"
"testing"
"time"
)
func TestTranscribeSpeakerStateRoundTripsPopulatedAndVersionedEmptyCollections(t *testing.T) {
now := time.Date(2026, 9, 21, 12, 0, 0, 0, time.UTC)
store := &LocalStore{}
zeroAudio := validTranscribeSpeakerRecord("Alice Table 1")
zeroAudio.AudioSize = 0
for _, test := range []struct {
name string
records map[string]TranscribeSpeakerRecord
}{
{
name: "populated",
records: map[string]TranscribeSpeakerRecord{
"Alice Table 1": validTranscribeSpeakerRecord("Alice Table 1"),
},
},
{name: "versioned empty", records: map[string]TranscribeSpeakerRecord{}},
{name: "zero size audio", records: map[string]TranscribeSpeakerRecord{"Alice Table 1": zeroAudio}},
} {
t.Run(test.name, func(t *testing.T) {
m := New("session", now)
m.Stages["transcribe"] = &StageRecord{
Name: "transcribe", Status: StatusSucceeded, CreatedAt: now, UpdatedAt: now,
TranscribeStateVersion: TranscribeStateContractVersion,
TranscribeSpeakers: test.records,
}
path := filepath.Join(t.TempDir(), "manifest.json")
if err := store.Save(context.Background(), path, m); err != nil {
t.Fatalf("Save() error = %v", err)
}
loaded, err := store.Load(context.Background(), path)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
stage := loaded.Stages["transcribe"]
if !stage.HasVersionedTranscribeState() {
t.Fatalf("transcribe state = %#v, want supported version", stage)
}
if len(stage.TranscribeSpeakers) != len(test.records) {
t.Fatalf("speaker count = %d, want %d", len(stage.TranscribeSpeakers), len(test.records))
}
if len(test.records) != 0 {
got := stage.TranscribeSpeakers["Alice Table 1"]
if got.AudioSize != test.records["Alice Table 1"].AudioSize {
t.Fatalf("audio size = %d, want %d", got.AudioSize, test.records["Alice Table 1"].AudioSize)
}
if got.AudioPath != "inputs/audio/Alice Table 1.flac" || got.Output.LocalPath != "transcripts/raw/Alice Table 1.json" {
t.Fatalf("round-tripped record = %#v", got)
}
}
})
}
}
func TestTranscribeSpeakerStateAllowsLegacyAggregateOnlyManifest(t *testing.T) {
legacy := `{
"schema_version": "narratio.session-manifest.v1",
"session_id": "session",
"created_at": "2026-09-21T12:00:00Z",
"updated_at": "2026-09-21T12:01:00Z",
"stages": {
"transcribe": {
"name": "transcribe",
"status": "succeeded",
"created_at": "2026-09-21T12:00:00Z",
"updated_at": "2026-09-21T12:01:00Z",
"outputs": [{"kind":"transcript_raw","local_path":"transcripts/raw/alice.json"}]
}
}
}`
loaded, err := (&LocalStore{}).LoadReader(context.Background(), strings.NewReader(legacy))
if err != nil {
t.Fatalf("LoadReader() error = %v", err)
}
if loaded.Stages["transcribe"].HasVersionedTranscribeState() {
t.Fatal("legacy aggregate-only manifest unexpectedly gained per-speaker authority")
}
if err := (&LocalStore{}).Save(context.Background(), filepath.Join(t.TempDir(), "manifest.json"), loaded); err != nil {
t.Fatalf("Save(legacy aggregate-only manifest) error = %v", err)
}
}
func TestValidateTranscribeSpeakerCollectionRejectsMalformedEvidence(t *testing.T) {
valid := validTranscribeSpeakerRecord("alice")
tests := []struct {
name string
speakerID string
mutate func(*TranscribeSpeakerRecord)
want string
}{
{name: "empty speaker ID", speakerID: "", want: "speaker ID is required"},
{name: "unsupported fingerprint version", speakerID: "alice", mutate: func(r *TranscribeSpeakerRecord) { r.FingerprintVersion = 2 }, want: "unsupported fingerprint version"},
{name: "bad fingerprint", speakerID: "alice", mutate: func(r *TranscribeSpeakerRecord) { r.Fingerprint = "bad" }, want: "fingerprint must be"},
{name: "unsafe audio path", speakerID: "alice", mutate: func(r *TranscribeSpeakerRecord) { r.AudioPath = "../alice.flac" }, want: "audio_path is unsafe"},
{name: "noncanonical audio path", speakerID: "alice", mutate: func(r *TranscribeSpeakerRecord) { r.AudioPath = "inputs/audio/../alice.flac" }, want: "not canonical"},
{name: "bad audio checksum", speakerID: "alice", mutate: func(r *TranscribeSpeakerRecord) { r.AudioSHA256 = strings.Repeat("G", 64) }, want: "audio_sha256 must be"},
{name: "negative audio size", speakerID: "alice", mutate: func(r *TranscribeSpeakerRecord) { r.AudioSize = -1 }, want: "audio_size"},
{name: "bad output kind", speakerID: "alice", mutate: func(r *TranscribeSpeakerRecord) { r.Output.Kind = "transcript" }, want: "must be transcript_raw"},
{name: "unsafe output path", speakerID: "alice", mutate: func(r *TranscribeSpeakerRecord) { r.Output.LocalPath = "/tmp/alice.json" }, want: "output local_path is unsafe"},
{name: "bad output checksum", speakerID: "alice", mutate: func(r *TranscribeSpeakerRecord) { r.Output.Checksum = "bad" }, want: "output checksum must be"},
{name: "bad output size", speakerID: "alice", mutate: func(r *TranscribeSpeakerRecord) { r.OutputSize = 0 }, want: "output_size must be positive"},
{name: "bad producer run", speakerID: "alice", mutate: func(r *TranscribeSpeakerRecord) { r.ProducerRunID = "bad/run" }, want: "producer_run_id is invalid"},
{name: "producer mismatch", speakerID: "alice", mutate: func(r *TranscribeSpeakerRecord) { r.Output.ProducerRunID = "other-run" }, want: "does not match record"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
record := valid
if test.mutate != nil {
test.mutate(&record)
}
err := ValidateTranscribeSpeakerCollection(TranscribeStateContractVersion, map[string]TranscribeSpeakerRecord{test.speakerID: record})
if err == nil || !strings.Contains(err.Error(), test.want) {
t.Fatalf("error = %v, want %q", err, test.want)
}
})
}
if err := ValidateTranscribeSpeakerCollection(99, nil); err == nil || !strings.Contains(err.Error(), "unsupported transcribe state version") {
t.Fatalf("unsupported version error = %v", err)
}
}
func TestTranscribeSpeakerStateValidationAppliesOnSaveAndLoad(t *testing.T) {
now := time.Date(2026, 9, 21, 12, 0, 0, 0, time.UTC)
invalid := validTranscribeSpeakerRecord("alice")
invalid.AudioSHA256 = "bad"
m := New("session", now)
m.Stages["transcribe"] = &StageRecord{
Name: "transcribe", Status: StatusSucceeded, CreatedAt: now, UpdatedAt: now,
TranscribeStateVersion: TranscribeStateContractVersion,
TranscribeSpeakers: map[string]TranscribeSpeakerRecord{"alice": invalid},
}
store := &LocalStore{}
if err := store.Save(context.Background(), filepath.Join(t.TempDir(), "manifest.json"), m); err == nil || !strings.Contains(err.Error(), "audio_sha256") {
t.Fatalf("Save() error = %v, want audio evidence validation", err)
}
data, err := json.Marshal(m)
if err != nil {
t.Fatalf("Marshal() error = %v", err)
}
if _, err := store.LoadReader(context.Background(), strings.NewReader(string(data))); err == nil || !strings.Contains(err.Error(), "audio_sha256") {
t.Fatalf("LoadReader() error = %v, want audio evidence validation", err)
}
}
func TestTranscribeSpeakerStateOwnershipAndVersionAreValidated(t *testing.T) {
now := time.Date(2026, 9, 21, 12, 0, 0, 0, time.UTC)
store := &LocalStore{}
for _, test := range []struct {
name string
stage *StageRecord
want string
}{
{
name: "wrong owner",
stage: &StageRecord{Name: "merge", Status: StatusSucceeded, CreatedAt: now, UpdatedAt: now,
TranscribeStateVersion: TranscribeStateContractVersion,
TranscribeSpeakers: map[string]TranscribeSpeakerRecord{}},
want: "cannot contain transcribe-owned state",
},
{
name: "missing version",
stage: &StageRecord{Name: "transcribe", Status: StatusSucceeded, CreatedAt: now, UpdatedAt: now,
TranscribeSpeakers: map[string]TranscribeSpeakerRecord{"alice": validTranscribeSpeakerRecord("alice")}},
want: "unsupported transcribe state version 0",
},
{
name: "unsupported version",
stage: &StageRecord{Name: "transcribe", Status: StatusSucceeded, CreatedAt: now, UpdatedAt: now,
TranscribeStateVersion: 2},
want: "unsupported transcribe state version 2",
},
} {
t.Run(test.name, func(t *testing.T) {
m := New("session", now)
m.Stages[test.stage.Name] = test.stage
if err := store.Save(context.Background(), filepath.Join(t.TempDir(), "manifest.json"), m); err == nil || !strings.Contains(err.Error(), test.want) {
t.Fatalf("Save() error = %v, want %q", err, test.want)
}
})
}
}
func TestCloneTranscribeSpeakerCollectionIsIsolated(t *testing.T) {
original := map[string]TranscribeSpeakerRecord{"alice": validTranscribeSpeakerRecord("alice")}
cloned := CloneTranscribeSpeakerCollection(original)
record := cloned["alice"]
record.AudioPath = "inputs/audio/replacement.flac"
record.Output.Checksum = strings.Repeat("c", 64)
cloned["alice"] = record
if got := original["alice"]; got.AudioPath != "inputs/audio/alice.flac" || got.Output.Checksum != strings.Repeat("b", 64) {
t.Fatalf("original changed through clone: %#v", got)
}
}
func TestTranscribeSpeakerStateSurvivesAggregateLifecycleTransitions(t *testing.T) {
now := time.Date(2026, 9, 21, 12, 0, 0, 0, time.UTC)
for _, transition := range []struct {
name string
apply func(*Manifest)
}{
{name: "running", apply: func(m *Manifest) { m.MarkStageRunning("transcribe", now.Add(time.Minute)) }},
{name: "failed", apply: func(m *Manifest) { m.MarkStageFailed("transcribe", now.Add(time.Minute), "request failed") }},
{name: "stale", apply: func(m *Manifest) { m.MarkStageStale("transcribe", now.Add(time.Minute), "audio changed") }},
} {
t.Run(transition.name, func(t *testing.T) {
m := New("session", now)
m.Stages["transcribe"] = &StageRecord{
Name: "transcribe", Status: StatusSucceeded, CreatedAt: now, UpdatedAt: now,
Outputs: []ArtifactRecord{{Kind: "transcript_raw", LocalPath: "transcripts/raw/old.json"}},
Logs: []string{"runs/old/transcribe.log"}, Metadata: map[string]any{"attempts": 1},
TranscribeStateVersion: TranscribeStateContractVersion,
TranscribeSpeakers: map[string]TranscribeSpeakerRecord{"alice": validTranscribeSpeakerRecord("alice")},
}
transition.apply(m)
stage := m.Stages["transcribe"]
if !stage.HasVersionedTranscribeState() || stage.TranscribeSpeakers["alice"].AudioPath != "inputs/audio/alice.flac" {
t.Fatalf("speaker completion was lost during %s: %#v", transition.name, stage)
}
if transition.name != "stale" && (len(stage.Outputs) != 0 || len(stage.Logs) != 0 || len(stage.Metadata) != 0) {
t.Fatalf("aggregate details survived %s: %#v", transition.name, stage)
}
})
}
}
func validTranscribeSpeakerRecord(speakerID string) TranscribeSpeakerRecord {
runID := "run-1"
return TranscribeSpeakerRecord{
FingerprintVersion: TranscribeFingerprintContractVersion,
Fingerprint: strings.Repeat("a", 64),
AudioPath: "inputs/audio/" + speakerID + ".flac",
AudioSHA256: strings.Repeat("1", 64),
AudioSize: 1024,
Output: ArtifactRecord{
Kind: "transcript_raw", LocalPath: "transcripts/raw/" + speakerID + ".json",
Checksum: strings.Repeat("b", 64), ProducerRunID: runID,
},
OutputSize: 256, ProducerRunID: runID,
}
}

View File

@@ -136,28 +136,26 @@ func (state *analyzeExecutionState) checkpointCollection(
return fmt.Errorf("validate analyze artifact collection: %w", err) return fmt.Errorf("validate analyze artifact collection: %w", err)
} }
candidate := *m
candidate.Stages = make(map[string]*manifest.StageRecord, len(m.Stages))
for name, record := range m.Stages {
candidate.Stages[name] = record
}
analyzeRecord := *m.Stages["analyze"]
analyzeRecord.AnalyzeStateVersion = manifest.AnalyzeStateContractVersion
analyzeRecord.AnalyzeArtifacts = manifest.CloneAnalyzeArtifactCollection(records)
candidate.Stages["analyze"] = &analyzeRecord
manifestPath := artifacts.SessionManifestPathForCampaign( manifestPath := artifacts.SessionManifestPathForCampaign(
state.execution.Paths.WorkspaceRoot, state.execution.Paths.WorkspaceRoot,
m.Campaign, m.Campaign,
m.SessionID, m.SessionID,
) )
if err := state.execution.Env.ManifestStore.Save(ctx, manifestPath, &candidate); err != nil { if err := checkpointStageCollection(
return fmt.Errorf("save session manifest %q: %w", manifestPath, err) ctx,
state.execution.Env.ManifestStore,
manifestPath,
m,
"analyze",
func(record *manifest.StageRecord) error {
record.AnalyzeStateVersion = manifest.AnalyzeStateContractVersion
record.AnalyzeArtifacts = manifest.CloneAnalyzeArtifactCollection(records)
return nil
},
); err != nil {
return err
} }
liveRecord := m.Stages["analyze"] liveRecord := m.Stages["analyze"]
liveRecord.AnalyzeStateVersion = candidate.Stages["analyze"].AnalyzeStateVersion
liveRecord.AnalyzeArtifacts = manifest.CloneAnalyzeArtifactCollection(candidate.Stages["analyze"].AnalyzeArtifacts)
m.UpdatedAt = candidate.UpdatedAt
state.records = manifest.CloneAnalyzeArtifactCollection(liveRecord.AnalyzeArtifacts) state.records = manifest.CloneAnalyzeArtifactCollection(liveRecord.AnalyzeArtifacts)
state.committedRecords = manifest.CloneAnalyzeArtifactCollection(liveRecord.AnalyzeArtifacts) state.committedRecords = manifest.CloneAnalyzeArtifactCollection(liveRecord.AnalyzeArtifacts)
return nil return nil

View File

@@ -0,0 +1,99 @@
package stage
import (
"context"
"fmt"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
// checkpointStageCollection isolates one stage-owned collection update from
// live authority until the existing manifest store reports a successful save.
// Callers own serialization and all stage-specific validation and policy.
func checkpointStageCollection(
ctx context.Context,
store manifest.Store,
path string,
live *manifest.Manifest,
stageName string,
update func(*manifest.StageRecord) error,
) error {
if live == nil {
return fmt.Errorf("session manifest is required")
}
if store == nil {
return fmt.Errorf("manifest store is required")
}
if update == nil {
return fmt.Errorf("checkpoint update is required")
}
candidate := cloneManifestForCheckpoint(live)
if candidate.Stages == nil {
candidate.Stages = make(map[string]*manifest.StageRecord)
}
target, exists := candidate.Stages[stageName]
if !exists {
target = &manifest.StageRecord{}
candidate.Stages[stageName] = target
} else if target == nil {
// Retain the invalid nil entry for Store.Save to reject rather than
// silently repairing manifest state. The detached record only gives the
// typed update a safe value to operate on before that validation.
target = &manifest.StageRecord{}
}
if err := update(target); err != nil {
return fmt.Errorf("update stage %q checkpoint: %w", stageName, err)
}
if err := store.Save(ctx, path, candidate); err != nil {
return fmt.Errorf("save session manifest %q: %w", path, err)
}
savedRecord := cloneStageRecordForCheckpoint(candidate.Stages[stageName])
if savedRecord == nil {
return fmt.Errorf("saved stage %q checkpoint record is nil", stageName)
}
if live.Stages != nil {
if current := live.Stages[stageName]; current != nil {
*current = *savedRecord
} else {
live.Stages[stageName] = savedRecord
}
} else {
live.Stages = map[string]*manifest.StageRecord{stageName: savedRecord}
}
live.UpdatedAt = candidate.UpdatedAt
return nil
}
// cloneManifestForCheckpoint copies the bounded mutable state touched by the
// two checkpoint callbacks or by manifest.LocalStore.Save. Metadata maps and
// other pointer fields remain shared read-only because neither boundary mutates
// them.
func cloneManifestForCheckpoint(source *manifest.Manifest) *manifest.Manifest {
clone := *source
clone.Inputs = append([]manifest.InputRecord(nil), source.Inputs...)
clone.Artifacts = append([]manifest.ArtifactRecord(nil), source.Artifacts...)
if source.Stages == nil {
clone.Stages = nil
return &clone
}
clone.Stages = make(map[string]*manifest.StageRecord, len(source.Stages))
for name, sourceRecord := range source.Stages {
clone.Stages[name] = cloneStageRecordForCheckpoint(sourceRecord)
}
return &clone
}
func cloneStageRecordForCheckpoint(source *manifest.StageRecord) *manifest.StageRecord {
if source == nil {
return nil
}
clone := *source
clone.Outputs = append([]manifest.ArtifactRecord(nil), source.Outputs...)
clone.Logs = append([]string(nil), source.Logs...)
clone.GeneratedConfigs = append([]string(nil), source.GeneratedConfigs...)
clone.TranscribeSpeakers = manifest.CloneTranscribeSpeakerCollection(source.TranscribeSpeakers)
clone.AnalyzeArtifacts = manifest.CloneAnalyzeArtifactCollection(source.AnalyzeArtifacts)
return &clone
}

View File

@@ -0,0 +1,327 @@
package stage
import (
"context"
"encoding/json"
"errors"
"path/filepath"
"reflect"
"slices"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
func TestCheckpointStageCollectionSavesThenPromotesTarget(t *testing.T) {
now := time.Date(2026, 9, 22, 12, 0, 0, 0, time.UTC)
live := checkpointTestManifest(now)
originalAnalyze := live.Stages["analyze"]
originalTranscribe := live.Stages["transcribe"]
originalMerge := live.Stages["merge"]
replacement := map[string]manifest.AnalyzeArtifactRecord{
"result": checkpointAnalyzeRecord(now.Add(time.Minute), []string{"zeta", "alpha"}),
}
path := filepath.Join(t.TempDir(), "manifest.json")
base := &manifest.LocalStore{}
store := &checkpointBoundaryStore{
Store: base,
BeforeSave: func(candidate *manifest.Manifest) {
if live.Stages["analyze"] != originalAnalyze || len(originalAnalyze.AnalyzeArtifacts) != 1 {
t.Fatal("live analyze authority changed before save")
}
if candidate.Stages["analyze"] == originalAnalyze || candidate.Stages["transcribe"] == originalTranscribe {
t.Fatal("checkpoint candidate shares stage records with live manifest")
}
},
}
err := checkpointStageCollection(context.Background(), store, path, live, "analyze", func(record *manifest.StageRecord) error {
record.AnalyzeStateVersion = manifest.AnalyzeStateContractVersion
record.AnalyzeArtifacts = replacement
return nil
})
if err != nil {
t.Fatalf("checkpointStageCollection() error = %v", err)
}
if store.Calls != 1 {
t.Fatalf("Save calls = %d, want 1", store.Calls)
}
if live.Stages["analyze"] != originalAnalyze {
t.Fatal("existing target record pointer identity changed")
}
if live.Stages["transcribe"] != originalTranscribe || live.Stages["merge"] != originalMerge {
t.Fatal("unrelated stage record identity changed")
}
if originalAnalyze.Status != manifest.StatusRunning || originalAnalyze.Outputs[0].LocalPath != "artifacts/aggregate.md" {
t.Fatalf("aggregate analyze data changed = %#v", originalAnalyze)
}
if got := originalAnalyze.AnalyzeArtifacts["result"].Dependencies; !slices.Equal(got, []string{"alpha", "zeta"}) {
t.Fatalf("promoted dependencies = %#v, want normalized saved state", got)
}
if !live.UpdatedAt.After(now) {
t.Fatalf("live UpdatedAt = %v, want save timestamp after %v", live.UpdatedAt, now)
}
durable, err := base.Load(context.Background(), path)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if !reflect.DeepEqual(durable.Stages["analyze"], live.Stages["analyze"]) {
t.Fatalf("durable analyze = %#v, live = %#v", durable.Stages["analyze"], live.Stages["analyze"])
}
if !reflect.DeepEqual(durable.Stages["transcribe"], originalTranscribe) || !reflect.DeepEqual(durable.Stages["merge"], originalMerge) {
t.Fatal("unrelated durable stage data changed")
}
changed := replacement["result"]
changed.Status = manifest.AnalyzeArtifactFailed
changed.Dependencies[0] = "changed"
replacement["result"] = changed
if got := live.Stages["analyze"].AnalyzeArtifacts["result"]; got.Status != manifest.AnalyzeArtifactMissing || got.Dependencies[0] != "alpha" {
t.Fatalf("caller edits mutated committed record = %#v", got)
}
}
func TestCheckpointStageCollectionFailureLeavesLiveAuthorityUnchanged(t *testing.T) {
t.Run("update", func(t *testing.T) {
live := checkpointTestManifest(time.Now().UTC())
before := checkpointManifestJSON(t, live)
cause := errors.New("update failed")
store := &checkpointBoundaryStore{Store: &manifest.LocalStore{}}
err := checkpointStageCollection(context.Background(), store, filepath.Join(t.TempDir(), "manifest.json"), live, "analyze", func(record *manifest.StageRecord) error {
record.AnalyzeArtifacts = map[string]manifest.AnalyzeArtifactRecord{}
return cause
})
if !errors.Is(err, cause) {
t.Fatalf("error = %v, want update cause", err)
}
if store.Calls != 0 {
t.Fatalf("Save calls = %d, want 0", store.Calls)
}
assertCheckpointManifestJSON(t, live, before)
})
t.Run("validation", func(t *testing.T) {
live := checkpointTestManifest(time.Now().UTC())
before := checkpointManifestJSON(t, live)
store := &checkpointBoundaryStore{Store: &manifest.LocalStore{}}
path := filepath.Join(t.TempDir(), "manifest.json")
err := checkpointStageCollection(context.Background(), store, path, live, "analyze", func(record *manifest.StageRecord) error {
record.Name = "wrong-stage"
return nil
})
if err == nil {
t.Fatal("error = nil, want manifest validation failure")
}
if store.Calls != 1 {
t.Fatalf("Save calls = %d, want 1", store.Calls)
}
assertCheckpointManifestJSON(t, live, before)
})
t.Run("save after candidate mutation", func(t *testing.T) {
now := time.Date(2026, 9, 22, 12, 0, 0, 0, time.UTC)
live := checkpointTestManifest(now)
analyze := live.Stages["analyze"].AnalyzeArtifacts["result"]
analyze.Dependencies = []string{"zeta", "alpha"}
live.Stages["analyze"].AnalyzeArtifacts["result"] = analyze
before := checkpointManifestJSON(t, live)
cause := errors.New("save failed after mutation")
store := &checkpointBoundaryStore{Store: &manifest.LocalStore{}, ReturnErr: cause}
path := filepath.Join(t.TempDir(), "manifest.json")
replacement := checkpointTranscribeCollection("bob", "run-2")
err := checkpointStageCollection(context.Background(), store, path, live, "transcribe", func(record *manifest.StageRecord) error {
record.TranscribeStateVersion = manifest.TranscribeStateContractVersion
record.TranscribeSpeakers = replacement
return nil
})
if !errors.Is(err, cause) {
t.Fatalf("error = %v, want save cause", err)
}
if !strings.Contains(err.Error(), path) {
t.Fatalf("error = %q, want manifest path context", err)
}
if store.Calls != 1 || store.SavedCandidate == nil {
t.Fatalf("Save calls/candidate = %d/%p, want one observed candidate", store.Calls, store.SavedCandidate)
}
if !store.SavedCandidate.UpdatedAt.After(now) {
t.Fatalf("candidate UpdatedAt = %v, want Store.Save mutation", store.SavedCandidate.UpdatedAt)
}
if got := store.SavedCandidate.Stages["analyze"].AnalyzeArtifacts["result"].Dependencies; !slices.Equal(got, []string{"alpha", "zeta"}) {
t.Fatalf("candidate dependencies = %#v, want save-time normalization", got)
}
if got := live.Stages["analyze"].AnalyzeArtifacts["result"].Dependencies; !slices.Equal(got, []string{"zeta", "alpha"}) {
t.Fatalf("live dependencies = %#v, save-time normalization leaked", got)
}
assertCheckpointManifestJSON(t, live, before)
changed := replacement["bob"]
changed.AudioPath = "audio/changed.flac"
replacement["bob"] = changed
if got := live.Stages["transcribe"].TranscribeSpeakers; len(got) != 1 || got["alice"].AudioPath != "audio/alice.flac" {
t.Fatalf("failed save changed live transcribe collection = %#v", got)
}
})
}
func TestCheckpointStageCollectionMissingTargetAppearsOnlyAfterSuccess(t *testing.T) {
now := time.Date(2026, 9, 22, 12, 0, 0, 0, time.UTC)
live := checkpointTestManifest(now)
delete(live.Stages, "transcribe")
path := filepath.Join(t.TempDir(), "manifest.json")
cause := errors.New("save failed")
failing := &checkpointBoundaryStore{Store: &manifest.LocalStore{}, ReturnErr: cause}
update := func(record *manifest.StageRecord) error {
record.Name = "transcribe"
record.Status = manifest.StatusRunning
record.CreatedAt = now
record.UpdatedAt = now
record.TranscribeStateVersion = manifest.TranscribeStateContractVersion
record.TranscribeSpeakers = checkpointTranscribeCollection("alice", "run-1")
return nil
}
if err := checkpointStageCollection(context.Background(), failing, path, live, "transcribe", update); !errors.Is(err, cause) {
t.Fatalf("failed checkpoint error = %v, want %v", err, cause)
}
if _, exists := live.Stages["transcribe"]; exists {
t.Fatal("missing live target was allocated after failed save")
}
successPath := filepath.Join(t.TempDir(), "manifest.json")
if err := checkpointStageCollection(context.Background(), &manifest.LocalStore{}, successPath, live, "transcribe", update); err != nil {
t.Fatalf("successful checkpoint error = %v", err)
}
created := live.Stages["transcribe"]
if created == nil || created.Name != "transcribe" || created.Status != manifest.StatusRunning {
t.Fatalf("promoted missing target = %#v", created)
}
durable, err := (&manifest.LocalStore{}).Load(context.Background(), successPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if !reflect.DeepEqual(durable.Stages["transcribe"], created) {
t.Fatalf("durable target = %#v, live = %#v", durable.Stages["transcribe"], created)
}
}
func TestCheckpointStageCollectionPreservesContextAndRequiredInputs(t *testing.T) {
now := time.Now().UTC()
live := checkpointTestManifest(now)
path := filepath.Join(t.TempDir(), "manifest.json")
update := func(*manifest.StageRecord) error { return nil }
store := &checkpointBoundaryStore{Store: &manifest.LocalStore{}}
ctx, cancel := context.WithCancel(context.Background())
cancel()
before := checkpointManifestJSON(t, live)
err := checkpointStageCollection(ctx, store, path, live, "analyze", update)
if !errors.Is(err, context.Canceled) {
t.Fatalf("canceled error = %v, want context.Canceled", err)
}
if !strings.Contains(err.Error(), path) {
t.Fatalf("canceled error = %q, want manifest path context", err)
}
assertCheckpointManifestJSON(t, live, before)
if err := checkpointStageCollection(context.Background(), store, path, nil, "analyze", update); err == nil {
t.Fatal("nil live manifest error = nil")
}
if err := checkpointStageCollection(context.Background(), nil, path, live, "analyze", update); err == nil {
t.Fatal("nil store error = nil")
}
if err := checkpointStageCollection(context.Background(), store, path, live, "analyze", nil); err == nil {
t.Fatal("nil update error = nil")
}
}
type checkpointBoundaryStore struct {
manifest.Store
Calls int
BeforeSave func(*manifest.Manifest)
ReturnErr error
SavedCandidate *manifest.Manifest
}
func (store *checkpointBoundaryStore) Save(ctx context.Context, path string, candidate *manifest.Manifest) error {
store.Calls++
if store.BeforeSave != nil {
store.BeforeSave(candidate)
}
if err := store.Store.Save(ctx, path, candidate); err != nil {
return err
}
store.SavedCandidate = candidate
return store.ReturnErr
}
func checkpointTestManifest(now time.Time) *manifest.Manifest {
live := manifest.New("session-1", now)
live.Campaign = "campaign-1"
live.RunID = "run-1"
live.Inputs = []manifest.InputRecord{{Kind: "audio", Path: "audio/alice.flac"}}
live.Artifacts = []manifest.ArtifactRecord{{Kind: "existing", LocalPath: "artifacts/existing.json"}}
live.Stages["analyze"] = &manifest.StageRecord{
Name: "analyze", Status: manifest.StatusRunning, CreatedAt: now, UpdatedAt: now,
Outputs: []manifest.ArtifactRecord{{Kind: "analysis", LocalPath: "artifacts/aggregate.md"}},
Logs: []string{"runs/run-1/analyze.log"},
GeneratedConfigs: []string{"runs/run-1/analyze.yml"},
Metadata: map[string]any{"shared_read_only": true},
AnalyzeStateVersion: manifest.AnalyzeStateContractVersion,
AnalyzeArtifacts: map[string]manifest.AnalyzeArtifactRecord{
"result": checkpointAnalyzeRecord(now, []string{"old_dependency"}),
},
}
live.Stages["transcribe"] = &manifest.StageRecord{
Name: "transcribe", Status: manifest.StatusFailed, CreatedAt: now, UpdatedAt: now,
TranscribeStateVersion: manifest.TranscribeStateContractVersion,
TranscribeSpeakers: checkpointTranscribeCollection("alice", "run-1"),
}
live.Stages["merge"] = &manifest.StageRecord{
Name: "merge", Status: manifest.StatusSucceeded, CreatedAt: now, UpdatedAt: now,
Outputs: []manifest.ArtifactRecord{{Kind: "transcript_base", LocalPath: "transcripts/base.json"}},
}
return live
}
func checkpointAnalyzeRecord(now time.Time, dependencies []string) manifest.AnalyzeArtifactRecord {
return manifest.AnalyzeArtifactRecord{
Key: "result", Status: manifest.AnalyzeArtifactMissing,
Dependencies: dependencies, ProducerRunID: "run-1", UpdatedAt: now,
}
}
func checkpointTranscribeCollection(speaker, runID string) map[string]manifest.TranscribeSpeakerRecord {
return map[string]manifest.TranscribeSpeakerRecord{
speaker: {
FingerprintVersion: manifest.TranscribeFingerprintContractVersion,
Fingerprint: strings.Repeat("b", 64),
AudioPath: "audio/" + speaker + ".flac",
AudioSHA256: strings.Repeat("a", 64),
AudioSize: 0,
Output: manifest.ArtifactRecord{
Kind: "transcript_raw", LocalPath: "transcripts/raw/" + speaker + ".json",
Checksum: strings.Repeat("c", 64), ProducerRunID: runID,
},
OutputSize: 1, ProducerRunID: runID,
},
}
}
func checkpointManifestJSON(t *testing.T, value *manifest.Manifest) []byte {
t.Helper()
data, err := json.Marshal(value)
if err != nil {
t.Fatalf("Marshal() error = %v", err)
}
return data
}
func assertCheckpointManifestJSON(t *testing.T, value *manifest.Manifest, want []byte) {
t.Helper()
if got := checkpointManifestJSON(t, value); !slices.Equal(got, want) {
t.Fatalf("live manifest changed on failure\ngot: %s\nwant: %s", got, want)
}
}

View File

@@ -146,7 +146,8 @@ func (extractStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (re
chunkCache = "refresh" chunkCache = "refresh"
} }
request := notarius.RunRequest{ request := notarius.RunRequest{
Binary: resolvedBinary, LLMAPIKey: notariusConfig.LLMAPIKey, ConfigPath: configPath, PipelineID: notariusConfig.PipelineID, Binary: resolvedBinary, LLMAPIKey: notariusConfig.LLMAPIKey, ConfigProfile: notariusConfig.ConfigProfile,
ConfigPath: configPath, PipelineID: notariusConfig.PipelineID,
InputPath: inputPath, OutputRoot: outputRoot, WorkingDirectory: workingDirectory, InputPath: inputPath, OutputRoot: outputRoot, WorkingDirectory: workingDirectory,
ReceiptPath: receiptPath, LogPath: logPath, Timeout: timeout, Resume: resume, ChunkCache: chunkCache, References: referenceBindings, ReceiptPath: receiptPath, LogPath: logPath, Timeout: timeout, Resume: resume, ChunkCache: chunkCache, References: referenceBindings,
} }
@@ -449,10 +450,11 @@ type fingerprintOutput struct {
} }
type fingerprintDocument struct { type fingerprintDocument struct {
PipelineID string `json:"pipeline_id"` PipelineID string `json:"pipeline_id"`
Input artifacts.ExtractionInputIdentity `json:"input"` ConfigProfile string `json:"config_profile,omitempty"`
References []extractReferenceIdentity `json:"references"` Input artifacts.ExtractionInputIdentity `json:"input"`
Outputs []fingerprintOutput `json:"outputs"` References []extractReferenceIdentity `json:"references"`
Outputs []fingerprintOutput `json:"outputs"`
} }
func extractionFingerprint( func extractionFingerprint(
@@ -484,7 +486,8 @@ func extractionFingerprint(
return sortedReferences[i].Path < sortedReferences[j].Path return sortedReferences[i].Path < sortedReferences[j].Path
}) })
payload, err := json.Marshal(fingerprintDocument{ payload, err := json.Marshal(fingerprintDocument{
PipelineID: cfg.PipelineID, Input: input, References: sortedReferences, Outputs: outputs, PipelineID: cfg.PipelineID, ConfigProfile: notariusConfigProfileReuseIdentity(cfg.ConfigProfile),
Input: input, References: sortedReferences, Outputs: outputs,
}) })
if err != nil { if err != nil {
return "", err return "", err

View File

@@ -2,6 +2,8 @@ package stage
import ( import (
"context" "context"
"crypto/sha256"
"encoding/hex"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
@@ -70,6 +72,8 @@ notarius:
env, m, fake := setupExtractEnv(t) env, m, fake := setupExtractEnv(t)
env.Config.Pipeline.Notarius.LLMAPIKey = loadedPipeline.Notarius.LLMAPIKey env.Config.Pipeline.Notarius.LLMAPIKey = loadedPipeline.Notarius.LLMAPIKey
configProfile := "inexpensive"
env.Config.Pipeline.Notarius.ConfigProfile = &configProfile
result, err := (extractStage{}).Run(context.Background(), env, m) result, err := (extractStage{}).Run(context.Background(), env, m)
if err != nil { if err != nil {
t.Fatalf("Run() error = %v", err) t.Fatalf("Run() error = %v", err)
@@ -102,6 +106,9 @@ notarius:
if req.LLMAPIKey != loadedPipeline.Notarius.LLMAPIKey || req.LLMAPIKey.EnvName() != credentialName || !req.LLMAPIKey.IsResolved() { if req.LLMAPIKey != loadedPipeline.Notarius.LLMAPIKey || req.LLMAPIKey.EnvName() != credentialName || !req.LLMAPIKey.IsResolved() {
t.Fatal("request did not retain the loaded Notarius credential reference") t.Fatal("request did not retain the loaded Notarius credential reference")
} }
if req.ConfigProfile == nil || *req.ConfigProfile != configProfile {
t.Fatalf("request config profile = %#v, want %q", req.ConfigProfile, configProfile)
}
if result.Metadata["reference_count"] != 0 || len(result.Metadata["references"].([]map[string]any)) != 0 { if result.Metadata["reference_count"] != 0 || len(result.Metadata["references"].([]map[string]any)) != 0 {
t.Fatalf("reference metadata = count %#v references %#v", result.Metadata["reference_count"], result.Metadata["references"]) t.Fatalf("reference metadata = count %#v references %#v", result.Metadata["reference_count"], result.Metadata["references"])
} }
@@ -134,6 +141,8 @@ func TestExtractStageAppliesNotariusCacheControlsAndForcePrecedence(t *testing.T
for _, test := range tests { for _, test := range tests {
t.Run(test.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
env, m, fake := setupExtractEnv(t) env, m, fake := setupExtractEnv(t)
configProfile := "inexpensive"
env.Config.Pipeline.Notarius.ConfigProfile = &configProfile
env.Config.Pipeline.Notarius.Resume = test.configuredResume env.Config.Pipeline.Notarius.Resume = test.configuredResume
env.Config.Pipeline.Notarius.ChunkCache = test.configuredMode env.Config.Pipeline.Notarius.ChunkCache = test.configuredMode
env.Force = test.force env.Force = test.force
@@ -148,6 +157,9 @@ func TestExtractStageAppliesNotariusCacheControlsAndForcePrecedence(t *testing.T
if request.Resume != test.wantResume || request.ChunkCache != test.wantMode { if request.Resume != test.wantResume || request.ChunkCache != test.wantMode {
t.Fatalf("cache controls = resume %v, mode %q; want resume %v, mode %q", request.Resume, request.ChunkCache, test.wantResume, test.wantMode) t.Fatalf("cache controls = resume %v, mode %q; want resume %v, mode %q", request.Resume, request.ChunkCache, test.wantResume, test.wantMode)
} }
if request.ConfigProfile == nil || *request.ConfigProfile != configProfile {
t.Fatalf("config profile = %#v, want %q", request.ConfigProfile, configProfile)
}
}) })
} }
} }
@@ -971,6 +983,60 @@ func TestExtractionFingerprintIsIndependentOfOutputMapOrder(t *testing.T) {
} }
} }
func TestExtractionFingerprintBaseSelectorsPreservePriorPayloadShape(t *testing.T) {
input := artifacts.ExtractionInputIdentity{
Checksum: "checksum", SourceID: artifacts.ArtifactTranscriptFinalTrimmed,
ProducerStage: "trim", OutputKind: artifacts.TranscriptOutputKindFinalTrimmed,
ProducerRunID: "trim-run", Provenance: "manifest.trim.outputs",
}
references := []extractReferenceIdentity{{
Selector: "party", SourceID: artifactpolicy.SourceInputParty,
Path: "inputs/party.yml", Checksum: "party-checksum", SizeBytes: 34,
}}
cfg := &config.NotariusConfig{
PipelineID: "pipeline",
Outputs: map[string]config.NotariusOutputConfig{
"alpha": {LaneID: "a", MediaType: "application/json", SchemaID: "a", SchemaVersion: "v1"},
},
}
legacyPayload, err := json.Marshal(struct {
PipelineID string `json:"pipeline_id"`
Input artifacts.ExtractionInputIdentity `json:"input"`
References []extractReferenceIdentity `json:"references"`
Outputs []fingerprintOutput `json:"outputs"`
}{
PipelineID: cfg.PipelineID,
Input: input,
References: references,
Outputs: []fingerprintOutput{{
Key: "alpha", LaneID: "a", MediaType: "application/json", SchemaID: "a", SchemaVersion: "v1",
}},
})
if err != nil {
t.Fatal(err)
}
legacyDigest := sha256.Sum256(legacyPayload)
want := hex.EncodeToString(legacyDigest[:])
for _, profile := range []*string{nil, stringPointerForExtractTest(""), stringPointerForExtractTest("default")} {
cfg.ConfigProfile = profile
got, err := extractionFingerprint(cfg, input, references)
if err != nil {
t.Fatal(err)
}
if got != want {
t.Fatalf("base selection fingerprint = %q, want prior no-profile payload digest %q", got, want)
}
}
cfg.ConfigProfile = stringPointerForExtractTest("inexpensive")
named, err := extractionFingerprint(cfg, input, references)
if err != nil {
t.Fatal(err)
}
if named == want {
t.Fatal("named selection retained prior no-profile fingerprint")
}
}
func TestExtractStageResumeValidationAcceptsCurrentImmutableResult(t *testing.T) { func TestExtractStageResumeValidationAcceptsCurrentImmutableResult(t *testing.T) {
env, m, _ := setupExtractEnv(t) env, m, _ := setupExtractEnv(t)
env.Config.Pipeline.Notarius.References = map[string]string{ env.Config.Pipeline.Notarius.References = map[string]string{
@@ -989,6 +1055,31 @@ func TestExtractStageResumeValidationAcceptsCurrentImmutableResult(t *testing.T)
} }
} }
func TestExtractStageResumeValidationTracksConfigProfile(t *testing.T) {
env, m, _ := setupExtractEnv(t)
selected := "inexpensive"
env.Config.Pipeline.Notarius.ConfigProfile = &selected
seedSucceededExtractResult(t, env, m)
validation, err := (extractStage{}).ValidateResume(context.Background(), env, m)
if err != nil {
t.Fatalf("ValidateResume(same selection) error = %v", err)
}
if !validation.Resumable {
t.Fatalf("same selection validation = %#v, want reusable recorded bundle", validation)
}
changed := "experimental"
env.Config.Pipeline.Notarius.ConfigProfile = &changed
validation, err = (extractStage{}).ValidateResume(context.Background(), env, m)
if err != nil {
t.Fatalf("ValidateResume(changed selection) error = %v", err)
}
if validation.Resumable || !strings.Contains(validation.Reason, "invocation contract changed") {
t.Fatalf("changed selection validation = %#v, want non-resumable invocation evidence", validation)
}
}
func TestExtractStageResumeValidationRejectsChangedPreparedReferenceIdentity(t *testing.T) { func TestExtractStageResumeValidationRejectsChangedPreparedReferenceIdentity(t *testing.T) {
env, m, _ := setupExtractEnv(t) env, m, _ := setupExtractEnv(t)
env.Config.Pipeline.Notarius.References = map[string]string{ env.Config.Pipeline.Notarius.References = map[string]string{
@@ -1277,6 +1368,8 @@ type extractFixture struct {
inputPath string inputPath string
} }
func stringPointerForExtractTest(value string) *string { return &value }
func extractFixtureFromEnv(t *testing.T, env *Env, m *manifest.Manifest) extractFixture { func extractFixtureFromEnv(t *testing.T, env *Env, m *manifest.Manifest) extractFixture {
t.Helper() t.Helper()
paths := sessionPathsForEnv(env, m.SessionID) paths := sessionPathsForEnv(env, m.SessionID)

View File

@@ -306,6 +306,12 @@ func normalizeMergeInputs(
} }
func discoverRawTranscripts(m *manifest.Manifest, paths artifacts.SessionPaths) ([]string, error) { func discoverRawTranscripts(m *manifest.Manifest, paths artifacts.SessionPaths) ([]string, error) {
if record, versioned, err := transcribeManifestState(m); err != nil {
return nil, err
} else if versioned {
return discoverVersionedRawTranscripts(record, paths)
}
fromManifest := make([]string, 0) fromManifest := make([]string, 0)
if m != nil && m.Stages != nil { if m != nil && m.Stages != nil {
if tr := m.Stages["transcribe"]; tr != nil { if tr := m.Stages["transcribe"]; tr != nil {
@@ -350,6 +356,64 @@ func discoverRawTranscripts(m *manifest.Manifest, paths artifacts.SessionPaths)
return dedupeAndSortPaths(out), nil return dedupeAndSortPaths(out), nil
} }
func discoverVersionedRawTranscripts(
record *manifest.StageRecord,
paths artifacts.SessionPaths,
) ([]string, error) {
if record.Status != manifest.StatusSucceeded {
return nil, fmt.Errorf("versioned transcribe state is not aggregate-successful (status %q)", record.Status)
}
if err := manifest.ValidateTranscribeSpeakerCollection(
record.TranscribeStateVersion,
record.TranscribeSpeakers,
); err != nil {
return nil, fmt.Errorf("validate versioned transcribe state: %w", err)
}
if len(record.TranscribeSpeakers) == 0 {
return nil, fmt.Errorf("versioned transcribe state has no completed speakers")
}
expected := make(map[string]string, len(record.TranscribeSpeakers))
for speaker, speakerRecord := range record.TranscribeSpeakers {
path := filepath.Clean(artifacts.ResolveSessionLocalPathForRead(paths, speakerRecord.Output.LocalPath))
if other, exists := expected[path]; exists {
return nil, fmt.Errorf("versioned transcribe speakers %q and %q declare the same output", other, speaker)
}
expected[path] = speaker
}
inputs := make([]string, 0, len(expected))
for _, output := range record.Outputs {
if output.Kind != "transcript_raw" {
continue
}
path := strings.TrimSpace(output.LocalPath)
if path == "" {
return nil, fmt.Errorf("versioned transcribe aggregate output has an empty local path")
}
path = filepath.Clean(artifacts.ResolveSessionLocalPathForRead(paths, path))
speaker, declared := expected[path]
if !declared {
return nil, fmt.Errorf("versioned transcribe aggregate output %q is not declared by current speaker state", output.LocalPath)
}
if speaker == "" {
return nil, fmt.Errorf("versioned transcribe aggregate output %q is duplicated", output.LocalPath)
}
expected[path] = ""
inputs = append(inputs, path)
}
missing := make([]string, 0)
for _, speaker := range expected {
if speaker != "" {
missing = append(missing, speaker)
}
}
if len(missing) > 0 {
sort.Strings(missing)
return nil, fmt.Errorf("versioned transcribe aggregate outputs are incomplete for speakers: %s", strings.Join(missing, ", "))
}
return dedupeAndSortPaths(inputs), nil
}
func dedupeAndSortPaths(paths []string) []string { func dedupeAndSortPaths(paths []string) []string {
if len(paths) == 0 { if len(paths) == 0 {
return nil return nil

View File

@@ -5,6 +5,7 @@ import (
"fmt" "fmt"
"os" "os"
"path/filepath" "path/filepath"
"slices"
"strings" "strings"
"testing" "testing"
"time" "time"
@@ -191,6 +192,70 @@ func TestMergeStageFallsBackToRawDirectoryWhenTranscribeOutputsMissing(t *testin
} }
} }
func TestMergeStageRejectsPartialVersionedTranscribeState(t *testing.T) {
env, m := setupMergeEnv(t)
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsRawDir, "alice.json"), `{"segments":[]}`)
writeFile(t, filepath.Join(paths.TranscriptsRawDir, "uncommitted.json"), `{"segments":[]}`)
writeFile(t, filepath.Join(paths.InputsDir, "speakers.yml"), "match: []\n")
writeFile(t, filepath.Join(paths.InputsDir, "autocorrect.yml"), "rules: []\n")
setVersionedTranscribeMergeState(t, m, manifest.StatusFailed, map[string]string{
"alice": "transcripts/raw/alice.json",
})
fake := &seriatim.FakeRunner{}
env.Seriatim = fake
_, err := (mergeStage{}).Run(context.Background(), env, m)
if err == nil || !strings.Contains(err.Error(), "not aggregate-successful") {
t.Fatalf("merge.Run() error = %v, want incomplete aggregate rejection", err)
}
if len(fake.Requests) != 0 || len(fake.NormalizeRequests) != 0 {
t.Fatalf("adapter requests = %#v / %#v, want none", fake.Requests, fake.NormalizeRequests)
}
}
func TestMergeStageUsesOnlyCurrentVersionedAggregateOutputs(t *testing.T) {
env, m := setupMergeEnv(t)
paths := sessionPathsForEnv(env, m.SessionID)
alice := filepath.Join(paths.TranscriptsRawDir, "alice.json")
writeFile(t, alice, `{"segments":[]}`)
writeFile(t, filepath.Join(paths.TranscriptsRawDir, "removed.json"), `{"segments":[]}`)
writeFile(t, filepath.Join(paths.InputsDir, "speakers.yml"), "match: []\n")
writeFile(t, filepath.Join(paths.InputsDir, "autocorrect.yml"), "rules: []\n")
setVersionedTranscribeMergeState(t, m, manifest.StatusSucceeded, map[string]string{
"alice": "transcripts/raw/alice.json",
})
fake := &seriatim.FakeRunner{}
env.Seriatim = fake
if _, err := (mergeStage{}).Run(context.Background(), env, m); err != nil {
t.Fatalf("merge.Run() error = %v", err)
}
if len(fake.NormalizeRequests) != 1 || fake.NormalizeRequests[0].InputTranscriptPath != alice {
t.Fatalf("normalize requests = %#v, want only current alice", fake.NormalizeRequests)
}
}
func TestMergeStageRejectsIncompleteVersionedAggregateOutputs(t *testing.T) {
env, m := setupMergeEnv(t)
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsRawDir, "alice.json"), `{"segments":[]}`)
writeFile(t, filepath.Join(paths.TranscriptsRawDir, "bob.json"), `{"segments":[]}`)
writeFile(t, filepath.Join(paths.InputsDir, "speakers.yml"), "match: []\n")
writeFile(t, filepath.Join(paths.InputsDir, "autocorrect.yml"), "rules: []\n")
setVersionedTranscribeMergeState(t, m, manifest.StatusSucceeded, map[string]string{
"alice": "transcripts/raw/alice.json",
"bob": "transcripts/raw/bob.json",
})
m.Stages["transcribe"].Outputs = m.Stages["transcribe"].Outputs[:1]
env.Seriatim = &seriatim.FakeRunner{}
_, err := (mergeStage{}).Run(context.Background(), env, m)
if err == nil || !strings.Contains(err.Error(), "incomplete for speakers: bob") {
t.Fatalf("merge.Run() error = %v, want incomplete aggregate output rejection", err)
}
}
func TestMergeStageResolvesWorkspaceQualifiedManifestOutputsWithoutDuplication(t *testing.T) { func TestMergeStageResolvesWorkspaceQualifiedManifestOutputsWithoutDuplication(t *testing.T) {
env, m := setupMergeEnvWithRelativeWorkspaceRoot(t) env, m := setupMergeEnvWithRelativeWorkspaceRoot(t)
paths := sessionPathsForEnv(env, m.SessionID) paths := sessionPathsForEnv(env, m.SessionID)
@@ -414,3 +479,44 @@ func setupMergeEnvWithWorkspace(t *testing.T, workspace string) (*Env, *manifest
ArtifactStore: store, ArtifactStore: store,
}, manifest.New("2026-05-03", time.Now().UTC()) }, manifest.New("2026-05-03", time.Now().UTC())
} }
func setVersionedTranscribeMergeState(
t *testing.T,
m *manifest.Manifest,
status manifest.StageStatus,
outputs map[string]string,
) {
t.Helper()
if m.RunID == "" {
m.RunID = "20260503T010203Z-a1b2c3d4"
}
aggregate := make([]manifest.ArtifactRecord, 0, len(outputs))
records := make(map[string]manifest.TranscribeSpeakerRecord, len(outputs))
for speaker, outputPath := range outputs {
output := manifest.ArtifactRecord{
Kind: "transcript_raw", LocalPath: outputPath,
ProducerRunID: m.RunID, Checksum: strings.Repeat("b", 64),
}
aggregate = append(aggregate, output)
records[speaker] = manifest.TranscribeSpeakerRecord{
FingerprintVersion: manifest.TranscribeFingerprintContractVersion,
Fingerprint: strings.Repeat("c", 64),
AudioPath: "audio/" + speaker + ".flac",
AudioSHA256: strings.Repeat("a", 64),
AudioSize: 1,
Output: output,
OutputSize: 1,
ProducerRunID: m.RunID,
}
}
slices.SortFunc(aggregate, func(a, b manifest.ArtifactRecord) int {
return strings.Compare(a.LocalPath, b.LocalPath)
})
m.MarkStageSucceeded("transcribe", time.Now().UTC(), aggregate)
record := m.Stages["transcribe"]
record.TranscribeStateVersion = manifest.TranscribeStateContractVersion
record.TranscribeSpeakers = records
if status != manifest.StatusSucceeded {
m.MarkStageFailed("transcribe", time.Now().UTC(), "incomplete")
}
}

View File

@@ -20,10 +20,18 @@ const (
) )
type extractSemanticConfig struct { type extractSemanticConfig struct {
Enabled bool `json:"enabled"` Enabled bool `json:"enabled"`
PipelineID string `json:"pipeline_id,omitempty"` PipelineID string `json:"pipeline_id,omitempty"`
References []extractSemanticReference `json:"references"` ConfigProfile string `json:"config_profile,omitempty"`
Outputs []extractSemanticOutput `json:"outputs"` References []extractSemanticReference `json:"references"`
Outputs []extractSemanticOutput `json:"outputs"`
}
func notariusConfigProfileReuseIdentity(profile *string) string {
if profile == nil || *profile == "" || *profile == "default" {
return ""
}
return *profile
} }
type extractSemanticReference struct { type extractSemanticReference struct {
@@ -77,6 +85,7 @@ func (extractStage) SemanticConfigFingerprint(env *Env) (manifest.SemanticConfig
} }
payload.Enabled = true payload.Enabled = true
payload.PipelineID = strings.TrimSpace(cfg.PipelineID) payload.PipelineID = strings.TrimSpace(cfg.PipelineID)
payload.ConfigProfile = notariusConfigProfileReuseIdentity(cfg.ConfigProfile)
for rawSelector, rawSourceID := range cfg.References { for rawSelector, rawSourceID := range cfg.References {
selector, err := notariusref.NormalizeSelector(rawSelector) selector, err := notariusref.NormalizeSelector(rawSelector)
if err != nil { if err != nil {

View File

@@ -9,6 +9,7 @@ import (
configkit "gitea.maximumdirect.net/eric/ConfigKit" configkit "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy" "gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
"gitea.maximumdirect.net/eric/narratio/internal/config" "gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
) )
func testSecretReference(name string) *configkit.Secret { func testSecretReference(name string) *configkit.Secret {
@@ -96,6 +97,74 @@ func TestExtractSemanticConfigSensitivityAndOperationalExclusions(t *testing.T)
} }
}) })
} }
profilePointer := func(value string) *string { return &value }
baseFingerprints := make([]manifest.SemanticConfigFingerprint, 0, 3)
for _, profile := range []*string{nil, profilePointer(""), profilePointer("default")} {
env := downstreamSemanticEnv()
env.Config.Pipeline.Notarius.ConfigProfile = profile
fingerprint, err := (extractStage{}).SemanticConfigFingerprint(env)
if err != nil {
t.Fatal(err)
}
baseFingerprints = append(baseFingerprints, fingerprint)
}
for index := 1; index < len(baseFingerprints); index++ {
if !baseFingerprints[0].Equal(baseFingerprints[index]) {
t.Fatal("nil, empty, and default Notarius selections must share base reuse evidence")
}
}
profiled := downstreamSemanticEnv()
profiled.Config.Pipeline.Notarius.ConfigProfile = profilePointer("inexpensive")
firstProfile, err := (extractStage{}).SemanticConfigFingerprint(profiled)
if err != nil {
t.Fatal(err)
}
if baseFingerprints[0].Equal(firstProfile) {
t.Fatal("base to named Notarius selection retained semantic fingerprint")
}
unchangedProfile, err := (extractStage{}).SemanticConfigFingerprint(profiled)
if err != nil {
t.Fatal(err)
}
if !firstProfile.Equal(unchangedProfile) {
t.Fatal("unchanged Notarius selection changed semantic fingerprint")
}
profiled.Config.Pipeline.Notarius.ConfigProfile = profilePointer("experimental")
secondProfile, err := (extractStage{}).SemanticConfigFingerprint(profiled)
if err != nil {
t.Fatal(err)
}
if firstProfile.Equal(secondProfile) {
t.Fatal("named to different named Notarius selection retained semantic fingerprint")
}
profiled.Config.Pipeline.Notarius.ConfigProfile = nil
returnedToBase, err := (extractStage{}).SemanticConfigFingerprint(profiled)
if err != nil {
t.Fatal(err)
}
if !baseFingerprints[0].Equal(returnedToBase) {
t.Fatal("named to base Notarius selection did not restore base semantic evidence")
}
if secondProfile.Equal(returnedToBase) {
t.Fatal("named to base Notarius selection retained named semantic evidence")
}
disabledBase := downstreamSemanticEnv()
disabledBase.Config.Pipeline.Notarius.Enabled = false
disabledNamed := downstreamSemanticEnv()
disabledNamed.Config.Pipeline.Notarius.Enabled = false
disabledNamed.Config.Pipeline.Notarius.ConfigProfile = profilePointer("ignored")
disabledBaseFingerprint, err := (extractStage{}).SemanticConfigFingerprint(disabledBase)
if err != nil {
t.Fatal(err)
}
disabledNamedFingerprint, err := (extractStage{}).SemanticConfigFingerprint(disabledNamed)
if err != nil {
t.Fatal(err)
}
if !disabledBaseFingerprint.Equal(disabledNamedFingerprint) {
t.Fatal("disabled extraction became sensitive to unused Notarius selection")
}
} }
func TestExtractResumeValidationExcludesOperationalSettings(t *testing.T) { func TestExtractResumeValidationExcludesOperationalSettings(t *testing.T) {

View File

@@ -8,7 +8,6 @@ import (
"path/filepath" "path/filepath"
"sort" "sort"
"strings" "strings"
"sync"
"golang.org/x/sync/errgroup" "golang.org/x/sync/errgroup"
@@ -28,11 +27,14 @@ func (transcribeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest)
if env.ArtifactStore == nil { if env.ArtifactStore == nil {
return nil, fmt.Errorf("transcribe: artifact store is required") return nil, fmt.Errorf("transcribe: artifact store is required")
} }
if env.ManifestStore == nil {
return nil, fmt.Errorf("transcribe: manifest store is required")
}
if env.Config.Pipeline == nil || env.Config.Session == nil { if env.Config.Pipeline == nil || env.Config.Session == nil {
return nil, fmt.Errorf("transcribe: resolved config must include pipeline and session") return nil, fmt.Errorf("transcribe: resolved config must include pipeline and session")
} }
if env.WhisperX == nil { if m == nil {
return nil, fmt.Errorf("transcribe: whisperx adapter is required") return nil, fmt.Errorf("transcribe: session manifest is required")
} }
var sessionID string var sessionID string
@@ -47,59 +49,40 @@ func (transcribeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest)
} }
paths := sessionPathsForEnv(env, sessionID) paths := sessionPathsForEnv(env, sessionID)
runLayout, err := resolveRunStageLayout(env, m, paths, sessionID, "transcribe") reconciliation, err := reconcileTranscribeSpeakers(ctx, env, m)
if err != nil { if err != nil {
return nil, fmt.Errorf("transcribe: resolve run-stage layout: %w", err) return nil, fmt.Errorf("transcribe: reconcile speakers: %w", err)
} }
audioFiles, err := discoverPreparedAudio(m, paths.AudioDir) state := newTranscribeExecutionState(env, m, paths, reconciliation.current)
if err != nil { if err := state.checkpointStartingCollection(ctx); err != nil {
return nil, fmt.Errorf("transcribe: resolve audio inputs: %w", err) return nil, fmt.Errorf("transcribe: checkpoint reconciled speaker collection before execution: %w", err)
} }
if len(audioFiles) == 0 {
return nil, fmt.Errorf("transcribe: no prepared audio files found") jobs := append([]transcribeReconciliationJob(nil), reconciliation.work...)
if len(jobs) > 0 && env.WhisperX == nil {
return nil, fmt.Errorf("transcribe: whisperx adapter is required")
}
if len(jobs) > 0 {
runLayout, err := resolveRunStageLayout(env, m, paths, sessionID, "transcribe")
if err != nil {
return nil, fmt.Errorf("transcribe: resolve run-stage layout: %w", err)
}
for index := range jobs {
jobs[index].runOutputPath, err = runLocalPathForCanonical(runLayout, paths, jobs[index].canonicalOutputPath)
if err != nil {
return nil, fmt.Errorf("transcribe: resolve run-local output path for %q: %w", jobs[index].speakerID, err)
}
}
} }
concurrency := 1 concurrency := 1
if env.Config.Pipeline.WhisperX.Concurrency != nil && *env.Config.Pipeline.WhisperX.Concurrency > 0 { if env.Config.Pipeline.WhisperX.Concurrency != nil && *env.Config.Pipeline.WhisperX.Concurrency > 0 {
concurrency = *env.Config.Pipeline.WhisperX.Concurrency concurrency = *env.Config.Pipeline.WhisperX.Concurrency
} }
if concurrency > len(audioFiles) { if len(jobs) > 0 && concurrency > len(jobs) {
concurrency = len(audioFiles) concurrency = len(jobs)
} }
type job struct {
speakerID string
audioPath string
outPath string
}
jobs := make([]job, 0, len(audioFiles))
seenSpeaker := map[string]string{}
for _, audioPath := range audioFiles {
base := strings.TrimSuffix(filepath.Base(audioPath), filepath.Ext(audioPath))
if base == "" {
return nil, fmt.Errorf("transcribe: could not derive speaker id from %q", audioPath)
}
if prev, ok := seenSpeaker[base]; ok {
return nil, fmt.Errorf("transcribe: duplicate speaker/audio basename %q from %q and %q", base, prev, audioPath)
}
seenSpeaker[base] = audioPath
canonicalOut := filepath.Join(paths.TranscriptsRawDir, base+".json")
runOut, err := runLocalPathForCanonical(runLayout, paths, canonicalOut)
if err != nil {
return nil, fmt.Errorf("transcribe: resolve run-local output path for %q: %w", base, err)
}
jobs = append(jobs, job{
speakerID: base,
audioPath: audioPath,
outPath: runOut,
})
}
var (
mu sync.Mutex
completed int
perFile = map[string]map[string]any{}
)
group, groupCtx := errgroup.WithContext(ctx) group, groupCtx := errgroup.WithContext(ctx)
group.SetLimit(concurrency) group.SetLimit(concurrency)
for _, job := range jobs { for _, job := range jobs {
@@ -108,40 +91,36 @@ func (transcribeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest)
if err := groupCtx.Err(); err != nil { if err := groupCtx.Err(); err != nil {
return err return err
} }
if err := state.checkpointFailure(); err != nil {
return err
}
res, runErr := env.WhisperX.Transcribe(groupCtx, whisperx.TranscribeRequest{ res, runErr := env.WhisperX.Transcribe(groupCtx, whisperx.TranscribeRequest{
SpeakerID: job.speakerID, SpeakerID: job.speakerID,
AudioPath: job.audioPath, AudioPath: job.audioPath,
OutputRawTranscriptPath: job.outPath, OutputRawTranscriptPath: job.runOutputPath,
}) })
if runErr != nil { if runErr != nil {
return fmt.Errorf("speaker %q (%s): %w", job.speakerID, filepath.Base(job.audioPath), runErr) return fmt.Errorf("speaker %q (%s): %w", job.speakerID, filepath.Base(job.audioPath), runErr)
} }
if _, err := authoritativeOutputPath(job.outPath, res.OutputRawTranscriptPath); err != nil {
return fmt.Errorf("speaker %q: %w", job.speakerID, err)
}
if err := validateTranscriptJSONFile(job.outPath); err != nil {
return fmt.Errorf("speaker %q output %q invalid: %w", job.speakerID, job.outPath, err)
}
meta := map[string]any{ meta := map[string]any{
"attempts": res.Attempts, "attempts": res.Attempts,
"http_status": res.HTTPStatus, "http_status": res.HTTPStatus,
"duration_ms": res.Duration.Milliseconds(), "duration_ms": res.Duration.Milliseconds(),
"output_path": job.outPath, "output_path": job.runOutputPath,
} }
mu.Lock() if err := state.acceptSpeaker(ctx, job, res.OutputRawTranscriptPath, meta); err != nil {
perFile[job.speakerID] = meta return fmt.Errorf("speaker %q: %w", job.speakerID, err)
completed++ }
mu.Unlock()
return nil return nil
}) })
} }
groupErr := group.Wait() groupErr := group.Wait()
mu.Lock() if err := state.checkpointFailure(); err != nil {
completedCount := completed return nil, err
mu.Unlock() }
completedCount := state.completedCount()
if err := ctx.Err(); err != nil { if err := ctx.Err(); err != nil {
return nil, fmt.Errorf("transcribe: canceled after completing %d of %d audio jobs: %w", completedCount, len(jobs), err) return nil, fmt.Errorf("transcribe: canceled after completing %d of %d audio jobs: %w", completedCount, len(jobs), err)
} }
@@ -151,35 +130,9 @@ func (transcribeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest)
if completedCount != len(jobs) { if completedCount != len(jobs) {
return nil, fmt.Errorf("transcribe: completed %d of %d planned audio jobs", completedCount, len(jobs)) return nil, fmt.Errorf("transcribe: completed %d of %d planned audio jobs", completedCount, len(jobs))
} }
outputs, outputPaths, runOutputPaths, orderedPerFile, err := state.successDetails()
speakers := make([]string, 0, len(perFile)) if err != nil {
for speaker := range perFile { return nil, fmt.Errorf("transcribe: assemble completed outputs: %w", err)
speakers = append(speakers, speaker)
}
sort.Strings(speakers)
outputs := make([]artifacts.Ref, 0, len(speakers))
runOutputPaths := make([]string, 0, len(speakers))
outputPaths := make([]string, 0, len(speakers))
orderedPerFile := make(map[string]any, len(speakers))
for _, speaker := range speakers {
var runOut string
for _, job := range jobs {
if job.speakerID == speaker {
runOut = job.outPath
break
}
}
runOutputPaths = append(runOutputPaths, runOut)
canonicalOut := filepath.Join(paths.TranscriptsRawDir, speaker+".json")
ref := artifacts.Ref{Kind: "transcript_raw", Category: "transcripts", SessionID: sessionID, AbsolutePath: runOut}
materialized, err := materializeRunLocalOutput(env.ArtifactStore, runOut, canonicalOut, ref)
if err != nil {
return nil, fmt.Errorf("transcribe: materialize %q output: %w", speaker, err)
}
outputs = append(outputs, materialized)
outputPaths = append(outputPaths, canonicalOut)
orderedPerFile[speaker] = perFile[speaker]
} }
retries := 0 retries := 0
@@ -191,7 +144,7 @@ func (transcribeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest)
Outputs: outputs, Outputs: outputs,
Metadata: map[string]any{ Metadata: map[string]any{
"stage": "transcribe", "stage": "transcribe",
"audio_files_count": len(audioFiles), "audio_files_count": len(reconciliation.jobs),
"language": env.Config.Pipeline.WhisperX.Language, "language": env.Config.Pipeline.WhisperX.Language,
"concurrency": concurrency, "concurrency": concurrency,
"retries": retries, "retries": retries,
@@ -204,7 +157,39 @@ func (transcribeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest)
}, nil }, nil
} }
func discoverPreparedAudio(m *manifest.Manifest, audioDir string) ([]string, error) { type transcribeJob struct {
speakerID string
audioPath string
canonicalOutputPath string
runOutputPath string
}
func buildTranscribeJobs(m *manifest.Manifest, paths artifacts.SessionPaths) ([]transcribeJob, error) {
audioFiles, err := discoverPreparedAudio(m, paths)
if err != nil {
return nil, err
}
jobs := make([]transcribeJob, 0, len(audioFiles))
seenSpeaker := map[string]string{}
for _, audioPath := range audioFiles {
base := strings.TrimSuffix(filepath.Base(audioPath), filepath.Ext(audioPath))
if base == "" {
return nil, fmt.Errorf("could not derive speaker id from %q", audioPath)
}
if previous, exists := seenSpeaker[base]; exists {
return nil, fmt.Errorf("duplicate speaker/audio basename %q from %q and %q", base, previous, audioPath)
}
seenSpeaker[base] = audioPath
jobs = append(jobs, transcribeJob{
speakerID: base,
audioPath: audioPath,
canonicalOutputPath: filepath.Join(paths.TranscriptsRawDir, base+".json"),
})
}
return jobs, nil
}
func discoverPreparedAudio(m *manifest.Manifest, paths artifacts.SessionPaths) ([]string, error) {
fromManifest := make([]string, 0) fromManifest := make([]string, 0)
if m != nil { if m != nil {
for _, in := range m.Inputs { for _, in := range m.Inputs {
@@ -215,7 +200,7 @@ func discoverPreparedAudio(m *manifest.Manifest, audioDir string) ([]string, err
if p == "" { if p == "" {
continue continue
} }
fromManifest = append(fromManifest, filepath.Clean(p)) fromManifest = append(fromManifest, artifacts.ResolveSessionLocalPathForRead(paths, p))
} }
} }
if len(fromManifest) > 0 { if len(fromManifest) > 0 {
@@ -226,9 +211,9 @@ func discoverPreparedAudio(m *manifest.Manifest, audioDir string) ([]string, err
return fromManifest, nil return fromManifest, nil
} }
entries, err := os.ReadDir(audioDir) entries, err := os.ReadDir(paths.AudioDir)
if err != nil { if err != nil {
return nil, fmt.Errorf("read audio directory %q: %w", audioDir, err) return nil, fmt.Errorf("read audio directory %q: %w", paths.AudioDir, err)
} }
out := make([]string, 0) out := make([]string, 0)
@@ -236,7 +221,7 @@ func discoverPreparedAudio(m *manifest.Manifest, audioDir string) ([]string, err
if entry.IsDir() { if entry.IsDir() {
continue continue
} }
p := filepath.Join(audioDir, entry.Name()) p := filepath.Join(paths.AudioDir, entry.Name())
if !isFlac(p) { if !isFlac(p) {
continue continue
} }

View File

@@ -0,0 +1,330 @@
package stage
import (
"context"
"errors"
"os"
"slices"
"sync/atomic"
"testing"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/whisperx"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
func TestTranscribeFailureCheckpointsEarlierSpeakersAndRetryRunsOnlyRemaining(t *testing.T) {
env, m := setupTranscribeEnv(t, []string{"a.flac", "b.flac", "c.flac", "d.flac"})
concurrency := 1
env.Config.Pipeline.WhisperX.Concurrency = &concurrency
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
t.Fatalf("prepare.Run() error = %v", err)
}
cause := errors.New("speaker c failed")
env.WhisperX = &whisperx.FakeClient{TranscribeFn: func(
_ context.Context,
req whisperx.TranscribeRequest,
) (whisperx.TranscribeResult, error) {
if req.SpeakerID == "c" {
return whisperx.TranscribeResult{}, cause
}
if req.SpeakerID == "d" {
t.Fatalf("speaker d was dispatched after failure")
}
if err := writeJSONFile(req.OutputRawTranscriptPath, map[string]any{"speaker": req.SpeakerID}); err != nil {
return whisperx.TranscribeResult{}, err
}
return whisperx.TranscribeResult{OutputRawTranscriptPath: req.OutputRawTranscriptPath}, nil
}}
if _, err := (transcribeStage{}).Run(context.Background(), env, m); !errors.Is(err, cause) {
t.Fatalf("Run() error = %v, want %v", err, cause)
}
durable, err := env.ManifestStore.Load(context.Background(), sessionPathsForEnv(env, m.SessionID).ManifestPath)
if err != nil {
t.Fatalf("load durable manifest: %v", err)
}
if got := sortedTranscribeSpeakerKeys(durable.Stages["transcribe"].TranscribeSpeakers); !slices.Equal(got, []string{"a", "b"}) {
t.Fatalf("durable speakers = %#v, want a and b", got)
}
retry := &whisperx.FakeClient{}
env.WhisperX = retry
result, err := (transcribeStage{}).Run(context.Background(), env, m)
if err != nil {
t.Fatalf("retry Run() error = %v", err)
}
requests := retry.RequestsSnapshot()
gotRequests := make([]string, 0, len(requests))
for _, request := range requests {
gotRequests = append(gotRequests, request.SpeakerID)
}
if !slices.Equal(gotRequests, []string{"c", "d"}) {
t.Fatalf("retry speakers = %#v, want c and d", gotRequests)
}
if len(result.Outputs) != 4 {
t.Fatalf("retry outputs = %d, want all four speakers", len(result.Outputs))
}
}
func TestTranscribeConcurrentCheckpointsDoNotLoseCompletions(t *testing.T) {
env, m := setupTranscribeEnv(t, []string{"a.flac", "b.flac", "c.flac", "d.flac"})
concurrency := 4
env.Config.Pipeline.WhisperX.Concurrency = &concurrency
env.WhisperX = &whisperx.FakeClient{}
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
t.Fatalf("prepare.Run() error = %v", err)
}
if _, err := (transcribeStage{}).Run(context.Background(), env, m); err != nil {
t.Fatalf("Run() error = %v", err)
}
durable, err := env.ManifestStore.Load(context.Background(), sessionPathsForEnv(env, m.SessionID).ManifestPath)
if err != nil {
t.Fatalf("load durable manifest: %v", err)
}
if got := sortedTranscribeSpeakerKeys(durable.Stages["transcribe"].TranscribeSpeakers); !slices.Equal(got, []string{"a", "b", "c", "d"}) {
t.Fatalf("durable speakers = %#v, want all completions", got)
}
}
func TestTranscribeStartingCheckpointFailurePreventsRequestsAndCanonicalWrites(t *testing.T) {
env, m := setupTranscribeEnv(t, []string{"alice.flac"})
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
t.Fatalf("prepare.Run() error = %v", err)
}
cause := errors.New("starting checkpoint failed")
base := env.ManifestStore
env.ManifestStore = &failingTranscribeManifestStore{Store: base, FailAt: 1, Err: cause}
fake := &whisperx.FakeClient{}
env.WhisperX = fake
if _, err := (transcribeStage{}).Run(context.Background(), env, m); !errors.Is(err, cause) {
t.Fatalf("Run() error = %v, want %v", err, cause)
}
if got := len(fake.RequestsSnapshot()); got != 0 {
t.Fatalf("adapter requests = %d, want none", got)
}
canonical := sessionPathsForEnv(env, m.SessionID).TranscriptsRawDir + "/alice.json"
if _, err := os.Stat(canonical); !os.IsNotExist(err) {
t.Fatalf("canonical output exists after starting checkpoint failure: %v", err)
}
}
func TestTranscribeCheckpointFailureRetainsEarlierAuthority(t *testing.T) {
env, m := setupTranscribeEnv(t, []string{"a.flac", "b.flac", "c.flac"})
concurrency := 1
env.Config.Pipeline.WhisperX.Concurrency = &concurrency
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
t.Fatalf("prepare.Run() error = %v", err)
}
cause := errors.New("speaker checkpoint failed")
base := env.ManifestStore
env.ManifestStore = &failingTranscribeManifestStore{Store: base, FailAt: 3, Err: cause}
fake := &whisperx.FakeClient{}
env.WhisperX = fake
if _, err := (transcribeStage{}).Run(context.Background(), env, m); !errors.Is(err, cause) {
t.Fatalf("Run() error = %v, want %v", err, cause)
}
requests := fake.RequestsSnapshot()
if len(requests) != 2 {
t.Fatalf("adapter requests = %d, want a and b only", len(requests))
}
durable, err := base.Load(context.Background(), sessionPathsForEnv(env, m.SessionID).ManifestPath)
if err != nil {
t.Fatalf("load durable manifest: %v", err)
}
if got := sortedTranscribeSpeakerKeys(durable.Stages["transcribe"].TranscribeSpeakers); !slices.Equal(got, []string{"a"}) {
t.Fatalf("durable speakers = %#v, want only a", got)
}
if got := sortedTranscribeSpeakerKeys(m.Stages["transcribe"].TranscribeSpeakers); !slices.Equal(got, []string{"a"}) {
t.Fatalf("live speakers = %#v, want last committed collection", got)
}
}
func TestTranscribeFailedForceRetryDoesNotResurrectOldSpeakers(t *testing.T) {
env, m := setupTranscribeEnv(t, []string{"a.flac", "b.flac"})
concurrency := 1
env.Config.Pipeline.WhisperX.Concurrency = &concurrency
env.WhisperX = &whisperx.FakeClient{}
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
t.Fatalf("prepare.Run() error = %v", err)
}
if _, err := (transcribeStage{}).Run(context.Background(), env, m); err != nil {
t.Fatalf("seed Run() error = %v", err)
}
m.RunID = "20260503T020304Z-b2c3d4e5"
env.Force = true
cause := errors.New("forced b failed")
env.WhisperX = &whisperx.FakeClient{TranscribeFn: func(
_ context.Context,
req whisperx.TranscribeRequest,
) (whisperx.TranscribeResult, error) {
if req.SpeakerID == "b" {
return whisperx.TranscribeResult{}, cause
}
if err := writeJSONFile(req.OutputRawTranscriptPath, map[string]any{"replacement": req.SpeakerID}); err != nil {
return whisperx.TranscribeResult{}, err
}
return whisperx.TranscribeResult{OutputRawTranscriptPath: req.OutputRawTranscriptPath}, nil
}}
if _, err := (transcribeStage{}).Run(context.Background(), env, m); !errors.Is(err, cause) {
t.Fatalf("forced Run() error = %v, want %v", err, cause)
}
if got := sortedTranscribeSpeakerKeys(m.Stages["transcribe"].TranscribeSpeakers); !slices.Equal(got, []string{"a"}) {
t.Fatalf("speakers after failed force = %#v, want only replacement a", got)
}
env.Force = false
retry := &whisperx.FakeClient{}
env.WhisperX = retry
if _, err := (transcribeStage{}).Run(context.Background(), env, m); err != nil {
t.Fatalf("retry Run() error = %v", err)
}
requests := retry.RequestsSnapshot()
if len(requests) != 1 || requests[0].SpeakerID != "b" {
t.Fatalf("retry requests = %#v, want only b", requests)
}
}
func TestTranscribeRejectsAudioChangedDuringRequest(t *testing.T) {
env, m := setupTranscribeEnv(t, []string{"alice.flac"})
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
t.Fatalf("prepare.Run() error = %v", err)
}
env.WhisperX = &whisperx.FakeClient{TranscribeFn: func(
_ context.Context,
req whisperx.TranscribeRequest,
) (whisperx.TranscribeResult, error) {
if err := writeJSONFile(req.OutputRawTranscriptPath, map[string]any{"speaker": req.SpeakerID}); err != nil {
return whisperx.TranscribeResult{}, err
}
if err := os.WriteFile(req.AudioPath, []byte("changed"), 0o644); err != nil {
return whisperx.TranscribeResult{}, err
}
return whisperx.TranscribeResult{OutputRawTranscriptPath: req.OutputRawTranscriptPath}, nil
}}
if _, err := (transcribeStage{}).Run(context.Background(), env, m); err == nil {
t.Fatal("Run() error = nil, want changed-audio failure")
}
if got := len(m.Stages["transcribe"].TranscribeSpeakers); got != 0 {
t.Fatalf("checkpointed speakers = %d, want none", got)
}
}
func TestTranscribeResumeValidationUsesSpeakerEvidence(t *testing.T) {
env, m := setupTranscribeEnv(t, []string{"alice.flac"})
env.WhisperX = &whisperx.FakeClient{}
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
t.Fatalf("prepare.Run() error = %v", err)
}
if _, err := (transcribeStage{}).Run(context.Background(), env, m); err != nil {
t.Fatalf("Run() error = %v", err)
}
validation, err := (transcribeStage{}).ValidateResume(context.Background(), env, m)
if err != nil || validation.Resumable {
t.Fatalf("ValidateResume() = %#v, %v, want missing aggregate repaired", validation, err)
}
if len(m.Stages["transcribe"].Outputs) != 0 {
t.Fatal("resume inspection mutated aggregate outputs")
}
m.Stages["transcribe"].Outputs = []manifest.ArtifactRecord{m.Stages["transcribe"].TranscribeSpeakers["alice"].Output}
validation, err = (transcribeStage{}).ValidateResume(context.Background(), env, m)
if err != nil || !validation.Resumable {
t.Fatalf("ValidateResume() = %#v, %v, want resumable", validation, err)
}
if err := os.WriteFile(m.Inputs[0].Path, []byte("changed"), 0o644); err != nil {
t.Fatalf("change audio: %v", err)
}
validation, err = (transcribeStage{}).ValidateResume(context.Background(), env, m)
if err != nil || validation.Resumable {
t.Fatalf("ValidateResume() = %#v, %v, want execution required", validation, err)
}
}
func TestTranscribeRebuildsAggregateFromCurrentSpeakersWithoutRequests(t *testing.T) {
env, m := setupTranscribeEnv(t, []string{"alice.flac", "bob.flac"})
fake := &whisperx.FakeClient{}
env.WhisperX = fake
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
t.Fatalf("prepare.Run() error = %v", err)
}
seed, err := (transcribeStage{}).Run(context.Background(), env, m)
if err != nil {
t.Fatalf("seed Run() error = %v", err)
}
originalRunID := m.RunID
m.MarkStageFailed("transcribe", m.UpdatedAt.Add(1), "aggregate save failed")
m.RunID = "20260503T030405Z-c3d4e5f6"
fake = &whisperx.FakeClient{}
env.WhisperX = fake
result, err := (transcribeStage{}).Run(context.Background(), env, m)
if err != nil {
t.Fatalf("rebuild Run() error = %v", err)
}
if got := len(fake.RequestsSnapshot()); got != 0 {
t.Fatalf("adapter requests = %d, want none", got)
}
if len(result.Outputs) != len(seed.Outputs) {
t.Fatalf("rebuilt outputs = %d, want %d", len(result.Outputs), len(seed.Outputs))
}
for index, output := range result.Outputs {
if output.ProducerRunID != originalRunID {
t.Fatalf("output %d producer = %q, want %q", index, output.ProducerRunID, originalRunID)
}
if output.Checksum != seed.Outputs[index].Checksum {
t.Fatalf("output %d checksum = %q, want %q", index, output.Checksum, seed.Outputs[index].Checksum)
}
}
}
func TestTranscribeReuseDoesNotDependOnRunDirectory(t *testing.T) {
env, m := setupTranscribeEnv(t, []string{"alice.flac"})
env.WhisperX = &whisperx.FakeClient{}
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
t.Fatalf("prepare.Run() error = %v", err)
}
if _, err := (transcribeStage{}).Run(context.Background(), env, m); err != nil {
t.Fatalf("Run() error = %v", err)
}
m.Stages["transcribe"].Outputs = []manifest.ArtifactRecord{m.Stages["transcribe"].TranscribeSpeakers["alice"].Output}
paths := sessionPathsForEnv(env, m.SessionID)
if err := os.RemoveAll(paths.RunsDir); err != nil {
t.Fatalf("remove run directory: %v", err)
}
if _, err := os.Stat(paths.TranscriptsRawDir + "/alice.json"); err != nil {
t.Fatalf("canonical transcript missing after run cleanup: %v", err)
}
validation, err := (transcribeStage{}).ValidateResume(context.Background(), env, m)
if err != nil || !validation.Resumable {
t.Fatalf("ValidateResume() = %#v, %v, want canonical reuse", validation, err)
}
}
type failingTranscribeManifestStore struct {
manifest.Store
Saves atomic.Int64
FailAt int64
Err error
}
func (store *failingTranscribeManifestStore) Save(
ctx context.Context,
path string,
m *manifest.Manifest,
) error {
if store.Saves.Add(1) == store.FailAt {
return store.Err
}
return store.Store.Save(ctx, path, m)
}
func sortedTranscribeSpeakerKeys(records map[string]manifest.TranscribeSpeakerRecord) []string {
keys := make([]string, 0, len(records))
for key := range records {
keys = append(keys, key)
}
slices.Sort(keys)
return keys
}

View File

@@ -0,0 +1,262 @@
package stage
import (
"context"
"fmt"
"reflect"
"sort"
"strings"
"sync"
"time"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
)
type transcribeExecutionState struct {
env *Env
manifest *manifest.Manifest
paths artifacts.SessionPaths
mu sync.Mutex
records map[string]manifest.TranscribeSpeakerRecord
checkpointErr error
completed int
perFile map[string]map[string]any
runOutputs map[string]string
}
func newTranscribeExecutionState(
env *Env,
m *manifest.Manifest,
paths artifacts.SessionPaths,
current map[string]manifest.TranscribeSpeakerRecord,
) *transcribeExecutionState {
return &transcribeExecutionState{
env: env,
manifest: m,
paths: paths,
records: manifest.CloneTranscribeSpeakerCollection(current),
perFile: make(map[string]map[string]any),
runOutputs: make(map[string]string),
}
}
func (state *transcribeExecutionState) checkpointStartingCollection(ctx context.Context) error {
state.mu.Lock()
defer state.mu.Unlock()
if record := transcribeStageRecord(state.manifest); record != nil &&
record.TranscribeStateVersion == manifest.TranscribeStateContractVersion &&
reflect.DeepEqual(record.TranscribeSpeakers, state.records) {
return nil
}
if err := state.checkpointCollectionLocked(ctx, state.records); err != nil {
state.checkpointErr = err
return err
}
return nil
}
func (state *transcribeExecutionState) acceptSpeaker(
ctx context.Context,
job transcribeReconciliationJob,
adapterOutputPath string,
metadata map[string]any,
) error {
if _, err := authoritativeOutputPath(job.runOutputPath, adapterOutputPath); err != nil {
return err
}
if err := validateTranscriptJSONFile(job.runOutputPath); err != nil {
return fmt.Errorf("output %q invalid: %w", job.runOutputPath, err)
}
audioSHA256, audioSize, err := hashRegularFileWithContext(ctx, job.audioPath)
if err != nil {
return fmt.Errorf("recheck audio %q: %w", job.audioPath, err)
}
if audioSHA256 != job.audioSHA256 || audioSize != job.audioSize {
return fmt.Errorf("audio %q changed during transcription", job.audioPath)
}
if err := state.checkpointFailure(); err != nil {
return err
}
producerRunID := strings.TrimSpace(state.manifest.RunID)
ref := artifacts.Ref{
Kind: "transcript_raw",
Category: "transcripts",
SessionID: state.manifest.SessionID,
AbsolutePath: job.runOutputPath,
ProducerRunID: producerRunID,
}
materialized, err := materializeRunLocalOutput(
state.env.ArtifactStore,
job.runOutputPath,
job.canonicalOutputPath,
ref,
)
if err != nil {
return fmt.Errorf("materialize output: %w", err)
}
outputSHA256, outputSize, err := hashRegularFileWithContext(ctx, job.canonicalOutputPath)
if err != nil {
return fmt.Errorf("inspect materialized output %q: %w", job.canonicalOutputPath, err)
}
if materialized.Checksum != outputSHA256 {
return fmt.Errorf("materialized output checksum changed while inspecting")
}
record := manifest.TranscribeSpeakerRecord{
FingerprintVersion: manifest.TranscribeFingerprintContractVersion,
Fingerprint: job.fingerprint,
AudioPath: job.audioRelativePath,
AudioSHA256: job.audioSHA256,
AudioSize: job.audioSize,
Output: manifest.ArtifactRecord{
Kind: "transcript_raw",
LocalPath: job.outputRelativePath,
ArchivePath: materialized.ArchivePath,
ProducerRunID: producerRunID,
Checksum: outputSHA256,
},
OutputSize: outputSize,
ProducerRunID: producerRunID,
}
state.mu.Lock()
defer state.mu.Unlock()
if state.checkpointErr != nil {
return state.checkpointErr
}
candidate := manifest.CloneTranscribeSpeakerCollection(state.records)
candidate[job.speakerID] = record
if err := state.checkpointCollectionLocked(ctx, candidate); err != nil {
state.checkpointErr = fmt.Errorf("transcribe: checkpoint speaker %q: %w", job.speakerID, err)
return state.checkpointErr
}
state.completed++
state.perFile[job.speakerID] = cloneStringAnyMap(metadata)
state.runOutputs[job.speakerID] = job.runOutputPath
return nil
}
func (state *transcribeExecutionState) checkpointCollectionLocked(
ctx context.Context,
records map[string]manifest.TranscribeSpeakerRecord,
) error {
if state.manifest == nil {
return fmt.Errorf("session manifest is required")
}
if state.env == nil || state.env.ManifestStore == nil {
return fmt.Errorf("manifest store is required")
}
if err := manifest.ValidateTranscribeSpeakerCollection(
manifest.TranscribeStateContractVersion,
records,
); err != nil {
return fmt.Errorf("validate transcribe speaker collection: %w", err)
}
missingRecord := transcribeStageRecord(state.manifest) == nil
if err := checkpointStageCollection(
ctx,
state.env.ManifestStore,
state.paths.ManifestPath,
state.manifest,
"transcribe",
func(record *manifest.StageRecord) error {
if missingRecord {
now := time.Now().UTC()
*record = manifest.StageRecord{
Name: "transcribe", Status: manifest.StatusRunning, CreatedAt: now, UpdatedAt: now,
}
}
record.TranscribeStateVersion = manifest.TranscribeStateContractVersion
record.TranscribeSpeakers = manifest.CloneTranscribeSpeakerCollection(records)
record.UpdatedAt = time.Now().UTC()
return nil
},
); err != nil {
return err
}
live := transcribeStageRecord(state.manifest)
state.records = manifest.CloneTranscribeSpeakerCollection(live.TranscribeSpeakers)
return nil
}
func (state *transcribeExecutionState) checkpointFailure() error {
state.mu.Lock()
defer state.mu.Unlock()
return state.checkpointErr
}
func (state *transcribeExecutionState) completedCount() int {
state.mu.Lock()
defer state.mu.Unlock()
return state.completed
}
func (state *transcribeExecutionState) successDetails() (
[]artifacts.Ref,
[]string,
[]string,
map[string]any,
error,
) {
state.mu.Lock()
defer state.mu.Unlock()
speakers := make([]string, 0, len(state.records))
for speaker := range state.records {
speakers = append(speakers, speaker)
}
sort.Strings(speakers)
outputs := make([]artifacts.Ref, 0, len(speakers))
outputPaths := make([]string, 0, len(speakers))
runOutputPaths := make([]string, 0, len(state.runOutputs))
orderedPerFile := make(map[string]any, len(state.perFile))
for _, speaker := range speakers {
record := state.records[speaker]
absolute, err := pathsafe.JoinSlashRelativeUnderRoot(state.paths.Root, record.Output.LocalPath)
if err != nil {
return nil, nil, nil, nil, fmt.Errorf("speaker %q output path: %w", speaker, err)
}
outputs = append(outputs, artifacts.Ref{
Kind: record.Output.Kind,
Category: "transcripts",
SessionID: state.manifest.SessionID,
RelativePath: record.Output.LocalPath,
AbsolutePath: absolute,
ArchivePath: record.Output.ArchivePath,
RemoteKey: record.Output.RemoteKey,
Checksum: record.Output.Checksum,
ProducerRunID: record.ProducerRunID,
})
outputPaths = append(outputPaths, absolute)
if runPath, ok := state.runOutputs[speaker]; ok {
runOutputPaths = append(runOutputPaths, runPath)
}
if metadata, ok := state.perFile[speaker]; ok {
orderedPerFile[speaker] = cloneStringAnyMap(metadata)
}
}
return outputs, outputPaths, runOutputPaths, orderedPerFile, nil
}
func transcribeStageRecord(m *manifest.Manifest) *manifest.StageRecord {
if m == nil || m.Stages == nil {
return nil
}
return m.Stages["transcribe"]
}
func cloneStringAnyMap(source map[string]any) map[string]any {
if source == nil {
return nil
}
clone := make(map[string]any, len(source))
for key, value := range source {
clone[key] = value
}
return clone
}

View File

@@ -0,0 +1,339 @@
package stage
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"os"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
)
type transcribeReconciliationJob struct {
transcribeJob
audioRelativePath string
audioSHA256 string
audioSize int64
outputRelativePath string
fingerprint string
}
type transcribeReconciliation struct {
jobs []transcribeReconciliationJob
current map[string]manifest.TranscribeSpeakerRecord
work []transcribeReconciliationJob
}
type transcribeSpeakerFingerprint struct {
SpeakerID string `json:"speaker_id"`
AudioPath string `json:"audio_path"`
AudioSHA256 string `json:"audio_sha256"`
AudioSize int64 `json:"audio_size"`
SemanticConfigVersion int `json:"semantic_config_version"`
SemanticConfigDigest string `json:"semantic_config_digest"`
}
// reconcileTranscribeSpeakers compares the current prepared audio set with
// manifest-owned per-speaker evidence without mutating either.
func reconcileTranscribeSpeakers(ctx context.Context, env *Env, m *manifest.Manifest) (transcribeReconciliation, error) {
if env == nil || env.Config == nil || env.Config.Pipeline == nil || env.Config.Session == nil {
return transcribeReconciliation{}, fmt.Errorf("transcribe reconciliation requires resolved pipeline and session config")
}
if env.ArtifactStore == nil {
return transcribeReconciliation{}, fmt.Errorf("transcribe reconciliation requires artifact store")
}
if err := contextError(ctx); err != nil {
return transcribeReconciliation{}, err
}
sessionID := strings.TrimSpace(env.Config.Session.SessionID)
if m != nil && strings.TrimSpace(m.SessionID) != "" {
sessionID = strings.TrimSpace(m.SessionID)
}
if sessionID == "" {
return transcribeReconciliation{}, fmt.Errorf("transcribe reconciliation requires session ID")
}
paths := sessionPathsForEnv(env, sessionID)
baseJobs, err := buildTranscribeJobs(m, paths)
if err != nil {
return transcribeReconciliation{}, fmt.Errorf("resolve audio inputs: %w", err)
}
if len(baseJobs) == 0 {
return transcribeReconciliation{}, fmt.Errorf("no prepared audio files found")
}
semantic, err := (transcribeStage{}).SemanticConfigFingerprint(env)
if err != nil {
return transcribeReconciliation{}, err
}
result := transcribeReconciliation{
jobs: make([]transcribeReconciliationJob, 0, len(baseJobs)),
current: map[string]manifest.TranscribeSpeakerRecord{},
work: make([]transcribeReconciliationJob, 0, len(baseJobs)),
}
for _, base := range baseJobs {
candidate, err := inspectTranscribeJob(ctx, paths, base, semantic)
if err != nil {
return transcribeReconciliation{}, fmt.Errorf("speaker %q: %w", base.speakerID, err)
}
result.jobs = append(result.jobs, candidate)
}
stageRecord, versioned, err := transcribeManifestState(m)
if err != nil {
return transcribeReconciliation{}, err
}
if env.Force || !versioned {
result.work = append(result.work, result.jobs...)
return result, nil
}
for _, candidate := range result.jobs {
record, exists := stageRecord.TranscribeSpeakers[candidate.speakerID]
if !exists {
result.work = append(result.work, candidate)
continue
}
if err := manifest.ValidateTranscribeSpeakerCollection(
stageRecord.TranscribeStateVersion,
map[string]manifest.TranscribeSpeakerRecord{candidate.speakerID: record},
); err != nil {
return transcribeReconciliation{}, fmt.Errorf("speaker %q manifest evidence: %w", candidate.speakerID, err)
}
if !sameTranscribeInputEvidence(candidate, record) {
result.work = append(result.work, candidate)
continue
}
current, err := inspectTranscribeOutput(ctx, paths, candidate, record)
if err != nil {
return transcribeReconciliation{}, fmt.Errorf("speaker %q output evidence: %w", candidate.speakerID, err)
}
if !current {
result.work = append(result.work, candidate)
continue
}
result.current[candidate.speakerID] = manifest.CloneTranscribeSpeakerCollection(
map[string]manifest.TranscribeSpeakerRecord{candidate.speakerID: record},
)[candidate.speakerID]
}
return result, nil
}
func inspectTranscribeJob(
ctx context.Context,
paths artifacts.SessionPaths,
job transcribeJob,
semantic manifest.SemanticConfigFingerprint,
) (transcribeReconciliationJob, error) {
audioRelative, err := pathsafe.SlashRelativeFromRoot(paths.Root, job.audioPath)
if err != nil {
return transcribeReconciliationJob{}, fmt.Errorf("derive session-relative audio path: %w", err)
}
outputRelative, err := pathsafe.SlashRelativeFromRoot(paths.Root, job.canonicalOutputPath)
if err != nil {
return transcribeReconciliationJob{}, fmt.Errorf("derive session-relative output path: %w", err)
}
audioSHA256, audioSize, err := hashRegularFileWithContext(ctx, job.audioPath)
if err != nil {
return transcribeReconciliationJob{}, fmt.Errorf("inspect audio %q: %w", job.audioPath, err)
}
fingerprint, err := fingerprintTranscribeSpeaker(transcribeSpeakerFingerprint{
SpeakerID: job.speakerID,
AudioPath: audioRelative,
AudioSHA256: audioSHA256,
AudioSize: audioSize,
SemanticConfigVersion: semantic.Version,
SemanticConfigDigest: semantic.Digest,
})
if err != nil {
return transcribeReconciliationJob{}, err
}
return transcribeReconciliationJob{
transcribeJob: job,
audioRelativePath: audioRelative,
audioSHA256: audioSHA256,
audioSize: audioSize,
outputRelativePath: outputRelative,
fingerprint: fingerprint,
}, nil
}
func fingerprintTranscribeSpeaker(payload transcribeSpeakerFingerprint) (string, error) {
fingerprint, err := FingerprintSemanticConfig(manifest.TranscribeFingerprintContractVersion, payload)
if err != nil {
return "", fmt.Errorf("build per-speaker fingerprint: %w", err)
}
return fingerprint.Digest, nil
}
func transcribeManifestState(m *manifest.Manifest) (*manifest.StageRecord, bool, error) {
if m == nil || m.Stages == nil || m.Stages["transcribe"] == nil {
return nil, false, nil
}
record := m.Stages["transcribe"]
if record.Name != "transcribe" {
return nil, false, fmt.Errorf("transcribe manifest record has stage name %q", record.Name)
}
if record.TranscribeStateVersion == 0 && record.TranscribeSpeakers == nil {
return record, false, nil
}
if record.TranscribeStateVersion != manifest.TranscribeStateContractVersion {
return nil, false, fmt.Errorf("unsupported transcribe state version %d", record.TranscribeStateVersion)
}
return record, true, nil
}
func sameTranscribeInputEvidence(candidate transcribeReconciliationJob, record manifest.TranscribeSpeakerRecord) bool {
return record.FingerprintVersion == manifest.TranscribeFingerprintContractVersion &&
record.Fingerprint == candidate.fingerprint &&
record.AudioPath == candidate.audioRelativePath &&
record.AudioSHA256 == candidate.audioSHA256 &&
record.AudioSize == candidate.audioSize &&
record.Output.LocalPath == candidate.outputRelativePath
}
func inspectTranscribeOutput(
ctx context.Context,
paths artifacts.SessionPaths,
candidate transcribeReconciliationJob,
record manifest.TranscribeSpeakerRecord,
) (bool, error) {
if record.Output.LocalPath != candidate.outputRelativePath {
return false, nil
}
absolute, err := pathsafe.JoinSlashRelativeUnderRoot(paths.Root, record.Output.LocalPath)
if err != nil {
return false, fmt.Errorf("resolve recorded output path: %w", err)
}
exists, err := requireSafeTranscribeOutput(paths.Root, record.Output.LocalPath)
if err != nil {
return false, err
}
if !exists {
return false, nil
}
if err := validateTranscriptJSONFile(absolute); err != nil {
return false, nil
}
checksum, size, err := hashRegularFileWithContext(ctx, absolute)
if err != nil {
return false, fmt.Errorf("verify output: %w", err)
}
return checksum == record.Output.Checksum && size == record.OutputSize, nil
}
func requireSafeTranscribeOutput(rootPath, relativePath string) (bool, error) {
normalized, err := pathsafe.NormalizeRelativeDestination(relativePath)
if err != nil {
return false, fmt.Errorf("recorded output path is unsafe: %w", err)
}
root, err := os.OpenRoot(rootPath)
if err != nil {
return false, fmt.Errorf("open session root: %w", err)
}
defer func() { _ = root.Close() }()
parts := strings.Split(normalized, "/")
current := root
opened := make([]*os.Root, 0, len(parts)-1)
defer func() {
for index := len(opened) - 1; index >= 0; index-- {
_ = opened[index].Close()
}
}()
for _, part := range parts[:len(parts)-1] {
info, err := current.Lstat(part)
if errors.Is(err, os.ErrNotExist) {
return false, nil
}
if err != nil {
return false, fmt.Errorf("inspect output ancestor %q: %w", part, err)
}
if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
return false, fmt.Errorf("output ancestor %q is not a directory without symlinks", part)
}
next, err := current.OpenRoot(part)
if err != nil {
return false, fmt.Errorf("open output ancestor %q: %w", part, err)
}
opened = append(opened, next)
current = next
}
leaf := parts[len(parts)-1]
info, err := current.Lstat(leaf)
if errors.Is(err, os.ErrNotExist) {
return false, nil
}
if err != nil {
return false, fmt.Errorf("inspect output %q: %w", normalized, err)
}
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
return false, fmt.Errorf("output %q is not a regular file without symlinks", normalized)
}
return true, nil
}
func hashRegularFileWithContext(ctx context.Context, path string) (string, int64, error) {
if err := contextError(ctx); err != nil {
return "", 0, err
}
file, err := os.Open(path)
if err != nil {
return "", 0, err
}
info, err := file.Stat()
if err != nil {
_ = file.Close()
return "", 0, err
}
if !info.Mode().IsRegular() {
_ = file.Close()
return "", 0, fmt.Errorf("path is not a regular file")
}
hash := sha256.New()
size, copyErr := io.Copy(hash, &contextReadChecker{ctx: ctx, reader: file})
closeErr := file.Close()
if copyErr != nil {
return "", 0, copyErr
}
if closeErr != nil {
return "", 0, closeErr
}
if err := contextError(ctx); err != nil {
return "", 0, err
}
if size != info.Size() {
return "", 0, fmt.Errorf("file size changed while reading")
}
return hex.EncodeToString(hash.Sum(nil)), size, nil
}
type contextReadChecker struct {
ctx context.Context
reader io.Reader
}
func (r *contextReadChecker) Read(buffer []byte) (int, error) {
if err := contextError(r.ctx); err != nil {
return 0, err
}
return r.reader.Read(buffer)
}
func contextError(ctx context.Context) error {
if ctx == nil {
return nil
}
select {
case <-ctx.Done():
return ctx.Err()
default:
return nil
}
}

View File

@@ -0,0 +1,316 @@
package stage
import (
"context"
"errors"
"os"
"path/filepath"
"reflect"
"slices"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
func TestReconcileTranscribeSpeakersReusesUnchangedCompletedEvidence(t *testing.T) {
env, m := setupCurrentTranscribeReconciliation(t, []string{"bob.flac", "alice.flac"})
reconciled, err := reconcileTranscribeSpeakers(context.Background(), env, m)
if err != nil {
t.Fatalf("reconcileTranscribeSpeakers() error = %v", err)
}
if got := reconciliationSpeakerIDs(reconciled.jobs); !slices.Equal(got, []string{"alice", "bob"}) {
t.Fatalf("jobs = %#v, want sorted speakers", got)
}
if len(reconciled.current) != 2 || len(reconciled.work) != 0 {
t.Fatalf("reconciliation = current %d work %d, want 2/0", len(reconciled.current), len(reconciled.work))
}
// Manifest paths may be session-relative; reconciliation must not resolve
// them against the process working directory.
for index := range m.Inputs {
if m.Inputs[index].Kind == "audio" {
relative, relErr := filepath.Rel(sessionPathsForEnv(env, m.SessionID).Root, m.Inputs[index].Path)
if relErr != nil {
t.Fatal(relErr)
}
m.Inputs[index].Path = filepath.ToSlash(relative)
}
}
reconciled, err = reconcileTranscribeSpeakers(context.Background(), env, m)
if err != nil || len(reconciled.current) != 2 || len(reconciled.work) != 0 {
t.Fatalf("session-relative reconciliation = %#v, error %v", reconciled, err)
}
}
func TestTranscribeResumeAcceptsEquivalentOutputsAndInspectsRemovalReadOnly(t *testing.T) {
env, m := setupCurrentTranscribeReconciliation(t, []string{"alice.flac", "bob.flac"})
record := m.Stages["transcribe"]
record.Status = manifest.StatusSucceeded
// Reverse speaker order and mix the supported local path representations.
bob := record.TranscribeSpeakers["bob"].Output
bob.LocalPath = filepath.Join(sessionPathsForEnv(env, m.SessionID).Root, filepath.FromSlash(bob.LocalPath))
record.Outputs = []manifest.ArtifactRecord{bob, record.TranscribeSpeakers["alice"].Output}
validation, err := (transcribeStage{}).ValidateResume(context.Background(), env, m)
if err != nil || !validation.Resumable {
t.Fatalf("equivalent aggregate: %#v, %v", validation, err)
}
m.Inputs = slices.DeleteFunc(m.Inputs, func(in manifest.InputRecord) bool { return filepath.Base(in.Path) == "bob.flac" })
before := cloneManifestForCheckpoint(m)
validation, err = (transcribeStage{}).ValidateResume(context.Background(), env, m)
if err != nil || validation.Resumable {
t.Fatalf("removed speaker: %#v, %v", validation, err)
}
if !reflect.DeepEqual(m, before) {
t.Fatal("resume inspection mutated manifest")
}
}
func TestReconcileTranscribeSpeakersReplacesOnlyChangedAudio(t *testing.T) {
env, m := setupCurrentTranscribeReconciliation(t, []string{"alice.flac", "bob.flac"})
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.AudioDir, "alice.flac"), "replacement audio")
reconciled, err := reconcileTranscribeSpeakers(context.Background(), env, m)
if err != nil {
t.Fatalf("reconcileTranscribeSpeakers() error = %v", err)
}
if got := reconciliationSpeakerIDs(reconciled.work); !slices.Equal(got, []string{"alice"}) {
t.Fatalf("work = %#v, want alice", got)
}
if _, current := reconciled.current["bob"]; !current || len(reconciled.current) != 1 {
t.Fatalf("current = %#v, want only bob", reconciled.current)
}
}
func TestReconcileTranscribeSpeakersTracksSemanticAndExcludesOperationalSettings(t *testing.T) {
for _, test := range []struct {
name string
mutate func(*Env)
work bool
}{
{name: "service URL", mutate: func(env *Env) { env.Config.Pipeline.WhisperX.TranscribeURL = "https://other.example/transcribe" }, work: true},
{name: "language", mutate: func(env *Env) { env.Config.Pipeline.WhisperX.Language = "fr" }, work: true},
{name: "timeout", mutate: func(env *Env) { env.Config.Pipeline.WhisperX.Timeout = "45m" }},
{name: "retry policy", mutate: func(env *Env) {
retries := 9
env.Config.Pipeline.WhisperX.Retries = &retries
env.Config.Pipeline.WhisperX.RetryDelay = "15s"
}},
{name: "concurrency", mutate: func(env *Env) {
concurrency := 1
env.Config.Pipeline.WhisperX.Concurrency = &concurrency
}},
} {
t.Run(test.name, func(t *testing.T) {
env, m := setupCurrentTranscribeReconciliation(t, []string{"alice.flac"})
test.mutate(env)
reconciled, err := reconcileTranscribeSpeakers(context.Background(), env, m)
if err != nil {
t.Fatalf("reconcileTranscribeSpeakers() error = %v", err)
}
if test.work {
if len(reconciled.work) != 1 || len(reconciled.current) != 0 {
t.Fatalf("semantic change reconciliation = current %d work %d", len(reconciled.current), len(reconciled.work))
}
} else if len(reconciled.work) != 0 || len(reconciled.current) != 1 {
t.Fatalf("operational change reconciliation = current %d work %d", len(reconciled.current), len(reconciled.work))
}
})
}
}
func TestReconcileTranscribeSpeakersHandlesAddedAndRemovedInputs(t *testing.T) {
env, m := setupCurrentTranscribeReconciliation(t, []string{"alice.flac", "bob.flac"})
paths := sessionPathsForEnv(env, m.SessionID)
caraPath := filepath.Join(paths.AudioDir, "cara.flac")
writeFile(t, caraPath, "cara audio")
m.Inputs = append(m.Inputs, manifest.InputRecord{Kind: "audio", Path: caraPath})
reconciled, err := reconcileTranscribeSpeakers(context.Background(), env, m)
if err != nil {
t.Fatalf("reconcile added input error = %v", err)
}
if got := reconciliationSpeakerIDs(reconciled.work); !slices.Equal(got, []string{"cara"}) || len(reconciled.current) != 2 {
t.Fatalf("added input reconciliation = current %#v work %#v", reconciled.current, got)
}
inputs := m.Inputs[:0]
for _, input := range m.Inputs {
if input.Kind != "audio" || filepath.Base(input.Path) != "bob.flac" {
inputs = append(inputs, input)
}
}
m.Inputs = inputs
reconciled, err = reconcileTranscribeSpeakers(context.Background(), env, m)
if err != nil {
t.Fatalf("reconcile removed input error = %v", err)
}
if got := reconciliationSpeakerIDs(reconciled.jobs); !slices.Equal(got, []string{"alice", "cara"}) {
t.Fatalf("jobs after removal = %#v", got)
}
if _, retained := reconciled.current["bob"]; retained {
t.Fatalf("removed bob record retained in current collection: %#v", reconciled.current)
}
}
func TestReconcileTranscribeSpeakersTreatsMissingAndCorruptRegularOutputsAsWork(t *testing.T) {
for _, test := range []struct {
name string
mutate func(*testing.T, string)
}{
{name: "missing", mutate: func(t *testing.T, path string) {
if err := os.Remove(path); err != nil {
t.Fatal(err)
}
}},
{name: "invalid JSON", mutate: func(t *testing.T, path string) { writeFile(t, path, "not-json") }},
{name: "checksum mismatch", mutate: func(t *testing.T, path string) { writeFile(t, path, `{"speaker":"changed"}`) }},
} {
t.Run(test.name, func(t *testing.T) {
env, m := setupCurrentTranscribeReconciliation(t, []string{"alice.flac"})
path := filepath.Join(sessionPathsForEnv(env, m.SessionID).TranscriptsRawDir, "alice.json")
test.mutate(t, path)
reconciled, err := reconcileTranscribeSpeakers(context.Background(), env, m)
if err != nil {
t.Fatalf("reconcileTranscribeSpeakers() error = %v", err)
}
if len(reconciled.current) != 0 || len(reconciled.work) != 1 {
t.Fatalf("reconciliation = current %d work %d, want 0/1", len(reconciled.current), len(reconciled.work))
}
})
}
}
func TestReconcileTranscribeSpeakersRejectsUnsafeEvidenceAndMissingAudio(t *testing.T) {
t.Run("unsafe record path", func(t *testing.T) {
env, m := setupCurrentTranscribeReconciliation(t, []string{"alice.flac"})
record := m.Stages["transcribe"].TranscribeSpeakers["alice"]
record.Output.LocalPath = "../alice.json"
m.Stages["transcribe"].TranscribeSpeakers["alice"] = record
_, err := reconcileTranscribeSpeakers(context.Background(), env, m)
if err == nil || !strings.Contains(err.Error(), "unsafe") {
t.Fatalf("error = %v, want unsafe record path", err)
}
})
t.Run("non-regular output", func(t *testing.T) {
env, m := setupCurrentTranscribeReconciliation(t, []string{"alice.flac"})
path := filepath.Join(sessionPathsForEnv(env, m.SessionID).TranscriptsRawDir, "alice.json")
if err := os.Remove(path); err != nil {
t.Fatal(err)
}
if err := os.Mkdir(path, 0o755); err != nil {
t.Fatal(err)
}
_, err := reconcileTranscribeSpeakers(context.Background(), env, m)
if err == nil || !strings.Contains(err.Error(), "not a regular file") {
t.Fatalf("error = %v, want non-regular safety error", err)
}
})
t.Run("missing audio", func(t *testing.T) {
env, m := setupCurrentTranscribeReconciliation(t, []string{"alice.flac"})
if err := os.Remove(filepath.Join(sessionPathsForEnv(env, m.SessionID).AudioDir, "alice.flac")); err != nil {
t.Fatal(err)
}
_, err := reconcileTranscribeSpeakers(context.Background(), env, m)
if err == nil || !strings.Contains(err.Error(), "not found") {
t.Fatalf("error = %v, want missing audio input error", err)
}
})
}
func TestReconcileTranscribeSpeakersForceAndLegacyStateSelectAllWork(t *testing.T) {
for _, test := range []struct {
name string
mutate func(*Env, *manifest.Manifest)
}{
{name: "force", mutate: func(env *Env, _ *manifest.Manifest) { env.Force = true }},
{name: "legacy aggregate only", mutate: func(_ *Env, m *manifest.Manifest) {
stage := m.Stages["transcribe"]
stage.TranscribeStateVersion = 0
stage.TranscribeSpeakers = nil
}},
{name: "versioned empty", mutate: func(_ *Env, m *manifest.Manifest) {
m.Stages["transcribe"].TranscribeSpeakers = map[string]manifest.TranscribeSpeakerRecord{}
}},
} {
t.Run(test.name, func(t *testing.T) {
env, m := setupCurrentTranscribeReconciliation(t, []string{"alice.flac", "bob.flac"})
test.mutate(env, m)
reconciled, err := reconcileTranscribeSpeakers(context.Background(), env, m)
if err != nil {
t.Fatalf("reconcileTranscribeSpeakers() error = %v", err)
}
if len(reconciled.current) != 0 || len(reconciled.work) != 2 {
t.Fatalf("reconciliation = current %d work %d, want 0/2", len(reconciled.current), len(reconciled.work))
}
})
}
}
func TestReconcileTranscribeSpeakersHonorsCancellationWhileInspectingInputs(t *testing.T) {
env, m := setupCurrentTranscribeReconciliation(t, []string{"alice.flac"})
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, err := reconcileTranscribeSpeakers(ctx, env, m)
if !errors.Is(err, context.Canceled) {
t.Fatalf("error = %v, want context cancellation", err)
}
}
func setupCurrentTranscribeReconciliation(t *testing.T, audioFiles []string) (*Env, *manifest.Manifest) {
t.Helper()
env, m := setupTranscribeEnv(t, audioFiles)
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
t.Fatalf("prepare.Run() error = %v", err)
}
initial, err := reconcileTranscribeSpeakers(context.Background(), env, m)
if err != nil {
t.Fatalf("initial reconciliation error = %v", err)
}
records := make(map[string]manifest.TranscribeSpeakerRecord, len(initial.work))
for _, candidate := range initial.work {
if err := writeJSONFile(candidate.canonicalOutputPath, map[string]any{"speaker": candidate.speakerID}); err != nil {
t.Fatal(err)
}
checksum, err := artifacts.SHA256File(candidate.canonicalOutputPath)
if err != nil {
t.Fatal(err)
}
info, err := os.Stat(candidate.canonicalOutputPath)
if err != nil {
t.Fatal(err)
}
records[candidate.speakerID] = manifest.TranscribeSpeakerRecord{
FingerprintVersion: manifest.TranscribeFingerprintContractVersion,
Fingerprint: candidate.fingerprint,
AudioPath: candidate.audioRelativePath,
AudioSHA256: candidate.audioSHA256,
AudioSize: candidate.audioSize,
Output: manifest.ArtifactRecord{
Kind: "transcript_raw", LocalPath: candidate.outputRelativePath,
Checksum: checksum, ProducerRunID: "run-current",
},
OutputSize: info.Size(), ProducerRunID: "run-current",
}
}
now := time.Date(2026, 9, 21, 12, 0, 0, 0, time.UTC)
m.Stages["transcribe"] = &manifest.StageRecord{
Name: "transcribe", Status: manifest.StatusFailed, CreatedAt: now, UpdatedAt: now,
TranscribeStateVersion: manifest.TranscribeStateContractVersion,
TranscribeSpeakers: records,
}
return env, m
}
func reconciliationSpeakerIDs(jobs []transcribeReconciliationJob) []string {
ids := make([]string, 0, len(jobs))
for _, job := range jobs {
ids = append(ids, job.speakerID)
}
return ids
}

View File

@@ -0,0 +1,60 @@
package stage
import (
"context"
"fmt"
"sort"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
func (transcribeStage) ValidateResume(
ctx context.Context,
env *Env,
m *manifest.Manifest,
) (ResumeValidation, error) {
reconciliation, err := reconcileTranscribeSpeakers(ctx, env, m)
if err != nil {
return ResumeValidation{}, fmt.Errorf("transcribe resume: reconcile speakers: %w", err)
}
if len(reconciliation.work) == 0 {
paths := sessionPathsForEnv(env, m.SessionID)
if !transcribeAggregateCurrent(m.Stages["transcribe"], reconciliation.current, paths) {
return NonResumable("speaker collection or aggregate outputs require reconciliation"), nil
}
return Resumable(), nil
}
speakers := make([]string, 0, len(reconciliation.work))
for _, job := range reconciliation.work {
speakers = append(speakers, job.speakerID)
}
sort.Strings(speakers)
return NonResumable("speaker transcripts require execution: " + strings.Join(speakers, ", ")), nil
}
// No HTTP work does not imply that the aggregate is current. The reconciled
// collection is a subset of the stored records, so its cardinality also detects
// removed speakers. Compare local output evidence independently of list order
// and of the supported absolute/session-relative path representations.
func transcribeAggregateCurrent(record *manifest.StageRecord, current map[string]manifest.TranscribeSpeakerRecord, paths artifacts.SessionPaths) bool {
if len(record.TranscribeSpeakers) != len(current) || len(record.Outputs) != len(current) {
return false
}
expected := make(map[string]manifest.ArtifactRecord, len(current))
for _, speaker := range current {
expected[artifacts.ResolveSessionLocalPathForRead(paths, speaker.Output.LocalPath)] = speaker.Output
}
for _, output := range record.Outputs {
path := artifacts.ResolveSessionLocalPathForRead(paths, output.LocalPath)
want, ok := expected[path]
if !ok || output.Kind != want.Kind || output.Checksum != want.Checksum || output.ProducerRunID != want.ProducerRunID {
return false
}
delete(expected, path)
}
return len(expected) == 0
}
var _ ResumeValidator = transcribeStage{}

View File

@@ -322,7 +322,7 @@ func TestTranscribeStagePreservesDistinctSameBasenameAudio(t *testing.T) {
} }
} }
func TestTranscribeStageCancellationDoesNotMaterializePartialOutputs(t *testing.T) { func TestTranscribeStageCancellationPreservesCheckpointedOutputs(t *testing.T) {
env, m := setupTranscribeEnv(t, []string{"alice.flac", "bob.flac", "cara.flac"}) env, m := setupTranscribeEnv(t, []string{"alice.flac", "bob.flac", "cara.flac"})
m.RunID = "20260810T214607Z-a1b2c3d4" m.RunID = "20260810T214607Z-a1b2c3d4"
fastComplete := make(chan struct{}) fastComplete := make(chan struct{})
@@ -351,6 +351,21 @@ func TestTranscribeStageCancellationDoesNotMaterializePartialOutputs(t *testing.
}() }()
select { select {
case <-fastComplete: case <-fastComplete:
manifestPath := sessionPathsForEnv(env, m.SessionID).ManifestPath
deadline := time.Now().Add(time.Second)
for {
durable, err := env.ManifestStore.Load(context.Background(), manifestPath)
if err == nil {
if _, ok := durable.Stages["transcribe"].TranscribeSpeakers["alice"]; ok {
break
}
}
if time.Now().After(deadline) {
cancel()
t.Fatal("alice completion was not checkpointed before cancellation")
}
time.Sleep(5 * time.Millisecond)
}
cancel() cancel()
case <-time.After(time.Second): case <-time.After(time.Second):
cancel() cancel()
@@ -365,8 +380,15 @@ func TestTranscribeStageCancellationDoesNotMaterializePartialOutputs(t *testing.
} }
canonical := filepath.Join(sessionPathsForEnv(env, m.SessionID).TranscriptsRawDir, "alice.json") canonical := filepath.Join(sessionPathsForEnv(env, m.SessionID).TranscriptsRawDir, "alice.json")
if _, err := os.Stat(canonical); !os.IsNotExist(err) { if _, err := os.Stat(canonical); err != nil {
t.Fatalf("partial canonical output should not exist, stat err = %v", err) t.Fatalf("checkpointed canonical output missing: %v", err)
}
durable, err := env.ManifestStore.Load(context.Background(), sessionPathsForEnv(env, m.SessionID).ManifestPath)
if err != nil {
t.Fatalf("load checkpointed manifest: %v", err)
}
if _, ok := durable.Stages["transcribe"].TranscribeSpeakers["alice"]; !ok {
t.Fatalf("checkpointed speakers = %#v, want alice", durable.Stages["transcribe"].TranscribeSpeakers)
} }
} }
@@ -454,8 +476,13 @@ func setupTranscribeEnv(t *testing.T, audioFiles []string) (*Env, *manifest.Mani
env := &Env{ env := &Env{
Config: cfg, Config: cfg,
ArtifactStore: artifacts.NewLocalStore(workspace), ArtifactStore: artifacts.NewLocalStore(workspace),
ManifestStore: &manifest.LocalStore{},
} }
return env, manifest.New("2026-05-03", time.Now().UTC()) m := manifest.New("2026-05-03", time.Now().UTC())
m.Campaign = "sample-campaign"
m.RunID = "20260503T010203Z-a1b2c3d4"
m.MarkStageRunning("transcribe", time.Now().UTC())
return env, m
} }
func writeJSONFile(path string, payload any) error { func writeJSONFile(path string, payload any) error {