Compare commits
58 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 792cbd0677 | |||
| 5dca34e0e0 | |||
| 54d335c820 | |||
| 96b4e73f53 | |||
| 6e6054f744 | |||
| dc3288d15a | |||
| d872edca94 | |||
| 2ae63dbc2b | |||
| 26aa0cce29 | |||
| 7b0bf99ed1 | |||
| 89ada1b269 | |||
| 3438fe82c9 | |||
| a51a513804 | |||
| 995354f1a5 | |||
| 931c9591cd | |||
| 46096fc4ba | |||
| d3ecd907ee | |||
| 442469b9f5 | |||
| c45e042462 | |||
| 207da1b6ef | |||
| 9a1609435e | |||
| 13b83d7ee2 | |||
| a0312ed034 | |||
| 8ff74e1251 | |||
| 1b480d4ed5 | |||
| e13d87a077 | |||
| 5202f16972 | |||
| 5086596f94 | |||
| f7fc64db10 | |||
| 6b0778b420 | |||
| 5b92b3e790 | |||
| cc43906ae9 | |||
| dcaa2ad9c6 | |||
| 9d68cfcf2f | |||
| 2d346733d6 | |||
| cd75bf6d5a | |||
| e08de98ae6 | |||
| df213bc2ba | |||
| 869e68e834 | |||
| ed4623bc41 | |||
| 3ce198197d | |||
| 0ac8072a9a | |||
| 3ec54b8c72 | |||
| 47b3e5b55b | |||
| 99d1a0b926 | |||
| 884a62e8a6 | |||
| f7a40afc7a | |||
| a4f37ff427 | |||
| 2b97ef3722 | |||
| 63a86c43f4 | |||
| 92432232ef | |||
| d31ba5a410 | |||
| f0225e937c | |||
| 9e40c729e1 | |||
| 0e6ba414b3 | |||
| 10a2e89965 | |||
| f8e3e651e6 | |||
| 1558c9ccae |
@@ -40,7 +40,7 @@ steps:
|
|||||||
set -eu
|
set -eu
|
||||||
output_dir="$(mktemp -d)"
|
output_dir="$(mktemp -d)"
|
||||||
trap 'rm -rf "$output_dir"' EXIT
|
trap 'rm -rf "$output_dir"' EXIT
|
||||||
for target in linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 windows/amd64 windows/arm64; do
|
for target in linux/amd64 linux/arm64 darwin/arm64; do
|
||||||
goos="${target%/*}"
|
goos="${target%/*}"
|
||||||
goarch="${target#*/}"
|
goarch="${target#*/}"
|
||||||
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" go build -o "$output_dir/narratio-$goos-$goarch" ./cmd/narratio
|
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" go build -o "$output_dir/narratio-$goos-$goarch" ./cmd/narratio
|
||||||
|
|||||||
62
docs/cli.md
62
docs/cli.md
@@ -20,7 +20,7 @@ Top-level commands:
|
|||||||
- `publish <session_id>`: force-run publish.
|
- `publish <session_id>`: force-run publish.
|
||||||
- `clean <session_id>` or `clean --all`: remove local work/spool state.
|
- `clean <session_id>` or `clean --all`: remove local work/spool state.
|
||||||
- `session <subcommand>`: session helper commands.
|
- `session <subcommand>`: session helper commands.
|
||||||
- `config <subcommand>`: validate, display, source-trace, or compare resolved pipeline configuration.
|
- `config <subcommand>`: validate, display, source-trace, catalog, or compare resolved pipeline configuration.
|
||||||
|
|
||||||
Session subcommands:
|
Session subcommands:
|
||||||
|
|
||||||
@@ -45,6 +45,7 @@ Most session-aware commands accept:
|
|||||||
- `--session-id <session_id>`
|
- `--session-id <session_id>`
|
||||||
- `--previous-session-id <session_id>`
|
- `--previous-session-id <session_id>`
|
||||||
- `--profile <name>`
|
- `--profile <name>`
|
||||||
|
- `--secrets-dir <directory>`
|
||||||
|
|
||||||
Rules:
|
Rules:
|
||||||
|
|
||||||
@@ -53,10 +54,13 @@ Rules:
|
|||||||
- if both positional `<session_id>` and `--session-id` are provided, values must match.
|
- if both positional `<session_id>` and `--session-id` are provided, values must match.
|
||||||
- `--previous-session-id` is a strict expectation: the selected session file
|
- `--previous-session-id` is a strict expectation: the selected session file
|
||||||
must contain the same `previous_session_id`.
|
must contain the same `previous_session_id`.
|
||||||
- `--profile` selects a declared pipeline profile. It may be supplied once;
|
- `--profile` selects `profiles/<name>.yml` relative to the pipeline root. It
|
||||||
an explicit empty or unknown value fails configuration resolution. When it is
|
may be supplied once; omitted, empty, and the exact name `default` select the
|
||||||
omitted, a declared `default_profile` is used. The same selection applies to
|
base configuration, while an unknown named profile fails. The same selection
|
||||||
all common-flag commands, including `regenerate-artifacts`.
|
applies to all common-flag commands, including `regenerate-artifacts`.
|
||||||
|
- `--secrets-dir` selects ConfigKit credential-file bootstrap explicitly. A
|
||||||
|
relative value uses caller CWD. Without the flag, a present `secrets/`
|
||||||
|
directory beside the resolved root pipeline is used; an absent one is skipped.
|
||||||
- `clean --all` cannot be combined with campaign/session selectors.
|
- `clean --all` cannot be combined with campaign/session selectors.
|
||||||
- notification delivery is currently limited to the configured `noop` mode; see
|
- notification delivery is currently limited to the configured `noop` mode; see
|
||||||
the [configuration reference](./config.md#notifications).
|
the [configuration reference](./config.md#notifications).
|
||||||
@@ -78,19 +82,24 @@ Commands with additional positionals keep their command-specific order:
|
|||||||
|
|
||||||
## Command Reference
|
## Command Reference
|
||||||
|
|
||||||
### `config validate`, `config show`, `config sources`, and `config diff`
|
### `config validate`, `config show`, `config sources`, `config artifacts`, and `config diff`
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio config validate [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>]
|
narratio config validate [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>] [--secrets-dir <directory>]
|
||||||
narratio config show [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>]
|
narratio config show [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>] [--secrets-dir <directory>]
|
||||||
narratio config sources [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>]
|
narratio config sources [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>] [--secrets-dir <directory>]
|
||||||
narratio config diff <left-profile> <right-profile> [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>]
|
narratio config artifacts [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>] [--secrets-dir <directory>]
|
||||||
|
narratio config diff <left-profile> <right-profile> [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--secrets-dir <directory>]
|
||||||
```
|
```
|
||||||
|
|
||||||
These commands resolve the selected profile, defaults, ordinary paths, and—if
|
These commands resolve the selected profile, defaults, ordinary paths, and—if
|
||||||
a campaign is selected—the campaign-owned party. They neither discover or load
|
a campaign is selected—the campaign-owned party. They neither discover or load
|
||||||
a session nor create a workspace, manifest, run, lock, adapter, remote
|
a session nor create a workspace, manifest, run, lock, adapter, or remote
|
||||||
connection, or credential environment.
|
connection. Like every pipeline-loading command, they bootstrap the selected or
|
||||||
|
conventional secrets directory before resolving configuration; `config show`
|
||||||
|
never includes bootstrap reports or secret values in its YAML output.
|
||||||
|
Typed credential references, including `notarius.llm_api_key`, must therefore
|
||||||
|
resolve even though these inspection commands never construct an adapter.
|
||||||
|
|
||||||
Campaign selection is optional for a pipeline without party-driven artifact
|
Campaign selection is optional for a pipeline without party-driven artifact
|
||||||
families. A pipeline with `scriptorium.artifact_families` needs a selected or
|
families. A pipeline with `scriptorium.artifact_families` needs a selected or
|
||||||
@@ -101,7 +110,7 @@ Session, range, force, and artifact-execution flags are not accepted.
|
|||||||
`config validate` writes a concise root-path, selected-profile (or `none`), and
|
`config validate` writes a concise root-path, selected-profile (or `none`), and
|
||||||
effective-digest summary after successful complete validation. `config show`
|
effective-digest summary after successful complete validation. `config show`
|
||||||
writes one deterministic, secret-free YAML document containing defaulted and
|
writes one deterministic, secret-free YAML document containing defaulted and
|
||||||
expanded concrete configuration. It omits composition declarations, artifact
|
expanded concrete configuration. It omits ConfigKit directives, artifact
|
||||||
family declarations, and runtime provenance.
|
family declarations, and runtime provenance.
|
||||||
|
|
||||||
`config sources` reports the same fully validated resolution without printing
|
`config sources` reports the same fully validated resolution without printing
|
||||||
@@ -113,8 +122,19 @@ and generated family ownership. A generated party member has
|
|||||||
one family record and one party record at the same logical path. The output
|
one family record and one party record at the same logical path. The output
|
||||||
never reads or prints secret values.
|
never reads or prints secret values.
|
||||||
|
|
||||||
`config diff` resolves both supplied profile names from one parsed root source
|
`config artifacts` prints the concrete catalog used by analysis, ordered by
|
||||||
set and compares their fully resolved, secret-free effective mappings. It does
|
artifact key. Its tab-separated header is `key`, `source`, `definitions`,
|
||||||
|
`enabled`, `output`, `dependencies`, `family`, `character`, `family_source`, and
|
||||||
|
`party_source`. Definitions and effective dependencies are sorted comma-separated
|
||||||
|
lists, and control characters in fields are escaped. Family-member rows identify
|
||||||
|
their family, character, defining family file, and party file. An empty catalog
|
||||||
|
prints only the header. Ordinary artifacts can be inspected without a campaign;
|
||||||
|
party-driven families retain the campaign requirement. See
|
||||||
|
[Scriptorium Artifact Entries](./config.md#scriptorium-artifact-entries) for the
|
||||||
|
configuration layout.
|
||||||
|
|
||||||
|
`config diff` resolves both supplied profile names independently and compares
|
||||||
|
their fully resolved, secret-free effective mappings. It does
|
||||||
not accept `--profile`; the two positional names must be distinct, declared
|
not accept `--profile`; the two positional names must be distinct, declared
|
||||||
profiles. When party-driven families are present, both profiles must resolve to
|
profiles. When party-driven families are present, both profiles must resolve to
|
||||||
the same selected campaign and party. Use `--campaign-file` if profile-specific
|
the same selected campaign and party. Use `--campaign-file` if profile-specific
|
||||||
@@ -157,6 +177,9 @@ Behavior:
|
|||||||
- rejects unknown endpoints and a `--from` endpoint after `--through`;
|
- rejects unknown endpoints and a `--from` endpoint after `--through`;
|
||||||
- runs `render` before `extract`; an omitted or disabled Notarius
|
- runs `render` before `extract`; an omitted or disabled Notarius
|
||||||
configuration records an explicit `notarius_disabled` self-skip;
|
configuration records an explicit `notarius_disabled` self-skip;
|
||||||
|
- resolves every selected typed credential during the initial configuration
|
||||||
|
load, including a Notarius credential when extract is disabled, outside the
|
||||||
|
selected range, or later skipped as resumable;
|
||||||
- skips already-succeeded stages unless `--force` is set or a stage-specific
|
- skips already-succeeded stages unless `--force` is set or a stage-specific
|
||||||
resume check finds its durable result obsolete;
|
resume check finds its durable result obsolete;
|
||||||
- applies `--force` only to stages in the selected range;
|
- applies `--force` only to stages in the selected range;
|
||||||
@@ -189,6 +212,10 @@ artifacts and any prerequisites required by those targets; without
|
|||||||
never run. Common session/configuration options and repeatable artifact values
|
never run. Common session/configuration options and repeatable artifact values
|
||||||
pass through unchanged.
|
pass through unchanged.
|
||||||
|
|
||||||
|
The forced extraction rebuilds the complete configured Notarius bundle,
|
||||||
|
disables Notarius checkpoint loading, and requests chunk-plan refresh.
|
||||||
|
`--artifacts` narrows only the analysis work; it never selects Notarius lanes.
|
||||||
|
|
||||||
Because the expansion owns `--force`, `--from`, and `--through`, callers cannot
|
Because the expansion owns `--force`, `--from`, and `--through`, callers cannot
|
||||||
supply those options. The shared `run` parser reports them as duplicate
|
supply those options. The shared `run` parser reports them as duplicate
|
||||||
singleton flags. The alias has no private execution options or behavior, and
|
singleton flags. The alias has no private execution options or behavior, and
|
||||||
@@ -231,6 +258,10 @@ Equivalent to:
|
|||||||
narratio run-stage analyze <session_id> --force [...common config flags]
|
narratio run-stage analyze <session_id> --force [...common config flags]
|
||||||
```
|
```
|
||||||
|
|
||||||
|
This selects only `analyze`: it does not invoke Notarius, replace extraction,
|
||||||
|
or remove existing extract outputs. `--artifacts` narrows the forced analysis
|
||||||
|
targets and their required analysis prerequisites.
|
||||||
|
|
||||||
### `publish`
|
### `publish`
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@@ -320,6 +351,7 @@ Options:
|
|||||||
|
|
||||||
- `--config <pipeline.yml>`
|
- `--config <pipeline.yml>`
|
||||||
- `--campaign <id>` or `--campaign-file <campaign.yml>`
|
- `--campaign <id>` or `--campaign-file <campaign.yml>`
|
||||||
|
- `--secrets-dir <directory>`
|
||||||
- `--previous-session-id <id>`
|
- `--previous-session-id <id>`
|
||||||
- `--date <YYYY-MM-DD>`
|
- `--date <YYYY-MM-DD>`
|
||||||
- `--title <text>`
|
- `--title <text>`
|
||||||
|
|||||||
251
docs/config.md
251
docs/config.md
@@ -44,13 +44,16 @@ the durable copied session input.
|
|||||||
|
|
||||||
### Read-only effective pipeline inspection
|
### Read-only effective pipeline inspection
|
||||||
|
|
||||||
`narratio config validate`, `narratio config show`, and `narratio config
|
`narratio config validate`, `narratio config show`, `narratio config sources`,
|
||||||
sources` use the same `--config`, `--campaign`, `--campaign-file`, and
|
and `narratio config artifacts` use the same `--config`, `--campaign`,
|
||||||
|
`--campaign-file`, and
|
||||||
`--profile` selection rules as pipeline commands, but do not select, discover,
|
`--profile` selection rules as pipeline commands, but do not select, discover,
|
||||||
or load a session. They do not read credential values or create runtime state.
|
or load a session. They resolve configured typed credential references as part
|
||||||
|
of the ordinary ConfigKit load but expose only their environment-variable names;
|
||||||
|
they do not create runtime state.
|
||||||
`narratio config diff <left-profile> <right-profile>` uses the same pipeline and
|
`narratio config diff <left-profile> <right-profile>` uses the same pipeline and
|
||||||
campaign selectors, resolves each named profile independently from one parsed
|
campaign selectors, resolves each named profile with an independent ConfigKit
|
||||||
root source set, and does not accept a separate `--profile` flag.
|
load, and does not accept a separate `--profile` flag.
|
||||||
|
|
||||||
Campaign selection is optional only when the resolved pipeline has no
|
Campaign selection is optional only when the resolved pipeline has no
|
||||||
`scriptorium.artifact_families`. When families are declared, Narratio selects a
|
`scriptorium.artifact_families`. When families are declared, Narratio selects a
|
||||||
@@ -59,10 +62,12 @@ then parses the campaign-owned party and expands concrete artifacts and any
|
|||||||
family publish rules before validation. `config validate` prints the resulting
|
family publish rules before validation. `config validate` prints the resulting
|
||||||
root, profile, and effective digest. `config show` emits the normalized
|
root, profile, and effective digest. `config show` emits the normalized
|
||||||
effective pipeline YAML, with defaults and concrete expansion included but
|
effective pipeline YAML, with defaults and concrete expansion included but
|
||||||
composition and family declarations omitted. `config sources` prints a stable
|
ConfigKit directives and family declarations omitted. This review projection is
|
||||||
|
not promised to be a reloadable authoring document. `config sources` prints a stable
|
||||||
source projection instead of effective values: root/import/profile/default
|
source projection instead of effective values: root/import/profile/default
|
||||||
ownership plus campaign/party and generated-family records. Canonical derived
|
ownership plus campaign/party and generated-family records. Canonical derived
|
||||||
players trace to the party. The [CLI reference](cli.md#config-validate-config-show-config-sources-and-config-diff)
|
players trace to the party. `config artifacts` lists the concrete artifact map,
|
||||||
|
effective dependencies, and defining sources. The [CLI reference](cli.md#config-validate-config-show-config-sources-config-artifacts-and-config-diff)
|
||||||
owns command syntax and output conventions.
|
owns command syntax and output conventions.
|
||||||
|
|
||||||
`config diff` compares normalized field values rather than YAML text or source
|
`config diff` compares normalized field values rather than YAML text or source
|
||||||
@@ -76,13 +81,14 @@ difference.
|
|||||||
### Migrating to the maintained bundle
|
### Migrating to the maintained bundle
|
||||||
|
|
||||||
Use the [production/testing bundle](../examples/production-testing/pipeline.yml)
|
Use the [production/testing bundle](../examples/production-testing/pipeline.yml)
|
||||||
as the complete copyable migration reference. Split stable pipeline settings
|
as the complete copyable migration reference. Keep production settings in the
|
||||||
into explicit additive imports, place production/testing differences in one
|
base, split them into explicit additive imports, and place testing differences
|
||||||
selected overlay, and retain a production `default_profile`. Convert campaign
|
in the fixed `profiles/testing.yml` overlay. Put ordinary artifacts and families
|
||||||
rosters to [canonical party input](integrations/party.md), then express
|
in the recommended one-definition files shown by that bundle. Convert campaign
|
||||||
character work as families. Inspect the result
|
rosters to [canonical party input](integrations/party.md), then express character
|
||||||
with `config validate`, `config show`, and `config sources`; use `config diff`
|
work as families. Inspect the result with `config validate`, `config show`,
|
||||||
to review profiles before running a session. Unversioned party documents are
|
`config sources`, and `config artifacts`; use `config diff default testing` to
|
||||||
|
review the overlay before running a session. Unversioned party documents are
|
||||||
rejected.
|
rejected.
|
||||||
|
|
||||||
### Identity segments
|
### Identity segments
|
||||||
@@ -96,20 +102,28 @@ remote state with an unsafe legacy identity must be migrated before use.
|
|||||||
|
|
||||||
## Validation and Merge Rules
|
## Validation and Merge Rules
|
||||||
|
|
||||||
- YAML decode is strict (`KnownFields(true)`) and accepts exactly one document:
|
- ConfigKit performs strict typed YAML decoding and accepts exactly one document;
|
||||||
unknown fields or trailing documents fail load.
|
unknown fields, duplicate keys, and trailing documents fail load.
|
||||||
- A pipeline file may explicitly import additive YAML fragments through the
|
- A pipeline file may explicitly import additive YAML fragments through the
|
||||||
root-only `composition.imports` list. Imported files contribute fields to one
|
root-only `configkit.imports` list. Imported files contribute fields to one
|
||||||
logical pipeline document; they do not override fields supplied by the root
|
logical pipeline document; they do not override fields supplied by the root
|
||||||
or another import.
|
or another import.
|
||||||
- A root pipeline may declare named profiles. Exactly one profile is selected
|
- An omitted profile, an empty profile option, and the exact name `default`
|
||||||
by an option-aware caller or by `composition.default_profile`; a caller's
|
select the base configuration. Any other name selects
|
||||||
explicit selection takes precedence. Declaring profiles without either form
|
`profiles/<name>.yml` relative to the pipeline root.
|
||||||
of selection is an error.
|
- Campaign, session, and canonical-party documents are strict single-source
|
||||||
|
ConfigKit schemas. They do not enable imports, profiles, or `configkit`
|
||||||
|
directives.
|
||||||
- Configured timeout and retry-delay durations must be positive. An omitted
|
- Configured timeout and retry-delay durations must be positive. An omitted
|
||||||
artifact timeout continues to inherit its configured Scriptorium timeout.
|
artifact timeout continues to inherit its configured Scriptorium timeout.
|
||||||
- Session files must be concrete; unresolved `{{ ... }}` placeholders fail load.
|
- Session files must be concrete; unresolved `{{ ... }}` placeholders fail load.
|
||||||
|
- Session and party inputs inspected before typed loading are limited to 8 MiB.
|
||||||
- Pipeline defaults are applied before validation.
|
- Pipeline defaults are applied before validation.
|
||||||
|
- Defaults apply only to absent fields. Explicit empty strings, zero values,
|
||||||
|
`false`, and `null` retain ConfigKit's typed meaning and are then checked by
|
||||||
|
Narratio domain validation; they are not silently replaced by defaults.
|
||||||
|
Explicit null optional stage sections stay absent, while required sections
|
||||||
|
fail validation.
|
||||||
- Campaign and session identities must agree.
|
- Campaign and session identities must agree.
|
||||||
- Required stable files (`speakers_file`, `autocorrect_file`, `glossary_file`,
|
- Required stable files (`speakers_file`, `autocorrect_file`, `glossary_file`,
|
||||||
`party_file`) are campaign-owned; the first three may be overridden by the
|
`party_file`) are campaign-owned; the first three may be overridden by the
|
||||||
@@ -123,22 +137,16 @@ remote state with an unsafe legacy identity must be migrated before use.
|
|||||||
- local (`audio_dir` or `audio_files`), or
|
- local (`audio_dir` or `audio_files`), or
|
||||||
- S3 (`audio_s3.prefix`).
|
- S3 (`audio_s3.prefix`).
|
||||||
|
|
||||||
### Pipeline composition
|
### Pipeline imports and profiles
|
||||||
|
|
||||||
Large pipeline configurations may be split into explicitly named fragments and
|
Large pipeline configurations may be split into explicitly named fragments and
|
||||||
may declare one overlay per selectable profile:
|
may use fixed-path profile overlays:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
composition:
|
configkit:
|
||||||
imports:
|
imports:
|
||||||
- config/storage.yml
|
- path: config/storage.yml
|
||||||
- config/integrations.yaml
|
- path: config/integrations.yaml
|
||||||
default_profile: production
|
|
||||||
profiles:
|
|
||||||
production:
|
|
||||||
overlay: profiles/production.yml
|
|
||||||
testing:
|
|
||||||
overlay: profiles/testing.yml
|
|
||||||
|
|
||||||
campaigns:
|
campaigns:
|
||||||
root: /usr/local/share/narratio/campaigns
|
root: /usr/local/share/narratio/campaigns
|
||||||
@@ -152,36 +160,29 @@ Imports are resolved relative to the directory containing the root pipeline
|
|||||||
file and are loaded in declaration order. Narratio does not scan directories or
|
file and are loaded in declaration order. Narratio does not scan directories or
|
||||||
infer fragments. Each import must be a confined regular `.yml` or `.yaml` file:
|
infer fragments. Each import must be a confined regular `.yml` or `.yaml` file:
|
||||||
absolute paths, traversal, symlinks, directories, duplicate files, and an
|
absolute paths, traversal, symlinks, directories, duplicate files, and an
|
||||||
import of the root pipeline itself are rejected. Only the root pipeline may
|
import of the root pipeline itself are rejected. Imported files cannot declare
|
||||||
contain `composition`; nested composition is rejected.
|
another `configkit` control section.
|
||||||
|
|
||||||
Composition is additive. A map may be extended by multiple files when every
|
Base merging is additive. A map may be extended by multiple files when every
|
||||||
leaf is distinct, but a scalar, list, or map/list/scalar kind cannot be claimed
|
leaf is distinct, but a scalar, list, or map/list/scalar kind cannot be claimed
|
||||||
more than once, even when the repeated values are identical. Conflict errors
|
more than once, even when the repeated values are identical. Conflict errors
|
||||||
name the full field path and every source that claimed it. The assembled YAML is
|
name the full field path and every source that claimed it. Narratio additionally
|
||||||
then decoded against the normal strict pipeline schema and defaults are applied
|
rejects an artifact or artifact-family key declared by more than one base file,
|
||||||
once.
|
even when those files contribute disjoint children.
|
||||||
|
|
||||||
Profile names are case-sensitive, non-empty, trimmed, and cannot contain
|
Profile names are case-sensitive and use ConfigKit's safe fixed-path naming
|
||||||
control characters. If `profiles` is present, it must contain at least one
|
rules. Only the selected overlay is opened; a malformed unselected overlay does
|
||||||
entry and every entry must contain only an `overlay` path. An explicit profile
|
not affect a base or different-profile load. Maps merge recursively, overlay
|
||||||
selection overrides `default_profile`; unknown and explicitly empty selections
|
scalars replace base scalars, and overlay lists replace base lists completely.
|
||||||
fail. Narratio never selects the first profile implicitly and does not read a
|
Explicit `false`, zero, empty strings, empty containers, and null values remain
|
||||||
profile selection from the environment.
|
explicit inputs and are not refilled by defaults. Profiles cannot inherit,
|
||||||
|
stack, import files, or declare another ConfigKit control section.
|
||||||
|
|
||||||
Every declared overlay is resolved relative to the root pipeline directory and
|
During ConfigKit loading, Narratio applies fresh typed defaults, recorded
|
||||||
must satisfy the same confined regular-YAML-file rules as an import. Narratio
|
entry-level defaults and normalization, read-only file-local validation, and
|
||||||
parses every declared overlay even when it is not selected, then applies only
|
ordinary path resolution before computing a deterministic effective
|
||||||
the selected one. Maps merge recursively, overlay scalars replace base scalars,
|
|
||||||
and overlay lists replace base lists completely. Explicit `false`, zero, empty
|
|
||||||
lists, and empty maps remain meaningful. YAML null cannot delete a value, and
|
|
||||||
kind changes are rejected. Profiles cannot inherit from or stack with other
|
|
||||||
profiles, and overlays cannot import files or declare profiles.
|
|
||||||
|
|
||||||
After composition, Narratio strictly decodes the result, applies centralized
|
|
||||||
defaults once, resolves ordinary paths, and computes a deterministic effective
|
|
||||||
configuration digest. The digest represents the normalized, secret-free
|
configuration digest. The digest represents the normalized, secret-free
|
||||||
runtime pipeline mapping; it excludes composition declarations, source
|
runtime pipeline mapping; it excludes ConfigKit directives, source
|
||||||
provenance, profile identity, and raw environment secret values. Equivalent
|
provenance, profile identity, and raw environment secret values. Equivalent
|
||||||
effective mappings therefore have the same digest regardless of how fields are
|
effective mappings therefore have the same digest regardless of how fields are
|
||||||
split among the root and imports.
|
split among the root and imports.
|
||||||
@@ -224,15 +225,31 @@ inputs:
|
|||||||
## Secrets Handling
|
## Secrets Handling
|
||||||
|
|
||||||
- Do not place raw secrets in YAML.
|
- Do not place raw secrets in YAML.
|
||||||
- Use env var names in config (for example `pipeline.audita.llm_api_key_env`).
|
- Credential fields contain environment-variable names, represented internally
|
||||||
- Optionally load credential files from `pipeline.secrets.env_dir`. Each valid
|
as ConfigKit typed secret references (for example
|
||||||
environment-variable filename supplies one value; trailing CR/LF is removed.
|
`pipeline.notarius.llm_api_key: NOTARIUS_LLM_API_KEY`). A configured reference
|
||||||
|
must resolve to a nonempty environment value during load.
|
||||||
|
- Every surviving reference is resolved during ordinary pipeline loading,
|
||||||
|
regardless of whether its integration is disabled, outside the selected
|
||||||
|
stage range, or eligible for resume. Omit the field or set it to `null` when
|
||||||
|
that deployment should not require the credential.
|
||||||
|
- Every command that loads a pipeline accepts `--secrets-dir <path>`. When the
|
||||||
|
flag is omitted, Narratio uses a present `secrets/` directory beside the
|
||||||
|
resolved root `pipeline.yml`; an absent conventional directory is ignored.
|
||||||
|
Explicit relative paths are resolved from the caller's working directory,
|
||||||
|
and a missing or empty explicit selection is an error.
|
||||||
|
- Each immediate regular file whose name is a valid environment-variable name
|
||||||
|
supplies one value. ConfigKit removes trailing CR/LF, permits a symlink as the
|
||||||
|
selected directory, rejects non-hidden symlink entries, and limits input to
|
||||||
|
1 MiB per file, 8 MiB total, and 1024 entries. Dot-prefixed entries (including
|
||||||
|
files, directories, and symlinks) are ignored and omitted from bootstrap
|
||||||
|
reports, but still count toward the directory entry limit.
|
||||||
- An existing process environment value takes precedence over a credential file.
|
- An existing process environment value takes precedence over a credential file.
|
||||||
- Credential directories and files must not be symlinks and must be regular,
|
- On POSIX, operators must provision the directory with no group/other access
|
||||||
bounded files (at most 8 KiB per value). On POSIX, provision the directory
|
(normally `0700`) and files with no group/other access (normally `0600`);
|
||||||
with no group/other access (normally `0700`) and files with no group/other
|
ConfigKit validates type and bounds but does not enforce permission modes.
|
||||||
access (normally `0600`).
|
- Bootstrap happens once per command, before any configuration load or adapter
|
||||||
- Commands that need storage/auth load filesystem secrets before constructing adapters.
|
construction. Safe reports contain filenames, paths, and statuses, never values.
|
||||||
|
|
||||||
## Publish Configuration Summary
|
## Publish Configuration Summary
|
||||||
|
|
||||||
@@ -274,22 +291,19 @@ Rules:
|
|||||||
|
|
||||||
| Field | Type | Required | Default / Rule |
|
| Field | Type | Required | Default / Rule |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| `composition.imports[]` | list of strings | No | explicit additive pipeline fragments relative to the root pipeline directory; `.yml` or `.yaml` regular files only |
|
| `configkit.imports[].path` | string | No | explicit additive pipeline fragment relative to the root pipeline directory; `.yml` or `.yaml` regular files only |
|
||||||
| `composition.default_profile` | string | Conditional | selected when profiles exist and no caller explicitly selects one; must name a declared profile |
|
|
||||||
| `composition.profiles.<name>.overlay` | string | Conditional | required for every declared profile; one confined `.yml` or `.yaml` overlay relative to the root pipeline directory |
|
|
||||||
| `pipeline.workspace.root` | string | No | `/var/lib/narratio` |
|
| `pipeline.workspace.root` | string | No | `/var/lib/narratio` |
|
||||||
| `pipeline.workspace.cleanup_after_publish` | bool | No | `false`; selects the published run's workspace target for final cleanup and requires effective `pipeline.publish.enabled=true` and `pipeline.publish.upload_run=true` |
|
| `pipeline.workspace.cleanup_after_publish` | bool | No | `false`; selects the published run's workspace target for final cleanup and requires effective `pipeline.publish.enabled=true` and `pipeline.publish.upload_run=true` |
|
||||||
| `pipeline.campaigns.root` | string | No | `/usr/local/share/narratio/campaigns` |
|
| `pipeline.campaigns.root` | string | No | `/usr/local/share/narratio/campaigns` |
|
||||||
| `pipeline.campaigns.default_campaign_id` | string | No | empty |
|
| `pipeline.campaigns.default_campaign_id` | string | No | empty |
|
||||||
| `pipeline.secrets.env_dir` | string | No | empty |
|
|
||||||
| `pipeline.storage.backend` | string | No | `local`; supported values are `local` and `s3` (case-insensitive) |
|
| `pipeline.storage.backend` | string | No | `local`; supported values are `local` and `s3` (case-insensitive) |
|
||||||
| `pipeline.storage.s3.bucket` | string | Conditional | required when backend is `s3` and S3 session-audio or publish upload is enabled |
|
| `pipeline.storage.s3.bucket` | string | Conditional | required when backend is `s3` and S3 session-audio or publish upload is enabled |
|
||||||
| `pipeline.storage.s3.root_prefix` | string | No | `dnd` |
|
| `pipeline.storage.s3.root_prefix` | string | No | `dnd` |
|
||||||
| `pipeline.storage.s3.region` | string | No | empty |
|
| `pipeline.storage.s3.region` | string | No | empty |
|
||||||
| `pipeline.storage.s3.endpoint` | string | No | empty |
|
| `pipeline.storage.s3.endpoint` | string | No | empty |
|
||||||
| `pipeline.storage.s3.force_path_style` | bool | No | `false` |
|
| `pipeline.storage.s3.force_path_style` | bool | No | `false` |
|
||||||
| `pipeline.storage.s3.access_key_id_env` | string | No | `OBJECT_STORAGE_KEY_ID` |
|
| `pipeline.storage.s3.access_key_id` | secret reference | Conditional | unset; must be paired with `secret_access_key` |
|
||||||
| `pipeline.storage.s3.secret_access_key_env` | string | No | `OBJECT_STORAGE_KEY` |
|
| `pipeline.storage.s3.secret_access_key` | secret reference | Conditional | unset; must be paired with `access_key_id` |
|
||||||
| `pipeline.spool.root` | string | No | `/var/spool/narratio` |
|
| `pipeline.spool.root` | string | No | `/var/spool/narratio` |
|
||||||
| `pipeline.spool.delete_audio_after_publish` | bool | No | `false`; selects the published run's spool-audio target for final cleanup and requires effective `pipeline.publish.enabled=true` and `pipeline.publish.upload_run=true` |
|
| `pipeline.spool.delete_audio_after_publish` | bool | No | `false`; selects the published run's spool-audio target for final cleanup and requires effective `pipeline.publish.enabled=true` and `pipeline.publish.upload_run=true` |
|
||||||
| `pipeline.cache.root` | string | No | `/var/cache/narratio` |
|
| `pipeline.cache.root` | string | No | `/var/cache/narratio` |
|
||||||
@@ -317,7 +331,7 @@ Rules:
|
|||||||
| `pipeline.seriatim.env.filler_max_duration` | float | No | unset |
|
| `pipeline.seriatim.env.filler_max_duration` | float | No | unset |
|
||||||
| `pipeline.audita.binary` | string | No | `audita` |
|
| `pipeline.audita.binary` | string | No | `audita` |
|
||||||
| `pipeline.audita.timeout` | duration | No | `3h` |
|
| `pipeline.audita.timeout` | duration | No | `3h` |
|
||||||
| `pipeline.audita.llm_api_key_env` | string | No | empty |
|
| `pipeline.audita.llm_api_key` | secret reference | No | unset |
|
||||||
| `pipeline.audita.modules[]` | list[string] | No | empty |
|
| `pipeline.audita.modules[]` | list[string] | No | empty |
|
||||||
| `pipeline.audita.base_url` | string | No | empty |
|
| `pipeline.audita.base_url` | string | No | empty |
|
||||||
| `pipeline.audita.model` | string | No | empty |
|
| `pipeline.audita.model` | string | No | empty |
|
||||||
@@ -345,10 +359,14 @@ Rules:
|
|||||||
| `pipeline.trim.seriatim.report` | bool | No | `false` |
|
| `pipeline.trim.seriatim.report` | bool | No | `false` |
|
||||||
| `pipeline.notarius.enabled` | bool | No | `false` |
|
| `pipeline.notarius.enabled` | bool | No | `false` |
|
||||||
| `pipeline.notarius.binary` | string | No | `notarius` |
|
| `pipeline.notarius.binary` | string | No | `notarius` |
|
||||||
|
| `pipeline.notarius.llm_api_key` | secret reference | No | unset; exact environment-variable name forwarded to Notarius |
|
||||||
|
| `pipeline.notarius.config_profile` | string | No | unset; exact Notarius ConfigKit profile selector |
|
||||||
| `pipeline.notarius.config_path` | string | Conditional | required when enabled; relative paths resolve from the pipeline file directory |
|
| `pipeline.notarius.config_path` | string | Conditional | required when enabled; relative paths resolve from the pipeline file directory |
|
||||||
| `pipeline.notarius.pipeline_id` | string | Conditional | required when enabled |
|
| `pipeline.notarius.pipeline_id` | string | Conditional | required when enabled |
|
||||||
| `pipeline.notarius.timeout` | duration | No | `3h`; must be positive |
|
| `pipeline.notarius.timeout` | duration | No | `3h`; must be positive |
|
||||||
| `pipeline.notarius.working_directory` | string | No | directory containing resolved `config_path`; relative paths resolve from the pipeline file directory |
|
| `pipeline.notarius.working_directory` | string | No | directory containing resolved `config_path`; relative paths resolve from the pipeline file directory |
|
||||||
|
| `pipeline.notarius.resume` | bool | No | `false`; request compatible Notarius checkpoint reuse during ordinary executed extraction |
|
||||||
|
| `pipeline.notarius.chunk_cache` | string | No | empty; when set, exactly `auto`, `bypass`, or `refresh` |
|
||||||
| `pipeline.notarius.references` | map[string]string | No | empty; maps normalized Notarius selectors to supported prepared Narratio source IDs; maximum 256 entries |
|
| `pipeline.notarius.references` | map[string]string | No | empty; maps normalized Notarius selectors to supported prepared Narratio source IDs; maximum 256 entries |
|
||||||
| `pipeline.notarius.outputs` | map | Conditional | at least one entry when enabled |
|
| `pipeline.notarius.outputs` | map | Conditional | at least one entry when enabled |
|
||||||
| `pipeline.render.enabled` | bool | No | `true` |
|
| `pipeline.render.enabled` | bool | No | `true` |
|
||||||
@@ -365,6 +383,73 @@ Rules:
|
|||||||
| `pipeline.scriptorium.artifact_families` | map | No | empty; expands one ordinary artifact per canonical party character |
|
| `pipeline.scriptorium.artifact_families` | map | No | empty; expands one ordinary artifact per canonical party character |
|
||||||
| `pipeline.notification.mode` | string | No | `noop`; the only supported notification mode until a provider is implemented |
|
| `pipeline.notification.mode` | string | No | `noop`; the only supported notification mode until a provider is implemented |
|
||||||
|
|
||||||
|
### Notarius Configuration Profile
|
||||||
|
|
||||||
|
`pipeline.notarius.config_profile` optionally selects a Notarius ConfigKit
|
||||||
|
configuration profile when extraction executes:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
notarius:
|
||||||
|
config_profile: downstream-economy
|
||||||
|
```
|
||||||
|
|
||||||
|
When the field is omitted or `null`, Narratio supplies no downstream profile
|
||||||
|
selector and preserves Notarius's existing configuration-selection behavior.
|
||||||
|
When present, Narratio forwards the exact string to Notarius, including an
|
||||||
|
empty string or `default`, which explicitly select Notarius's composed base
|
||||||
|
configuration. Notarius owns selector interpretation and validation; Narratio
|
||||||
|
does not restrict profile names, resolve profile files, or rewrite the value.
|
||||||
|
|
||||||
|
This selection is independent of Narratio's own `--profile` selection. A
|
||||||
|
Narratio overlay may explicitly select a differently named Notarius overlay,
|
||||||
|
but Narratio never forwards its own profile name automatically. It is also
|
||||||
|
independent of `pipeline.notarius.llm_api_key`: an overlay may change either or
|
||||||
|
both fields according to the deployed Notarius configuration. See the
|
||||||
|
[Notarius integration contract](./integrations/notarius.md) for invocation and
|
||||||
|
failure behavior.
|
||||||
|
|
||||||
|
### Notarius Credential
|
||||||
|
|
||||||
|
`pipeline.notarius.llm_api_key` optionally names the one environment variable
|
||||||
|
to forward to the Notarius child process:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
notarius:
|
||||||
|
llm_api_key: OPENROUTER_API_KEY
|
||||||
|
```
|
||||||
|
|
||||||
|
The field contains the variable name, not a credential value. That name must
|
||||||
|
match `api_key_env` in the Notarius/PromptKit profile selected by the deployed
|
||||||
|
Notarius pipeline; Narratio does not infer or remap provider names. Omission or
|
||||||
|
`null` forwards nothing. A non-null selected reference must resolve at pipeline
|
||||||
|
load time through the process environment or existing secrets-directory
|
||||||
|
bootstrap, even when Notarius is disabled, extract is outside the requested
|
||||||
|
range, or a completed extraction can be reused.
|
||||||
|
|
||||||
|
The Notarius child receives only Narratio's minimal execution environment plus
|
||||||
|
this exact selected name and loaded value. Other credentials are not forwarded.
|
||||||
|
Changing the value, name, presence, or absence does not invalidate extract's
|
||||||
|
semantic resume evidence. The public name remains part of safe effective
|
||||||
|
inspection, so renaming it may appear in `config diff` and change the effective
|
||||||
|
configuration digest; rotating only its value does neither.
|
||||||
|
|
||||||
|
See the [Notarius integration contract](./integrations/notarius.md) for the
|
||||||
|
delivery and redaction boundary.
|
||||||
|
|
||||||
|
### Notarius Cache Controls
|
||||||
|
|
||||||
|
`pipeline.notarius.resume` defaults to `false`. When true, an executed ordinary
|
||||||
|
extraction passes `--resume`; the deployed Notarius configuration must already
|
||||||
|
enable checkpoint recording and should select an explicit persistent checkpoint
|
||||||
|
root. Narratio does not inspect or generate that configuration.
|
||||||
|
|
||||||
|
`pipeline.notarius.chunk_cache` is empty by default, leaving Notarius's
|
||||||
|
configured mode in control. A nonempty value must be `auto`, `bypass`, or
|
||||||
|
`refresh` and is passed as `--chunk_cache`. Forced extraction takes precedence
|
||||||
|
over both fields: it omits `--resume` and passes `--chunk_cache refresh`.
|
||||||
|
Neither field changes Narratio's semantic extraction fingerprint, so changing
|
||||||
|
them alone does not rerun an otherwise complete durable extraction.
|
||||||
|
|
||||||
### Notarius Reference Bindings
|
### Notarius Reference Bindings
|
||||||
|
|
||||||
`pipeline.notarius.references` maps a Notarius CLI selector to a prepared
|
`pipeline.notarius.references` maps a Notarius CLI selector to a prepared
|
||||||
@@ -394,8 +479,8 @@ It validates only selector structure and the prepared source vocabulary;
|
|||||||
Notarius owns target-slot declarations and media compatibility.
|
Notarius owns target-slot declarations and media compatibility.
|
||||||
|
|
||||||
Before extraction, Narratio resolves every binding from the current prepared
|
Before extraction, Narratio resolves every binding from the current prepared
|
||||||
session manifest and streams it into a verified invocation-local snapshot whose
|
session manifest and streams it into a verified, content-addressed session
|
||||||
absolute path is passed to Notarius. Missing, unsafe, empty,
|
snapshot whose absolute path is passed to Notarius. Missing, unsafe, empty,
|
||||||
changed-during-copy, or checksum-inconsistent prepared evidence fails with
|
changed-during-copy, or checksum-inconsistent prepared evidence fails with
|
||||||
guidance to force `prepare`. Bindings are sorted by normalized selector and are
|
guidance to force `prepare`. Bindings are sorted by normalized selector and are
|
||||||
part of extraction fingerprint and resume identity. See the
|
part of extraction fingerprint and resume identity. See the
|
||||||
@@ -430,7 +515,7 @@ For each `pipeline.scriptorium.artifacts.<name>`:
|
|||||||
| Field | Type | Required | Rule |
|
| Field | Type | Required | Rule |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| `enabled` | bool | No | `false` if omitted |
|
| `enabled` | bool | No | `false` if omitted |
|
||||||
| `depends_on[]` | list[string] | No | must reference configured artifact keys; no self-reference; configured graph must be acyclic |
|
| `depends_on[]` | list[string] | No | explicit ordering-only edges; must reference configured artifact keys, may not self-reference, and join inferred input edges in one acyclic graph |
|
||||||
| `render_debug` | bool | No | per-artifact override |
|
| `render_debug` | bool | No | per-artifact override |
|
||||||
| `prompt_id` | string | Conditional | required when artifact is enabled |
|
| `prompt_id` | string | Conditional | required when artifact is enabled |
|
||||||
| `profile_id` | string | No | empty |
|
| `profile_id` | string | No | empty |
|
||||||
@@ -444,7 +529,7 @@ Narratio adds `session_id=narratio-session-<session_id>` to every Scriptorium re
|
|||||||
Without `--artifacts`, analyze executes enabled configured artifacts. With an
|
Without `--artifacts`, analyze executes enabled configured artifacts. With an
|
||||||
explicit `--artifacts` list, the exact named configured artifacts are the
|
explicit `--artifacts` list, the exact named configured artifacts are the
|
||||||
one-invocation targets even if their `enabled` values are false. Analyze closes
|
one-invocation targets even if their `enabled` values are false. Analyze closes
|
||||||
those targets over `depends_on`: a current prerequisite is reused, while a
|
those targets over the effective dependency graph: a current prerequisite is reused, while a
|
||||||
stale, missing, or failed prerequisite is rebuilt before its dependent.
|
stale, missing, or failed prerequisite is rebuilt before its dependent.
|
||||||
Unrelated artifacts are not executed. Named targets and any prerequisite that
|
Unrelated artifacts are not executed. Named targets and any prerequisite that
|
||||||
may require rebuilding must therefore have valid executable fields. This
|
may require rebuilding must therefore have valid executable fields. This
|
||||||
@@ -462,6 +547,15 @@ For each artifact input `pipeline.scriptorium.artifacts.<name>.inputs.<input_nam
|
|||||||
loading. Use the canonical `source` identifier to select the input; Narratio
|
loading. Use the canonical `source` identifier to select the input; Narratio
|
||||||
does not provide adapter-specific input passthrough fields.
|
does not provide adapter-specific input passthrough fields.
|
||||||
|
|
||||||
|
Each `narratio.artifact.<name>` input source automatically contributes an
|
||||||
|
effective dependency on that configured artifact. Narratio unions these inferred
|
||||||
|
edges with explicit `depends_on` entries, then trims, deduplicates, and sorts the
|
||||||
|
result during final configuration resolution. Authors use `depends_on` only for
|
||||||
|
ordering constraints that are not already expressed by an artifact input.
|
||||||
|
Use [`narratio config artifacts`](./cli.md#config-validate-config-show-config-sources-config-artifacts-and-config-diff)
|
||||||
|
to inspect keys, definition sources, outputs, and effective dependencies without
|
||||||
|
loading a session or creating runtime state.
|
||||||
|
|
||||||
### Scriptorium Artifact Families
|
### Scriptorium Artifact Families
|
||||||
|
|
||||||
`pipeline.scriptorium.artifact_families` declares a shared artifact template
|
`pipeline.scriptorium.artifact_families` declares a shared artifact template
|
||||||
@@ -493,9 +587,10 @@ Their resolved values are strings. A member variable may not reuse a static
|
|||||||
other Scriptorium artifact.
|
other Scriptorium artifact.
|
||||||
|
|
||||||
Within a family only, an input source may use
|
Within a family only, an input source may use
|
||||||
`narratio.member_artifact.<family>`. The referenced family must be named in
|
`narratio.member_artifact.<family>`. Resolution rewrites the source to the
|
||||||
that family's `member_dependencies`; resolution rewrites the source to the
|
corresponding ordinary `narratio.artifact.<family>_<character_id>` source and
|
||||||
corresponding ordinary `narratio.artifact.<family>_<character_id>` source.
|
infers the concrete member dependency. Use `member_dependencies` only for
|
||||||
|
additional same-character ordering constraints that are not already inputs.
|
||||||
This syntax is rejected in explicit artifacts and never reaches runtime stages
|
This syntax is rejected in explicit artifacts and never reaches runtime stages
|
||||||
or Scriptorium.
|
or Scriptorium.
|
||||||
|
|
||||||
|
|||||||
@@ -58,7 +58,7 @@ in the [Release Procedure](release.md).
|
|||||||
|
|
||||||
Woodpecker also runs `go test -race -shuffle=on -count=3 ./...` on its scheduled
|
Woodpecker also runs `go test -race -shuffle=on -count=3 ./...` on its scheduled
|
||||||
job to expose ordering and repeatability defects. Current runners cross-compile
|
job to expose ordering and repeatability defects. Current runners cross-compile
|
||||||
for macOS and Windows, but do not provide native macOS or Windows execution.
|
the supported Linux amd64 and arm64 targets and the best-effort macOS arm64
|
||||||
Those cross-builds establish compilation only, not platform-equivalent runtime
|
target. The macOS cross-build establishes compilation only, not native runtime
|
||||||
evidence. Add native checks only when official runner labels and successful
|
evidence. Add a native check only when an official runner label and successful
|
||||||
native-run evidence are available.
|
native-run evidence are available.
|
||||||
|
|||||||
@@ -41,7 +41,6 @@ Construction fails for invalid static config values, including:
|
|||||||
|
|
||||||
Run fails for:
|
Run fails for:
|
||||||
- missing required request paths;
|
- missing required request paths;
|
||||||
- missing required credential env var when configured (`llm_api_key_env`);
|
|
||||||
- subprocess execution failure;
|
- subprocess execution failure;
|
||||||
- invalid processed transcript JSON (`segments` array required);
|
- invalid processed transcript JSON (`segments` array required);
|
||||||
- invalid report JSON when reporting is enabled.
|
- invalid report JSON when reporting is enabled.
|
||||||
@@ -53,6 +52,9 @@ Failure results still include output/log/config/exit metadata for diagnostics.
|
|||||||
|
|
||||||
## Deterministic Behavior
|
## Deterministic Behavior
|
||||||
- CLI args are built from runner config + request in a fixed order.
|
- CLI args are built from runner config + request in a fixed order.
|
||||||
|
- A configured typed `llm_api_key` reference is resolved during pipeline load;
|
||||||
|
the captured value is supplied only as the child's `AUDITA_LLM_API_KEY`
|
||||||
|
override and is never re-read from the parent environment.
|
||||||
- Generated invocation YAML (`audita.generated.v1`) is emitted when requested.
|
- Generated invocation YAML (`audita.generated.v1`) is emitted when requested.
|
||||||
- Manifest writes are stage-owned; adapter itself is stateless.
|
- Manifest writes are stage-owned; adapter itself is stateless.
|
||||||
|
|
||||||
|
|||||||
@@ -7,14 +7,14 @@ lanes from the final trimmed Seriatim transcript. Narratio owns invocation,
|
|||||||
safe bundle discovery, lane selection, and its own artifact metadata. Notarius
|
safe bundle discovery, lane selection, and its own artifact metadata. Notarius
|
||||||
owns pipeline definitions, lane schemas, the receipt, and bundle formats.
|
owns pipeline definitions, lane schemas, the receipt, and bundle formats.
|
||||||
|
|
||||||
Canonical Notarius v0.6.0 references:
|
Canonical Notarius v0.7.0 references:
|
||||||
|
|
||||||
- [CLI reference](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.6.0/docs/cli.md)
|
- [CLI reference](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/cli.md)
|
||||||
- [Subprocess consumer contract](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.6.0/docs/consumers/subprocess.md)
|
- [Subprocess consumer contract](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/consumers/subprocess.md)
|
||||||
- [D&D pipeline and lane contracts](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.6.0/docs/consumers/dnd-pipeline.md)
|
- [D&D pipeline and lane contracts](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/consumers/dnd-pipeline.md)
|
||||||
- [Run-result receipt](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.6.0/docs/integrations/run-result.md)
|
- [Run-result receipt](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/integrations/run-result.md)
|
||||||
- [JSON output bundle](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.6.0/docs/integrations/json-output.md)
|
- [JSON output bundle](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/integrations/json-output.md)
|
||||||
- [D&D spell-catalog overlay](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.6.0/docs/integrations/dnd-spell-catalog-overlays.md)
|
- [D&D spell-catalog overlay](https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/integrations/dnd-spell-catalog-overlays.md)
|
||||||
|
|
||||||
The [complete Narratio example](../../examples/pipeline.full.annotated.yml)
|
The [complete Narratio example](../../examples/pipeline.full.annotated.yml)
|
||||||
records the exact current constraints for all ten D&D lanes. Treat the linked
|
records the exact current constraints for all ten D&D lanes. Treat the linked
|
||||||
@@ -25,19 +25,38 @@ duplicate the complete schemas.
|
|||||||
|
|
||||||
When `pipeline.notarius.enabled` is true, Narratio resolves the executable,
|
When `pipeline.notarius.enabled` is true, Narratio resolves the executable,
|
||||||
configuration path, input path, output directory, and working directory to
|
configuration path, input path, output directory, and working directory to
|
||||||
absolute paths. Narratio requires the Notarius v0.6.0 CLI contract when
|
absolute paths. Configured references require support for the `--reference`
|
||||||
references are configured and invokes each binding as a separate argument
|
contract introduced in Notarius v0.6.0. Cache controls require support for the
|
||||||
before `--json`:
|
selected `--resume` and `--chunk_cache` flags. Notarius v0.7.0 is the tested
|
||||||
|
cache integration baseline, not an unconditional minimum version; Narratio
|
||||||
|
does not impose a runtime version gate. Each reference binding is a separate
|
||||||
|
argument before `--json`. When `pipeline.notarius.config_profile` is present,
|
||||||
|
its exact value is another argument pair before `--json`:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
notarius run <pipeline_id> --config <config_path> --input <trimmed_json> --output-dir <staging_dir> [--reference <selector>=<verified_snapshot_path>]... --json
|
notarius run <pipeline_id> --config <config_path> --input <trimmed_json> --output-dir <staging_dir> [--resume] [--chunk_cache <mode>] [--reference <selector>=<verified_snapshot_path>]... [--config-profile <name>] --json
|
||||||
```
|
```
|
||||||
|
|
||||||
Reference paths are absolute invocation-local snapshots streamed from the
|
Notarius v0.7.0 is a known supporting version for `--config-profile`; using the
|
||||||
manifest-verified canonical files prepared inside the current Narratio session
|
option requires a compatible Notarius executable, but deployments that omit the
|
||||||
workspace. Narratio verifies snapshot checksum and size before and after the
|
field acquire no additional version requirement. Omitted or null configuration
|
||||||
subprocess, and passes only configured bindings, ordered lexically by normalized
|
adds no flag. Supplied empty, `default`, and named selectors are forwarded
|
||||||
selector, as direct argument-vector entries without shell interpretation. A CLI
|
without normalization so Notarius can apply its own ConfigKit semantics. This
|
||||||
|
option selects a Notarius configuration overlay, not a PromptKit profile through
|
||||||
|
`--llm-profile`, and Narratio never substitutes its own selected profile name.
|
||||||
|
A missing or invalid downstream overlay is an ordinary Notarius subprocess
|
||||||
|
failure: Narratio does not retry without the selector or fall back to another
|
||||||
|
profile.
|
||||||
|
|
||||||
|
Reference paths are absolute stable snapshots streamed from manifest-verified
|
||||||
|
canonical files prepared inside the current Narratio session workspace. They
|
||||||
|
live at `cache/notarius/references/<sha256>/<prepared-basename>` and retain the
|
||||||
|
full lowercase content digest. Narratio verifies snapshot checksum and size
|
||||||
|
before and after the subprocess, reconstructs missing or corrupt regular files
|
||||||
|
from a still-valid prepared source, and passes only configured bindings,
|
||||||
|
ordered lexically by normalized selector, as direct argument-vector entries
|
||||||
|
without shell interpretation. Symlinks, directories, other non-regular
|
||||||
|
conflicts, and prepared-source mismatches fail rather than being deleted. A CLI
|
||||||
binding takes precedence over a matching external path in Notarius
|
binding takes precedence over a matching external path in Notarius
|
||||||
configuration. Narratio never emits `--without-reference`.
|
configuration. Narratio never emits `--without-reference`.
|
||||||
|
|
||||||
@@ -75,9 +94,41 @@ It does not pass a Narratio session ID or run `notarius config validate`
|
|||||||
automatically; the configured working directory and Narratio's minimal child
|
automatically; the configured working directory and Narratio's minimal child
|
||||||
environment apply to the subprocess.
|
environment apply to the subprocess.
|
||||||
|
|
||||||
|
Ordinary extraction maps `pipeline.notarius.resume: true` to `--resume` and a
|
||||||
|
nonempty `pipeline.notarius.chunk_cache` to `--chunk_cache <mode>`. Forced
|
||||||
|
extraction always omits `--resume` and supplies `--chunk_cache refresh`.
|
||||||
|
Notarius owns checkpoint enablement, recording, identity, and both cache roots;
|
||||||
|
Narratio neither reads nor synthesizes its configuration. Use explicit
|
||||||
|
persistent roots in Notarius when reuse must span subprocess invocations.
|
||||||
|
Changing a reference path/digest or other upstream identity input correctly
|
||||||
|
causes a cold miss. Chunk refresh bypasses lookup and publishes the accepted
|
||||||
|
replacement for the active source; it does not delete unrelated plans or
|
||||||
|
checkpoint state.
|
||||||
|
|
||||||
|
When `pipeline.notarius.llm_api_key` is absent or null, Narratio adds no
|
||||||
|
credential to that environment. When configured, the field names an environment
|
||||||
|
variable whose already-loaded nonempty value is forwarded under that exact
|
||||||
|
name. Operators must use the same name as the `api_key_env` in the selected
|
||||||
|
Notarius/PromptKit profile. Narratio does not inspect the Notarius configuration
|
||||||
|
to infer a provider or destination name, and it does not forward unrelated
|
||||||
|
credentials from its parent environment or secrets directory.
|
||||||
|
|
||||||
|
The selected name is registered with Narratio's shared streaming redactor.
|
||||||
|
Credential bytes written by the child to captured stdout, captured stderr, or
|
||||||
|
stderr-derived failure diagnostics are replaced with `<redacted>`, including
|
||||||
|
custom names not known to the built-in sensitive-name list. Narratio does not
|
||||||
|
serialize the credential into arguments, stage metadata, manifests, or
|
||||||
|
generated configuration.
|
||||||
|
|
||||||
|
This captured-stream protection is not a filesystem sandbox. Notarius remains
|
||||||
|
responsible for keeping credentials out of files it writes directly, including
|
||||||
|
bundle artifacts and its own diagnostics. Narratio does not scan or rewrite
|
||||||
|
arbitrary downstream files and cannot promise redaction of transformed or
|
||||||
|
encoded credentials.
|
||||||
|
|
||||||
## Accepted Result
|
## Accepted Result
|
||||||
|
|
||||||
Narratio's supported invocation baseline is Notarius v0.6.0. The accepted
|
Narratio's tested invocation baseline is Notarius v0.7.0. The accepted
|
||||||
receipt remains `notarius.run-result.v2`; reference flags do not change the
|
receipt remains `notarius.run-result.v2`; reference flags do not change the
|
||||||
receipt or ten-lane output contract. The receipt
|
receipt or ten-lane output contract. The receipt
|
||||||
must identify the configured pipeline, and its `index_file` must be exactly
|
must identify the configured pipeline, and its `index_file` must be exactly
|
||||||
@@ -102,6 +153,8 @@ version, and, when configured, module key. Missing, duplicate, rejected, or
|
|||||||
incompatible required lanes fail extraction even if Notarius exited zero. A
|
incompatible required lanes fail extraction even if Notarius exited zero. A
|
||||||
configured lane whose v2 validation summary is `rejected` or `incomplete` also
|
configured lane whose v2 validation summary is `rejected` or `incomplete` also
|
||||||
fails extraction.
|
fails extraction.
|
||||||
|
An accepted reused producer may report `producer_attempt_count: 0`; Narratio
|
||||||
|
preserves that valid summary rather than treating it as a missing field.
|
||||||
Unconfigured lanes may remain in the preserved bundle but do not become
|
Unconfigured lanes may remain in the preserved bundle but do not become
|
||||||
selectable Narratio sources.
|
selectable Narratio sources.
|
||||||
|
|
||||||
@@ -130,3 +183,18 @@ Configuration fields and defaults are in [Configuration](../config.md).
|
|||||||
Operator paths, rerun procedures, and bundle retention are in
|
Operator paths, rerun procedures, and bundle retention are in
|
||||||
[Operations](../operations.md). See [Troubleshooting](../troubleshooting.md)
|
[Operations](../operations.md). See [Troubleshooting](../troubleshooting.md)
|
||||||
for failure recovery.
|
for failure recovery.
|
||||||
|
|
||||||
|
## Opt-in Integration Test
|
||||||
|
|
||||||
|
The production-boundary cache test requires an absolute path to a locally
|
||||||
|
built Notarius v0.7.0-compatible executable. It uses only a local
|
||||||
|
OpenAI-compatible `httptest` provider and temporary cache/config/output roots:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
NARRATIO_TEST_NOTARIUS_BINARY=/absolute/path/to/notarius \
|
||||||
|
GOWORK=off go test ./internal/stage \
|
||||||
|
-run '^TestNotariusCacheReuseIntegration$' -count=1 -v
|
||||||
|
```
|
||||||
|
|
||||||
|
When the variable is unset, the normal test suite skips this test and never
|
||||||
|
searches `PATH`, a sibling checkout, the network, or a real provider.
|
||||||
|
|||||||
@@ -17,11 +17,20 @@ Narratio sends an HTTP `POST` to the configured transcription URL using
|
|||||||
The server must return a `2xx` response whose body is valid JSON. Narratio does
|
The server must return a `2xx` response whose body is valid JSON. Narratio does
|
||||||
not currently require a more specific response schema at this boundary.
|
not currently require a more specific response schema at this boundary.
|
||||||
|
|
||||||
The transcription URL must be an absolute `http` or `https` URL. The audio body
|
The transcription URL must be an absolute `http` or `https` URL. Each
|
||||||
is streamed through a fresh multipart writer for every attempt, so its memory
|
application attempt creates one bounded multipart framing and streams the audio
|
||||||
use is bounded by the transport buffer rather than by the complete audio file.
|
from an open file between that framing; it does not buffer the complete audio
|
||||||
|
or spool a multipart copy. The file field precedes the language field.
|
||||||
WhisperX response acquisition is capped at 10 MiB.
|
WhisperX response acquisition is capped at 10 MiB.
|
||||||
|
|
||||||
|
The initial request body and any transport-requested replay use the same
|
||||||
|
multipart boundary and framing. Every body opens an independent audio handle
|
||||||
|
and replay starts at the beginning. Narratio closes all bodies created for the
|
||||||
|
attempt on response, failure, timeout, or cancellation, including when a
|
||||||
|
transport abandons an earlier body. Genuine source read/close failures remain
|
||||||
|
request failures; deliberate early closure of an abandoned original does not
|
||||||
|
poison a valid replay.
|
||||||
|
|
||||||
## Request And Result Contract
|
## Request And Result Contract
|
||||||
|
|
||||||
Each adapter request identifies a speaker, a readable audio file, and the
|
Each adapter request identifies a speaker, a readable audio file, and the
|
||||||
@@ -43,6 +52,11 @@ transcript output.
|
|||||||
- Narratio waits the configured retry delay between attempts and aborts that
|
- Narratio waits the configured retry delay between attempts and aborts that
|
||||||
wait when the parent context is canceled.
|
wait when the parent context is canceled.
|
||||||
|
|
||||||
|
Transport replay occurs inside one application attempt and does not consume an
|
||||||
|
entry from the configured retry budget or increment the reported attempt
|
||||||
|
count. The standard HTTP redirect policy remains in effect; body replay can
|
||||||
|
also support body-preserving redirects.
|
||||||
|
|
||||||
## Validation And Failure Semantics
|
## Validation And Failure Semantics
|
||||||
|
|
||||||
Client construction rejects a missing or non-HTTP(S) absolute transcription URL,
|
Client construction rejects a missing or non-HTTP(S) absolute transcription URL,
|
||||||
@@ -55,12 +69,19 @@ failure to install the output causes the transcription to fail. Errors include
|
|||||||
attempt context, and the result retains attempts, final status when available,
|
attempt context, and the result retains attempts, final status when available,
|
||||||
and elapsed duration for diagnostics.
|
and elapsed duration for diagnostics.
|
||||||
|
|
||||||
|
Replay makes a fresh request body available to the HTTP transport; it does not
|
||||||
|
provide an exactly-once server guarantee and does not prove recovery from every
|
||||||
|
proxy or protocol-level failure. A server may have received work before the
|
||||||
|
client learns that a request failed. WhisperX operations should therefore be
|
||||||
|
safe to retry for the same input when practical.
|
||||||
|
|
||||||
## Determinism And Concurrency
|
## Determinism And Concurrency
|
||||||
|
|
||||||
Each audio request has stable multipart field names, and successful bytes are
|
Each audio request has stable multipart field names, and successful bytes are
|
||||||
installed atomically. The transcribe stage may process speaker files in
|
installed atomically. The transcribe stage may process speaker files in
|
||||||
parallel, bounded by the configured concurrency. It records results in stable
|
parallel, bounded by the configured concurrency. Each validated speaker is
|
||||||
speaker order after all work completes; any speaker failure fails the stage.
|
checkpointed independently; any speaker failure still fails the aggregate
|
||||||
|
stage, while committed earlier speakers remain reusable on an ordinary retry.
|
||||||
|
|
||||||
## Related Canonical Docs
|
## Related Canonical Docs
|
||||||
|
|
||||||
|
|||||||
@@ -65,6 +65,14 @@ Notarius is composed only when extraction is enabled; the extract stage owns
|
|||||||
prepared reference resolution, receipt, bundle, and configured-lane policy.
|
prepared reference resolution, receipt, bundle, and configured-lane policy.
|
||||||
The adapter validates the ordered selector/absolute-path pairs and is the sole
|
The adapter validates the ordered selector/absolute-path pairs and is the sole
|
||||||
owner of serializing them as repeated `--reference` arguments before `--json`.
|
owner of serializing them as repeated `--reference` arguments before `--json`.
|
||||||
|
It also serializes an effective resume request as `--resume` and a nonempty
|
||||||
|
validated chunk mode as `--chunk_cache <mode>`. The extract stage, not the
|
||||||
|
adapter, resolves force precedence before constructing the request.
|
||||||
|
Its request may also carry one already-resolved opaque ConfigKit secret
|
||||||
|
reference. Only the Notarius adapter reads that value, rejects an unresolved or
|
||||||
|
empty programmatic reference, and maps the exact configured name/value into the
|
||||||
|
shared subprocess override and sensitive-name fields. It never rereads the
|
||||||
|
parent environment. A nil reference preserves the previous invocation.
|
||||||
|
|
||||||
Application-owned object-store construction loads configured filesystem
|
Application-owned object-store construction loads configured filesystem
|
||||||
secrets before invoking the invocation's storage factory. Production commands
|
secrets before invoking the invocation's storage factory. Production commands
|
||||||
@@ -78,15 +86,15 @@ in-memory backend without changing package state.
|
|||||||
- Subprocess adapters persist stage logs/generated configs through stage-managed paths.
|
- Subprocess adapters persist stage logs/generated configs through stage-managed paths.
|
||||||
- Shared subprocess execution keeps private lifecycle mechanics separate from
|
- Shared subprocess execution keeps private lifecycle mechanics separate from
|
||||||
Narratio's environment, diagnostics, redaction, and error policy. It starts
|
Narratio's environment, diagnostics, redaction, and error policy. It starts
|
||||||
an owned process group on Linux/macOS or a
|
an owned process group on Linux and macOS. Every terminal path disposes of
|
||||||
kill-on-close job object on Windows. Every terminal path disposes of that
|
that owned tree before returning. After a natural leader exit, the runner checks
|
||||||
owned tree before returning. After a natural leader exit, Unix checks for
|
for remaining group members and uses bounded graceful then forceful termination.
|
||||||
remaining group members and uses bounded graceful then forceful termination;
|
Cancellation, deadlines, and
|
||||||
Windows closes the job so kill-on-close applies. Cancellation, deadlines, and
|
|
||||||
diagnostic limits use the same terminal disposal path without losing their
|
diagnostic limits use the same terminal disposal path without losing their
|
||||||
original result classification. Child environments contain only the execution
|
original result classification. Child environments contain only the execution
|
||||||
baseline and adapter-specified values; configured credentials are explicit
|
baseline and adapter-specified values; configured credentials are resolved
|
||||||
sensitive values. Stdout and stderr are redacted while streaming into separate
|
once by ConfigKit and passed explicitly as sensitive values. Adapters do not
|
||||||
|
re-read their named environment references. Stdout and stderr are redacted while streaming into separate
|
||||||
8 MiB diagnostic captures; a bounded wait closes a stream retained by a
|
8 MiB diagnostic captures; a bounded wait closes a stream retained by a
|
||||||
departed leader's descendant. Unsupported platforms reject owned command
|
departed leader's descendant. Unsupported platforms reject owned command
|
||||||
execution.
|
execution.
|
||||||
|
|||||||
@@ -58,10 +58,12 @@ Configured definitions are always registered. Without an explicit selection,
|
|||||||
the effective analyze set contains enabled definitions. With `--artifacts`, the
|
the effective analyze set contains enabled definitions. With `--artifacts`, the
|
||||||
exact named configured definitions become the effective set for that invocation,
|
exact named configured definitions become the effective set for that invocation,
|
||||||
regardless of their `enabled` value. The effective-set resolver itself does not
|
regardless of their `enabled` value. The effective-set resolver itself does not
|
||||||
expand dependencies; the analyze work planner closes those targets over their
|
expand dependencies. Configuration has already materialized the sorted union of
|
||||||
configured prerequisite graph. Availability is separate from executability.
|
explicit ordering edges and configured-artifact input edges, and the analyze
|
||||||
Configuration may normalize a family selection into its concrete generated
|
work planner closes targets over that one concrete prerequisite graph.
|
||||||
members before this resolver runs. The effective set retains optional family
|
Availability is separate from executability. Configuration may normalize a
|
||||||
|
family selection into its concrete generated members before this resolver runs.
|
||||||
|
The effective set retains optional family
|
||||||
and character origin metadata, but its keys, catalog sources, and runtime
|
and character origin metadata, but its keys, catalog sources, and runtime
|
||||||
lookups remain concrete configured-artifact identities.
|
lookups remain concrete configured-artifact identities.
|
||||||
Family publish policies are likewise expanded into ordinary configured-source
|
Family publish policies are likewise expanded into ordinary configured-source
|
||||||
|
|||||||
@@ -8,30 +8,21 @@ pipeline-loading boundary implemented by `internal/config`.
|
|||||||
|
|
||||||
`LoadPipeline` assembles and validates a pipeline in this order:
|
`LoadPipeline` assembles and validates a pipeline in this order:
|
||||||
|
|
||||||
1. Parse the root YAML into a presence-aware composition tree. The tree retains
|
1. Give ConfigKit a fresh typed `PipelineConfig` default value, the root file,
|
||||||
source names, full field paths, node kinds, declaration order, and explicit
|
enabled imports, and the caller's profile selection. Omitted, empty, and
|
||||||
zero, false, empty-map, and empty-list values.
|
exact `default` selections use the base; named profiles use the fixed
|
||||||
2. Decode the root-only `composition` envelope, validate its declaration shapes
|
`profiles/<name>.yml` convention.
|
||||||
and cross-field profile semantics, then clone the root without that envelope
|
2. ConfigKit parses the root and declared imports, additively merges base
|
||||||
as the ordinary pipeline body. The parsed source document is not mutated. A
|
sources, applies only the selected overlay, and strictly decodes the result.
|
||||||
load option retains the difference between omitted and explicitly empty
|
3. The definition resolver applies recorded entry-level and inherited defaults,
|
||||||
profile selection.
|
runs deterministic domain normalization, records derived publish destinations,
|
||||||
3. Open each import relative to the root pipeline directory through the
|
and resolves ordinary paths relative to the root pipeline file.
|
||||||
confined regular-file boundary. Imports must use a `.yml` or `.yaml`
|
4. The definition validator performs read-only checks that require only the
|
||||||
extension and cannot traverse, use symlinks, repeat a file, import the root,
|
pipeline file. Checks requiring canonical-party family expansion remain at
|
||||||
or contain another composition envelope.
|
the existing explicit pipeline/campaign resolution boundary.
|
||||||
4. Resolve and structurally parse every declared profile overlay through the
|
5. Narratio inspects public ConfigKit metadata to reject artifact and family keys
|
||||||
same confined regular-file boundary. Missing or malformed unselected
|
declared by multiple base files, adapts source ownership records, attaches
|
||||||
overlays fail the load. Overlays cannot contain a composition envelope.
|
runtime-only state, and computes the normalized effective digest.
|
||||||
5. Additively merge the root body and imports. Distinct map leaves compose;
|
|
||||||
repeated scalar or list paths and node-kind disagreements are conflicts.
|
|
||||||
6. Select exactly one declared profile from an explicit option or the default,
|
|
||||||
then recursively merge its overlay. Overlay leaves replace base leaves,
|
|
||||||
lists are atomic replacements, and null or kind changes fail.
|
|
||||||
7. Emit deterministic canonical YAML and strictly decode it into
|
|
||||||
`PipelineConfig`.
|
|
||||||
8. Apply pipeline defaults once, resolve ordinary relative pipeline paths from
|
|
||||||
the root pipeline file, and digest the normalized effective mapping.
|
|
||||||
|
|
||||||
This ordering preserves monolithic configuration behavior. Moving a field to
|
This ordering preserves monolithic configuration behavior. Moving a field to
|
||||||
an imported fragment changes its source ownership, not its path base, default,
|
an imported fragment changes its source ownership, not its path base, default,
|
||||||
@@ -53,25 +44,35 @@ download retain that exact pipeline object and its private provenance. Removing
|
|||||||
a temporary downloaded session file therefore cannot invalidate the resolved
|
a temporary downloaded session file therefore cannot invalidate the resolved
|
||||||
pipeline or campaign context.
|
pipeline or campaign context.
|
||||||
|
|
||||||
|
Campaign and session files use independent ConfigKit definitions with imports
|
||||||
|
and profiles disabled. Campaign files normally load directly from their source;
|
||||||
|
session files are first read through the 8 MiB bounded-read boundary because the
|
||||||
|
application must reject unresolved template placeholders before passing an exact
|
||||||
|
byte snapshot to ConfigKit. Local and remotely downloaded session bytes then use
|
||||||
|
the same typed loader and retain the existing post-load identity expectations,
|
||||||
|
stable-input precedence, and source-relative path semantics.
|
||||||
|
|
||||||
The application also has a separate read-only inspection resolver for `config
|
The application also has a separate read-only inspection resolver for `config
|
||||||
validate`, `config show`, and `config sources`. It uses the same production root/profile and
|
validate`, `config show`, `config sources`, and `config artifacts`. It uses the same production root/profile and
|
||||||
campaign selection functions, but never routes through session discovery,
|
campaign selection functions, but never routes through session discovery,
|
||||||
remote-session download, secret loading, adapter composition, workspace
|
remote-session download, adapter composition, workspace
|
||||||
initialization, manifest access, or cleanup. A pipeline with retained artifact
|
initialization, manifest access, or cleanup. A pipeline with retained artifact
|
||||||
family declarations must resolve its selected campaign before ordinary pipeline
|
family declarations must resolve its selected campaign before ordinary pipeline
|
||||||
validation, which expands its canonical-party members and generated publish
|
validation, which expands its canonical-party members and generated publish
|
||||||
rules. A pipeline without those declarations may be validated by itself.
|
rules. A pipeline without those declarations may be validated by itself.
|
||||||
|
Secret-directory bootstrap remains invocation-scoped and happens before the
|
||||||
|
ordinary ConfigKit load because explicit typed references must resolve even for
|
||||||
|
inspection.
|
||||||
|
|
||||||
`MarshalEffectivePipeline` is the configuration-owned projection for `config
|
`MarshalEffectivePipeline` is the configuration-owned projection for `config
|
||||||
show`. It serializes the typed, defaulted effective mapping through the
|
show`. It serializes one explicit typed safe projection with the existing YAML
|
||||||
deterministic composition renderer, then removes resolution-only artifact
|
encoder. The projection has no slots for resolution-only artifact family
|
||||||
family declarations. The result contains no composition envelope or private
|
declarations or private provenance. The result has one trailing newline;
|
||||||
provenance fields and has one trailing newline; commands do not marshal runtime
|
commands do not marshal runtime objects directly.
|
||||||
objects directly.
|
|
||||||
|
|
||||||
`EffectivePipelineSources` and `EffectiveCampaignSources` provide the separate
|
`EffectivePipelineSources` and `EffectiveCampaignSources` provide the separate
|
||||||
safe provenance projection for `config sources`. Pipeline ownership begins with
|
safe provenance projection for `config sources`. Pipeline ownership begins with
|
||||||
the complete logical field paths retained during composition and classifies
|
the complete logical field paths projected from the schema and classifies
|
||||||
each contributor as root, import, profile, or centralized default. The
|
each contributor as root, import, profile, or centralized default. The
|
||||||
projection replaces generated concrete member paths with paired family and
|
projection replaces generated concrete member paths with paired family and
|
||||||
canonical-party records, and does the same for generated publish rules.
|
canonical-party records, and does the same for generated publish rules.
|
||||||
@@ -79,12 +80,18 @@ Campaign records identify campaign-owned fields and party inputs; canonical
|
|||||||
derived players point to the party source. The application command only joins these sorted
|
derived players point to the party source. The application command only joins these sorted
|
||||||
records with selection metadata and never reparses configuration files.
|
records with selection metadata and never reparses configuration files.
|
||||||
|
|
||||||
`config diff` uses a paired profile loader that parses the root, imports, and
|
`EffectiveArtifactCatalog` projects the same finalized concrete artifact map for
|
||||||
declared overlays once, then clones the additive base before independently
|
`config artifacts`. It derives source identity from each map key, carries sorted
|
||||||
selecting, decoding, defaulting, and finalizing each profile. When campaign
|
effective dependencies, aggregates ConfigKit definition contributors, and adds
|
||||||
|
family/character/party provenance for generated members. It does not inspect
|
||||||
|
files named after artifacts or infer another runtime graph.
|
||||||
|
|
||||||
|
`config diff` uses a thin paired loader that performs two independent ConfigKit
|
||||||
|
loads. When campaign
|
||||||
resolution is needed, the command loads one selected campaign and party and
|
resolution is needed, the command loads one selected campaign and party and
|
||||||
expands both effective pipelines from that same party value. The configuration
|
expands both effective pipelines from that same party value. The configuration
|
||||||
owner projects each normalized effective mapping into sorted logical paths;
|
owner projects each normalized effective mapping directly from that same typed
|
||||||
|
projection into sorted logical paths using JSON number preservation;
|
||||||
mapping leaves are compared individually while sequence values remain atomic.
|
mapping leaves are compared individually while sequence values remain atomic.
|
||||||
Values are compact deterministic JSON representations for command output, not
|
Values are compact deterministic JSON representations for command output, not
|
||||||
raw YAML fragments, ownership records, or secret material. A differing digest
|
raw YAML fragments, ownership records, or secret material. A differing digest
|
||||||
@@ -98,13 +105,16 @@ configured roster authority; player and session-party fields are unsupported.
|
|||||||
|
|
||||||
## Canonical Party Domain
|
## Canonical Party Domain
|
||||||
|
|
||||||
`ParseParty` is the package-owned boundary for parsing a party source. It
|
`ParseParty` is the package-owned boundary for parsing a party source. Its
|
||||||
requires a top-level `schema_version` and strictly validates the
|
single-source ConfigKit definition disables imports and profiles, requires
|
||||||
`narratio.party.v1` contract into ordered character domain values. The
|
`schema_version` through `VersionRule`, and strictly decodes the
|
||||||
canonical value retains a separate exact byte copy of its source so consumers
|
character-keyed typed schema before applying Narratio's party-domain checks.
|
||||||
can materialize the authored party document without reserializing it. Its
|
Successful metadata coordinates recover authored character-map order without a
|
||||||
`PlayersYAML` method deterministically derives the versioned players-only
|
second YAML parse; class and alias slice order comes directly from typed decode.
|
||||||
projection.
|
The canonical value retains a separate exact byte copy of its source so
|
||||||
|
consumers can materialize the authored party document without reserializing it.
|
||||||
|
Its `PlayersYAML` method deterministically derives the ID-sorted, versioned
|
||||||
|
players-only projection.
|
||||||
|
|
||||||
An unversioned source is rejected before resolution, family expansion, or
|
An unversioned source is rejected before resolution, family expansion, or
|
||||||
prepare-stage materialization.
|
prepare-stage materialization.
|
||||||
@@ -117,58 +127,67 @@ for core pipeline/session rules, external adapters, publish/storage, Notarius,
|
|||||||
Scriptorium, and shared scalar/path checks. These are package-local ownership
|
Scriptorium, and shared scalar/path checks. These are package-local ownership
|
||||||
boundaries rather than extension points or a general validation framework.
|
boundaries rather than extension points or a general validation framework.
|
||||||
|
|
||||||
Validation fails fast and does not define which of several independent invalid
|
Validation is read-only, fails fast, and does not define which of several
|
||||||
fields is reported first. Map-backed declarations are nevertheless traversed
|
independent invalid fields is reported first. Map-backed declarations are
|
||||||
deterministically. Notarius validates scalar policy, builds normalized reference
|
nevertheless traversed deterministically. Reusable domain normalization is an
|
||||||
bindings and output lanes independently, and installs all candidate values only
|
explicit preceding operation for both loaded and programmatically constructed
|
||||||
after every check succeeds. Scriptorium validates artifact declarations and
|
pipelines. Notarius normalization commits canonical scalar, reference, and output
|
||||||
outputs, dependencies and inputs, variables, referenced outputs, and the final
|
values only after the complete normalization succeeds; its validator then checks
|
||||||
|
that candidate without changing it. Scriptorium validates artifact declarations
|
||||||
|
and outputs, dependencies and inputs, variables, referenced outputs, and the final
|
||||||
dependency graph in deterministic artifact order.
|
dependency graph in deterministic artifact order.
|
||||||
|
|
||||||
Diagnostics identify the relevant semantic field. Composition conflicts also
|
Diagnostics identify the relevant semantic field. Merge conflicts also
|
||||||
retain every claiming source needed to correct an additive ownership conflict.
|
retain every claiming source needed to correct an additive ownership conflict.
|
||||||
Complete prose, punctuation, formatting, and private validation choreography are
|
Complete prose, punctuation, formatting, and private validation choreography are
|
||||||
not compatibility contracts.
|
not compatibility contracts.
|
||||||
|
|
||||||
## Diagnostics And Runtime Metadata
|
## Diagnostics And Runtime Metadata
|
||||||
|
|
||||||
Syntax, duplicate-key, composition, conflict, and schema failures include the
|
Syntax, duplicate-key, merge, profile, and schema failures include the
|
||||||
relevant source name and full field path. Additive conflicts report every
|
relevant source name and full field path. Additive conflicts report every
|
||||||
claiming source so operators can repair the split without repeatedly
|
claiming source so operators can repair the split without repeatedly
|
||||||
rediscovering additional conflicts.
|
rediscovering additional conflicts.
|
||||||
|
|
||||||
The loaded pipeline retains private runtime metadata for the absolute root
|
The loaded pipeline retains private runtime metadata for the absolute root
|
||||||
path, ordered imports, selected profile name and selection source, selected
|
path, ordered imports, selected profile name and selection source, selected
|
||||||
overlay, contributing sources, effective digest, and leaf ownership. Base
|
overlay, original ConfigKit metadata and source snapshots, effective digest,
|
||||||
|
and leaf ownership. Base
|
||||||
leaves retain their root/import owners, replaced leaves belong to the selected
|
leaves retain their root/import owners, replaced leaves belong to the selected
|
||||||
overlay, and centrally supplied values use the synthetic `default` owner. This
|
overlay, and centrally supplied values use the synthetic `default` owner. This
|
||||||
metadata does not participate in YAML decoding or alter the public
|
metadata does not participate in YAML decoding or alter the public
|
||||||
configuration model.
|
configuration model.
|
||||||
|
|
||||||
The effective digest is SHA-256 over deterministic canonical YAML produced from
|
The effective digest is SHA-256 over a versioned deterministic JSON encoding of
|
||||||
the defaulted `PipelineConfig`. Runtime Notarius paths remain absolute for
|
the same typed safe projection used by show and diff. Runtime Notarius paths remain absolute for
|
||||||
execution, but the digest substitutes their normalized logical values captured
|
execution, but the digest substitutes their normalized logical values captured
|
||||||
before root-relative resolution, so relocating an equivalent configuration
|
before root-relative resolution, so relocating an equivalent configuration
|
||||||
bundle does not change provenance. Because composition and resolution metadata
|
bundle does not change provenance. Because ConfigKit and resolution metadata
|
||||||
are private, the digest excludes source layout, profile name, and ownership.
|
are private, the digest excludes source layout, profile name, and ownership.
|
||||||
Configuration stores environment variable names rather than resolving raw
|
Configuration stores credentials as ConfigKit typed references. A successful
|
||||||
credentials, so raw secret values are neither loaded nor hashed.
|
load resolves their values inside the opaque typed boundary, while this safe
|
||||||
|
projection serializes only `EnvName()` identities. Credential-value changes are
|
||||||
|
therefore never hashed; reference-name changes remain ordinary provenance.
|
||||||
|
The optional Notarius credential follows the same path even for disabled or
|
||||||
|
resumable extraction; normalization copies the opaque reference without reading
|
||||||
|
or resolving it again. Direct YAML serialization has a configuration-owned safe
|
||||||
|
projection so only the reference name can leave this boundary.
|
||||||
`recomputePipelineEffectiveDigest` is the single package-owned refresh point
|
`recomputePipelineEffectiveDigest` is the single package-owned refresh point
|
||||||
for later runtime expansion.
|
for later runtime expansion.
|
||||||
|
|
||||||
## Test Surfaces
|
## Test Surfaces
|
||||||
|
|
||||||
`composition_test.go` protects the presence and merge algebra independently of
|
`pipeline_configkit_test.go` exercises real imports and fixed-path profiles,
|
||||||
the public schema. `pipeline_composition_test.go` exercises explicit imports,
|
base selection, unselected-overlay isolation, source attribution, and Narratio's
|
||||||
confinement, conflicts, strict decoding, metadata, and root-relative path
|
unique artifact/family base-declaration rule. Existing focused configuration
|
||||||
behavior through `LoadPipeline`. `pipeline_profiles_test.go` covers selection,
|
tests protect defaults, explicit-value semantics, strict decoding, validation,
|
||||||
all-overlay validation, overlay behavior, provenance, option propagation, and
|
metadata, and root-relative path behavior. Application configuration-loader tests protect the
|
||||||
effective-digest stability. Application configuration-loader tests protect the
|
|
||||||
single-read boundary by changing the pipeline file after its initial load and
|
single-read boundary by changing the pipeline file after its initial load and
|
||||||
confirming local session resolution retains the original pipeline. Other
|
confirming local session resolution retains the original pipeline. Other
|
||||||
configuration tests continue to protect defaults and validation after assembly.
|
configuration tests continue to protect defaults and validation after assembly.
|
||||||
`party_test.go` protects the versioned party schema, domain invariants, and
|
`party_test.go` protects the versioned typed party schema, domain invariants,
|
||||||
deterministic players projection without involving campaign or runtime wiring.
|
authored character order, retained bytes, and deterministic players projection
|
||||||
|
without involving campaign or runtime wiring.
|
||||||
`party_resolution_test.go` protects campaign-owned party loading, canonical
|
`party_resolution_test.go` protects campaign-owned party loading, canonical
|
||||||
input restrictions, source provenance, and virtual players input selection.
|
input restrictions, source provenance, and virtual players input selection.
|
||||||
|
|
||||||
|
|||||||
@@ -50,6 +50,45 @@ The model admits these stage states:
|
|||||||
- `stale`
|
- `stale`
|
||||||
- `interrupted`
|
- `interrupted`
|
||||||
|
|
||||||
|
### Transcribe-owned speaker state
|
||||||
|
|
||||||
|
A `transcribe` stage record may carry `transcribe_state_version: 1` and a
|
||||||
|
`transcribe_speakers` map keyed by the existing derived speaker ID. The version
|
||||||
|
is the authority marker: a versioned empty map means that no earlier speaker
|
||||||
|
completion is reusable. Absence of both fields is accepted as legacy
|
||||||
|
aggregate-only state so current Narratio can read older manifests.
|
||||||
|
|
||||||
|
Each map entry represents one completed speaker only. It binds a versioned
|
||||||
|
fingerprint to the session-relative audio path, audio SHA-256 and size, one
|
||||||
|
canonical `transcript_raw` output with its path, SHA-256 and size, and the
|
||||||
|
producer run ID. Paths must be canonical and session-relative, checksums must
|
||||||
|
be lowercase SHA-256 values, and the record and output producer identities
|
||||||
|
must agree. Other stages cannot carry this transcribe-owned state.
|
||||||
|
Audio size is nonnegative; WhisperX determines whether that audio can be
|
||||||
|
transcribed. Output size remains positive and the transcript must be valid JSON.
|
||||||
|
|
||||||
|
Transcribe reconciles this collection against the current prepared audio set
|
||||||
|
and observable WhisperX semantics. Missing or corrupt regular outputs become
|
||||||
|
work; unsafe or non-regular evidence is rejected. It saves the reconciled
|
||||||
|
collection before HTTP dispatch and saves a detached candidate after each
|
||||||
|
validated completion. Only after a save succeeds is the candidate installed as
|
||||||
|
live authority. Canonical bytes produced before an unsuccessful save are not a
|
||||||
|
completion record.
|
||||||
|
|
||||||
|
Aggregate transcribe status still describes the whole invocation. A failed,
|
||||||
|
running, stale, or interrupted aggregate may therefore retain reusable speaker
|
||||||
|
records, while aggregate outputs are declared only after the current
|
||||||
|
collection is complete. Ordinary retry reconciles and reuses those records.
|
||||||
|
Force first checkpoints an empty current collection, preventing later retries
|
||||||
|
from reviving pre-force results.
|
||||||
|
|
||||||
|
Older aggregate-only transcribe success is conservatively regenerated once
|
||||||
|
when transcribe is selected because it does not bind individual outputs to the
|
||||||
|
audio bytes used. The session-manifest schema identifier remains
|
||||||
|
`narratio.session-manifest.v1`; this additive state is accepted by current
|
||||||
|
Narratio, but older strict-decoding binaries may reject manifests after these
|
||||||
|
fields have been written.
|
||||||
|
|
||||||
### Analyze-owned artifact state
|
### Analyze-owned artifact state
|
||||||
|
|
||||||
Every `analyze` stage record carries `analyze_state_version: 1` and an
|
Every `analyze` stage record carries `analyze_state_version: 1` and an
|
||||||
@@ -156,10 +195,10 @@ establish a current immutable snapshot.
|
|||||||
|
|
||||||
Manifest replacement provides atomic visibility, not a crash- or power-loss
|
Manifest replacement provides atomic visibility, not a crash- or power-loss
|
||||||
durability guarantee. Callers reload after a returned save error before retrying.
|
durability guarantee. Callers reload after a returned save error before retrying.
|
||||||
An analyze checkpoint error leaves the prior manifest as per-artifact authority.
|
A transcribe or analyze checkpoint error leaves the prior manifest collection
|
||||||
The runner then attempts to persist the bounded failed/stale retry collection;
|
as authority. The runner then attempts to persist the aggregate failure; if
|
||||||
if that terminal save also fails, its error is joined with the originating
|
that terminal save also fails, its error is joined with the originating failure
|
||||||
failure and the next invocation must reconcile from the last readable manifest.
|
and the next invocation must reconcile from the last readable manifest.
|
||||||
|
|
||||||
## Execution Semantics
|
## Execution Semantics
|
||||||
|
|
||||||
@@ -193,11 +232,11 @@ only fields returned by the new result. Marking a record stale does not clear
|
|||||||
those details because resume validation and diagnosis may still require them
|
those details because resume validation and diagnosis may still require them
|
||||||
before execution begins.
|
before execution begins.
|
||||||
|
|
||||||
Aggregate lifecycle clearing deliberately preserves the analyze-owned
|
Aggregate lifecycle clearing deliberately preserves both the transcribe-owned
|
||||||
per-artifact collection. This lets later reconciliation replace only evaluated
|
speaker collection and the analyze-owned per-artifact collection. This lets
|
||||||
entries without erasing unrelated current results. Other stages retain their
|
later reconciliation replace only non-current entries without erasing durable
|
||||||
existing aggregate-only lifecycle behavior and are forbidden from carrying the
|
completions. Other stages retain aggregate-only lifecycle behavior and are
|
||||||
analyze-specific fields.
|
forbidden from carrying either collection.
|
||||||
|
|
||||||
A stage may explicitly return a skipped disposition and stable reason. The
|
A stage may explicitly return a skipped disposition and stable reason. The
|
||||||
runner persists that outcome in the session manifest, clears older outputs,
|
runner persists that outcome in the session manifest, clears older outputs,
|
||||||
@@ -228,7 +267,7 @@ evidence and focused durable validators:
|
|||||||
| Work | Reuse authority | Focused owners |
|
| Work | Reuse authority | Focused owners |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| prepare | aggregate semantic fingerprint | [prepare](stage-prepare.md) |
|
| prepare | aggregate semantic fingerprint | [prepare](stage-prepare.md) |
|
||||||
| transcribe | aggregate semantic fingerprint | [transcribe](stage-transcribe.md), [WhisperX](../integrations/whisperx.md) |
|
| transcribe speakers | aggregate semantic fingerprint plus per-speaker input/output reconciliation | [transcribe](stage-transcribe.md), [WhisperX](../integrations/whisperx.md) |
|
||||||
| merge | aggregate semantic fingerprint | [merge](stage-merge.md), [Seriatim](../integrations/seriatim.md) |
|
| merge | aggregate semantic fingerprint | [merge](stage-merge.md), [Seriatim](../integrations/seriatim.md) |
|
||||||
| polish | aggregate semantic fingerprint | [polish](stage-polish.md), [Audita](../integrations/audita.md) |
|
| polish | aggregate semantic fingerprint | [polish](stage-polish.md), [Audita](../integrations/audita.md) |
|
||||||
| normalize | aggregate semantic fingerprint | [normalize](stage-normalize.md), [Seriatim](../integrations/seriatim.md) |
|
| normalize | aggregate semantic fingerprint | [normalize](stage-normalize.md), [Seriatim](../integrations/seriatim.md) |
|
||||||
@@ -310,6 +349,7 @@ where the last session transition can require operator interpretation.
|
|||||||
- Remote commit model and readers: `internal/artifacts/remote_commit.go`,
|
- Remote commit model and readers: `internal/artifacts/remote_commit.go`,
|
||||||
`internal/artifacts/current_state_commit.go`,
|
`internal/artifacts/current_state_commit.go`,
|
||||||
- Persistence and validation: `internal/manifest/store.go`
|
- Persistence and validation: `internal/manifest/store.go`
|
||||||
|
- Transcribe speaker state: `internal/manifest/transcribe_state.go`
|
||||||
- Package tests: `internal/manifest/*_test.go`
|
- Package tests: `internal/manifest/*_test.go`
|
||||||
- Assembled execution behavior: `internal/app/runner_test.go`,
|
- Assembled execution behavior: `internal/app/runner_test.go`,
|
||||||
`internal/app/run_stage_test.go`
|
`internal/app/run_stage_test.go`
|
||||||
|
|||||||
@@ -37,7 +37,7 @@ failed stage or required manifest save prevents later stages from starting.
|
|||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| Executable | `cmd/narratio` | Process entry, standard stream wiring, argument handoff, and exit status. |
|
| Executable | `cmd/narratio` | Process entry, standard stream wiring, argument handoff, and exit status. |
|
||||||
| Application orchestration | `internal/app` | Command dispatch, configuration selection, secret-file environment loading, production composition, session locking, planning, execution, restore, and user-facing reporting. |
|
| Application orchestration | `internal/app` | Command dispatch, configuration selection, secret-file environment loading, production composition, session locking, planning, execution, restore, and user-facing reporting. |
|
||||||
| Configuration | [`internal/config`](configuration.md) | Presence-aware root/import/profile composition, canonical party and family expansion, strict YAML loading, defaults, normalization, session templating, and validation. |
|
| Configuration | [`internal/config`](configuration.md) | ConfigKit-backed pipeline imports and profiles, canonical party and family expansion, strict YAML loading, defaults, normalization, session templating, and validation. |
|
||||||
| Pipeline stages | `internal/stage` | Canonical stage registry, shared stage contract, execution dependencies, and implemented stage behavior. |
|
| Pipeline stages | `internal/stage` | Canonical stage registry, shared stage contract, execution dependencies, and implemented stage behavior. |
|
||||||
| External boundaries | `internal/adapters`, `internal/audio` | WhisperX HTTP, downstream subprocesses, notification, object storage, and S3 audio materialization behind Narratio contracts. |
|
| External boundaries | `internal/adapters`, `internal/audio` | WhisperX HTTP, downstream subprocesses, notification, object storage, and S3 audio materialization behind Narratio contracts. |
|
||||||
| Manifests | `internal/manifest` | Durable session progress, stage transitions, validation, and atomic persistence. |
|
| Manifests | `internal/manifest` | Durable session progress, stage transitions, validation, and atomic persistence. |
|
||||||
@@ -53,11 +53,12 @@ adapters. The normative rules for these relationships remain in
|
|||||||
[Architecture](../policy/architecture.md).
|
[Architecture](../policy/architecture.md).
|
||||||
|
|
||||||
Command execution normalizes one private dependency value per invocation.
|
Command execution normalizes one private dependency value per invocation.
|
||||||
Configuration reads, secret-directory reads, remote downloads, object-store
|
Pipeline configuration-path discovery, configuration reads, secret-directory
|
||||||
construction, and restore phase functions use that value throughout the
|
reads, remote downloads, object-store construction, and restore phase functions
|
||||||
command. Public command entry points supply production defaults, while focused
|
use that value throughout the command. Public command entry points supply the
|
||||||
tests replace only the external or sequencing boundary they exercise without
|
canonical discovery and production defaults, while focused tests may provide
|
||||||
mutating package state.
|
invocation-owned discovery candidates or replace only the external or sequencing
|
||||||
|
boundary they exercise without mutating package state.
|
||||||
|
|
||||||
Pipeline execution and `session plan` share the same inclusive contiguous-range
|
Pipeline execution and `session plan` share the same inclusive contiguous-range
|
||||||
model and the same pure pre-execution decision over recorded status, force, and
|
model and the same pure pre-execution decision over recorded status, force, and
|
||||||
@@ -105,8 +106,8 @@ so any earlier replacement can stale a prior cleanup result.
|
|||||||
|
|
||||||
## Focused Documentation
|
## Focused Documentation
|
||||||
|
|
||||||
- [Configuration Internals](configuration.md): pipeline composition, import
|
- [Configuration Internals](configuration.md): pipeline imports and profiles,
|
||||||
confinement, field ownership, decoding, and root-relative path semantics.
|
field ownership, decoding, and root-relative path semantics.
|
||||||
- [Adapter Internals](adapters.md): external adapter boundaries, composition,
|
- [Adapter Internals](adapters.md): external adapter boundaries, composition,
|
||||||
failure behavior, and test surfaces.
|
failure behavior, and test surfaces.
|
||||||
- [Artifact Internals](artifacts.md): source identities, runtime catalog,
|
- [Artifact Internals](artifacts.md): source identities, runtime catalog,
|
||||||
|
|||||||
@@ -10,7 +10,9 @@ dependency order, and safely materialize validated outputs.
|
|||||||
- ordinary configured artifacts from `pipeline.scriptorium.artifacts`; canonical
|
- ordinary configured artifacts from `pipeline.scriptorium.artifacts`; canonical
|
||||||
party artifact families have already expanded into this map during
|
party artifact families have already expanded into this map during
|
||||||
configuration resolution, including corresponding member dependencies and
|
configuration resolution, including corresponding member dependencies and
|
||||||
rewritten member-artifact input sources
|
rewritten member-artifact input sources. Each artifact's dependency list is
|
||||||
|
already the normalized union of explicit ordering edges and configured-artifact
|
||||||
|
input edges.
|
||||||
- optional selected artifact keys supplied through the stage environment
|
- optional selected artifact keys supplied through the stage environment
|
||||||
- built-in, configured, extraction, and previous-session source references in
|
- built-in, configured, extraction, and previous-session source references in
|
||||||
artifact inputs
|
artifact inputs
|
||||||
@@ -51,6 +53,9 @@ Supported source families:
|
|||||||
manifest record and durable output pass the configured-artifact evidence
|
manifest record and durable output pass the configured-artifact evidence
|
||||||
contract; an incidental or stale canonical file is unavailable.
|
contract; an incidental or stale canonical file is unavailable.
|
||||||
- validates selected artifact dependency order (cycle-safe topo ordering).
|
- validates selected artifact dependency order (cycle-safe topo ordering).
|
||||||
|
- consumes only the concrete dependency graph finalized by configuration;
|
||||||
|
planning, prerequisite closure, execution, reuse, invalidation, and semantic
|
||||||
|
evidence do not infer a second graph from inputs.
|
||||||
- resolves required/optional inputs per artifact source definition into an
|
- resolves required/optional inputs per artifact source definition into an
|
||||||
ordered semantic identity. Each identity records the configured input name,
|
ordered semantic identity. Each identity records the configured input name,
|
||||||
canonical source ID, required policy, explicit presence, source contract,
|
canonical source ID, required policy, explicit presence, source contract,
|
||||||
|
|||||||
@@ -19,12 +19,17 @@ procedures belong in [Operations](../operations.md).
|
|||||||
1. resolves the final trimmed transcript from the shared artifact catalog;
|
1. resolves the final trimmed transcript from the shared artifact catalog;
|
||||||
2. resolves every configured prepared reference through the shared
|
2. resolves every configured prepared reference through the shared
|
||||||
manifest-authoritative identity resolver before creating run-local output;
|
manifest-authoritative identity resolver before creating run-local output;
|
||||||
3. streams each verified reference into an invocation-local snapshot and
|
3. resolves each reference to a stable content-addressed session snapshot,
|
||||||
rejects any source change observed while copying;
|
reusing verified bytes or reconstructing a missing/corrupt regular entry,
|
||||||
|
and rejects any source change observed while copying;
|
||||||
4. fingerprints the byte- and provenance-bearing Notarius invocation evidence,
|
4. fingerprints the byte- and provenance-bearing Notarius invocation evidence,
|
||||||
including sorted reference identities;
|
including sorted reference identities and the explicit downstream
|
||||||
|
configuration-profile selection;
|
||||||
5. creates a run-local staging directory and invokes the injected
|
5. creates a run-local staging directory and invokes the injected
|
||||||
`notarius.Runner`;
|
`notarius.Runner`, carrying the optional already-resolved credential
|
||||||
|
reference without exposing its value, the optional exact configuration
|
||||||
|
profile, plus effective resume and chunk-cache controls after force
|
||||||
|
precedence;
|
||||||
6. revalidates the reference snapshots, then validates the v2 successful
|
6. revalidates the reference snapshots, then validates the v2 successful
|
||||||
receipt, confined index, management documents, configured required lane
|
receipt, confined index, management documents, configured required lane
|
||||||
descriptors, validation summaries, and regular payload files;
|
descriptors, validation summaries, and regular payload files;
|
||||||
@@ -41,14 +46,22 @@ Narratio run ID, the resolved trimmed-input identity, and invocation
|
|||||||
fingerprint. The input identity binds the exact transcript bytes, canonical
|
fingerprint. The input identity binds the exact transcript bytes, canonical
|
||||||
source ID, producer stage/output/run identity, and resolution provenance.
|
source ID, producer stage/output/run identity, and resolution provenance.
|
||||||
Reference metadata contains only selector, source ID, canonical session-relative
|
Reference metadata contains only selector, source ID, canonical session-relative
|
||||||
path, checksum, and size; adapter requests receive selector and absolute
|
path, checksum, and size; adapter requests receive selector and absolute stable
|
||||||
invocation-local snapshot path, never payload contents. Snapshot bytes must
|
snapshot path, never payload contents. Snapshot bytes must
|
||||||
match the prepared identity both before and after Notarius runs, so a concurrent
|
match the prepared identity both before and after Notarius runs, so a concurrent
|
||||||
prepared-file replacement cannot make recorded provenance describe different
|
prepared-file replacement cannot make recorded provenance describe different
|
||||||
bytes from those supplied to Notarius.
|
bytes from those supplied to Notarius.
|
||||||
Validation completes before the single durable rename, so a rejected result
|
Validation completes before the single durable rename, so a rejected result
|
||||||
cannot expose a partial durable bundle.
|
cannot expose a partial durable bundle.
|
||||||
|
|
||||||
|
Snapshot destinations are owned by `internal/artifacts` under
|
||||||
|
`cache/notarius/references/<sha256>/<prepared-basename>`. They contain no run ID
|
||||||
|
or timestamp and can be reused across invocations. Identical bytes and basename
|
||||||
|
share a verified file; changed bytes select a different digest directory.
|
||||||
|
Unsafe path components and non-regular conflicts fail without recursive
|
||||||
|
deletion. Snapshot paths are delivery details and never become manifest output
|
||||||
|
evidence or prerequisites for reuse of an already complete extract bundle.
|
||||||
|
|
||||||
Any executed extraction outcome that replaces a different effective outcome
|
Any executed extraction outcome that replaces a different effective outcome
|
||||||
marks succeeded analysis and delivery dependents stale. Render is an independent
|
marks succeeded analysis and delivery dependents stale. Render is an independent
|
||||||
sibling and remains current. Repeating the same disabled self-skip with no
|
sibling and remains current. Repeating the same disabled self-skip with no
|
||||||
@@ -59,17 +72,28 @@ outputs is stable and does not repeatedly invalidate dependent stages.
|
|||||||
Before the focused validator runs, the application compares extract's versioned
|
Before the focused validator runs, the application compares extract's versioned
|
||||||
semantic fingerprint. It covers enablement, Notarius pipeline identity, sorted
|
semantic fingerprint. It covers enablement, Notarius pipeline identity, sorted
|
||||||
reference selector/source mappings, sorted declared output contracts, and each
|
reference selector/source mappings, sorted declared output contracts, and each
|
||||||
canonical `narratio.extraction.<key>` output identity. It excludes executable,
|
canonical `narratio.extraction.<key>` output identity. It also covers the
|
||||||
|
explicit Notarius configuration-profile selection; omitted, empty, and exact
|
||||||
|
`default` selectors share the base-selection identity. It excludes executable,
|
||||||
timeout, working directory, config path, and private Notarius config contents.
|
timeout, working directory, config path, and private Notarius config contents.
|
||||||
|
Credential presence, reference name, and resolved value are likewise
|
||||||
|
operational-only and excluded. Ordinary command loading still resolves a
|
||||||
|
selected reference before resume evaluation, but a credential-only change does
|
||||||
|
not rerun a valid completed extraction or stale its dependents.
|
||||||
|
|
||||||
`internal/stage/extract_resume.go` then permits a skip only when the existing
|
`internal/stage/extract_resume.go` then permits a skip only when the existing
|
||||||
stage record still matches the current byte- and provenance-bearing invocation
|
stage record still matches the current byte- and provenance-bearing invocation
|
||||||
evidence. That evidence covers the current direct trimmed-transcript identity,
|
evidence. That evidence covers the current direct trimmed-transcript identity,
|
||||||
sorted prepared-reference identities, pipeline identity, and configured output
|
sorted prepared-reference identities, pipeline identity, and configured output
|
||||||
contracts. The same reference helper and transcript identity are resolved again
|
contracts, plus the same downstream profile-selection identity. The same
|
||||||
for artifact evidence, so changing current transcript bytes, reference bytes,
|
reference helper and transcript identity are resolved again for artifact
|
||||||
or producer identity makes the prior extraction obsolete. Operational runner
|
evidence, so changing current transcript bytes, reference bytes, producer
|
||||||
settings do not invalidate otherwise current durable evidence.
|
identity, or a named downstream selection makes the prior extraction obsolete.
|
||||||
|
Operational runner settings do not invalidate otherwise current durable
|
||||||
|
evidence. This includes `resume` and `chunk_cache`; changing them alone cannot
|
||||||
|
make a complete extraction non-resumable. If extraction executes, ordinary
|
||||||
|
runs forward the configured values, while force disables resume and selects
|
||||||
|
`refresh`.
|
||||||
|
|
||||||
A valid prepared-reference change makes extraction non-resumable. Missing,
|
A valid prepared-reference change makes extraction non-resumable. Missing,
|
||||||
unsafe, or checksum-inconsistent prepared evidence is a hard validation error
|
unsafe, or checksum-inconsistent prepared evidence is a hard validation error
|
||||||
@@ -82,10 +106,12 @@ contracts and provenance, regular-file status, and stored checksums. Missing or
|
|||||||
obsolete results are non-resumable and run again; unsafe filesystem conditions
|
obsolete results are non-resumable and run again; unsafe filesystem conditions
|
||||||
return an error rather than silently accepting or replacing data.
|
return an error rather than silently accepting or replacing data.
|
||||||
|
|
||||||
Neither contract can observe files imported by Notarius configuration, profile
|
This sensitivity is limited to the explicit downstream selector. Narratio's
|
||||||
contents, prompt/module definitions, or other transitive inputs. Operators must
|
own profile label and whole effective-configuration digest are not blanket
|
||||||
force extraction after changing any such private input behind a stable
|
extraction invalidators. Neither contract can observe files imported by
|
||||||
identifier.
|
Notarius configuration, profile contents, prompt/module definitions, or other
|
||||||
|
transitive inputs. Operators must force extraction after changing any such
|
||||||
|
private input behind a stable identifier.
|
||||||
|
|
||||||
## Failure Behavior
|
## Failure Behavior
|
||||||
|
|
||||||
@@ -110,9 +136,13 @@ recovery.
|
|||||||
`internal/stage/extract_resume.go`,
|
`internal/stage/extract_resume.go`,
|
||||||
`internal/stage/extract_test.go`,
|
`internal/stage/extract_test.go`,
|
||||||
`internal/stage/semantic_contracts_delivery.go`
|
`internal/stage/semantic_contracts_delivery.go`
|
||||||
|
- Stable snapshot materialization and verification:
|
||||||
|
`internal/stage/extract_references.go`
|
||||||
- Subprocess boundary: `internal/adapters/notarius/subprocess.go`,
|
- Subprocess boundary: `internal/adapters/notarius/subprocess.go`,
|
||||||
`internal/adapters/notarius/subprocess_test.go`
|
`internal/adapters/notarius/subprocess_test.go`
|
||||||
- Catalog hydration: `internal/artifacts/extraction_catalog.go`,
|
- Catalog hydration: `internal/artifacts/extraction_catalog.go`,
|
||||||
`internal/artifacts/extraction_catalog_test.go`
|
`internal/artifacts/extraction_catalog_test.go`
|
||||||
- Composition and downstream behavior: `internal/app/runner_test.go`,
|
- Composition and downstream behavior: `internal/app/runner_test.go`,
|
||||||
`internal/stage/analyze_test.go`, `internal/stage/publish_test.go`
|
`internal/stage/analyze_test.go`, `internal/stage/publish_test.go`
|
||||||
|
- Opt-in Notarius v0.7.0 production-boundary coverage:
|
||||||
|
`internal/stage/notarius_cache_integration_test.go`
|
||||||
|
|||||||
@@ -17,7 +17,11 @@ Normalize raw transcript inputs and merge into base transcript via Seriatim.
|
|||||||
|
|
||||||
## Key Behavior
|
## Key Behavior
|
||||||
|
|
||||||
- discovers and validates raw transcript inputs.
|
- discovers and validates raw transcript inputs. For versioned transcribe
|
||||||
|
state, discovery requires aggregate transcribe success and consumes only the
|
||||||
|
aggregate outputs that exactly match the current per-speaker collection.
|
||||||
|
Partial checkpoints and orphaned files are not merge inputs. Legacy/manual
|
||||||
|
workflows without versioned speaker state retain directory discovery.
|
||||||
- normalizes each raw transcript (`seriatim.Normalize`) into run-local scratch output.
|
- normalizes each raw transcript (`seriatim.Normalize`) into run-local scratch output.
|
||||||
- merges normalized inputs (`seriatim.Run`) into base transcript.
|
- merges normalized inputs (`seriatim.Run`) into base transcript.
|
||||||
- validates merged transcript and optional report JSON.
|
- validates merged transcript and optional report JSON.
|
||||||
@@ -26,6 +30,8 @@ Normalize raw transcript inputs and merge into base transcript via Seriatim.
|
|||||||
## Invariants
|
## Invariants
|
||||||
|
|
||||||
- merge always consumes normalized forms of raw inputs.
|
- merge always consumes normalized forms of raw inputs.
|
||||||
|
- a versioned transcribe collection is usable only through its complete,
|
||||||
|
successful aggregate output declaration.
|
||||||
- base transcript must validate before stage success.
|
- base transcript must validate before stage success.
|
||||||
- report output is config-gated.
|
- report output is config-gated.
|
||||||
|
|
||||||
|
|||||||
@@ -35,8 +35,9 @@ validation model, module set, transcript description, output schema, selected
|
|||||||
external configuration path, and canonical polished-transcript identity. Module
|
external configuration path, and canonical polished-transcript identity. Module
|
||||||
ordering is normalized because the configured modules form a set. Audita's
|
ordering is normalized because the configured modules form a set. Audita's
|
||||||
executable, timeouts, concurrency, report and debug behavior, work retention,
|
executable, timeouts, concurrency, report and debug behavior, work retention,
|
||||||
and credential environment name are operational and do not invalidate a
|
and resolved credential value are operational and do not invalidate a
|
||||||
successful result.
|
successful result. The public credential reference name may participate in
|
||||||
|
configuration provenance but never exposes its resolved value.
|
||||||
|
|
||||||
Narratio can fingerprint a selected model, module, or configuration identifier,
|
Narratio can fingerprint a selected model, module, or configuration identifier,
|
||||||
but it cannot inspect content that Audita privately resolves behind that stable
|
but it cannot inspect content that Audita privately resolves behind that stable
|
||||||
|
|||||||
@@ -14,12 +14,19 @@ Generate raw per-speaker transcripts from prepared audio using WhisperX.
|
|||||||
|
|
||||||
## Key Behavior
|
## Key Behavior
|
||||||
|
|
||||||
- discovers prepared audio from manifest inputs or canonical audio directory.
|
- discovers prepared audio from manifest inputs or the canonical audio
|
||||||
- derives the transcript identity from the prepared `.flac` filename.
|
directory and derives each speaker identity from its `.flac` filename.
|
||||||
- dispatches one WhisperX request per prepared file through a bounded errgroup.
|
- hashes the current audio and reconciles it with the versioned per-speaker
|
||||||
- validates each output as JSON.
|
completion collection before starting HTTP work.
|
||||||
- writes run-local outputs then materializes canonical transcript outputs only
|
- reuses only records whose audio identity, observable WhisperX semantics,
|
||||||
after every planned request succeeds.
|
canonical output location, JSON, checksum, and size remain current.
|
||||||
|
- durably checkpoints the reconciled collection before dispatch, then sends
|
||||||
|
only missing or obsolete speakers through the bounded worker group.
|
||||||
|
- validates each successful run-local response, rechecks the audio bytes,
|
||||||
|
materializes the canonical raw transcript, and checkpoints that speaker
|
||||||
|
before accepting another completion.
|
||||||
|
- rebuilds aggregate stage outputs in stable speaker order from the complete
|
||||||
|
current collection, including records produced by earlier invocations.
|
||||||
|
|
||||||
## Invariants
|
## Invariants
|
||||||
|
|
||||||
@@ -28,21 +35,48 @@ Generate raw per-speaker transcripts from prepared audio using WhisperX.
|
|||||||
- output path returned by adapter must match requested output path.
|
- output path returned by adapter must match requested output path.
|
||||||
- an empty adapter result path means the requested path; adapters cannot select
|
- an empty adapter result path means the requested path; adapters cannot select
|
||||||
an alternate destination.
|
an alternate destination.
|
||||||
- each successful output is validated before stage success, and cancellation or
|
- candidate collection construction, manifest save, and installation into live
|
||||||
incomplete dispatch cannot be reported as a successful result.
|
state are serialized. Expensive hashing and HTTP work remain outside that
|
||||||
|
critical section.
|
||||||
|
- a canonical file has no completion authority until its manifest checkpoint
|
||||||
|
succeeds. A save failure preserves the last successfully committed
|
||||||
|
collection for ordinary retry.
|
||||||
|
- each successful output is validated before its checkpoint, and cancellation,
|
||||||
|
incomplete dispatch, or a failed aggregate save cannot be reported as
|
||||||
|
aggregate stage success.
|
||||||
|
- removed audio inputs are omitted from the current collection and aggregate
|
||||||
|
output list; reconciliation does not delete arbitrary transcript files.
|
||||||
|
- merge cannot consume a partial versioned collection: it requires transcribe
|
||||||
|
aggregate success and an exact match between current speaker records and
|
||||||
|
declared aggregate outputs.
|
||||||
|
|
||||||
## Resume Evidence
|
## Resume Evidence
|
||||||
|
|
||||||
Transcribe records a versioned semantic-configuration fingerprint containing
|
Each completed speaker record binds the speaker ID, session-relative audio
|
||||||
the Narratio-visible WhisperX service URL and recognition language. Changes to
|
path, streamed audio checksum and size, the Narratio-visible WhisperX service
|
||||||
either rerun transcription and stale its fixed descendants while leaving
|
URL and recognition language, and the canonical transcript path, checksum,
|
||||||
prepare reusable. Retry count/delay, concurrency, timeout, credentials, and
|
size, and producer run. Changed audio bytes select that speaker for replacement;
|
||||||
diagnostic locations are operational and do not change this evidence.
|
a changed URL or language selects every speaker. Either case stales fixed
|
||||||
|
descendants through the normal runner decision. Retry count/delay, concurrency,
|
||||||
|
timeout, credentials, and diagnostic locations are operational and do not
|
||||||
|
change this evidence.
|
||||||
|
|
||||||
|
An ordinary retry reuses every valid speaker checkpoint even when aggregate
|
||||||
|
transcribe status is failed, running, or stale. A succeeded stage is skipped only
|
||||||
|
when its stored speaker set and aggregate output evidence also match the current
|
||||||
|
collection. Otherwise, execution repairs those records without retranscribing
|
||||||
|
reusable speakers and stales dependents through the normal runner path.
|
||||||
|
`--force` first durably replaces
|
||||||
|
the collection with an empty versioned collection, so a later ordinary retry
|
||||||
|
cannot resurrect pre-force results; speakers checkpointed by that forced
|
||||||
|
attempt remain reusable.
|
||||||
|
|
||||||
WhisperX models or private service configuration not exposed by Narratio's
|
WhisperX models or private service configuration not exposed by Narratio's
|
||||||
adapter contract cannot be fingerprinted; use `--force` after changing them.
|
adapter contract cannot be fingerprinted; use `--force` after changing them.
|
||||||
An existing successful transcribe record without evidence reruns once when
|
An older aggregate-only transcribe record has no trustworthy audio-to-output
|
||||||
selected.
|
binding. When transcribe is next selected, every current speaker is therefore
|
||||||
|
generated once and recorded in the versioned collection; later invocations can
|
||||||
|
reuse it normally.
|
||||||
|
|
||||||
## Related Contracts And Tests
|
## Related Contracts And Tests
|
||||||
|
|
||||||
@@ -51,6 +85,9 @@ selected.
|
|||||||
- [Configuration](../config.md#pipeline) owns concurrency and other
|
- [Configuration](../config.md#pipeline) owns concurrency and other
|
||||||
operator-selected values.
|
operator-selected values.
|
||||||
- Implementation and tests: `internal/stage/transcribe.go`,
|
- Implementation and tests: `internal/stage/transcribe.go`,
|
||||||
|
`internal/stage/transcribe_reconciliation.go`,
|
||||||
|
`internal/stage/transcribe_execution_state.go`,
|
||||||
|
`internal/stage/transcribe_resume.go`,
|
||||||
`internal/stage/transcribe_test.go`,
|
`internal/stage/transcribe_test.go`,
|
||||||
`internal/stage/semantic_contracts_initial.go`, and
|
`internal/stage/transcribe_reconciliation_test.go`, and
|
||||||
`internal/stage/semantic_contracts_initial_test.go`
|
`internal/stage/transcribe_checkpoint_test.go`
|
||||||
|
|||||||
@@ -37,6 +37,12 @@ validates and renames the external bundle to the unique immutable Notarius
|
|||||||
bundle path supplied by `internal/artifacts`. Exact physical paths belong in
|
bundle path supplied by `internal/artifacts`. Exact physical paths belong in
|
||||||
[Operations](../operations.md#extraction-workflow).
|
[Operations](../operations.md#extraction-workflow).
|
||||||
|
|
||||||
|
Prepared references delivered to Notarius use session-durable paths owned by
|
||||||
|
`internal/artifacts` beneath
|
||||||
|
`cache/notarius/references/<sha256>/<prepared-basename>`. These verified
|
||||||
|
content-addressed snapshots are outside run-local layout, survive ordinary
|
||||||
|
published-run cleanup, and are excluded from publish and restore enumeration.
|
||||||
|
|
||||||
## Locking
|
## Locking
|
||||||
|
|
||||||
`artifacts.LocalStore` enforces the single-writer session lock via an
|
`artifacts.LocalStore` enforces the single-writer session lock via an
|
||||||
|
|||||||
@@ -76,12 +76,15 @@ value from the last value persisted in the session manifest, which helps
|
|||||||
diagnose profile switches without changing resume authority.
|
diagnose profile switches without changing resume authority.
|
||||||
|
|
||||||
Before switching an operational profile, compare its effective meaning with the
|
Before switching an operational profile, compare its effective meaning with the
|
||||||
current selection through `narratio config diff <left-profile> <right-profile>`.
|
production base through `narratio config diff default <profile>`.
|
||||||
The command is read-only and succeeds whether it finds differences or not. Its
|
The command is read-only and succeeds whether it finds differences or not. Its
|
||||||
sorted records describe defaulted, expanded concrete configuration—not source
|
sorted records describe defaulted, expanded concrete configuration—not source
|
||||||
file layout—so it can be used to review model, artifact, and publish changes
|
file layout—so it can be used to review model, artifact, and publish changes
|
||||||
without creating a session or run. Select the same campaign explicitly when
|
without creating a session or run. Use `narratio config artifacts` to review the
|
||||||
profiles could resolve different campaign paths; see the [CLI reference](cli.md#config-validate-config-show-config-sources-and-config-diff) for syntax and record format.
|
concrete artifact identities, definitions, outputs, and dependencies. Select the
|
||||||
|
same campaign explicitly when profiles could resolve different campaign paths;
|
||||||
|
see the [CLI reference](cli.md#config-validate-config-show-config-sources-config-artifacts-and-config-diff)
|
||||||
|
for syntax and record format.
|
||||||
|
|
||||||
## Stage Execution and Continuation Behavior
|
## Stage Execution and Continuation Behavior
|
||||||
|
|
||||||
@@ -113,8 +116,8 @@ Execution rules:
|
|||||||
perpetually rerun dependent work.
|
perpetually rerun dependent work.
|
||||||
|
|
||||||
Every aggregate stage except analyze currently provides semantic-configuration
|
Every aggregate stage except analyze currently provides semantic-configuration
|
||||||
evidence; analyze retains its more precise per-artifact fingerprints and
|
evidence; transcribe additionally reconciles per-speaker completion evidence,
|
||||||
validator.
|
while analyze retains its more precise per-artifact fingerprints and validator.
|
||||||
Prepare additionally validates current stable/local/S3 source identity and the
|
Prepare additionally validates current stable/local/S3 source identity and the
|
||||||
checksums of its durable prepared copies before reuse. Changed bytes, audio
|
checksums of its durable prepared copies before reuse. Changed bytes, audio
|
||||||
membership, S3 object identity, or missing/tampered copies rerun prepare and
|
membership, S3 object identity, or missing/tampered copies rerun prepare and
|
||||||
@@ -184,7 +187,68 @@ shared session lifecycle. In particular, render does not require Notarius or
|
|||||||
Scriptorium, extract does not require Scriptorium, and analyze does not require
|
Scriptorium, extract does not require Scriptorium, and analyze does not require
|
||||||
the transcription, Seriatim, Audita, or Notarius adapters.
|
the transcription, Seriatim, Audita, or Notarius adapters.
|
||||||
|
|
||||||
For the common post-transcript development loop, use:
|
### Transcription continuation and replacement
|
||||||
|
|
||||||
|
Transcribe checkpoints each validated speaker independently. If a later
|
||||||
|
speaker exhausts HTTP retries, the process is canceled, or the aggregate stage
|
||||||
|
save fails, the aggregate stage is not successful and downstream merge cannot
|
||||||
|
consume the partial set. The last successfully saved speaker records and their
|
||||||
|
canonical `transcripts/raw/<speaker>.json` files remain durable, however.
|
||||||
|
|
||||||
|
After correcting the underlying problem, retry normally:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio run-stage transcribe 2026-04-04
|
||||||
|
```
|
||||||
|
|
||||||
|
Narratio hashes the current prepared audio, validates recorded transcript JSON,
|
||||||
|
checksum, size, and location, and calls WhisperX only for missing or obsolete
|
||||||
|
speakers. A changed audio file replaces only its speaker; a changed WhisperX
|
||||||
|
URL or language makes every current speaker non-reusable. Retry, timeout, and
|
||||||
|
concurrency tuning do not invalidate completed speakers.
|
||||||
|
|
||||||
|
Use force only when every current transcript must be replaced, including after
|
||||||
|
a WhisperX model or private server-side configuration changes behind the same
|
||||||
|
URL and language:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio run-stage transcribe 2026-04-04 --force
|
||||||
|
```
|
||||||
|
|
||||||
|
Force durably clears the reusable collection before sending requests. If that
|
||||||
|
forced attempt fails, a later ordinary retry reuses only replacement speakers
|
||||||
|
checkpointed after the reset; it does not revive pre-force results. Do not edit
|
||||||
|
speaker checksums or copy older JSON into the canonical directory to influence
|
||||||
|
reuse.
|
||||||
|
|
||||||
|
The first current Narratio invocation that selects transcribe for an older
|
||||||
|
aggregate-only manifest retranscribes every current speaker once because the
|
||||||
|
old record cannot prove which audio bytes produced each file. Subsequent runs
|
||||||
|
reuse the new versioned evidence normally. The overall session-manifest schema
|
||||||
|
identifier is unchanged, but older strict-decoding Narratio binaries may reject
|
||||||
|
a manifest after current Narratio writes the additive speaker fields; retain a
|
||||||
|
backup or complete the upgrade before relying on downgrade.
|
||||||
|
|
||||||
|
Restoring a manifest does not implicitly publish or restore every raw speaker
|
||||||
|
file. If a recorded canonical transcript is absent locally, ordinary
|
||||||
|
reconciliation schedules that speaker again. Removing run directories does not
|
||||||
|
remove canonical checkpointed transcripts and does not by itself invalidate
|
||||||
|
them.
|
||||||
|
|
||||||
|
## Artifact Development Workflows
|
||||||
|
|
||||||
|
Choose the narrowest workflow that matches the intended replacement:
|
||||||
|
|
||||||
|
| Intent | Command | Preserved and rebuilt state |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Rebuild analysis only | [`narratio analyze 2026-04-04`](./cli.md#analyze) | Preserves transcripts and the current extraction bundle; force-runs selected analysis artifacts and their required analysis prerequisites. |
|
||||||
|
| Rebuild extraction and analysis | [`narratio regenerate-artifacts 2026-04-04`](./cli.md#regenerate-artifacts) | Preserves transcripts; force-runs the complete configured extraction bundle, then selected analysis. `--artifacts` narrows analysis only. |
|
||||||
|
| Continue bounded work normally | [`narratio run 2026-04-04 --from extract --through analyze`](./cli.md#run) | Uses ordinary resume decisions: current extraction and analysis can be skipped, while incomplete or obsolete work runs. |
|
||||||
|
|
||||||
|
All three stop before publication. Inspect the results and run
|
||||||
|
[`narratio publish`](./cli.md#publish) separately when delivery is intended.
|
||||||
|
|
||||||
|
For the extraction-plus-analysis development loop, use:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio regenerate-artifacts 2026-04-04
|
narratio regenerate-artifacts 2026-04-04
|
||||||
@@ -217,6 +281,30 @@ earlier checkpoints are reused and execution resumes with failed, stale, or
|
|||||||
missing work in dependency order. `--force` is unnecessary unless the operator
|
missing work in dependency order. `--force` is unnecessary unless the operator
|
||||||
intends to replace otherwise-current artifacts.
|
intends to replace otherwise-current artifacts.
|
||||||
|
|
||||||
|
### Developing With A Notarius Configuration Profile
|
||||||
|
|
||||||
|
A Narratio development overlay can explicitly select a Notarius overlay without
|
||||||
|
requiring the names to match. For example, `profiles/testing.yml` may contain:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
notarius:
|
||||||
|
config_profile: downstream-economy
|
||||||
|
```
|
||||||
|
|
||||||
|
Selecting Narratio's `testing` profile then supplies `downstream-economy` only
|
||||||
|
when extraction executes. Switching that explicit selector makes a recorded
|
||||||
|
extraction non-current and reruns it when `extract` is in the selected stage
|
||||||
|
range; the usual dependent invalidation then applies. Selecting a profile does
|
||||||
|
not create a separate workspace or retain a separate current extraction result.
|
||||||
|
|
||||||
|
Narratio cannot observe edits inside Notarius configuration behind an unchanged
|
||||||
|
selector. Force extraction after such an edit when a replacement is intended.
|
||||||
|
Use `regenerate-artifacts` when both extraction and analysis must be rebuilt;
|
||||||
|
use `analyze` when the existing extraction bundle should be preserved. Field
|
||||||
|
semantics belong in [Configuration](./config.md#notarius-configuration-profile),
|
||||||
|
and the exact downstream invocation belongs in the
|
||||||
|
[Notarius integration contract](./integrations/notarius.md#invocation).
|
||||||
|
|
||||||
## Artifact Selection
|
## Artifact Selection
|
||||||
|
|
||||||
For a configured artifact family, selecting its family key expands to every
|
For a configured artifact family, selecting its family key expands to every
|
||||||
@@ -257,9 +345,14 @@ are not selectable or published implicitly.
|
|||||||
Configured Notarius references resolve only from the current manifest-backed
|
Configured Notarius references resolve only from the current manifest-backed
|
||||||
prepared inputs. Their canonical locations are `inputs/party.yml`,
|
prepared inputs. Their canonical locations are `inputs/party.yml`,
|
||||||
`inputs/players.yml`, `inputs/glossary.yml`, and, when configured,
|
`inputs/players.yml`, `inputs/glossary.yml`, and, when configured,
|
||||||
`inputs/spell_catalog.json`. Extraction supplies Notarius with verified copies
|
`inputs/spell_catalog.json`. Extraction supplies Notarius with verified,
|
||||||
under `runs/<run_id>/extract/references/` so a concurrent refresh of canonical
|
content-addressed copies under
|
||||||
prepared files cannot change the bytes consumed by an in-flight invocation.
|
`cache/notarius/references/<sha256>/<prepared-basename>`. The full lowercase
|
||||||
|
SHA-256 digest identifies the reference bytes, so retries and later invocations
|
||||||
|
reuse the same path while changed bytes select a new path. Older snapshots are
|
||||||
|
retained and are not current artifact evidence. A concurrent refresh of
|
||||||
|
canonical prepared files therefore cannot change the bytes consumed by an
|
||||||
|
in-flight invocation.
|
||||||
For a canonical party, preparation retains the validated authored party bytes
|
For a canonical party, preparation retains the validated authored party bytes
|
||||||
at `inputs/party.yml` and generates `inputs/players.yml` from that roster.
|
at `inputs/party.yml` and generates `inputs/players.yml` from that roster.
|
||||||
The manifest records their checksums separately, with the players input marked
|
The manifest records their checksums separately, with the players input marked
|
||||||
@@ -294,6 +387,28 @@ Notarius extraction validates its run-local bundle and moves it once to the
|
|||||||
unique durable bundle path. A pre-existing durable destination is a collision;
|
unique durable bundle path. A pre-existing durable destination is a collision;
|
||||||
no duplicate promotion tree is created.
|
no duplicate promotion tree is created.
|
||||||
|
|
||||||
|
`pipeline.notarius.resume` and `pipeline.notarius.chunk_cache` control work
|
||||||
|
inside an executed Notarius invocation. Ordinary extraction forwards their
|
||||||
|
configured values. Forced extraction always disables checkpoint loading and
|
||||||
|
requests chunk-plan `refresh`, even when the configured mode is `bypass`.
|
||||||
|
Refresh regenerates and replaces only the active input's accepted chunk plan;
|
||||||
|
it is not cache deletion.
|
||||||
|
|
||||||
|
Notarius checkpoint recording must be enabled in the deployed Notarius
|
||||||
|
configuration. Narratio neither creates a Notarius pipeline/configuration nor
|
||||||
|
reads it to infer checkpoint availability. Configure explicit persistent
|
||||||
|
Notarius chunk-plan and checkpoint roots when reuse must survive separate
|
||||||
|
Narratio subprocesses. A changed reference, transcript, selected profile,
|
||||||
|
profile content, endpoint, or other Notarius checkpoint identity input can
|
||||||
|
produce a legitimate cold miss.
|
||||||
|
|
||||||
|
There are two independent resume boundaries: Narratio may skip a completed
|
||||||
|
`extract` stage without invoking Notarius, while `--resume` allows Notarius to
|
||||||
|
reuse compatible accepted work only after Narratio has decided extraction must
|
||||||
|
execute. Deleting Narratio's local reference snapshots does not invalidate a
|
||||||
|
complete durable extraction; missing snapshots are reconstructed only when a
|
||||||
|
later extraction actually runs.
|
||||||
|
|
||||||
## External Command Lifecycle
|
## External Command Lifecycle
|
||||||
|
|
||||||
When an external command is cancelled or times out, Narratio terminates its
|
When an external command is cancelled or times out, Narratio terminates its
|
||||||
@@ -334,13 +449,13 @@ narratio run-stage extract 2026-04-04 --force
|
|||||||
Narratio automatically reruns extraction when its recorded invocation contract,
|
Narratio automatically reruns extraction when its recorded invocation contract,
|
||||||
prepared Narratio reference identities, or durable output validation changes.
|
prepared Narratio reference identities, or durable output validation changes.
|
||||||
The semantic portion covers Notarius enablement, pipeline identity, declared
|
The semantic portion covers Notarius enablement, pipeline identity, declared
|
||||||
reference mapping, and output contracts. Executable, timeout, working directory,
|
reference mapping, output contracts, and the explicit Notarius configuration
|
||||||
and private config-file paths are operational and do not invalidate a current
|
profile selector. Executable, timeout, working directory, and private
|
||||||
result.
|
config-file paths are operational and do not invalidate a current result.
|
||||||
It cannot fingerprint configuration files, profiles, prompts, modules, or
|
It cannot fingerprint configuration or profile contents, prompts, modules, or
|
||||||
other references loaded transitively by Notarius itself. Force extraction after
|
other references loaded transitively by Notarius itself. Force extraction after
|
||||||
changing any of those inputs, even when the top-level Narratio and Notarius
|
changing any of those inputs behind an unchanged selector, even when the
|
||||||
config paths remain the same. A forced extract
|
top-level Narratio and Notarius config paths remain the same. A forced extract
|
||||||
marks successful downstream stages stale. Ordinary extraction failures or
|
marks successful downstream stages stale. Ordinary extraction failures or
|
||||||
outcome changes also stale affected downstream stages, while an identical
|
outcome changes also stale affected downstream stages, while an identical
|
||||||
repeated `notarius_disabled` self-skip does not repeatedly invalidate them.
|
repeated `notarius_disabled` self-skip does not repeatedly invalidate them.
|
||||||
@@ -506,6 +621,7 @@ Durable session paths:
|
|||||||
- `logs/**`
|
- `logs/**`
|
||||||
- `config/**`
|
- `config/**`
|
||||||
- `runs/**`
|
- `runs/**`
|
||||||
|
- `cache/notarius/references/<sha256>/<prepared-basename>`
|
||||||
|
|
||||||
`manifest.json` is the versioned session-progress document. Narratio accepts
|
`manifest.json` is the versioned session-progress document. Narratio accepts
|
||||||
only `narratio.session-manifest.v1`. Do not edit unsupported session state to
|
only `narratio.session-manifest.v1`. Do not edit unsupported session state to
|
||||||
@@ -516,6 +632,13 @@ relevant workflow requires.
|
|||||||
Validated Notarius bundles live below `artifacts/notarius/{run_id}/`; receipt,
|
Validated Notarius bundles live below `artifacts/notarius/{run_id}/`; receipt,
|
||||||
stderr, and pre-promotion output remain in the producing run's `extract`
|
stderr, and pre-promotion output remain in the producing run's `extract`
|
||||||
directory as described in [Extraction Workflow](#extraction-workflow).
|
directory as described in [Extraction Workflow](#extraction-workflow).
|
||||||
|
Stable reference snapshots live in the session cache path above. Ordinary
|
||||||
|
published-run cleanup removes its selected run-local target but retains this
|
||||||
|
cache, and publish/restore never enumerate it as an artifact. Remove a snapshot
|
||||||
|
or digest directory manually only while no command is active for the session;
|
||||||
|
regular missing or corrupt content is reconstructed on the next executed
|
||||||
|
extraction, while symlinks, directories, and other unsafe conflicts must be
|
||||||
|
investigated and removed deliberately.
|
||||||
|
|
||||||
Run-local layout:
|
Run-local layout:
|
||||||
|
|
||||||
@@ -550,18 +673,28 @@ converge on `0664`. Narratio explicitly applies these modes so a restrictive
|
|||||||
caller umask does not remove group write or setgid. It does not change file or
|
caller umask does not remove group write or setgid. It does not change file or
|
||||||
directory ownership: the configured workspace's existing group is inherited.
|
directory ownership: the configured workspace's existing group is inherited.
|
||||||
|
|
||||||
Windows does not implement POSIX mode bits or setgid semantics. Configure the
|
|
||||||
workspace, spool, and cache locations with an ACL that grants the collaborating
|
|
||||||
group read/write access, and configure credential locations with an ACL limited
|
|
||||||
to the intended credential owner. Do not use POSIX mode displays as evidence of
|
|
||||||
Windows access control.
|
|
||||||
|
|
||||||
API keys are credentials, not ordinary workspace data. Store them outside the
|
API keys are credentials, not ordinary workspace data. Store them outside the
|
||||||
shared workspace or in a separately restricted credential location; ordinary
|
shared workspace or in a separately restricted credential location; ordinary
|
||||||
workspace group access must never be treated as authorization to read keys.
|
workspace group access must never be treated as authorization to read keys.
|
||||||
On POSIX, provision a credential directory as `0700` and credential files as
|
Provision a credential directory as `0700` and credential files as `0600`;
|
||||||
`0600`; Narratio rejects group- or other-readable configured credential paths.
|
these permission modes are an operator requirement and are not enforced by
|
||||||
On Windows, restrict the directory and files with ACLs to the credential owner.
|
ConfigKit.
|
||||||
|
|
||||||
|
Every pipeline-loading command bootstraps credentials once, before the first
|
||||||
|
configuration load and before any adapter is constructed. `--secrets-dir`
|
||||||
|
selects an explicit directory (relative paths use caller CWD). Otherwise a
|
||||||
|
present `secrets/` beside the resolved root pipeline is used, while an absent
|
||||||
|
conventional directory is skipped. Explicit missing/empty selections and
|
||||||
|
invalid existing directories fail the command. ConfigKit follows a symlink used
|
||||||
|
as the selected directory, but immediate entries must be regular non-symlink
|
||||||
|
files. Existing environment variables—including empty values—always win.
|
||||||
|
|
||||||
|
S3 `access_key_id` and `secret_access_key` are an optional pair of typed
|
||||||
|
environment references. When both are absent, the AWS SDK retains its default
|
||||||
|
credential provider chain. When both are configured, Narratio supplies their
|
||||||
|
load-time resolved values through an explicit static provider. Audita's optional
|
||||||
|
`llm_api_key` is likewise captured at pipeline load and supplied only through
|
||||||
|
the child process's `AUDITA_LLM_API_KEY` override.
|
||||||
|
|
||||||
External adapter results are individually bounded before Narratio validates or
|
External adapter results are individually bounded before Narratio validates or
|
||||||
materializes them. These per-file limits do not reserve disk space: prevent hard
|
materializes them. These per-file limits do not reserve disk space: prevent hard
|
||||||
@@ -624,4 +757,5 @@ normal migration path.
|
|||||||
- Publish requires prerequisite stages through `render` and `analyze` to be succeeded.
|
- Publish requires prerequisite stages through `render` and `analyze` to be succeeded.
|
||||||
- Markdown publish defaults require render outputs (`transcripts/final.md` and `transcripts/final.trimmed.md`).
|
- Markdown publish defaults require render outputs (`transcripts/final.md` and `transcripts/final.trimmed.md`).
|
||||||
- Restore requires configured object storage and committed remote current state.
|
- Restore requires configured object storage and committed remote current state.
|
||||||
- Storage-backed commands load filesystem secrets before object-store initialization.
|
- Pipeline-loading commands bootstrap filesystem secrets once before configuration
|
||||||
|
resolution and object-store initialization.
|
||||||
|
|||||||
@@ -146,7 +146,7 @@ Configuration is strict, explicit, centralized, and operator-oriented.
|
|||||||
- YAML decoding rejects unknown fields.
|
- YAML decoding rejects unknown fields.
|
||||||
- Defaults are centralized and testable.
|
- Defaults are centralized and testable.
|
||||||
- Empty configured values do not silently replace meaningful defaults.
|
- Empty configured values do not silently replace meaningful defaults.
|
||||||
- Validation rejects invalid composition before stage execution where
|
- Validation rejects invalid configuration before stage execution where
|
||||||
practical.
|
practical.
|
||||||
- Root-owned imports and one selected profile resolve deterministically through
|
- Root-owned imports and one selected profile resolve deterministically through
|
||||||
the configuration owner; commands do not implement their own merge rules.
|
the configuration owner; commands do not implement their own merge rules.
|
||||||
@@ -234,20 +234,26 @@ configuration, logs, reports, diagnostics, and Notarius artifacts—is
|
|||||||
intentionally shareable with the configured workspace group. API-key material
|
intentionally shareable with the configured workspace group. API-key material
|
||||||
is sensitive and is not covered by the ordinary workspace-sharing policy.
|
is sensitive and is not covered by the ordinary workspace-sharing policy.
|
||||||
|
|
||||||
On POSIX systems, Narratio-created ordinary workspace directories converge on
|
Narratio supports Linux on amd64 and arm64. macOS arm64 is best effort: it has
|
||||||
|
a cross-build gate and release asset, but no native runtime release gate.
|
||||||
|
Windows and macOS amd64 are unsupported and have no compatibility promise or
|
||||||
|
release asset.
|
||||||
|
|
||||||
|
On supported systems, Narratio-created ordinary workspace directories converge on
|
||||||
setgid `02775` and ordinary workspace files on `0664`, even when the caller's
|
setgid `02775` and ordinary workspace files on `0664`, even when the caller's
|
||||||
umask is restrictive. This preserves the existing workspace group for nested
|
umask is restrictive. This preserves the existing workspace group for nested
|
||||||
creation and atomic replacements without changing ownership. API-key storage
|
creation and atomic replacements without changing ownership. API-key storage
|
||||||
uses a separate restrictive contract. On Windows, POSIX mode bits and setgid
|
uses a separate restrictive contract.
|
||||||
are not authoritative; operators must provide the equivalent shared-group and
|
|
||||||
credential-restricted ACLs described in [Operations](../operations.md#workspace-permissions).
|
|
||||||
|
|
||||||
Raw secrets must not be stored in pipeline, campaign, or session YAML or written
|
Raw secrets must not be stored in pipeline, campaign, or session YAML or written
|
||||||
to manifests, logs, generated configuration, reports, publish metadata,
|
to manifests, logs, generated configuration, reports, publish metadata,
|
||||||
documentation, or examples. Secrets enter through configured environment
|
documentation, or examples. Pipeline configuration names environment variables;
|
||||||
variable names or secret-file references. Diagnostics should avoid transcript
|
command-owned startup may populate previously unset variables from ConfigKit's
|
||||||
and prompt content unless a deliberate, bounded inspection mechanism requires
|
explicit or conventional secrets directory. This bootstrap occurs once per
|
||||||
it.
|
invocation before any typed configuration load or adapter construction and is
|
||||||
|
not hidden inside library loaders. Safe bootstrap diagnostics may identify
|
||||||
|
files and statuses but never values. Diagnostics should avoid transcript and
|
||||||
|
prompt content unless a deliberate, bounded inspection mechanism requires it.
|
||||||
|
|
||||||
Logs, reports, generated invocation files, generated configuration, and render
|
Logs, reports, generated invocation files, generated configuration, and render
|
||||||
debug files are diagnostics, not canonical pipeline products. They should be
|
debug files are diagnostics, not canonical pipeline products. They should be
|
||||||
|
|||||||
@@ -58,14 +58,14 @@ scripts/check-release-candidate.sh vMAJOR.MINOR.PATCH
|
|||||||
|
|
||||||
It validates the version and matching note, module hygiene, formatting,
|
It validates the version and matching note, module hygiene, formatting,
|
||||||
whitespace, uncached tests, race tests, static checks, documentation, examples,
|
whitespace, uncached tests, race tests, static checks, documentation, examples,
|
||||||
and six official cross-build assets. It uses `GOWORK=off`, does not contact
|
and three official cross-build assets. It uses `GOWORK=off`, does not contact
|
||||||
application services, CI, or Gitea, and does not create tags or modify tracked
|
application services, CI, or Gitea, and does not create tags or modify tracked
|
||||||
source. Fix any failure on `main`, commit it, push it normally, and rerun the
|
source. Fix any failure on `main`, commit it, push it normally, and rerun the
|
||||||
checker.
|
checker.
|
||||||
|
|
||||||
The checker builds Linux, macOS, and Windows assets for `amd64` and `arm64`.
|
The checker builds Linux assets for `amd64` and `arm64` and a macOS asset for
|
||||||
Cross-builds prove compilation; they are not native macOS or Windows runtime
|
`arm64`. The macOS cross-build proves compilation; it is not native macOS
|
||||||
evidence.
|
runtime evidence.
|
||||||
|
|
||||||
## Publish the tag
|
## Publish the tag
|
||||||
|
|
||||||
|
|||||||
@@ -22,5 +22,7 @@ See the [release procedure](../release.md) for creating a candidate and tag.
|
|||||||
When asynchronous publication succeeds, the corresponding Gitea release is the
|
When asynchronous publication succeeds, the corresponding Gitea release is the
|
||||||
canonical source for downloadable binaries and checksums.
|
canonical source for downloadable binaries and checksums.
|
||||||
|
|
||||||
|
- [v2.0.0](v2.0.0.md)
|
||||||
|
- [v1.7.0](v1.7.0.md)
|
||||||
- [v1.6.0](v1.6.0.md)
|
- [v1.6.0](v1.6.0.md)
|
||||||
- [v1.5.0](v1.5.0.md)
|
- [v1.5.0](v1.5.0.md)
|
||||||
|
|||||||
94
docs/releases/v2.0.0.md
Normal file
94
docs/releases/v2.0.0.md
Normal file
@@ -0,0 +1,94 @@
|
|||||||
|
# Narratio v2.0.0
|
||||||
|
|
||||||
|
Narratio v2.0.0 replaces its configuration infrastructure with ConfigKit,
|
||||||
|
simplifies the supported release platforms, and makes artifact definitions and
|
||||||
|
their effective dependency graph directly inspectable.
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
Pipeline roots now use ConfigKit v0.1.2 for strict YAML loading, explicit
|
||||||
|
imports, fixed-path named profiles, provenance, defaults, and typed credential
|
||||||
|
references. The recommended production layout keeps production values in the
|
||||||
|
base, defines each ordinary artifact and artifact family in its own imported
|
||||||
|
file, and keeps testing changes in `profiles/testing.yml`.
|
||||||
|
|
||||||
|
Configured-artifact inputs now create their dependency edges automatically.
|
||||||
|
`narratio config artifacts` reports the finalized concrete catalog, including
|
||||||
|
generated party-family members, without loading a session or initializing
|
||||||
|
runtime adapters.
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
This is a breaking configuration and platform release. There is no runtime
|
||||||
|
fallback for the retired Narratio composition/profile catalog, `pipeline.secrets`
|
||||||
|
or `secrets.env_dir`, credential `_env` fields, or the previous YAML
|
||||||
|
presence/default behavior. Defaults apply only to absent values; explicit empty,
|
||||||
|
zero, false, and null values retain their typed meaning and may fail Narratio
|
||||||
|
domain validation.
|
||||||
|
|
||||||
|
The maintained binaries are Linux amd64, Linux arm64, and macOS arm64. Windows
|
||||||
|
and macOS amd64 builds are no longer produced or supported. Existing v1 release
|
||||||
|
notes remain historical descriptions of their releases.
|
||||||
|
|
||||||
|
Inspection commands still avoid session, manifest, workspace, adapter, and
|
||||||
|
remote-service activity, but they are not credential-free loaders: every
|
||||||
|
explicit typed credential reference in the selected effective configuration must
|
||||||
|
resolve; references replaced or removed by that profile need not resolve.
|
||||||
|
Pipelines containing artifact families also require a campaign so the canonical
|
||||||
|
party can be expanded.
|
||||||
|
`config show` is a secret-safe review projection, not a promised reloadable
|
||||||
|
authoring file.
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
Migrate each operational bundle before running v2:
|
||||||
|
|
||||||
|
1. Replace old composition syntax with root-owned
|
||||||
|
[`configkit.imports`](../config.md#pipeline-imports-and-profiles). Imports are
|
||||||
|
additive base sources; they do not override one another.
|
||||||
|
2. Fold the old production overlay into the base pipeline. Move testing changes
|
||||||
|
to `profiles/testing.yml`; compare the two independent resolutions with
|
||||||
|
`narratio config diff default testing`.
|
||||||
|
3. Follow the maintained
|
||||||
|
[`examples/production-testing`](../../examples/production-testing/pipeline.yml)
|
||||||
|
layout: one explicit artifact entry per `artifacts/<key>.yml` and one family
|
||||||
|
entry per `artifact-families/<key>.yml`. The map key, not the filename, is the
|
||||||
|
stable artifact identity.
|
||||||
|
4. Move credential files to an untracked `secrets/` directory beside
|
||||||
|
`pipeline.yml`, or pass `--secrets-dir`. Rename S3 fields to `access_key_id`
|
||||||
|
and `secret_access_key`, and rename the Audita field to `llm_api_key`.
|
||||||
|
5. For static S3 credentials, configure both S3 references (commonly
|
||||||
|
`OBJECT_STORAGE_KEY_ID` and `OBJECT_STORAGE_KEY`) and make them resolvable.
|
||||||
|
To use instance roles, workload identity, shared SDK files, or another AWS SDK
|
||||||
|
discovery provider, omit both references. A partial pair is invalid.
|
||||||
|
6. Remove a `depends_on` entry when it only duplicates a
|
||||||
|
`narratio.artifact.<key>` input. Keep explicit entries that express ordering
|
||||||
|
without supplying the upstream output as an input.
|
||||||
|
7. Review ConfigKit's stricter explicit-value/null semantics and correct values
|
||||||
|
that previously acquired defaults after being explicitly empty. Do not use
|
||||||
|
`config show` output as a source migration mechanism.
|
||||||
|
8. Run `config validate`, `config show`, `config sources`, and `config artifacts`
|
||||||
|
with the same profile, campaign, and secrets selection intended for runtime.
|
||||||
|
Resolve the campaign-owned party whenever families are present.
|
||||||
|
9. Update deployment and download automation to one of the three maintained
|
||||||
|
platform assets before removing the v1 binary.
|
||||||
|
|
||||||
|
The [configuration reference](../config.md), [CLI reference](../cli.md), and
|
||||||
|
[maintained examples](../../examples/README.md) are the canonical detailed
|
||||||
|
guides.
|
||||||
|
|
||||||
|
## Changes
|
||||||
|
|
||||||
|
- Adopted ConfigKit v0.1.2 across pipeline, campaign, session, and party loading
|
||||||
|
while keeping Narratio domain validation and post-campaign expansion explicit.
|
||||||
|
- Added ConfigKit imports and fixed `profiles/<name>.yml` pipeline overlays,
|
||||||
|
unified safe effective show/diff/digest projection, and source provenance.
|
||||||
|
- Added invocation-scoped ConfigKit secrets-directory bootstrap and typed S3 and
|
||||||
|
Audita credential references without exposing resolved values.
|
||||||
|
- Inferred, normalized, and validated configured-artifact input dependencies as
|
||||||
|
part of the one concrete graph consumed by analyze planning and resume logic.
|
||||||
|
- Added `config artifacts` with stable concrete identities, definition sources,
|
||||||
|
effective dependencies, and generated family/party provenance.
|
||||||
|
- Added the maintained production-base/testing-overlay artifact authoring layout.
|
||||||
|
- Reduced release assets and active platform support to Linux amd64/arm64 and
|
||||||
|
macOS arm64.
|
||||||
70
docs/releases/v2.1.0.md
Normal file
70
docs/releases/v2.1.0.md
Normal file
@@ -0,0 +1,70 @@
|
|||||||
|
# Narratio v2.1.0
|
||||||
|
|
||||||
|
This release adds explicit Notarius credential forwarding and cache reuse,
|
||||||
|
with predictable forced regeneration and clearer artifact-development guidance.
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
Narratio can pass one typed API-key reference to Notarius and opt into reuse of
|
||||||
|
its accepted execution checkpoints. Stable, verified reference snapshots allow
|
||||||
|
unchanged references to retain their cache identity across Narratio invocations.
|
||||||
|
Configuration loading now uses ConfigKit v0.3.0.
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
The new `notarius.llm_api_key`, `notarius.resume`, and `notarius.chunk_cache`
|
||||||
|
fields are optional. Existing v2 configuration needs no schema migration.
|
||||||
|
Configured credential references must resolve during configuration loading,
|
||||||
|
even when extraction will not execute; omitted references add no requirement.
|
||||||
|
|
||||||
|
Forced extraction now omits Notarius checkpoint loading and requests chunk-plan
|
||||||
|
refresh. This includes `regenerate-artifacts`, which remains an alias for a
|
||||||
|
forced run from extract through analyze. `analyze` continues to force analysis
|
||||||
|
only, preserving extraction results and transcripts.
|
||||||
|
|
||||||
|
Notarius v0.7.0 is the tested cache integration baseline, not an unconditional
|
||||||
|
minimum-version gate. The executable must support the flags selected for the
|
||||||
|
invocation, including `--chunk_cache refresh` for forced extraction. Reference
|
||||||
|
bindings retain the contract introduced in Notarius v0.6.0. Receipt and artifact
|
||||||
|
schemas are unchanged; reused producers may report zero production attempts.
|
||||||
|
|
||||||
|
Supported release platforms remain Linux amd64/arm64 and best-effort macOS
|
||||||
|
arm64. macOS validation is cross-compilation, not native runtime testing.
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
No special migration is required for existing v2 configuration. To use the new
|
||||||
|
features:
|
||||||
|
|
||||||
|
- Declare `notarius.llm_api_key` with the environment-variable name expected by
|
||||||
|
the deployed Notarius profile, supplying its value through the existing
|
||||||
|
environment or secrets-directory mechanism.
|
||||||
|
- Enable checkpoint recording in Notarius configuration before setting
|
||||||
|
Narratio's `notarius.resume: true`. Prefer explicit persistent Notarius cache
|
||||||
|
roots for service deployments. An optional `notarius.chunk_cache` overrides
|
||||||
|
its ordinary chunk-plan policy.
|
||||||
|
- Expect an initial checkpoint miss when moving from old run-local reference
|
||||||
|
paths to stable session snapshots. No migration or deletion of old caches is
|
||||||
|
required. Force refreshes the current input's plan, not the whole cache.
|
||||||
|
- Use `narratio analyze SESSION` for analysis-only iteration, and
|
||||||
|
`narratio regenerate-artifacts SESSION` when extraction should also be rebuilt.
|
||||||
|
|
||||||
|
See [Configuration](../config.md), the
|
||||||
|
[Notarius integration contract](../integrations/notarius.md), and
|
||||||
|
[Artifact Development Workflows](../operations.md#artifact-development-workflows)
|
||||||
|
for current details.
|
||||||
|
|
||||||
|
## Changes
|
||||||
|
|
||||||
|
- Upgraded ConfigKit from v0.1.2 to v0.3.0 and isolated pipeline-discovery tests
|
||||||
|
from machine configuration.
|
||||||
|
- Added optional typed Notarius credentials, exact-name environment delivery,
|
||||||
|
safe configuration projections, and captured-stream redaction for custom names.
|
||||||
|
- Added Notarius resume and chunk-cache controls with force taking precedence.
|
||||||
|
- Added verified, content-addressed session reference snapshots that survive
|
||||||
|
ordinary run cleanup without becoming published artifacts.
|
||||||
|
- Preserved relative-workspace support for snapshot delivery and confinement.
|
||||||
|
- Kept credentials and cache-policy controls outside extract's semantic resume
|
||||||
|
identity, so operational changes alone do not invalidate completed results.
|
||||||
|
- Added an opt-in real Notarius integration test using a local provider to
|
||||||
|
verify reuse, partial-failure recovery, and forced refresh without paid calls.
|
||||||
33
docs/releases/v2.1.1.md
Normal file
33
docs/releases/v2.1.1.md
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
# Narratio v2.1.1
|
||||||
|
|
||||||
|
This release fixes configuration loading for publish rules that explicitly set
|
||||||
|
requiredness while relying on a derived destination.
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
Publish rules with a derivable source and omitted `dest` now load correctly
|
||||||
|
whether `required` is omitted, explicitly true, or explicitly false. Previously,
|
||||||
|
explicit requiredness could trigger ConfigKit's `resolve.unrecorded_change`
|
||||||
|
diagnostic instead of accepting valid configuration.
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
This is a backward-compatible bookkeeping correction. Publish schemas,
|
||||||
|
destination derivation, requiredness defaults, and YAML syntax support are
|
||||||
|
unchanged. Explicit destinations and explicit false values remain intact.
|
||||||
|
No ConfigKit upgrade or publication/storage behavior change is involved.
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
No special upgrade action or configuration migration is required. Operators
|
||||||
|
need not add a derivable `dest`, remove explicit `required` values, or change
|
||||||
|
flow-style YAML to block-style YAML. See the
|
||||||
|
[configuration reference](../config.md#publish-configuration-summary).
|
||||||
|
|
||||||
|
## Changes
|
||||||
|
|
||||||
|
- Recorded derived publish destinations at ConfigKit's atomic sequence boundary
|
||||||
|
while retaining field-level resolution records.
|
||||||
|
- Added real load/validation and safe effective-configuration regression coverage
|
||||||
|
for built-in and configured-artifact sources, explicit requiredness, and an
|
||||||
|
explicit destination that must remain unchanged.
|
||||||
26
docs/releases/v2.1.2.md
Normal file
26
docs/releases/v2.1.2.md
Normal file
@@ -0,0 +1,26 @@
|
|||||||
|
# Narratio v2.1.2
|
||||||
|
|
||||||
|
This release allows secrets directories to contain hidden metadata without
|
||||||
|
preventing credential loading.
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
ConfigKit v0.3.1 ignores dot-prefixed entries such as `.git`, `.gitignore`, and
|
||||||
|
`.DS_Store` during secrets bootstrap. These entries no longer block otherwise
|
||||||
|
valid credentials from loading and are omitted from bootstrap reports.
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
This is a backward-compatible dependency bug fix. Configuration schemas,
|
||||||
|
existing-environment precedence, and validation of non-hidden entries are
|
||||||
|
unchanged. Hidden entries still count toward the directory entry limit.
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
No special upgrade action or configuration migration is required. See the
|
||||||
|
[credential configuration reference](../config.md) for secrets-directory usage.
|
||||||
|
|
||||||
|
## Changes
|
||||||
|
|
||||||
|
- Updated ConfigKit from v0.3.0 to v0.3.1.
|
||||||
|
- Clarified hidden-entry handling in the secrets-directory documentation.
|
||||||
53
docs/releases/v2.2.0.md
Normal file
53
docs/releases/v2.2.0.md
Normal file
@@ -0,0 +1,53 @@
|
|||||||
|
# Narratio v2.2.0
|
||||||
|
|
||||||
|
This release adds resumable per-speaker transcription, replayable WhisperX
|
||||||
|
uploads, and explicit Notarius configuration profile selection.
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
Successfully checkpointed speaker transcripts survive later transcription
|
||||||
|
failures. Ordinary retry reuses valid completed speakers and processes only the
|
||||||
|
remaining work. WhisperX uploads can now be replayed by the HTTP transport
|
||||||
|
without buffering whole audio files.
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
Existing CLI commands, speaker identities, transcript formats, and pipeline
|
||||||
|
ordering are unchanged. Session manifests gain optional per-speaker completion
|
||||||
|
evidence. Older manifests remain readable, but older Narratio binaries may
|
||||||
|
reject manifests containing the new fields; downgrading after transcription
|
||||||
|
is not transparent.
|
||||||
|
|
||||||
|
The optional `pipeline.notarius.config_profile` field selects a downstream
|
||||||
|
Notarius configuration profile independently of Narratio's own profile. Using
|
||||||
|
it requires a Notarius version supporting `--config-profile`; v0.7.0 is a known
|
||||||
|
supporting version. Omitting the field retains downstream default selection.
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
No configuration migration is required. When transcribe is next selected for
|
||||||
|
an older aggregate-only manifest, all current speakers are transcribed once
|
||||||
|
to establish reusable evidence. Previously unrecorded results from failed runs
|
||||||
|
are not automatically imported.
|
||||||
|
|
||||||
|
Use ordinary run/resume to retain valid checkpoints. Forcing transcribe resets
|
||||||
|
reuse of prior results; an ordinary retry after a failed forced invocation can
|
||||||
|
retain replacements already checkpointed by that attempt. See the
|
||||||
|
[operations guide](../operations.md) and
|
||||||
|
[Notarius profile configuration](../config.md#notarius-configuration-profile).
|
||||||
|
|
||||||
|
## Changes
|
||||||
|
|
||||||
|
- Persist validated speaker completions independently of aggregate stage status,
|
||||||
|
with serialized saves for concurrent work and input/output integrity checks.
|
||||||
|
- Rebuild aggregate transcript references when speakers are removed or those
|
||||||
|
references need repair, without retranscribing otherwise-current speakers.
|
||||||
|
- Keep removed and incomplete speaker results out of versioned merge inputs.
|
||||||
|
- Permit checkpoint evidence for zero-byte audio when WhisperX accepts it;
|
||||||
|
service rejection still fails normally.
|
||||||
|
- Provide fresh streaming multipart bodies for HTTP transport replay while
|
||||||
|
preserving application retries, cancellation, and resource cleanup. This
|
||||||
|
addresses the missing-body-replay limitation, not persistent upstream
|
||||||
|
protocol failures or exactly-once service execution.
|
||||||
|
- Pass explicit Notarius configuration profiles through the CLI and include
|
||||||
|
their selection in extraction reuse decisions.
|
||||||
102
docs/roadmap/feature-request-notarius-config-profile.yml
Normal file
102
docs/roadmap/feature-request-notarius-config-profile.yml
Normal file
@@ -0,0 +1,102 @@
|
|||||||
|
---
|
||||||
|
title: Select a Notarius configuration profile through Narratio
|
||||||
|
status: proposed
|
||||||
|
consumer: rakestraw.dnd Ansible collection and its pipeline role
|
||||||
|
|
||||||
|
use_case: >-
|
||||||
|
The Ansible pipeline role deploys Narratio and Notarius as independently
|
||||||
|
configured applications. Operators select Narratio's testing profile to
|
||||||
|
develop prompts and other pipeline assets with inexpensive models while
|
||||||
|
retaining production settings in the base configuration. That selection
|
||||||
|
currently cannot select a Notarius configuration profile, leaving extraction
|
||||||
|
on its production settings.
|
||||||
|
|
||||||
|
desired_outcome: >-
|
||||||
|
Narratio's testing overlay can explicitly select Notarius's testing overlay.
|
||||||
|
Notarius owns the resulting model choices and any other testing-specific
|
||||||
|
settings. The operator selects testing once when invoking Narratio, without
|
||||||
|
editing production files or separately invoking Notarius.
|
||||||
|
|
||||||
|
rationale: >-
|
||||||
|
Both applications already use ConfigKit configuration overlays. Selecting
|
||||||
|
the downstream overlay preserves each application's configuration ownership
|
||||||
|
and supports future Notarius testing adjustments without adding a Narratio
|
||||||
|
configuration field for every downstream setting. This is explicit profile
|
||||||
|
composition, not a requirement for arbitrary recursive orchestration.
|
||||||
|
|
||||||
|
requested_interface:
|
||||||
|
field: pipeline.notarius.config_profile
|
||||||
|
value: Optional Notarius ConfigKit configuration profile name
|
||||||
|
supplied: >-
|
||||||
|
Invoke Notarius with --config-profile and the selected name when extraction
|
||||||
|
executes. The setting is available in Narratio's base configuration and
|
||||||
|
its ordinary configuration overlays.
|
||||||
|
omitted_or_null: >-
|
||||||
|
Do not supply a profile-selection flag. Preserve existing Notarius
|
||||||
|
configuration selection behavior.
|
||||||
|
downstream_semantics: >-
|
||||||
|
Retain Notarius's existing selector semantics, including the default
|
||||||
|
selector for its base configuration. Notarius resolves profiles relative
|
||||||
|
to its own configuration root, not Narratio's root.
|
||||||
|
explicit_selection: >-
|
||||||
|
Do not automatically forward Narratio's profile name. The selected
|
||||||
|
Notarius profile may have the same name or a different name.
|
||||||
|
distinction: >-
|
||||||
|
This selects a ConfigKit configuration overlay through --config-profile,
|
||||||
|
not a PromptKit LLM profile through --llm-profile. The Notarius overlay
|
||||||
|
can itself select the desired LLM profiles using native Notarius settings.
|
||||||
|
|
||||||
|
illustrative_consumer_configuration:
|
||||||
|
note: >-
|
||||||
|
These are separate files managed by the Ansible suite, not a proposed
|
||||||
|
combined configuration document. The Notarius pipeline and LLM profile
|
||||||
|
names are illustrative; the referenced LLM profile must be available to
|
||||||
|
Notarius.
|
||||||
|
narratio_profiles_testing_yml: |
|
||||||
|
notarius:
|
||||||
|
config_profile: testing
|
||||||
|
notarius_profiles_testing_yml: |
|
||||||
|
pipelines:
|
||||||
|
dnd-session:
|
||||||
|
llm_profile: gemini-flash-latest
|
||||||
|
operator_workflow: >-
|
||||||
|
Select --profile testing on the ordinary Narratio invocation. Narratio
|
||||||
|
selects Notarius's testing configuration overlay when it runs extraction;
|
||||||
|
other Narratio testing overrides continue to apply normally.
|
||||||
|
|
||||||
|
observable_behavior:
|
||||||
|
- >-
|
||||||
|
Notarius owns overlay discovery, composition, validation, and the effective
|
||||||
|
downstream settings. A missing or invalid selected overlay must not silently
|
||||||
|
fall back to production or be reported as successful extraction.
|
||||||
|
- >-
|
||||||
|
Changing the selected Notarius configuration profile must not silently
|
||||||
|
reuse Narratio's completed extraction from the previous selection.
|
||||||
|
Existing downstream invalidation behavior applies when extraction changes.
|
||||||
|
- >-
|
||||||
|
Editing Notarius overlay contents behind an unchanged selector retains
|
||||||
|
the existing operator responsibility to force extraction when needed.
|
||||||
|
This request does not require inspecting or tracking private Notarius files.
|
||||||
|
- >-
|
||||||
|
Existing credential forwarding remains independently configurable through
|
||||||
|
notarius.llm_api_key, including in the Narratio testing overlay when the
|
||||||
|
selected Notarius settings require a different credential reference.
|
||||||
|
|
||||||
|
consumer_follow_up: >-
|
||||||
|
After Narratio releases this interface, the standalone Narratio Ansible role
|
||||||
|
can expose an optional variable and omit the field when unset. The suite role
|
||||||
|
can deploy Notarius's testing overlay and select it from Narratio's testing
|
||||||
|
overlay, then distribute the changes through the collection. These consumer
|
||||||
|
updates are separate from this upstream feature request.
|
||||||
|
|
||||||
|
scope_boundaries:
|
||||||
|
- No Notarius change is currently required; v0.7.0 supports --config-profile.
|
||||||
|
- No separate Narratio-side Notarius LLM-profile selector is requested.
|
||||||
|
- No automatic profile-name propagation or downstream configuration rewriting.
|
||||||
|
- No new credential discovery, copying, or provider-name remapping.
|
||||||
|
- This request specifies consumer behavior, not internal Narratio code changes.
|
||||||
|
|
||||||
|
references:
|
||||||
|
narratio_configuration: ../config.md
|
||||||
|
narratio_notarius_integration: ../integrations/notarius.md
|
||||||
|
notarius_cli: https://gitea.maximumdirect.net/eric/notarius/src/tag/v0.7.0/docs/cli.md
|
||||||
171
docs/roadmap/fix-publish-output-destination-resolution.md
Normal file
171
docs/roadmap/fix-publish-output-destination-resolution.md
Normal file
@@ -0,0 +1,171 @@
|
|||||||
|
# Bugfix Request: Preserve Publish Destination Derivation with Explicit Rules
|
||||||
|
|
||||||
|
Status: Implemented; release target v2.1.1
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
Narratio should successfully derive a publish destination when a valid
|
||||||
|
`publish.outputs` rule omits `dest`, regardless of whether that rule explicitly
|
||||||
|
sets `required`. Narratio v2.1.0 currently rejects this valid configuration in
|
||||||
|
some cases with a ConfigKit `resolve.unrecorded_change` diagnostic.
|
||||||
|
|
||||||
|
This is a configuration-resolution bookkeeping defect, not a YAML formatting
|
||||||
|
restriction. The fix should preserve the existing public configuration
|
||||||
|
contract: operators may omit a derivable destination and may independently set
|
||||||
|
`required` to either `true` or `false`.
|
||||||
|
|
||||||
|
## Observed Behavior
|
||||||
|
|
||||||
|
A pipeline containing a rule equivalent to the following fails during native
|
||||||
|
configuration loading:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
publish:
|
||||||
|
outputs: [{"source": "narratio.transcript.final", "required": true}]
|
||||||
|
```
|
||||||
|
|
||||||
|
Narratio reports:
|
||||||
|
|
||||||
|
```text
|
||||||
|
error [resolve.unrecorded_change] /publish/outputs: application resolution changed a value without a covering record
|
||||||
|
```
|
||||||
|
|
||||||
|
The same failure is possible with ordinary block-style YAML when `required` is
|
||||||
|
explicit and `dest` is omitted. Flow-style YAML is valid input and is not the
|
||||||
|
cause of the defect.
|
||||||
|
|
||||||
|
The existing destination-derivation tests do not expose this case because
|
||||||
|
their rules also omit `required`. Narratio then supplies the default value for
|
||||||
|
`required` and records a change covering the complete `publish.outputs`
|
||||||
|
sequence. That broader record happens to cover the destination mutation as
|
||||||
|
well, masking the missing resolution record that occurs when `required` is
|
||||||
|
already explicit.
|
||||||
|
|
||||||
|
## Expected Contract
|
||||||
|
|
||||||
|
The current configuration contract remains unchanged:
|
||||||
|
|
||||||
|
- `publish.outputs[].source` is required;
|
||||||
|
- `publish.outputs[].dest` may be omitted when Narratio can derive it from the
|
||||||
|
selected built-in or configured artifact source;
|
||||||
|
- `publish.outputs[].required` defaults to `true`, but an operator may set it
|
||||||
|
explicitly to either `true` or `false`; and
|
||||||
|
- equivalent valid YAML representations have equivalent configuration
|
||||||
|
semantics.
|
||||||
|
|
||||||
|
For example, both rules below should load and validate, retain their explicit
|
||||||
|
requiredness, and receive the canonical derived destination:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
publish:
|
||||||
|
outputs:
|
||||||
|
- source: narratio.transcript.final
|
||||||
|
required: true
|
||||||
|
- source: narratio.transcript.final_markdown
|
||||||
|
required: false
|
||||||
|
```
|
||||||
|
|
||||||
|
## Root Cause
|
||||||
|
|
||||||
|
ConfigKit intentionally treats sequences as atomic when comparing values
|
||||||
|
before and after application resolution. Changing a field within one
|
||||||
|
`publish.outputs` entry is therefore observed as a change to the
|
||||||
|
`/publish/outputs` sequence.
|
||||||
|
|
||||||
|
Narratio currently derives each missing destination by recording a
|
||||||
|
normalization at the individual `publish.outputs[index].dest` path. That child
|
||||||
|
record does not cover the atomic parent-sequence change. A separate
|
||||||
|
container-level record is currently created only when Narratio also defaults
|
||||||
|
an omitted `required` value, so configurations with explicit `required` values
|
||||||
|
lose that incidental coverage.
|
||||||
|
|
||||||
|
## Recommended Fix
|
||||||
|
|
||||||
|
Make Narratio's publish-output resolution bookkeeping conform to ConfigKit's
|
||||||
|
atomic-sequence contract. Whenever resolution derives one or more destinations
|
||||||
|
inside `publish.outputs`, the recorded resolution changes must cover the
|
||||||
|
`/publish/outputs` container. Destination derivation must not depend on an
|
||||||
|
unrelated `required` default being applied in the same list.
|
||||||
|
|
||||||
|
The implementation may retain more specific provenance where useful, but it
|
||||||
|
must provide valid container-level coverage for the sequence mutation. Do not
|
||||||
|
change ConfigKit's sequence comparison semantics merely to accommodate this
|
||||||
|
call site.
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
- Correct destination derivation for publish rules that explicitly set
|
||||||
|
`required` and omit a derivable `dest`.
|
||||||
|
- Preserve explicit `required: true` and `required: false` values.
|
||||||
|
- Preserve canonical destination derivation for built-in transcript sources
|
||||||
|
and configured artifact sources.
|
||||||
|
- Add focused regression coverage through Narratio's real configuration load
|
||||||
|
and validation boundary.
|
||||||
|
- Prepare a backward-compatible patch release after the fix passes the normal
|
||||||
|
repository validation.
|
||||||
|
|
||||||
|
## Non-goals
|
||||||
|
|
||||||
|
- Changing the `publish.outputs` schema, defaults, source vocabulary, or
|
||||||
|
destination derivation rules.
|
||||||
|
- Requiring operators or configuration generators to supply `dest` when it is
|
||||||
|
derivable.
|
||||||
|
- Requiring operators to omit explicit `required: true` values.
|
||||||
|
- Treating flow-style YAML or JSON-compatible YAML values as unsupported.
|
||||||
|
- Changing ConfigKit's documented atomic treatment of sequences.
|
||||||
|
- Adding storage, publication, or network integration behavior.
|
||||||
|
|
||||||
|
## Regression Coverage
|
||||||
|
|
||||||
|
Add one focused package-level behavioral test that loads and validates a
|
||||||
|
pipeline with omitted destinations and explicit requiredness. A representative
|
||||||
|
list should include both `required: true` and `required: false`, then assert
|
||||||
|
that:
|
||||||
|
|
||||||
|
- loading and validation succeed;
|
||||||
|
- each destination is derived from its source; and
|
||||||
|
- each explicit `required` value is preserved.
|
||||||
|
|
||||||
|
Using the compact flow-style representation that exposed the integration
|
||||||
|
failure is reasonable, while the existing block-style cases continue to cover
|
||||||
|
omitted `required`. The test should exercise the public configuration behavior,
|
||||||
|
not ConfigKit call choreography or Narratio's private recording mechanism.
|
||||||
|
|
||||||
|
No network service, object store, credential, or test double is required.
|
||||||
|
|
||||||
|
## Target End State
|
||||||
|
|
||||||
|
All valid publish rules with derivable sources behave consistently across
|
||||||
|
these cases:
|
||||||
|
|
||||||
|
| `dest` | `required` | Result |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| omitted | omitted | destination derived; required defaults to `true` |
|
||||||
|
| omitted | `true` | destination derived; explicit `true` retained |
|
||||||
|
| omitted | `false` | destination derived; explicit `false` retained |
|
||||||
|
| explicit | either value | explicit destination and requiredness retained |
|
||||||
|
|
||||||
|
`narratio config validate`, `narratio config show`, and ordinary pipeline
|
||||||
|
loading accept the corrected cases without an unrecorded-change diagnostic.
|
||||||
|
No operator configuration migration or workaround is required.
|
||||||
|
|
||||||
|
## Release
|
||||||
|
|
||||||
|
This is a backward-compatible correction to documented configuration behavior
|
||||||
|
and should be released as Narratio `v2.1.1`. The release note should identify
|
||||||
|
the corrected explicit-required/derived-destination interaction and state that
|
||||||
|
no special upgrade action is required.
|
||||||
|
|
||||||
|
## Acceptance Criteria
|
||||||
|
|
||||||
|
- The focused regression test fails against v2.1.0 behavior and passes after
|
||||||
|
the fix.
|
||||||
|
- Publish rules with explicit `required: true` or `required: false` and omitted
|
||||||
|
derivable destinations load and validate successfully.
|
||||||
|
- Derived destinations and explicit requiredness appear correctly in the safe
|
||||||
|
effective configuration.
|
||||||
|
- Existing destination derivation, publish validation, imports, profiles, and
|
||||||
|
configuration inspection tests remain green.
|
||||||
|
- The complete repository validation and release-candidate checks pass.
|
||||||
|
- The correction is published as the immutable `v2.1.1` tag with the required
|
||||||
|
release note.
|
||||||
431
docs/roadmap/implementation.md
Normal file
431
docs/roadmap/implementation.md
Normal file
@@ -0,0 +1,431 @@
|
|||||||
|
# Implementation Plan: Speaker Checkpoints and Replayable Uploads
|
||||||
|
|
||||||
|
Status: Implementation complete; Stages 1-7 complete.
|
||||||
|
|
||||||
|
## Scope and Instructions
|
||||||
|
|
||||||
|
Implement [the feature roadmap](transcribe-checkpoints-and-replay.md). This plan
|
||||||
|
replaces the completed Notarius-profile plan; that work remains in Git history.
|
||||||
|
Complete the numbered stages in order, one bounded implementation prompt per
|
||||||
|
stage for gpt-5.6-terra. Record completion and actual verification results here.
|
||||||
|
Intermediate stages are not a finished feature and must not be released alone.
|
||||||
|
|
||||||
|
Before working, read `AGENTS.md`, `docs/development.md`, all three policies in
|
||||||
|
`docs/policy/`, the roadmap, and the stage-specific preparation below. Use the
|
||||||
|
maintained Go 1.26 toolchain. Preserve operator choices, speaker filenames,
|
||||||
|
bounded concurrency, streaming, and existing credentials/settings. Do not add
|
||||||
|
new commands, dependencies, service requirements, arbitrary speaker limits,
|
||||||
|
strict transcript schemas, or a generic checkpoint framework. No release or
|
||||||
|
upstream service work is part of this plan.
|
||||||
|
|
||||||
|
## Settled Design Decisions
|
||||||
|
|
||||||
|
### Progress Representation and Recovery
|
||||||
|
|
||||||
|
Add optional `transcribe_state_version` (version 1) and `transcribe_speakers`
|
||||||
|
to `manifest.StageRecord`. The latter is a map keyed by the existing derived
|
||||||
|
speaker ID, containing completed results only. Absence means that work must be
|
||||||
|
done; a separate pending/running/failed state machine is unnecessary. Aggregate
|
||||||
|
stage status/error remains authoritative for invocation failure. A versioned
|
||||||
|
empty collection is meaningful: no earlier completions are reusable.
|
||||||
|
|
||||||
|
Use a typed `TranscribeSpeakerRecord` with fingerprint version/digest,
|
||||||
|
session-relative audio path, audio SHA-256 and size, one `ArtifactRecord` output
|
||||||
|
of kind `transcript_raw`, output size, and producer run ID. The output carries
|
||||||
|
its session-relative canonical path and SHA-256. No new transcript source IDs,
|
||||||
|
synthetic schema contracts, timestamps, or historical attempt collections are
|
||||||
|
needed. Existing aggregate diagnostics can retain attempt/status/duration data
|
||||||
|
for work performed in the current invocation; do not fabricate that information
|
||||||
|
for reused outputs.
|
||||||
|
|
||||||
|
Keep the session-manifest schema identifier unchanged for this optional additive
|
||||||
|
state; validate supported per-stage versions and retain strict decoding. New
|
||||||
|
Narratio reads old manifests. Old binaries may reject the added fields under
|
||||||
|
their existing strict decoding; document this downgrade limitation.
|
||||||
|
|
||||||
|
For pre-feature manifests with no speaker evidence, regenerate transcription
|
||||||
|
once when transcribe is next selected, even if the aggregate status succeeded.
|
||||||
|
Old records do not bind individual transcript results to the audio bytes they
|
||||||
|
used. Do not guess that association from current files, old `per_file` metadata,
|
||||||
|
or abandoned run directories. This is the conservative upgrade path, not a new
|
||||||
|
general migration system. No other stage or unselected range is forcibly run.
|
||||||
|
|
||||||
|
### Reuse and Persistence
|
||||||
|
|
||||||
|
Compute a versioned per-speaker fingerprint from speaker ID, session-relative
|
||||||
|
audio identity, actual audio SHA-256/size, and the existing transcribe semantic
|
||||||
|
configuration fingerprint (URL and language). Do not include invocation ID,
|
||||||
|
absolute workspace root, concurrency, retry settings, timeout, or credentials.
|
||||||
|
Stream input hashing with cancellation; do not trust stored input checksums
|
||||||
|
alone. Recheck input bytes before accepting a new result to catch changes
|
||||||
|
observed during transcription; fail that job without checkpointing mismatched
|
||||||
|
evidence. This is not a hostile-filesystem snapshot or concurrent-editor
|
||||||
|
transaction guarantee and requires no extra audio copies or file locks.
|
||||||
|
|
||||||
|
Validate reusable output identity, canonical location, JSON, checksum, and size.
|
||||||
|
Use existing confinement/file primitives. Missing or corrupt regular outputs
|
||||||
|
become work; unsafe paths/non-regular conflicts follow existing safety errors,
|
||||||
|
not recursive deletion. Missing required audio is an input error, not permission
|
||||||
|
to reuse an unverifiable transcript. Removed inputs are removed from the current
|
||||||
|
collection and aggregate outputs without deleting arbitrary files.
|
||||||
|
|
||||||
|
One stage-owned mutex serializes candidate collection construction, manifest
|
||||||
|
save, and installation of committed state. Save a detached candidate and only
|
||||||
|
then update live authority, following analyze's pattern. Workers must not mutate
|
||||||
|
the live manifest outside that serialized section. Keep expensive HTTP work
|
||||||
|
outside the mutex. Stop dispatch/acceptance on checkpoint failure, join workers
|
||||||
|
before return, and preserve the last successfully committed collection.
|
||||||
|
|
||||||
|
Before launching requests, durably checkpoint the reconciled collection. For
|
||||||
|
force, that collection is empty: this prevents later ordinary retries from
|
||||||
|
resurrecting pre-force outputs. If this initial save fails, do not start requests
|
||||||
|
or overwrite outputs. Force again clears the current collection again; ordinary
|
||||||
|
retry retains replacements already checkpointed after the previous force.
|
||||||
|
|
||||||
|
Save each validated result at its canonical raw-transcript location before
|
||||||
|
checkpointing it. Run-local files remain staging/diagnostics, not reuse
|
||||||
|
dependencies. Bytes left by an unsuccessful checkpoint have no completion
|
||||||
|
authority. A worker may checkpoint a completed result using the parent stage
|
||||||
|
context even if a sibling canceled the work context; parent cancellation or a
|
||||||
|
checkpoint failure may prevent that save. Guarantee retention of committed
|
||||||
|
work, not all responses that happened to finish during cancellation.
|
||||||
|
|
||||||
|
### Multipart Replay Mechanism
|
||||||
|
|
||||||
|
Replace the pipe/producer upload mechanism with a small pull-based multipart
|
||||||
|
body backed by the audio reader. Use `mime/multipart.Writer` to encode a bounded
|
||||||
|
metadata prefix and suffix around the file payload: capture the prefix after
|
||||||
|
`CreateFormFile`, then encode the language field and closing boundary as the
|
||||||
|
suffix. Generate this framing once per application attempt. Each body combines
|
||||||
|
fresh readers for prefix, freshly opened audio, and suffix using `io.MultiReader`.
|
||||||
|
No full-audio buffer or temporary multipart spool is needed.
|
||||||
|
|
||||||
|
The initial body and `Request.GetBody` use the same factory/framing. Each owns
|
||||||
|
an independent audio handle. Give bodies idempotent Close and context-triggered
|
||||||
|
closure (for example `context.AfterFunc`), without holding a mutex across a
|
||||||
|
blocking Read. Read/Close coordination must release a blocked source and retain
|
||||||
|
genuine source-read/close errors while not treating deliberate early closure as
|
||||||
|
source failure. The attempt owns cleanup of all created bodies, including
|
||||||
|
replays; reject new body creation after its context expires.
|
||||||
|
|
||||||
|
Remove the old producer `Wait` dependency. Track body-local source errors with
|
||||||
|
synchronization so a swallowed upload read error cannot become a successful
|
||||||
|
transcript, and a deliberately closed original body cannot poison a successful
|
||||||
|
replay. Keep the outer retry policy and per-attempt context. Do not add replay
|
||||||
|
counters to the user retry budget or depend on Go's exact internal retry count.
|
||||||
|
Preserve ordinary HTTP redirect policy; body replay can also enable Go's
|
||||||
|
body-preserving redirects. Do not introduce a new redirect restriction.
|
||||||
|
|
||||||
|
## Stage 1: Add Durable Speaker Completion State
|
||||||
|
|
||||||
|
Status: Complete.
|
||||||
|
|
||||||
|
Completion marker (2026-09-21): Added optional versioned transcribe-owned
|
||||||
|
per-speaker completion records, strict load/save validation, isolated collection
|
||||||
|
cloning, legacy aggregate-only compatibility, and lifecycle-preservation
|
||||||
|
coverage. Verified with `go test ./internal/manifest` and `git diff --check`.
|
||||||
|
|
||||||
|
Preparation: `internal/manifest/manifest.go`, `store.go`, `analyze_state.go`,
|
||||||
|
their tests, and `docs/internal/manifest.md`.
|
||||||
|
|
||||||
|
1. Implement the additive fields, record type, version constants, validation,
|
||||||
|
and deep-copy helper described above in a focused transcribe state file.
|
||||||
|
2. Validate ownership by the transcribe stage, supported versions, required
|
||||||
|
completed-record evidence, output kind, safe relative paths, SHA-256 values,
|
||||||
|
sizes, and producer identity consistency. Use existing speaker IDs as-is;
|
||||||
|
do not apply analysis-artifact key regexes or copy unrelated analysis limits.
|
||||||
|
3. Integrate validation with both load and save. Preserve speaker state through
|
||||||
|
running/failed/stale transitions and aggregate result clearing. Generic stage
|
||||||
|
invalidation does not discard checkpoints; reconciliation checks their data.
|
||||||
|
4. Test round-trip populated and versioned-empty collections, old manifests
|
||||||
|
without fields, malformed evidence, deep-copy isolation, and lifecycle
|
||||||
|
preservation. Avoid imposing JSON formatting or map order as contracts.
|
||||||
|
|
||||||
|
Verify: `go test ./internal/manifest`.
|
||||||
|
|
||||||
|
Done when durable storage can represent partial completion independently of
|
||||||
|
aggregate status, without activating new stage behavior yet.
|
||||||
|
|
||||||
|
## Stage 2: Implement Read-Only Speaker Reconciliation
|
||||||
|
|
||||||
|
Status: Complete.
|
||||||
|
|
||||||
|
Completion marker (2026-09-21): Extracted shared deterministic transcribe job
|
||||||
|
construction and added read-only, cancellation-aware per-speaker input and
|
||||||
|
output reconciliation. Covered unchanged reuse, semantic and operational
|
||||||
|
settings, changed/added/removed inputs, legacy and forced work, missing/corrupt
|
||||||
|
outputs, unsafe evidence, and missing audio without connecting runtime skip or
|
||||||
|
checkpoint behavior. Verified with
|
||||||
|
`go test ./internal/stage ./internal/manifest` and `git diff --check`.
|
||||||
|
|
||||||
|
Preparation: `internal/stage/transcribe.go`, `semantic_contracts_initial.go`,
|
||||||
|
`analyze_reconciliation.go`, `extract_resume.go`, their tests, and the artifact
|
||||||
|
path/integrity helpers they use.
|
||||||
|
|
||||||
|
1. Extract the existing audio discovery/job construction into a shared,
|
||||||
|
transcribe-local reconciliation path. Preserve current input selection and
|
||||||
|
speaker identity behavior. Build sorted jobs and classify completed results
|
||||||
|
against the current input set and evidence defined above.
|
||||||
|
2. Add pure fingerprint construction and streamed input/output inspection.
|
||||||
|
Resolve manifest input paths with existing session-relative/absolute reading
|
||||||
|
helpers; do not change input precedence or assume the current working directory
|
||||||
|
is the session root. Newly written checkpoint paths are always relative.
|
||||||
|
3. Return current records and work-to-do without writing files or manifest state.
|
||||||
|
Force selects all work and an empty starting collection; old unversioned
|
||||||
|
transcribe state supplies no reusable records. Do not connect aggregate skip
|
||||||
|
validation until execution can produce the new evidence in Stage 3.
|
||||||
|
4. Test unchanged reuse; one changed audio among unchanged speakers; URL/language
|
||||||
|
changes; operational-setting exclusions; added/removed speakers; missing and
|
||||||
|
corrupt regular outputs; unsafe record paths; missing audio; and force.
|
||||||
|
Prefer compact tables and real temporary files over mocked hashing.
|
||||||
|
|
||||||
|
Verify: `go test ./internal/stage ./internal/manifest`.
|
||||||
|
|
||||||
|
Done when one read-only decision path describes exactly which speakers need
|
||||||
|
work, without altering existing runtime behavior prematurely.
|
||||||
|
|
||||||
|
## Stage 3: Execute and Checkpoint Speakers Incrementally
|
||||||
|
|
||||||
|
Status: Complete.
|
||||||
|
|
||||||
|
Completion marker (2026-09-21): Connected reconciliation to bounded execution,
|
||||||
|
durably checkpointed the starting and per-speaker collections through detached
|
||||||
|
manifest candidates, preserved committed work across failures and cancellation,
|
||||||
|
rebuilt deterministic aggregate outputs from reusable evidence, and added
|
||||||
|
speaker-aware resume validation. Updated production-style fixtures and covered
|
||||||
|
serialized concurrent saves, force/retry replacement, input rechecks, save
|
||||||
|
failures, partial recovery, and aggregate reconstruction. Verified with
|
||||||
|
`go test ./internal/stage ./internal/app`, `go test -race ./internal/stage`, and
|
||||||
|
`git diff --check`.
|
||||||
|
|
||||||
|
Preparation: `internal/stage/transcribe.go`, `analyze_execution_state.go`,
|
||||||
|
`run_local.go`,
|
||||||
|
`internal/app/runner.go`, and transcribe/checkpoint test fixtures.
|
||||||
|
|
||||||
|
1. Wire reconciliation into Run, require the existing ManifestStore collaborator,
|
||||||
|
and checkpoint the starting collection before launching HTTP work. Implement
|
||||||
|
detached candidate saves and serialized live-state updates. Audit Store.Save's
|
||||||
|
normalization side effects so a failed candidate save cannot mutate shared
|
||||||
|
authority; use detached records/collections wherever it may write.
|
||||||
|
2. Retain bounded errgroup execution and fail-fast cancellation. Check the work
|
||||||
|
context before beginning each queued request. After success, validate adapter
|
||||||
|
output path/JSON, recheck audio evidence, materialize canonical bytes, compute
|
||||||
|
output evidence, and checkpoint the new record. Check a checkpoint-failure
|
||||||
|
latch under the serialization lock before accepting further completions.
|
||||||
|
3. Preserve committed successes on all error/cancellation exits, including
|
||||||
|
starting-save, canonical-write, per-speaker-save, and aggregate-save failures.
|
||||||
|
Return meaningful speaker/error context and let the runner retain its existing
|
||||||
|
terminal failure behavior. Do not report partial aggregate outputs as success.
|
||||||
|
4. Assemble final outputs in deterministic speaker order from the reconciled
|
||||||
|
completed records, including reused speakers, with original producer IDs and
|
||||||
|
checksums. If all records are current but aggregate status is failed/running,
|
||||||
|
finish without WhisperX calls and rebuild the aggregate success.
|
||||||
|
5. Implement `transcribeStage.ValidateResume` using the same read-only
|
||||||
|
reconciliation, so a succeeded stage cannot skip changed audio or invalid
|
||||||
|
transcripts. Retain the existing URL/language semantic check and runner
|
||||||
|
invalidation behavior; no whole-config digest invalidation.
|
||||||
|
6. Adapt existing tests that require no canonical partial files: the new contract
|
||||||
|
permits checkpointed files but forbids aggregate success. Update fixtures to
|
||||||
|
provide real manifest stores and records rather than a production fallback
|
||||||
|
that silently disables checkpointing.
|
||||||
|
7. Test A/B completion followed by C failure and D unattempted (concurrency 1),
|
||||||
|
reload and retry only C/D; concurrently finishing speakers without lost saves;
|
||||||
|
failure and cancellation after a checkpoint; save failure retaining earlier
|
||||||
|
authority; and force failure followed by ordinary retry without old-result reuse.
|
||||||
|
|
||||||
|
Verify: `go test ./internal/stage ./internal/app` and
|
||||||
|
`go test -race ./internal/stage`.
|
||||||
|
|
||||||
|
Done when incremental progress and aggregate success work together safely under
|
||||||
|
ordinary, concurrent, canceled, and forced execution.
|
||||||
|
|
||||||
|
## Stage 4: Verify Runner, Merge, and Durable Lifecycle Integration
|
||||||
|
|
||||||
|
Status: Complete.
|
||||||
|
|
||||||
|
Completion marker (2026-09-22): Added real-persistence application coverage for
|
||||||
|
cross-invocation retry, interrupted running state, failed aggregate saves,
|
||||||
|
force/reset persistence, downstream invalidation, and one-time legacy upgrade.
|
||||||
|
Versioned merge input discovery now requires aggregate success and an exact
|
||||||
|
match between current speaker evidence and declared aggregate outputs, while
|
||||||
|
legacy/manual directory fallback remains available. Restore coverage preserves
|
||||||
|
session-relative speaker paths, leaves unpublished raw transcripts absent, and
|
||||||
|
canonical transcripts remain reusable after run-directory cleanup. Verified
|
||||||
|
with `go test ./internal/app ./internal/stage ./internal/manifest`,
|
||||||
|
`go test -race ./internal/app`, and `git diff --check`.
|
||||||
|
|
||||||
|
Preparation: `internal/app/runner.go`, `runner_runtime.go`,
|
||||||
|
`analyze_checkpoint_recovery_test.go`, `runner_terminal_failure_test.go`,
|
||||||
|
`restore_manifest_paths.go`, `internal/stage/merge.go`, and existing publish,
|
||||||
|
restore, and cleanup tests routed from `docs/internal/overview.md`.
|
||||||
|
|
||||||
|
1. Add application-level tests with real manifest persistence and fake WhisperX:
|
||||||
|
failed stage across separate invocations; all speakers committed before a
|
||||||
|
failed aggregate save; interrupted running state; force/reset persistence;
|
||||||
|
and unchanged upstream/succeeded downstream invalidation behavior. Inject
|
||||||
|
errors at durable boundaries, not every private method call.
|
||||||
|
2. Close `discoverRawTranscripts`' directory-scan fallback for versioned
|
||||||
|
transcribe state: incomplete aggregate state is not usable, and completed
|
||||||
|
state consumes only declared current aggregate outputs. Keep legacy/manual
|
||||||
|
fallback behavior for manifests without the new state rather than banning
|
||||||
|
previously allowed standalone workflows. Test that partial files and removed
|
||||||
|
speaker files cannot leak into a versioned merge, including direct stage calls.
|
||||||
|
3. Audit manifest copies/projections, remote manifest publication, restored path
|
||||||
|
handling, and cleanup. Preserve relative speaker paths on restore and verify
|
||||||
|
they resolve under the destination session, not the producer machine. Reuse
|
||||||
|
existing confinement validation, not a second path-policy layer.
|
||||||
|
4. Add focused round-trip coverage through existing publish/restore fixtures for
|
||||||
|
speaker records. Restored outputs absent from the published selection are
|
||||||
|
non-reusable locally; do not silently expand remote publication to every raw
|
||||||
|
speaker or upload audio as a new default. Removing run directories must not
|
||||||
|
remove canonical checkpointed transcripts or make reuse require run files.
|
||||||
|
5. Verify old succeeded manifests are readable and rerun once when transcribe is
|
||||||
|
selected, then reuse normally. Keep old manifests usable for other commands
|
||||||
|
and bounded ranges. Do not automatically import old unrecorded JSON files.
|
||||||
|
|
||||||
|
Verify: `go test ./internal/app ./internal/stage ./internal/manifest` and
|
||||||
|
`go test -race ./internal/app`.
|
||||||
|
|
||||||
|
Done when the new authority survives runner and storage lifecycles and cannot
|
||||||
|
expose incomplete transcripts downstream.
|
||||||
|
|
||||||
|
## Stage 5: Implement Replayable Streaming Multipart Bodies
|
||||||
|
|
||||||
|
Status: Complete.
|
||||||
|
|
||||||
|
Completion marker (2026-09-22): Replaced the producer-goroutine/pipe upload
|
||||||
|
with bounded standard-library multipart framing and independently opened,
|
||||||
|
pull-based streaming bodies shared by the initial request and `GetBody`.
|
||||||
|
Bodies are tracked, idempotently closed, canceled with their attempt context,
|
||||||
|
and retain genuine read/close failures without treating an intentionally
|
||||||
|
abandoned original as a replay failure. Added focused coverage for identical
|
||||||
|
framing, partial-read restart, simultaneous handles, source failures, and
|
||||||
|
blocked-reader release. Verified with `go test ./internal/adapters/whisperx`,
|
||||||
|
`go test -race ./internal/adapters/whisperx` (including ten repeated runs), and
|
||||||
|
`git diff --check`.
|
||||||
|
|
||||||
|
Preparation: `internal/adapters/whisperx/http.go`, `http_test.go`,
|
||||||
|
`docs/integrations/whisperx.md`, and the installed Go net/http Request/GetBody
|
||||||
|
contract. Keep this change inside the WhisperX adapter.
|
||||||
|
|
||||||
|
1. Implement the pull-based framing/body factory described above, using standard
|
||||||
|
multipart encoding rather than hand-escaped MIME headers. Add no new payload
|
||||||
|
cap or upload-size setting. Preserve file-before-language wire semantics and
|
||||||
|
the existing form names.
|
||||||
|
2. Install the initial body and GetBody with the same per-attempt boundary.
|
||||||
|
Track and close all opened bodies even if a custom transport returns early;
|
||||||
|
make cleanup/context callbacks safe against overlapping reads and closes.
|
||||||
|
A GetBody failure must be returned, not converted into an empty upload.
|
||||||
|
3. Replace pipe-specific Wait/error paths in doTranscribeAttempt with source-error
|
||||||
|
handling appropriate to pull bodies. Genuine source errors remain visible;
|
||||||
|
intentional closure during transport retry is not itself a source error.
|
||||||
|
Keep response bounds, JSON validation, output installation, retry
|
||||||
|
classification/delay, cancellation, and application attempt reporting intact.
|
||||||
|
4. Remove obsolete producer-only helpers rather than retaining two upload paths.
|
||||||
|
Adapt tests tied to producer goroutines to assert streaming and resource
|
||||||
|
release, which are the actual contracts.
|
||||||
|
5. Add focused body tests: original/replayed bytes parse identically; replay
|
||||||
|
after partial consumption starts from the beginning; simultaneous bodies use
|
||||||
|
independent handles; source failures propagate; Close/cancel releases a
|
||||||
|
blocked reader without deadlock. Use synthetic audio and gated readers.
|
||||||
|
|
||||||
|
Verify: `go test ./internal/adapters/whisperx` and
|
||||||
|
`go test -race ./internal/adapters/whisperx`.
|
||||||
|
|
||||||
|
Done when replay is available without whole-file buffering or producer-lifetime
|
||||||
|
ambiguity, and all existing HTTP behavior tests still pass.
|
||||||
|
|
||||||
|
## Stage 6: Verify Replay Through the HTTP Adapter Boundary
|
||||||
|
|
||||||
|
Status: Complete.
|
||||||
|
|
||||||
|
Completion marker (2026-09-22): Added a deterministic transport regression
|
||||||
|
that abandons a partially consumed original request, obtains and parses a fresh
|
||||||
|
multipart replay from `GetBody`, and installs the returned transcript within one
|
||||||
|
application attempt. Covered replay open/read failures, early responses,
|
||||||
|
parent cancellation during replay, and disposal of every opened source, while
|
||||||
|
proving an abandoned original does not poison a valid replay. Added a real HTTP
|
||||||
|
retry-exhaustion integration that durably retains an earlier speaker and resumes
|
||||||
|
only the uncommitted speakers. This controlled boundary verifies Narratio's
|
||||||
|
replay contract, not recovery from every real proxy protocol failure. Verified
|
||||||
|
with `go test ./internal/adapters/whisperx ./internal/app ./internal/stage`,
|
||||||
|
`go test -race ./internal/adapters/whisperx ./internal/app ./internal/stage`,
|
||||||
|
ten repeated race-enabled adapter runs, and `git diff --check`.
|
||||||
|
|
||||||
|
Preparation: Existing injectable HTTPClient/RoundTripper seam and local-server
|
||||||
|
tests in `internal/adapters/whisperx/http_test.go`; app WhisperX wiring tests.
|
||||||
|
|
||||||
|
1. Add a deterministic transport-boundary regression: consume part of the
|
||||||
|
original request, close it as a transport would, call GetBody, consume and
|
||||||
|
parse the replay using the request Content-Type, then return valid JSON.
|
||||||
|
Verify exact audio/language, installed output, and one application attempt.
|
||||||
|
This must fail before replay support; merely checking non-nil GetBody is
|
||||||
|
insufficient. Do not assert Go's internal retry count or require a flaky
|
||||||
|
handcrafted HTTP/2 peer in the default suite.
|
||||||
|
2. Cover replay-open/read failure, early response, parent cancellation during
|
||||||
|
replay, and disposal of all body handles. Reuse Stage 5 body coverage rather
|
||||||
|
than repeat every low-level case. Prove an intentionally abandoned original
|
||||||
|
body does not turn a valid replay response into failure.
|
||||||
|
3. Retain local HTTP tests for application retries, non-retryable responses,
|
||||||
|
response limits, invalid JSON, and bounded streaming; exercise that an
|
||||||
|
exhausted HTTP failure still leaves earlier speaker checkpoints reusable
|
||||||
|
using the existing stage/app seams.
|
||||||
|
4. Record the verification boundary honestly: controlled transport replay proves
|
||||||
|
Narratio's body contract, not recovery from every real proxy PROTOCOL_ERROR.
|
||||||
|
|
||||||
|
Verify: `go test ./internal/adapters/whisperx ./internal/app ./internal/stage`
|
||||||
|
and race tests for changed packages.
|
||||||
|
|
||||||
|
Done when the replay defect is protected at the adapter boundary and the
|
||||||
|
application retry/checkpoint behavior remains intact.
|
||||||
|
|
||||||
|
## Stage 7: Document and Validate the Completed Feature
|
||||||
|
|
||||||
|
Status: Complete.
|
||||||
|
|
||||||
|
Completion marker (2026-09-22): Updated the canonical transcribe, merge,
|
||||||
|
manifest, WhisperX, operations, and troubleshooting documentation for durable
|
||||||
|
speaker reconciliation, serialized checkpoints, aggregate gating, durable
|
||||||
|
force reset, legacy upgrade and downgrade limits, replayable streaming bodies,
|
||||||
|
resource ownership, and the absence of an exactly-once remote guarantee.
|
||||||
|
Reviewed the completed implementation against the feature roadmap and
|
||||||
|
implementation-discipline guidance. Verified with `go test ./...`,
|
||||||
|
`go test -race ./...`, `go vet ./...`, `go build ./...`,
|
||||||
|
`go test ./internal/doccheck`,
|
||||||
|
`go test ./internal/config -run '^TestExamplesLoadAndValidate$'`, and
|
||||||
|
`git diff --check`.
|
||||||
|
|
||||||
|
1. Update canonical transcribe and manifest internals for the collection,
|
||||||
|
reconciliation, durable-reset force behavior, serialized saves, and aggregate
|
||||||
|
gate. Remove obsolete all-speakers-before-canonical-write descriptions.
|
||||||
|
2. Update the WhisperX contract for replayable bounded streaming, application
|
||||||
|
attempts versus transport replay, resource ownership, and the lack of an
|
||||||
|
exactly-once server guarantee. Do not claim the remote protocol issue is fixed.
|
||||||
|
3. Update operations/troubleshooting with ordinary resume versus force, durable
|
||||||
|
partial success, old-manifest one-time retranscription on selecting transcribe,
|
||||||
|
downgrade limitations, and private-model-change guidance. No new CLI flags
|
||||||
|
or promises of automatic historical-file recovery. Keep detailed wire and
|
||||||
|
storage contracts in their canonical owners.
|
||||||
|
4. Review changes against the feature roadmap and the implementation-discipline
|
||||||
|
guidance. Do not elevate selected storage encoding, internal ordering, test
|
||||||
|
synchronization, or exact error wording into new product guarantees.
|
||||||
|
5. Run the full repository gates below; document results and update completion
|
||||||
|
statuses only after they pass. Leave release preparation for a separate task.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go test ./...
|
||||||
|
go test -race ./...
|
||||||
|
go vet ./...
|
||||||
|
go build ./...
|
||||||
|
go test ./internal/doccheck
|
||||||
|
go test ./internal/config -run '^TestExamplesLoadAndValidate$'
|
||||||
|
git diff --check
|
||||||
|
```
|
||||||
|
|
||||||
|
## Open Questions
|
||||||
|
|
||||||
|
None requiring user input. The agreed roadmap supplies the product decisions;
|
||||||
|
this plan selects a completed-only manifest collection, conservative old-record
|
||||||
|
handling, and pull-based replay as implementation choices. No stronger remote
|
||||||
|
reliability guarantee, new operator restriction, or generic orchestration
|
||||||
|
surface is proposed.
|
||||||
151
docs/roadmap/notarius-config-profile.md
Normal file
151
docs/roadmap/notarius-config-profile.md
Normal file
@@ -0,0 +1,151 @@
|
|||||||
|
# Notarius Configuration Profile Selection
|
||||||
|
|
||||||
|
Status: Implemented.
|
||||||
|
|
||||||
|
## Goal and Authority
|
||||||
|
|
||||||
|
Implement the consumer behavior in the
|
||||||
|
[feature request](feature-request-notarius-config-profile.yml): let a Narratio
|
||||||
|
configuration overlay explicitly select a Notarius configuration overlay.
|
||||||
|
Operators can select testing once in Narratio while each application retains
|
||||||
|
ownership of its own model choices and other settings.
|
||||||
|
|
||||||
|
This roadmap describes the target state and required change areas, not an
|
||||||
|
implementation sequence; the [implementation plan](implementation.md) owns
|
||||||
|
the ordered work and implementation decisions. The feature request is the scope
|
||||||
|
authority; the mechanisms suggested below do not establish additional
|
||||||
|
user-facing contracts.
|
||||||
|
|
||||||
|
## Underlying Issue
|
||||||
|
|
||||||
|
Narratio can select its own testing profile, but extraction currently invokes
|
||||||
|
Notarius without a configuration-profile selector. Consequently, a Narratio
|
||||||
|
testing invocation cannot explicitly select the independently deployed Notarius
|
||||||
|
testing overlay through pipeline configuration.
|
||||||
|
|
||||||
|
Adding individual downstream model or tuning fields would duplicate Notarius's
|
||||||
|
configuration responsibilities. Passing its existing profile selector provides
|
||||||
|
the requested flexibility without that duplication.
|
||||||
|
|
||||||
|
## Targeted End State
|
||||||
|
|
||||||
|
### Explicit Configuration Composition
|
||||||
|
|
||||||
|
`pipeline.notarius.config_profile` is an optional Notarius configuration-profile
|
||||||
|
name, usable in Narratio's base configuration and ordinary overlays. For
|
||||||
|
example, Narratio's `profiles/testing.yml` may contain:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
notarius:
|
||||||
|
config_profile: testing
|
||||||
|
```
|
||||||
|
|
||||||
|
Selecting Narratio's `--profile testing` then selects that explicit downstream
|
||||||
|
profile whenever extraction executes. The two profile names need not match.
|
||||||
|
Narratio does not automatically forward its own profile name.
|
||||||
|
|
||||||
|
When supplied, the selection is passed through Notarius's `--config-profile`
|
||||||
|
argument. Omitted or null values add no profile-selection flag and preserve
|
||||||
|
existing downstream selection behavior. Notarius's selector semantics remain
|
||||||
|
authoritative, including empty and `default` selections for the composed base
|
||||||
|
configuration. Narratio does not introduce a stricter profile-name policy.
|
||||||
|
|
||||||
|
This field selects a ConfigKit configuration overlay, not a PromptKit profile
|
||||||
|
through `--llm-profile`. The Notarius overlay owns any resulting LLM-profile
|
||||||
|
selection. Notarius resolves overlays relative to its own configuration root;
|
||||||
|
Narratio neither resolves those files against its own root nor rewrites them.
|
||||||
|
|
||||||
|
### Execution, Reuse, and Failure
|
||||||
|
|
||||||
|
A changed downstream profile selection must not silently reuse Narratio's
|
||||||
|
completed extraction from the previous selection. Existing dependent-stage
|
||||||
|
invalidation applies when extraction changes; the feature does not change the
|
||||||
|
pipeline order or transcript-generation behavior.
|
||||||
|
|
||||||
|
Notarius continues to own downstream overlay discovery, composition, validation,
|
||||||
|
and cache compatibility. A missing or invalid selected overlay must fail
|
||||||
|
extraction through the existing subprocess error path, without silently falling
|
||||||
|
back to production or reporting successful extraction.
|
||||||
|
|
||||||
|
Existing resume and chunk-cache controls, including force precedence, remain
|
||||||
|
in place. Profile selection does not itself mean forced regeneration or cache
|
||||||
|
deletion. Narratio's decision to execute extraction and Notarius's decision to
|
||||||
|
reuse compatible work remain separate responsibilities.
|
||||||
|
|
||||||
|
Changes to private Notarius configuration behind an unchanged selector retain
|
||||||
|
the existing operator responsibility to force extraction when needed. There is
|
||||||
|
no new downstream file inspection or content-tracking requirement.
|
||||||
|
|
||||||
|
Credential forwarding remains independently configured through
|
||||||
|
`notarius.llm_api_key`. A Narratio testing overlay may select a different
|
||||||
|
credential reference alongside the downstream profile; no new credential
|
||||||
|
discovery or provider-name mapping is introduced.
|
||||||
|
|
||||||
|
## Required Change Areas
|
||||||
|
|
||||||
|
### Configuration and Inspection
|
||||||
|
|
||||||
|
Extend the typed Notarius configuration and its existing safe effective-YAML
|
||||||
|
projection to carry the optional field. Preserve the ordinary configuration
|
||||||
|
composition behavior for base values, overlay overrides, and null values.
|
||||||
|
Configuration inspection must reflect the selected field without exposing
|
||||||
|
credential values. No replacement configuration loader or downstream preflight
|
||||||
|
loader is needed.
|
||||||
|
|
||||||
|
### Stage and Adapter Boundary
|
||||||
|
|
||||||
|
Carry the selection from resolved configuration through the extraction request
|
||||||
|
to the subprocess adapter. Construct the optional CLI flag using the existing
|
||||||
|
argument-vector mechanism, alongside the current configuration path, references,
|
||||||
|
credential forwarding, and cache controls. Retain normal Notarius error handling.
|
||||||
|
|
||||||
|
### Extraction Reuse Evidence
|
||||||
|
|
||||||
|
Account for the downstream selection in the existing extraction reuse evidence.
|
||||||
|
The current semantic-configuration and invocation fingerprints are the relevant
|
||||||
|
change points. Keep their treatment of selection consistent so that a profile
|
||||||
|
change cannot bypass the required rerun and downstream invalidation.
|
||||||
|
|
||||||
|
This roadmap does not require a new manifest schema, separate profile-indexed
|
||||||
|
workspaces, historical per-profile result retention, or an upgrade-wide forced
|
||||||
|
rerun.
|
||||||
|
|
||||||
|
### Verification and Documentation
|
||||||
|
|
||||||
|
Use focused behavioral coverage for configuration composition and inspection,
|
||||||
|
explicit argument forwarding and omission, changed-selection reuse decisions,
|
||||||
|
and ordinary subprocess failure handling. Reuse existing test coverage where
|
||||||
|
it already protects unchanged credentials, force behavior, and invalidation;
|
||||||
|
do not duplicate Notarius's complete profile-validation or cache test suites.
|
||||||
|
|
||||||
|
When implemented, update the canonical
|
||||||
|
[configuration reference](../config.md),
|
||||||
|
[Notarius integration contract](../integrations/notarius.md),
|
||||||
|
[extract internals](../internal/stage-extract.md), and relevant maintained
|
||||||
|
examples. The development workflow in [Operations](../operations.md) should
|
||||||
|
explain explicit profile composition and the existing need to force extraction
|
||||||
|
after private downstream edits. Clarify that profile selection does not create
|
||||||
|
an isolated testing workspace or separate current extraction state.
|
||||||
|
|
||||||
|
## Compatibility and Scope Boundaries
|
||||||
|
|
||||||
|
Notarius v0.7.0 already documents `--config-profile`; no upstream change is
|
||||||
|
required. Document the capability needed when the option is used, without adding
|
||||||
|
an unconditional runtime version gate for existing unconfigured deployments.
|
||||||
|
|
||||||
|
This feature adds no Narratio CLI selector, automatic profile propagation,
|
||||||
|
Notarius LLM-profile selector, downstream configuration rewriting, new
|
||||||
|
credential mechanism, or generalized recursive orchestration. Existing output
|
||||||
|
and receipt contracts remain unchanged.
|
||||||
|
|
||||||
|
The consumer's Ansible role and suite updates are follow-up work outside this
|
||||||
|
repository. They can deploy downstream overlays and populate this field after
|
||||||
|
the interface is released.
|
||||||
|
|
||||||
|
## Scope Review
|
||||||
|
|
||||||
|
No stronger requirements or new invariants are proposed beyond the feature
|
||||||
|
request and existing repository contracts. In particular, minimizing one-time
|
||||||
|
upgrade invalidation and avoiding reruns between equivalent base selectors are
|
||||||
|
implementation considerations, not additional acceptance guarantees imposed
|
||||||
|
by this roadmap. No user decisions are currently required before planning.
|
||||||
104
docs/roadmap/remediation.md
Normal file
104
docs/roadmap/remediation.md
Normal file
@@ -0,0 +1,104 @@
|
|||||||
|
# Transcription Checkpoint Review Remediation
|
||||||
|
|
||||||
|
Status: Complete.
|
||||||
|
|
||||||
|
Reviewed commits `7b0bf99` through `96b4e73` against the speaker-checkpoint and
|
||||||
|
upload-replay roadmap and the original user request. No production changes were
|
||||||
|
made during review. The two failures below were reproduced with temporary tests
|
||||||
|
using the existing stage fixtures; those temporary tests were removed afterward.
|
||||||
|
|
||||||
|
## 1. Reconcile Aggregate State Even When No Speaker Needs HTTP Work
|
||||||
|
|
||||||
|
Priority: P2. Status: Complete.
|
||||||
|
|
||||||
|
Implemented read-only aggregate identity/evidence checks before resume skips.
|
||||||
|
Regression tests cover removed inputs, missing/duplicate/mismatched outputs,
|
||||||
|
repair without HTTP requests, downstream invalidation, and exclusion of removed
|
||||||
|
speakers from merge without deleting their files. Equivalent path forms and
|
||||||
|
output ordering remain reusable. Focused app/stage tests and documentation
|
||||||
|
checks pass.
|
||||||
|
|
||||||
|
Location: `internal/stage/transcribe_resume.go:21` and the reconciliation result
|
||||||
|
consumed there.
|
||||||
|
|
||||||
|
`ValidateResume` returns reusable whenever `reconciliation.work` is empty. That
|
||||||
|
only establishes that the remaining speakers need no transcription; it does
|
||||||
|
not establish that the stored speaker collection and aggregate outputs match
|
||||||
|
the current input set. Removing a speaker while the remaining inputs are
|
||||||
|
unchanged produces an empty work list, so a succeeded record can be skipped
|
||||||
|
without pruning the removed speaker. Its old aggregate output can then remain
|
||||||
|
available to merge. Likewise, missing/inconsistent aggregate output references
|
||||||
|
are not repaired when all individual checkpoints remain valid. A prepare rerun
|
||||||
|
may already stale transcribe in some workflows, but the resume validator cannot
|
||||||
|
rely on that to satisfy its own current-input/output checks.
|
||||||
|
|
||||||
|
Reproduction: create complete alice/bob checkpoint records and aggregate
|
||||||
|
outputs with the existing fixture, mark the stage succeeded, remove bob from
|
||||||
|
the manifest's audio inputs, and call `ValidateResume`. It returns reusable
|
||||||
|
despite still retaining bob in the stored collection and aggregate outputs.
|
||||||
|
|
||||||
|
Required change:
|
||||||
|
|
||||||
|
- Distinguish no HTTP work from no reconciliation work. Validate that the
|
||||||
|
committed speaker set and aggregate transcript references represent the
|
||||||
|
reconciled current set, including relevant identity/evidence, before skipping.
|
||||||
|
- When only collection/output repair is needed, return non-resumable and let
|
||||||
|
normal execution checkpoint the pruned collection and rebuild aggregate
|
||||||
|
outputs without retranscribing reusable speakers. Preserve existing dependent
|
||||||
|
invalidation; do not force otherwise-current speaker jobs.
|
||||||
|
- Keep resume inspection read-only. Do not fix this by deleting incidental
|
||||||
|
transcript files or changing stage selection rules.
|
||||||
|
|
||||||
|
Verification: add removed-only input coverage (not just removal combined with
|
||||||
|
an added speaker), plus incomplete/inconsistent aggregate output coverage.
|
||||||
|
Exercise the runner skip path and assert zero additional WhisperX requests,
|
||||||
|
correct rebuilt outputs, and no removed speaker reaching merge.
|
||||||
|
|
||||||
|
## 2. Remove the Unrequested Positive Audio-Size Requirement
|
||||||
|
|
||||||
|
Priority: P2. Status: Complete.
|
||||||
|
|
||||||
|
Audio evidence now permits zero size while still rejecting negative values.
|
||||||
|
Regression tests verify zero-size manifest round trips, successful service
|
||||||
|
results being checkpointed and reused, and service rejection being propagated
|
||||||
|
without creating a completion record. Nonempty valid-JSON output validation is
|
||||||
|
unchanged. Both new success regressions failed before the fix.
|
||||||
|
|
||||||
|
Location: `internal/manifest/transcribe_state.go:95`.
|
||||||
|
|
||||||
|
The new record validator requires `AudioSize > 0`. Previously Narratio accepted
|
||||||
|
a regular `.flac` file and delegated transcription acceptance to WhisperX;
|
||||||
|
neither the user request nor the roadmap introduced minimum audio size or a new
|
||||||
|
audio-content validation policy. Zero-byte content still has a valid checksum
|
||||||
|
and can be represented by the checkpoint mechanism. The current check runs
|
||||||
|
after a successful response and output materialization, causing the entire
|
||||||
|
stage to fail instead of retaining the completed work.
|
||||||
|
|
||||||
|
Reproduction: prepare one speaker, replace its prepared audio with an empty
|
||||||
|
regular file, and use the existing successful fake WhisperX client. Stage
|
||||||
|
execution fails at checkpoint validation with `audio_size must be positive`.
|
||||||
|
This demonstrates the extra Narratio restriction; it does not assert that a
|
||||||
|
particular production WhisperX deployment accepts empty audio.
|
||||||
|
|
||||||
|
Required change:
|
||||||
|
|
||||||
|
- Permit zero audio size and reject negative sizes. Keep byte hashing and
|
||||||
|
size equality checks. Let the downstream service decide whether the input
|
||||||
|
can be transcribed, as before.
|
||||||
|
- Do not remove the nonempty valid-JSON output requirement, which has a separate
|
||||||
|
existing justification. Do not add a new early empty-audio rejection instead.
|
||||||
|
- Update any new tests/documentation that promote strictly positive input size
|
||||||
|
to a contract.
|
||||||
|
|
||||||
|
Verification: cover a zero-size record round-trip and successful adapter result
|
||||||
|
being checkpointed/reused for that input; retain negative-size rejection and
|
||||||
|
ordinary downstream-failure tests.
|
||||||
|
|
||||||
|
## Verification After Remediation
|
||||||
|
|
||||||
|
Completed on 2026-09-22 with Go 1.26.8: focused stage/application tests,
|
||||||
|
changed-package race tests, `go test ./...`, `go test -race ./...`,
|
||||||
|
`go vet ./...`, `go build ./...`, `go test ./internal/doccheck`,
|
||||||
|
`go test ./internal/config -run '^TestExamplesLoadAndValidate$'`, and
|
||||||
|
`git diff --check` all passed. Existing replay and cancellation regression
|
||||||
|
tests remain intact; the HTTP adapter required no changes.
|
||||||
183
docs/roadmap/transcribe-checkpoints-and-replay.md
Normal file
183
docs/roadmap/transcribe-checkpoints-and-replay.md
Normal file
@@ -0,0 +1,183 @@
|
|||||||
|
# Resumable Speaker Transcription and Replayable Uploads
|
||||||
|
|
||||||
|
Status: Implemented.
|
||||||
|
|
||||||
|
## Goal and User Intent
|
||||||
|
|
||||||
|
Preserve completed speaker transcriptions when another speaker fails, so an
|
||||||
|
ordinary run or resume does not repeat expensive successful work. Separately,
|
||||||
|
allow Go's HTTP transport to replay WhisperX uploads when it encounters a
|
||||||
|
recoverable connection or stream failure.
|
||||||
|
|
||||||
|
The motivating incident exhausted four transcription attempts for one speaker
|
||||||
|
after three other speakers had completed. The final error reported an HTTP/2
|
||||||
|
peer `PROTOCOL_ERROR` and an inability to replay the request body without
|
||||||
|
`Request.GetBody`.
|
||||||
|
|
||||||
|
These are related reliability improvements, not a promise to eliminate all
|
||||||
|
service failures. This document owns the target state and scope; the
|
||||||
|
[implementation plan](implementation.md) owns ordered changes and concrete
|
||||||
|
storage and replay decisions.
|
||||||
|
|
||||||
|
## Pre-implementation Limitations
|
||||||
|
|
||||||
|
The [transcribe stage](../internal/stage-transcribe.md) writes successful
|
||||||
|
responses into the invocation's run directory. It installs canonical transcripts
|
||||||
|
and returns its per-file metadata only after every speaker succeeds. A failure
|
||||||
|
therefore leaves no durable per-speaker progress that ordinary resume can use,
|
||||||
|
even though some completed JSON files may remain on disk.
|
||||||
|
|
||||||
|
The [WhisperX adapter](../integrations/whisperx.md) already retries requests at
|
||||||
|
the application level, reopening audio for each attempt. Its multipart request
|
||||||
|
body is streamed through a pipe and has no replay callback. Transport-level
|
||||||
|
recovery within an attempt cannot reconstruct a consumed body. Providing that
|
||||||
|
capability addresses the replay limitation, not the underlying peer protocol
|
||||||
|
error; persistent failures can still require service or proxy investigation.
|
||||||
|
|
||||||
|
## Targeted End State
|
||||||
|
|
||||||
|
### Durable Per-Speaker Completion
|
||||||
|
|
||||||
|
Each successfully transcribed speaker file is validated, saved, and recorded
|
||||||
|
durably before that work is treated as reusable. The session manifest remains
|
||||||
|
the progress authority. Follow the existing analyze-stage checkpoint pattern
|
||||||
|
without turning speakers into pipeline stages or introducing a general task
|
||||||
|
engine.
|
||||||
|
|
||||||
|
Ordinary execution reconciles the current prepared audio set with recorded
|
||||||
|
speaker results. It reuses valid completed work and schedules failed,
|
||||||
|
unattempted, missing, or obsolete results. A result is not reusable merely
|
||||||
|
because a matching filename exists. Evidence binds it to the speaker/audio
|
||||||
|
identity, audio content, Narratio-visible recognition settings, and the saved
|
||||||
|
transcript's integrity.
|
||||||
|
|
||||||
|
The relevant recognition settings remain the WhisperX service URL and language.
|
||||||
|
Retry count/delay, concurrency, timeout, credentials, and diagnostic locations
|
||||||
|
remain operational settings rather than reasons to discard completed results.
|
||||||
|
Changes to private service models or configuration remain an operator reason
|
||||||
|
to force transcription; Narratio does not inspect the service's private state.
|
||||||
|
|
||||||
|
Changes to individual audio inputs require replacement of affected results,
|
||||||
|
not automatic retranscription of unrelated unchanged speakers. New speakers
|
||||||
|
need transcription; removed speakers do not remain in the current aggregate
|
||||||
|
output set. Records must describe the inputs actually used, not grant reuse to
|
||||||
|
old bytes under a changed identity.
|
||||||
|
|
||||||
|
### Aggregate Completion, Failure, and Force
|
||||||
|
|
||||||
|
The overall stage succeeds only when every currently required speaker has a
|
||||||
|
validated, durably recorded result. Incomplete per-speaker progress does not
|
||||||
|
make the aggregate stage successful or authorize downstream consumption of an
|
||||||
|
incomplete collection. Existing dependent-stage invalidation remains in effect.
|
||||||
|
|
||||||
|
Retain bounded concurrency and the current fail-fast behavior: a speaker
|
||||||
|
failure cancels outstanding work. Results already checkpointed remain reusable
|
||||||
|
after failure, cancellation, or process interruption. A later ordinary run or
|
||||||
|
resume retries the remaining work; it does not need a new CLI command.
|
||||||
|
|
||||||
|
Checkpoint writes must be serialized so concurrently finishing speakers cannot
|
||||||
|
overwrite each other's progress. A failed persistence operation must not
|
||||||
|
advertise an unrecorded result as complete. Previously committed progress remains
|
||||||
|
the recovery authority, and checkpoint errors must be surfaced rather than
|
||||||
|
reported as successful transcription.
|
||||||
|
|
||||||
|
`--force` on transcribe regenerates all currently required speakers instead of
|
||||||
|
reusing their prior completions. Successful replacements are checkpointed too,
|
||||||
|
so an ordinary retry after a failed forced invocation can retain the newly
|
||||||
|
completed work. Old results for speakers not successfully regenerated must not
|
||||||
|
satisfy that unfinished forced replacement on the next ordinary retry.
|
||||||
|
|
||||||
|
This deliberately relaxes the current all-speakers-before-canonical-installation
|
||||||
|
implementation: individual durable results may exist while the stage remains
|
||||||
|
failed or incomplete. The all-speakers-before-stage-success boundary remains.
|
||||||
|
|
||||||
|
### Replayable WhisperX Requests
|
||||||
|
|
||||||
|
Provide `Request.GetBody` so the HTTP transport can obtain a fresh body for the
|
||||||
|
same request. Replay reopens the audio and reproduces the multipart boundary,
|
||||||
|
filename, fields, and framing used by the original request. Independent bodies
|
||||||
|
must not share a consumed reader or mutable read position.
|
||||||
|
|
||||||
|
Preserve bounded-memory streaming rather than buffering entire audio files in
|
||||||
|
memory. Correctly close every original and replay body and its audio resources;
|
||||||
|
preserve source-read errors and cancellation, including partial uploads and
|
||||||
|
early server responses. Successful response handling must not accidentally wait
|
||||||
|
on, or report errors solely from, an abandoned body instead of the replay that
|
||||||
|
produced the response.
|
||||||
|
|
||||||
|
Keep the existing application-level retry count, delay, classification, and
|
||||||
|
per-attempt timeout semantics. Transport replays occur within an application
|
||||||
|
attempt and its context; they are not additional Narratio-configured attempts.
|
||||||
|
Do not add another custom retry loop, silently increase retries, or disable
|
||||||
|
HTTP/2 globally as a substitute for replay support.
|
||||||
|
|
||||||
|
The adapter still validates and installs only successful JSON responses.
|
||||||
|
Request replay does not provide exactly-once server execution: a lost response
|
||||||
|
can cause repeated transcription work, as can existing application retries.
|
||||||
|
No new service-side idempotency protocol is required by this feature.
|
||||||
|
|
||||||
|
## Required Change Areas
|
||||||
|
|
||||||
|
- **Manifest and persistence:** Add transcribe-owned per-speaker progress and
|
||||||
|
evidence, with safe checkpoint updates and preservation through runner failure
|
||||||
|
handling. Review manifest validation, copying, serialization, and existing
|
||||||
|
publish/restore handling wherever the new records pass through those paths.
|
||||||
|
- **Stage execution and reuse:** Reconcile speaker records before scheduling,
|
||||||
|
checkpoint validated completions, rebuild aggregate outputs from the current
|
||||||
|
set, and account for partial progress in ordinary and forced execution.
|
||||||
|
Review successful-stage reuse as well as failed-stage retries so stale audio
|
||||||
|
or missing/corrupt transcript evidence is not bypassed by an aggregate skip.
|
||||||
|
- **Artifact lifecycle:** Use existing path, installation, integrity, and manifest
|
||||||
|
mechanisms. Ensure checkpointed results do not depend on an ephemeral run
|
||||||
|
directory surviving cleanup. Do not introduce a second authoritative ledger
|
||||||
|
or infer completion by scanning abandoned runs.
|
||||||
|
- **HTTP adapter:** Make multipart bodies reproducible and correctly manage
|
||||||
|
every replay's lifetime, without changing the WhisperX HTTP schema or requiring
|
||||||
|
an upstream service change.
|
||||||
|
- **Tests:** Cover per-speaker recovery after a later failure, interruption and
|
||||||
|
checkpoint failure; changed/missing inputs or outputs; force followed by an
|
||||||
|
ordinary retry; concurrent successful completions; and aggregate downstream
|
||||||
|
gating. Extend existing fixtures rather than duplicating analyze's complete
|
||||||
|
test suite. Exercise actual body reconstruction after partial consumption,
|
||||||
|
multipart equivalence, cancellation/resource cleanup, and existing bounded
|
||||||
|
streaming and retry behavior—not just the presence of a callback. Default
|
||||||
|
tests remain offline and use local/fake HTTP boundaries, never paid services.
|
||||||
|
- **Documentation:** Update the canonical
|
||||||
|
[stage guide](../internal/stage-transcribe.md),
|
||||||
|
[manifest guide](../internal/manifest.md),
|
||||||
|
[WhisperX contract](../integrations/whisperx.md),
|
||||||
|
[operations guide](../operations.md), and
|
||||||
|
[troubleshooting guide](../troubleshooting.md) when implemented. Explain
|
||||||
|
partial completion, ordinary retry versus force, and the distinction between
|
||||||
|
request replay limitations and persistent peer protocol failures.
|
||||||
|
|
||||||
|
## Compatibility and Scope Boundaries
|
||||||
|
|
||||||
|
Existing speaker identities, raw transcript formats, and the fixed pipeline
|
||||||
|
order remain unchanged. Existing manifests remain readable, but must not be
|
||||||
|
treated as containing per-speaker evidence they never recorded. Recovery relies
|
||||||
|
on verified recorded evidence, not guessed associations between old transcripts
|
||||||
|
and current audio. No general legacy-migration framework is part of the feature.
|
||||||
|
|
||||||
|
Automatic adoption of unrecorded outputs from previously failed runs is outside
|
||||||
|
this feature. Such files may be recoverable after verifying their inputs, but
|
||||||
|
their presence alone does not prove completion. No recovery/import command,
|
||||||
|
speaker-selection CLI, historical invocation ledger, new service configuration
|
||||||
|
discovery, or broader orchestration refactor is included.
|
||||||
|
|
||||||
|
The feature does not diagnose the remote service/proxy's underlying HTTP/2
|
||||||
|
problem, guarantee recovery from persistent protocol errors, or change the
|
||||||
|
policy for continuing unrelated work after failure.
|
||||||
|
|
||||||
|
## Completion Criteria
|
||||||
|
|
||||||
|
For speakers A, B, C, and D: if A and B are checkpointed, C fails, and D has not
|
||||||
|
completed, ordinary resume retains A and B, retries C, and completes D. No
|
||||||
|
downstream stage receives an incomplete successful transcription collection.
|
||||||
|
Force replaces all four, while an ordinary retry can reuse checkpoints from
|
||||||
|
that forced replacement attempt.
|
||||||
|
|
||||||
|
The HTTP transport can reconstruct the upload after consuming the original
|
||||||
|
body, without whole-file memory buffering, lost source errors, or stranded
|
||||||
|
upload resources. A persistent peer error still fails normally after the
|
||||||
|
configured application attempts, without discarding checkpointed speakers.
|
||||||
@@ -213,6 +213,134 @@ input validation.
|
|||||||
|
|
||||||
Relevant reference: [Operations: Stage Execution and Continuation Behavior](./operations.md#stage-execution-and-continuation-behavior).
|
Relevant reference: [Operations: Stage Execution and Continuation Behavior](./operations.md#stage-execution-and-continuation-behavior).
|
||||||
|
|
||||||
|
## Transcription retries speakers that previously finished
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- an ordinary transcribe retry calls WhisperX for a speaker that appeared to
|
||||||
|
finish earlier; or
|
||||||
|
- an older successful session retranscribes all speakers once after upgrade.
|
||||||
|
|
||||||
|
Likely causes:
|
||||||
|
|
||||||
|
- the earlier response or canonical file was not followed by a successful
|
||||||
|
speaker manifest checkpoint;
|
||||||
|
- the prepared audio bytes/path, WhisperX URL, or language changed;
|
||||||
|
- the canonical transcript is missing, unsafe, non-regular, invalid JSON, or
|
||||||
|
no longer matches its recorded size/checksum;
|
||||||
|
- the previous invocation used `--force`, which durably cleared pre-force
|
||||||
|
speaker authority; or
|
||||||
|
- the manifest has only legacy aggregate transcribe state and cannot bind old
|
||||||
|
outputs to their source audio.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- inspect the reported speaker and filesystem error, repair the prepared input
|
||||||
|
or canonical-output path when needed, then retry transcribe normally;
|
||||||
|
- allow a legacy aggregate-only record to regenerate once; and
|
||||||
|
- do not edit the manifest or copy an old JSON file into place as a substitute
|
||||||
|
for a successful checkpoint.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio run-stage transcribe 2026-04-04
|
||||||
|
```
|
||||||
|
|
||||||
|
Valid earlier speaker checkpoints remain reusable even while aggregate
|
||||||
|
transcribe status is failed or interrupted. Merge remains gated until
|
||||||
|
transcribe reaches aggregate success with the complete current set.
|
||||||
|
|
||||||
|
Relevant reference: [Operations: Transcription continuation and replacement](./operations.md#transcription-continuation-and-replacement).
|
||||||
|
|
||||||
|
## WhisperX model changed but transcription was reused
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- WhisperX model weights, prompts, or private server configuration changed at
|
||||||
|
the same configured URL, but Narratio still considers speaker transcripts
|
||||||
|
current.
|
||||||
|
|
||||||
|
Likely cause:
|
||||||
|
|
||||||
|
- Narratio can fingerprint its configured URL and language, not private state
|
||||||
|
loaded behind the service boundary.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- explicitly force transcribe to establish a durable reset, then use ordinary
|
||||||
|
retry if the replacement attempt is interrupted.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio run-stage transcribe 2026-04-04 --force
|
||||||
|
```
|
||||||
|
|
||||||
|
Transport replay and application retries do not provide an exactly-once remote
|
||||||
|
execution guarantee. Investigate server-side failures or duplicate work using
|
||||||
|
WhisperX diagnostics; do not infer completion from a server log without the
|
||||||
|
local speaker checkpoint.
|
||||||
|
|
||||||
|
Relevant references: [WhisperX integration](./integrations/whisperx.md) and
|
||||||
|
[Transcribe internals](./internal/stage-transcribe.md).
|
||||||
|
|
||||||
|
## Transcription manifest checkpoint failed
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- transcribe reports that a speaker completed but the session manifest could
|
||||||
|
not be saved.
|
||||||
|
|
||||||
|
Likely causes:
|
||||||
|
|
||||||
|
- the workspace is unwritable or out of space;
|
||||||
|
- the manifest path or parent directory is unsafe or the wrong type; or
|
||||||
|
- a transient filesystem error interrupted atomic replacement.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- correct the reported filesystem or permission problem and retry transcribe
|
||||||
|
normally. Narratio reconciles from the last readable collection and reuses
|
||||||
|
only successfully checkpointed speakers.
|
||||||
|
|
||||||
|
A canonical raw transcript left by the failed checkpoint is incidental and
|
||||||
|
does not become reusable authority. Preserve the manifest and diagnostics if
|
||||||
|
the failure repeats; do not advertise the file by editing manifest evidence.
|
||||||
|
|
||||||
|
Relevant references: [Manifest persistence](./internal/manifest.md#persistence-semantics)
|
||||||
|
and [Transcribe internals](./internal/stage-transcribe.md).
|
||||||
|
|
||||||
|
## Notarius credential reference unavailable
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- configuration loading reports `secret.unset`, `secret.empty`, or an invalid
|
||||||
|
name for `pipeline.notarius.llm_api_key`, including during inspection or a
|
||||||
|
run that would not execute extraction.
|
||||||
|
|
||||||
|
Likely causes:
|
||||||
|
|
||||||
|
- the named environment variable is absent or empty;
|
||||||
|
- the selected secrets directory lacks a file with that exact name;
|
||||||
|
- a process environment variable exists but is empty, so bootstrap correctly
|
||||||
|
leaves it in place; or
|
||||||
|
- the Narratio reference name does not match `api_key_env` in the selected
|
||||||
|
Notarius/PromptKit profile.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- set the exact nonempty environment variable or provide an equivalently named
|
||||||
|
file through the conventional or explicit secrets directory;
|
||||||
|
- remove or set the field to `null` when that deployment needs no credential;
|
||||||
|
and
|
||||||
|
- use `config show`, `config sources`, and `config diff` to inspect the public
|
||||||
|
reference name and ownership without exposing its value.
|
||||||
|
|
||||||
|
Narratio resolves selected credential references during ordinary pipeline
|
||||||
|
loading, before stage selection or resume. Credential changes alone do not
|
||||||
|
invalidate a completed extraction, but the current reference must still resolve
|
||||||
|
for the command to proceed.
|
||||||
|
|
||||||
|
Relevant references: [Secrets](./config.md#secrets-handling) and
|
||||||
|
[Notarius credential](./config.md#notarius-credential).
|
||||||
|
|
||||||
## Notarius executable missing
|
## Notarius executable missing
|
||||||
|
|
||||||
Symptom:
|
Symptom:
|
||||||
@@ -289,6 +417,59 @@ Do not point Notarius directly at the original source path or edit the manifest
|
|||||||
checksum. Relevant references: [Notarius reference configuration](./config.md#notarius-reference-bindings)
|
checksum. Relevant references: [Notarius reference configuration](./config.md#notarius-reference-bindings)
|
||||||
and [Operations: Extraction Workflow](./operations.md#extraction-workflow).
|
and [Operations: Extraction Workflow](./operations.md#extraction-workflow).
|
||||||
|
|
||||||
|
## Notarius reference snapshot cache conflict
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- extraction reports that a path beneath
|
||||||
|
`cache/notarius/references/<sha256>/` is a symlink, directory, or another
|
||||||
|
non-regular or unsafe entry.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- stop commands for the affected session and inspect the exact reported path;
|
||||||
|
- investigate unexpected links or special files as possible tampering; and
|
||||||
|
- remove only the reported conflicting leaf or digest directory after
|
||||||
|
confirming it belongs to the reconstructible reference cache.
|
||||||
|
|
||||||
|
A missing snapshot or corrupt regular snapshot is reconstructed automatically
|
||||||
|
from currently verified prepared input. Do not recursively delete the session
|
||||||
|
root, edit the manifest checksum, or remove cache entries while extraction is
|
||||||
|
active.
|
||||||
|
|
||||||
|
Relevant reference: [Operations: Extraction Workflow](./operations.md#extraction-workflow).
|
||||||
|
|
||||||
|
## Notarius resume executes work instead of reusing it
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- Narratio executes `extract`, and Notarius makes provider calls despite
|
||||||
|
`pipeline.notarius.resume: true`.
|
||||||
|
|
||||||
|
Likely causes:
|
||||||
|
|
||||||
|
- checkpoint recording is disabled in the Notarius configuration;
|
||||||
|
- Notarius is using a nonpersistent or different checkpoint root;
|
||||||
|
- transcript/reference bytes, reference paths, pipeline/profile content,
|
||||||
|
local-backend endpoint, selected lanes, or another checkpoint identity input
|
||||||
|
changed; or
|
||||||
|
- Narratio extraction was forced, which deliberately omits `--resume`.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- validate the deployed Notarius configuration and configure explicit
|
||||||
|
persistent chunk-plan and checkpoint roots;
|
||||||
|
- inspect Notarius checkpoint decision reason codes for a legitimate cold miss;
|
||||||
|
and
|
||||||
|
- retry without force only when reuse is intended.
|
||||||
|
|
||||||
|
Remember that Narratio skipping a completed `extract` stage is separate from
|
||||||
|
Notarius reusing work inside an extraction that does execute. Narratio does not
|
||||||
|
read the Notarius configuration to infer checkpoint enablement.
|
||||||
|
|
||||||
|
Relevant references: [Notarius cache controls](./config.md#notarius-cache-controls)
|
||||||
|
and [Notarius integration](./integrations/notarius.md).
|
||||||
|
|
||||||
## Notarius reference selector or generated-handoff collision
|
## Notarius reference selector or generated-handoff collision
|
||||||
|
|
||||||
Symptom:
|
Symptom:
|
||||||
@@ -361,10 +542,10 @@ Symptom:
|
|||||||
|
|
||||||
Likely causes:
|
Likely causes:
|
||||||
|
|
||||||
- the executable/config path, pipeline ID, timeout, working directory, or
|
- the pipeline ID or configured output contracts changed;
|
||||||
configured output contracts changed;
|
|
||||||
- a configured prepared reference selector, source, path, checksum, or byte
|
- a configured prepared reference selector, source, path, checksum, or byte
|
||||||
size changed;
|
size changed;
|
||||||
|
- the direct trimmed transcript identity or bytes changed; or
|
||||||
- the durable bundle, index, lane set, provenance, regular-file status, or
|
- the durable bundle, index, lane set, provenance, regular-file status, or
|
||||||
checksum no longer validates.
|
checksum no longer validates.
|
||||||
|
|
||||||
@@ -374,6 +555,11 @@ Safe fix:
|
|||||||
an unsafe path or bundle symlink error as filesystem corruption or tampering
|
an unsafe path or bundle symlink error as filesystem corruption or tampering
|
||||||
and investigate it rather than replacing files manually.
|
and investigate it rather than replacing files manually.
|
||||||
|
|
||||||
|
Executable, config path, timeout, working directory, and Notarius credential
|
||||||
|
changes are operational and do not by themselves invalidate reusable extraction
|
||||||
|
evidence. A newly selected credential must nevertheless resolve during command
|
||||||
|
configuration loading.
|
||||||
|
|
||||||
## Notarius transitive configuration changed
|
## Notarius transitive configuration changed
|
||||||
|
|
||||||
Symptom:
|
Symptom:
|
||||||
@@ -488,18 +674,22 @@ Symptom:
|
|||||||
or unsupported; or
|
or unsupported; or
|
||||||
- restore rejects a committed remote session manifest for the same reason.
|
- restore rejects a committed remote session manifest for the same reason.
|
||||||
|
|
||||||
Likely cause:
|
Likely causes:
|
||||||
|
|
||||||
- the manifest predates the supported canonical state contract. Narratio
|
- the manifest predates the supported canonical state contract. Narratio
|
||||||
rejects it at the schema boundary rather than interpreting aggregate output
|
rejects it at the schema boundary rather than interpreting aggregate output
|
||||||
lists or incidental files as current authority.
|
lists or incidental files as current authority; or
|
||||||
|
- an older strict-decoding Narratio binary is reading a current manifest that
|
||||||
|
contains additive transcribe speaker state it does not recognize.
|
||||||
|
|
||||||
Safe fix:
|
Safe fix:
|
||||||
|
|
||||||
- for unsupported local state, move the old manifest aside and regenerate, or
|
- for unsupported local state, move the old manifest aside and regenerate, or
|
||||||
restore a verified current committed snapshot when one is authoritative;
|
restore a verified current committed snapshot when one is authoritative;
|
||||||
- for unsupported committed remote state, regenerate healthy local state as
|
- for unsupported committed remote state, regenerate healthy local state as
|
||||||
needed and republish the session.
|
needed and republish the session; and
|
||||||
|
- for a downgrade rejection, return to the current binary or restore a verified
|
||||||
|
pre-upgrade backup. Do not delete only the speaker fields by hand.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
narratio run 2026-04-04
|
narratio run 2026-04-04
|
||||||
@@ -511,7 +701,7 @@ old files into canonical paths merely to influence selection; availability is
|
|||||||
manifest-owned.
|
manifest-owned.
|
||||||
|
|
||||||
Relevant references: [Operations: Stage Execution and Continuation Behavior](./operations.md#stage-execution-and-continuation-behavior)
|
Relevant references: [Operations: Stage Execution and Continuation Behavior](./operations.md#stage-execution-and-continuation-behavior)
|
||||||
and [Manifest Internals](./internal/manifest.md#analyze-owned-artifact-state).
|
and [Manifest Internals](./internal/manifest.md#transcribe-owned-speaker-state).
|
||||||
|
|
||||||
## Scriptorium private input changed without a rerun
|
## Scriptorium private input changed without a rerun
|
||||||
|
|
||||||
@@ -717,7 +907,8 @@ Symptom:
|
|||||||
|
|
||||||
Likely causes:
|
Likely causes:
|
||||||
|
|
||||||
- invalid `pipeline.secrets.env_dir`;
|
- missing, unreadable, or invalid explicit `--secrets-dir`;
|
||||||
|
- invalid entries in the conventional `secrets/` directory beside the root pipeline;
|
||||||
- missing credential environment variables;
|
- missing credential environment variables;
|
||||||
- invalid S3 endpoint/bucket settings.
|
- invalid S3 endpoint/bucket settings.
|
||||||
|
|
||||||
@@ -725,12 +916,12 @@ Diagnostics:
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
ls -la /path/to/secrets_dir
|
ls -la /path/to/secrets_dir
|
||||||
env | sed 's/=.*//' | grep -E 'OBJECT_STORAGE|AWS|AUDITA|SCRIPTORIUM'
|
env | sed 's/=.*//' | grep -E 'OBJECT_STORAGE|AWS|AUDITA|NOTARIUS|SCRIPTORIUM'
|
||||||
```
|
```
|
||||||
|
|
||||||
Safe fix:
|
Safe fix:
|
||||||
|
|
||||||
- correct secret-file path and permissions;
|
- correct `--secrets-dir`, secret-file type/name, and operator-managed permissions;
|
||||||
- provide required env vars;
|
- provide required env vars;
|
||||||
- keep secret values out of YAML.
|
- keep secret values out of YAML.
|
||||||
|
|
||||||
|
|||||||
@@ -11,18 +11,26 @@ in the [configuration reference](../docs/config.md).
|
|||||||
WhisperX URL.
|
WhisperX URL.
|
||||||
- [Production-shaped pipeline](pipeline.production.yml): S3 storage, publish,
|
- [Production-shaped pipeline](pipeline.production.yml): S3 storage, publish,
|
||||||
external tools, and configured Scriptorium artifacts.
|
external tools, and configured Scriptorium artifacts.
|
||||||
- [Production/testing split bundle](production-testing/pipeline.yml): explicit
|
- [Production-base/testing-overlay bundle](production-testing/pipeline.yml):
|
||||||
`conf.d` imports, a production default, and selectable production/testing
|
explicit `conf.d` imports, one ordinary artifact per `artifacts/<key>.yml`,
|
||||||
overlays. It also demonstrates canonical-party artifact families and a
|
one family per `artifact-families/<key>.yml`, and the fixed
|
||||||
testing-only disabled artifact. Validate it with:
|
`profiles/testing.yml` overlay. Production values live in the base. The
|
||||||
|
bundle demonstrates an input-only artifact dependency, two character families
|
||||||
|
sharing the campaign party, and a testing-only disabled artifact. Inspect it
|
||||||
|
offline with:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
narratio config validate --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml
|
narratio config validate --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml
|
||||||
narratio config diff production testing --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml
|
narratio config show --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml --profile testing
|
||||||
|
narratio config sources --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml --profile testing
|
||||||
|
narratio config artifacts --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml --profile testing
|
||||||
|
narratio config diff default testing --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml
|
||||||
```
|
```
|
||||||
|
|
||||||
`config show` and `config sources` accept the same selectors and remain
|
These inspection commands load configuration and resolve explicit credential
|
||||||
read-only.
|
references, but do not load a session or create runtime state. This bundle is
|
||||||
|
intentionally local-storage and credential-reference free so the walkthrough
|
||||||
|
needs no tools or cloud services.
|
||||||
- [Full annotated pipeline](pipeline.full.annotated.yml): every implemented
|
- [Full annotated pipeline](pipeline.full.annotated.yml): every implemented
|
||||||
pipeline section with explanatory comments.
|
pipeline section with explanatory comments.
|
||||||
- [Extraction subset pipeline](pipeline.extraction-subset.yml): a focused
|
- [Extraction subset pipeline](pipeline.extraction-subset.yml): a focused
|
||||||
@@ -52,11 +60,20 @@ with the sample campaign and a compatible local- or S3-audio session.
|
|||||||
[glossary](campaigns/sample-campaign/glossary.yml), and canonical
|
[glossary](campaigns/sample-campaign/glossary.yml), and canonical
|
||||||
[party](campaigns/sample-campaign/party.yml) fixture, plus an optional
|
[party](campaigns/sample-campaign/party.yml) fixture, plus an optional
|
||||||
[spell-catalog overlay](campaigns/sample-campaign/spell_catalog.json) that
|
[spell-catalog overlay](campaigns/sample-campaign/spell_catalog.json) that
|
||||||
follows the Notarius v0.6 contract. Narratio derives the players projection
|
follows the Notarius v0.7 contract. Narratio derives the players projection
|
||||||
from this party source.
|
from this party source.
|
||||||
- [Sample speaker audio](audio/sample-speaker.flac) is a text placeholder that
|
- [Sample speaker audio](audio/sample-speaker.flac) is a text placeholder that
|
||||||
reserves the expected filename and directory shape. Replace it with a real
|
reserves the expected filename and directory shape. Replace it with a real
|
||||||
FLAC file before running transcription.
|
FLAC file before running transcription.
|
||||||
|
|
||||||
The examples contain environment-variable names but no credential values. They
|
The examples contain environment-variable names but no credential values. Set
|
||||||
use fictional campaign content and reserved example domains.
|
`OBJECT_STORAGE_KEY_ID`, `OBJECT_STORAGE_KEY`, `AUDITA_LLM_API_KEY`, and
|
||||||
|
`NOTARIUS_LLM_API_KEY` to nonempty synthetic or real values before loading the
|
||||||
|
examples that reference them. They use fictional campaign content and reserved
|
||||||
|
example domains.
|
||||||
|
|
||||||
|
For a deployed bundle, keep credential files out of source control in a
|
||||||
|
`secrets/` directory beside the root `pipeline.yml`, one file per referenced
|
||||||
|
environment-variable name, or select another directory with `--secrets-dir`.
|
||||||
|
Static S3 credentials are optional: omit both S3 credential references to use
|
||||||
|
the AWS SDK discovery chain.
|
||||||
|
|||||||
@@ -7,9 +7,8 @@ workspace:
|
|||||||
# Optional: remove run-scoped workdir after successful publish commit.
|
# Optional: remove run-scoped workdir after successful publish commit.
|
||||||
cleanup_after_publish: false
|
cleanup_after_publish: false
|
||||||
|
|
||||||
# Optional: local secret file loader (directory of ENV_VAR_NAME files).
|
# Credential files, when used, belong in a sibling secrets/ directory or in the
|
||||||
# secrets:
|
# directory selected with --secrets-dir; they are not pipeline configuration.
|
||||||
# env_dir: ./secrets
|
|
||||||
|
|
||||||
storage:
|
storage:
|
||||||
# Defaults to "local". Use "s3" explicitly for publish + S3 audio workflows.
|
# Defaults to "local". Use "s3" explicitly for publish + S3 audio workflows.
|
||||||
@@ -24,8 +23,8 @@ storage:
|
|||||||
endpoint: ""
|
endpoint: ""
|
||||||
force_path_style: false
|
force_path_style: false
|
||||||
# Optional; defaults shown explicitly.
|
# Optional; defaults shown explicitly.
|
||||||
access_key_id_env: OBJECT_STORAGE_KEY_ID
|
access_key_id: OBJECT_STORAGE_KEY_ID
|
||||||
secret_access_key_env: OBJECT_STORAGE_KEY
|
secret_access_key: OBJECT_STORAGE_KEY
|
||||||
|
|
||||||
campaigns:
|
campaigns:
|
||||||
# Optional; defaults to /usr/local/share/narratio/campaigns.
|
# Optional; defaults to /usr/local/share/narratio/campaigns.
|
||||||
@@ -94,7 +93,7 @@ audita:
|
|||||||
# Optional overrides; defaults shown explicitly where applicable.
|
# Optional overrides; defaults shown explicitly where applicable.
|
||||||
binary: audita
|
binary: audita
|
||||||
timeout: 3h
|
timeout: 3h
|
||||||
llm_api_key_env: AUDITA_LLM_API_KEY
|
llm_api_key: AUDITA_LLM_API_KEY
|
||||||
modules: [glossary, homophones, spoken_word, grammar]
|
modules: [glossary, homophones, spoken_word, grammar]
|
||||||
base_url: ""
|
base_url: ""
|
||||||
model: ""
|
model: ""
|
||||||
@@ -129,13 +128,26 @@ trim:
|
|||||||
report: false
|
report: false
|
||||||
|
|
||||||
notarius:
|
notarius:
|
||||||
# Optional structured extraction between trim and render.
|
# Optional structured extraction after render and before analyze.
|
||||||
enabled: true
|
enabled: true
|
||||||
binary: notarius
|
binary: notarius
|
||||||
|
# Optional environment-variable reference. Match this name to api_key_env in
|
||||||
|
# the selected Notarius/PromptKit profile; never put the credential here.
|
||||||
|
llm_api_key: NOTARIUS_LLM_API_KEY
|
||||||
|
# Optionally select a Notarius ConfigKit overlay when extraction runs. This
|
||||||
|
# is independent of Narratio's selected profile and may use a different name.
|
||||||
|
# See docs/config.md and docs/integrations/notarius.md for the contracts.
|
||||||
|
# config_profile: downstream-economy
|
||||||
config_path: /usr/local/etc/notarius/config.yml
|
config_path: /usr/local/etc/notarius/config.yml
|
||||||
pipeline_id: dnd-session
|
pipeline_id: dnd-session
|
||||||
timeout: 3h
|
timeout: 3h
|
||||||
working_directory: /usr/local/etc/notarius
|
working_directory: /usr/local/etc/notarius
|
||||||
|
# Reuse compatible accepted Notarius checkpoints during executed extraction.
|
||||||
|
# The Notarius config must enable recording and provide persistent cache roots.
|
||||||
|
resume: true
|
||||||
|
# auto uses or creates a source-addressed chunk plan. Forced Narratio
|
||||||
|
# extraction overrides this with refresh and disables checkpoint loading.
|
||||||
|
chunk_cache: auto
|
||||||
# External campaign references use prepared Narratio source IDs. Omit an
|
# External campaign references use prepared Narratio source IDs. Omit an
|
||||||
# optional binding when the selected Notarius pipeline does not need it.
|
# optional binding when the selected Notarius pipeline does not need it.
|
||||||
references:
|
references:
|
||||||
|
|||||||
@@ -8,8 +8,8 @@ storage:
|
|||||||
bucket: my-dnd-archive
|
bucket: my-dnd-archive
|
||||||
root_prefix: dnd
|
root_prefix: dnd
|
||||||
region: us-east-1
|
region: us-east-1
|
||||||
access_key_id_env: OBJECT_STORAGE_KEY_ID
|
access_key_id: OBJECT_STORAGE_KEY_ID
|
||||||
secret_access_key_env: OBJECT_STORAGE_KEY
|
secret_access_key: OBJECT_STORAGE_KEY
|
||||||
|
|
||||||
campaigns:
|
campaigns:
|
||||||
root: /usr/local/share/narratio/campaigns
|
root: /usr/local/share/narratio/campaigns
|
||||||
@@ -57,7 +57,7 @@ seriatim:
|
|||||||
audita:
|
audita:
|
||||||
binary: audita
|
binary: audita
|
||||||
timeout: 3h
|
timeout: 3h
|
||||||
llm_api_key_env: AUDITA_LLM_API_KEY
|
llm_api_key: AUDITA_LLM_API_KEY
|
||||||
modules: [glossary, homophones, spoken_word, grammar]
|
modules: [glossary, homophones, spoken_word, grammar]
|
||||||
output_schema: audita-v1
|
output_schema: audita-v1
|
||||||
work_dir_retention: auto
|
work_dir_retention: auto
|
||||||
|
|||||||
@@ -1,23 +1,11 @@
|
|||||||
scriptorium:
|
scriptorium:
|
||||||
binary: scriptorium
|
|
||||||
config_path: ./scriptorium/config.yml
|
|
||||||
artifact_families:
|
artifact_families:
|
||||||
character_meta:
|
|
||||||
enabled: true
|
|
||||||
for_each: party.characters
|
|
||||||
prompt_id: dnd.character_meta
|
|
||||||
output_path_pattern: artifacts/characters/{character_id}/meta.md
|
|
||||||
member_vars:
|
|
||||||
character_id: character_id
|
|
||||||
character_name: character.name
|
|
||||||
player_name: player.name
|
|
||||||
class_summary: character.class_summary
|
|
||||||
character_items:
|
character_items:
|
||||||
enabled: true
|
enabled: true
|
||||||
for_each: party.characters
|
for_each: party.characters
|
||||||
prompt_id: dnd.character_items
|
prompt_id: dnd.character_items
|
||||||
|
profile_id: production-placeholder
|
||||||
output_path_pattern: artifacts/characters/{character_id}/items.md
|
output_path_pattern: artifacts/characters/{character_id}/items.md
|
||||||
member_dependencies: [character_meta]
|
|
||||||
inputs:
|
inputs:
|
||||||
character_meta:
|
character_meta:
|
||||||
source: narratio.member_artifact.character_meta
|
source: narratio.member_artifact.character_meta
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
scriptorium:
|
||||||
|
artifact_families:
|
||||||
|
character_meta:
|
||||||
|
enabled: true
|
||||||
|
for_each: party.characters
|
||||||
|
prompt_id: dnd.character_meta
|
||||||
|
profile_id: production-placeholder
|
||||||
|
output_path_pattern: artifacts/characters/{character_id}/meta.md
|
||||||
|
member_vars:
|
||||||
|
character_id: character_id
|
||||||
|
character_name: character.name
|
||||||
|
player_name: player.name
|
||||||
|
class_summary: character.class_summary
|
||||||
11
examples/production-testing/artifacts/player_handout.yml
Normal file
11
examples/production-testing/artifacts/player_handout.yml
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
scriptorium:
|
||||||
|
artifacts:
|
||||||
|
player_handout:
|
||||||
|
enabled: true
|
||||||
|
prompt_id: dnd.player_handout
|
||||||
|
profile_id: production-placeholder
|
||||||
|
output_path: artifacts/player-handout.md
|
||||||
|
inputs:
|
||||||
|
recap:
|
||||||
|
source: narratio.artifact.session_recap
|
||||||
|
required: true
|
||||||
11
examples/production-testing/artifacts/session_recap.yml
Normal file
11
examples/production-testing/artifacts/session_recap.yml
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
scriptorium:
|
||||||
|
artifacts:
|
||||||
|
session_recap:
|
||||||
|
enabled: true
|
||||||
|
prompt_id: dnd.session_recap
|
||||||
|
profile_id: production-placeholder
|
||||||
|
output_path: artifacts/session-recap.md
|
||||||
|
inputs:
|
||||||
|
transcript:
|
||||||
|
source: narratio.transcript.final_trimmed
|
||||||
|
required: true
|
||||||
@@ -10,6 +10,8 @@ audita:
|
|||||||
binary: audita
|
binary: audita
|
||||||
modules: [glossary, grammar]
|
modules: [glossary, grammar]
|
||||||
output_schema: audita-v1
|
output_schema: audita-v1
|
||||||
|
model: narratio-production-model-placeholder
|
||||||
|
validation_model: narratio-production-validator-placeholder
|
||||||
|
|
||||||
normalize:
|
normalize:
|
||||||
output_path: transcripts/final.json
|
output_path: transcripts/final.json
|
||||||
|
|||||||
@@ -1,15 +1,14 @@
|
|||||||
# Copyable production/testing pipeline entry point. Every fragment is named
|
# Copyable production-base/testing-overlay entry point. ConfigKit expands only
|
||||||
# explicitly; Narratio never scans conf.d automatically.
|
# these explicit path and glob imports; Narratio never scans directories itself.
|
||||||
composition:
|
configkit:
|
||||||
imports:
|
imports:
|
||||||
- conf.d/platform.yml
|
- path: conf.d/platform.yml
|
||||||
- conf.d/storage.yml
|
- path: conf.d/storage.yml
|
||||||
- conf.d/transcript.yml
|
- path: conf.d/transcript.yml
|
||||||
- conf.d/artifacts.yml
|
- path: conf.d/publish.yml
|
||||||
- conf.d/publish.yml
|
- glob: artifacts/*.yml
|
||||||
default_profile: production
|
- glob: artifact-families/*.yml
|
||||||
profiles:
|
|
||||||
production:
|
scriptorium:
|
||||||
overlay: profiles/production.yml
|
binary: scriptorium
|
||||||
testing:
|
config_path: ./scriptorium/config.yml
|
||||||
overlay: profiles/testing.yml
|
|
||||||
|
|||||||
@@ -1,10 +0,0 @@
|
|||||||
audita:
|
|
||||||
model: narratio-production-model-placeholder
|
|
||||||
validation_model: narratio-production-validator-placeholder
|
|
||||||
|
|
||||||
scriptorium:
|
|
||||||
artifact_families:
|
|
||||||
character_meta:
|
|
||||||
profile_id: production-placeholder
|
|
||||||
character_items:
|
|
||||||
profile_id: production-placeholder
|
|
||||||
@@ -4,6 +4,10 @@ audita:
|
|||||||
|
|
||||||
scriptorium:
|
scriptorium:
|
||||||
artifacts:
|
artifacts:
|
||||||
|
session_recap:
|
||||||
|
profile_id: testing-placeholder
|
||||||
|
player_handout:
|
||||||
|
profile_id: testing-placeholder
|
||||||
testing_notes:
|
testing_notes:
|
||||||
enabled: false
|
enabled: false
|
||||||
output_path: artifacts/testing-notes.md
|
output_path: artifacts/testing-notes.md
|
||||||
|
|||||||
2
go.mod
2
go.mod
@@ -3,6 +3,7 @@ module gitea.maximumdirect.net/eric/narratio
|
|||||||
go 1.26.0
|
go 1.26.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
gitea.maximumdirect.net/eric/ConfigKit v0.3.1
|
||||||
github.com/aws/aws-sdk-go-v2/config v1.32.17
|
github.com/aws/aws-sdk-go-v2/config v1.32.17
|
||||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.16
|
github.com/aws/aws-sdk-go-v2/credentials v1.19.16
|
||||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
|
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
|
||||||
@@ -28,4 +29,5 @@ require (
|
|||||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect
|
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect
|
||||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect
|
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect
|
||||||
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect
|
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect
|
||||||
|
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
|
||||||
)
|
)
|
||||||
|
|||||||
4
go.sum
4
go.sum
@@ -1,3 +1,5 @@
|
|||||||
|
gitea.maximumdirect.net/eric/ConfigKit v0.3.1 h1:etfTI/dxO3NnS18La0V6IeUh/Iv+3PjVfEyaN8hTuOM=
|
||||||
|
gitea.maximumdirect.net/eric/ConfigKit v0.3.1/go.mod h1:tYWpsrbLyu05r8+1MapWp5fN6ZrX9I8hU2xEPedYmYk=
|
||||||
github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=
|
github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=
|
||||||
github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=
|
github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=
|
||||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho=
|
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho=
|
||||||
@@ -36,6 +38,8 @@ github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
|
|||||||
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||||
|
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
|
||||||
|
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
||||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/url"
|
"net/url"
|
||||||
"os"
|
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
@@ -24,7 +23,8 @@ const MaxReportOutputBytes int64 = 16 * 1024 * 1024
|
|||||||
type SubprocessRunnerConfig struct {
|
type SubprocessRunnerConfig struct {
|
||||||
Binary string
|
Binary string
|
||||||
Timeout time.Duration
|
Timeout time.Duration
|
||||||
LLMAPIKeyEnv string
|
LLMAPIKey string
|
||||||
|
LLMAPIKeyReference string
|
||||||
Modules []string
|
Modules []string
|
||||||
BaseURL string
|
BaseURL string
|
||||||
Model string
|
Model string
|
||||||
@@ -43,7 +43,8 @@ type SubprocessRunnerConfig struct {
|
|||||||
type SubprocessRunner struct {
|
type SubprocessRunner struct {
|
||||||
binary string
|
binary string
|
||||||
timeout time.Duration
|
timeout time.Duration
|
||||||
llmAPIKeyEnv string
|
llmAPIKey string
|
||||||
|
llmAPIKeyReference string
|
||||||
modules []string
|
modules []string
|
||||||
baseURL string
|
baseURL string
|
||||||
model string
|
model string
|
||||||
@@ -62,7 +63,8 @@ type SubprocessRunner struct {
|
|||||||
func NewSubprocessRunnerFromConfigValues(
|
func NewSubprocessRunnerFromConfigValues(
|
||||||
binary string,
|
binary string,
|
||||||
timeout string,
|
timeout string,
|
||||||
llmAPIKeyEnv string,
|
llmAPIKey string,
|
||||||
|
llmAPIKeyReference string,
|
||||||
modules []string,
|
modules []string,
|
||||||
baseURL string,
|
baseURL string,
|
||||||
model string,
|
model string,
|
||||||
@@ -86,7 +88,8 @@ func NewSubprocessRunnerFromConfigValues(
|
|||||||
return NewSubprocessRunner(SubprocessRunnerConfig{
|
return NewSubprocessRunner(SubprocessRunnerConfig{
|
||||||
Binary: binary,
|
Binary: binary,
|
||||||
Timeout: parsedTimeout,
|
Timeout: parsedTimeout,
|
||||||
LLMAPIKeyEnv: llmAPIKeyEnv,
|
LLMAPIKey: llmAPIKey,
|
||||||
|
LLMAPIKeyReference: llmAPIKeyReference,
|
||||||
Modules: modules,
|
Modules: modules,
|
||||||
BaseURL: baseURL,
|
BaseURL: baseURL,
|
||||||
Model: model,
|
Model: model,
|
||||||
@@ -110,6 +113,9 @@ func NewSubprocessRunner(cfg SubprocessRunnerConfig) (*SubprocessRunner, error)
|
|||||||
if cfg.Timeout <= 0 {
|
if cfg.Timeout <= 0 {
|
||||||
return nil, fmt.Errorf("audita timeout must be > 0")
|
return nil, fmt.Errorf("audita timeout must be > 0")
|
||||||
}
|
}
|
||||||
|
if (cfg.LLMAPIKey == "") != (strings.TrimSpace(cfg.LLMAPIKeyReference) == "") {
|
||||||
|
return nil, fmt.Errorf("audita credential value and reference must be provided together")
|
||||||
|
}
|
||||||
for i, module := range cfg.Modules {
|
for i, module := range cfg.Modules {
|
||||||
if strings.TrimSpace(module) == "" {
|
if strings.TrimSpace(module) == "" {
|
||||||
return nil, fmt.Errorf("audita module at index %d is empty", i)
|
return nil, fmt.Errorf("audita module at index %d is empty", i)
|
||||||
@@ -152,7 +158,8 @@ func NewSubprocessRunner(cfg SubprocessRunnerConfig) (*SubprocessRunner, error)
|
|||||||
return &SubprocessRunner{
|
return &SubprocessRunner{
|
||||||
binary: strings.TrimSpace(cfg.Binary),
|
binary: strings.TrimSpace(cfg.Binary),
|
||||||
timeout: cfg.Timeout,
|
timeout: cfg.Timeout,
|
||||||
llmAPIKeyEnv: strings.TrimSpace(cfg.LLMAPIKeyEnv),
|
llmAPIKey: cfg.LLMAPIKey,
|
||||||
|
llmAPIKeyReference: strings.TrimSpace(cfg.LLMAPIKeyReference),
|
||||||
modules: modules,
|
modules: modules,
|
||||||
baseURL: strings.TrimSpace(cfg.BaseURL),
|
baseURL: strings.TrimSpace(cfg.BaseURL),
|
||||||
model: strings.TrimSpace(cfg.Model),
|
model: strings.TrimSpace(cfg.Model),
|
||||||
@@ -195,16 +202,10 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
|
|||||||
}
|
}
|
||||||
args := r.buildArgs(req, reqModules)
|
args := r.buildArgs(req, reqModules)
|
||||||
|
|
||||||
credentialEnvVar := strings.TrimSpace(r.llmAPIKeyEnv)
|
credentialPresent := r.llmAPIKeyReference != ""
|
||||||
credentialPresent := false
|
|
||||||
env := map[string]string{}
|
env := map[string]string{}
|
||||||
if credentialEnvVar != "" {
|
if credentialPresent {
|
||||||
credential, ok := os.LookupEnv(credentialEnvVar)
|
env["AUDITA_LLM_API_KEY"] = r.llmAPIKey
|
||||||
if !ok || strings.TrimSpace(credential) == "" {
|
|
||||||
return PolishResult{}, fmt.Errorf("audita: required credential environment variable %s is not set", credentialEnvVar)
|
|
||||||
}
|
|
||||||
env["AUDITA_LLM_API_KEY"] = credential
|
|
||||||
credentialPresent = true
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if req.GeneratedConfigPath != "" {
|
if req.GeneratedConfigPath != "" {
|
||||||
@@ -269,7 +270,7 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
|
|||||||
"total_llm_concurrency": r.totalLLMConcurrency,
|
"total_llm_concurrency": r.totalLLMConcurrency,
|
||||||
"proposal_llm_concurrency": r.proposalLLMConcurrency,
|
"proposal_llm_concurrency": r.proposalLLMConcurrency,
|
||||||
"validation_llm_concurrency": r.validationLLMConcurrency,
|
"validation_llm_concurrency": r.validationLLMConcurrency,
|
||||||
"credential_env_var": r.llmAPIKeyEnv,
|
"credential_env_var": r.llmAPIKeyReference,
|
||||||
"credential_present": credentialPresent,
|
"credential_present": credentialPresent,
|
||||||
},
|
},
|
||||||
}, nil
|
}, nil
|
||||||
@@ -299,7 +300,7 @@ func (r *SubprocessRunner) failureResult(req PolishRequest, modules []string, ru
|
|||||||
"total_llm_concurrency": r.totalLLMConcurrency,
|
"total_llm_concurrency": r.totalLLMConcurrency,
|
||||||
"proposal_llm_concurrency": r.proposalLLMConcurrency,
|
"proposal_llm_concurrency": r.proposalLLMConcurrency,
|
||||||
"validation_llm_concurrency": r.validationLLMConcurrency,
|
"validation_llm_concurrency": r.validationLLMConcurrency,
|
||||||
"credential_env_var": r.llmAPIKeyEnv,
|
"credential_env_var": r.llmAPIKeyReference,
|
||||||
"credential_present": credentialPresent,
|
"credential_present": credentialPresent,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -375,7 +376,7 @@ func (r *SubprocessRunner) writeInvocationConfig(req PolishRequest, args []strin
|
|||||||
"output_path": req.OutputProcessedPath,
|
"output_path": req.OutputProcessedPath,
|
||||||
"report_path": req.ReportPath,
|
"report_path": req.ReportPath,
|
||||||
"work_dir": req.WorkDir,
|
"work_dir": req.WorkDir,
|
||||||
"credential_env_var": r.llmAPIKeyEnv,
|
"credential_env_var": r.llmAPIKeyReference,
|
||||||
"credential_present": credentialPresent,
|
"credential_present": credentialPresent,
|
||||||
}
|
}
|
||||||
return writeGeneratedYAML(req.GeneratedConfigPath, payload)
|
return writeGeneratedYAML(req.GeneratedConfigPath, payload)
|
||||||
|
|||||||
@@ -6,16 +6,12 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"runtime"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||||
t.Setenv("AUDITA_HELPER_MODE", "success")
|
t.Setenv("AUDITA_HELPER_MODE", "success")
|
||||||
@@ -31,7 +27,8 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
|||||||
runner, err := NewSubprocessRunner(SubprocessRunnerConfig{
|
runner, err := NewSubprocessRunner(SubprocessRunnerConfig{
|
||||||
Binary: wrapper,
|
Binary: wrapper,
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
LLMAPIKey: "super-secret",
|
||||||
|
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||||
Modules: []string{"glossary", "homophones", "glossary"},
|
Modules: []string{"glossary", "homophones", "glossary"},
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
@@ -48,6 +45,7 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewSubprocessRunner() error = %v", err)
|
t.Fatalf("NewSubprocessRunner() error = %v", err)
|
||||||
}
|
}
|
||||||
|
t.Setenv("OPENAI_KEY_SOURCE", "changed-after-resolution")
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
req := PolishRequest{
|
req := PolishRequest{
|
||||||
@@ -132,47 +130,31 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerMissingConfiguredCredentialFails(t *testing.T) {
|
func TestSubprocessRunnerRejectsIncompleteResolvedCredential(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
_, err := NewSubprocessRunner(SubprocessRunnerConfig{
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
}
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
LLMAPIKeyReference: "MISSING_AUDITA_KEY",
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
|
||||||
LLMAPIKeyEnv: "MISSING_AUDITA_KEY",
|
|
||||||
Modules: []string{"glossary"},
|
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
|
||||||
Report: false,
|
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, false)
|
if err == nil || !strings.Contains(err.Error(), "credential value and reference must be provided together") {
|
||||||
|
t.Fatalf("NewSubprocessRunner() error = %v, want credential pair error", err)
|
||||||
_, err := runner.Run(context.Background(), req)
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected error, got nil")
|
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), "required credential environment variable MISSING_AUDITA_KEY is not set") {
|
|
||||||
t.Fatalf("error = %q, want missing credential guidance", err.Error())
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerUnconfiguredCredentialEnvOmitsCredential(t *testing.T) {
|
func TestSubprocessRunnerUnconfiguredCredentialEnvOmitsCredential(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||||
t.Setenv("AUDITA_HELPER_MODE", "success")
|
t.Setenv("AUDITA_HELPER_MODE", "success")
|
||||||
recordPath := filepath.Join(t.TempDir(), "record.json")
|
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||||
|
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
LLMAPIKeyEnv: "",
|
LLMAPIKey: "",
|
||||||
Modules: []string{"glossary"},
|
Modules: []string{"glossary"},
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
Report: false,
|
Report: false,
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, false)
|
req := auditaReqForTest(t, false)
|
||||||
|
|
||||||
@@ -190,9 +172,6 @@ func TestSubprocessRunnerUnconfiguredCredentialEnvOmitsCredential(t *testing.T)
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerOmitsUnspecifiedParentEnvironment(t *testing.T) {
|
func TestSubprocessRunnerOmitsUnspecifiedParentEnvironment(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||||
t.Setenv("AUDITA_HELPER_MODE", "success")
|
t.Setenv("AUDITA_HELPER_MODE", "success")
|
||||||
t.Setenv("AUDITA_INHERITED_MARKER", "inherited-from-parent")
|
t.Setenv("AUDITA_INHERITED_MARKER", "inherited-from-parent")
|
||||||
@@ -200,13 +179,13 @@ func TestSubprocessRunnerOmitsUnspecifiedParentEnvironment(t *testing.T) {
|
|||||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||||
|
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
Modules: []string{"glossary"},
|
Modules: []string{"glossary"},
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
Report: false,
|
Report: false,
|
||||||
LLMAPIKeyEnv: "",
|
LLMAPIKey: "",
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, false)
|
req := auditaReqForTest(t, false)
|
||||||
if _, err := runner.Run(context.Background(), req); err != nil {
|
if _, err := runner.Run(context.Background(), req); err != nil {
|
||||||
@@ -220,21 +199,18 @@ func TestSubprocessRunnerOmitsUnspecifiedParentEnvironment(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerOmitsModulesFlagWhenNotConfigured(t *testing.T) {
|
func TestSubprocessRunnerOmitsModulesFlagWhenNotConfigured(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||||
t.Setenv("AUDITA_HELPER_MODE", "success")
|
t.Setenv("AUDITA_HELPER_MODE", "success")
|
||||||
recordPath := filepath.Join(t.TempDir(), "record.json")
|
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||||
|
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
LLMAPIKeyEnv: "",
|
LLMAPIKey: "",
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
Report: false,
|
Report: false,
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, false)
|
req := auditaReqForTest(t, false)
|
||||||
if _, err := runner.Run(context.Background(), req); err != nil {
|
if _, err := runner.Run(context.Background(), req); err != nil {
|
||||||
@@ -249,19 +225,16 @@ func TestSubprocessRunnerOmitsModulesFlagWhenNotConfigured(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerOmitsBaseURLAndModelFlagsWhenNotConfigured(t *testing.T) {
|
func TestSubprocessRunnerOmitsBaseURLAndModelFlagsWhenNotConfigured(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||||
t.Setenv("AUDITA_HELPER_MODE", "success")
|
t.Setenv("AUDITA_HELPER_MODE", "success")
|
||||||
recordPath := filepath.Join(t.TempDir(), "record.json")
|
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||||
|
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
LLMAPIKeyEnv: "",
|
LLMAPIKey: "",
|
||||||
Report: false,
|
Report: false,
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, false)
|
req := auditaReqForTest(t, false)
|
||||||
if _, err := runner.Run(context.Background(), req); err != nil {
|
if _, err := runner.Run(context.Background(), req); err != nil {
|
||||||
@@ -279,22 +252,20 @@ func TestSubprocessRunnerOmitsBaseURLAndModelFlagsWhenNotConfigured(t *testing.T
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
|
func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||||
t.Setenv("AUDITA_HELPER_MODE", "fail")
|
t.Setenv("AUDITA_HELPER_MODE", "fail")
|
||||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
|
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
LLMAPIKey: "super-secret",
|
||||||
Modules: []string{"glossary"},
|
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
Modules: []string{"glossary"},
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Report: true,
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
|
Report: true,
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, true)
|
req := auditaReqForTest(t, true)
|
||||||
_, err := runner.Run(context.Background(), req)
|
_, err := runner.Run(context.Background(), req)
|
||||||
@@ -310,25 +281,26 @@ func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
|
|||||||
if !strings.Contains(err.Error(), req.StdoutLogPath) || !strings.Contains(err.Error(), req.StderrLogPath) {
|
if !strings.Contains(err.Error(), req.StdoutLogPath) || !strings.Contains(err.Error(), req.StderrLogPath) {
|
||||||
t.Fatalf("error = %q, want stdout/stderr log paths", err.Error())
|
t.Fatalf("error = %q, want stdout/stderr log paths", err.Error())
|
||||||
}
|
}
|
||||||
|
if strings.Contains(err.Error(), "super-secret") {
|
||||||
|
t.Fatalf("failure diagnostic exposed credential value: %q", err.Error())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerSubprocessFailureAddsStderrDescriptorHint(t *testing.T) {
|
func TestSubprocessRunnerSubprocessFailureAddsStderrDescriptorHint(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||||
t.Setenv("AUDITA_HELPER_MODE", "fail_bad_fd")
|
t.Setenv("AUDITA_HELPER_MODE", "fail_bad_fd")
|
||||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
|
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
LLMAPIKey: "super-secret",
|
||||||
Modules: []string{"glossary"},
|
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
Modules: []string{"glossary"},
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Report: true,
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
|
Report: true,
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, true)
|
req := auditaReqForTest(t, true)
|
||||||
_, err := runner.Run(context.Background(), req)
|
_, err := runner.Run(context.Background(), req)
|
||||||
@@ -341,22 +313,20 @@ func TestSubprocessRunnerSubprocessFailureAddsStderrDescriptorHint(t *testing.T)
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerMissingOutputFails(t *testing.T) {
|
func TestSubprocessRunnerMissingOutputFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||||
t.Setenv("AUDITA_HELPER_MODE", "missing_output")
|
t.Setenv("AUDITA_HELPER_MODE", "missing_output")
|
||||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
|
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
LLMAPIKey: "super-secret",
|
||||||
Modules: []string{"glossary"},
|
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
Modules: []string{"glossary"},
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Report: false,
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
|
Report: false,
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, false)
|
req := auditaReqForTest(t, false)
|
||||||
_, err := runner.Run(context.Background(), req)
|
_, err := runner.Run(context.Background(), req)
|
||||||
@@ -369,22 +339,20 @@ func TestSubprocessRunnerMissingOutputFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) {
|
func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||||
t.Setenv("AUDITA_HELPER_MODE", "invalid_output")
|
t.Setenv("AUDITA_HELPER_MODE", "invalid_output")
|
||||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
|
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
LLMAPIKey: "super-secret",
|
||||||
Modules: []string{"glossary"},
|
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
Modules: []string{"glossary"},
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Report: false,
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
|
Report: false,
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, false)
|
req := auditaReqForTest(t, false)
|
||||||
_, err := runner.Run(context.Background(), req)
|
_, err := runner.Run(context.Background(), req)
|
||||||
@@ -397,22 +365,20 @@ func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerSegmentsMissingFails(t *testing.T) {
|
func TestSubprocessRunnerSegmentsMissingFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||||
t.Setenv("AUDITA_HELPER_MODE", "segments_missing")
|
t.Setenv("AUDITA_HELPER_MODE", "segments_missing")
|
||||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
|
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
LLMAPIKey: "super-secret",
|
||||||
Modules: []string{"glossary"},
|
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
Modules: []string{"glossary"},
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Report: false,
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
|
Report: false,
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, false)
|
req := auditaReqForTest(t, false)
|
||||||
_, err := runner.Run(context.Background(), req)
|
_, err := runner.Run(context.Background(), req)
|
||||||
@@ -425,22 +391,20 @@ func TestSubprocessRunnerSegmentsMissingFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
|
func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||||
t.Setenv("AUDITA_HELPER_MODE", "invalid_report")
|
t.Setenv("AUDITA_HELPER_MODE", "invalid_report")
|
||||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
|
|
||||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||||
Binary: writeAuditaHelperWrapper(t),
|
Binary: writeAuditaHelperWrapper(t),
|
||||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
LLMAPIKey: "super-secret",
|
||||||
Modules: []string{"glossary"},
|
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||||
BaseURL: "https://openrouter.ai/api/v1",
|
Modules: []string{"glossary"},
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
BaseURL: "https://openrouter.ai/api/v1",
|
||||||
Report: true,
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
|
Report: true,
|
||||||
})
|
})
|
||||||
req := auditaReqForTest(t, true)
|
req := auditaReqForTest(t, true)
|
||||||
_, err := runner.Run(context.Background(), req)
|
_, err := runner.Run(context.Background(), req)
|
||||||
@@ -453,21 +417,22 @@ func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerConstructorValidation(t *testing.T) {
|
func TestSubprocessRunnerConstructorValidation(t *testing.T) {
|
||||||
_, err := NewSubprocessRunnerFromConfigValues("", "3h", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", "", "", "", "", nil, nil, "", nil, true)
|
_, err := NewSubprocessRunnerFromConfigValues("", "3h", "synthetic-secret", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", "", "", "", "", nil, nil, "", nil, true)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected binary validation error")
|
t.Fatal("expected binary validation error")
|
||||||
}
|
}
|
||||||
_, err = NewSubprocessRunnerFromConfigValues("audita", "bad", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", "", "", "", "", nil, nil, "", nil, true)
|
_, err = NewSubprocessRunnerFromConfigValues("audita", "bad", "synthetic-secret", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", "", "", "", "", nil, nil, "", nil, true)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected timeout parse error")
|
t.Fatal("expected timeout parse error")
|
||||||
}
|
}
|
||||||
_, err = NewSubprocessRunner(SubprocessRunnerConfig{
|
_, err = NewSubprocessRunner(SubprocessRunnerConfig{
|
||||||
Binary: "audita",
|
Binary: "audita",
|
||||||
Timeout: mustParseAuditaDuration(t, "1s"),
|
Timeout: mustParseAuditaDuration(t, "1s"),
|
||||||
LLMAPIKeyEnv: "AUDITA_LLM_API_KEY",
|
LLMAPIKey: "synthetic-secret",
|
||||||
Modules: []string{"glossary"},
|
LLMAPIKeyReference: "AUDITA_LLM_API_KEY",
|
||||||
BaseURL: "://",
|
Modules: []string{"glossary"},
|
||||||
Model: "openrouter/google/gemma-4-31b-it",
|
BaseURL: "://",
|
||||||
|
Model: "openrouter/google/gemma-4-31b-it",
|
||||||
})
|
})
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected base url validation error")
|
t.Fatal("expected base url validation error")
|
||||||
|
|||||||
@@ -15,6 +15,10 @@ func (f *FakeRunner) Run(ctx context.Context, req RunRequest) (RunResult, error)
|
|||||||
return RunResult{}, err
|
return RunResult{}, err
|
||||||
}
|
}
|
||||||
copyRequest := req
|
copyRequest := req
|
||||||
|
if req.ConfigProfile != nil {
|
||||||
|
profile := *req.ConfigProfile
|
||||||
|
copyRequest.ConfigProfile = &profile
|
||||||
|
}
|
||||||
copyRequest.References = append([]ReferenceBinding(nil), req.References...)
|
copyRequest.References = append([]ReferenceBinding(nil), req.References...)
|
||||||
f.Requests = append(f.Requests, copyRequest)
|
f.Requests = append(f.Requests, copyRequest)
|
||||||
if f.Err != nil {
|
if f.Err != nil {
|
||||||
|
|||||||
@@ -4,6 +4,8 @@ package notarius
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
)
|
)
|
||||||
|
|
||||||
const ReceiptSchemaVersion = "notarius.run-result.v2"
|
const ReceiptSchemaVersion = "notarius.run-result.v2"
|
||||||
@@ -22,7 +24,10 @@ type ReferenceBinding struct {
|
|||||||
|
|
||||||
// RunRequest contains the resolved inputs and diagnostic destinations for one invocation.
|
// RunRequest contains the resolved inputs and diagnostic destinations for one invocation.
|
||||||
type RunRequest struct {
|
type RunRequest struct {
|
||||||
Binary string
|
Binary string
|
||||||
|
// LLMAPIKey is an optional already-resolved credential reference.
|
||||||
|
LLMAPIKey *configkit.Secret
|
||||||
|
ConfigProfile *string
|
||||||
ConfigPath string
|
ConfigPath string
|
||||||
PipelineID string
|
PipelineID string
|
||||||
InputPath string
|
InputPath string
|
||||||
@@ -31,6 +36,8 @@ type RunRequest struct {
|
|||||||
ReceiptPath string
|
ReceiptPath string
|
||||||
LogPath string
|
LogPath string
|
||||||
Timeout time.Duration
|
Timeout time.Duration
|
||||||
|
Resume bool
|
||||||
|
ChunkCache string
|
||||||
References []ReferenceBinding
|
References []ReferenceBinding
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -52,6 +52,20 @@ func (r *SubprocessRunner) Run(ctx context.Context, req RunRequest) (RunResult,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return RunResult{}, err
|
return RunResult{}, err
|
||||||
}
|
}
|
||||||
|
var envOverrides map[string]string
|
||||||
|
var sensitiveEnvNames []string
|
||||||
|
if req.LLMAPIKey != nil {
|
||||||
|
if !req.LLMAPIKey.IsResolved() {
|
||||||
|
return RunResult{}, fmt.Errorf("notarius LLM API credential must be resolved and nonempty")
|
||||||
|
}
|
||||||
|
name := req.LLMAPIKey.EnvName()
|
||||||
|
value := req.LLMAPIKey.Value()
|
||||||
|
if value == "" {
|
||||||
|
return RunResult{}, fmt.Errorf("notarius LLM API credential must be resolved and nonempty")
|
||||||
|
}
|
||||||
|
envOverrides = map[string]string{name: value}
|
||||||
|
sensitiveEnvNames = []string{name}
|
||||||
|
}
|
||||||
|
|
||||||
args := []string{
|
args := []string{
|
||||||
"run", req.PipelineID,
|
"run", req.PipelineID,
|
||||||
@@ -59,18 +73,29 @@ func (r *SubprocessRunner) Run(ctx context.Context, req RunRequest) (RunResult,
|
|||||||
"--input", req.InputPath,
|
"--input", req.InputPath,
|
||||||
"--output-dir", req.OutputRoot,
|
"--output-dir", req.OutputRoot,
|
||||||
}
|
}
|
||||||
|
if req.Resume {
|
||||||
|
args = append(args, "--resume")
|
||||||
|
}
|
||||||
|
if req.ChunkCache != "" {
|
||||||
|
args = append(args, "--chunk_cache", req.ChunkCache)
|
||||||
|
}
|
||||||
for _, reference := range references {
|
for _, reference := range references {
|
||||||
args = append(args, "--reference", reference.Selector+"="+reference.Path)
|
args = append(args, "--reference", reference.Selector+"="+reference.Path)
|
||||||
}
|
}
|
||||||
|
if req.ConfigProfile != nil {
|
||||||
|
args = append(args, "--config-profile", *req.ConfigProfile)
|
||||||
|
}
|
||||||
args = append(args, "--json")
|
args = append(args, "--json")
|
||||||
processResult, err := r.run(ctx, subprocess.RunRequest{
|
processResult, err := r.run(ctx, subprocess.RunRequest{
|
||||||
Executable: req.Binary,
|
Executable: req.Binary,
|
||||||
Args: args,
|
Args: args,
|
||||||
WorkingDir: req.WorkingDirectory,
|
WorkingDir: req.WorkingDirectory,
|
||||||
Timeout: req.Timeout,
|
Timeout: req.Timeout,
|
||||||
DiagnosticOwner: "notarius",
|
DiagnosticOwner: "notarius",
|
||||||
StdoutLogPath: req.ReceiptPath,
|
StdoutLogPath: req.ReceiptPath,
|
||||||
StderrLogPath: req.LogPath,
|
StderrLogPath: req.LogPath,
|
||||||
|
EnvOverrides: envOverrides,
|
||||||
|
SensitiveEnvNames: sensitiveEnvNames,
|
||||||
})
|
})
|
||||||
baseResult := RunResult{
|
baseResult := RunResult{
|
||||||
ReceiptPath: req.ReceiptPath,
|
ReceiptPath: req.ReceiptPath,
|
||||||
@@ -143,6 +168,9 @@ func validateRunRequest(req RunRequest) ([]ReferenceBinding, error) {
|
|||||||
if req.Timeout <= 0 {
|
if req.Timeout <= 0 {
|
||||||
return nil, fmt.Errorf("notarius timeout must be positive")
|
return nil, fmt.Errorf("notarius timeout must be positive")
|
||||||
}
|
}
|
||||||
|
if !validChunkCacheMode(req.ChunkCache) {
|
||||||
|
return nil, fmt.Errorf("notarius chunk cache mode must be empty, auto, bypass, or refresh")
|
||||||
|
}
|
||||||
for label, path := range map[string]string{
|
for label, path := range map[string]string{
|
||||||
"config": req.ConfigPath,
|
"config": req.ConfigPath,
|
||||||
"input": req.InputPath,
|
"input": req.InputPath,
|
||||||
@@ -201,6 +229,15 @@ func validateRunRequest(req RunRequest) ([]ReferenceBinding, error) {
|
|||||||
return references, nil
|
return references, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validChunkCacheMode(mode string) bool {
|
||||||
|
switch mode {
|
||||||
|
case "", "auto", "bypass", "refresh":
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type receiptDocument struct {
|
type receiptDocument struct {
|
||||||
SchemaVersion string `json:"schema_version"`
|
SchemaVersion string `json:"schema_version"`
|
||||||
RunID string `json:"run_id"`
|
RunID string `json:"run_id"`
|
||||||
@@ -248,7 +285,7 @@ func validateValidationSummaries(documents []validationSummaryDocument) ([]Valid
|
|||||||
if document.Status != "complete" && document.Status != "rejected" && document.Status != "incomplete" {
|
if document.Status != "complete" && document.Status != "rejected" && document.Status != "incomplete" {
|
||||||
return nil, fmt.Errorf("notarius validation summary status %q is invalid", document.Status)
|
return nil, fmt.Errorf("notarius validation summary status %q is invalid", document.Status)
|
||||||
}
|
}
|
||||||
if document.ProducerAttemptCount == nil || *document.ProducerAttemptCount <= 0 || !validTerminalAction(document.TerminalAction) {
|
if document.ProducerAttemptCount == nil || *document.ProducerAttemptCount < 0 || !validTerminalAction(document.TerminalAction) {
|
||||||
return nil, fmt.Errorf("notarius validation summary is missing required fields")
|
return nil, fmt.Errorf("notarius validation summary is missing required fields")
|
||||||
}
|
}
|
||||||
summaries = append(summaries, ValidationSummary{
|
summaries = append(summaries, ValidationSummary{
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
sharedsubprocess "gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
sharedsubprocess "gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -76,6 +77,174 @@ func TestSubprocessRunnerBuildsExactInvocationAndDiscoversBundle(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSubprocessRunnerMapsCacheControlsToArguments(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
resume bool
|
||||||
|
chunkCache string
|
||||||
|
want []string
|
||||||
|
}{
|
||||||
|
{name: "resume", resume: true, want: []string{"--resume"}},
|
||||||
|
{name: "auto", chunkCache: "auto", want: []string{"--chunk_cache", "auto"}},
|
||||||
|
{name: "bypass", chunkCache: "bypass", want: []string{"--chunk_cache", "bypass"}},
|
||||||
|
{name: "refresh", chunkCache: "refresh", want: []string{"--chunk_cache", "refresh"}},
|
||||||
|
{name: "resume and mode", resume: true, chunkCache: "auto", want: []string{"--resume", "--chunk_cache", "auto"}},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
req := validRunRequest(t)
|
||||||
|
req.Resume = test.resume
|
||||||
|
req.ChunkCache = test.chunkCache
|
||||||
|
var captured sharedsubprocess.RunRequest
|
||||||
|
runner := &SubprocessRunner{run: func(_ context.Context, processReq sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
|
||||||
|
captured = processReq
|
||||||
|
writeValidBundleAndReceipt(t, req, false)
|
||||||
|
return sharedsubprocess.RunResult{ExitCode: 0}, nil
|
||||||
|
}}
|
||||||
|
|
||||||
|
if _, err := runner.Run(context.Background(), req); err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
want := []string{
|
||||||
|
"run", req.PipelineID, "--config", req.ConfigPath, "--input", req.InputPath,
|
||||||
|
"--output-dir", req.OutputRoot,
|
||||||
|
}
|
||||||
|
want = append(want, test.want...)
|
||||||
|
want = append(want, "--json")
|
||||||
|
if !reflect.DeepEqual(captured.Args, want) {
|
||||||
|
t.Fatalf("subprocess args = %#v, want %#v", captured.Args, want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessRunnerMapsConfigProfileToArgumentPair(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
supplied bool
|
||||||
|
value string
|
||||||
|
}{
|
||||||
|
{name: "omitted"},
|
||||||
|
{name: "empty", supplied: true},
|
||||||
|
{name: "default", supplied: true, value: "default"},
|
||||||
|
{name: "named", supplied: true, value: "inexpensive"},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
req := validRunRequest(t)
|
||||||
|
if test.supplied {
|
||||||
|
req.ConfigProfile = &test.value
|
||||||
|
}
|
||||||
|
var captured sharedsubprocess.RunRequest
|
||||||
|
runner := &SubprocessRunner{run: func(_ context.Context, processReq sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
|
||||||
|
captured = processReq
|
||||||
|
writeValidBundleAndReceipt(t, req, false)
|
||||||
|
return sharedsubprocess.RunResult{ExitCode: 0}, nil
|
||||||
|
}}
|
||||||
|
|
||||||
|
if _, err := runner.Run(context.Background(), req); err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
flagIndex := -1
|
||||||
|
for index, arg := range captured.Args {
|
||||||
|
if arg == "--config-profile" {
|
||||||
|
flagIndex = index
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !test.supplied {
|
||||||
|
if flagIndex >= 0 {
|
||||||
|
t.Fatalf("subprocess args unexpectedly contain config profile: %#v", captured.Args)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if flagIndex < 0 || flagIndex+2 >= len(captured.Args) || captured.Args[flagIndex+1] != test.value || captured.Args[flagIndex+2] != "--json" {
|
||||||
|
t.Fatalf("config profile argument boundary = %#v, want flag, exact value, then --json", captured.Args)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessRunnerRejectsInvalidChunkCacheBeforeLaunch(t *testing.T) {
|
||||||
|
for _, mode := range []string{"AUTO", "invalid", " auto "} {
|
||||||
|
t.Run(mode, func(t *testing.T) {
|
||||||
|
req := validRunRequest(t)
|
||||||
|
req.ChunkCache = mode
|
||||||
|
started := false
|
||||||
|
runner := &SubprocessRunner{run: func(context.Context, sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
|
||||||
|
started = true
|
||||||
|
return sharedsubprocess.RunResult{}, nil
|
||||||
|
}}
|
||||||
|
_, err := runner.Run(context.Background(), req)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "chunk cache mode") {
|
||||||
|
t.Fatalf("Run() error = %v, want chunk cache validation failure", err)
|
||||||
|
}
|
||||||
|
if started {
|
||||||
|
t.Fatal("subprocess started with invalid chunk cache mode")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessRunnerForwardsLoadedCredentialWithoutRereadingEnvironment(t *testing.T) {
|
||||||
|
const envName = "NARRATIO_NOTARIUS_CUSTOM_LLM_KEY"
|
||||||
|
const loadedValue = "loaded-notarius-credential"
|
||||||
|
credential := loadTestCredential(t, envName, loadedValue)
|
||||||
|
t.Setenv(envName, "changed-parent-credential")
|
||||||
|
|
||||||
|
req := validRunRequest(t)
|
||||||
|
req.LLMAPIKey = credential
|
||||||
|
var captured sharedsubprocess.RunRequest
|
||||||
|
runner := &SubprocessRunner{run: func(_ context.Context, processReq sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
|
||||||
|
captured = processReq
|
||||||
|
writeValidBundleAndReceipt(t, req, false)
|
||||||
|
return sharedsubprocess.RunResult{ExitCode: 0}, nil
|
||||||
|
}}
|
||||||
|
if _, err := runner.Run(context.Background(), req); err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(captured.EnvOverrides) != 1 || captured.EnvOverrides[envName] != loadedValue {
|
||||||
|
t.Fatal("subprocess did not receive exactly the loaded credential override")
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(captured.SensitiveEnvNames, []string{envName}) {
|
||||||
|
t.Fatalf("sensitive environment names = %#v, want configured reference name", captured.SensitiveEnvNames)
|
||||||
|
}
|
||||||
|
wantArgs := []string{
|
||||||
|
"run", req.PipelineID, "--config", req.ConfigPath, "--input", req.InputPath,
|
||||||
|
"--output-dir", req.OutputRoot, "--json",
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(captured.Args, wantArgs) {
|
||||||
|
t.Fatalf("subprocess args = %#v, want %#v", captured.Args, wantArgs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessRunnerRejectsInvalidProgrammaticCredentialBeforeLaunch(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
reference configkit.Secret
|
||||||
|
}{
|
||||||
|
{name: "unresolved", reference: configkit.SecretEnv("NARRATIO_NOTARIUS_UNRESOLVED_KEY")},
|
||||||
|
{name: "empty", reference: configkit.SecretEnv("")},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
req := validRunRequest(t)
|
||||||
|
req.LLMAPIKey = &test.reference
|
||||||
|
started := false
|
||||||
|
runner := &SubprocessRunner{run: func(context.Context, sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
|
||||||
|
started = true
|
||||||
|
return sharedsubprocess.RunResult{}, nil
|
||||||
|
}}
|
||||||
|
_, err := runner.Run(context.Background(), req)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "resolved and nonempty") {
|
||||||
|
t.Fatalf("Run() error = %v, want credential validation failure", err)
|
||||||
|
}
|
||||||
|
if started {
|
||||||
|
t.Fatal("subprocess started with an invalid credential")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerBuildsOrderedReferenceArguments(t *testing.T) {
|
func TestSubprocessRunnerBuildsOrderedReferenceArguments(t *testing.T) {
|
||||||
req := validRunRequest(t)
|
req := validRunRequest(t)
|
||||||
referenceRoot := t.TempDir()
|
referenceRoot := t.TempDir()
|
||||||
@@ -172,16 +341,23 @@ func TestSubprocessRunnerRejectsInvalidReferencesBeforeLaunch(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerUsesMinimalEnvironmentAndSeparatesStreams(t *testing.T) {
|
func TestSubprocessRunnerUsesMinimalEnvironmentAndSeparatesStreams(t *testing.T) {
|
||||||
|
const envName = "NARRATIO_NOTARIUS_CUSTOM_CAPTURE_KEY"
|
||||||
|
const canary = "notarius-captured-stream-canary"
|
||||||
|
const unrelated = "NARRATIO_NOTARIUS_UNRELATED_PARENT_KEY"
|
||||||
req := validRunRequest(t)
|
req := validRunRequest(t)
|
||||||
|
req.LLMAPIKey = loadTestCredential(t, envName, canary)
|
||||||
writeValidBundleAndReceipt(t, req, false)
|
writeValidBundleAndReceipt(t, req, false)
|
||||||
receiptFixture := req.ReceiptPath + ".fixture"
|
receiptFixture := req.ReceiptPath + ".fixture"
|
||||||
data, err := os.ReadFile(req.ReceiptPath)
|
data, err := os.ReadFile(req.ReceiptPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("ReadFile(receipt) error = %v", err)
|
t.Fatalf("ReadFile(receipt) error = %v", err)
|
||||||
}
|
}
|
||||||
if err := os.WriteFile(receiptFixture, data, 0o644); err != nil {
|
var receipt map[string]any
|
||||||
t.Fatalf("WriteFile(receipt fixture) error = %v", err)
|
if err := json.Unmarshal(data, &receipt); err != nil {
|
||||||
|
t.Fatalf("decode receipt fixture: %v", err)
|
||||||
}
|
}
|
||||||
|
receipt["credential_echo"] = canary
|
||||||
|
writeJSONFile(t, receiptFixture, receipt)
|
||||||
if err := os.Remove(req.ReceiptPath); err != nil {
|
if err := os.Remove(req.ReceiptPath); err != nil {
|
||||||
t.Fatalf("Remove(receipt) error = %v", err)
|
t.Fatalf("Remove(receipt) error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -193,27 +369,55 @@ func TestSubprocessRunnerUsesMinimalEnvironmentAndSeparatesStreams(t *testing.T)
|
|||||||
script := writeShellScript(t, `#!/bin/sh
|
script := writeShellScript(t, `#!/bin/sh
|
||||||
pwd > "$NOTARIUS_CAPTURE_DIR/working-directory"
|
pwd > "$NOTARIUS_CAPTURE_DIR/working-directory"
|
||||||
printf '%s' "$NOTARIUS_INHERITED_VALUE" > "$NOTARIUS_CAPTURE_DIR/environment"
|
printf '%s' "$NOTARIUS_INHERITED_VALUE" > "$NOTARIUS_CAPTURE_DIR/environment"
|
||||||
printf 'diagnostic stream\n' >&2
|
if [ "$`+envName+`" != '`+canary+`' ]; then
|
||||||
|
printf 'selected credential mismatch\n' >&2
|
||||||
|
exit 8
|
||||||
|
fi
|
||||||
|
if [ -n "$`+unrelated+`" ]; then
|
||||||
|
printf 'unrelated credential inherited\n' >&2
|
||||||
|
exit 9
|
||||||
|
fi
|
||||||
|
printf '%s\n' "$`+envName+`" >&2
|
||||||
cat "$NOTARIUS_RECEIPT_FIXTURE"
|
cat "$NOTARIUS_RECEIPT_FIXTURE"
|
||||||
`)
|
`)
|
||||||
req.Binary = script
|
req.Binary = script
|
||||||
t.Setenv("NOTARIUS_CAPTURE_DIR", captureDir)
|
t.Setenv("NOTARIUS_CAPTURE_DIR", captureDir)
|
||||||
t.Setenv("NOTARIUS_INHERITED_VALUE", "inherited-value")
|
t.Setenv("NOTARIUS_INHERITED_VALUE", "inherited-value")
|
||||||
t.Setenv("NOTARIUS_RECEIPT_FIXTURE", receiptFixture)
|
t.Setenv("NOTARIUS_RECEIPT_FIXTURE", receiptFixture)
|
||||||
|
t.Setenv(unrelated, "unrelated-parent-credential")
|
||||||
|
|
||||||
if _, err := NewSubprocessRunner().Run(context.Background(), req); err != nil {
|
if _, err := NewSubprocessRunner().Run(context.Background(), req); err != nil {
|
||||||
t.Fatalf("Run() error = %v", err)
|
t.Fatalf("Run() error = %v", err)
|
||||||
}
|
}
|
||||||
assertTextFile(t, filepath.Join(captureDir, "working-directory"), req.WorkingDirectory+"\n")
|
assertTextFile(t, filepath.Join(captureDir, "working-directory"), req.WorkingDirectory+"\n")
|
||||||
assertTextFile(t, filepath.Join(captureDir, "environment"), "")
|
assertTextFile(t, filepath.Join(captureDir, "environment"), "")
|
||||||
assertTextFile(t, req.LogPath, "diagnostic stream\n")
|
assertRedactedCredentialFile(t, req.LogPath, canary)
|
||||||
receiptBytes, err := os.ReadFile(req.ReceiptPath)
|
receiptBytes, err := os.ReadFile(req.ReceiptPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("ReadFile(receipt) error = %v", err)
|
t.Fatalf("ReadFile(receipt) error = %v", err)
|
||||||
}
|
}
|
||||||
if strings.Contains(string(receiptBytes), "diagnostic stream") {
|
assertRedactedCredentialBytes(t, receiptBytes, canary)
|
||||||
t.Fatal("receipt contains stderr output")
|
}
|
||||||
|
|
||||||
|
func TestSubprocessRunnerRedactsCredentialOnProcessFailure(t *testing.T) {
|
||||||
|
const envName = "NARRATIO_NOTARIUS_CUSTOM_FAILURE_KEY"
|
||||||
|
const canary = "notarius-failure-stream-canary"
|
||||||
|
req := validRunRequest(t)
|
||||||
|
req.LLMAPIKey = loadTestCredential(t, envName, canary)
|
||||||
|
req.Binary = writeShellScript(t, `#!/bin/sh
|
||||||
|
printf '%s\n' "$`+envName+`"
|
||||||
|
printf '%s\n' "$`+envName+`" >&2
|
||||||
|
exit 7
|
||||||
|
`)
|
||||||
|
_, err := NewSubprocessRunner().Run(context.Background(), req)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Run() error = nil, want process failure")
|
||||||
}
|
}
|
||||||
|
if strings.Contains(err.Error(), canary) || !strings.Contains(err.Error(), "<redacted>") {
|
||||||
|
t.Fatal("process failure did not return redacted diagnostics")
|
||||||
|
}
|
||||||
|
assertRedactedCredentialFile(t, req.ReceiptPath, canary)
|
||||||
|
assertRedactedCredentialFile(t, req.LogPath, canary)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerReturnsProcessFailuresWithoutParsingStdout(t *testing.T) {
|
func TestSubprocessRunnerReturnsProcessFailuresWithoutParsingStdout(t *testing.T) {
|
||||||
@@ -231,6 +435,10 @@ func TestSubprocessRunnerReturnsProcessFailuresWithoutParsingStdout(t *testing.T
|
|||||||
for _, test := range tests {
|
for _, test := range tests {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
req := validRunRequest(t)
|
req := validRunRequest(t)
|
||||||
|
if test.name == "nonzero" {
|
||||||
|
profile := "missing-profile"
|
||||||
|
req.ConfigProfile = &profile
|
||||||
|
}
|
||||||
req.Binary = writeShellScript(t, "#!/bin/sh\n"+test.scriptBody)
|
req.Binary = writeShellScript(t, "#!/bin/sh\n"+test.scriptBody)
|
||||||
req.Timeout = test.timeout
|
req.Timeout = test.timeout
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -527,7 +735,8 @@ func TestLoadDiagnosticSummariesValidateBoundsAndTolerateUnknownFields(t *testin
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestFakeRunnerCapturesRequestsAndHonorsContextAndError(t *testing.T) {
|
func TestFakeRunnerCapturesRequestsAndHonorsContextAndError(t *testing.T) {
|
||||||
req := RunRequest{PipelineID: "pipeline", References: []ReferenceBinding{{Selector: "party", Path: "/references/party.json"}}}
|
profile := "testing"
|
||||||
|
req := RunRequest{PipelineID: "pipeline", ConfigProfile: &profile, References: []ReferenceBinding{{Selector: "party", Path: "/references/party.json"}}}
|
||||||
want := RunResult{BundleRoot: "/bundle"}
|
want := RunResult{BundleRoot: "/bundle"}
|
||||||
fake := &FakeRunner{Result: want}
|
fake := &FakeRunner{Result: want}
|
||||||
got, err := fake.Run(context.Background(), req)
|
got, err := fake.Run(context.Background(), req)
|
||||||
@@ -538,6 +747,10 @@ func TestFakeRunnerCapturesRequestsAndHonorsContextAndError(t *testing.T) {
|
|||||||
if fake.Requests[0].References[0].Path != "/references/party.json" {
|
if fake.Requests[0].References[0].Path != "/references/party.json" {
|
||||||
t.Fatalf("fake retained aliased request references: %#v", fake.Requests[0].References)
|
t.Fatalf("fake retained aliased request references: %#v", fake.Requests[0].References)
|
||||||
}
|
}
|
||||||
|
profile = "changed"
|
||||||
|
if *fake.Requests[0].ConfigProfile != "testing" {
|
||||||
|
t.Fatalf("fake retained aliased config profile: %q", *fake.Requests[0].ConfigProfile)
|
||||||
|
}
|
||||||
|
|
||||||
wantErr := errors.New("configured failure")
|
wantErr := errors.New("configured failure")
|
||||||
fake.Err = wantErr
|
fake.Err = wantErr
|
||||||
@@ -579,6 +792,56 @@ func validRunRequest(t *testing.T) RunRequest {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func loadTestCredential(t *testing.T, envName, value string) *configkit.Secret {
|
||||||
|
t.Helper()
|
||||||
|
t.Setenv(envName, value)
|
||||||
|
type testConfig struct {
|
||||||
|
Credential *configkit.Secret `yaml:"credential"`
|
||||||
|
}
|
||||||
|
result, err := configkit.Load(configkit.Bytes("credential.yml", []byte("credential: "+envName+"\n")), configkit.Options[testConfig]{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load test credential: %v", err)
|
||||||
|
}
|
||||||
|
return result.Config.Credential
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertRedactedCredentialFile(t *testing.T, path, canary string) {
|
||||||
|
t.Helper()
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadFile(%q) error = %v", path, err)
|
||||||
|
}
|
||||||
|
assertRedactedCredentialBytes(t, data, canary)
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertRedactedCredentialBytes(t *testing.T, data []byte, canary string) {
|
||||||
|
t.Helper()
|
||||||
|
text := string(data)
|
||||||
|
if strings.Contains(text, canary) || !strings.Contains(text, "<redacted>") {
|
||||||
|
t.Fatal("captured stream did not contain the required redaction marker")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateValidationSummariesAcceptsReusedProducerWithoutAttempts(t *testing.T) {
|
||||||
|
zero := 0
|
||||||
|
summaries, err := validateValidationSummaries([]validationSummaryDocument{{
|
||||||
|
Stage: "chunk", Status: "complete", ProducerAttemptCount: &zero, TerminalAction: "accepted",
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("validateValidationSummaries() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(summaries) != 1 || summaries[0].ProducerAttemptCount != 0 {
|
||||||
|
t.Fatalf("summaries = %#v, want one reused producer with zero attempts", summaries)
|
||||||
|
}
|
||||||
|
|
||||||
|
negative := -1
|
||||||
|
if _, err := validateValidationSummaries([]validationSummaryDocument{{
|
||||||
|
Stage: "chunk", Status: "complete", ProducerAttemptCount: &negative, TerminalAction: "accepted",
|
||||||
|
}}); err == nil {
|
||||||
|
t.Fatal("validateValidationSummaries() negative producer attempts error = nil")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func writeValidBundleAndReceipt(t *testing.T, req RunRequest, includeUnknown bool) {
|
func writeValidBundleAndReceipt(t *testing.T, req RunRequest, includeUnknown bool) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
bundle := filepath.Join(req.OutputRoot, "notarius-run-1")
|
bundle := filepath.Join(req.OutputRoot, "notarius-run-1")
|
||||||
|
|||||||
@@ -6,16 +6,12 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"runtime"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestSubprocessRunnerRunSuccessBuildsDeterministicArgsAndCapturesLogs(t *testing.T) {
|
func TestSubprocessRunnerRunSuccessBuildsDeterministicArgsAndCapturesLogs(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
|
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
|
||||||
t.Setenv("SCRIPTORIUM_HELPER_MODE", "run_success")
|
t.Setenv("SCRIPTORIUM_HELPER_MODE", "run_success")
|
||||||
@@ -95,9 +91,6 @@ func TestSubprocessRunnerRunSuccessBuildsDeterministicArgsAndCapturesLogs(t *tes
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerRunExitCodeOneFails(t *testing.T) {
|
func TestSubprocessRunnerRunExitCodeOneFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
|
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
|
||||||
t.Setenv("SCRIPTORIUM_HELPER_MODE", "fail_exit1")
|
t.Setenv("SCRIPTORIUM_HELPER_MODE", "fail_exit1")
|
||||||
t.Setenv("SCRIPTORIUM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SCRIPTORIUM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -118,9 +111,6 @@ func TestSubprocessRunnerRunExitCodeOneFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerRunExitCodeTwoReturnsValidationFailureAndPreservesOutput(t *testing.T) {
|
func TestSubprocessRunnerRunExitCodeTwoReturnsValidationFailureAndPreservesOutput(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
|
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
|
||||||
t.Setenv("SCRIPTORIUM_HELPER_MODE", "fail_exit2_with_output")
|
t.Setenv("SCRIPTORIUM_HELPER_MODE", "fail_exit2_with_output")
|
||||||
t.Setenv("SCRIPTORIUM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SCRIPTORIUM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -144,9 +134,6 @@ func TestSubprocessRunnerRunExitCodeTwoReturnsValidationFailureAndPreservesOutpu
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerRunMissingOutputOnSuccessFails(t *testing.T) {
|
func TestSubprocessRunnerRunMissingOutputOnSuccessFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
|
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
|
||||||
t.Setenv("SCRIPTORIUM_HELPER_MODE", "run_success_missing_output")
|
t.Setenv("SCRIPTORIUM_HELPER_MODE", "run_success_missing_output")
|
||||||
t.Setenv("SCRIPTORIUM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SCRIPTORIUM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -166,9 +153,6 @@ func TestSubprocessRunnerRunMissingOutputOnSuccessFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerRenderSuccess(t *testing.T) {
|
func TestSubprocessRunnerRenderSuccess(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
|
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
|
||||||
t.Setenv("SCRIPTORIUM_HELPER_MODE", "render_success")
|
t.Setenv("SCRIPTORIUM_HELPER_MODE", "render_success")
|
||||||
recordPath := filepath.Join(t.TempDir(), "record.json")
|
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||||
|
|||||||
@@ -6,16 +6,12 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"runtime"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestSubprocessRunnerSuccessWithReportArgsAndEnv(t *testing.T) {
|
func TestSubprocessRunnerSuccessWithReportArgsAndEnv(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "success")
|
t.Setenv("SERIATIM_HELPER_MODE", "success")
|
||||||
@@ -131,9 +127,6 @@ func TestSubprocessRunnerSuccessWithReportArgsAndEnv(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
|
func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "fail")
|
t.Setenv("SERIATIM_HELPER_MODE", "fail")
|
||||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -153,9 +146,6 @@ func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerMissingOutputFails(t *testing.T) {
|
func TestSubprocessRunnerMissingOutputFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "missing_output")
|
t.Setenv("SERIATIM_HELPER_MODE", "missing_output")
|
||||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -172,9 +162,6 @@ func TestSubprocessRunnerMissingOutputFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) {
|
func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "invalid_output")
|
t.Setenv("SERIATIM_HELPER_MODE", "invalid_output")
|
||||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -191,9 +178,6 @@ func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
|
func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "invalid_report")
|
t.Setenv("SERIATIM_HELPER_MODE", "invalid_report")
|
||||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -210,9 +194,6 @@ func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerTrimSuccessInvocationAndProvenance(t *testing.T) {
|
func TestSubprocessRunnerTrimSuccessInvocationAndProvenance(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "trim_success")
|
t.Setenv("SERIATIM_HELPER_MODE", "trim_success")
|
||||||
@@ -297,9 +278,6 @@ func TestSubprocessRunnerTrimSuccessInvocationAndProvenance(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerTrimSubprocessFailure(t *testing.T) {
|
func TestSubprocessRunnerTrimSubprocessFailure(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "fail")
|
t.Setenv("SERIATIM_HELPER_MODE", "fail")
|
||||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -319,9 +297,6 @@ func TestSubprocessRunnerTrimSubprocessFailure(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerTrimMissingOutputFails(t *testing.T) {
|
func TestSubprocessRunnerTrimMissingOutputFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "missing_output")
|
t.Setenv("SERIATIM_HELPER_MODE", "missing_output")
|
||||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -338,9 +313,6 @@ func TestSubprocessRunnerTrimMissingOutputFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerTrimInvalidOutputJSONFails(t *testing.T) {
|
func TestSubprocessRunnerTrimInvalidOutputJSONFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "invalid_output")
|
t.Setenv("SERIATIM_HELPER_MODE", "invalid_output")
|
||||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -357,9 +329,6 @@ func TestSubprocessRunnerTrimInvalidOutputJSONFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerTrimOutputMissingSegmentsFails(t *testing.T) {
|
func TestSubprocessRunnerTrimOutputMissingSegmentsFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "trim_missing_segments")
|
t.Setenv("SERIATIM_HELPER_MODE", "trim_missing_segments")
|
||||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -376,9 +345,6 @@ func TestSubprocessRunnerTrimOutputMissingSegmentsFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerNormalizeSuccessInvocationAndProvenance(t *testing.T) {
|
func TestSubprocessRunnerNormalizeSuccessInvocationAndProvenance(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "normalize_success")
|
t.Setenv("SERIATIM_HELPER_MODE", "normalize_success")
|
||||||
@@ -453,9 +419,6 @@ func TestSubprocessRunnerNormalizeSuccessInvocationAndProvenance(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerNormalizeSubprocessFailure(t *testing.T) {
|
func TestSubprocessRunnerNormalizeSubprocessFailure(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "fail")
|
t.Setenv("SERIATIM_HELPER_MODE", "fail")
|
||||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -475,9 +438,6 @@ func TestSubprocessRunnerNormalizeSubprocessFailure(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerNormalizeMissingOutputFails(t *testing.T) {
|
func TestSubprocessRunnerNormalizeMissingOutputFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "normalize_missing_output")
|
t.Setenv("SERIATIM_HELPER_MODE", "normalize_missing_output")
|
||||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -494,9 +454,6 @@ func TestSubprocessRunnerNormalizeMissingOutputFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerNormalizeInvalidOutputJSONFails(t *testing.T) {
|
func TestSubprocessRunnerNormalizeInvalidOutputJSONFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "normalize_invalid_output")
|
t.Setenv("SERIATIM_HELPER_MODE", "normalize_invalid_output")
|
||||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -513,9 +470,6 @@ func TestSubprocessRunnerNormalizeInvalidOutputJSONFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerNormalizeOutputMissingSegmentsFails(t *testing.T) {
|
func TestSubprocessRunnerNormalizeOutputMissingSegmentsFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "normalize_missing_segments")
|
t.Setenv("SERIATIM_HELPER_MODE", "normalize_missing_segments")
|
||||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -532,9 +486,6 @@ func TestSubprocessRunnerNormalizeOutputMissingSegmentsFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerNormalizeReportMissingFails(t *testing.T) {
|
func TestSubprocessRunnerNormalizeReportMissingFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "normalize_report_missing")
|
t.Setenv("SERIATIM_HELPER_MODE", "normalize_report_missing")
|
||||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -551,9 +502,6 @@ func TestSubprocessRunnerNormalizeReportMissingFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerNormalizeInvalidReportJSONFails(t *testing.T) {
|
func TestSubprocessRunnerNormalizeInvalidReportJSONFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "normalize_invalid_report")
|
t.Setenv("SERIATIM_HELPER_MODE", "normalize_invalid_report")
|
||||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -570,9 +518,6 @@ func TestSubprocessRunnerNormalizeInvalidReportJSONFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerRenderSuccessInvocationAndProvenance(t *testing.T) {
|
func TestSubprocessRunnerRenderSuccessInvocationAndProvenance(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "render_success")
|
t.Setenv("SERIATIM_HELPER_MODE", "render_success")
|
||||||
@@ -639,9 +584,6 @@ func TestSubprocessRunnerRenderSuccessInvocationAndProvenance(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerRenderWithoutTitleOmitsTitleArg(t *testing.T) {
|
func TestSubprocessRunnerRenderWithoutTitleOmitsTitleArg(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "render_success")
|
t.Setenv("SERIATIM_HELPER_MODE", "render_success")
|
||||||
recordPath := filepath.Join(t.TempDir(), "record.json")
|
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||||
@@ -663,9 +605,6 @@ func TestSubprocessRunnerRenderWithoutTitleOmitsTitleArg(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerRenderSubprocessFailure(t *testing.T) {
|
func TestSubprocessRunnerRenderSubprocessFailure(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "fail")
|
t.Setenv("SERIATIM_HELPER_MODE", "fail")
|
||||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -682,9 +621,6 @@ func TestSubprocessRunnerRenderSubprocessFailure(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerRenderMissingOutputFails(t *testing.T) {
|
func TestSubprocessRunnerRenderMissingOutputFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "missing_output")
|
t.Setenv("SERIATIM_HELPER_MODE", "missing_output")
|
||||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
@@ -701,9 +637,6 @@ func TestSubprocessRunnerRenderMissingOutputFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerRenderEmptyOutputFails(t *testing.T) {
|
func TestSubprocessRunnerRenderEmptyOutputFails(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
|
||||||
}
|
|
||||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||||
t.Setenv("SERIATIM_HELPER_MODE", "render_empty_output")
|
t.Setenv("SERIATIM_HELPER_MODE", "render_empty_output")
|
||||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||||
|
|||||||
@@ -37,8 +37,12 @@ type s3ClientOptions struct {
|
|||||||
Region string
|
Region string
|
||||||
Endpoint string
|
Endpoint string
|
||||||
ForcePathStyle bool
|
ForcePathStyle bool
|
||||||
AccessKeyID string
|
Credentials *s3StaticCredentials
|
||||||
SecretKey string
|
}
|
||||||
|
|
||||||
|
type s3StaticCredentials struct {
|
||||||
|
AccessKeyID string
|
||||||
|
SecretAccessKey string
|
||||||
}
|
}
|
||||||
|
|
||||||
var newS3Client = func(ctx context.Context, opts s3ClientOptions) (s3API, error) {
|
var newS3Client = func(ctx context.Context, opts s3ClientOptions) (s3API, error) {
|
||||||
@@ -46,11 +50,11 @@ var newS3Client = func(ctx context.Context, opts s3ClientOptions) (s3API, error)
|
|||||||
if strings.TrimSpace(opts.Region) != "" {
|
if strings.TrimSpace(opts.Region) != "" {
|
||||||
loadOpts = append(loadOpts, awsconfig.WithRegion(strings.TrimSpace(opts.Region)))
|
loadOpts = append(loadOpts, awsconfig.WithRegion(strings.TrimSpace(opts.Region)))
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(opts.AccessKeyID) != "" && strings.TrimSpace(opts.SecretKey) != "" {
|
if opts.Credentials != nil {
|
||||||
loadOpts = append(loadOpts, awsconfig.WithCredentialsProvider(
|
loadOpts = append(loadOpts, awsconfig.WithCredentialsProvider(
|
||||||
credentials.NewStaticCredentialsProvider(
|
credentials.NewStaticCredentialsProvider(
|
||||||
strings.TrimSpace(opts.AccessKeyID),
|
opts.Credentials.AccessKeyID,
|
||||||
strings.TrimSpace(opts.SecretKey),
|
opts.Credentials.SecretAccessKey,
|
||||||
"",
|
"",
|
||||||
),
|
),
|
||||||
))
|
))
|
||||||
@@ -75,14 +79,24 @@ func NewS3BackendFromConfig(ctx context.Context, cfg config.StorageS3Config) (*S
|
|||||||
if bucket == "" {
|
if bucket == "" {
|
||||||
return nil, fmt.Errorf("storage.s3.bucket is required")
|
return nil, fmt.Errorf("storage.s3.bucket is required")
|
||||||
}
|
}
|
||||||
|
if (cfg.AccessKeyID == nil) != (cfg.SecretAccessKey == nil) {
|
||||||
|
return nil, fmt.Errorf("storage.s3 access key ID and secret access key references must be configured together")
|
||||||
|
}
|
||||||
|
if cfg.AccessKeyID != nil && (!cfg.AccessKeyID.IsResolved() || !cfg.SecretAccessKey.IsResolved()) {
|
||||||
|
return nil, fmt.Errorf("storage.s3 configured credential references must be resolved")
|
||||||
|
}
|
||||||
|
|
||||||
client, err := newS3Client(ctx, s3ClientOptions{
|
clientOptions := s3ClientOptions{
|
||||||
Region: cfg.Region,
|
Region: cfg.Region,
|
||||||
Endpoint: cfg.Endpoint,
|
Endpoint: cfg.Endpoint,
|
||||||
ForcePathStyle: cfg.ForcePathStyle,
|
ForcePathStyle: cfg.ForcePathStyle,
|
||||||
AccessKeyID: s3CredentialFromEnv(orDefaultEnvName(cfg.AccessKeyIDEnv, config.DefaultS3AccessKeyIDEnv)),
|
}
|
||||||
SecretKey: s3CredentialFromEnv(orDefaultEnvName(cfg.SecretKeyEnv, config.DefaultS3SecretAccessKeyEnv)),
|
if cfg.AccessKeyID != nil && cfg.SecretAccessKey != nil {
|
||||||
})
|
clientOptions.Credentials = &s3StaticCredentials{
|
||||||
|
AccessKeyID: cfg.AccessKeyID.Value(), SecretAccessKey: cfg.SecretAccessKey.Value(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
client, err := newS3Client(ctx, clientOptions)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("build s3 client: %w", err)
|
return nil, fmt.Errorf("build s3 client: %w", err)
|
||||||
}
|
}
|
||||||
@@ -93,26 +107,6 @@ func NewS3BackendFromConfig(ctx context.Context, cfg config.StorageS3Config) (*S
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func s3CredentialFromEnv(envVarName string) string {
|
|
||||||
name := strings.TrimSpace(envVarName)
|
|
||||||
if name == "" {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
value, ok := os.LookupEnv(name)
|
|
||||||
if !ok {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
return strings.TrimSpace(value)
|
|
||||||
}
|
|
||||||
|
|
||||||
func orDefaultEnvName(name, fallback string) string {
|
|
||||||
trimmed := strings.TrimSpace(name)
|
|
||||||
if trimmed == "" {
|
|
||||||
return fallback
|
|
||||||
}
|
|
||||||
return trimmed
|
|
||||||
}
|
|
||||||
|
|
||||||
// List returns objects under prefix.
|
// List returns objects under prefix.
|
||||||
func (b *S3Backend) List(ctx context.Context, prefix string) ([]ObjectInfo, error) {
|
func (b *S3Backend) List(ctx context.Context, prefix string) ([]ObjectInfo, error) {
|
||||||
normalizedPrefix := normalizeObjectKey(prefix)
|
normalizedPrefix := normalizeObjectKey(prefix)
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import (
|
|||||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||||
"github.com/aws/smithy-go"
|
"github.com/aws/smithy-go"
|
||||||
|
|
||||||
|
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -319,8 +320,10 @@ func TestS3BackendDeleteHonorsCancellation(t *testing.T) {
|
|||||||
func TestNewS3BackendFromConfigUsesClientOptions(t *testing.T) {
|
func TestNewS3BackendFromConfigUsesClientOptions(t *testing.T) {
|
||||||
original := newS3Client
|
original := newS3Client
|
||||||
t.Cleanup(func() { newS3Client = original })
|
t.Cleanup(func() { newS3Client = original })
|
||||||
t.Setenv("OBJECT_STORAGE_KEY_ID", "id-123")
|
accessKey := resolvedTestSecret(t, "NARRATIO_S3_TEST_ACCESS", "id-123")
|
||||||
t.Setenv("OBJECT_STORAGE_KEY", "secret-abc")
|
secretKey := resolvedTestSecret(t, "NARRATIO_S3_TEST_SECRET", "secret-abc")
|
||||||
|
t.Setenv("NARRATIO_S3_TEST_ACCESS", "changed-id")
|
||||||
|
t.Setenv("NARRATIO_S3_TEST_SECRET", "changed-secret")
|
||||||
|
|
||||||
var got s3ClientOptions
|
var got s3ClientOptions
|
||||||
newS3Client = func(_ context.Context, opts s3ClientOptions) (s3API, error) {
|
newS3Client = func(_ context.Context, opts s3ClientOptions) (s3API, error) {
|
||||||
@@ -329,10 +332,12 @@ func TestNewS3BackendFromConfigUsesClientOptions(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
backend, err := NewS3BackendFromConfig(context.Background(), config.StorageS3Config{
|
backend, err := NewS3BackendFromConfig(context.Background(), config.StorageS3Config{
|
||||||
Bucket: "my-archive",
|
Bucket: "my-archive",
|
||||||
Region: "us-east-1",
|
Region: "us-east-1",
|
||||||
Endpoint: "http://localhost:9000",
|
Endpoint: "http://localhost:9000",
|
||||||
ForcePathStyle: true,
|
ForcePathStyle: true,
|
||||||
|
AccessKeyID: accessKey,
|
||||||
|
SecretAccessKey: secretKey,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewS3BackendFromConfig() error = %v", err)
|
t.Fatalf("NewS3BackendFromConfig() error = %v", err)
|
||||||
@@ -343,7 +348,7 @@ func TestNewS3BackendFromConfigUsesClientOptions(t *testing.T) {
|
|||||||
if got.Region != "us-east-1" || got.Endpoint != "http://localhost:9000" || !got.ForcePathStyle {
|
if got.Region != "us-east-1" || got.Endpoint != "http://localhost:9000" || !got.ForcePathStyle {
|
||||||
t.Fatalf("client options = %#v, want region/endpoint/path-style values", got)
|
t.Fatalf("client options = %#v, want region/endpoint/path-style values", got)
|
||||||
}
|
}
|
||||||
if got.AccessKeyID != "id-123" || got.SecretKey != "secret-abc" {
|
if got.Credentials == nil || got.Credentials.AccessKeyID != "id-123" || got.Credentials.SecretAccessKey != "secret-abc" {
|
||||||
t.Fatalf("client options credentials = %#v, want env-resolved static credentials", got)
|
t.Fatalf("client options credentials = %#v, want env-resolved static credentials", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -355,7 +360,21 @@ func TestNewS3BackendFromConfigRequiresBucket(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNewS3BackendFromConfigFallsBackWhenCredentialEnvMissing(t *testing.T) {
|
func TestNewS3BackendFromConfigRejectsIncompleteOrUnresolvedCredentials(t *testing.T) {
|
||||||
|
unresolved := configkit.SecretEnv("NARRATIO_UNRESOLVED_S3_KEY")
|
||||||
|
for name, cfg := range map[string]config.StorageS3Config{
|
||||||
|
"partial": {Bucket: "my-archive", AccessKeyID: &unresolved},
|
||||||
|
"unresolved": {Bucket: "my-archive", AccessKeyID: &unresolved, SecretAccessKey: &unresolved},
|
||||||
|
} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
if _, err := NewS3BackendFromConfig(context.Background(), cfg); err == nil {
|
||||||
|
t.Fatal("NewS3BackendFromConfig() succeeded, want credential error")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewS3BackendFromConfigUsesDefaultProviderWithoutCredentialReferences(t *testing.T) {
|
||||||
original := newS3Client
|
original := newS3Client
|
||||||
t.Cleanup(func() { newS3Client = original })
|
t.Cleanup(func() { newS3Client = original })
|
||||||
|
|
||||||
@@ -366,19 +385,30 @@ func TestNewS3BackendFromConfigFallsBackWhenCredentialEnvMissing(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
_, err := NewS3BackendFromConfig(context.Background(), config.StorageS3Config{
|
_, err := NewS3BackendFromConfig(context.Background(), config.StorageS3Config{
|
||||||
Bucket: "my-archive",
|
Bucket: "my-archive",
|
||||||
Region: "us-east-1",
|
Region: "us-east-1",
|
||||||
AccessKeyIDEnv: "MISSING_ACCESS_KEY_ID",
|
|
||||||
SecretKeyEnv: "MISSING_SECRET_KEY",
|
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewS3BackendFromConfig() error = %v", err)
|
t.Fatalf("NewS3BackendFromConfig() error = %v", err)
|
||||||
}
|
}
|
||||||
if got.AccessKeyID != "" || got.SecretKey != "" {
|
if got.Credentials != nil {
|
||||||
t.Fatalf("client options credentials = %#v, want empty fallback values", got)
|
t.Fatalf("client options credentials = %#v, want default provider chain", got.Credentials)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func resolvedTestSecret(t *testing.T, name, value string) *configkit.Secret {
|
||||||
|
t.Helper()
|
||||||
|
t.Setenv(name, value)
|
||||||
|
type document struct {
|
||||||
|
Secret *configkit.Secret `yaml:"secret"`
|
||||||
|
}
|
||||||
|
result, err := configkit.Load(configkit.Bytes("test.yml", []byte("secret: "+name+"\n")), configkit.Options[document]{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolve test secret: %v", err)
|
||||||
|
}
|
||||||
|
return result.Config.Secret
|
||||||
|
}
|
||||||
|
|
||||||
func strPtr(v string) *string { return &v }
|
func strPtr(v string) *string { return &v }
|
||||||
func int64Ptr(v int64) *int64 { return &v }
|
func int64Ptr(v int64) *int64 { return &v }
|
||||||
func boolPtr(v bool) *bool { return &v }
|
func boolPtr(v bool) *bool { return &v }
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
//go:build linux || darwin || windows
|
//go:build linux || darwin
|
||||||
|
|
||||||
package subprocess
|
package subprocess
|
||||||
|
|
||||||
|
|||||||
@@ -1,10 +1,9 @@
|
|||||||
//go:build !linux && !darwin && !windows
|
//go:build !linux && !darwin
|
||||||
|
|
||||||
package subprocess
|
package subprocess
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"runtime"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func newOwnedProcessTree() (ownedProcessTree, error) {
|
func newOwnedProcessTree() (ownedProcessTree, error) {
|
||||||
|
|||||||
@@ -1,145 +0,0 @@
|
|||||||
//go:build windows
|
|
||||||
|
|
||||||
package subprocess
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"os/exec"
|
|
||||||
"syscall"
|
|
||||||
"unsafe"
|
|
||||||
|
|
||||||
"golang.org/x/sys/windows"
|
|
||||||
)
|
|
||||||
|
|
||||||
type windowsProcessTree struct {
|
|
||||||
job windows.Handle
|
|
||||||
}
|
|
||||||
|
|
||||||
func newOwnedProcessTree() (ownedProcessTree, error) {
|
|
||||||
return &windowsProcessTree{}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (tree *windowsProcessTree) Start(cmd *exec.Cmd) error {
|
|
||||||
job, err := windows.CreateJobObject(nil, nil)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("create job object: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
limits := windows.JOBOBJECT_EXTENDED_LIMIT_INFORMATION{}
|
|
||||||
limits.BasicLimitInformation.LimitFlags = windows.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
|
|
||||||
if _, err := windows.SetInformationJobObject(job, windows.JobObjectExtendedLimitInformation, uintptr(unsafe.Pointer(&limits)), uint32(unsafe.Sizeof(limits))); err != nil {
|
|
||||||
_ = windows.CloseHandle(job)
|
|
||||||
return fmt.Errorf("configure job object: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cmd.SysProcAttr = &syscall.SysProcAttr{CreationFlags: windows.CREATE_SUSPENDED}
|
|
||||||
if err := cmd.Start(); err != nil {
|
|
||||||
_ = windows.CloseHandle(job)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
var assignErr error
|
|
||||||
withHandleErr := cmd.Process.WithHandle(func(handle uintptr) {
|
|
||||||
assignErr = windows.AssignProcessToJobObject(job, windows.Handle(handle))
|
|
||||||
})
|
|
||||||
if withHandleErr != nil {
|
|
||||||
return failWindowsProcessStart(cmd, job, fmt.Errorf("acquire stable process handle for job assignment: %w", withHandleErr))
|
|
||||||
}
|
|
||||||
if assignErr != nil {
|
|
||||||
return failWindowsProcessStart(cmd, job, fmt.Errorf("assign process to job object: %w", assignErr))
|
|
||||||
}
|
|
||||||
if err := resumeInitialThread(uint32(cmd.Process.Pid)); err != nil {
|
|
||||||
return failWindowsProcessStart(cmd, job, fmt.Errorf("resume initial process thread: %w", err))
|
|
||||||
}
|
|
||||||
|
|
||||||
tree.job = job
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func failWindowsProcessStart(cmd *exec.Cmd, job windows.Handle, setupErr error) error {
|
|
||||||
killErr := cmd.Process.Kill()
|
|
||||||
waitErr := cmd.Wait()
|
|
||||||
var exitErr *exec.ExitError
|
|
||||||
if killErr == nil && errors.As(waitErr, &exitErr) {
|
|
||||||
// A killed suspended child normally reports a non-zero exit from Wait;
|
|
||||||
// successful reaping is not an additional setup failure.
|
|
||||||
waitErr = nil
|
|
||||||
}
|
|
||||||
return joinErrors(
|
|
||||||
setupErr,
|
|
||||||
wrapWindowsProcessCleanupError("kill suspended child", killErr),
|
|
||||||
wrapWindowsProcessCleanupError("reap suspended child", waitErr),
|
|
||||||
wrapWindowsProcessCleanupError("close unowned job object", windows.CloseHandle(job)),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
func wrapWindowsProcessCleanupError(operation string, err error) error {
|
|
||||||
if err == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return fmt.Errorf("%s: %w", operation, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func resumeInitialThread(processID uint32) error {
|
|
||||||
snapshot, err := windows.CreateToolhelp32Snapshot(windows.TH32CS_SNAPTHREAD, 0)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("snapshot initial thread: %w", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = windows.CloseHandle(snapshot) }()
|
|
||||||
|
|
||||||
entry := windows.ThreadEntry32{Size: uint32(unsafe.Sizeof(windows.ThreadEntry32{}))}
|
|
||||||
if err := windows.Thread32First(snapshot, &entry); err != nil {
|
|
||||||
return fmt.Errorf("find initial thread: %w", err)
|
|
||||||
}
|
|
||||||
for {
|
|
||||||
if entry.OwnerProcessID != processID {
|
|
||||||
// Keep enumerating until the suspended process's only initial thread
|
|
||||||
// is found.
|
|
||||||
} else {
|
|
||||||
thread, openErr := windows.OpenThread(windows.THREAD_SUSPEND_RESUME, false, entry.ThreadID)
|
|
||||||
if openErr != nil {
|
|
||||||
return fmt.Errorf("open initial thread: %w", openErr)
|
|
||||||
}
|
|
||||||
defer func() { _ = windows.CloseHandle(thread) }()
|
|
||||||
if _, resumeErr := windows.ResumeThread(thread); resumeErr != nil {
|
|
||||||
return fmt.Errorf("resume initial thread: %w", resumeErr)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := windows.Thread32Next(snapshot, &entry); err != nil {
|
|
||||||
if errors.Is(err, windows.ERROR_NO_MORE_FILES) {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
return fmt.Errorf("find initial thread: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return fmt.Errorf("find initial thread: no thread found for process %d", processID)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (tree *windowsProcessTree) TerminateGracefully() error {
|
|
||||||
// Windows jobs have no portable graceful signal. Terminating the owned job
|
|
||||||
// is the safe fallback and prevents a descendant from escaping cleanup.
|
|
||||||
return tree.terminate()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (tree *windowsProcessTree) TerminateForcefully() error {
|
|
||||||
return tree.terminate()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (tree *windowsProcessTree) Dispose() error {
|
|
||||||
if tree.job == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
err := windows.CloseHandle(tree.job)
|
|
||||||
tree.job = 0
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (tree *windowsProcessTree) terminate() error {
|
|
||||||
if tree.job == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return windows.TerminateJobObject(tree.job, 1)
|
|
||||||
}
|
|
||||||
@@ -8,7 +8,6 @@ import (
|
|||||||
"os/exec"
|
"os/exec"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"runtime"
|
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"syscall"
|
"syscall"
|
||||||
@@ -137,9 +136,6 @@ func TestRunFailureRedactsSensitiveTail(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunRejectsSymlinkDiagnosticWithoutTruncatingTarget(t *testing.T) {
|
func TestRunRejectsSymlinkDiagnosticWithoutTruncatingTarget(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("creating symlinks requires privileges that are not available on every Windows runner")
|
|
||||||
}
|
|
||||||
exe, err := os.Executable()
|
exe, err := os.Executable()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("os.Executable() error = %v", err)
|
t.Fatalf("os.Executable() error = %v", err)
|
||||||
|
|||||||
@@ -6,14 +6,11 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"mime/multipart"
|
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||||
@@ -21,7 +18,6 @@ import (
|
|||||||
|
|
||||||
const (
|
const (
|
||||||
defaultMaxWhisperXResponseBytes int64 = 10 * 1024 * 1024
|
defaultMaxWhisperXResponseBytes int64 = 10 * 1024 * 1024
|
||||||
whisperXUploadBufferSize = 32 * 1024
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// HTTPClientConfig contains parsed, deterministic WhisperX HTTP client settings.
|
// HTTPClientConfig contains parsed, deterministic WhisperX HTTP client settings.
|
||||||
@@ -192,21 +188,32 @@ func (c *HTTPClient) Transcribe(ctx context.Context, req TranscribeRequest) (Tra
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (c *HTTPClient) doTranscribeAttempt(ctx context.Context, audioPath string) (int, []byte, error) {
|
func (c *HTTPClient) doTranscribeAttempt(ctx context.Context, audioPath string) (int, []byte, error) {
|
||||||
upload := newMultipartUpload(ctx, audioPath, c.language, c.openAudio)
|
upload, err := newMultipartBodyFactory(ctx, audioPath, c.language, c.openAudio)
|
||||||
defer upload.Close()
|
if err != nil {
|
||||||
|
return 0, nil, fmt.Errorf("prepare whisperx request body: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = upload.Close() }()
|
||||||
|
body, err := upload.NewBody()
|
||||||
|
if err != nil {
|
||||||
|
return 0, nil, fmt.Errorf("open whisperx request body: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.url.String(), upload)
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.url.String(), body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, nil, fmt.Errorf("build whisperx request: %w", err)
|
return 0, nil, fmt.Errorf("build whisperx request: %w", err)
|
||||||
}
|
}
|
||||||
req.Header.Set("Content-Type", upload.contentType)
|
req.Header.Set("Content-Type", upload.contentType)
|
||||||
req.Header.Set("Expect", "100-continue")
|
req.Header.Set("Expect", "100-continue")
|
||||||
|
req.GetBody = upload.NewBody
|
||||||
|
|
||||||
resp, err := c.httpClient.Do(req)
|
resp, err := c.httpClient.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = upload.Close()
|
_ = upload.Close()
|
||||||
if producerErr := upload.Wait(); producerErr != nil {
|
if sourceErr := upload.SourceError(true); sourceErr != nil {
|
||||||
return 0, nil, fmt.Errorf("stream whisperx request body: %w", producerErr)
|
return 0, nil, fmt.Errorf("stream whisperx request body: %w", sourceErr)
|
||||||
|
}
|
||||||
|
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||||
|
return 0, nil, fmt.Errorf("perform whisperx request: %w", ctxErr)
|
||||||
}
|
}
|
||||||
return 0, nil, fmt.Errorf("perform whisperx request: %w", err)
|
return 0, nil, fmt.Errorf("perform whisperx request: %w", err)
|
||||||
}
|
}
|
||||||
@@ -214,8 +221,8 @@ func (c *HTTPClient) doTranscribeAttempt(ctx context.Context, audioPath string)
|
|||||||
|
|
||||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||||
_ = upload.Close()
|
_ = upload.Close()
|
||||||
if producerErr := upload.Wait(); producerErr != nil {
|
if sourceErr := upload.SourceError(false); sourceErr != nil {
|
||||||
return resp.StatusCode, nil, fmt.Errorf("stream whisperx request body: %w", producerErr)
|
return resp.StatusCode, nil, fmt.Errorf("stream whisperx request body: %w", sourceErr)
|
||||||
}
|
}
|
||||||
if _, err := readWhisperXResponse(resp.Body, c.maxResponseBytes); err != nil {
|
if _, err := readWhisperXResponse(resp.Body, c.maxResponseBytes); err != nil {
|
||||||
return resp.StatusCode, nil, fmt.Errorf("read whisperx response body: %w", err)
|
return resp.StatusCode, nil, fmt.Errorf("read whisperx response body: %w", err)
|
||||||
@@ -223,13 +230,19 @@ func (c *HTTPClient) doTranscribeAttempt(ctx context.Context, audioPath string)
|
|||||||
return resp.StatusCode, nil, fmt.Errorf("whisperx returned status %d", resp.StatusCode)
|
return resp.StatusCode, nil, fmt.Errorf("whisperx returned status %d", resp.StatusCode)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := upload.Wait(); err != nil {
|
data, responseErr := readWhisperXResponse(resp.Body, c.maxResponseBytes)
|
||||||
return resp.StatusCode, nil, fmt.Errorf("stream whisperx request body: %w", err)
|
_ = upload.Close()
|
||||||
|
if sourceErr := upload.SourceError(false); sourceErr != nil {
|
||||||
|
if responseErr != nil {
|
||||||
|
return resp.StatusCode, nil, fmt.Errorf("stream whisperx request body: %w", errors.Join(sourceErr, responseErr))
|
||||||
|
}
|
||||||
|
return resp.StatusCode, nil, fmt.Errorf("stream whisperx request body: %w", sourceErr)
|
||||||
}
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
data, err := readWhisperXResponse(resp.Body, c.maxResponseBytes)
|
return resp.StatusCode, nil, fmt.Errorf("complete whisperx request: %w", err)
|
||||||
if err != nil {
|
}
|
||||||
return resp.StatusCode, nil, fmt.Errorf("read whisperx response body: %w", err)
|
if responseErr != nil {
|
||||||
|
return resp.StatusCode, nil, fmt.Errorf("read whisperx response body: %w", responseErr)
|
||||||
}
|
}
|
||||||
return resp.StatusCode, data, nil
|
return resp.StatusCode, data, nil
|
||||||
}
|
}
|
||||||
@@ -286,161 +299,6 @@ func isHTTPURLScheme(scheme string) bool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type multipartUpload struct {
|
|
||||||
reader *io.PipeReader
|
|
||||||
writer *io.PipeWriter
|
|
||||||
contentType string
|
|
||||||
done chan struct{}
|
|
||||||
|
|
||||||
mu sync.Mutex
|
|
||||||
audio io.Closer
|
|
||||||
err error
|
|
||||||
aborted bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func newMultipartUpload(ctx context.Context, audioPath, language string, openAudio func(string) (io.ReadCloser, error)) *multipartUpload {
|
|
||||||
reader, writer := io.Pipe()
|
|
||||||
multipartWriter := multipart.NewWriter(writer)
|
|
||||||
upload := &multipartUpload{
|
|
||||||
reader: reader,
|
|
||||||
writer: writer,
|
|
||||||
contentType: multipartWriter.FormDataContentType(),
|
|
||||||
done: make(chan struct{}),
|
|
||||||
}
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
err := upload.write(ctx, multipartWriter, audioPath, language, openAudio)
|
|
||||||
if err != nil {
|
|
||||||
_ = writer.CloseWithError(err)
|
|
||||||
} else {
|
|
||||||
_ = writer.Close()
|
|
||||||
}
|
|
||||||
upload.mu.Lock()
|
|
||||||
upload.err = err
|
|
||||||
upload.audio = nil
|
|
||||||
upload.mu.Unlock()
|
|
||||||
close(upload.done)
|
|
||||||
}()
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
upload.abort()
|
|
||||||
case <-upload.done:
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
return upload
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u *multipartUpload) Read(p []byte) (int, error) {
|
|
||||||
return u.reader.Read(p)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u *multipartUpload) Close() error {
|
|
||||||
u.abort()
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u *multipartUpload) Wait() error {
|
|
||||||
<-u.done
|
|
||||||
u.mu.Lock()
|
|
||||||
defer u.mu.Unlock()
|
|
||||||
return u.err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u *multipartUpload) write(ctx context.Context, writer *multipart.Writer, audioPath, language string, openAudio func(string) (io.ReadCloser, error)) error {
|
|
||||||
fileWriter, err := writer.CreateFormFile("file", filepath.Base(audioPath))
|
|
||||||
if err != nil {
|
|
||||||
return u.producerError(ctx, fmt.Errorf("create multipart file field: %w", err))
|
|
||||||
}
|
|
||||||
|
|
||||||
audioFile, err := openAudio(audioPath)
|
|
||||||
if err != nil {
|
|
||||||
return u.producerError(ctx, fmt.Errorf("open audio file %q: %w", audioPath, err))
|
|
||||||
}
|
|
||||||
u.setAudio(audioFile)
|
|
||||||
|
|
||||||
_, copyErr := io.CopyBuffer(fileWriter, &contextReader{ctx: ctx, reader: audioFile}, make([]byte, whisperXUploadBufferSize))
|
|
||||||
closeErr := audioFile.Close()
|
|
||||||
u.clearAudio(audioFile)
|
|
||||||
if copyErr != nil {
|
|
||||||
return u.producerError(ctx, fmt.Errorf("copy audio file %q: %w", audioPath, copyErr))
|
|
||||||
}
|
|
||||||
if closeErr != nil {
|
|
||||||
return u.producerError(ctx, fmt.Errorf("close audio file %q: %w", audioPath, closeErr))
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := writer.WriteField("language", language); err != nil {
|
|
||||||
return u.producerError(ctx, fmt.Errorf("write language form field: %w", err))
|
|
||||||
}
|
|
||||||
if err := writer.Close(); err != nil {
|
|
||||||
return u.producerError(ctx, fmt.Errorf("close multipart writer: %w", err))
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u *multipartUpload) producerError(ctx context.Context, err error) error {
|
|
||||||
if ctx.Err() != nil {
|
|
||||||
return ctx.Err()
|
|
||||||
}
|
|
||||||
u.mu.Lock()
|
|
||||||
aborted := u.aborted
|
|
||||||
u.mu.Unlock()
|
|
||||||
if aborted {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u *multipartUpload) setAudio(audio io.Closer) {
|
|
||||||
u.mu.Lock()
|
|
||||||
u.audio = audio
|
|
||||||
aborted := u.aborted
|
|
||||||
u.mu.Unlock()
|
|
||||||
if aborted {
|
|
||||||
_ = audio.Close()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u *multipartUpload) clearAudio(audio io.Closer) {
|
|
||||||
u.mu.Lock()
|
|
||||||
if u.audio == audio {
|
|
||||||
u.audio = nil
|
|
||||||
}
|
|
||||||
u.mu.Unlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u *multipartUpload) abort() {
|
|
||||||
u.mu.Lock()
|
|
||||||
if u.aborted {
|
|
||||||
u.mu.Unlock()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
u.aborted = true
|
|
||||||
audio := u.audio
|
|
||||||
u.mu.Unlock()
|
|
||||||
|
|
||||||
_ = u.reader.Close()
|
|
||||||
if audio != nil {
|
|
||||||
_ = audio.Close()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type contextReader struct {
|
|
||||||
ctx context.Context
|
|
||||||
reader io.Reader
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *contextReader) Read(p []byte) (int, error) {
|
|
||||||
select {
|
|
||||||
case <-r.ctx.Done():
|
|
||||||
return 0, r.ctx.Err()
|
|
||||||
default:
|
|
||||||
return r.reader.Read(p)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeFileAtomic(path string, data []byte, perm os.FileMode) error {
|
func writeFileAtomic(path string, data []byte, perm os.FileMode) error {
|
||||||
if strings.TrimSpace(path) == "" {
|
if strings.TrimSpace(path) == "" {
|
||||||
return fmt.Errorf("path is required")
|
return fmt.Errorf("path is required")
|
||||||
|
|||||||
261
internal/adapters/whisperx/http_replay_test.go
Normal file
261
internal/adapters/whisperx/http_replay_test.go
Normal file
@@ -0,0 +1,261 @@
|
|||||||
|
package whisperx
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestHTTPClientTransportReplaysAbandonedMultipartBody(t *testing.T) {
|
||||||
|
var opens atomic.Int32
|
||||||
|
var closes atomic.Int32
|
||||||
|
transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
|
partial := make([]byte, 17)
|
||||||
|
if _, err := io.ReadFull(request.Body, partial); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := request.Body.Close(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if request.GetBody == nil {
|
||||||
|
return nil, errors.New("request GetBody is nil")
|
||||||
|
}
|
||||||
|
replay, err := request.GetBody()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
replayedBytes, err := io.ReadAll(replay)
|
||||||
|
if err != nil {
|
||||||
|
_ = replay.Close()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := replay.Close(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
parts := parseMultipartParts(t, request.Header.Get("Content-Type"), replayedBytes)
|
||||||
|
if len(parts) != 2 || parts[0].name != "file" || parts[0].filename != "audio.flac" || string(parts[0].data) != "audio-data" {
|
||||||
|
t.Fatalf("replayed file part = %#v, want exact audio.flac payload", parts)
|
||||||
|
}
|
||||||
|
if parts[1].name != "language" || string(parts[1].data) != "en" {
|
||||||
|
t.Fatalf("replayed language part = %#v, want language=en", parts[1])
|
||||||
|
}
|
||||||
|
return whisperXJSONResponse(request), nil
|
||||||
|
})
|
||||||
|
client := newReplayTestHTTPClient(t, transport)
|
||||||
|
client.openAudio = func(string) (io.ReadCloser, error) {
|
||||||
|
opens.Add(1)
|
||||||
|
return &countedReadCloser{Reader: strings.NewReader("audio-data"), closes: &closes}, nil
|
||||||
|
}
|
||||||
|
outPath := filepath.Join(t.TempDir(), "raw.json")
|
||||||
|
|
||||||
|
result, err := client.Transcribe(context.Background(), TranscribeRequest{
|
||||||
|
AudioPath: "audio.flac",
|
||||||
|
OutputRawTranscriptPath: outPath,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Transcribe() error = %v", err)
|
||||||
|
}
|
||||||
|
if result.Attempts != 1 {
|
||||||
|
t.Fatalf("application attempts = %d, want 1", result.Attempts)
|
||||||
|
}
|
||||||
|
verifyJSONFile(t, outPath)
|
||||||
|
if got := opens.Load(); got != 2 {
|
||||||
|
t.Fatalf("audio opens = %d, want original and replay", got)
|
||||||
|
}
|
||||||
|
if got := closes.Load(); got != 2 {
|
||||||
|
t.Fatalf("audio closes = %d, want all body handles disposed", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHTTPClientTransportReturnsReplayOpenFailure(t *testing.T) {
|
||||||
|
replayErr := errors.New("replay open failed")
|
||||||
|
var opens atomic.Int32
|
||||||
|
var closes atomic.Int32
|
||||||
|
transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
|
if _, err := io.ReadFull(request.Body, make([]byte, 1)); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
_ = request.Body.Close()
|
||||||
|
_, err := request.GetBody()
|
||||||
|
return nil, err
|
||||||
|
})
|
||||||
|
client := newReplayTestHTTPClient(t, transport)
|
||||||
|
client.openAudio = func(string) (io.ReadCloser, error) {
|
||||||
|
if opens.Add(1) == 2 {
|
||||||
|
return nil, replayErr
|
||||||
|
}
|
||||||
|
return &countedReadCloser{Reader: strings.NewReader("audio-data"), closes: &closes}, nil
|
||||||
|
}
|
||||||
|
outPath := filepath.Join(t.TempDir(), "raw.json")
|
||||||
|
|
||||||
|
result, err := client.Transcribe(context.Background(), TranscribeRequest{
|
||||||
|
AudioPath: "audio.flac",
|
||||||
|
OutputRawTranscriptPath: outPath,
|
||||||
|
})
|
||||||
|
if !errors.Is(err, replayErr) {
|
||||||
|
t.Fatalf("Transcribe() error = %v, want replay open failure", err)
|
||||||
|
}
|
||||||
|
if result.Attempts != 1 {
|
||||||
|
t.Fatalf("application attempts = %d, want 1", result.Attempts)
|
||||||
|
}
|
||||||
|
if got := closes.Load(); got != 1 {
|
||||||
|
t.Fatalf("audio closes = %d, want original disposed", got)
|
||||||
|
}
|
||||||
|
if _, statErr := os.Stat(outPath); !errors.Is(statErr, os.ErrNotExist) {
|
||||||
|
t.Fatalf("output should not exist, stat error = %v", statErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHTTPClientTransportCannotSwallowReplayReadFailure(t *testing.T) {
|
||||||
|
replayErr := errors.New("replay read failed")
|
||||||
|
var opens atomic.Int32
|
||||||
|
transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
|
if _, err := io.ReadFull(request.Body, make([]byte, 1)); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
_ = request.Body.Close()
|
||||||
|
replay, err := request.GetBody()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
_, _ = io.ReadAll(replay)
|
||||||
|
_ = replay.Close()
|
||||||
|
return whisperXJSONResponse(request), nil
|
||||||
|
})
|
||||||
|
client := newReplayTestHTTPClient(t, transport)
|
||||||
|
client.openAudio = func(string) (io.ReadCloser, error) {
|
||||||
|
if opens.Add(1) == 2 {
|
||||||
|
return &failingReadCloser{first: []byte("partial"), err: replayErr}, nil
|
||||||
|
}
|
||||||
|
return io.NopCloser(strings.NewReader("audio-data")), nil
|
||||||
|
}
|
||||||
|
outPath := filepath.Join(t.TempDir(), "raw.json")
|
||||||
|
|
||||||
|
_, err := client.Transcribe(context.Background(), TranscribeRequest{
|
||||||
|
AudioPath: "audio.flac",
|
||||||
|
OutputRawTranscriptPath: outPath,
|
||||||
|
})
|
||||||
|
if !errors.Is(err, replayErr) {
|
||||||
|
t.Fatalf("Transcribe() error = %v, want replay read failure", err)
|
||||||
|
}
|
||||||
|
if _, statErr := os.Stat(outPath); !errors.Is(statErr, os.ErrNotExist) {
|
||||||
|
t.Fatalf("output should not exist, stat error = %v", statErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHTTPClientTransportEarlyResponseDisposesOriginalBody(t *testing.T) {
|
||||||
|
var closes atomic.Int32
|
||||||
|
client := newReplayTestHTTPClient(t, roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
|
return whisperXJSONResponse(request), nil
|
||||||
|
}))
|
||||||
|
client.openAudio = func(string) (io.ReadCloser, error) {
|
||||||
|
return &countedReadCloser{Reader: strings.NewReader("audio-data"), closes: &closes}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := client.Transcribe(context.Background(), TranscribeRequest{
|
||||||
|
AudioPath: "audio.flac",
|
||||||
|
OutputRawTranscriptPath: filepath.Join(t.TempDir(), "raw.json"),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Transcribe() error = %v", err)
|
||||||
|
}
|
||||||
|
if got := closes.Load(); got != 1 {
|
||||||
|
t.Fatalf("audio closes = %d, want early-response body disposed", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHTTPClientParentCancellationDuringReplayClosesEveryBody(t *testing.T) {
|
||||||
|
var opens atomic.Int32
|
||||||
|
var originalCloses atomic.Int32
|
||||||
|
replaySource := newSignalingBlockedReadCloser()
|
||||||
|
transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
|
if _, err := io.ReadFull(request.Body, make([]byte, 1)); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
_ = request.Body.Close()
|
||||||
|
replay, err := request.GetBody()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
_, _ = io.ReadAll(replay)
|
||||||
|
_ = replay.Close()
|
||||||
|
return nil, request.Context().Err()
|
||||||
|
})
|
||||||
|
client := newReplayTestHTTPClient(t, transport)
|
||||||
|
client.openAudio = func(string) (io.ReadCloser, error) {
|
||||||
|
if opens.Add(1) == 1 {
|
||||||
|
return &countedReadCloser{Reader: strings.NewReader("audio-data"), closes: &originalCloses}, nil
|
||||||
|
}
|
||||||
|
return replaySource, nil
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
_, err := client.Transcribe(ctx, TranscribeRequest{
|
||||||
|
AudioPath: "audio.flac",
|
||||||
|
OutputRawTranscriptPath: filepath.Join(t.TempDir(), "raw.json"),
|
||||||
|
})
|
||||||
|
done <- err
|
||||||
|
}()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-replaySource.blocked:
|
||||||
|
cancel()
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
cancel()
|
||||||
|
t.Fatal("replay source did not block")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("Transcribe() error = %v, want context cancellation", err)
|
||||||
|
}
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("Transcribe() did not return after replay cancellation")
|
||||||
|
}
|
||||||
|
if got := originalCloses.Load(); got != 1 {
|
||||||
|
t.Fatalf("original audio closes = %d, want 1", got)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-replaySource.closed:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("replay audio source was not closed")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type roundTripFunc func(*http.Request) (*http.Response, error)
|
||||||
|
|
||||||
|
func (roundTrip roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) {
|
||||||
|
return roundTrip(request)
|
||||||
|
}
|
||||||
|
|
||||||
|
func newReplayTestHTTPClient(t *testing.T, transport http.RoundTripper) *HTTPClient {
|
||||||
|
t.Helper()
|
||||||
|
client, err := NewHTTPClient(HTTPClientConfig{
|
||||||
|
TranscribeURL: "http://whisperx.test/transcribe",
|
||||||
|
Language: "en",
|
||||||
|
Timeout: time.Second,
|
||||||
|
HTTPClient: &http.Client{Transport: transport},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewHTTPClient() error = %v", err)
|
||||||
|
}
|
||||||
|
return client
|
||||||
|
}
|
||||||
|
|
||||||
|
func whisperXJSONResponse(request *http.Request) *http.Response {
|
||||||
|
return &http.Response{
|
||||||
|
StatusCode: http.StatusOK,
|
||||||
|
Status: "200 OK",
|
||||||
|
Header: make(http.Header),
|
||||||
|
Body: io.NopCloser(strings.NewReader(`{"source":"transport-replay","segments":[]}`)),
|
||||||
|
Request: request,
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -365,7 +365,7 @@ func TestHTTPClientEarlyServerResponseReturns(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHTTPClientCancellationReleasesBlockedProducer(t *testing.T) {
|
func TestHTTPClientCancellationReleasesBlockedSource(t *testing.T) {
|
||||||
release := make(chan struct{})
|
release := make(chan struct{})
|
||||||
source := newGatedReadCloser([]byte("audio-data"), release)
|
source := newGatedReadCloser([]byte("audio-data"), release)
|
||||||
firstByteReceived := make(chan struct{})
|
firstByteReceived := make(chan struct{})
|
||||||
|
|||||||
263
internal/adapters/whisperx/multipart_body.go
Normal file
263
internal/adapters/whisperx/multipart_body.go
Normal file
@@ -0,0 +1,263 @@
|
|||||||
|
package whisperx
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"mime/multipart"
|
||||||
|
"path/filepath"
|
||||||
|
"sync"
|
||||||
|
)
|
||||||
|
|
||||||
|
type multipartBodyFactory struct {
|
||||||
|
ctx context.Context
|
||||||
|
audioPath string
|
||||||
|
openAudio func(string) (io.ReadCloser, error)
|
||||||
|
prefix []byte
|
||||||
|
suffix []byte
|
||||||
|
contentType string
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
closed bool
|
||||||
|
bodies []*multipartBody
|
||||||
|
}
|
||||||
|
|
||||||
|
func newMultipartBodyFactory(
|
||||||
|
ctx context.Context,
|
||||||
|
audioPath string,
|
||||||
|
language string,
|
||||||
|
openAudio func(string) (io.ReadCloser, error),
|
||||||
|
) (*multipartBodyFactory, error) {
|
||||||
|
if ctx == nil {
|
||||||
|
ctx = context.Background()
|
||||||
|
}
|
||||||
|
if openAudio == nil {
|
||||||
|
return nil, fmt.Errorf("audio opener is required")
|
||||||
|
}
|
||||||
|
var framing bytes.Buffer
|
||||||
|
writer := multipart.NewWriter(&framing)
|
||||||
|
if _, err := writer.CreateFormFile("file", filepath.Base(audioPath)); err != nil {
|
||||||
|
return nil, fmt.Errorf("create multipart file field: %w", err)
|
||||||
|
}
|
||||||
|
prefixLength := framing.Len()
|
||||||
|
if err := writer.WriteField("language", language); err != nil {
|
||||||
|
return nil, fmt.Errorf("write language form field: %w", err)
|
||||||
|
}
|
||||||
|
if err := writer.Close(); err != nil {
|
||||||
|
return nil, fmt.Errorf("close multipart framing: %w", err)
|
||||||
|
}
|
||||||
|
framingBytes := framing.Bytes()
|
||||||
|
return &multipartBodyFactory{
|
||||||
|
ctx: ctx,
|
||||||
|
audioPath: audioPath,
|
||||||
|
openAudio: openAudio,
|
||||||
|
prefix: append([]byte(nil), framingBytes[:prefixLength]...),
|
||||||
|
suffix: append([]byte(nil), framingBytes[prefixLength:]...),
|
||||||
|
contentType: writer.FormDataContentType(),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (factory *multipartBodyFactory) NewBody() (io.ReadCloser, error) {
|
||||||
|
factory.mu.Lock()
|
||||||
|
if factory.closed {
|
||||||
|
factory.mu.Unlock()
|
||||||
|
return nil, fmt.Errorf("multipart body factory is closed")
|
||||||
|
}
|
||||||
|
if err := factory.ctx.Err(); err != nil {
|
||||||
|
factory.mu.Unlock()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
audio, err := factory.openAudio(factory.audioPath)
|
||||||
|
if err != nil {
|
||||||
|
factory.mu.Unlock()
|
||||||
|
return nil, fmt.Errorf("open audio file %q: %w", factory.audioPath, err)
|
||||||
|
}
|
||||||
|
if err := factory.ctx.Err(); err != nil {
|
||||||
|
factory.mu.Unlock()
|
||||||
|
_ = audio.Close()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
source := &multipartAudioSource{audio: audio, closeDone: make(chan struct{})}
|
||||||
|
body := &multipartBody{
|
||||||
|
ctx: factory.ctx,
|
||||||
|
source: source,
|
||||||
|
reader: io.MultiReader(bytes.NewReader(factory.prefix), source, bytes.NewReader(factory.suffix)),
|
||||||
|
}
|
||||||
|
factory.bodies = append(factory.bodies, body)
|
||||||
|
factory.mu.Unlock()
|
||||||
|
stopCancel := context.AfterFunc(factory.ctx, func() {
|
||||||
|
_ = body.Close()
|
||||||
|
})
|
||||||
|
body.mu.Lock()
|
||||||
|
body.stopCancel = stopCancel
|
||||||
|
closed := body.closed
|
||||||
|
body.mu.Unlock()
|
||||||
|
if closed {
|
||||||
|
stopCancel()
|
||||||
|
}
|
||||||
|
if err := factory.ctx.Err(); err != nil {
|
||||||
|
_ = body.Close()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return body, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (factory *multipartBodyFactory) Close() error {
|
||||||
|
factory.mu.Lock()
|
||||||
|
if factory.closed {
|
||||||
|
bodies := append([]*multipartBody(nil), factory.bodies...)
|
||||||
|
factory.mu.Unlock()
|
||||||
|
return multipartBodyCloseErrors(bodies)
|
||||||
|
}
|
||||||
|
factory.closed = true
|
||||||
|
bodies := append([]*multipartBody(nil), factory.bodies...)
|
||||||
|
factory.mu.Unlock()
|
||||||
|
for _, body := range bodies {
|
||||||
|
_ = body.Close()
|
||||||
|
}
|
||||||
|
return multipartBodyCloseErrors(bodies)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (factory *multipartBodyFactory) SourceError(includeAbandoned bool) error {
|
||||||
|
factory.mu.Lock()
|
||||||
|
bodies := append([]*multipartBody(nil), factory.bodies...)
|
||||||
|
factory.mu.Unlock()
|
||||||
|
var result error
|
||||||
|
for _, body := range bodies {
|
||||||
|
if err := body.sourceError(includeAbandoned); err != nil {
|
||||||
|
result = errors.Join(result, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
func multipartBodyCloseErrors(bodies []*multipartBody) error {
|
||||||
|
var result error
|
||||||
|
for _, body := range bodies {
|
||||||
|
if err := body.sourceError(false); err != nil {
|
||||||
|
result = errors.Join(result, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
type multipartBody struct {
|
||||||
|
ctx context.Context
|
||||||
|
reader io.Reader
|
||||||
|
source *multipartAudioSource
|
||||||
|
stopCancel func() bool
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
closed bool
|
||||||
|
abandoned bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (body *multipartBody) Read(buffer []byte) (int, error) {
|
||||||
|
if err := body.ctx.Err(); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
body.mu.Lock()
|
||||||
|
closed := body.closed
|
||||||
|
body.mu.Unlock()
|
||||||
|
if closed {
|
||||||
|
return 0, io.ErrClosedPipe
|
||||||
|
}
|
||||||
|
return body.reader.Read(buffer)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (body *multipartBody) Close() error {
|
||||||
|
body.mu.Lock()
|
||||||
|
if body.closed {
|
||||||
|
body.mu.Unlock()
|
||||||
|
return body.source.Close()
|
||||||
|
}
|
||||||
|
body.closed = true
|
||||||
|
body.abandoned = !body.source.reachedEOF()
|
||||||
|
stopCancel := body.stopCancel
|
||||||
|
body.mu.Unlock()
|
||||||
|
if stopCancel != nil {
|
||||||
|
stopCancel()
|
||||||
|
}
|
||||||
|
return body.source.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (body *multipartBody) sourceError(includeAbandoned bool) error {
|
||||||
|
body.mu.Lock()
|
||||||
|
abandoned := body.abandoned
|
||||||
|
body.mu.Unlock()
|
||||||
|
err := body.source.Err()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if abandoned && !includeAbandoned {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type multipartAudioSource struct {
|
||||||
|
audio io.ReadCloser
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
closing bool
|
||||||
|
eof bool
|
||||||
|
readErr error
|
||||||
|
closeErr error
|
||||||
|
closeDone chan struct{}
|
||||||
|
closeResult error
|
||||||
|
closed bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (source *multipartAudioSource) Read(buffer []byte) (int, error) {
|
||||||
|
n, err := source.audio.Read(buffer)
|
||||||
|
source.mu.Lock()
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, io.EOF):
|
||||||
|
source.eof = true
|
||||||
|
case err != nil && !source.closing:
|
||||||
|
source.readErr = errors.Join(source.readErr, err)
|
||||||
|
}
|
||||||
|
source.mu.Unlock()
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (source *multipartAudioSource) Close() error {
|
||||||
|
source.mu.Lock()
|
||||||
|
if source.closed {
|
||||||
|
done := source.closeDone
|
||||||
|
source.mu.Unlock()
|
||||||
|
<-done
|
||||||
|
source.mu.Lock()
|
||||||
|
err := source.closeResult
|
||||||
|
source.mu.Unlock()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
source.closed = true
|
||||||
|
source.closing = true
|
||||||
|
reachedEOF := source.eof
|
||||||
|
source.mu.Unlock()
|
||||||
|
|
||||||
|
err := source.audio.Close()
|
||||||
|
source.mu.Lock()
|
||||||
|
source.closeResult = err
|
||||||
|
if err != nil && reachedEOF {
|
||||||
|
source.closeErr = err
|
||||||
|
}
|
||||||
|
close(source.closeDone)
|
||||||
|
source.mu.Unlock()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (source *multipartAudioSource) reachedEOF() bool {
|
||||||
|
source.mu.Lock()
|
||||||
|
defer source.mu.Unlock()
|
||||||
|
return source.eof
|
||||||
|
}
|
||||||
|
|
||||||
|
func (source *multipartAudioSource) Err() error {
|
||||||
|
source.mu.Lock()
|
||||||
|
defer source.mu.Unlock()
|
||||||
|
return errors.Join(source.readErr, source.closeErr)
|
||||||
|
}
|
||||||
327
internal/adapters/whisperx/multipart_body_test.go
Normal file
327
internal/adapters/whisperx/multipart_body_test.go
Normal file
@@ -0,0 +1,327 @@
|
|||||||
|
package whisperx
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"mime"
|
||||||
|
"mime/multipart"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMultipartBodyReplayUsesIdenticalFraming(t *testing.T) {
|
||||||
|
factory := newTestMultipartBodyFactory(t, context.Background(), "audio-data")
|
||||||
|
defer factory.Close()
|
||||||
|
|
||||||
|
original, err := factory.NewBody()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewBody() original error = %v", err)
|
||||||
|
}
|
||||||
|
replay, err := factory.NewBody()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewBody() replay error = %v", err)
|
||||||
|
}
|
||||||
|
originalBytes, err := io.ReadAll(original)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadAll(original) error = %v", err)
|
||||||
|
}
|
||||||
|
replayBytes, err := io.ReadAll(replay)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadAll(replay) error = %v", err)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(originalBytes, replayBytes) {
|
||||||
|
t.Fatal("original and replay multipart bytes differ")
|
||||||
|
}
|
||||||
|
|
||||||
|
parts := parseMultipartParts(t, factory.contentType, replayBytes)
|
||||||
|
if len(parts) != 2 {
|
||||||
|
t.Fatalf("part count = %d, want 2", len(parts))
|
||||||
|
}
|
||||||
|
if parts[0].name != "file" || parts[0].filename != "audio.flac" || string(parts[0].data) != "audio-data" {
|
||||||
|
t.Fatalf("file part = %#v, want file/audio.flac with exact audio", parts[0])
|
||||||
|
}
|
||||||
|
if parts[1].name != "language" || string(parts[1].data) != "en" {
|
||||||
|
t.Fatalf("language part = %#v, want language=en", parts[1])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMultipartBodyReplayRestartsAfterPartialOriginal(t *testing.T) {
|
||||||
|
factory := newTestMultipartBodyFactory(t, context.Background(), "audio-data")
|
||||||
|
defer factory.Close()
|
||||||
|
|
||||||
|
original, err := factory.NewBody()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewBody() original error = %v", err)
|
||||||
|
}
|
||||||
|
partial := make([]byte, len(factory.prefix)+3)
|
||||||
|
if _, err := io.ReadFull(original, partial); err != nil {
|
||||||
|
t.Fatalf("ReadFull(original) error = %v", err)
|
||||||
|
}
|
||||||
|
if err := original.Close(); err != nil {
|
||||||
|
t.Fatalf("Close(original) error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
replay, err := factory.NewBody()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewBody() replay error = %v", err)
|
||||||
|
}
|
||||||
|
replayBytes, err := io.ReadAll(replay)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadAll(replay) error = %v", err)
|
||||||
|
}
|
||||||
|
parts := parseMultipartParts(t, factory.contentType, replayBytes)
|
||||||
|
if got := string(parts[0].data); got != "audio-data" {
|
||||||
|
t.Fatalf("replayed audio = %q, want complete payload", got)
|
||||||
|
}
|
||||||
|
if err := factory.SourceError(false); err != nil {
|
||||||
|
t.Fatalf("SourceError(false) = %v, want abandoned original ignored", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMultipartBodySimultaneousBodiesUseIndependentSources(t *testing.T) {
|
||||||
|
var opens atomic.Int32
|
||||||
|
var closes atomic.Int32
|
||||||
|
factory, err := newMultipartBodyFactory(context.Background(), "audio.flac", "en", func(string) (io.ReadCloser, error) {
|
||||||
|
opens.Add(1)
|
||||||
|
return &countedReadCloser{Reader: strings.NewReader("audio-data"), closes: &closes}, nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("newMultipartBodyFactory() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
first, err := factory.NewBody()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewBody() first error = %v", err)
|
||||||
|
}
|
||||||
|
second, err := factory.NewBody()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewBody() second error = %v", err)
|
||||||
|
}
|
||||||
|
var firstBytes, secondBytes []byte
|
||||||
|
var firstErr, secondErr error
|
||||||
|
var group sync.WaitGroup
|
||||||
|
group.Add(2)
|
||||||
|
go func() {
|
||||||
|
defer group.Done()
|
||||||
|
firstBytes, firstErr = io.ReadAll(first)
|
||||||
|
}()
|
||||||
|
go func() {
|
||||||
|
defer group.Done()
|
||||||
|
secondBytes, secondErr = io.ReadAll(second)
|
||||||
|
}()
|
||||||
|
group.Wait()
|
||||||
|
if firstErr != nil || secondErr != nil {
|
||||||
|
t.Fatalf("concurrent reads returned (%v, %v)", firstErr, secondErr)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(firstBytes, secondBytes) {
|
||||||
|
t.Fatal("independent body bytes differ")
|
||||||
|
}
|
||||||
|
if got := opens.Load(); got != 2 {
|
||||||
|
t.Fatalf("audio opens = %d, want 2", got)
|
||||||
|
}
|
||||||
|
if err := factory.Close(); err != nil {
|
||||||
|
t.Fatalf("Close() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := factory.Close(); err != nil {
|
||||||
|
t.Fatalf("second Close() error = %v", err)
|
||||||
|
}
|
||||||
|
if got := closes.Load(); got != 2 {
|
||||||
|
t.Fatalf("audio closes = %d, want 2", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMultipartBodyReportsSourceFailures(t *testing.T) {
|
||||||
|
t.Run("read", func(t *testing.T) {
|
||||||
|
sourceErr := errors.New("read source")
|
||||||
|
factory, err := newMultipartBodyFactory(context.Background(), "audio.flac", "en", func(string) (io.ReadCloser, error) {
|
||||||
|
return &failingReadCloser{first: []byte("partial"), err: sourceErr}, nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("newMultipartBodyFactory() error = %v", err)
|
||||||
|
}
|
||||||
|
body, err := factory.NewBody()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewBody() error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := io.ReadAll(body); !errors.Is(err, sourceErr) {
|
||||||
|
t.Fatalf("ReadAll() error = %v, want source failure", err)
|
||||||
|
}
|
||||||
|
_ = body.Close()
|
||||||
|
if err := factory.SourceError(false); !errors.Is(err, sourceErr) {
|
||||||
|
t.Fatalf("SourceError(false) = %v, want source failure", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("close after complete read", func(t *testing.T) {
|
||||||
|
closeErr := errors.New("close source")
|
||||||
|
factory, err := newMultipartBodyFactory(context.Background(), "audio.flac", "en", func(string) (io.ReadCloser, error) {
|
||||||
|
return &errorCloseReadCloser{Reader: strings.NewReader("audio-data"), err: closeErr}, nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("newMultipartBodyFactory() error = %v", err)
|
||||||
|
}
|
||||||
|
body, err := factory.NewBody()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewBody() error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := io.ReadAll(body); err != nil {
|
||||||
|
t.Fatalf("ReadAll() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := body.Close(); !errors.Is(err, closeErr) {
|
||||||
|
t.Fatalf("Close() error = %v, want source close failure", err)
|
||||||
|
}
|
||||||
|
if err := factory.SourceError(false); !errors.Is(err, closeErr) {
|
||||||
|
t.Fatalf("SourceError(false) = %v, want source close failure", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMultipartBodyCloseAndCancelReleaseBlockedSource(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
release func(context.CancelFunc, io.ReadCloser)
|
||||||
|
}{
|
||||||
|
{name: "close", release: func(_ context.CancelFunc, body io.ReadCloser) { _ = body.Close() }},
|
||||||
|
{name: "cancel", release: func(cancel context.CancelFunc, _ io.ReadCloser) { cancel() }},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
source := newSignalingBlockedReadCloser()
|
||||||
|
factory, err := newMultipartBodyFactory(ctx, "audio.flac", "en", func(string) (io.ReadCloser, error) {
|
||||||
|
return source, nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("newMultipartBodyFactory() error = %v", err)
|
||||||
|
}
|
||||||
|
defer factory.Close()
|
||||||
|
body, err := factory.NewBody()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewBody() error = %v", err)
|
||||||
|
}
|
||||||
|
readDone := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
_, err := io.ReadAll(body)
|
||||||
|
readDone <- err
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-source.blocked:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("audio source did not block")
|
||||||
|
}
|
||||||
|
test.release(cancel, body)
|
||||||
|
select {
|
||||||
|
case <-source.closed:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("audio source was not closed")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-readDone:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("blocked body read did not return")
|
||||||
|
}
|
||||||
|
if test.name == "cancel" {
|
||||||
|
if _, err := factory.NewBody(); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("NewBody() after cancellation error = %v, want context.Canceled", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newTestMultipartBodyFactory(t *testing.T, ctx context.Context, audio string) *multipartBodyFactory {
|
||||||
|
t.Helper()
|
||||||
|
factory, err := newMultipartBodyFactory(ctx, "audio.flac", "en", func(string) (io.ReadCloser, error) {
|
||||||
|
return io.NopCloser(strings.NewReader(audio)), nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("newMultipartBodyFactory() error = %v", err)
|
||||||
|
}
|
||||||
|
return factory
|
||||||
|
}
|
||||||
|
|
||||||
|
type multipartTestPart struct {
|
||||||
|
name string
|
||||||
|
filename string
|
||||||
|
data []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseMultipartParts(t *testing.T, contentType string, body []byte) []multipartTestPart {
|
||||||
|
t.Helper()
|
||||||
|
mediaType, params, err := mime.ParseMediaType(contentType)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ParseMediaType() error = %v", err)
|
||||||
|
}
|
||||||
|
if mediaType != "multipart/form-data" {
|
||||||
|
t.Fatalf("media type = %q, want multipart/form-data", mediaType)
|
||||||
|
}
|
||||||
|
reader := multipart.NewReader(bytes.NewReader(body), params["boundary"])
|
||||||
|
var parts []multipartTestPart
|
||||||
|
for {
|
||||||
|
part, err := reader.NextPart()
|
||||||
|
if errors.Is(err, io.EOF) {
|
||||||
|
return parts
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NextPart() error = %v", err)
|
||||||
|
}
|
||||||
|
data, err := io.ReadAll(part)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadAll(part %q) error = %v", part.FormName(), err)
|
||||||
|
}
|
||||||
|
parts = append(parts, multipartTestPart{name: part.FormName(), filename: part.FileName(), data: data})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type countedReadCloser struct {
|
||||||
|
io.Reader
|
||||||
|
closes *atomic.Int32
|
||||||
|
once sync.Once
|
||||||
|
}
|
||||||
|
|
||||||
|
func (reader *countedReadCloser) Close() error {
|
||||||
|
reader.once.Do(func() { reader.closes.Add(1) })
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type errorCloseReadCloser struct {
|
||||||
|
io.Reader
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (reader *errorCloseReadCloser) Close() error { return reader.err }
|
||||||
|
|
||||||
|
type signalingBlockedReadCloser struct {
|
||||||
|
blocked chan struct{}
|
||||||
|
closed chan struct{}
|
||||||
|
once sync.Once
|
||||||
|
read bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func newSignalingBlockedReadCloser() *signalingBlockedReadCloser {
|
||||||
|
return &signalingBlockedReadCloser{blocked: make(chan struct{}), closed: make(chan struct{})}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (reader *signalingBlockedReadCloser) Read(buffer []byte) (int, error) {
|
||||||
|
if !reader.read {
|
||||||
|
reader.read = true
|
||||||
|
return copy(buffer, "audio-data"), nil
|
||||||
|
}
|
||||||
|
reader.once.Do(func() { close(reader.blocked) })
|
||||||
|
<-reader.closed
|
||||||
|
return 0, errors.New("audio source closed")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (reader *signalingBlockedReadCloser) Close() error {
|
||||||
|
select {
|
||||||
|
case <-reader.closed:
|
||||||
|
default:
|
||||||
|
close(reader.closed)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -272,13 +272,18 @@ func TestExecuteAnalyzeRejectsPositionalArgsAndForceFlag(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestExecuteAnalyzeMissingConfigUsesRunStageLoadingPath(t *testing.T) {
|
func TestExecuteAnalyzeMissingConfigUsesRunStageLoadingPath(t *testing.T) {
|
||||||
|
deps, missingPath, executor := guardedMissingPipelineDiscovery(t)
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute([]string{"analyze", "2026-05-03"}, &stdout, &stderr)
|
code := executeWithDependencies([]string{"analyze", "2026-05-03"}, &stdout, &stderr, deps, executor)
|
||||||
if code == 0 {
|
if code == 0 {
|
||||||
t.Fatal("exit code = 0, want non-zero")
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
}
|
}
|
||||||
if !strings.Contains(stderr.String(), "analyze: no pipeline config path provided and no default pipeline config found; searched:") {
|
if stdout.Len() != 0 {
|
||||||
|
t.Fatalf("stdout = %q, want empty", stdout.String())
|
||||||
|
}
|
||||||
|
if got := stderr.String(); !strings.Contains(got, "analyze: no pipeline config path provided") ||
|
||||||
|
!strings.Contains(got, "no default pipeline config found") || !strings.Contains(got, missingPath) {
|
||||||
t.Fatalf("stderr = %q, want pipeline discovery error", stderr.String())
|
t.Fatalf("stderr = %q, want pipeline discovery error", stderr.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -371,13 +376,18 @@ func TestExecutePublishUnknownArtifactFailsValidation(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestExecutePublishMissingConfigUsesRunStageLoadingPath(t *testing.T) {
|
func TestExecutePublishMissingConfigUsesRunStageLoadingPath(t *testing.T) {
|
||||||
|
deps, missingPath, executor := guardedMissingPipelineDiscovery(t)
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute([]string{"publish", "2026-05-03"}, &stdout, &stderr)
|
code := executeWithDependencies([]string{"publish", "2026-05-03"}, &stdout, &stderr, deps, executor)
|
||||||
if code == 0 {
|
if code == 0 {
|
||||||
t.Fatal("exit code = 0, want non-zero")
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
}
|
}
|
||||||
if !strings.Contains(stderr.String(), "publish: no pipeline config path provided and no default pipeline config found; searched:") {
|
if stdout.Len() != 0 {
|
||||||
|
t.Fatalf("stdout = %q, want empty", stdout.String())
|
||||||
|
}
|
||||||
|
if got := stderr.String(); !strings.Contains(got, "publish: no pipeline config path provided") ||
|
||||||
|
!strings.Contains(got, "no default pipeline config found") || !strings.Contains(got, missingPath) {
|
||||||
t.Fatalf("stderr = %q, want pipeline discovery error", stderr.String())
|
t.Fatalf("stderr = %q, want pipeline discovery error", stderr.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -254,10 +254,10 @@ func TestAssembledSplitBundleCommandsUseOneEffectiveFamilyConfiguration(t *testi
|
|||||||
|
|
||||||
var validated bytes.Buffer
|
var validated bytes.Buffer
|
||||||
if err := ConfigValidate(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath}, &validated); err != nil {
|
if err := ConfigValidate(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath}, &validated); err != nil {
|
||||||
t.Fatalf("validate production default: %v", err)
|
t.Fatalf("validate production base: %v", err)
|
||||||
}
|
}
|
||||||
if !strings.Contains(validated.String(), "profile=production") {
|
if !strings.Contains(validated.String(), "profile=none") {
|
||||||
t.Fatalf("validate output = %q, want production profile", validated.String())
|
t.Fatalf("validate output = %q, want unprofiled production base", validated.String())
|
||||||
}
|
}
|
||||||
for _, command := range []func(context.Context, []string, io.Writer) error{ConfigShow, ConfigSources} {
|
for _, command := range []func(context.Context, []string, io.Writer) error{ConfigShow, ConfigSources} {
|
||||||
if err := command(context.Background(), []string{
|
if err := command(context.Background(), []string{
|
||||||
@@ -268,7 +268,7 @@ func TestAssembledSplitBundleCommandsUseOneEffectiveFamilyConfiguration(t *testi
|
|||||||
}
|
}
|
||||||
var diff bytes.Buffer
|
var diff bytes.Buffer
|
||||||
if err := ConfigDiff(context.Background(), []string{
|
if err := ConfigDiff(context.Background(), []string{
|
||||||
"production", "testing", "--config", pipelinePath, "--campaign-file", campaignPath,
|
"default", "testing", "--config", pipelinePath, "--campaign-file", campaignPath,
|
||||||
}, &diff); err != nil {
|
}, &diff); err != nil {
|
||||||
t.Fatalf("compare profiles: %v", err)
|
t.Fatalf("compare profiles: %v", err)
|
||||||
}
|
}
|
||||||
@@ -365,7 +365,7 @@ func TestAssembledSplitBundleProfileSwitchRecordsProvenanceAndLimitsReuse(t *tes
|
|||||||
if !reflect.DeepEqual(productionRuns, make([]int, len(names))) {
|
if !reflect.DeepEqual(productionRuns, make([]int, len(names))) {
|
||||||
t.Fatalf("production runs = %v, want complete reuse", productionRuns)
|
t.Fatalf("production runs = %v, want complete reuse", productionRuns)
|
||||||
}
|
}
|
||||||
assertAssembledProvenance(t, store, productionSummary, "production", config.EffectivePipelineDigest(production.Pipeline))
|
assertAssembledProvenance(t, store, productionSummary, "", config.EffectivePipelineDigest(production.Pipeline))
|
||||||
|
|
||||||
testingRuns := make([]int, len(names))
|
testingRuns := make([]int, len(names))
|
||||||
testingStages := assembledSemanticStages(names, providers, testingRuns)
|
testingStages := assembledSemanticStages(names, providers, testingRuns)
|
||||||
@@ -415,7 +415,9 @@ func assertAssembledProvenance(t *testing.T, store *manifest.LocalStore, summary
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
value := session.EffectiveConfig
|
value := session.EffectiveConfig
|
||||||
if value == nil || value.SelectedProfile == nil || value.SelectedProfile.Name != profile || value.EffectiveConfigDigest != digest {
|
profileMatches := value != nil && ((profile == "" && value.SelectedProfile == nil) ||
|
||||||
|
(profile != "" && value.SelectedProfile != nil && value.SelectedProfile.Name == profile))
|
||||||
|
if !profileMatches || value.EffectiveConfigDigest != digest {
|
||||||
t.Fatalf("effective configuration provenance = %#v, want profile=%q digest=%q", value, profile, digest)
|
t.Fatalf("effective configuration provenance = %#v, want profile=%q digest=%q", value, profile, digest)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -45,7 +45,7 @@ func cleanSessionWithDependencies(ctx context.Context, flags commonConfigFlags,
|
|||||||
if strings.TrimSpace(flags.sessionID) == "" {
|
if strings.TrimSpace(flags.sessionID) == "" {
|
||||||
return fmt.Errorf("clean: session_id is required unless --all is set")
|
return fmt.Errorf("clean: session_id is required unless --all is set")
|
||||||
}
|
}
|
||||||
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), deps)
|
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), flags.secretsDir.pointer(), deps)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("clean: %w", err)
|
return fmt.Errorf("clean: %w", err)
|
||||||
}
|
}
|
||||||
@@ -93,7 +93,7 @@ func cleanAllLocalWithDependencies(flags commonConfigFlags, dryRun, clearCache b
|
|||||||
strings.TrimSpace(flags.previousSessionID) != "" {
|
strings.TrimSpace(flags.previousSessionID) != "" {
|
||||||
return fmt.Errorf("clean: --all cannot be combined with --campaign, --campaign-file, --session, a session_id, or --previous-session-id")
|
return fmt.Errorf("clean: --all cannot be combined with --campaign, --campaign-file, --session, a session_id, or --previous-session-id")
|
||||||
}
|
}
|
||||||
_, pipelineCfg, err := loadPipelineConfigWithDependencies(flags.pipelinePath, config.PipelineLoadOptions{Profile: flags.profile.pointer()}, deps)
|
_, pipelineCfg, err := loadPipelineConfigWithDependencies(flags.pipelinePath, config.PipelineLoadOptions{Profile: flags.profile.pointer()}, flags.secretsDir.pointer(), deps)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("clean: %w", err)
|
return fmt.Errorf("clean: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,16 +2,20 @@ package app
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"os"
|
"io"
|
||||||
|
|
||||||
|
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
type commandDependencies struct {
|
type commandDependencies struct {
|
||||||
|
diagnostics io.Writer
|
||||||
|
resolvePipelineConfigPath func(string) (string, error)
|
||||||
loadPipelineConfig func(string, config.PipelineLoadOptions) (*config.PipelineConfig, error)
|
loadPipelineConfig func(string, config.PipelineLoadOptions) (*config.PipelineConfig, error)
|
||||||
downloadObjectToTemp func(context.Context, storage.ObjectStore, string, string) (string, error)
|
downloadObjectToTemp func(context.Context, storage.ObjectStore, string, string) (string, error)
|
||||||
readSecretsDirectory func(string) ([]os.DirEntry, error)
|
loadSecretsDir func(string) (*configkit.SecretsReport, error)
|
||||||
|
secretsBootstrap *secretsBootstrap
|
||||||
newObjectStore func(context.Context, *config.Config) (storage.ObjectStore, error)
|
newObjectStore func(context.Context, *config.Config) (storage.ObjectStore, error)
|
||||||
discoverRemoteCurrentState func(context.Context, *config.Config, storage.ObjectStore) (*RemoteCurrentState, error)
|
discoverRemoteCurrentState func(context.Context, *config.Config, storage.ObjectStore) (*RemoteCurrentState, error)
|
||||||
buildRestorePlan func(context.Context, *config.Config, *RemoteCurrentState, storage.ObjectStore, RestorePlanOptions) (*RestorePlan, error)
|
buildRestorePlan func(context.Context, *config.Config, *RemoteCurrentState, storage.ObjectStore, RestorePlanOptions) (*RestorePlan, error)
|
||||||
@@ -19,14 +23,23 @@ type commandDependencies struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func normalizeCommandDependencies(deps commandDependencies) commandDependencies {
|
func normalizeCommandDependencies(deps commandDependencies) commandDependencies {
|
||||||
|
if deps.diagnostics == nil {
|
||||||
|
deps.diagnostics = io.Discard
|
||||||
|
}
|
||||||
|
if deps.resolvePipelineConfigPath == nil {
|
||||||
|
deps.resolvePipelineConfigPath = resolvePipelineConfigPath
|
||||||
|
}
|
||||||
if deps.loadPipelineConfig == nil {
|
if deps.loadPipelineConfig == nil {
|
||||||
deps.loadPipelineConfig = config.LoadPipelineWithOptions
|
deps.loadPipelineConfig = config.LoadPipelineWithOptions
|
||||||
}
|
}
|
||||||
if deps.downloadObjectToTemp == nil {
|
if deps.downloadObjectToTemp == nil {
|
||||||
deps.downloadObjectToTemp = storage.DownloadObjectToTemp
|
deps.downloadObjectToTemp = storage.DownloadObjectToTemp
|
||||||
}
|
}
|
||||||
if deps.readSecretsDirectory == nil {
|
if deps.loadSecretsDir == nil {
|
||||||
deps.readSecretsDirectory = os.ReadDir
|
deps.loadSecretsDir = configkit.LoadSecretsDir
|
||||||
|
}
|
||||||
|
if deps.secretsBootstrap == nil {
|
||||||
|
deps.secretsBootstrap = &secretsBootstrap{}
|
||||||
}
|
}
|
||||||
if deps.newObjectStore == nil {
|
if deps.newObjectStore == nil {
|
||||||
deps.newObjectStore = storage.NewObjectStoreFromConfig
|
deps.newObjectStore = storage.NewObjectStoreFromConfig
|
||||||
|
|||||||
@@ -19,6 +19,9 @@ func executeWithPipelineExecutor(args []string, stdout, stderr io.Writer, execut
|
|||||||
}
|
}
|
||||||
|
|
||||||
func executeWithDependencies(args []string, stdout, stderr io.Writer, deps commandDependencies, executor pipelineExecutor) int {
|
func executeWithDependencies(args []string, stdout, stderr io.Writer, deps commandDependencies, executor pipelineExecutor) int {
|
||||||
|
if deps.diagnostics == nil {
|
||||||
|
deps.diagnostics = stderr
|
||||||
|
}
|
||||||
deps = normalizeCommandDependencies(deps)
|
deps = normalizeCommandDependencies(deps)
|
||||||
if len(args) == 0 {
|
if len(args) == 0 {
|
||||||
printUsage(stderr)
|
printUsage(stderr)
|
||||||
|
|||||||
@@ -59,9 +59,10 @@ func TestExecuteValidCommands(t *testing.T) {
|
|||||||
|
|
||||||
func TestExecuteMissingRequiredFlags(t *testing.T) {
|
func TestExecuteMissingRequiredFlags(t *testing.T) {
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
name string
|
name string
|
||||||
args []string
|
args []string
|
||||||
want string
|
want string
|
||||||
|
controlledDiscovery bool
|
||||||
}{
|
}{
|
||||||
{name: "run missing session", args: []string{"run"}, want: "run: session_id is required"},
|
{name: "run missing session", args: []string{"run"}, want: "run: session_id is required"},
|
||||||
{name: "plan old top-level removed", args: []string{"plan"}, want: `unknown command: "plan"`},
|
{name: "plan old top-level removed", args: []string{"plan"}, want: `unknown command: "plan"`},
|
||||||
@@ -69,7 +70,7 @@ func TestExecuteMissingRequiredFlags(t *testing.T) {
|
|||||||
{name: "resume removed", args: []string{"resume"}, want: `unknown command: "resume"`},
|
{name: "resume removed", args: []string{"resume"}, want: `unknown command: "resume"`},
|
||||||
{name: "run-stage missing name", args: []string{"run-stage", "--config", "a", "--session", "b"}, want: "run-stage: expected stage name and session_id"},
|
{name: "run-stage missing name", args: []string{"run-stage", "--config", "a", "--session", "b"}, want: "run-stage: expected stage name and session_id"},
|
||||||
{name: "run-stage missing session", args: []string{"run-stage", "polish"}, want: "run-stage: expected stage name and session_id"},
|
{name: "run-stage missing session", args: []string{"run-stage", "polish"}, want: "run-stage: expected stage name and session_id"},
|
||||||
{name: "run missing config uses defaults", args: []string{"run", "2026-05-03", "--session", "session.yml"}, want: "run: no pipeline config path provided and no default pipeline config found; searched:"},
|
{name: "run missing config uses defaults", args: []string{"run", "2026-05-03", "--session", "session.yml"}, want: "run: no pipeline config path provided", controlledDiscovery: true},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range cases {
|
for _, tc := range cases {
|
||||||
@@ -77,7 +78,13 @@ func TestExecuteMissingRequiredFlags(t *testing.T) {
|
|||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
code := Execute(tc.args, &stdout, &stderr)
|
deps := commandDependencies{}
|
||||||
|
executor := pipelineExecutor(executePlan)
|
||||||
|
missingPath := ""
|
||||||
|
if tc.controlledDiscovery {
|
||||||
|
deps, missingPath, executor = guardedMissingPipelineDiscovery(t)
|
||||||
|
}
|
||||||
|
code := executeWithDependencies(tc.args, &stdout, &stderr, deps, executor)
|
||||||
if code == 0 {
|
if code == 0 {
|
||||||
t.Fatalf("exit code = 0, want non-zero")
|
t.Fatalf("exit code = 0, want non-zero")
|
||||||
}
|
}
|
||||||
@@ -87,6 +94,11 @@ func TestExecuteMissingRequiredFlags(t *testing.T) {
|
|||||||
if !strings.Contains(stderr.String(), tc.want) {
|
if !strings.Contains(stderr.String(), tc.want) {
|
||||||
t.Fatalf("stderr = %q, want to contain %q", stderr.String(), tc.want)
|
t.Fatalf("stderr = %q, want to contain %q", stderr.String(), tc.want)
|
||||||
}
|
}
|
||||||
|
if tc.controlledDiscovery {
|
||||||
|
if got := stderr.String(); !strings.Contains(got, "no default pipeline config found") || !strings.Contains(got, missingPath) {
|
||||||
|
t.Fatalf("stderr = %q, want controlled searched path %q", got, missingPath)
|
||||||
|
}
|
||||||
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -192,7 +204,7 @@ func TestExecuteRunStagePolishLoadsCredentialFromSecretsDir(t *testing.T) {
|
|||||||
configDir := t.TempDir()
|
configDir := t.TempDir()
|
||||||
sessionID := "2026-05-03"
|
sessionID := "2026-05-03"
|
||||||
secretsDir := filepath.Join(configDir, "secrets")
|
secretsDir := filepath.Join(configDir, "secrets")
|
||||||
if err := os.MkdirAll(secretsDir, secretDirectoryPrivateMode); err != nil {
|
if err := os.MkdirAll(secretsDir, 0o755); err != nil {
|
||||||
t.Fatalf("MkdirAll(%q): %v", secretsDir, err)
|
t.Fatalf("MkdirAll(%q): %v", secretsDir, err)
|
||||||
}
|
}
|
||||||
if err := os.WriteFile(filepath.Join(secretsDir, "OPENROUTER_API_KEY"), []byte("from-secret-file\n"), 0o600); err != nil {
|
if err := os.WriteFile(filepath.Join(secretsDir, "OPENROUTER_API_KEY"), []byte("from-secret-file\n"), 0o600); err != nil {
|
||||||
@@ -211,15 +223,13 @@ func TestExecuteRunStagePolishLoadsCredentialFromSecretsDir(t *testing.T) {
|
|||||||
root: ` + workspaceRoot + `
|
root: ` + workspaceRoot + `
|
||||||
storage:
|
storage:
|
||||||
backend: local
|
backend: local
|
||||||
secrets:
|
|
||||||
env_dir: ./secrets
|
|
||||||
whisperx:
|
whisperx:
|
||||||
transcribe_url: https://example.com/transcribe
|
transcribe_url: https://example.com/transcribe
|
||||||
seriatim:
|
seriatim:
|
||||||
binary: ` + seriatimBinary + `
|
binary: ` + seriatimBinary + `
|
||||||
audita:
|
audita:
|
||||||
binary: ` + auditaBinary + `
|
binary: ` + auditaBinary + `
|
||||||
llm_api_key_env: OPENROUTER_API_KEY
|
llm_api_key: OPENROUTER_API_KEY
|
||||||
notification:
|
notification:
|
||||||
mode: noop
|
mode: noop
|
||||||
`
|
`
|
||||||
@@ -264,9 +274,31 @@ inputs:
|
|||||||
if !strings.Contains(stdout.String(), "stage=polish executed=1 skipped=0") {
|
if !strings.Contains(stdout.String(), "stage=polish executed=1 skipped=0") {
|
||||||
t.Fatalf("stdout = %q, want polish execution", stdout.String())
|
t.Fatalf("stdout = %q, want polish execution", stdout.String())
|
||||||
}
|
}
|
||||||
|
foundReference := false
|
||||||
|
if err := filepath.WalkDir(workRoot, func(path string, entry os.DirEntry, walkErr error) error {
|
||||||
|
if walkErr != nil || entry.IsDir() {
|
||||||
|
return walkErr
|
||||||
|
}
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if bytes.Contains(data, []byte("from-secret-file")) {
|
||||||
|
t.Fatalf("workspace output %q exposed credential value", path)
|
||||||
|
}
|
||||||
|
if bytes.Contains(data, []byte("OPENROUTER_API_KEY")) {
|
||||||
|
foundReference = true
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("inspect polish outputs: %v", err)
|
||||||
|
}
|
||||||
|
if !foundReference {
|
||||||
|
t.Fatal("polish outputs omitted safe credential reference")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExecuteRunFailsWhenConfiguredSecretsDirMissing(t *testing.T) {
|
func TestExecuteRunFailsWhenExplicitSecretsDirMissing(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
configDir := t.TempDir()
|
configDir := t.TempDir()
|
||||||
pipelinePath := filepath.Join(configDir, "pipeline.yml")
|
pipelinePath := filepath.Join(configDir, "pipeline.yml")
|
||||||
@@ -277,8 +309,6 @@ func TestExecuteRunFailsWhenConfiguredSecretsDirMissing(t *testing.T) {
|
|||||||
root: ` + workspaceRoot + `
|
root: ` + workspaceRoot + `
|
||||||
storage:
|
storage:
|
||||||
backend: local
|
backend: local
|
||||||
secrets:
|
|
||||||
env_dir: ./missing-secrets
|
|
||||||
whisperx:
|
whisperx:
|
||||||
transcribe_url: https://example.com/transcribe
|
transcribe_url: https://example.com/transcribe
|
||||||
seriatim:
|
seriatim:
|
||||||
@@ -305,11 +335,11 @@ inputs:
|
|||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute([]string{"run", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
code := Execute([]string{"run", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--secrets-dir", filepath.Join(configDir, "missing-secrets")}, &stdout, &stderr)
|
||||||
if code == 0 {
|
if code == 0 {
|
||||||
t.Fatal("exit code = 0, want non-zero")
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
}
|
}
|
||||||
if !strings.Contains(stderr.String(), "validate secrets env_dir") {
|
if !strings.Contains(stderr.String(), "bootstrap secrets directory") {
|
||||||
t.Fatalf("stderr = %q, want secrets validation error context", stderr.String())
|
t.Fatalf("stderr = %q, want secrets validation error context", stderr.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
211
internal/app/config_artifacts_test.go
Normal file
211
internal/app/config_artifacts_test.go
Normal file
@@ -0,0 +1,211 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
const artifactCatalogHeader = "key\tsource\tdefinitions\tenabled\toutput\tdependencies\tfamily\tcharacter\tfamily_source\tparty_source\n"
|
||||||
|
|
||||||
|
func TestConfigArtifactsReportsOrdinaryCatalogWithoutCampaign(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||||
|
mustWriteTestFile(t, pipelinePath, `workspace:
|
||||||
|
root: `+filepath.Join(dir, "workspace")+`
|
||||||
|
whisperx:
|
||||||
|
transcribe_url: https://transcription.example.com/transcribe
|
||||||
|
scriptorium:
|
||||||
|
binary: scriptorium
|
||||||
|
artifacts:
|
||||||
|
zeta:
|
||||||
|
enabled: false
|
||||||
|
output_path: artifacts/zeta.md
|
||||||
|
alpha:
|
||||||
|
enabled: false
|
||||||
|
output_path: artifacts/alpha.md
|
||||||
|
inputs:
|
||||||
|
upstream: {source: narratio.artifact.zeta, required: true}
|
||||||
|
`)
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
if err := ConfigArtifacts(context.Background(), []string{"--config", pipelinePath}, &out); err != nil {
|
||||||
|
t.Fatalf("ConfigArtifacts() error = %v", err)
|
||||||
|
}
|
||||||
|
lines := strings.Split(strings.TrimSuffix(out.String(), "\n"), "\n")
|
||||||
|
if len(lines) != 3 || !strings.HasPrefix(lines[1], "alpha\t") || !strings.HasPrefix(lines[2], "zeta\t") {
|
||||||
|
t.Fatalf("catalog order/output = %q", out.String())
|
||||||
|
}
|
||||||
|
alpha := strings.Split(lines[1], "\t")
|
||||||
|
if len(alpha) != 10 {
|
||||||
|
t.Fatalf("alpha fields = %#v", alpha)
|
||||||
|
}
|
||||||
|
if alpha[1] != "narratio.artifact.alpha" || alpha[2] != pipelinePath || alpha[3] != "false" || alpha[4] != "artifacts/alpha.md" || alpha[5] != "zeta" {
|
||||||
|
t.Fatalf("alpha record = %#v", alpha)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigArtifactsEmptyCatalogWritesOnlyHeader(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||||
|
mustWriteTestFile(t, pipelinePath, "workspace:\n root: "+filepath.Join(dir, "workspace")+"\nwhisperx:\n transcribe_url: https://transcription.example.com/transcribe\n")
|
||||||
|
var out bytes.Buffer
|
||||||
|
if err := ConfigArtifacts(context.Background(), []string{"--config", pipelinePath}, &out); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if out.String() != artifactCatalogHeader {
|
||||||
|
t.Fatalf("empty catalog = %q, want header only", out.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigArtifactsReportsGeneratedFamilyProvenance(t *testing.T) {
|
||||||
|
pipelinePath, campaignPath := writeInspectionFamilyConfig(t)
|
||||||
|
var out bytes.Buffer
|
||||||
|
if err := ConfigArtifacts(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath}, &out); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
records := catalogRecordsByKey(t, out.String())
|
||||||
|
meta := records["character_meta_arannis"]
|
||||||
|
note := records["character_note_arannis"]
|
||||||
|
partyPath := filepath.Join(filepath.Dir(campaignPath), "party.yml")
|
||||||
|
if len(meta) != 10 || meta[6] != "character_meta" || meta[7] != "arannis" || meta[8] != pipelinePath || meta[9] != partyPath {
|
||||||
|
t.Fatalf("generated meta record = %#v", meta)
|
||||||
|
}
|
||||||
|
if len(note) != 10 || note[5] != "character_meta_arannis" || note[6] != "character_note" {
|
||||||
|
t.Fatalf("generated note record = %#v", note)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigArtifactsReportsProfiledFamilyContributors(t *testing.T) {
|
||||||
|
pipelinePath, campaignPath := writeInspectionFamilyConfig(t)
|
||||||
|
profilePath := filepath.Join(filepath.Dir(pipelinePath), "profiles", "testing.yml")
|
||||||
|
mustWriteTestFile(t, profilePath, `scriptorium:
|
||||||
|
artifact_families:
|
||||||
|
character_note:
|
||||||
|
prompt_id: dnd.character_note.testing
|
||||||
|
`)
|
||||||
|
|
||||||
|
args := []string{"--config", pipelinePath, "--campaign-file", campaignPath, "--profile", "testing"}
|
||||||
|
var catalog bytes.Buffer
|
||||||
|
if err := ConfigArtifacts(context.Background(), args, &catalog); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
records := catalogRecordsByKey(t, catalog.String())
|
||||||
|
if got := strings.Split(records["character_note_arannis"][2], ","); !reflect.DeepEqual(got, []string{pipelinePath, profilePath}) {
|
||||||
|
t.Fatalf("generated family definitions = %#v", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
var sources bytes.Buffer
|
||||||
|
if err := ConfigSources(context.Background(), args, &sources); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := "scriptorium.artifacts.character_note_arannis.prompt_id\tfamily\t" + profilePath
|
||||||
|
if !strings.Contains(sources.String(), want) {
|
||||||
|
t.Fatalf("sources output missing profiled family contributor %q:\n%s", want, sources.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigArtifactsReportsBaseAndProfileContributors(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||||
|
profilePath := filepath.Join(dir, "profiles", "testing.yml")
|
||||||
|
mustWriteTestFile(t, pipelinePath, `workspace:
|
||||||
|
root: /srv/narratio
|
||||||
|
whisperx:
|
||||||
|
transcribe_url: https://transcription.example.com/transcribe
|
||||||
|
scriptorium:
|
||||||
|
binary: scriptorium
|
||||||
|
artifacts:
|
||||||
|
notes:
|
||||||
|
enabled: false
|
||||||
|
output_path: artifacts/notes.md
|
||||||
|
`)
|
||||||
|
mustWriteTestFile(t, profilePath, `scriptorium:
|
||||||
|
artifacts:
|
||||||
|
notes:
|
||||||
|
enabled: true
|
||||||
|
prompt_id: notes
|
||||||
|
profile_only:
|
||||||
|
enabled: false
|
||||||
|
output_path: artifacts/profile-only.md
|
||||||
|
`)
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
if err := ConfigArtifacts(context.Background(), []string{"--config", pipelinePath, "--profile", "testing"}, &out); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
records := catalogRecordsByKey(t, out.String())
|
||||||
|
if got := strings.Split(records["notes"][2], ","); !reflect.DeepEqual(got, []string{pipelinePath, profilePath}) {
|
||||||
|
t.Fatalf("notes definitions = %#v", got)
|
||||||
|
}
|
||||||
|
if got := records["profile_only"][2]; got != profilePath {
|
||||||
|
t.Fatalf("profile-only definition = %q, want %q", got, profilePath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigArtifactsIdentityDoesNotDependOnDefinitionFilename(t *testing.T) {
|
||||||
|
inlineDir := t.TempDir()
|
||||||
|
inlinePath := filepath.Join(inlineDir, "pipeline.yml")
|
||||||
|
mustWriteTestFile(t, inlinePath, artifactIdentityPipeline("", "recap"))
|
||||||
|
importDir := t.TempDir()
|
||||||
|
importPath := filepath.Join(importDir, "pipeline.yml")
|
||||||
|
mustWriteTestFile(t, importPath, artifactIdentityPipeline("artifacts.yml", ""))
|
||||||
|
mustWriteTestFile(t, filepath.Join(importDir, "artifacts.yml"), "scriptorium:\n artifacts:\n recap:\n enabled: false\n output_path: artifacts/recap.md\n")
|
||||||
|
|
||||||
|
for _, path := range []string{inlinePath, importPath} {
|
||||||
|
var out bytes.Buffer
|
||||||
|
if err := ConfigArtifacts(context.Background(), []string{"--config", path}, &out); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := catalogRecordsByKey(t, out.String())["recap"][1]; got != "narratio.artifact.recap" {
|
||||||
|
t.Fatalf("source identity for %s = %q", path, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigArtifactsPropagatesWriterErrorsAndEscapesFields(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||||
|
mustWriteTestFile(t, pipelinePath, "workspace:\n root: /srv/narratio\nwhisperx:\n transcribe_url: https://transcription.example.com/transcribe\n")
|
||||||
|
want := errors.New("write rejected")
|
||||||
|
err := ConfigArtifacts(context.Background(), []string{"--config", pipelinePath}, rejectingCatalogWriter{err: want})
|
||||||
|
if !errors.Is(err, want) {
|
||||||
|
t.Fatalf("ConfigArtifacts() error = %v, want writer error", err)
|
||||||
|
}
|
||||||
|
if got := escapeCatalogField("a\\b\tc\nd\r"); got != `a\\b\tc\nd\r` {
|
||||||
|
t.Fatalf("escaped field = %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func catalogRecordsByKey(t *testing.T, output string) map[string][]string {
|
||||||
|
t.Helper()
|
||||||
|
lines := strings.Split(strings.TrimSuffix(output, "\n"), "\n")
|
||||||
|
if len(lines) == 0 || lines[0]+"\n" != artifactCatalogHeader {
|
||||||
|
t.Fatalf("catalog header = %q", output)
|
||||||
|
}
|
||||||
|
records := make(map[string][]string, len(lines)-1)
|
||||||
|
for _, line := range lines[1:] {
|
||||||
|
fields := strings.Split(line, "\t")
|
||||||
|
records[fields[0]] = fields
|
||||||
|
}
|
||||||
|
return records
|
||||||
|
}
|
||||||
|
|
||||||
|
func artifactIdentityPipeline(importPath, inlineKey string) string {
|
||||||
|
document := "workspace:\n root: /srv/narratio\nwhisperx:\n transcribe_url: https://transcription.example.com/transcribe\nscriptorium:\n binary: scriptorium\n"
|
||||||
|
if importPath != "" {
|
||||||
|
document = "configkit:\n imports:\n - path: " + importPath + "\n" + document
|
||||||
|
}
|
||||||
|
if inlineKey != "" {
|
||||||
|
document += " artifacts:\n " + inlineKey + ":\n enabled: false\n output_path: artifacts/recap.md\n"
|
||||||
|
}
|
||||||
|
return document
|
||||||
|
}
|
||||||
|
|
||||||
|
type rejectingCatalogWriter struct{ err error }
|
||||||
|
|
||||||
|
func (w rejectingCatalogWriter) Write([]byte) (int, error) { return 0, w.err }
|
||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"sort"
|
"sort"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
@@ -17,6 +18,7 @@ type inspectionFlags struct {
|
|||||||
campaignPath string
|
campaignPath string
|
||||||
campaignFilePath string
|
campaignFilePath string
|
||||||
profile singletonStringFlag
|
profile singletonStringFlag
|
||||||
|
secretsDir singletonStringFlag
|
||||||
}
|
}
|
||||||
|
|
||||||
type inspectionConfig struct {
|
type inspectionConfig struct {
|
||||||
@@ -35,7 +37,7 @@ func Config(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
func configWithDependencies(ctx context.Context, args []string, out io.Writer, deps commandDependencies) error {
|
func configWithDependencies(ctx context.Context, args []string, out io.Writer, deps commandDependencies) error {
|
||||||
deps = normalizeCommandDependencies(deps)
|
deps = normalizeCommandDependencies(deps)
|
||||||
if len(args) == 0 {
|
if len(args) == 0 {
|
||||||
return fmt.Errorf("config: expected subcommand: validate|show|sources|diff")
|
return fmt.Errorf("config: expected subcommand: validate|show|sources|artifacts|diff")
|
||||||
}
|
}
|
||||||
if args[0] == "--help" || args[0] == "-h" {
|
if args[0] == "--help" || args[0] == "-h" {
|
||||||
writeConfigCommandUsage(out)
|
writeConfigCommandUsage(out)
|
||||||
@@ -48,8 +50,10 @@ func configWithDependencies(ctx context.Context, args []string, out io.Writer, d
|
|||||||
return configShowWithDependencies(ctx, args[1:], out, deps)
|
return configShowWithDependencies(ctx, args[1:], out, deps)
|
||||||
case "sources":
|
case "sources":
|
||||||
return configSourcesWithDependencies(ctx, args[1:], out, deps)
|
return configSourcesWithDependencies(ctx, args[1:], out, deps)
|
||||||
|
case "artifacts":
|
||||||
|
return configArtifactsWithDependencies(ctx, args[1:], out, deps)
|
||||||
case "diff":
|
case "diff":
|
||||||
return ConfigDiff(ctx, args[1:], out)
|
return configDiffWithDependencies(ctx, args[1:], out, deps)
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("config: unknown subcommand %q", args[0])
|
return fmt.Errorf("config: unknown subcommand %q", args[0])
|
||||||
}
|
}
|
||||||
@@ -139,6 +143,70 @@ func configSourcesWithDependencies(_ context.Context, args []string, out io.Writ
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ConfigArtifacts validates and reports the effective concrete artifact catalog
|
||||||
|
// without loading session state or runtime collaborators.
|
||||||
|
func ConfigArtifacts(ctx context.Context, args []string, out io.Writer) error {
|
||||||
|
return configArtifactsWithDependencies(ctx, args, out, commandDependencies{})
|
||||||
|
}
|
||||||
|
|
||||||
|
func configArtifactsWithDependencies(_ context.Context, args []string, out io.Writer, deps commandDependencies) error {
|
||||||
|
flags, err := parseInspectionFlags("config artifacts", args, out)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, flag.ErrHelp) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
resolved, err := resolveInspectionConfigWithDependencies(flags, deps)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("config artifacts: %w", err)
|
||||||
|
}
|
||||||
|
records, err := config.EffectiveArtifactCatalog(resolved.Pipeline, resolved.Party)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("config artifacts: %w", err)
|
||||||
|
}
|
||||||
|
if _, err := fmt.Fprintln(out, "key\tsource\tdefinitions\tenabled\toutput\tdependencies\tfamily\tcharacter\tfamily_source\tparty_source"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, record := range records {
|
||||||
|
fields := []string{
|
||||||
|
record.Key,
|
||||||
|
record.SourceID,
|
||||||
|
strings.Join(normalizedCatalogPaths(record.DefinitionSources), ","),
|
||||||
|
strconv.FormatBool(record.Enabled),
|
||||||
|
record.OutputPath,
|
||||||
|
strings.Join(record.Dependencies, ","),
|
||||||
|
record.FamilyKey,
|
||||||
|
record.CharacterID,
|
||||||
|
config.NormalizedConfigurationPath(record.FamilySource),
|
||||||
|
config.NormalizedConfigurationPath(record.PartySource),
|
||||||
|
}
|
||||||
|
for index := range fields {
|
||||||
|
fields[index] = escapeCatalogField(fields[index])
|
||||||
|
}
|
||||||
|
if _, err := fmt.Fprintln(out, strings.Join(fields, "\t")); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizedCatalogPaths(paths []string) []string {
|
||||||
|
result := make([]string, len(paths))
|
||||||
|
for index, path := range paths {
|
||||||
|
result[index] = config.NormalizedConfigurationPath(path)
|
||||||
|
}
|
||||||
|
sort.Strings(result)
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
func escapeCatalogField(value string) string {
|
||||||
|
value = strings.ReplaceAll(value, "\\", "\\\\")
|
||||||
|
value = strings.ReplaceAll(value, "\t", "\\t")
|
||||||
|
value = strings.ReplaceAll(value, "\r", "\\r")
|
||||||
|
return strings.ReplaceAll(value, "\n", "\\n")
|
||||||
|
}
|
||||||
|
|
||||||
func parseInspectionFlags(command string, args []string, out io.Writer) (inspectionFlags, error) {
|
func parseInspectionFlags(command string, args []string, out io.Writer) (inspectionFlags, error) {
|
||||||
fs := flag.NewFlagSet(command, flag.ContinueOnError)
|
fs := flag.NewFlagSet(command, flag.ContinueOnError)
|
||||||
fs.SetOutput(out)
|
fs.SetOutput(out)
|
||||||
@@ -148,8 +216,10 @@ func parseInspectionFlags(command string, args []string, out io.Writer) (inspect
|
|||||||
fs.StringVar(&flags.campaignFilePath, "campaign-file", "", "path to campaign.yml")
|
fs.StringVar(&flags.campaignFilePath, "campaign-file", "", "path to campaign.yml")
|
||||||
flags.profile.name = "profile"
|
flags.profile.name = "profile"
|
||||||
fs.Var(&flags.profile, "profile", "named pipeline profile")
|
fs.Var(&flags.profile, "profile", "named pipeline profile")
|
||||||
|
flags.secretsDir.name = "secrets-dir"
|
||||||
|
fs.Var(&flags.secretsDir, "secrets-dir", "secrets directory (optional; defaults beside pipeline.yml)")
|
||||||
fs.Usage = func() {
|
fs.Usage = func() {
|
||||||
fmt.Fprintf(out, "Usage: narratio %s [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>]\n\n", command)
|
fmt.Fprintf(out, "Usage: narratio %s [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>] [--secrets-dir <path>]\n\n", command)
|
||||||
fmt.Fprintln(out, "Resolves configuration without a session, workspace, manifest, adapters, or external services.")
|
fmt.Fprintln(out, "Resolves configuration without a session, workspace, manifest, adapters, or external services.")
|
||||||
fmt.Fprintln(out)
|
fmt.Fprintln(out)
|
||||||
fmt.Fprintln(out, "Flags:")
|
fmt.Fprintln(out, "Flags:")
|
||||||
@@ -168,7 +238,7 @@ func parseInspectionFlags(command string, args []string, out io.Writer) (inspect
|
|||||||
}
|
}
|
||||||
|
|
||||||
func resolveInspectionConfigWithDependencies(flags inspectionFlags, deps commandDependencies) (*inspectionConfig, error) {
|
func resolveInspectionConfigWithDependencies(flags inspectionFlags, deps commandDependencies) (*inspectionConfig, error) {
|
||||||
pipelinePath, pipeline, err := loadPipelineConfigWithDependencies(flags.pipelinePath, config.PipelineLoadOptions{Profile: flags.profile.pointer()}, deps)
|
pipelinePath, pipeline, err := loadPipelineConfigWithDependencies(flags.pipelinePath, config.PipelineLoadOptions{Profile: flags.profile.pointer()}, flags.secretsDir.pointer(), deps)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -253,9 +323,9 @@ func inspectionProfile(pipeline *config.PipelineConfig) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func writeConfigCommandUsage(out io.Writer) {
|
func writeConfigCommandUsage(out io.Writer) {
|
||||||
fmt.Fprintln(out, "Usage: narratio config <validate|show|sources|diff>")
|
fmt.Fprintln(out, "Usage: narratio config <validate|show|sources|artifacts|diff>")
|
||||||
fmt.Fprintln(out)
|
fmt.Fprintln(out)
|
||||||
fmt.Fprintln(out, "Use config validate to check an effective pipeline, config show to print normalized YAML, config sources to report ownership, or config diff to compare two profiles.")
|
fmt.Fprintln(out, "Use config validate to check an effective pipeline, config show to print normalized YAML, config sources to report ownership, config artifacts to list concrete artifacts, or config diff to compare two profiles.")
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeInspectionSourceHeader(out io.Writer, resolved *inspectionConfig) error {
|
func writeInspectionSourceHeader(out io.Writer, resolved *inspectionConfig) error {
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -21,13 +22,14 @@ func TestConfigCommandHelpAndDispatch(t *testing.T) {
|
|||||||
want string
|
want string
|
||||||
code int
|
code int
|
||||||
}{
|
}{
|
||||||
{name: "top level help", args: []string{"config", "--help"}, want: "Usage: narratio config <validate|show|sources|diff>"},
|
{name: "top level help", args: []string{"config", "--help"}, want: "Usage: narratio config <validate|show|sources|artifacts|diff>"},
|
||||||
{name: "validate help", args: []string{"config", "validate", "--help"}, want: "Usage: narratio config validate"},
|
{name: "validate help", args: []string{"config", "validate", "--help"}, want: "Usage: narratio config validate"},
|
||||||
{name: "show help", args: []string{"config", "show", "--help"}, want: "Usage: narratio config show"},
|
{name: "show help", args: []string{"config", "show", "--help"}, want: "Usage: narratio config show"},
|
||||||
{name: "sources help", args: []string{"config", "sources", "--help"}, want: "Usage: narratio config sources"},
|
{name: "sources help", args: []string{"config", "sources", "--help"}, want: "Usage: narratio config sources"},
|
||||||
|
{name: "artifacts help", args: []string{"config", "artifacts", "--help"}, want: "Usage: narratio config artifacts"},
|
||||||
{name: "diff help", args: []string{"config", "diff", "--help"}, want: "Usage: narratio config diff"},
|
{name: "diff help", args: []string{"config", "diff", "--help"}, want: "Usage: narratio config diff"},
|
||||||
{name: "unknown", args: []string{"config", "unknown"}, want: `config: unknown subcommand "unknown"`, code: 1},
|
{name: "unknown", args: []string{"config", "unknown"}, want: `config: unknown subcommand "unknown"`, code: 1},
|
||||||
{name: "missing", args: []string{"config"}, want: "config: expected subcommand: validate|show|sources|diff", code: 1},
|
{name: "missing", args: []string{"config"}, want: "config: expected subcommand: validate|show|sources|artifacts|diff", code: 1},
|
||||||
{name: "session flag", args: []string{"config", "validate", "--session", "session.yml"}, want: "flag provided but not defined", code: 1},
|
{name: "session flag", args: []string{"config", "validate", "--session", "session.yml"}, want: "flag provided but not defined", code: 1},
|
||||||
{name: "stage flag", args: []string{"config", "show", "--from", "prepare"}, want: "flag provided but not defined", code: 1},
|
{name: "stage flag", args: []string{"config", "show", "--from", "prepare"}, want: "flag provided but not defined", code: 1},
|
||||||
{name: "force flag", args: []string{"config", "show", "--force"}, want: "flag provided but not defined", code: 1},
|
{name: "force flag", args: []string{"config", "show", "--force"}, want: "flag provided but not defined", code: 1},
|
||||||
@@ -52,7 +54,7 @@ func TestExecuteConfigUsesInvocationPipelineLoader(t *testing.T) {
|
|||||||
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||||
mustWriteTestFile(t, pipelinePath, "workspace:\n root: "+filepath.Join(dir, "workspace")+"\nwhisperx:\n transcribe_url: https://transcription.example.com/transcribe\n")
|
mustWriteTestFile(t, pipelinePath, "workspace:\n root: "+filepath.Join(dir, "workspace")+"\nwhisperx:\n transcribe_url: https://transcription.example.com/transcribe\n")
|
||||||
|
|
||||||
for _, subcommand := range []string{"validate", "show", "sources"} {
|
for _, subcommand := range []string{"validate", "show", "sources", "artifacts"} {
|
||||||
t.Run(subcommand, func(t *testing.T) {
|
t.Run(subcommand, func(t *testing.T) {
|
||||||
calls := 0
|
calls := 0
|
||||||
deps := commandDependencies{
|
deps := commandDependencies{
|
||||||
@@ -91,8 +93,6 @@ func TestConfigValidateAndShowPipelineOnlyAreSideEffectFree(t *testing.T) {
|
|||||||
defer server.Close()
|
defer server.Close()
|
||||||
mustWriteTestFile(t, pipelinePath, `workspace:
|
mustWriteTestFile(t, pipelinePath, `workspace:
|
||||||
root: `+workspaceRoot+`
|
root: `+workspaceRoot+`
|
||||||
secrets:
|
|
||||||
env_dir: `+secretsDir+`
|
|
||||||
whisperx:
|
whisperx:
|
||||||
transcribe_url: `+server.URL+`
|
transcribe_url: `+server.URL+`
|
||||||
`)
|
`)
|
||||||
@@ -207,6 +207,56 @@ scriptorium:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestConfigShowPreservesNullAndEmptyCollections(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||||
|
mustWriteTestFile(t, pipelinePath, `workspace:
|
||||||
|
root: /tmp/narratio-work
|
||||||
|
whisperx:
|
||||||
|
transcribe_url: https://transcription.example.test/transcribe
|
||||||
|
audita:
|
||||||
|
modules: null
|
||||||
|
scriptorium:
|
||||||
|
artifacts: null
|
||||||
|
`)
|
||||||
|
mustWriteTestFile(t, filepath.Join(dir, "profiles", "empty.yml"), `audita:
|
||||||
|
modules: []
|
||||||
|
scriptorium:
|
||||||
|
artifacts: {}
|
||||||
|
`)
|
||||||
|
|
||||||
|
var base, empty bytes.Buffer
|
||||||
|
if err := ConfigShow(context.Background(), []string{"--config", pipelinePath}, &base); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := ConfigShow(context.Background(), []string{"--config", pipelinePath, "--profile", "empty"}, &empty); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, want := range []string{"modules: null\n", "artifacts: null\n"} {
|
||||||
|
if !strings.Contains(base.String(), want) {
|
||||||
|
t.Fatalf("base show output missing %q:\n%s", want, base.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, want := range []string{"modules: []\n", "artifacts: {}\n"} {
|
||||||
|
if !strings.Contains(empty.String(), want) {
|
||||||
|
t.Fatalf("empty-profile show output missing %q:\n%s", want, empty.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var diff bytes.Buffer
|
||||||
|
if err := ConfigDiff(context.Background(), []string{"default", "empty", "--config", pipelinePath}, &diff); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, want := range []string{
|
||||||
|
"changed\taudita.modules\tnull\t[]\n",
|
||||||
|
"changed\tscriptorium.artifacts\tnull\t{}\n",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(diff.String(), want) {
|
||||||
|
t.Fatalf("diff output missing %q:\n%s", want, diff.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestConfigDiffComparesExpandedFamiliesAndPublishRules(t *testing.T) {
|
func TestConfigDiffComparesExpandedFamiliesAndPublishRules(t *testing.T) {
|
||||||
pipelinePath, campaignPath := writeInspectionFamilyConfig(t)
|
pipelinePath, campaignPath := writeInspectionFamilyConfig(t)
|
||||||
writeInspectionProfiles(t, pipelinePath, `scriptorium:
|
writeInspectionProfiles(t, pipelinePath, `scriptorium:
|
||||||
@@ -276,6 +326,137 @@ func TestConfigDiffReportsEqualityAndRejectsInvalidInput(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestConfigDiffBootstrapsSecretsOnceForBothProfiles(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, _, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
writeInspectionProfiles(t, pipelinePath, "whisperx:\n language: en\n", "whisperx:\n language: fr\n")
|
||||||
|
secretsDir := t.TempDir()
|
||||||
|
const secretName = "NARRATIO_DIFF_BOOTSTRAP_REPORT"
|
||||||
|
restoreEnvAfterTest(t, secretName)
|
||||||
|
if err := os.Unsetenv(secretName); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
mustWriteSecretFile(t, filepath.Join(secretsDir, secretName), "diff-secret-must-not-appear\n")
|
||||||
|
bootstrapCalls := 0
|
||||||
|
var diagnostics bytes.Buffer
|
||||||
|
deps := commandDependencies{loadSecretsDir: func(got string) (*configkit.SecretsReport, error) {
|
||||||
|
bootstrapCalls++
|
||||||
|
if got != secretsDir {
|
||||||
|
t.Fatalf("secrets directory = %q, want %q", got, secretsDir)
|
||||||
|
}
|
||||||
|
return configkit.LoadSecretsDir(got)
|
||||||
|
}, diagnostics: &diagnostics}
|
||||||
|
|
||||||
|
if err := configDiffWithDependencies(context.Background(), []string{
|
||||||
|
"production", "testing", "--config", pipelinePath, "--secrets-dir", secretsDir,
|
||||||
|
}, io.Discard, deps); err != nil {
|
||||||
|
t.Fatalf("config diff: %v", err)
|
||||||
|
}
|
||||||
|
if bootstrapCalls != 1 {
|
||||||
|
t.Fatalf("bootstrap calls = %d, want 1", bootstrapCalls)
|
||||||
|
}
|
||||||
|
if got := diagnostics.String(); strings.Count(got, `name="`+secretName+`"`) != 1 {
|
||||||
|
t.Fatalf("bootstrap report count = %d, want 1; diagnostics = %q", strings.Count(got, `name="`+secretName+`"`), got)
|
||||||
|
}
|
||||||
|
if strings.Contains(diagnostics.String(), "diff-secret-must-not-appear") {
|
||||||
|
t.Fatalf("bootstrap diagnostics exposed secret value: %q", diagnostics.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigInspectionOutputsNeverExposeResolvedCredentialValues(t *testing.T) {
|
||||||
|
const firstCanary = "inspection-credential-canary-one"
|
||||||
|
const secondCanary = "inspection-credential-canary-two"
|
||||||
|
for name, value := range map[string]string{
|
||||||
|
"NARRATIO_INSPECT_ACCESS": firstCanary,
|
||||||
|
"NARRATIO_INSPECT_SECRET": firstCanary,
|
||||||
|
"NARRATIO_INSPECT_LLM_ONE": firstCanary,
|
||||||
|
"NARRATIO_INSPECT_LLM_TWO": secondCanary,
|
||||||
|
"NARRATIO_INSPECT_NOTARIUS_ONE": firstCanary,
|
||||||
|
"NARRATIO_INSPECT_NOTARIUS_TWO": secondCanary,
|
||||||
|
} {
|
||||||
|
t.Setenv(name, value)
|
||||||
|
}
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, _, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
pipelineData, err := os.ReadFile(pipelinePath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
pipelineYAML := strings.Replace(string(pipelineData), " bucket: test-bucket\n", ` bucket: test-bucket
|
||||||
|
access_key_id: NARRATIO_INSPECT_ACCESS
|
||||||
|
secret_access_key: NARRATIO_INSPECT_SECRET
|
||||||
|
`, 1)
|
||||||
|
pipelineYAML += `notarius:
|
||||||
|
enabled: false
|
||||||
|
llm_api_key: NARRATIO_INSPECT_NOTARIUS_ONE
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
writeInspectionProfiles(t, pipelinePath,
|
||||||
|
"audita:\n model: production\n llm_api_key: NARRATIO_INSPECT_LLM_ONE\n",
|
||||||
|
"audita:\n model: testing\n llm_api_key: NARRATIO_INSPECT_LLM_TWO\nnotarius:\n llm_api_key: NARRATIO_INSPECT_NOTARIUS_TWO\n",
|
||||||
|
)
|
||||||
|
mustWriteTestFile(t, filepath.Join(filepath.Dir(pipelinePath), "profiles", "without-notarius-key.yml"), "notarius:\n llm_api_key: null\n")
|
||||||
|
|
||||||
|
var validate bytes.Buffer
|
||||||
|
if err := ConfigValidate(context.Background(), []string{"--config", pipelinePath, "--profile", "production"}, &validate); err != nil {
|
||||||
|
t.Fatalf("config validate: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var show bytes.Buffer
|
||||||
|
if err := ConfigShow(context.Background(), []string{"--config", pipelinePath, "--profile", "production"}, &show); err != nil {
|
||||||
|
t.Fatalf("config show: %v", err)
|
||||||
|
}
|
||||||
|
var sources bytes.Buffer
|
||||||
|
if err := ConfigSources(context.Background(), []string{"--config", pipelinePath, "--profile", "production"}, &sources); err != nil {
|
||||||
|
t.Fatalf("config sources: %v", err)
|
||||||
|
}
|
||||||
|
var catalog bytes.Buffer
|
||||||
|
if err := ConfigArtifacts(context.Background(), []string{"--config", pipelinePath, "--profile", "production"}, &catalog); err != nil {
|
||||||
|
t.Fatalf("config artifacts: %v", err)
|
||||||
|
}
|
||||||
|
var diff bytes.Buffer
|
||||||
|
if err := ConfigDiff(context.Background(), []string{"production", "testing", "--config", pipelinePath}, &diff); err != nil {
|
||||||
|
t.Fatalf("config diff: %v", err)
|
||||||
|
}
|
||||||
|
var removed bytes.Buffer
|
||||||
|
if err := ConfigShow(context.Background(), []string{"--config", pipelinePath, "--profile", "without-notarius-key"}, &removed); err != nil {
|
||||||
|
t.Fatalf("config show with removed Notarius credential: %v", err)
|
||||||
|
}
|
||||||
|
for label, output := range map[string]string{"show": show.String(), "sources": sources.String(), "artifacts": catalog.String(), "diff": diff.String()} {
|
||||||
|
if strings.Contains(output, firstCanary) || strings.Contains(output, secondCanary) {
|
||||||
|
t.Fatalf("%s output exposed credential value: %q", label, output)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, reference := range []string{"NARRATIO_INSPECT_ACCESS", "NARRATIO_INSPECT_SECRET", "NARRATIO_INSPECT_LLM_ONE", "NARRATIO_INSPECT_NOTARIUS_ONE"} {
|
||||||
|
if !strings.Contains(show.String(), reference) {
|
||||||
|
t.Fatalf("show omitted safe credential reference %q: %q", reference, show.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !strings.Contains(diff.String(), "NARRATIO_INSPECT_LLM_ONE") || !strings.Contains(diff.String(), "NARRATIO_INSPECT_LLM_TWO") ||
|
||||||
|
!strings.Contains(diff.String(), "NARRATIO_INSPECT_NOTARIUS_ONE") || !strings.Contains(diff.String(), "NARRATIO_INSPECT_NOTARIUS_TWO") {
|
||||||
|
t.Fatalf("diff omitted safe credential references: %q", diff.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(sources.String(), "notarius.llm_api_key") {
|
||||||
|
t.Fatalf("sources omitted Notarius credential ownership: %q", sources.String())
|
||||||
|
}
|
||||||
|
if strings.Contains(removed.String(), "llm_api_key") || strings.Contains(removed.String(), "NARRATIO_INSPECT_NOTARIUS_ONE") {
|
||||||
|
t.Fatalf("profile null did not remove Notarius credential: %q", removed.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigValidateRejectsUnavailableNotariusCredential(t *testing.T) {
|
||||||
|
const envName = "NARRATIO_INSPECT_NOTARIUS_MISSING"
|
||||||
|
restoreEnvAfterTest(t, envName)
|
||||||
|
path := filepath.Join(t.TempDir(), "pipeline.yml")
|
||||||
|
mustWriteTestFile(t, path, "whisperx:\n transcribe_url: https://transcription.example.test/transcribe\nnotarius:\n enabled: false\n llm_api_key: "+envName+"\n")
|
||||||
|
err := ConfigValidate(context.Background(), []string{"--config", path}, io.Discard)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "secret.unset") {
|
||||||
|
t.Fatalf("ConfigValidate() error = %v, want unavailable credential failure", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestConfigCommandsInspectMaintainedSplitBundleWithoutRuntimeState(t *testing.T) {
|
func TestConfigCommandsInspectMaintainedSplitBundleWithoutRuntimeState(t *testing.T) {
|
||||||
examplesDir := filepath.Join("..", "..", "examples")
|
examplesDir := filepath.Join("..", "..", "examples")
|
||||||
pipelinePath := filepath.Join(examplesDir, "production-testing", "pipeline.yml")
|
pipelinePath := filepath.Join(examplesDir, "production-testing", "pipeline.yml")
|
||||||
@@ -285,13 +466,13 @@ func TestConfigCommandsInspectMaintainedSplitBundleWithoutRuntimeState(t *testin
|
|||||||
t.Fatalf("example workspace stat = %v, want absent", err)
|
t.Fatalf("example workspace stat = %v, want absent", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, command := range []func(context.Context, []string, io.Writer) error{ConfigValidate, ConfigShow, ConfigSources} {
|
for _, command := range []func(context.Context, []string, io.Writer) error{ConfigValidate, ConfigShow, ConfigSources, ConfigArtifacts} {
|
||||||
if err := command(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath, "--profile", "production"}, io.Discard); err != nil {
|
if err := command(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath}, io.Discard); err != nil {
|
||||||
t.Fatalf("inspection command %T error = %v", command, err)
|
t.Fatalf("inspection command %T error = %v", command, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
var diff bytes.Buffer
|
var diff bytes.Buffer
|
||||||
if err := ConfigDiff(context.Background(), []string{"production", "testing", "--config", pipelinePath, "--campaign-file", campaignPath}, &diff); err != nil {
|
if err := ConfigDiff(context.Background(), []string{"default", "testing", "--config", pipelinePath, "--campaign-file", campaignPath}, &diff); err != nil {
|
||||||
t.Fatalf("ConfigDiff() error = %v", err)
|
t.Fatalf("ConfigDiff() error = %v", err)
|
||||||
}
|
}
|
||||||
if !strings.Contains(diff.String(), "audita.model") {
|
if !strings.Contains(diff.String(), "audita.model") {
|
||||||
@@ -304,17 +485,9 @@ func TestConfigCommandsInspectMaintainedSplitBundleWithoutRuntimeState(t *testin
|
|||||||
|
|
||||||
func writeInspectionProfiles(t *testing.T, pipelinePath, production, testing string) {
|
func writeInspectionProfiles(t *testing.T, pipelinePath, production, testing string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
data, err := os.ReadFile(pipelinePath)
|
profilesDir := filepath.Join(filepath.Dir(pipelinePath), "profiles")
|
||||||
if err != nil {
|
mustWriteTestFile(t, filepath.Join(profilesDir, "production.yml"), production)
|
||||||
t.Fatal(err)
|
mustWriteTestFile(t, filepath.Join(profilesDir, "testing.yml"), testing)
|
||||||
}
|
|
||||||
composition := "composition:\n default_profile: production\n profiles:\n production:\n overlay: production.yml\n testing:\n overlay: testing.yml\n"
|
|
||||||
if err := os.WriteFile(pipelinePath, append([]byte(composition), data...), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
dir := filepath.Dir(pipelinePath)
|
|
||||||
mustWriteTestFile(t, filepath.Join(dir, "production.yml"), production)
|
|
||||||
mustWriteTestFile(t, filepath.Join(dir, "testing.yml"), testing)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestConfigCommandsSelectProfilesAndCampaigns(t *testing.T) {
|
func TestConfigCommandsSelectProfilesAndCampaigns(t *testing.T) {
|
||||||
@@ -326,7 +499,7 @@ func TestConfigCommandsSelectProfilesAndCampaigns(t *testing.T) {
|
|||||||
if err := ConfigValidate(context.Background(), []string{"--config", pipelinePath}, &defaultOut); err != nil {
|
if err := ConfigValidate(context.Background(), []string{"--config", pipelinePath}, &defaultOut); err != nil {
|
||||||
t.Fatalf("default ConfigValidate() error = %v", err)
|
t.Fatalf("default ConfigValidate() error = %v", err)
|
||||||
}
|
}
|
||||||
if !strings.Contains(defaultOut.String(), "profile=production") {
|
if !strings.Contains(defaultOut.String(), "profile=none") {
|
||||||
t.Fatalf("default output = %q", defaultOut.String())
|
t.Fatalf("default output = %q", defaultOut.String())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -352,7 +525,7 @@ func TestConfigCommandsSelectProfilesAndCampaigns(t *testing.T) {
|
|||||||
func TestConfigCommandsExpandFamiliesAndRequireCampaign(t *testing.T) {
|
func TestConfigCommandsExpandFamiliesAndRequireCampaign(t *testing.T) {
|
||||||
pipelinePath, campaignPath := writeInspectionFamilyConfig(t)
|
pipelinePath, campaignPath := writeInspectionFamilyConfig(t)
|
||||||
|
|
||||||
for _, command := range []func(context.Context, []string, io.Writer) error{ConfigValidate, ConfigShow} {
|
for _, command := range []func(context.Context, []string, io.Writer) error{ConfigValidate, ConfigShow, ConfigArtifacts} {
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
err := command(context.Background(), []string{"--config", pipelinePath}, &out)
|
err := command(context.Background(), []string{"--config", pipelinePath}, &out)
|
||||||
if err == nil || !strings.Contains(err.Error(), "artifact_families requires a campaign") {
|
if err == nil || !strings.Contains(err.Error(), "artifact_families requires a campaign") {
|
||||||
@@ -398,8 +571,9 @@ func TestConfigValidateReportsCanonicalPartyAndImportErrors(t *testing.T) {
|
|||||||
|
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
rootPath := filepath.Join(dir, "pipeline.yml")
|
rootPath := filepath.Join(dir, "pipeline.yml")
|
||||||
mustWriteTestFile(t, rootPath, `composition:
|
mustWriteTestFile(t, rootPath, `configkit:
|
||||||
imports: [conf.yml]
|
imports:
|
||||||
|
- path: conf.yml
|
||||||
workspace:
|
workspace:
|
||||||
root: /one
|
root: /one
|
||||||
whisperx:
|
whisperx:
|
||||||
@@ -407,7 +581,7 @@ whisperx:
|
|||||||
`)
|
`)
|
||||||
mustWriteTestFile(t, filepath.Join(dir, "conf.yml"), "workspace:\n root: /two\n")
|
mustWriteTestFile(t, filepath.Join(dir, "conf.yml"), "workspace:\n root: /two\n")
|
||||||
err := ConfigValidate(context.Background(), []string{"--config", rootPath}, io.Discard)
|
err := ConfigValidate(context.Background(), []string{"--config", rootPath}, io.Discard)
|
||||||
if err == nil || !strings.Contains(err.Error(), "workspace.root") || !strings.Contains(err.Error(), "pipeline.yml") || !strings.Contains(err.Error(), "conf.yml") {
|
if err == nil || !strings.Contains(err.Error(), "/workspace/root") || !strings.Contains(err.Error(), "pipeline.yml") || !strings.Contains(err.Error(), "conf.yml") {
|
||||||
t.Fatalf("import diagnostic = %v", err)
|
t.Fatalf("import diagnostic = %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -432,7 +606,7 @@ func TestConfigSourcesReportsStableOwnership(t *testing.T) {
|
|||||||
for _, want := range []string{
|
for _, want := range []string{
|
||||||
"root: ",
|
"root: ",
|
||||||
"imports: none",
|
"imports: none",
|
||||||
"profile: name=testing selection=cli overlay=",
|
"profile: name=testing selection=explicit overlay=",
|
||||||
"campaign: id=sample-campaign source=",
|
"campaign: id=sample-campaign source=",
|
||||||
"party: source=",
|
"party: source=",
|
||||||
"digest: ",
|
"digest: ",
|
||||||
@@ -475,15 +649,7 @@ func TestConfigSourcesReportsCanonicalFamilyAndPublishOrigins(t *testing.T) {
|
|||||||
|
|
||||||
func TestConfigSourcesPreservesProfileOwnershipThroughFamilyExpansion(t *testing.T) {
|
func TestConfigSourcesPreservesProfileOwnershipThroughFamilyExpansion(t *testing.T) {
|
||||||
pipelinePath, campaignPath := writeInspectionFamilyConfig(t)
|
pipelinePath, campaignPath := writeInspectionFamilyConfig(t)
|
||||||
data, err := os.ReadFile(pipelinePath)
|
profilePath := filepath.Join(filepath.Dir(pipelinePath), "profiles", "testing.yml")
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
profilePath := filepath.Join(filepath.Dir(pipelinePath), "testing.yml")
|
|
||||||
composition := "composition:\n default_profile: testing\n profiles:\n testing:\n overlay: testing.yml\n"
|
|
||||||
if err := os.WriteFile(pipelinePath, append([]byte(composition), data...), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
mustWriteTestFile(t, profilePath, `scriptorium:
|
mustWriteTestFile(t, profilePath, `scriptorium:
|
||||||
artifact_families:
|
artifact_families:
|
||||||
character_note:
|
character_note:
|
||||||
@@ -492,7 +658,7 @@ func TestConfigSourcesPreservesProfileOwnershipThroughFamilyExpansion(t *testing
|
|||||||
`)
|
`)
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
if err := ConfigSources(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath}, &out); err != nil {
|
if err := ConfigSources(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath, "--profile", "testing"}, &out); err != nil {
|
||||||
t.Fatalf("ConfigSources() error = %v", err)
|
t.Fatalf("ConfigSources() error = %v", err)
|
||||||
}
|
}
|
||||||
want := "scriptorium.artifacts.character_note_arannis.enabled\tfamily\t" + profilePath
|
want := "scriptorium.artifacts.character_note_arannis.enabled\tfamily\t" + profilePath
|
||||||
@@ -501,17 +667,74 @@ func TestConfigSourcesPreservesProfileOwnershipThroughFamilyExpansion(t *testing
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestConfigSourcesAttributesInferredArtifactDependencies(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||||
|
inputPath := filepath.Join(dir, "input-only.yml")
|
||||||
|
profilePath := filepath.Join(dir, "profiles", "testing.yml")
|
||||||
|
mustWriteTestFile(t, pipelinePath, `configkit:
|
||||||
|
imports:
|
||||||
|
- path: input-only.yml
|
||||||
|
workspace:
|
||||||
|
root: /srv/narratio
|
||||||
|
whisperx:
|
||||||
|
transcribe_url: https://transcription.example.test/transcribe
|
||||||
|
scriptorium:
|
||||||
|
artifacts:
|
||||||
|
first:
|
||||||
|
enabled: false
|
||||||
|
output_path: artifacts/first.md
|
||||||
|
second:
|
||||||
|
enabled: false
|
||||||
|
output_path: artifacts/second.md
|
||||||
|
mixed:
|
||||||
|
enabled: false
|
||||||
|
output_path: artifacts/mixed.md
|
||||||
|
depends_on: [first]
|
||||||
|
`)
|
||||||
|
mustWriteTestFile(t, inputPath, `scriptorium:
|
||||||
|
artifacts:
|
||||||
|
input_only:
|
||||||
|
enabled: false
|
||||||
|
output_path: artifacts/input-only.md
|
||||||
|
inputs:
|
||||||
|
prior:
|
||||||
|
source: narratio.artifact.first
|
||||||
|
`)
|
||||||
|
mustWriteTestFile(t, profilePath, `scriptorium:
|
||||||
|
artifacts:
|
||||||
|
mixed:
|
||||||
|
inputs:
|
||||||
|
prior:
|
||||||
|
source: narratio.artifact.second
|
||||||
|
`)
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
if err := ConfigSources(context.Background(), []string{"--config", pipelinePath, "--profile", "testing"}, &out); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, want := range []string{
|
||||||
|
"scriptorium.artifacts.input_only.depends_on\timport\t" + inputPath,
|
||||||
|
"scriptorium.artifacts.mixed.depends_on\troot\t" + pipelinePath,
|
||||||
|
"scriptorium.artifacts.mixed.depends_on\tprofile\t" + profilePath,
|
||||||
|
} {
|
||||||
|
if !strings.Contains(out.String(), want) {
|
||||||
|
t.Fatalf("sources output missing dependency contributor %q:\n%s", want, out.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Contains(out.String(), "scriptorium.artifacts.input_only.depends_on\tdefault\tdefault") {
|
||||||
|
t.Fatalf("inferred dependency retained default attribution:\n%s", out.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestConfigSourcesReportsRootImportProfileAndDefaultOwnership(t *testing.T) {
|
func TestConfigSourcesReportsRootImportProfileAndDefaultOwnership(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
rootPath := filepath.Join(dir, "pipeline.yml")
|
rootPath := filepath.Join(dir, "pipeline.yml")
|
||||||
importPath := filepath.Join(dir, "base.yml")
|
importPath := filepath.Join(dir, "base.yml")
|
||||||
profilePath := filepath.Join(dir, "testing.yml")
|
profilePath := filepath.Join(dir, "profiles", "testing.yml")
|
||||||
mustWriteTestFile(t, rootPath, `composition:
|
mustWriteTestFile(t, rootPath, `configkit:
|
||||||
imports: [base.yml]
|
imports:
|
||||||
default_profile: testing
|
- path: base.yml
|
||||||
profiles:
|
|
||||||
testing:
|
|
||||||
overlay: testing.yml
|
|
||||||
workspace:
|
workspace:
|
||||||
root: /srv/narratio
|
root: /srv/narratio
|
||||||
whisperx:
|
whisperx:
|
||||||
@@ -521,13 +744,13 @@ whisperx:
|
|||||||
mustWriteTestFile(t, profilePath, "whisperx:\n language: fr\n")
|
mustWriteTestFile(t, profilePath, "whisperx:\n language: fr\n")
|
||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
if err := ConfigSources(context.Background(), []string{"--config", rootPath}, &out); err != nil {
|
if err := ConfigSources(context.Background(), []string{"--config", rootPath, "--profile", "testing"}, &out); err != nil {
|
||||||
t.Fatalf("ConfigSources() error = %v", err)
|
t.Fatalf("ConfigSources() error = %v", err)
|
||||||
}
|
}
|
||||||
output := out.String()
|
output := out.String()
|
||||||
for _, want := range []string{
|
for _, want := range []string{
|
||||||
"import: " + importPath,
|
"import: " + importPath,
|
||||||
"profile: name=testing selection=default overlay=" + profilePath,
|
"profile: name=testing selection=explicit overlay=" + profilePath,
|
||||||
"workspace.root\troot\t" + rootPath,
|
"workspace.root\troot\t" + rootPath,
|
||||||
"storage.backend\timport\t" + importPath,
|
"storage.backend\timport\t" + importPath,
|
||||||
"whisperx.language\tprofile\t" + profilePath,
|
"whisperx.language\tprofile\t" + profilePath,
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ type configDiffFlags struct {
|
|||||||
pipelinePath singletonStringFlag
|
pipelinePath singletonStringFlag
|
||||||
campaignPath singletonStringFlag
|
campaignPath singletonStringFlag
|
||||||
campaignFilePath singletonStringFlag
|
campaignFilePath singletonStringFlag
|
||||||
|
secretsDir singletonStringFlag
|
||||||
}
|
}
|
||||||
|
|
||||||
type configDiffRequest struct {
|
type configDiffRequest struct {
|
||||||
@@ -40,7 +41,11 @@ type configDiffRecord struct {
|
|||||||
// read-only configuration resolution boundary used by config validate, show,
|
// read-only configuration resolution boundary used by config validate, show,
|
||||||
// and sources. Differences describe normalized effective values rather than
|
// and sources. Differences describe normalized effective values rather than
|
||||||
// the layout of root, import, or overlay files.
|
// the layout of root, import, or overlay files.
|
||||||
func ConfigDiff(_ context.Context, args []string, out io.Writer) error {
|
func ConfigDiff(ctx context.Context, args []string, out io.Writer) error {
|
||||||
|
return configDiffWithDependencies(ctx, args, out, commandDependencies{})
|
||||||
|
}
|
||||||
|
|
||||||
|
func configDiffWithDependencies(_ context.Context, args []string, out io.Writer, deps commandDependencies) error {
|
||||||
request, err := parseConfigDiffRequest(args, out)
|
request, err := parseConfigDiffRequest(args, out)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, flag.ErrHelp) {
|
if errors.Is(err, flag.ErrHelp) {
|
||||||
@@ -48,7 +53,7 @@ func ConfigDiff(_ context.Context, args []string, out io.Writer) error {
|
|||||||
}
|
}
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
resolved, err := resolveConfigDiff(request)
|
resolved, err := resolveConfigDiff(request, deps)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("config diff: %w", err)
|
return fmt.Errorf("config diff: %w", err)
|
||||||
}
|
}
|
||||||
@@ -102,11 +107,13 @@ func parseConfigDiffRequest(args []string, out io.Writer) (configDiffRequest, er
|
|||||||
request.flags.pipelinePath.name = "config"
|
request.flags.pipelinePath.name = "config"
|
||||||
request.flags.campaignPath.name = "campaign"
|
request.flags.campaignPath.name = "campaign"
|
||||||
request.flags.campaignFilePath.name = "campaign-file"
|
request.flags.campaignFilePath.name = "campaign-file"
|
||||||
|
request.flags.secretsDir.name = "secrets-dir"
|
||||||
fs.Var(&request.flags.pipelinePath, "config", "path to pipeline.yml (optional; defaults searched)")
|
fs.Var(&request.flags.pipelinePath, "config", "path to pipeline.yml (optional; defaults searched)")
|
||||||
fs.Var(&request.flags.campaignPath, "campaign", "campaign ID")
|
fs.Var(&request.flags.campaignPath, "campaign", "campaign ID")
|
||||||
fs.Var(&request.flags.campaignFilePath, "campaign-file", "path to campaign.yml")
|
fs.Var(&request.flags.campaignFilePath, "campaign-file", "path to campaign.yml")
|
||||||
|
fs.Var(&request.flags.secretsDir, "secrets-dir", "secrets directory (optional; defaults beside pipeline.yml)")
|
||||||
fs.Usage = func() {
|
fs.Usage = func() {
|
||||||
fmt.Fprintln(out, "Usage: narratio config diff <left-profile> <right-profile> [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>]")
|
fmt.Fprintln(out, "Usage: narratio config diff <left-profile> <right-profile> [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--secrets-dir <path>]")
|
||||||
fmt.Fprintln(out)
|
fmt.Fprintln(out)
|
||||||
fmt.Fprintln(out, "Compares fully resolved profiles without a session, workspace, manifest, adapters, or external services.")
|
fmt.Fprintln(out, "Compares fully resolved profiles without a session, workspace, manifest, adapters, or external services.")
|
||||||
fmt.Fprintln(out)
|
fmt.Fprintln(out)
|
||||||
@@ -120,6 +127,9 @@ func parseConfigDiffRequest(args []string, out io.Writer) (configDiffRequest, er
|
|||||||
if len(profiles) != 2 || strings.TrimSpace(profiles[0]) == "" || strings.TrimSpace(profiles[1]) == "" {
|
if len(profiles) != 2 || strings.TrimSpace(profiles[0]) == "" || strings.TrimSpace(profiles[1]) == "" {
|
||||||
return configDiffRequest{}, fmt.Errorf("config diff: exactly two non-empty profile names are required")
|
return configDiffRequest{}, fmt.Errorf("config diff: exactly two non-empty profile names are required")
|
||||||
}
|
}
|
||||||
|
if profiles[0] == profiles[1] {
|
||||||
|
return configDiffRequest{}, fmt.Errorf("config diff: profile names must differ")
|
||||||
|
}
|
||||||
if request.flags.campaignPath.set && request.flags.campaignFilePath.set {
|
if request.flags.campaignPath.set && request.flags.campaignFilePath.set {
|
||||||
return configDiffRequest{}, fmt.Errorf("config diff: --campaign and --campaign-file are mutually exclusive")
|
return configDiffRequest{}, fmt.Errorf("config diff: --campaign and --campaign-file are mutually exclusive")
|
||||||
}
|
}
|
||||||
@@ -156,21 +166,22 @@ func splitConfigDiffArguments(args []string) (profiles, flagArgs []string) {
|
|||||||
|
|
||||||
func configDiffStringFlag(name string) bool {
|
func configDiffStringFlag(name string) bool {
|
||||||
switch name {
|
switch name {
|
||||||
case "config", "campaign", "campaign-file", "profile":
|
case "config", "campaign", "campaign-file", "profile", "secrets-dir":
|
||||||
return true
|
return true
|
||||||
default:
|
default:
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func resolveConfigDiff(request configDiffRequest) (*configDiffResolved, error) {
|
func resolveConfigDiff(request configDiffRequest, deps commandDependencies) (*configDiffResolved, error) {
|
||||||
pipelinePath, err := resolvePipelineConfigPath(request.flags.pipelinePath.value)
|
deps = normalizeCommandDependencies(deps)
|
||||||
|
pipelinePath, leftPipeline, err := loadPipelineConfigWithDependencies(request.flags.pipelinePath.value, config.PipelineLoadOptions{Profile: &request.leftProfile}, request.flags.secretsDir.pointer(), deps)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, fmt.Errorf("load left pipeline profile: %w", err)
|
||||||
}
|
}
|
||||||
leftPipeline, rightPipeline, err := config.LoadPipelineProfilePair(pipelinePath, request.leftProfile, request.rightProfile)
|
_, rightPipeline, err := loadPipelineConfigWithDependencies(request.flags.pipelinePath.value, config.PipelineLoadOptions{Profile: &request.rightProfile}, request.flags.secretsDir.pointer(), deps)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, fmt.Errorf("load right pipeline profile: %w", err)
|
||||||
}
|
}
|
||||||
needsCampaign := pipelineHasArtifactFamilies(leftPipeline) || pipelineHasArtifactFamilies(rightPipeline)
|
needsCampaign := pipelineHasArtifactFamilies(leftPipeline) || pipelineHasArtifactFamilies(rightPipeline)
|
||||||
hasCampaignSelection := request.flags.campaignPath.set || request.flags.campaignFilePath.set
|
hasCampaignSelection := request.flags.campaignPath.set || request.flags.campaignFilePath.set
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ func retainedCommandConfig(cfg *config.Config) *commandConfig {
|
|||||||
return &commandConfig{Config: cfg}
|
return &commandConfig{Config: cfg}
|
||||||
}
|
}
|
||||||
|
|
||||||
func loadCommandConfigWithDependencies(ctx context.Context, pipelineFlag, campaignFlag, campaignFileFlag, sessionFlag string, sessionOpts config.SessionLoadOptions, deps commandDependencies) (loaded *commandConfig, err error) {
|
func loadCommandConfigWithDependencies(ctx context.Context, pipelineFlag, campaignFlag, campaignFileFlag, sessionFlag string, sessionOpts config.SessionLoadOptions, secretsDir *string, deps commandDependencies) (loaded *commandConfig, err error) {
|
||||||
deps = normalizeCommandDependencies(deps)
|
deps = normalizeCommandDependencies(deps)
|
||||||
var cleanup func() error
|
var cleanup func() error
|
||||||
defer func() {
|
defer func() {
|
||||||
@@ -46,7 +46,7 @@ func loadCommandConfigWithDependencies(ctx context.Context, pipelineFlag, campai
|
|||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
base, err := loadPipelineCampaignConfigWithDependencies(pipelineFlag, campaignFlag, campaignFileFlag, config.PipelineLoadOptions{Profile: sessionOpts.Profile}, deps)
|
base, err := loadPipelineCampaignConfigWithDependencies(pipelineFlag, campaignFlag, campaignFileFlag, config.PipelineLoadOptions{Profile: sessionOpts.Profile}, secretsDir, deps)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -135,9 +135,9 @@ func loadCommandConfigWithDependencies(ctx context.Context, pipelineFlag, campai
|
|||||||
return loaded, nil
|
return loaded, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func loadPipelineCampaignConfigWithDependencies(pipelineFlag, campaignFlag, campaignFileFlag string, pipelineOpts config.PipelineLoadOptions, deps commandDependencies) (*pipelineCampaignConfig, error) {
|
func loadPipelineCampaignConfigWithDependencies(pipelineFlag, campaignFlag, campaignFileFlag string, pipelineOpts config.PipelineLoadOptions, secretsDir *string, deps commandDependencies) (*pipelineCampaignConfig, error) {
|
||||||
deps = normalizeCommandDependencies(deps)
|
deps = normalizeCommandDependencies(deps)
|
||||||
loadedPipelinePath, pipelineCfg, err := loadPipelineConfigWithDependencies(pipelineFlag, pipelineOpts, deps)
|
loadedPipelinePath, pipelineCfg, err := loadPipelineConfigWithDependencies(pipelineFlag, pipelineOpts, secretsDir, deps)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -161,12 +161,19 @@ func loadPipelineCampaignConfigWithDependencies(pipelineFlag, campaignFlag, camp
|
|||||||
return &loaded, nil
|
return &loaded, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func loadPipelineConfigWithDependencies(pipelineFlag string, opts config.PipelineLoadOptions, deps commandDependencies) (string, *config.PipelineConfig, error) {
|
func loadPipelineConfigWithDependencies(pipelineFlag string, opts config.PipelineLoadOptions, secretsDir *string, deps commandDependencies) (string, *config.PipelineConfig, error) {
|
||||||
deps = normalizeCommandDependencies(deps)
|
deps = normalizeCommandDependencies(deps)
|
||||||
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelineFlag)
|
resolvedPipelinePath, err := deps.resolvePipelineConfigPath(pipelineFlag)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", nil, err
|
return "", nil, err
|
||||||
}
|
}
|
||||||
|
if !deps.secretsBootstrap.selectionSet {
|
||||||
|
deps.secretsBootstrap.selection = secretsDir
|
||||||
|
deps.secretsBootstrap.selectionSet = true
|
||||||
|
}
|
||||||
|
if err := deps.secretsBootstrap.load(resolvedPipelinePath, deps.loadSecretsDir, deps.diagnostics); err != nil {
|
||||||
|
return "", nil, err
|
||||||
|
}
|
||||||
pipelineCfg, err := deps.loadPipelineConfig(resolvedPipelinePath, opts)
|
pipelineCfg, err := deps.loadPipelineConfig(resolvedPipelinePath, opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", nil, err
|
return "", nil, err
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ func TestLoadCommandConfigRetainsInitiallyLoadedPipeline(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
loaded, err := loadCommandConfigWithDependencies(context.Background(), pipelinePath, "", campaignPath, sessionPath, config.SessionLoadOptions{SessionID: "2026-05-03"}, deps)
|
loaded, err := loadCommandConfigWithDependencies(context.Background(), pipelinePath, "", campaignPath, sessionPath, config.SessionLoadOptions{SessionID: "2026-05-03"}, nil, deps)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("loadCommandConfig() error = %v", err)
|
t.Fatalf("loadCommandConfig() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,11 +7,13 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/notarius"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/notarius"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
|
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
|
||||||
@@ -181,12 +183,16 @@ func TestExtractLifecyclePrepareBindsVerifiedReferenceSnapshots(t *testing.T) {
|
|||||||
for index, expected := range want {
|
for index, expected := range want {
|
||||||
binding := request.References[index]
|
binding := request.References[index]
|
||||||
canonical := filepath.Join(paths.InputsDir, expected.filename)
|
canonical := filepath.Join(paths.InputsDir, expected.filename)
|
||||||
snapshot := filepath.Join(
|
descriptor, ok := artifactpolicy.DescribePreparedInputSource(expected.sourceID)
|
||||||
artifacts.SessionRunNotariusReferencesDirForCampaign(
|
if !ok {
|
||||||
cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, loaded.RunID,
|
t.Fatalf("DescribePreparedInputSource(%q) ok = false", expected.sourceID)
|
||||||
),
|
}
|
||||||
expected.filename,
|
snapshot, err := artifacts.SessionNotariusReferenceSnapshotPath(
|
||||||
|
paths, lifecycleInputChecksum(t, loaded, descriptor.ManifestKind), expected.filename,
|
||||||
)
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("SessionNotariusReferenceSnapshotPath(%q) error = %v", expected.sourceID, err)
|
||||||
|
}
|
||||||
if binding.Selector != expected.selector || binding.Path != snapshot || binding.Path == canonical || binding.Path == originalPaths[expected.sourceID] {
|
if binding.Selector != expected.selector || binding.Path != snapshot || binding.Path == canonical || binding.Path == originalPaths[expected.sourceID] {
|
||||||
t.Fatalf("reference[%d] = %#v, want selector %q snapshot %q and not prepared/source paths", index, binding, expected.selector, snapshot)
|
t.Fatalf("reference[%d] = %#v, want selector %q snapshot %q and not prepared/source paths", index, binding, expected.selector, snapshot)
|
||||||
}
|
}
|
||||||
@@ -393,6 +399,140 @@ func TestExtractLifecycleFailureInvalidatesAndOrdinaryRetryRerunsDownstream(t *t
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExtractLifecycleRetryReusesStableSnapshotsAndChangedReferenceSelectsNewPath(t *testing.T) {
|
||||||
|
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||||
|
configureLifecycleReferences(t, cfg)
|
||||||
|
runner.failuresRemaining = 1
|
||||||
|
plan := prepareExtractLifecyclePlan(t)
|
||||||
|
|
||||||
|
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err == nil {
|
||||||
|
t.Fatal("first executeStages() error = nil, want extraction failure")
|
||||||
|
}
|
||||||
|
if len(runner.requests) != 1 {
|
||||||
|
t.Fatalf("failed requests = %d, want 1", len(runner.requests))
|
||||||
|
}
|
||||||
|
firstPaths := lifecycleReferencePaths(runner.requests[0])
|
||||||
|
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err != nil {
|
||||||
|
t.Fatalf("retry executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(runner.requests) != 2 {
|
||||||
|
t.Fatalf("retry requests = %d, want 2", len(runner.requests))
|
||||||
|
}
|
||||||
|
if secondPaths := lifecycleReferencePaths(runner.requests[1]); !reflect.DeepEqual(secondPaths, firstPaths) {
|
||||||
|
t.Fatalf("snapshot paths changed on retry:\nfirst %#v\nsecond %#v", firstPaths, secondPaths)
|
||||||
|
}
|
||||||
|
|
||||||
|
partyDocument, err := config.ParseParty([]byte(`schema_version: narratio.party.v1
|
||||||
|
characters:
|
||||||
|
changed:
|
||||||
|
player: {name: Eric}
|
||||||
|
character:
|
||||||
|
name: Changed
|
||||||
|
classes: [{name: wizard}]
|
||||||
|
`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ParseParty(changed party) error = %v", err)
|
||||||
|
}
|
||||||
|
cfg.Party.Canonical = partyDocument
|
||||||
|
persisted := loadLifecycleManifest(t, cfg)
|
||||||
|
prepare, err := stage.Select("prepare")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("stage.Select(prepare) error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := prepare.Run(context.Background(), env, persisted); err != nil {
|
||||||
|
t.Fatalf("prepare.Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := (&manifest.LocalStore{}).Save(context.Background(), manifestPathFor(cfg), persisted); err != nil {
|
||||||
|
t.Fatalf("Save(reprepared manifest) error = %v", err)
|
||||||
|
}
|
||||||
|
extractPlan, _ := BuildSingleStagePlan("extract")
|
||||||
|
if _, err := executeStages(context.Background(), cfg, extractPlan, RunOptions{Env: env}); err != nil {
|
||||||
|
t.Fatalf("changed-reference executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(runner.requests) != 3 {
|
||||||
|
t.Fatalf("changed-reference requests = %d, want 3", len(runner.requests))
|
||||||
|
}
|
||||||
|
thirdPaths := lifecycleReferencePaths(runner.requests[2])
|
||||||
|
if thirdPaths["party"] == firstPaths["party"] {
|
||||||
|
t.Fatalf("changed party retained snapshot path %q", thirdPaths["party"])
|
||||||
|
}
|
||||||
|
if thirdPaths["players"] == firstPaths["players"] {
|
||||||
|
t.Fatalf("changed party retained derived players snapshot path %q", thirdPaths["players"])
|
||||||
|
}
|
||||||
|
for selector, firstPath := range firstPaths {
|
||||||
|
if selector != "party" && selector != "players" && thirdPaths[selector] != firstPath {
|
||||||
|
t.Fatalf("unchanged %s snapshot path = %q, want %q", selector, thirdPaths[selector], firstPath)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(firstPath); err != nil {
|
||||||
|
t.Fatalf("old %s snapshot %q was not retained: %v", selector, firstPath, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractLifecycleCacheControlsAndSnapshotLossDoNotInvalidateCompleteResult(t *testing.T) {
|
||||||
|
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||||
|
configureLifecycleReferences(t, cfg)
|
||||||
|
if _, err := executeStages(context.Background(), cfg, prepareExtractLifecyclePlan(t), RunOptions{Env: env}); err != nil {
|
||||||
|
t.Fatalf("initial executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
cacheRoot := artifacts.SessionNotariusReferenceCacheDir(paths)
|
||||||
|
if err := os.RemoveAll(cacheRoot); err != nil {
|
||||||
|
t.Fatalf("RemoveAll(snapshot cache) error = %v", err)
|
||||||
|
}
|
||||||
|
cfg.Pipeline.Notarius.Resume = true
|
||||||
|
cfg.Pipeline.Notarius.ChunkCache = "bypass"
|
||||||
|
extractPlan, _ := BuildSingleStagePlan("extract")
|
||||||
|
summary, err := executeStages(context.Background(), cfg, extractPlan, RunOptions{Env: env})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resume executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(summary.Executed) != 0 || !reflect.DeepEqual(summary.Skipped, []string{"extract"}) || len(runner.requests) != 1 {
|
||||||
|
t.Fatalf("resume summary = %#v requests=%d", summary, len(runner.requests))
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(cacheRoot); !errors.Is(err, os.ErrNotExist) {
|
||||||
|
t.Fatalf("skipped extraction recreated snapshot cache: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnalyzeOnlyLifecyclePreservesExtractionWithoutInvokingNotarius(t *testing.T) {
|
||||||
|
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||||
|
extractPlan, _ := BuildSingleStagePlan("extract")
|
||||||
|
if _, err := executeStages(context.Background(), cfg, extractPlan, RunOptions{Env: env}); err != nil {
|
||||||
|
t.Fatalf("extract executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
before := loadLifecycleManifest(t, cfg).Stages["extract"]
|
||||||
|
beforeJSON, _ := json.Marshal(before)
|
||||||
|
analyzeRuns := 0
|
||||||
|
if _, err := executeStages(context.Background(), cfg, []stage.Stage{countingStage{name: "analyze", runs: &analyzeRuns}}, RunOptions{Env: env, Force: true}); err != nil {
|
||||||
|
t.Fatalf("analyze-only executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
after := loadLifecycleManifest(t, cfg).Stages["extract"]
|
||||||
|
afterJSON, _ := json.Marshal(after)
|
||||||
|
if analyzeRuns != 1 || len(runner.requests) != 1 || string(afterJSON) != string(beforeJSON) {
|
||||||
|
t.Fatalf("analyze-only lifecycle changed extraction: analyze=%d requests=%d\nbefore=%s\nafter=%s", analyzeRuns, len(runner.requests), beforeJSON, afterJSON)
|
||||||
|
}
|
||||||
|
for _, output := range after.Outputs {
|
||||||
|
if _, err := os.Stat(output.LocalPath); err != nil {
|
||||||
|
t.Fatalf("analyze-only lifecycle removed extract output %q: %v", output.LocalPath, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestForcedArtifactRegenerationUsesOrdinaryExtractAndAnalyzePath(t *testing.T) {
|
||||||
|
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||||
|
cfg.Pipeline.Notarius.Resume = true
|
||||||
|
cfg.Pipeline.Notarius.ChunkCache = "bypass"
|
||||||
|
analyzeRuns := 0
|
||||||
|
plan := extractionLifecyclePlan(t, &analyzeRuns)
|
||||||
|
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env, Force: true, SelectedArtifacts: []string{"session_recap"}}); err != nil {
|
||||||
|
t.Fatalf("forced artifact regeneration error = %v", err)
|
||||||
|
}
|
||||||
|
if len(runner.requests) != 1 || runner.requests[0].Resume || runner.requests[0].ChunkCache != "refresh" || analyzeRuns != 1 {
|
||||||
|
t.Fatalf("forced path request=%#v analyze=%d", runner.requests, analyzeRuns)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExtractLifecycleForcedSelfSkipInvalidatesDownstream(t *testing.T) {
|
func TestExtractLifecycleForcedSelfSkipInvalidatesDownstream(t *testing.T) {
|
||||||
cfg, env, _ := extractionLifecycleFixture(t, false)
|
cfg, env, _ := extractionLifecycleFixture(t, false)
|
||||||
markLifecycleStageSucceeded(t, cfg, "analyze")
|
markLifecycleStageSucceeded(t, cfg, "analyze")
|
||||||
@@ -485,6 +625,273 @@ func TestExtractLifecycleSkipsCurrentResumableResult(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExtractLifecycleCredentialChangesRemainResumable(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
before func(*testing.T) *configkit.Secret
|
||||||
|
after func(*testing.T) *configkit.Secret
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "added",
|
||||||
|
after: func(t *testing.T) *configkit.Secret {
|
||||||
|
return loadLifecycleNotariusCredential(t, "NARRATIO_LIFECYCLE_ADDED_KEY", "added-value")
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "removed",
|
||||||
|
before: func(t *testing.T) *configkit.Secret {
|
||||||
|
return loadLifecycleNotariusCredential(t, "NARRATIO_LIFECYCLE_REMOVED_KEY", "removed-value")
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "renamed",
|
||||||
|
before: func(t *testing.T) *configkit.Secret {
|
||||||
|
return loadLifecycleNotariusCredential(t, "NARRATIO_LIFECYCLE_FIRST_KEY", "first-value")
|
||||||
|
},
|
||||||
|
after: func(t *testing.T) *configkit.Secret {
|
||||||
|
return loadLifecycleNotariusCredential(t, "NARRATIO_LIFECYCLE_SECOND_KEY", "second-value")
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "value rotated",
|
||||||
|
before: func(t *testing.T) *configkit.Secret {
|
||||||
|
return loadLifecycleNotariusCredential(t, "NARRATIO_LIFECYCLE_ROTATED_KEY", "first-value")
|
||||||
|
},
|
||||||
|
after: func(t *testing.T) *configkit.Secret {
|
||||||
|
return loadLifecycleNotariusCredential(t, "NARRATIO_LIFECYCLE_ROTATED_KEY", "second-value")
|
||||||
|
},
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||||
|
if test.before != nil {
|
||||||
|
cfg.Pipeline.Notarius.LLMAPIKey = test.before(t)
|
||||||
|
}
|
||||||
|
analyzeRuns := 0
|
||||||
|
plan := extractionLifecyclePlan(t, &analyzeRuns)
|
||||||
|
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err != nil {
|
||||||
|
t.Fatalf("initial executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
cfg.Pipeline.Notarius.LLMAPIKey = nil
|
||||||
|
if test.after != nil {
|
||||||
|
cfg.Pipeline.Notarius.LLMAPIKey = test.after(t)
|
||||||
|
}
|
||||||
|
resumed, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resume executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(resumed.Executed) != 0 || len(resumed.Skipped) != 2 || len(runner.requests) != 1 || analyzeRuns != 1 {
|
||||||
|
t.Fatalf("credential-only change reexecuted work: summary=%#v requests=%d analyze=%d", resumed, len(runner.requests), analyzeRuns)
|
||||||
|
}
|
||||||
|
loaded := loadLifecycleManifest(t, cfg)
|
||||||
|
if loaded.Stages["extract"].Status != manifest.StatusSucceeded || loaded.Stages["analyze"].Status != manifest.StatusSucceeded {
|
||||||
|
t.Fatalf("credential-only change invalidated stages: extract=%q analyze=%q", loaded.Stages["extract"].Status, loaded.Stages["analyze"].Status)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractLifecycleConfigProfileChangesRerunExtractionAndDependents(t *testing.T) {
|
||||||
|
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||||
|
selected := "inexpensive"
|
||||||
|
cfg.Pipeline.Notarius.ConfigProfile = &selected
|
||||||
|
analyzeRuns := 0
|
||||||
|
plan := extractionLifecyclePlan(t, &analyzeRuns)
|
||||||
|
|
||||||
|
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err != nil {
|
||||||
|
t.Fatalf("initial executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(runner.requests) != 1 || runner.requests[0].ConfigProfile == nil || *runner.requests[0].ConfigProfile != selected || analyzeRuns != 1 {
|
||||||
|
t.Fatalf("initial requests=%#v analyze=%d, want selected extraction and analysis", runner.requests, analyzeRuns)
|
||||||
|
}
|
||||||
|
reused, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("reuse executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(reused.Executed) != 0 || len(runner.requests) != 1 || analyzeRuns != 1 {
|
||||||
|
t.Fatalf("same-profile reuse summary=%#v requests=%d analyze=%d", reused, len(runner.requests), analyzeRuns)
|
||||||
|
}
|
||||||
|
|
||||||
|
markLifecycleStageSucceeded(t, cfg, "render")
|
||||||
|
markLifecycleStageSucceeded(t, cfg, "publish")
|
||||||
|
before := loadLifecycleManifest(t, cfg)
|
||||||
|
trimBefore, err := json.Marshal(before.Stages["trim"])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
changed := "experimental"
|
||||||
|
cfg.Pipeline.Notarius.ConfigProfile = &changed
|
||||||
|
rerun, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("changed-profile executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(rerun.Executed) != 2 || len(runner.requests) != 2 || runner.requests[1].ConfigProfile == nil || *runner.requests[1].ConfigProfile != changed || analyzeRuns != 2 {
|
||||||
|
t.Fatalf("changed-profile summary=%#v requests=%#v analyze=%d", rerun, runner.requests, analyzeRuns)
|
||||||
|
}
|
||||||
|
after := loadLifecycleManifest(t, cfg)
|
||||||
|
trimAfter, err := json.Marshal(after.Stages["trim"])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if string(trimAfter) != string(trimBefore) {
|
||||||
|
t.Fatalf("profile change altered upstream trim record:\nbefore=%s\nafter=%s", trimBefore, trimAfter)
|
||||||
|
}
|
||||||
|
if after.Stages["render"].Status != manifest.StatusSucceeded || after.Stages["analyze"].Status != manifest.StatusSucceeded || after.Stages["publish"].Status != manifest.StatusStale {
|
||||||
|
t.Fatalf("changed-profile lifecycle render=%#v analyze=%#v publish=%#v", after.Stages["render"], after.Stages["analyze"], after.Stages["publish"])
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg.Pipeline.Notarius.ConfigProfile = nil
|
||||||
|
returned, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("return-to-base executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(returned.Executed) != 2 || len(runner.requests) != 3 || runner.requests[2].ConfigProfile != nil || analyzeRuns != 3 {
|
||||||
|
t.Fatalf("return-to-base summary=%#v requests=%#v analyze=%d", returned, runner.requests, analyzeRuns)
|
||||||
|
}
|
||||||
|
final := loadLifecycleManifest(t, cfg)
|
||||||
|
if final.Stages["trim"].Status != manifest.StatusSucceeded || final.Stages["render"].Status != manifest.StatusSucceeded {
|
||||||
|
t.Fatalf("return-to-base invalidated upstream siblings: trim=%#v render=%#v", final.Stages["trim"], final.Stages["render"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractLifecycleConfigProfileFailureClearsCurrentResultAndDependents(t *testing.T) {
|
||||||
|
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||||
|
selected := "inexpensive"
|
||||||
|
cfg.Pipeline.Notarius.ConfigProfile = &selected
|
||||||
|
analyzeRuns := 0
|
||||||
|
plan := extractionLifecyclePlan(t, &analyzeRuns)
|
||||||
|
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err != nil {
|
||||||
|
t.Fatalf("initial executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
markLifecycleStageSucceeded(t, cfg, "render")
|
||||||
|
markLifecycleStageSucceeded(t, cfg, "publish")
|
||||||
|
|
||||||
|
changed := "missing-profile"
|
||||||
|
cfg.Pipeline.Notarius.ConfigProfile = &changed
|
||||||
|
runner.failuresRemaining = 1
|
||||||
|
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err == nil {
|
||||||
|
t.Fatal("changed-profile executeStages() error = nil, want downstream failure")
|
||||||
|
}
|
||||||
|
failed := loadLifecycleManifest(t, cfg)
|
||||||
|
extract := failed.Stages["extract"]
|
||||||
|
if extract.Status != manifest.StatusFailed || len(extract.Outputs) != 0 || len(extract.Logs) != 0 || len(extract.GeneratedConfigs) != 0 || len(extract.Metadata) != 0 {
|
||||||
|
t.Fatalf("failed changed-profile extraction advertised prior success: %#v", extract)
|
||||||
|
}
|
||||||
|
if failed.Stages["trim"].Status != manifest.StatusSucceeded || failed.Stages["render"].Status != manifest.StatusSucceeded ||
|
||||||
|
failed.Stages["analyze"].Status != manifest.StatusStale || failed.Stages["publish"].Status != manifest.StatusStale || analyzeRuns != 1 {
|
||||||
|
t.Fatalf("failed changed-profile lifecycle trim=%#v render=%#v analyze=%#v publish=%#v analyzeRuns=%d",
|
||||||
|
failed.Stages["trim"], failed.Stages["render"], failed.Stages["analyze"], failed.Stages["publish"], analyzeRuns)
|
||||||
|
}
|
||||||
|
if len(runner.requests) != 2 || runner.requests[1].ConfigProfile == nil || *runner.requests[1].ConfigProfile != changed {
|
||||||
|
t.Fatalf("failed request = %#v, want changed selector", runner.requests)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractLifecycleRealConfigProfilesSelectNotariusExplicitly(t *testing.T) {
|
||||||
|
seed, env, runner := extractionLifecycleFixture(t, true)
|
||||||
|
workspace := seed.Pipeline.Workspace.Root
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspace)
|
||||||
|
pipelineFile, err := os.OpenFile(pipelinePath, os.O_APPEND|os.O_WRONLY, 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_, writeErr := pipelineFile.WriteString(`notarius:
|
||||||
|
enabled: true
|
||||||
|
binary: ` + filepath.Join(workspace, "notarius") + `
|
||||||
|
config_path: notarius.yml
|
||||||
|
pipeline_id: dnd-session
|
||||||
|
timeout: 45m
|
||||||
|
outputs:
|
||||||
|
npc_registry:
|
||||||
|
lane_id: npc-registry
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.npc_registry
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/npc-registry
|
||||||
|
`)
|
||||||
|
closeErr := pipelineFile.Close()
|
||||||
|
if writeErr != nil || closeErr != nil {
|
||||||
|
t.Fatalf("write Notarius configuration: %v / %v", writeErr, closeErr)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(filepath.Dir(pipelinePath), "notarius.yml"), []byte("pipelines: {}\n"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
profilesDir := filepath.Join(filepath.Dir(pipelinePath), "profiles")
|
||||||
|
if err := os.MkdirAll(profilesDir, 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for name, contents := range map[string]string{
|
||||||
|
"testing": "whisperx:\n language: fr\nnotarius:\n config_profile: downstream-economy\n",
|
||||||
|
"narratio-only": "whisperx:\n language: de\n",
|
||||||
|
"narratio-only-two": "whisperx:\n language: es\n",
|
||||||
|
} {
|
||||||
|
if err := os.WriteFile(filepath.Join(profilesDir, name+".yml"), []byte(contents), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
load := func(profile string) *config.Config {
|
||||||
|
t.Helper()
|
||||||
|
loaded, err := config.LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{Profile: &profile})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load profile %q: %v", profile, err)
|
||||||
|
}
|
||||||
|
return loaded
|
||||||
|
}
|
||||||
|
plan, err := BuildSingleStagePlan("extract")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
testingConfig := load("testing")
|
||||||
|
runner.cfg = testingConfig.Pipeline.Notarius
|
||||||
|
if _, err := executeStages(context.Background(), testingConfig, plan, RunOptions{Env: env}); err != nil {
|
||||||
|
t.Fatalf("execute testing profile: %v", err)
|
||||||
|
}
|
||||||
|
if len(runner.requests) != 1 || runner.requests[0].ConfigProfile == nil || *runner.requests[0].ConfigProfile != "downstream-economy" {
|
||||||
|
t.Fatalf("testing profile request = %#v, want explicit differently named downstream selector", runner.requests)
|
||||||
|
}
|
||||||
|
|
||||||
|
narratioOnly := load("narratio-only")
|
||||||
|
runner.cfg = narratioOnly.Pipeline.Notarius
|
||||||
|
if _, err := executeStages(context.Background(), narratioOnly, plan, RunOptions{Env: env}); err != nil {
|
||||||
|
t.Fatalf("execute Narratio-only profile: %v", err)
|
||||||
|
}
|
||||||
|
if len(runner.requests) != 2 || runner.requests[1].ConfigProfile != nil {
|
||||||
|
t.Fatalf("Narratio-only profile request = %#v, want no automatic downstream selector", runner.requests)
|
||||||
|
}
|
||||||
|
|
||||||
|
narratioOnlyTwo := load("narratio-only-two")
|
||||||
|
runner.cfg = narratioOnlyTwo.Pipeline.Notarius
|
||||||
|
reused, err := executeStages(context.Background(), narratioOnlyTwo, plan, RunOptions{Env: env})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("execute second Narratio-only profile: %v", err)
|
||||||
|
}
|
||||||
|
if len(reused.Executed) != 0 || len(runner.requests) != 2 {
|
||||||
|
t.Fatalf("Narratio profile label/effective digest broadened invalidation: summary=%#v requests=%#v", reused, runner.requests)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPreparedPipelineAndManifestExcludeNotariusCredentialValue(t *testing.T) {
|
||||||
|
const name = "NARRATIO_LIFECYCLE_PERSISTED_KEY"
|
||||||
|
const canary = "notarius-persisted-credential-canary"
|
||||||
|
cfg, env, _ := extractionLifecycleFixture(t, true)
|
||||||
|
cfg.Pipeline.Notarius.LLMAPIKey = loadLifecycleNotariusCredential(t, name, canary)
|
||||||
|
if _, err := executeStages(context.Background(), cfg, prepareExtractLifecyclePlan(t), RunOptions{Env: env}); err != nil {
|
||||||
|
t.Fatalf("executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
for _, path := range []string{paths.ManifestPath, filepath.Join(paths.InputsDir, "pipeline.resolved.yml")} {
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadFile(%q) error = %v", path, err)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(data), canary) {
|
||||||
|
t.Fatalf("persisted Narratio file %q contains the credential value", path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExtractLifecycleRerunsAfterDirectTranscriptChange(t *testing.T) {
|
func TestExtractLifecycleRerunsAfterDirectTranscriptChange(t *testing.T) {
|
||||||
cfg, env, runner := extractionLifecycleFixture(t, true)
|
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||||
analyzeRuns := 0
|
analyzeRuns := 0
|
||||||
@@ -696,6 +1103,14 @@ func lifecycleInputChecksum(t *testing.T, m *manifest.Manifest, kind string) str
|
|||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func lifecycleReferencePaths(request notarius.RunRequest) map[string]string {
|
||||||
|
paths := make(map[string]string, len(request.References))
|
||||||
|
for _, reference := range request.References {
|
||||||
|
paths[reference.Selector] = reference.Path
|
||||||
|
}
|
||||||
|
return paths
|
||||||
|
}
|
||||||
|
|
||||||
func lifecycleDndOutputs() map[string]config.NotariusOutputConfig {
|
func lifecycleDndOutputs() map[string]config.NotariusOutputConfig {
|
||||||
return map[string]config.NotariusOutputConfig{
|
return map[string]config.NotariusOutputConfig{
|
||||||
"item_registry": {LaneID: "item-registry", MediaType: "application/json", SchemaID: "notarius.dnd.item_registry", SchemaVersion: "v1", ModuleKey: "dnd/item-registry"},
|
"item_registry": {LaneID: "item-registry", MediaType: "application/json", SchemaID: "notarius.dnd.item_registry", SchemaVersion: "v1", ModuleKey: "dnd/item-registry"},
|
||||||
@@ -761,3 +1176,17 @@ func extractionLifecycleFixture(t *testing.T, enabled bool) (*config.Config, *st
|
|||||||
runner := &materializingNotariusRunner{cfg: cfg.Pipeline.Notarius}
|
runner := &materializingNotariusRunner{cfg: cfg.Pipeline.Notarius}
|
||||||
return cfg, &stage.Env{Notarius: runner}, runner
|
return cfg, &stage.Env{Notarius: runner}, runner
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func loadLifecycleNotariusCredential(t *testing.T, name, value string) *configkit.Secret {
|
||||||
|
t.Helper()
|
||||||
|
t.Setenv(name, value)
|
||||||
|
path := filepath.Join(t.TempDir(), "pipeline.yml")
|
||||||
|
if err := os.WriteFile(path, []byte("whisperx:\n transcribe_url: https://transcription.example.test/transcribe\nnotarius:\n enabled: false\n llm_api_key: "+name+"\n"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
pipeline, err := config.LoadPipeline(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return pipeline.Notarius.LLMAPIKey
|
||||||
|
}
|
||||||
|
|||||||
@@ -11,9 +11,6 @@ import (
|
|||||||
|
|
||||||
func newCommandObjectStoreWithDependencies(ctx context.Context, cfg *config.Config, logger *slog.Logger, deps commandDependencies) (storage.ObjectStore, error) {
|
func newCommandObjectStoreWithDependencies(ctx context.Context, cfg *config.Config, logger *slog.Logger, deps commandDependencies) (storage.ObjectStore, error) {
|
||||||
deps = normalizeCommandDependencies(deps)
|
deps = normalizeCommandDependencies(deps)
|
||||||
if _, err := loadSecretsFromConfigWithDependencies(cfg, logger, deps); err != nil {
|
|
||||||
return nil, fmt.Errorf("load secrets from files: %w", err)
|
|
||||||
}
|
|
||||||
return initializeObjectStoreBackend(ctx, cfg, deps.newObjectStore)
|
return initializeObjectStoreBackend(ctx, cfg, deps.newObjectStore)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -3,8 +3,6 @@ package app
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
@@ -12,89 +10,8 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestNewCommandObjectStoreLoadsSecretsBeforeFactory(t *testing.T) {
|
|
||||||
accessKeyEnv := "NARRATIO_TEST_COMMAND_STORE_KEY_ID"
|
|
||||||
secretKeyEnv := "NARRATIO_TEST_COMMAND_STORE_SECRET"
|
|
||||||
restoreEnvAfterTest(t, accessKeyEnv, secretKeyEnv)
|
|
||||||
|
|
||||||
secretsDir := t.TempDir()
|
|
||||||
mustWriteSecretFile(t, filepath.Join(secretsDir, accessKeyEnv), "loaded-key-id\n")
|
|
||||||
mustWriteSecretFile(t, filepath.Join(secretsDir, secretKeyEnv), "loaded-secret\n")
|
|
||||||
|
|
||||||
cfg := commandObjectStoreTestConfig(secretsDir)
|
|
||||||
fake := &storage.FakeBackend{}
|
|
||||||
called := false
|
|
||||||
deps := commandDependencies{newObjectStore: func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
|
||||||
called = true
|
|
||||||
if got := os.Getenv(accessKeyEnv); got != "loaded-key-id" {
|
|
||||||
return nil, errors.New("access key was not loaded before object store init")
|
|
||||||
}
|
|
||||||
if got := os.Getenv(secretKeyEnv); got != "loaded-secret" {
|
|
||||||
return nil, errors.New("secret key was not loaded before object store init")
|
|
||||||
}
|
|
||||||
return fake, nil
|
|
||||||
}}
|
|
||||||
|
|
||||||
store, err := newCommandObjectStoreWithDependencies(context.Background(), cfg, nil, deps)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("newCommandObjectStore() error = %v", err)
|
|
||||||
}
|
|
||||||
if store != fake {
|
|
||||||
t.Fatalf("store = %#v, want fake backend", store)
|
|
||||||
}
|
|
||||||
if !called {
|
|
||||||
t.Fatal("object store factory was not called")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewCommandObjectStorePreservesExistingEnv(t *testing.T) {
|
|
||||||
accessKeyEnv := "NARRATIO_TEST_COMMAND_STORE_EXISTING_KEY_ID"
|
|
||||||
secretKeyEnv := "NARRATIO_TEST_COMMAND_STORE_EXISTING_SECRET"
|
|
||||||
t.Setenv(accessKeyEnv, "existing-key-id")
|
|
||||||
t.Setenv(secretKeyEnv, "existing-secret")
|
|
||||||
|
|
||||||
secretsDir := t.TempDir()
|
|
||||||
mustWriteSecretFile(t, filepath.Join(secretsDir, accessKeyEnv), "file-key-id\n")
|
|
||||||
mustWriteSecretFile(t, filepath.Join(secretsDir, secretKeyEnv), "file-secret\n")
|
|
||||||
|
|
||||||
cfg := commandObjectStoreTestConfig(secretsDir)
|
|
||||||
deps := commandDependencies{newObjectStore: func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
|
||||||
if got := os.Getenv(accessKeyEnv); got != "existing-key-id" {
|
|
||||||
return nil, errors.New("existing access key was overwritten")
|
|
||||||
}
|
|
||||||
if got := os.Getenv(secretKeyEnv); got != "existing-secret" {
|
|
||||||
return nil, errors.New("existing secret key was overwritten")
|
|
||||||
}
|
|
||||||
return &storage.FakeBackend{}, nil
|
|
||||||
}}
|
|
||||||
|
|
||||||
if _, err := newCommandObjectStoreWithDependencies(context.Background(), cfg, nil, deps); err != nil {
|
|
||||||
t.Fatalf("newCommandObjectStore() error = %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewCommandObjectStoreSecretErrorStopsFactory(t *testing.T) {
|
|
||||||
cfg := commandObjectStoreTestConfig(filepath.Join(t.TempDir(), "missing"))
|
|
||||||
called := false
|
|
||||||
deps := commandDependencies{newObjectStore: func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
|
||||||
called = true
|
|
||||||
return &storage.FakeBackend{}, nil
|
|
||||||
}}
|
|
||||||
|
|
||||||
_, err := newCommandObjectStoreWithDependencies(context.Background(), cfg, nil, deps)
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected error, got nil")
|
|
||||||
}
|
|
||||||
if called {
|
|
||||||
t.Fatal("object store factory was called after secret load failure")
|
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), "load secrets from files") {
|
|
||||||
t.Fatalf("error = %q, want secret loading context", err.Error())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewCommandObjectStoreFactoryErrorIsContextual(t *testing.T) {
|
func TestNewCommandObjectStoreFactoryErrorIsContextual(t *testing.T) {
|
||||||
cfg := commandObjectStoreTestConfig("")
|
cfg := commandObjectStoreTestConfig()
|
||||||
deps := commandDependencies{newObjectStore: func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
deps := commandDependencies{newObjectStore: func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||||
return nil, errors.New("factory boom")
|
return nil, errors.New("factory boom")
|
||||||
}}
|
}}
|
||||||
@@ -108,21 +25,17 @@ func TestNewCommandObjectStoreFactoryErrorIsContextual(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func commandObjectStoreTestConfig(secretsDir string) *config.Config {
|
func commandObjectStoreTestConfig() *config.Config {
|
||||||
cfg := &config.Config{
|
return &config.Config{
|
||||||
Pipeline: &config.PipelineConfig{
|
Pipeline: &config.PipelineConfig{
|
||||||
Storage: config.StorageConfig{
|
Storage: config.StorageConfig{
|
||||||
Backend: "s3",
|
Backend: "s3",
|
||||||
S3: &config.StorageS3Config{
|
S3: &config.StorageS3Config{
|
||||||
Bucket: "test-bucket",
|
Bucket: "test-bucket",
|
||||||
AccessKeyIDEnv: "NARRATIO_TEST_COMMAND_STORE_KEY_ID",
|
AccessKeyID: testSecretReference("NARRATIO_TEST_COMMAND_STORE_KEY_ID"),
|
||||||
SecretKeyEnv: "NARRATIO_TEST_COMMAND_STORE_SECRET",
|
SecretAccessKey: testSecretReference("NARRATIO_TEST_COMMAND_STORE_SECRET"),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(secretsDir) != "" {
|
|
||||||
cfg.Pipeline.Secrets = &config.SecretsConfig{EnvDir: secretsDir}
|
|
||||||
}
|
|
||||||
return cfg
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ type commonConfigFlags struct {
|
|||||||
sessionID string
|
sessionID string
|
||||||
previousSessionID string
|
previousSessionID string
|
||||||
profile singletonStringFlag
|
profile singletonStringFlag
|
||||||
|
secretsDir singletonStringFlag
|
||||||
}
|
}
|
||||||
|
|
||||||
func addCommonConfigFlags(fs *flag.FlagSet, flags *commonConfigFlags) {
|
func addCommonConfigFlags(fs *flag.FlagSet, flags *commonConfigFlags) {
|
||||||
@@ -35,6 +36,8 @@ func addCommonConfigFlags(fs *flag.FlagSet, flags *commonConfigFlags) {
|
|||||||
fs.StringVar(&flags.previousSessionID, "previous-session-id", "", "expected previous session identifier")
|
fs.StringVar(&flags.previousSessionID, "previous-session-id", "", "expected previous session identifier")
|
||||||
flags.profile.name = "profile"
|
flags.profile.name = "profile"
|
||||||
fs.Var(&flags.profile, "profile", "named pipeline profile")
|
fs.Var(&flags.profile, "profile", "named pipeline profile")
|
||||||
|
flags.secretsDir.name = "secrets-dir"
|
||||||
|
fs.Var(&flags.secretsDir, "secrets-dir", "secrets directory (optional; defaults beside pipeline.yml)")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f commonConfigFlags) sessionOptions() config.SessionLoadOptions {
|
func (f commonConfigFlags) sessionOptions() config.SessionLoadOptions {
|
||||||
@@ -112,7 +115,7 @@ func artifactsWithDependencies(ctx context.Context, args []string, out io.Writer
|
|||||||
|
|
||||||
func loadHelperContextWithDependencies(ctx context.Context, flags commonConfigFlags, needStore bool, deps commandDependencies) (*config.Config, storage.ObjectStore, []artifacts.RemoteLock, *manifest.Manifest, func(), error) {
|
func loadHelperContextWithDependencies(ctx context.Context, flags commonConfigFlags, needStore bool, deps commandDependencies) (*config.Config, storage.ObjectStore, []artifacts.RemoteLock, *manifest.Manifest, func(), error) {
|
||||||
deps = normalizeCommandDependencies(deps)
|
deps = normalizeCommandDependencies(deps)
|
||||||
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), deps)
|
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), flags.secretsDir.pointer(), deps)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, nil, nil, nil, err
|
return nil, nil, nil, nil, nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -164,19 +164,19 @@ func TestExecuteSessionInitRequiresSessionID(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestExecuteSessionInitMissingDefaultConfigReportsSearchedPaths(t *testing.T) {
|
func TestExecuteSessionInitMissingDefaultConfigReportsSearchedPaths(t *testing.T) {
|
||||||
origPipelineDefaults := append([]string(nil), config.DefaultPipelineConfigSearchPaths...)
|
deps, missingPath, executor := guardedMissingPipelineDiscovery(t)
|
||||||
config.DefaultPipelineConfigSearchPaths = []string{filepath.Join(t.TempDir(), "missing-pipeline.yml")}
|
|
||||||
t.Cleanup(func() {
|
|
||||||
config.DefaultPipelineConfigSearchPaths = origPipelineDefaults
|
|
||||||
})
|
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute([]string{"session", "init", "2026-06-07", "--remote"}, &stdout, &stderr)
|
code := executeWithDependencies([]string{"session", "init", "2026-06-07", "--remote"}, &stdout, &stderr, deps, executor)
|
||||||
if code == 0 {
|
if code == 0 {
|
||||||
t.Fatal("exit code = 0, want non-zero")
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
}
|
}
|
||||||
if !strings.Contains(stderr.String(), "session init: no pipeline config path provided and no default pipeline config found; searched:") {
|
if stdout.Len() != 0 {
|
||||||
|
t.Fatalf("stdout = %q, want empty", stdout.String())
|
||||||
|
}
|
||||||
|
if got := stderr.String(); !strings.Contains(got, "session init: no pipeline config path provided") ||
|
||||||
|
!strings.Contains(got, "no default pipeline config found") || !strings.Contains(got, missingPath) {
|
||||||
t.Fatalf("stderr = %q, want default pipeline searched-path error", stderr.String())
|
t.Fatalf("stderr = %q, want default pipeline searched-path error", stderr.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -131,7 +131,7 @@ func locksRemoveWithDependencies(ctx context.Context, args []string, out io.Writ
|
|||||||
|
|
||||||
func loadLockCommandContextWithDependencies(ctx context.Context, flags commonConfigFlags, deps commandDependencies) (*config.Config, storage.ObjectStore, string, func(), error) {
|
func loadLockCommandContextWithDependencies(ctx context.Context, flags commonConfigFlags, deps commandDependencies) (*config.Config, storage.ObjectStore, string, func(), error) {
|
||||||
deps = normalizeCommandDependencies(deps)
|
deps = normalizeCommandDependencies(deps)
|
||||||
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), deps)
|
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), flags.secretsDir.pointer(), deps)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, "", nil, err
|
return nil, nil, "", nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ func sessionInitWithDependencies(ctx context.Context, args []string, out io.Writ
|
|||||||
fs.SetOutput(io.Discard)
|
fs.SetOutput(io.Discard)
|
||||||
var pipelinePath, campaignPath, campaignFilePath, sessionID, previousSessionID, date, title, output, audioS3Prefix, audioDir string
|
var pipelinePath, campaignPath, campaignFilePath, sessionID, previousSessionID, date, title, output, audioS3Prefix, audioDir string
|
||||||
var profile singletonStringFlag
|
var profile singletonStringFlag
|
||||||
|
var secretsDir singletonStringFlag
|
||||||
var remote, force bool
|
var remote, force bool
|
||||||
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
||||||
fs.StringVar(&campaignPath, "campaign", "", "campaign ID")
|
fs.StringVar(&campaignPath, "campaign", "", "campaign ID")
|
||||||
@@ -41,6 +42,8 @@ func sessionInitWithDependencies(ctx context.Context, args []string, out io.Writ
|
|||||||
fs.StringVar(&audioDir, "audio-dir", "", "local audio directory")
|
fs.StringVar(&audioDir, "audio-dir", "", "local audio directory")
|
||||||
profile.name = "profile"
|
profile.name = "profile"
|
||||||
fs.Var(&profile, "profile", "named pipeline profile")
|
fs.Var(&profile, "profile", "named pipeline profile")
|
||||||
|
secretsDir.name = "secrets-dir"
|
||||||
|
fs.Var(&secretsDir, "secrets-dir", "secrets directory (optional; defaults beside pipeline.yml)")
|
||||||
fs.BoolVar(&remote, "remote", false, "write session.yml to S3 session prefix")
|
fs.BoolVar(&remote, "remote", false, "write session.yml to S3 session prefix")
|
||||||
fs.BoolVar(&force, "force", false, "overwrite existing target")
|
fs.BoolVar(&force, "force", false, "overwrite existing target")
|
||||||
if err := parseSessionAwareFlags("session init", fs, args, &sessionID); err != nil {
|
if err := parseSessionAwareFlags("session init", fs, args, &sessionID); err != nil {
|
||||||
@@ -56,7 +59,7 @@ func sessionInitWithDependencies(ctx context.Context, args []string, out io.Writ
|
|||||||
return fmt.Errorf("session init: --audio-dir and --audio-s3-prefix are mutually exclusive")
|
return fmt.Errorf("session init: --audio-dir and --audio-s3-prefix are mutually exclusive")
|
||||||
}
|
}
|
||||||
|
|
||||||
base, err := loadPipelineCampaignConfigWithDependencies(pipelinePath, campaignPath, campaignFilePath, config.PipelineLoadOptions{Profile: profile.pointer()}, deps)
|
base, err := loadPipelineCampaignConfigWithDependencies(pipelinePath, campaignPath, campaignFilePath, config.PipelineLoadOptions{Profile: profile.pointer()}, secretsDir.pointer(), deps)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("session init: %w", err)
|
return fmt.Errorf("session init: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ func sessionValidateWithDependencies(ctx context.Context, args []string, out io.
|
|||||||
}
|
}
|
||||||
|
|
||||||
findings := []finding{}
|
findings := []finding{}
|
||||||
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), deps)
|
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), flags.secretsDir.pointer(), deps)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
findings = append(findings, errorFinding("config", err.Error()))
|
findings = append(findings, errorFinding("config", err.Error()))
|
||||||
return renderFindings(out, "", "", findings)
|
return renderFindings(out, "", "", findings)
|
||||||
|
|||||||
@@ -31,7 +31,7 @@ func statusWithDependencies(ctx context.Context, args []string, out io.Writer, d
|
|||||||
if strings.TrimSpace(flags.sessionID) == "" {
|
if strings.TrimSpace(flags.sessionID) == "" {
|
||||||
return fmt.Errorf("status: session_id is required")
|
return fmt.Errorf("status: session_id is required")
|
||||||
}
|
}
|
||||||
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), deps)
|
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), flags.secretsDir.pointer(), deps)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("status: %w", err)
|
return fmt.Errorf("status: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,10 +1,16 @@
|
|||||||
package app
|
package app
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestResolvePipelineConfigPathWithCandidatesExplicitWins(t *testing.T) {
|
func TestResolvePipelineConfigPathWithCandidatesExplicitWins(t *testing.T) {
|
||||||
@@ -55,11 +61,57 @@ func TestResolvePipelineConfigPathWithCandidatesPrecedence(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestResolvePipelineConfigPathWithCandidatesMissing(t *testing.T) {
|
func TestResolvePipelineConfigPathWithCandidatesMissing(t *testing.T) {
|
||||||
_, err := resolvePipelineConfigPathWithCandidates("", []string{"/does/not/exist/one.yml", "/does/not/exist/two.yml"})
|
directory := t.TempDir()
|
||||||
|
first := filepath.Join(directory, "one.yml")
|
||||||
|
second := filepath.Join(directory, "two.yml")
|
||||||
|
_, err := resolvePipelineConfigPathWithCandidates("", []string{first, second})
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected error, got nil")
|
t.Fatal("expected error, got nil")
|
||||||
}
|
}
|
||||||
if !strings.Contains(err.Error(), "no default pipeline config found") {
|
if !strings.Contains(err.Error(), "no default pipeline config found") {
|
||||||
t.Fatalf("error = %q, want missing-defaults context", err.Error())
|
t.Fatalf("error = %q, want missing-defaults context", err.Error())
|
||||||
}
|
}
|
||||||
|
if !strings.Contains(err.Error(), first) || !strings.Contains(err.Error(), second) {
|
||||||
|
t.Fatalf("error = %q, want controlled searched paths", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteMissingExplicitConfigDiscoversExistingDefaultBeforeLoading(t *testing.T) {
|
||||||
|
directory := t.TempDir()
|
||||||
|
missing := filepath.Join(directory, "missing.yml")
|
||||||
|
existing := filepath.Join(directory, "pipeline.yml")
|
||||||
|
if err := os.WriteFile(existing, []byte("workspace:\n root: ./tmp\n"), 0o644); err != nil {
|
||||||
|
t.Fatalf("write default pipeline: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
controlledErr := errors.New("controlled pipeline loader boundary")
|
||||||
|
loadedPath := ""
|
||||||
|
deps := commandDependenciesForPipelineCandidates(missing, existing)
|
||||||
|
deps.loadPipelineConfig = func(path string, _ config.PipelineLoadOptions) (*config.PipelineConfig, error) {
|
||||||
|
loadedPath = path
|
||||||
|
return nil, controlledErr
|
||||||
|
}
|
||||||
|
deps.loadSecretsDir = func(string) (*configkit.SecretsReport, error) {
|
||||||
|
t.Fatal("absent conventional secrets directory unexpectedly invoked bootstrap loader")
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := executeWithDependencies([]string{"config", "validate"}, &stdout, &stderr, deps, func(context.Context, *config.Config, BoundedPlan, RunOptions) (*RunSummary, error) {
|
||||||
|
t.Fatal("config validation unexpectedly invoked pipeline executor")
|
||||||
|
return nil, nil
|
||||||
|
})
|
||||||
|
if code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want controlled loader failure")
|
||||||
|
}
|
||||||
|
if loadedPath != existing {
|
||||||
|
t.Fatalf("loader path = %q, want discovered default %q", loadedPath, existing)
|
||||||
|
}
|
||||||
|
if stdout.Len() != 0 {
|
||||||
|
t.Fatalf("stdout = %q, want empty", stdout.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), controlledErr.Error()) {
|
||||||
|
t.Fatalf("stderr = %q, want controlled loader error", stderr.String())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -31,7 +31,7 @@ func planWithDependencies(ctx context.Context, args []string, out io.Writer, dep
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
flags := request.Config
|
flags := request.Config
|
||||||
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), deps)
|
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), flags.secretsDir.pointer(), deps)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("plan: %w", err)
|
return fmt.Errorf("plan: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -88,7 +88,7 @@ func TestPlanShowsRunAndSkipFromManifest(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPlanDoesNotLoadConfiguredSecrets(t *testing.T) {
|
func TestPlanSkipsAbsentConventionalSecretsDirectory(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
configDir := t.TempDir()
|
configDir := t.TempDir()
|
||||||
pipelinePath := filepath.Join(configDir, "pipeline.yml")
|
pipelinePath := filepath.Join(configDir, "pipeline.yml")
|
||||||
@@ -99,8 +99,6 @@ func TestPlanDoesNotLoadConfiguredSecrets(t *testing.T) {
|
|||||||
root: ` + workspaceRoot + `
|
root: ` + workspaceRoot + `
|
||||||
storage:
|
storage:
|
||||||
backend: local
|
backend: local
|
||||||
secrets:
|
|
||||||
env_dir: ./missing-secrets
|
|
||||||
whisperx:
|
whisperx:
|
||||||
transcribe_url: https://example.com/transcribe
|
transcribe_url: https://example.com/transcribe
|
||||||
seriatim:
|
seriatim:
|
||||||
@@ -128,7 +126,7 @@ inputs:
|
|||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
err := Plan(context.Background(), []string{"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &out)
|
err := Plan(context.Background(), []string{"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &out)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Plan() error = %v, want missing runtime secrets ignored", err)
|
t.Fatalf("Plan() error = %v, want absent conventional secrets directory skipped", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -37,10 +37,10 @@ func TestRunProfileSelectionFlowsThroughSharedLoader(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if language != "en" {
|
if language != "en" {
|
||||||
t.Fatalf("default profile language = %q, want en", language)
|
t.Fatalf("base configuration language = %q, want en", language)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, profile := range []string{"unknown", ""} {
|
for _, profile := range []string{"unknown"} {
|
||||||
err := runWithPipelineExecutor(context.Background(), []string{
|
err := runWithPipelineExecutor(context.Background(), []string{
|
||||||
"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--profile", profile,
|
"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--profile", profile,
|
||||||
}, io.Discard, executor)
|
}, io.Discard, executor)
|
||||||
@@ -48,19 +48,20 @@ func TestRunProfileSelectionFlowsThroughSharedLoader(t *testing.T) {
|
|||||||
t.Fatalf("profile %q error = %v, want selection rejection", profile, err)
|
t.Fatalf("profile %q error = %v, want selection rejection", profile, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if err := runWithPipelineExecutor(context.Background(), []string{
|
||||||
|
"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--profile", "",
|
||||||
|
}, io.Discard, executor); err != nil {
|
||||||
|
t.Fatalf("explicit empty profile should select base: %v", err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func enableCommandTestProfiles(t *testing.T, pipelinePath string) {
|
func enableCommandTestProfiles(t *testing.T, pipelinePath string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
data, err := os.ReadFile(pipelinePath)
|
dir := filepath.Join(filepath.Dir(pipelinePath), "profiles")
|
||||||
if err != nil {
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
composition := "composition:\n default_profile: production\n profiles:\n production:\n overlay: production.yml\n testing:\n overlay: testing.yml\n"
|
|
||||||
if err := os.WriteFile(pipelinePath, append([]byte(composition), data...), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
dir := filepath.Dir(pipelinePath)
|
|
||||||
if err := os.WriteFile(filepath.Join(dir, "production.yml"), []byte("whisperx:\n language: en\n"), 0o644); err != nil {
|
if err := os.WriteFile(filepath.Join(dir, "production.yml"), []byte("whisperx:\n language: en\n"), 0o644); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
@@ -78,7 +79,7 @@ inputs:
|
|||||||
setSessionConfigSearchPaths(t, []string{filepath.Join(t.TempDir(), "session.yml")})
|
setSessionConfigSearchPaths(t, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||||
deps, storeInitCalls := remoteSessionLoaderDependencies(fake)
|
deps, storeInitCalls := remoteSessionLoaderDependencies(fake)
|
||||||
|
|
||||||
loaded, err := loadCommandConfigWithDependencies(context.Background(), pipelinePath, "", campaignPath, "", config.SessionLoadOptions{SessionID: "2026-05-03"}, deps)
|
loaded, err := loadCommandConfigWithDependencies(context.Background(), pipelinePath, "", campaignPath, "", config.SessionLoadOptions{SessionID: "2026-05-03"}, nil, deps)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("loadCommandConfig() error = %v", err)
|
t.Fatalf("loadCommandConfig() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -200,7 +201,7 @@ inputs:
|
|||||||
|
|
||||||
var downloadedPath string
|
var downloadedPath string
|
||||||
deps.downloadObjectToTemp = captureRemoteSessionTempPath(&downloadedPath)
|
deps.downloadObjectToTemp = captureRemoteSessionTempPath(&downloadedPath)
|
||||||
loaded, err := loadCommandConfigWithDependencies(context.Background(), pipelinePath, "", campaignPath, "", config.SessionLoadOptions{SessionID: "2026-05-03"}, deps)
|
loaded, err := loadCommandConfigWithDependencies(context.Background(), pipelinePath, "", campaignPath, "", config.SessionLoadOptions{SessionID: "2026-05-03"}, nil, deps)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("loadCommandConfig() error = %v", err)
|
t.Fatalf("loadCommandConfig() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -246,12 +247,18 @@ inputs:
|
|||||||
prefix: audio/
|
prefix: audio/
|
||||||
`)
|
`)
|
||||||
setSessionConfigSearchPaths(t, []string{filepath.Join(t.TempDir(), "session.yml")})
|
setSessionConfigSearchPaths(t, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||||
deps := commandDependencies{newObjectStore: func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
bootstrapCalls := 0
|
||||||
if os.Getenv(accessKeyEnv) != "remote-session-key-id" || os.Getenv(secretKeyEnv) != "remote-session-secret" {
|
deps := commandDependencies{
|
||||||
return nil, fmt.Errorf("secrets were not loaded before remote session object store init")
|
loadSecretsDir: func(directory string) (*configkit.SecretsReport, error) {
|
||||||
}
|
bootstrapCalls++
|
||||||
return fake, nil
|
return configkit.LoadSecretsDir(directory)
|
||||||
}}
|
},
|
||||||
|
newObjectStore: func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||||
|
if os.Getenv(accessKeyEnv) != "remote-session-key-id" || os.Getenv(secretKeyEnv) != "remote-session-secret" {
|
||||||
|
return nil, fmt.Errorf("secrets were not loaded before remote session object store init")
|
||||||
|
}
|
||||||
|
return fake, nil
|
||||||
|
}}
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
@@ -259,6 +266,9 @@ inputs:
|
|||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("exit code = %d, want 0; stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
t.Fatalf("exit code = %d, want 0; stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||||
}
|
}
|
||||||
|
if bootstrapCalls != 1 {
|
||||||
|
t.Fatalf("bootstrap calls = %d, want 1", bootstrapCalls)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExecuteExplicitLocalSessionPrecedenceSkipsRemote(t *testing.T) {
|
func TestExecuteExplicitLocalSessionPrecedenceSkipsRemote(t *testing.T) {
|
||||||
@@ -468,10 +478,13 @@ func addSecretsToPipelineConfig(t *testing.T, pipelinePath, secretsDir, accessKe
|
|||||||
pipelineYAML := strings.Replace(
|
pipelineYAML := strings.Replace(
|
||||||
string(pipelineData),
|
string(pipelineData),
|
||||||
"storage:\n backend: s3\n s3:\n bucket: test-bucket\n",
|
"storage:\n backend: s3\n s3:\n bucket: test-bucket\n",
|
||||||
"storage:\n backend: s3\n s3:\n bucket: test-bucket\n access_key_id_env: "+accessKeyEnv+"\n secret_access_key_env: "+secretKeyEnv+"\nsecrets:\n env_dir: "+secretsDir+"\n",
|
"storage:\n backend: s3\n s3:\n bucket: test-bucket\n access_key_id: "+accessKeyEnv+"\n secret_access_key: "+secretKeyEnv+"\n",
|
||||||
1,
|
1,
|
||||||
)
|
)
|
||||||
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||||
t.Fatalf("write pipeline: %v", err)
|
t.Fatalf("write pipeline: %v", err)
|
||||||
}
|
}
|
||||||
|
if err := os.Symlink(secretsDir, filepath.Join(filepath.Dir(pipelinePath), "secrets")); err != nil {
|
||||||
|
t.Fatalf("link conventional secrets directory: %v", err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -53,7 +53,7 @@ func restoreWithDependencies(ctx context.Context, args []string, out io.Writer,
|
|||||||
if strings.TrimSpace(flags.sessionID) == "" {
|
if strings.TrimSpace(flags.sessionID) == "" {
|
||||||
return fmt.Errorf("restore: session_id is required")
|
return fmt.Errorf("restore: session_id is required")
|
||||||
}
|
}
|
||||||
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), deps)
|
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), flags.secretsDir.pointer(), deps)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("restore: %w", err)
|
return fmt.Errorf("restore: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -82,6 +82,28 @@ func TestExecuteRestoreRoundTripsPublishedExtractionAndManifestMetadata(t *testi
|
|||||||
remoteManifest.RunID = "20260519T010203Z-a1b2c3d4"
|
remoteManifest.RunID = "20260519T010203Z-a1b2c3d4"
|
||||||
producerRoot := "/prior/workspace/work/sample-campaign/2026-05-03"
|
producerRoot := "/prior/workspace/work/sample-campaign/2026-05-03"
|
||||||
remoteManifest.LocalWorkDir = filepath.Join(producerRoot, "runs", remoteManifest.RunID)
|
remoteManifest.LocalWorkDir = filepath.Join(producerRoot, "runs", remoteManifest.RunID)
|
||||||
|
rawRelative := "transcripts/raw/alice.json"
|
||||||
|
rawOutput := manifest.ArtifactRecord{
|
||||||
|
Kind: "transcript_raw", LocalPath: filepath.Join(producerRoot, filepath.FromSlash(rawRelative)),
|
||||||
|
ProducerRunID: remoteManifest.RunID, Checksum: strings.Repeat("b", 64),
|
||||||
|
}
|
||||||
|
remoteManifest.MarkStageSucceeded("transcribe", time.Now().UTC(), []manifest.ArtifactRecord{rawOutput})
|
||||||
|
remoteManifest.Stages["transcribe"].TranscribeStateVersion = manifest.TranscribeStateContractVersion
|
||||||
|
remoteManifest.Stages["transcribe"].TranscribeSpeakers = map[string]manifest.TranscribeSpeakerRecord{
|
||||||
|
"alice": {
|
||||||
|
FingerprintVersion: manifest.TranscribeFingerprintContractVersion,
|
||||||
|
Fingerprint: strings.Repeat("c", 64),
|
||||||
|
AudioPath: "audio/alice.flac",
|
||||||
|
AudioSHA256: strings.Repeat("a", 64),
|
||||||
|
AudioSize: 12,
|
||||||
|
Output: manifest.ArtifactRecord{
|
||||||
|
Kind: "transcript_raw", LocalPath: rawRelative,
|
||||||
|
ProducerRunID: remoteManifest.RunID, Checksum: strings.Repeat("b", 64),
|
||||||
|
},
|
||||||
|
OutputSize: 15,
|
||||||
|
ProducerRunID: remoteManifest.RunID,
|
||||||
|
},
|
||||||
|
}
|
||||||
remoteManifest.Stages["extract"] = &manifest.StageRecord{
|
remoteManifest.Stages["extract"] = &manifest.StageRecord{
|
||||||
Name: "extract", Status: manifest.StatusSucceeded,
|
Name: "extract", Status: manifest.StatusSucceeded,
|
||||||
Outputs: []manifest.ArtifactRecord{{
|
Outputs: []manifest.ArtifactRecord{{
|
||||||
@@ -121,6 +143,13 @@ func TestExecuteRestoreRoundTripsPublishedExtractionAndManifestMetadata(t *testi
|
|||||||
if want := filepath.Join(sessionRoot, "artifacts", "encounters.json"); lane.LocalPath != want {
|
if want := filepath.Join(sessionRoot, "artifacts", "encounters.json"); lane.LocalPath != want {
|
||||||
t.Fatalf("rebased extraction path = %q, want %q", lane.LocalPath, want)
|
t.Fatalf("rebased extraction path = %q, want %q", lane.LocalPath, want)
|
||||||
}
|
}
|
||||||
|
speaker := restored.Stages["transcribe"].TranscribeSpeakers["alice"]
|
||||||
|
if speaker.AudioPath != "audio/alice.flac" || speaker.Output.LocalPath != rawRelative {
|
||||||
|
t.Fatalf("restored speaker paths = %#v, want session-relative paths", speaker)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(sessionRoot, filepath.FromSlash(rawRelative))); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("unpublished raw transcript should not be restored; stat err=%v", err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExecuteRestoreIncludeAudioRestoresAudio(t *testing.T) {
|
func TestExecuteRestoreIncludeAudioRestoresAudio(t *testing.T) {
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ func TestRestorePlanDefaultScope(t *testing.T) {
|
|||||||
seedCurrentRestoreObject(t, store, current, current.SessionPrefix+"audio/alice.flac", []byte("audio"))
|
seedCurrentRestoreObject(t, store, current, current.SessionPrefix+"audio/alice.flac", []byte("audio"))
|
||||||
seedRestoreObject(store, current.SessionPrefix+"runs/20260519T010203Z-a1b2/manifest.json", []byte("{}"))
|
seedRestoreObject(store, current.SessionPrefix+"runs/20260519T010203Z-a1b2/manifest.json", []byte("{}"))
|
||||||
seedRestoreObject(store, current.SessionPrefix+"logs/publish.log", []byte("log"))
|
seedRestoreObject(store, current.SessionPrefix+"logs/publish.log", []byte("log"))
|
||||||
|
seedRestoreObject(store, current.SessionPrefix+"cache/notarius/references/ignored/party.yml", []byte("cache"))
|
||||||
|
|
||||||
plan, err := buildRestorePlan(context.Background(), cfg, current, store, RestorePlanOptions{})
|
plan, err := buildRestorePlan(context.Background(), cfg, current, store, RestorePlanOptions{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -179,13 +179,8 @@ func TestExecuteRestoreLoadsSecretsBeforeObjectStoreInit(t *testing.T) {
|
|||||||
secretsDir := filepath.Join(t.TempDir(), "secrets")
|
secretsDir := filepath.Join(t.TempDir(), "secrets")
|
||||||
mustWriteSecretFile(t, filepath.Join(secretsDir, accessKeyEnv), "test-access-key-id\n")
|
mustWriteSecretFile(t, filepath.Join(secretsDir, accessKeyEnv), "test-access-key-id\n")
|
||||||
mustWriteSecretFile(t, filepath.Join(secretsDir, secretKeyEnv), "test-secret-key\n")
|
mustWriteSecretFile(t, filepath.Join(secretsDir, secretKeyEnv), "test-secret-key\n")
|
||||||
f, err := os.OpenFile(pipelinePath, os.O_APPEND|os.O_WRONLY, 0)
|
if err := os.Symlink(secretsDir, filepath.Join(filepath.Dir(pipelinePath), "secrets")); err != nil {
|
||||||
if err != nil {
|
t.Fatalf("link conventional secrets directory: %v", err)
|
||||||
t.Fatalf("open pipeline config for append: %v", err)
|
|
||||||
}
|
|
||||||
defer f.Close()
|
|
||||||
if _, err := f.WriteString("\nsecrets:\n env_dir: " + secretsDir + "\n"); err != nil {
|
|
||||||
t.Fatalf("append secrets config: %v", err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
storeInitCalled := false
|
storeInitCalled := false
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ func runWithDependencies(ctx context.Context, args []string, out io.Writer, exec
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
flags := request.Config
|
flags := request.Config
|
||||||
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), deps)
|
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), flags.secretsDir.pointer(), deps)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("run: %w", err)
|
return fmt.Errorf("run: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -80,6 +80,8 @@ func runStageWithDependencies(ctx context.Context, args []string, out io.Writer,
|
|||||||
SessionPath: flags.sessionPath,
|
SessionPath: flags.sessionPath,
|
||||||
SessionID: flags.sessionID,
|
SessionID: flags.sessionID,
|
||||||
PreviousSessionID: flags.previousSessionID,
|
PreviousSessionID: flags.previousSessionID,
|
||||||
|
Profile: flags.profile.pointer(),
|
||||||
|
SecretsDir: flags.secretsDir.pointer(),
|
||||||
Force: force,
|
Force: force,
|
||||||
SelectedArtifacts: normalizedArtifacts,
|
SelectedArtifacts: normalizedArtifacts,
|
||||||
}, executor, deps)
|
}, executor, deps)
|
||||||
@@ -137,6 +139,8 @@ func analyzeWithDependencies(ctx context.Context, args []string, out io.Writer,
|
|||||||
SessionPath: flags.sessionPath,
|
SessionPath: flags.sessionPath,
|
||||||
SessionID: flags.sessionID,
|
SessionID: flags.sessionID,
|
||||||
PreviousSessionID: flags.previousSessionID,
|
PreviousSessionID: flags.previousSessionID,
|
||||||
|
Profile: flags.profile.pointer(),
|
||||||
|
SecretsDir: flags.secretsDir.pointer(),
|
||||||
Force: true,
|
Force: true,
|
||||||
SelectedArtifacts: normalizedArtifacts,
|
SelectedArtifacts: normalizedArtifacts,
|
||||||
}, executor, deps)
|
}, executor, deps)
|
||||||
@@ -192,6 +196,8 @@ func publishWithDependencies(ctx context.Context, args []string, out io.Writer,
|
|||||||
SessionPath: flags.sessionPath,
|
SessionPath: flags.sessionPath,
|
||||||
SessionID: flags.sessionID,
|
SessionID: flags.sessionID,
|
||||||
PreviousSessionID: flags.previousSessionID,
|
PreviousSessionID: flags.previousSessionID,
|
||||||
|
Profile: flags.profile.pointer(),
|
||||||
|
SecretsDir: flags.secretsDir.pointer(),
|
||||||
Force: true,
|
Force: true,
|
||||||
SelectedArtifacts: normalizedArtifacts,
|
SelectedArtifacts: normalizedArtifacts,
|
||||||
}, executor, deps)
|
}, executor, deps)
|
||||||
@@ -218,6 +224,8 @@ type singleStageCommand struct {
|
|||||||
SessionPath string
|
SessionPath string
|
||||||
SessionID string
|
SessionID string
|
||||||
PreviousSessionID string
|
PreviousSessionID string
|
||||||
|
Profile *string
|
||||||
|
SecretsDir *string
|
||||||
Force bool
|
Force bool
|
||||||
SelectedArtifacts []string
|
SelectedArtifacts []string
|
||||||
}
|
}
|
||||||
@@ -232,7 +240,8 @@ func runSingleStageCommand(ctx context.Context, req singleStageCommand, executor
|
|||||||
loaded, err := loadCommandConfigWithDependencies(ctx, req.PipelinePath, req.CampaignPath, req.CampaignFilePath, req.SessionPath, config.SessionLoadOptions{
|
loaded, err := loadCommandConfigWithDependencies(ctx, req.PipelinePath, req.CampaignPath, req.CampaignFilePath, req.SessionPath, config.SessionLoadOptions{
|
||||||
SessionID: req.SessionID,
|
SessionID: req.SessionID,
|
||||||
PreviousSessionID: req.PreviousSessionID,
|
PreviousSessionID: req.PreviousSessionID,
|
||||||
}, deps)
|
Profile: req.Profile,
|
||||||
|
}, req.SecretsDir, deps)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%s: %w", req.CommandName, err)
|
return nil, fmt.Errorf("%s: %w", req.CommandName, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -211,11 +211,18 @@ func buildDefaultAuditaRunner(cfg *config.Config) (audita.Runner, error) {
|
|||||||
if a.Report != nil {
|
if a.Report != nil {
|
||||||
report = *a.Report
|
report = *a.Report
|
||||||
}
|
}
|
||||||
|
llmAPIKey := ""
|
||||||
|
llmAPIKeyReference := ""
|
||||||
|
if a.LLMAPIKey != nil {
|
||||||
|
llmAPIKey = a.LLMAPIKey.Value()
|
||||||
|
llmAPIKeyReference = a.LLMAPIKey.EnvName()
|
||||||
|
}
|
||||||
|
|
||||||
runner, err := audita.NewSubprocessRunnerFromConfigValues(
|
runner, err := audita.NewSubprocessRunnerFromConfigValues(
|
||||||
a.Binary,
|
a.Binary,
|
||||||
a.Timeout,
|
a.Timeout,
|
||||||
a.LLMAPIKeyEnv,
|
llmAPIKey,
|
||||||
|
llmAPIKeyReference,
|
||||||
append([]string(nil), a.Modules...),
|
append([]string(nil), a.Modules...),
|
||||||
a.BaseURL,
|
a.BaseURL,
|
||||||
a.Model,
|
a.Model,
|
||||||
|
|||||||
@@ -157,9 +157,6 @@ func initializeLockedInvocation(runtime *invocationRuntime) error {
|
|||||||
if err := runtime.manifestStore.Save(runtime.ctx, runtime.manifestPath, m); err != nil {
|
if err := runtime.manifestStore.Save(runtime.ctx, runtime.manifestPath, m); err != nil {
|
||||||
return fmt.Errorf("save manifest identity %q: %w", runtime.manifestPath, err)
|
return fmt.Errorf("save manifest identity %q: %w", runtime.manifestPath, err)
|
||||||
}
|
}
|
||||||
if _, err := loadSecretsFromConfig(runtime.env.Config, runtime.env.Logger); err != nil {
|
|
||||||
return runtime.persistTerminalFailure(fmt.Errorf("load secrets from files: %w", err))
|
|
||||||
}
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user