Compare commits
19 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| ed4623bc41 | |||
| 3ce198197d | |||
| 0ac8072a9a | |||
| 3ec54b8c72 | |||
| 47b3e5b55b | |||
| 99d1a0b926 | |||
| 884a62e8a6 | |||
| f7a40afc7a | |||
| a4f37ff427 | |||
| 2b97ef3722 | |||
| 63a86c43f4 | |||
| 92432232ef | |||
| d31ba5a410 | |||
| f0225e937c | |||
| 9e40c729e1 | |||
| 0e6ba414b3 | |||
| 10a2e89965 | |||
| f8e3e651e6 | |||
| 1558c9ccae |
@@ -40,7 +40,7 @@ steps:
|
||||
set -eu
|
||||
output_dir="$(mktemp -d)"
|
||||
trap 'rm -rf "$output_dir"' EXIT
|
||||
for target in linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 windows/amd64 windows/arm64; do
|
||||
for target in linux/amd64 linux/arm64 darwin/arm64; do
|
||||
goos="${target%/*}"
|
||||
goarch="${target#*/}"
|
||||
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" go build -o "$output_dir/narratio-$goos-$goarch" ./cmd/narratio
|
||||
|
||||
49
docs/cli.md
49
docs/cli.md
@@ -20,7 +20,7 @@ Top-level commands:
|
||||
- `publish <session_id>`: force-run publish.
|
||||
- `clean <session_id>` or `clean --all`: remove local work/spool state.
|
||||
- `session <subcommand>`: session helper commands.
|
||||
- `config <subcommand>`: validate, display, source-trace, or compare resolved pipeline configuration.
|
||||
- `config <subcommand>`: validate, display, source-trace, catalog, or compare resolved pipeline configuration.
|
||||
|
||||
Session subcommands:
|
||||
|
||||
@@ -45,6 +45,7 @@ Most session-aware commands accept:
|
||||
- `--session-id <session_id>`
|
||||
- `--previous-session-id <session_id>`
|
||||
- `--profile <name>`
|
||||
- `--secrets-dir <directory>`
|
||||
|
||||
Rules:
|
||||
|
||||
@@ -53,10 +54,13 @@ Rules:
|
||||
- if both positional `<session_id>` and `--session-id` are provided, values must match.
|
||||
- `--previous-session-id` is a strict expectation: the selected session file
|
||||
must contain the same `previous_session_id`.
|
||||
- `--profile` selects a declared pipeline profile. It may be supplied once;
|
||||
an explicit empty or unknown value fails configuration resolution. When it is
|
||||
omitted, a declared `default_profile` is used. The same selection applies to
|
||||
all common-flag commands, including `regenerate-artifacts`.
|
||||
- `--profile` selects `profiles/<name>.yml` relative to the pipeline root. It
|
||||
may be supplied once; omitted, empty, and the exact name `default` select the
|
||||
base configuration, while an unknown named profile fails. The same selection
|
||||
applies to all common-flag commands, including `regenerate-artifacts`.
|
||||
- `--secrets-dir` selects ConfigKit credential-file bootstrap explicitly. A
|
||||
relative value uses caller CWD. Without the flag, a present `secrets/`
|
||||
directory beside the resolved root pipeline is used; an absent one is skipped.
|
||||
- `clean --all` cannot be combined with campaign/session selectors.
|
||||
- notification delivery is currently limited to the configured `noop` mode; see
|
||||
the [configuration reference](./config.md#notifications).
|
||||
@@ -78,19 +82,22 @@ Commands with additional positionals keep their command-specific order:
|
||||
|
||||
## Command Reference
|
||||
|
||||
### `config validate`, `config show`, `config sources`, and `config diff`
|
||||
### `config validate`, `config show`, `config sources`, `config artifacts`, and `config diff`
|
||||
|
||||
```bash
|
||||
narratio config validate [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>]
|
||||
narratio config show [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>]
|
||||
narratio config sources [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>]
|
||||
narratio config diff <left-profile> <right-profile> [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>]
|
||||
narratio config validate [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>] [--secrets-dir <directory>]
|
||||
narratio config show [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>] [--secrets-dir <directory>]
|
||||
narratio config sources [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>] [--secrets-dir <directory>]
|
||||
narratio config artifacts [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>] [--secrets-dir <directory>]
|
||||
narratio config diff <left-profile> <right-profile> [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--secrets-dir <directory>]
|
||||
```
|
||||
|
||||
These commands resolve the selected profile, defaults, ordinary paths, and—if
|
||||
a campaign is selected—the campaign-owned party. They neither discover or load
|
||||
a session nor create a workspace, manifest, run, lock, adapter, remote
|
||||
connection, or credential environment.
|
||||
a session nor create a workspace, manifest, run, lock, adapter, or remote
|
||||
connection. Like every pipeline-loading command, they bootstrap the selected or
|
||||
conventional secrets directory before resolving configuration; `config show`
|
||||
never includes bootstrap reports or secret values in its YAML output.
|
||||
|
||||
Campaign selection is optional for a pipeline without party-driven artifact
|
||||
families. A pipeline with `scriptorium.artifact_families` needs a selected or
|
||||
@@ -101,7 +108,7 @@ Session, range, force, and artifact-execution flags are not accepted.
|
||||
`config validate` writes a concise root-path, selected-profile (or `none`), and
|
||||
effective-digest summary after successful complete validation. `config show`
|
||||
writes one deterministic, secret-free YAML document containing defaulted and
|
||||
expanded concrete configuration. It omits composition declarations, artifact
|
||||
expanded concrete configuration. It omits ConfigKit directives, artifact
|
||||
family declarations, and runtime provenance.
|
||||
|
||||
`config sources` reports the same fully validated resolution without printing
|
||||
@@ -113,8 +120,19 @@ and generated family ownership. A generated party member has
|
||||
one family record and one party record at the same logical path. The output
|
||||
never reads or prints secret values.
|
||||
|
||||
`config diff` resolves both supplied profile names from one parsed root source
|
||||
set and compares their fully resolved, secret-free effective mappings. It does
|
||||
`config artifacts` prints the concrete catalog used by analysis, ordered by
|
||||
artifact key. Its tab-separated header is `key`, `source`, `definitions`,
|
||||
`enabled`, `output`, `dependencies`, `family`, `character`, `family_source`, and
|
||||
`party_source`. Definitions and effective dependencies are sorted comma-separated
|
||||
lists, and control characters in fields are escaped. Family-member rows identify
|
||||
their family, character, defining family file, and party file. An empty catalog
|
||||
prints only the header. Ordinary artifacts can be inspected without a campaign;
|
||||
party-driven families retain the campaign requirement. See
|
||||
[Scriptorium Artifact Entries](./config.md#scriptorium-artifact-entries) for the
|
||||
configuration layout.
|
||||
|
||||
`config diff` resolves both supplied profile names independently and compares
|
||||
their fully resolved, secret-free effective mappings. It does
|
||||
not accept `--profile`; the two positional names must be distinct, declared
|
||||
profiles. When party-driven families are present, both profiles must resolve to
|
||||
the same selected campaign and party. Use `--campaign-file` if profile-specific
|
||||
@@ -320,6 +338,7 @@ Options:
|
||||
|
||||
- `--config <pipeline.yml>`
|
||||
- `--campaign <id>` or `--campaign-file <campaign.yml>`
|
||||
- `--secrets-dir <directory>`
|
||||
- `--previous-session-id <id>`
|
||||
- `--date <YYYY-MM-DD>`
|
||||
- `--title <text>`
|
||||
|
||||
170
docs/config.md
170
docs/config.md
@@ -44,13 +44,16 @@ the durable copied session input.
|
||||
|
||||
### Read-only effective pipeline inspection
|
||||
|
||||
`narratio config validate`, `narratio config show`, and `narratio config
|
||||
sources` use the same `--config`, `--campaign`, `--campaign-file`, and
|
||||
`narratio config validate`, `narratio config show`, `narratio config sources`,
|
||||
and `narratio config artifacts` use the same `--config`, `--campaign`,
|
||||
`--campaign-file`, and
|
||||
`--profile` selection rules as pipeline commands, but do not select, discover,
|
||||
or load a session. They do not read credential values or create runtime state.
|
||||
or load a session. They resolve configured typed credential references as part
|
||||
of the ordinary ConfigKit load but expose only their environment-variable names;
|
||||
they do not create runtime state.
|
||||
`narratio config diff <left-profile> <right-profile>` uses the same pipeline and
|
||||
campaign selectors, resolves each named profile independently from one parsed
|
||||
root source set, and does not accept a separate `--profile` flag.
|
||||
campaign selectors, resolves each named profile with an independent ConfigKit
|
||||
load, and does not accept a separate `--profile` flag.
|
||||
|
||||
Campaign selection is optional only when the resolved pipeline has no
|
||||
`scriptorium.artifact_families`. When families are declared, Narratio selects a
|
||||
@@ -59,10 +62,12 @@ then parses the campaign-owned party and expands concrete artifacts and any
|
||||
family publish rules before validation. `config validate` prints the resulting
|
||||
root, profile, and effective digest. `config show` emits the normalized
|
||||
effective pipeline YAML, with defaults and concrete expansion included but
|
||||
composition and family declarations omitted. `config sources` prints a stable
|
||||
ConfigKit directives and family declarations omitted. This review projection is
|
||||
not promised to be a reloadable authoring document. `config sources` prints a stable
|
||||
source projection instead of effective values: root/import/profile/default
|
||||
ownership plus campaign/party and generated-family records. Canonical derived
|
||||
players trace to the party. The [CLI reference](cli.md#config-validate-config-show-config-sources-and-config-diff)
|
||||
players trace to the party. `config artifacts` lists the concrete artifact map,
|
||||
effective dependencies, and defining sources. The [CLI reference](cli.md#config-validate-config-show-config-sources-config-artifacts-and-config-diff)
|
||||
owns command syntax and output conventions.
|
||||
|
||||
`config diff` compares normalized field values rather than YAML text or source
|
||||
@@ -76,13 +81,14 @@ difference.
|
||||
### Migrating to the maintained bundle
|
||||
|
||||
Use the [production/testing bundle](../examples/production-testing/pipeline.yml)
|
||||
as the complete copyable migration reference. Split stable pipeline settings
|
||||
into explicit additive imports, place production/testing differences in one
|
||||
selected overlay, and retain a production `default_profile`. Convert campaign
|
||||
rosters to [canonical party input](integrations/party.md), then express
|
||||
character work as families. Inspect the result
|
||||
with `config validate`, `config show`, and `config sources`; use `config diff`
|
||||
to review profiles before running a session. Unversioned party documents are
|
||||
as the complete copyable migration reference. Keep production settings in the
|
||||
base, split them into explicit additive imports, and place testing differences
|
||||
in the fixed `profiles/testing.yml` overlay. Put ordinary artifacts and families
|
||||
in the recommended one-definition files shown by that bundle. Convert campaign
|
||||
rosters to [canonical party input](integrations/party.md), then express character
|
||||
work as families. Inspect the result with `config validate`, `config show`,
|
||||
`config sources`, and `config artifacts`; use `config diff default testing` to
|
||||
review the overlay before running a session. Unversioned party documents are
|
||||
rejected.
|
||||
|
||||
### Identity segments
|
||||
@@ -96,20 +102,28 @@ remote state with an unsafe legacy identity must be migrated before use.
|
||||
|
||||
## Validation and Merge Rules
|
||||
|
||||
- YAML decode is strict (`KnownFields(true)`) and accepts exactly one document:
|
||||
unknown fields or trailing documents fail load.
|
||||
- ConfigKit performs strict typed YAML decoding and accepts exactly one document;
|
||||
unknown fields, duplicate keys, and trailing documents fail load.
|
||||
- A pipeline file may explicitly import additive YAML fragments through the
|
||||
root-only `composition.imports` list. Imported files contribute fields to one
|
||||
root-only `configkit.imports` list. Imported files contribute fields to one
|
||||
logical pipeline document; they do not override fields supplied by the root
|
||||
or another import.
|
||||
- A root pipeline may declare named profiles. Exactly one profile is selected
|
||||
by an option-aware caller or by `composition.default_profile`; a caller's
|
||||
explicit selection takes precedence. Declaring profiles without either form
|
||||
of selection is an error.
|
||||
- An omitted profile, an empty profile option, and the exact name `default`
|
||||
select the base configuration. Any other name selects
|
||||
`profiles/<name>.yml` relative to the pipeline root.
|
||||
- Campaign, session, and canonical-party documents are strict single-source
|
||||
ConfigKit schemas. They do not enable imports, profiles, or `configkit`
|
||||
directives.
|
||||
- Configured timeout and retry-delay durations must be positive. An omitted
|
||||
artifact timeout continues to inherit its configured Scriptorium timeout.
|
||||
- Session files must be concrete; unresolved `{{ ... }}` placeholders fail load.
|
||||
- Session and party inputs inspected before typed loading are limited to 8 MiB.
|
||||
- Pipeline defaults are applied before validation.
|
||||
- Defaults apply only to absent fields. Explicit empty strings, zero values,
|
||||
`false`, and `null` retain ConfigKit's typed meaning and are then checked by
|
||||
Narratio domain validation; they are not silently replaced by defaults.
|
||||
Explicit null optional stage sections stay absent, while required sections
|
||||
fail validation.
|
||||
- Campaign and session identities must agree.
|
||||
- Required stable files (`speakers_file`, `autocorrect_file`, `glossary_file`,
|
||||
`party_file`) are campaign-owned; the first three may be overridden by the
|
||||
@@ -123,22 +137,16 @@ remote state with an unsafe legacy identity must be migrated before use.
|
||||
- local (`audio_dir` or `audio_files`), or
|
||||
- S3 (`audio_s3.prefix`).
|
||||
|
||||
### Pipeline composition
|
||||
### Pipeline imports and profiles
|
||||
|
||||
Large pipeline configurations may be split into explicitly named fragments and
|
||||
may declare one overlay per selectable profile:
|
||||
may use fixed-path profile overlays:
|
||||
|
||||
```yaml
|
||||
composition:
|
||||
configkit:
|
||||
imports:
|
||||
- config/storage.yml
|
||||
- config/integrations.yaml
|
||||
default_profile: production
|
||||
profiles:
|
||||
production:
|
||||
overlay: profiles/production.yml
|
||||
testing:
|
||||
overlay: profiles/testing.yml
|
||||
- path: config/storage.yml
|
||||
- path: config/integrations.yaml
|
||||
|
||||
campaigns:
|
||||
root: /usr/local/share/narratio/campaigns
|
||||
@@ -152,36 +160,29 @@ Imports are resolved relative to the directory containing the root pipeline
|
||||
file and are loaded in declaration order. Narratio does not scan directories or
|
||||
infer fragments. Each import must be a confined regular `.yml` or `.yaml` file:
|
||||
absolute paths, traversal, symlinks, directories, duplicate files, and an
|
||||
import of the root pipeline itself are rejected. Only the root pipeline may
|
||||
contain `composition`; nested composition is rejected.
|
||||
import of the root pipeline itself are rejected. Imported files cannot declare
|
||||
another `configkit` control section.
|
||||
|
||||
Composition is additive. A map may be extended by multiple files when every
|
||||
Base merging is additive. A map may be extended by multiple files when every
|
||||
leaf is distinct, but a scalar, list, or map/list/scalar kind cannot be claimed
|
||||
more than once, even when the repeated values are identical. Conflict errors
|
||||
name the full field path and every source that claimed it. The assembled YAML is
|
||||
then decoded against the normal strict pipeline schema and defaults are applied
|
||||
once.
|
||||
name the full field path and every source that claimed it. Narratio additionally
|
||||
rejects an artifact or artifact-family key declared by more than one base file,
|
||||
even when those files contribute disjoint children.
|
||||
|
||||
Profile names are case-sensitive, non-empty, trimmed, and cannot contain
|
||||
control characters. If `profiles` is present, it must contain at least one
|
||||
entry and every entry must contain only an `overlay` path. An explicit profile
|
||||
selection overrides `default_profile`; unknown and explicitly empty selections
|
||||
fail. Narratio never selects the first profile implicitly and does not read a
|
||||
profile selection from the environment.
|
||||
Profile names are case-sensitive and use ConfigKit's safe fixed-path naming
|
||||
rules. Only the selected overlay is opened; a malformed unselected overlay does
|
||||
not affect a base or different-profile load. Maps merge recursively, overlay
|
||||
scalars replace base scalars, and overlay lists replace base lists completely.
|
||||
Explicit `false`, zero, empty strings, empty containers, and null values remain
|
||||
explicit inputs and are not refilled by defaults. Profiles cannot inherit,
|
||||
stack, import files, or declare another ConfigKit control section.
|
||||
|
||||
Every declared overlay is resolved relative to the root pipeline directory and
|
||||
must satisfy the same confined regular-YAML-file rules as an import. Narratio
|
||||
parses every declared overlay even when it is not selected, then applies only
|
||||
the selected one. Maps merge recursively, overlay scalars replace base scalars,
|
||||
and overlay lists replace base lists completely. Explicit `false`, zero, empty
|
||||
lists, and empty maps remain meaningful. YAML null cannot delete a value, and
|
||||
kind changes are rejected. Profiles cannot inherit from or stack with other
|
||||
profiles, and overlays cannot import files or declare profiles.
|
||||
|
||||
After composition, Narratio strictly decodes the result, applies centralized
|
||||
defaults once, resolves ordinary paths, and computes a deterministic effective
|
||||
During ConfigKit loading, Narratio applies fresh typed defaults, recorded
|
||||
entry-level defaults and normalization, read-only file-local validation, and
|
||||
ordinary path resolution before computing a deterministic effective
|
||||
configuration digest. The digest represents the normalized, secret-free
|
||||
runtime pipeline mapping; it excludes composition declarations, source
|
||||
runtime pipeline mapping; it excludes ConfigKit directives, source
|
||||
provenance, profile identity, and raw environment secret values. Equivalent
|
||||
effective mappings therefore have the same digest regardless of how fields are
|
||||
split among the root and imports.
|
||||
@@ -224,15 +225,25 @@ inputs:
|
||||
## Secrets Handling
|
||||
|
||||
- Do not place raw secrets in YAML.
|
||||
- Use env var names in config (for example `pipeline.audita.llm_api_key_env`).
|
||||
- Optionally load credential files from `pipeline.secrets.env_dir`. Each valid
|
||||
environment-variable filename supplies one value; trailing CR/LF is removed.
|
||||
- Credential fields contain environment-variable names, represented internally
|
||||
as ConfigKit typed secret references (for example
|
||||
`pipeline.audita.llm_api_key: AUDITA_LLM_API_KEY`). A configured reference
|
||||
must resolve to a nonempty environment value during load.
|
||||
- Every command that loads a pipeline accepts `--secrets-dir <path>`. When the
|
||||
flag is omitted, Narratio uses a present `secrets/` directory beside the
|
||||
resolved root `pipeline.yml`; an absent conventional directory is ignored.
|
||||
Explicit relative paths are resolved from the caller's working directory,
|
||||
and a missing or empty explicit selection is an error.
|
||||
- Each immediate regular file whose name is a valid environment-variable name
|
||||
supplies one value. ConfigKit removes trailing CR/LF, permits a symlink as the
|
||||
selected directory, rejects symlink entries, and limits input to 1 MiB per
|
||||
file, 8 MiB total, and 1024 entries.
|
||||
- An existing process environment value takes precedence over a credential file.
|
||||
- Credential directories and files must not be symlinks and must be regular,
|
||||
bounded files (at most 8 KiB per value). On POSIX, provision the directory
|
||||
with no group/other access (normally `0700`) and files with no group/other
|
||||
access (normally `0600`).
|
||||
- Commands that need storage/auth load filesystem secrets before constructing adapters.
|
||||
- On POSIX, operators must provision the directory with no group/other access
|
||||
(normally `0700`) and files with no group/other access (normally `0600`);
|
||||
ConfigKit validates type and bounds but does not enforce permission modes.
|
||||
- Bootstrap happens once per command, before any configuration load or adapter
|
||||
construction. Safe reports contain filenames, paths, and statuses, never values.
|
||||
|
||||
## Publish Configuration Summary
|
||||
|
||||
@@ -274,22 +285,19 @@ Rules:
|
||||
|
||||
| Field | Type | Required | Default / Rule |
|
||||
| --- | --- | --- | --- |
|
||||
| `composition.imports[]` | list of strings | No | explicit additive pipeline fragments relative to the root pipeline directory; `.yml` or `.yaml` regular files only |
|
||||
| `composition.default_profile` | string | Conditional | selected when profiles exist and no caller explicitly selects one; must name a declared profile |
|
||||
| `composition.profiles.<name>.overlay` | string | Conditional | required for every declared profile; one confined `.yml` or `.yaml` overlay relative to the root pipeline directory |
|
||||
| `configkit.imports[].path` | string | No | explicit additive pipeline fragment relative to the root pipeline directory; `.yml` or `.yaml` regular files only |
|
||||
| `pipeline.workspace.root` | string | No | `/var/lib/narratio` |
|
||||
| `pipeline.workspace.cleanup_after_publish` | bool | No | `false`; selects the published run's workspace target for final cleanup and requires effective `pipeline.publish.enabled=true` and `pipeline.publish.upload_run=true` |
|
||||
| `pipeline.campaigns.root` | string | No | `/usr/local/share/narratio/campaigns` |
|
||||
| `pipeline.campaigns.default_campaign_id` | string | No | empty |
|
||||
| `pipeline.secrets.env_dir` | string | No | empty |
|
||||
| `pipeline.storage.backend` | string | No | `local`; supported values are `local` and `s3` (case-insensitive) |
|
||||
| `pipeline.storage.s3.bucket` | string | Conditional | required when backend is `s3` and S3 session-audio or publish upload is enabled |
|
||||
| `pipeline.storage.s3.root_prefix` | string | No | `dnd` |
|
||||
| `pipeline.storage.s3.region` | string | No | empty |
|
||||
| `pipeline.storage.s3.endpoint` | string | No | empty |
|
||||
| `pipeline.storage.s3.force_path_style` | bool | No | `false` |
|
||||
| `pipeline.storage.s3.access_key_id_env` | string | No | `OBJECT_STORAGE_KEY_ID` |
|
||||
| `pipeline.storage.s3.secret_access_key_env` | string | No | `OBJECT_STORAGE_KEY` |
|
||||
| `pipeline.storage.s3.access_key_id` | secret reference | Conditional | unset; must be paired with `secret_access_key` |
|
||||
| `pipeline.storage.s3.secret_access_key` | secret reference | Conditional | unset; must be paired with `access_key_id` |
|
||||
| `pipeline.spool.root` | string | No | `/var/spool/narratio` |
|
||||
| `pipeline.spool.delete_audio_after_publish` | bool | No | `false`; selects the published run's spool-audio target for final cleanup and requires effective `pipeline.publish.enabled=true` and `pipeline.publish.upload_run=true` |
|
||||
| `pipeline.cache.root` | string | No | `/var/cache/narratio` |
|
||||
@@ -317,7 +325,7 @@ Rules:
|
||||
| `pipeline.seriatim.env.filler_max_duration` | float | No | unset |
|
||||
| `pipeline.audita.binary` | string | No | `audita` |
|
||||
| `pipeline.audita.timeout` | duration | No | `3h` |
|
||||
| `pipeline.audita.llm_api_key_env` | string | No | empty |
|
||||
| `pipeline.audita.llm_api_key` | secret reference | No | unset |
|
||||
| `pipeline.audita.modules[]` | list[string] | No | empty |
|
||||
| `pipeline.audita.base_url` | string | No | empty |
|
||||
| `pipeline.audita.model` | string | No | empty |
|
||||
@@ -430,7 +438,7 @@ For each `pipeline.scriptorium.artifacts.<name>`:
|
||||
| Field | Type | Required | Rule |
|
||||
| --- | --- | --- | --- |
|
||||
| `enabled` | bool | No | `false` if omitted |
|
||||
| `depends_on[]` | list[string] | No | must reference configured artifact keys; no self-reference; configured graph must be acyclic |
|
||||
| `depends_on[]` | list[string] | No | explicit ordering-only edges; must reference configured artifact keys, may not self-reference, and join inferred input edges in one acyclic graph |
|
||||
| `render_debug` | bool | No | per-artifact override |
|
||||
| `prompt_id` | string | Conditional | required when artifact is enabled |
|
||||
| `profile_id` | string | No | empty |
|
||||
@@ -444,7 +452,7 @@ Narratio adds `session_id=narratio-session-<session_id>` to every Scriptorium re
|
||||
Without `--artifacts`, analyze executes enabled configured artifacts. With an
|
||||
explicit `--artifacts` list, the exact named configured artifacts are the
|
||||
one-invocation targets even if their `enabled` values are false. Analyze closes
|
||||
those targets over `depends_on`: a current prerequisite is reused, while a
|
||||
those targets over the effective dependency graph: a current prerequisite is reused, while a
|
||||
stale, missing, or failed prerequisite is rebuilt before its dependent.
|
||||
Unrelated artifacts are not executed. Named targets and any prerequisite that
|
||||
may require rebuilding must therefore have valid executable fields. This
|
||||
@@ -462,6 +470,15 @@ For each artifact input `pipeline.scriptorium.artifacts.<name>.inputs.<input_nam
|
||||
loading. Use the canonical `source` identifier to select the input; Narratio
|
||||
does not provide adapter-specific input passthrough fields.
|
||||
|
||||
Each `narratio.artifact.<name>` input source automatically contributes an
|
||||
effective dependency on that configured artifact. Narratio unions these inferred
|
||||
edges with explicit `depends_on` entries, then trims, deduplicates, and sorts the
|
||||
result during final configuration resolution. Authors use `depends_on` only for
|
||||
ordering constraints that are not already expressed by an artifact input.
|
||||
Use [`narratio config artifacts`](./cli.md#config-validate-config-show-config-sources-config-artifacts-and-config-diff)
|
||||
to inspect keys, definition sources, outputs, and effective dependencies without
|
||||
loading a session or creating runtime state.
|
||||
|
||||
### Scriptorium Artifact Families
|
||||
|
||||
`pipeline.scriptorium.artifact_families` declares a shared artifact template
|
||||
@@ -493,9 +510,10 @@ Their resolved values are strings. A member variable may not reuse a static
|
||||
other Scriptorium artifact.
|
||||
|
||||
Within a family only, an input source may use
|
||||
`narratio.member_artifact.<family>`. The referenced family must be named in
|
||||
that family's `member_dependencies`; resolution rewrites the source to the
|
||||
corresponding ordinary `narratio.artifact.<family>_<character_id>` source.
|
||||
`narratio.member_artifact.<family>`. Resolution rewrites the source to the
|
||||
corresponding ordinary `narratio.artifact.<family>_<character_id>` source and
|
||||
infers the concrete member dependency. Use `member_dependencies` only for
|
||||
additional same-character ordering constraints that are not already inputs.
|
||||
This syntax is rejected in explicit artifacts and never reaches runtime stages
|
||||
or Scriptorium.
|
||||
|
||||
|
||||
@@ -58,7 +58,7 @@ in the [Release Procedure](release.md).
|
||||
|
||||
Woodpecker also runs `go test -race -shuffle=on -count=3 ./...` on its scheduled
|
||||
job to expose ordering and repeatability defects. Current runners cross-compile
|
||||
for macOS and Windows, but do not provide native macOS or Windows execution.
|
||||
Those cross-builds establish compilation only, not platform-equivalent runtime
|
||||
evidence. Add native checks only when official runner labels and successful
|
||||
the supported Linux amd64 and arm64 targets and the best-effort macOS arm64
|
||||
target. The macOS cross-build establishes compilation only, not native runtime
|
||||
evidence. Add a native check only when an official runner label and successful
|
||||
native-run evidence are available.
|
||||
|
||||
@@ -41,7 +41,6 @@ Construction fails for invalid static config values, including:
|
||||
|
||||
Run fails for:
|
||||
- missing required request paths;
|
||||
- missing required credential env var when configured (`llm_api_key_env`);
|
||||
- subprocess execution failure;
|
||||
- invalid processed transcript JSON (`segments` array required);
|
||||
- invalid report JSON when reporting is enabled.
|
||||
@@ -53,6 +52,9 @@ Failure results still include output/log/config/exit metadata for diagnostics.
|
||||
|
||||
## Deterministic Behavior
|
||||
- CLI args are built from runner config + request in a fixed order.
|
||||
- A configured typed `llm_api_key` reference is resolved during pipeline load;
|
||||
the captured value is supplied only as the child's `AUDITA_LLM_API_KEY`
|
||||
override and is never re-read from the parent environment.
|
||||
- Generated invocation YAML (`audita.generated.v1`) is emitted when requested.
|
||||
- Manifest writes are stage-owned; adapter itself is stateless.
|
||||
|
||||
|
||||
@@ -78,15 +78,15 @@ in-memory backend without changing package state.
|
||||
- Subprocess adapters persist stage logs/generated configs through stage-managed paths.
|
||||
- Shared subprocess execution keeps private lifecycle mechanics separate from
|
||||
Narratio's environment, diagnostics, redaction, and error policy. It starts
|
||||
an owned process group on Linux/macOS or a
|
||||
kill-on-close job object on Windows. Every terminal path disposes of that
|
||||
owned tree before returning. After a natural leader exit, Unix checks for
|
||||
remaining group members and uses bounded graceful then forceful termination;
|
||||
Windows closes the job so kill-on-close applies. Cancellation, deadlines, and
|
||||
an owned process group on Linux and macOS. Every terminal path disposes of
|
||||
that owned tree before returning. After a natural leader exit, the runner checks
|
||||
for remaining group members and uses bounded graceful then forceful termination.
|
||||
Cancellation, deadlines, and
|
||||
diagnostic limits use the same terminal disposal path without losing their
|
||||
original result classification. Child environments contain only the execution
|
||||
baseline and adapter-specified values; configured credentials are explicit
|
||||
sensitive values. Stdout and stderr are redacted while streaming into separate
|
||||
baseline and adapter-specified values; configured credentials are resolved
|
||||
once by ConfigKit and passed explicitly as sensitive values. Adapters do not
|
||||
re-read their named environment references. Stdout and stderr are redacted while streaming into separate
|
||||
8 MiB diagnostic captures; a bounded wait closes a stream retained by a
|
||||
departed leader's descendant. Unsupported platforms reject owned command
|
||||
execution.
|
||||
|
||||
@@ -58,10 +58,12 @@ Configured definitions are always registered. Without an explicit selection,
|
||||
the effective analyze set contains enabled definitions. With `--artifacts`, the
|
||||
exact named configured definitions become the effective set for that invocation,
|
||||
regardless of their `enabled` value. The effective-set resolver itself does not
|
||||
expand dependencies; the analyze work planner closes those targets over their
|
||||
configured prerequisite graph. Availability is separate from executability.
|
||||
Configuration may normalize a family selection into its concrete generated
|
||||
members before this resolver runs. The effective set retains optional family
|
||||
expand dependencies. Configuration has already materialized the sorted union of
|
||||
explicit ordering edges and configured-artifact input edges, and the analyze
|
||||
work planner closes targets over that one concrete prerequisite graph.
|
||||
Availability is separate from executability. Configuration may normalize a
|
||||
family selection into its concrete generated members before this resolver runs.
|
||||
The effective set retains optional family
|
||||
and character origin metadata, but its keys, catalog sources, and runtime
|
||||
lookups remain concrete configured-artifact identities.
|
||||
Family publish policies are likewise expanded into ordinary configured-source
|
||||
|
||||
@@ -8,30 +8,21 @@ pipeline-loading boundary implemented by `internal/config`.
|
||||
|
||||
`LoadPipeline` assembles and validates a pipeline in this order:
|
||||
|
||||
1. Parse the root YAML into a presence-aware composition tree. The tree retains
|
||||
source names, full field paths, node kinds, declaration order, and explicit
|
||||
zero, false, empty-map, and empty-list values.
|
||||
2. Decode the root-only `composition` envelope, validate its declaration shapes
|
||||
and cross-field profile semantics, then clone the root without that envelope
|
||||
as the ordinary pipeline body. The parsed source document is not mutated. A
|
||||
load option retains the difference between omitted and explicitly empty
|
||||
profile selection.
|
||||
3. Open each import relative to the root pipeline directory through the
|
||||
confined regular-file boundary. Imports must use a `.yml` or `.yaml`
|
||||
extension and cannot traverse, use symlinks, repeat a file, import the root,
|
||||
or contain another composition envelope.
|
||||
4. Resolve and structurally parse every declared profile overlay through the
|
||||
same confined regular-file boundary. Missing or malformed unselected
|
||||
overlays fail the load. Overlays cannot contain a composition envelope.
|
||||
5. Additively merge the root body and imports. Distinct map leaves compose;
|
||||
repeated scalar or list paths and node-kind disagreements are conflicts.
|
||||
6. Select exactly one declared profile from an explicit option or the default,
|
||||
then recursively merge its overlay. Overlay leaves replace base leaves,
|
||||
lists are atomic replacements, and null or kind changes fail.
|
||||
7. Emit deterministic canonical YAML and strictly decode it into
|
||||
`PipelineConfig`.
|
||||
8. Apply pipeline defaults once, resolve ordinary relative pipeline paths from
|
||||
the root pipeline file, and digest the normalized effective mapping.
|
||||
1. Give ConfigKit a fresh typed `PipelineConfig` default value, the root file,
|
||||
enabled imports, and the caller's profile selection. Omitted, empty, and
|
||||
exact `default` selections use the base; named profiles use the fixed
|
||||
`profiles/<name>.yml` convention.
|
||||
2. ConfigKit parses the root and declared imports, additively merges base
|
||||
sources, applies only the selected overlay, and strictly decodes the result.
|
||||
3. The definition resolver applies recorded entry-level and inherited defaults,
|
||||
runs deterministic domain normalization, records derived publish destinations,
|
||||
and resolves ordinary paths relative to the root pipeline file.
|
||||
4. The definition validator performs read-only checks that require only the
|
||||
pipeline file. Checks requiring canonical-party family expansion remain at
|
||||
the existing explicit pipeline/campaign resolution boundary.
|
||||
5. Narratio inspects public ConfigKit metadata to reject artifact and family keys
|
||||
declared by multiple base files, adapts source ownership records, attaches
|
||||
runtime-only state, and computes the normalized effective digest.
|
||||
|
||||
This ordering preserves monolithic configuration behavior. Moving a field to
|
||||
an imported fragment changes its source ownership, not its path base, default,
|
||||
@@ -53,25 +44,35 @@ download retain that exact pipeline object and its private provenance. Removing
|
||||
a temporary downloaded session file therefore cannot invalidate the resolved
|
||||
pipeline or campaign context.
|
||||
|
||||
Campaign and session files use independent ConfigKit definitions with imports
|
||||
and profiles disabled. Campaign files normally load directly from their source;
|
||||
session files are first read through the 8 MiB bounded-read boundary because the
|
||||
application must reject unresolved template placeholders before passing an exact
|
||||
byte snapshot to ConfigKit. Local and remotely downloaded session bytes then use
|
||||
the same typed loader and retain the existing post-load identity expectations,
|
||||
stable-input precedence, and source-relative path semantics.
|
||||
|
||||
The application also has a separate read-only inspection resolver for `config
|
||||
validate`, `config show`, and `config sources`. It uses the same production root/profile and
|
||||
validate`, `config show`, `config sources`, and `config artifacts`. It uses the same production root/profile and
|
||||
campaign selection functions, but never routes through session discovery,
|
||||
remote-session download, secret loading, adapter composition, workspace
|
||||
remote-session download, adapter composition, workspace
|
||||
initialization, manifest access, or cleanup. A pipeline with retained artifact
|
||||
family declarations must resolve its selected campaign before ordinary pipeline
|
||||
validation, which expands its canonical-party members and generated publish
|
||||
rules. A pipeline without those declarations may be validated by itself.
|
||||
Secret-directory bootstrap remains invocation-scoped and happens before the
|
||||
ordinary ConfigKit load because explicit typed references must resolve even for
|
||||
inspection.
|
||||
|
||||
`MarshalEffectivePipeline` is the configuration-owned projection for `config
|
||||
show`. It serializes the typed, defaulted effective mapping through the
|
||||
deterministic composition renderer, then removes resolution-only artifact
|
||||
family declarations. The result contains no composition envelope or private
|
||||
provenance fields and has one trailing newline; commands do not marshal runtime
|
||||
objects directly.
|
||||
show`. It serializes one explicit typed safe projection with the existing YAML
|
||||
encoder. The projection has no slots for resolution-only artifact family
|
||||
declarations or private provenance. The result has one trailing newline;
|
||||
commands do not marshal runtime objects directly.
|
||||
|
||||
`EffectivePipelineSources` and `EffectiveCampaignSources` provide the separate
|
||||
safe provenance projection for `config sources`. Pipeline ownership begins with
|
||||
the complete logical field paths retained during composition and classifies
|
||||
the complete logical field paths projected from the schema and classifies
|
||||
each contributor as root, import, profile, or centralized default. The
|
||||
projection replaces generated concrete member paths with paired family and
|
||||
canonical-party records, and does the same for generated publish rules.
|
||||
@@ -79,12 +80,18 @@ Campaign records identify campaign-owned fields and party inputs; canonical
|
||||
derived players point to the party source. The application command only joins these sorted
|
||||
records with selection metadata and never reparses configuration files.
|
||||
|
||||
`config diff` uses a paired profile loader that parses the root, imports, and
|
||||
declared overlays once, then clones the additive base before independently
|
||||
selecting, decoding, defaulting, and finalizing each profile. When campaign
|
||||
`EffectiveArtifactCatalog` projects the same finalized concrete artifact map for
|
||||
`config artifacts`. It derives source identity from each map key, carries sorted
|
||||
effective dependencies, aggregates ConfigKit definition contributors, and adds
|
||||
family/character/party provenance for generated members. It does not inspect
|
||||
files named after artifacts or infer another runtime graph.
|
||||
|
||||
`config diff` uses a thin paired loader that performs two independent ConfigKit
|
||||
loads. When campaign
|
||||
resolution is needed, the command loads one selected campaign and party and
|
||||
expands both effective pipelines from that same party value. The configuration
|
||||
owner projects each normalized effective mapping into sorted logical paths;
|
||||
owner projects each normalized effective mapping directly from that same typed
|
||||
projection into sorted logical paths using JSON number preservation;
|
||||
mapping leaves are compared individually while sequence values remain atomic.
|
||||
Values are compact deterministic JSON representations for command output, not
|
||||
raw YAML fragments, ownership records, or secret material. A differing digest
|
||||
@@ -98,13 +105,16 @@ configured roster authority; player and session-party fields are unsupported.
|
||||
|
||||
## Canonical Party Domain
|
||||
|
||||
`ParseParty` is the package-owned boundary for parsing a party source. It
|
||||
requires a top-level `schema_version` and strictly validates the
|
||||
`narratio.party.v1` contract into ordered character domain values. The
|
||||
canonical value retains a separate exact byte copy of its source so consumers
|
||||
can materialize the authored party document without reserializing it. Its
|
||||
`PlayersYAML` method deterministically derives the versioned players-only
|
||||
projection.
|
||||
`ParseParty` is the package-owned boundary for parsing a party source. Its
|
||||
single-source ConfigKit definition disables imports and profiles, requires
|
||||
`schema_version` through `VersionRule`, and strictly decodes the
|
||||
character-keyed typed schema before applying Narratio's party-domain checks.
|
||||
Successful metadata coordinates recover authored character-map order without a
|
||||
second YAML parse; class and alias slice order comes directly from typed decode.
|
||||
The canonical value retains a separate exact byte copy of its source so
|
||||
consumers can materialize the authored party document without reserializing it.
|
||||
Its `PlayersYAML` method deterministically derives the ID-sorted, versioned
|
||||
players-only projection.
|
||||
|
||||
An unversioned source is rejected before resolution, family expansion, or
|
||||
prepare-stage materialization.
|
||||
@@ -117,58 +127,63 @@ for core pipeline/session rules, external adapters, publish/storage, Notarius,
|
||||
Scriptorium, and shared scalar/path checks. These are package-local ownership
|
||||
boundaries rather than extension points or a general validation framework.
|
||||
|
||||
Validation fails fast and does not define which of several independent invalid
|
||||
fields is reported first. Map-backed declarations are nevertheless traversed
|
||||
deterministically. Notarius validates scalar policy, builds normalized reference
|
||||
bindings and output lanes independently, and installs all candidate values only
|
||||
after every check succeeds. Scriptorium validates artifact declarations and
|
||||
outputs, dependencies and inputs, variables, referenced outputs, and the final
|
||||
Validation is read-only, fails fast, and does not define which of several
|
||||
independent invalid fields is reported first. Map-backed declarations are
|
||||
nevertheless traversed deterministically. Reusable domain normalization is an
|
||||
explicit preceding operation for both loaded and programmatically constructed
|
||||
pipelines. Notarius normalization commits canonical scalar, reference, and output
|
||||
values only after the complete normalization succeeds; its validator then checks
|
||||
that candidate without changing it. Scriptorium validates artifact declarations
|
||||
and outputs, dependencies and inputs, variables, referenced outputs, and the final
|
||||
dependency graph in deterministic artifact order.
|
||||
|
||||
Diagnostics identify the relevant semantic field. Composition conflicts also
|
||||
Diagnostics identify the relevant semantic field. Merge conflicts also
|
||||
retain every claiming source needed to correct an additive ownership conflict.
|
||||
Complete prose, punctuation, formatting, and private validation choreography are
|
||||
not compatibility contracts.
|
||||
|
||||
## Diagnostics And Runtime Metadata
|
||||
|
||||
Syntax, duplicate-key, composition, conflict, and schema failures include the
|
||||
Syntax, duplicate-key, merge, profile, and schema failures include the
|
||||
relevant source name and full field path. Additive conflicts report every
|
||||
claiming source so operators can repair the split without repeatedly
|
||||
rediscovering additional conflicts.
|
||||
|
||||
The loaded pipeline retains private runtime metadata for the absolute root
|
||||
path, ordered imports, selected profile name and selection source, selected
|
||||
overlay, contributing sources, effective digest, and leaf ownership. Base
|
||||
overlay, original ConfigKit metadata and source snapshots, effective digest,
|
||||
and leaf ownership. Base
|
||||
leaves retain their root/import owners, replaced leaves belong to the selected
|
||||
overlay, and centrally supplied values use the synthetic `default` owner. This
|
||||
metadata does not participate in YAML decoding or alter the public
|
||||
configuration model.
|
||||
|
||||
The effective digest is SHA-256 over deterministic canonical YAML produced from
|
||||
the defaulted `PipelineConfig`. Runtime Notarius paths remain absolute for
|
||||
The effective digest is SHA-256 over a versioned deterministic JSON encoding of
|
||||
the same typed safe projection used by show and diff. Runtime Notarius paths remain absolute for
|
||||
execution, but the digest substitutes their normalized logical values captured
|
||||
before root-relative resolution, so relocating an equivalent configuration
|
||||
bundle does not change provenance. Because composition and resolution metadata
|
||||
bundle does not change provenance. Because ConfigKit and resolution metadata
|
||||
are private, the digest excludes source layout, profile name, and ownership.
|
||||
Configuration stores environment variable names rather than resolving raw
|
||||
credentials, so raw secret values are neither loaded nor hashed.
|
||||
Configuration stores credentials as ConfigKit typed references. A successful
|
||||
load resolves their values inside the opaque typed boundary, while this safe
|
||||
projection serializes only `EnvName()` identities. Credential-value changes are
|
||||
therefore never hashed; reference-name changes remain ordinary provenance.
|
||||
`recomputePipelineEffectiveDigest` is the single package-owned refresh point
|
||||
for later runtime expansion.
|
||||
|
||||
## Test Surfaces
|
||||
|
||||
`composition_test.go` protects the presence and merge algebra independently of
|
||||
the public schema. `pipeline_composition_test.go` exercises explicit imports,
|
||||
confinement, conflicts, strict decoding, metadata, and root-relative path
|
||||
behavior through `LoadPipeline`. `pipeline_profiles_test.go` covers selection,
|
||||
all-overlay validation, overlay behavior, provenance, option propagation, and
|
||||
effective-digest stability. Application configuration-loader tests protect the
|
||||
`pipeline_configkit_test.go` exercises real imports and fixed-path profiles,
|
||||
base selection, unselected-overlay isolation, source attribution, and Narratio's
|
||||
unique artifact/family base-declaration rule. Existing focused configuration
|
||||
tests protect defaults, explicit-value semantics, strict decoding, validation,
|
||||
metadata, and root-relative path behavior. Application configuration-loader tests protect the
|
||||
single-read boundary by changing the pipeline file after its initial load and
|
||||
confirming local session resolution retains the original pipeline. Other
|
||||
configuration tests continue to protect defaults and validation after assembly.
|
||||
`party_test.go` protects the versioned party schema, domain invariants, and
|
||||
deterministic players projection without involving campaign or runtime wiring.
|
||||
`party_test.go` protects the versioned typed party schema, domain invariants,
|
||||
authored character order, retained bytes, and deterministic players projection
|
||||
without involving campaign or runtime wiring.
|
||||
`party_resolution_test.go` protects campaign-owned party loading, canonical
|
||||
input restrictions, source provenance, and virtual players input selection.
|
||||
|
||||
|
||||
@@ -37,7 +37,7 @@ failed stage or required manifest save prevents later stages from starting.
|
||||
| --- | --- | --- |
|
||||
| Executable | `cmd/narratio` | Process entry, standard stream wiring, argument handoff, and exit status. |
|
||||
| Application orchestration | `internal/app` | Command dispatch, configuration selection, secret-file environment loading, production composition, session locking, planning, execution, restore, and user-facing reporting. |
|
||||
| Configuration | [`internal/config`](configuration.md) | Presence-aware root/import/profile composition, canonical party and family expansion, strict YAML loading, defaults, normalization, session templating, and validation. |
|
||||
| Configuration | [`internal/config`](configuration.md) | ConfigKit-backed pipeline imports and profiles, canonical party and family expansion, strict YAML loading, defaults, normalization, session templating, and validation. |
|
||||
| Pipeline stages | `internal/stage` | Canonical stage registry, shared stage contract, execution dependencies, and implemented stage behavior. |
|
||||
| External boundaries | `internal/adapters`, `internal/audio` | WhisperX HTTP, downstream subprocesses, notification, object storage, and S3 audio materialization behind Narratio contracts. |
|
||||
| Manifests | `internal/manifest` | Durable session progress, stage transitions, validation, and atomic persistence. |
|
||||
@@ -105,8 +105,8 @@ so any earlier replacement can stale a prior cleanup result.
|
||||
|
||||
## Focused Documentation
|
||||
|
||||
- [Configuration Internals](configuration.md): pipeline composition, import
|
||||
confinement, field ownership, decoding, and root-relative path semantics.
|
||||
- [Configuration Internals](configuration.md): pipeline imports and profiles,
|
||||
field ownership, decoding, and root-relative path semantics.
|
||||
- [Adapter Internals](adapters.md): external adapter boundaries, composition,
|
||||
failure behavior, and test surfaces.
|
||||
- [Artifact Internals](artifacts.md): source identities, runtime catalog,
|
||||
|
||||
@@ -10,7 +10,9 @@ dependency order, and safely materialize validated outputs.
|
||||
- ordinary configured artifacts from `pipeline.scriptorium.artifacts`; canonical
|
||||
party artifact families have already expanded into this map during
|
||||
configuration resolution, including corresponding member dependencies and
|
||||
rewritten member-artifact input sources
|
||||
rewritten member-artifact input sources. Each artifact's dependency list is
|
||||
already the normalized union of explicit ordering edges and configured-artifact
|
||||
input edges.
|
||||
- optional selected artifact keys supplied through the stage environment
|
||||
- built-in, configured, extraction, and previous-session source references in
|
||||
artifact inputs
|
||||
@@ -51,6 +53,9 @@ Supported source families:
|
||||
manifest record and durable output pass the configured-artifact evidence
|
||||
contract; an incidental or stale canonical file is unavailable.
|
||||
- validates selected artifact dependency order (cycle-safe topo ordering).
|
||||
- consumes only the concrete dependency graph finalized by configuration;
|
||||
planning, prerequisite closure, execution, reuse, invalidation, and semantic
|
||||
evidence do not infer a second graph from inputs.
|
||||
- resolves required/optional inputs per artifact source definition into an
|
||||
ordered semantic identity. Each identity records the configured input name,
|
||||
canonical source ID, required policy, explicit presence, source contract,
|
||||
|
||||
@@ -35,8 +35,9 @@ validation model, module set, transcript description, output schema, selected
|
||||
external configuration path, and canonical polished-transcript identity. Module
|
||||
ordering is normalized because the configured modules form a set. Audita's
|
||||
executable, timeouts, concurrency, report and debug behavior, work retention,
|
||||
and credential environment name are operational and do not invalidate a
|
||||
successful result.
|
||||
and resolved credential value are operational and do not invalidate a
|
||||
successful result. The public credential reference name may participate in
|
||||
configuration provenance but never exposes its resolved value.
|
||||
|
||||
Narratio can fingerprint a selected model, module, or configuration identifier,
|
||||
but it cannot inspect content that Audita privately resolves behind that stable
|
||||
|
||||
@@ -76,12 +76,15 @@ value from the last value persisted in the session manifest, which helps
|
||||
diagnose profile switches without changing resume authority.
|
||||
|
||||
Before switching an operational profile, compare its effective meaning with the
|
||||
current selection through `narratio config diff <left-profile> <right-profile>`.
|
||||
production base through `narratio config diff default <profile>`.
|
||||
The command is read-only and succeeds whether it finds differences or not. Its
|
||||
sorted records describe defaulted, expanded concrete configuration—not source
|
||||
file layout—so it can be used to review model, artifact, and publish changes
|
||||
without creating a session or run. Select the same campaign explicitly when
|
||||
profiles could resolve different campaign paths; see the [CLI reference](cli.md#config-validate-config-show-config-sources-and-config-diff) for syntax and record format.
|
||||
without creating a session or run. Use `narratio config artifacts` to review the
|
||||
concrete artifact identities, definitions, outputs, and dependencies. Select the
|
||||
same campaign explicitly when profiles could resolve different campaign paths;
|
||||
see the [CLI reference](cli.md#config-validate-config-show-config-sources-config-artifacts-and-config-diff)
|
||||
for syntax and record format.
|
||||
|
||||
## Stage Execution and Continuation Behavior
|
||||
|
||||
@@ -550,18 +553,28 @@ converge on `0664`. Narratio explicitly applies these modes so a restrictive
|
||||
caller umask does not remove group write or setgid. It does not change file or
|
||||
directory ownership: the configured workspace's existing group is inherited.
|
||||
|
||||
Windows does not implement POSIX mode bits or setgid semantics. Configure the
|
||||
workspace, spool, and cache locations with an ACL that grants the collaborating
|
||||
group read/write access, and configure credential locations with an ACL limited
|
||||
to the intended credential owner. Do not use POSIX mode displays as evidence of
|
||||
Windows access control.
|
||||
|
||||
API keys are credentials, not ordinary workspace data. Store them outside the
|
||||
shared workspace or in a separately restricted credential location; ordinary
|
||||
workspace group access must never be treated as authorization to read keys.
|
||||
On POSIX, provision a credential directory as `0700` and credential files as
|
||||
`0600`; Narratio rejects group- or other-readable configured credential paths.
|
||||
On Windows, restrict the directory and files with ACLs to the credential owner.
|
||||
Provision a credential directory as `0700` and credential files as `0600`;
|
||||
these permission modes are an operator requirement and are not enforced by
|
||||
ConfigKit.
|
||||
|
||||
Every pipeline-loading command bootstraps credentials once, before the first
|
||||
configuration load and before any adapter is constructed. `--secrets-dir`
|
||||
selects an explicit directory (relative paths use caller CWD). Otherwise a
|
||||
present `secrets/` beside the resolved root pipeline is used, while an absent
|
||||
conventional directory is skipped. Explicit missing/empty selections and
|
||||
invalid existing directories fail the command. ConfigKit follows a symlink used
|
||||
as the selected directory, but immediate entries must be regular non-symlink
|
||||
files. Existing environment variables—including empty values—always win.
|
||||
|
||||
S3 `access_key_id` and `secret_access_key` are an optional pair of typed
|
||||
environment references. When both are absent, the AWS SDK retains its default
|
||||
credential provider chain. When both are configured, Narratio supplies their
|
||||
load-time resolved values through an explicit static provider. Audita's optional
|
||||
`llm_api_key` is likewise captured at pipeline load and supplied only through
|
||||
the child process's `AUDITA_LLM_API_KEY` override.
|
||||
|
||||
External adapter results are individually bounded before Narratio validates or
|
||||
materializes them. These per-file limits do not reserve disk space: prevent hard
|
||||
@@ -624,4 +637,5 @@ normal migration path.
|
||||
- Publish requires prerequisite stages through `render` and `analyze` to be succeeded.
|
||||
- Markdown publish defaults require render outputs (`transcripts/final.md` and `transcripts/final.trimmed.md`).
|
||||
- Restore requires configured object storage and committed remote current state.
|
||||
- Storage-backed commands load filesystem secrets before object-store initialization.
|
||||
- Pipeline-loading commands bootstrap filesystem secrets once before configuration
|
||||
resolution and object-store initialization.
|
||||
|
||||
@@ -146,7 +146,7 @@ Configuration is strict, explicit, centralized, and operator-oriented.
|
||||
- YAML decoding rejects unknown fields.
|
||||
- Defaults are centralized and testable.
|
||||
- Empty configured values do not silently replace meaningful defaults.
|
||||
- Validation rejects invalid composition before stage execution where
|
||||
- Validation rejects invalid configuration before stage execution where
|
||||
practical.
|
||||
- Root-owned imports and one selected profile resolve deterministically through
|
||||
the configuration owner; commands do not implement their own merge rules.
|
||||
@@ -234,20 +234,26 @@ configuration, logs, reports, diagnostics, and Notarius artifacts—is
|
||||
intentionally shareable with the configured workspace group. API-key material
|
||||
is sensitive and is not covered by the ordinary workspace-sharing policy.
|
||||
|
||||
On POSIX systems, Narratio-created ordinary workspace directories converge on
|
||||
Narratio supports Linux on amd64 and arm64. macOS arm64 is best effort: it has
|
||||
a cross-build gate and release asset, but no native runtime release gate.
|
||||
Windows and macOS amd64 are unsupported and have no compatibility promise or
|
||||
release asset.
|
||||
|
||||
On supported systems, Narratio-created ordinary workspace directories converge on
|
||||
setgid `02775` and ordinary workspace files on `0664`, even when the caller's
|
||||
umask is restrictive. This preserves the existing workspace group for nested
|
||||
creation and atomic replacements without changing ownership. API-key storage
|
||||
uses a separate restrictive contract. On Windows, POSIX mode bits and setgid
|
||||
are not authoritative; operators must provide the equivalent shared-group and
|
||||
credential-restricted ACLs described in [Operations](../operations.md#workspace-permissions).
|
||||
uses a separate restrictive contract.
|
||||
|
||||
Raw secrets must not be stored in pipeline, campaign, or session YAML or written
|
||||
to manifests, logs, generated configuration, reports, publish metadata,
|
||||
documentation, or examples. Secrets enter through configured environment
|
||||
variable names or secret-file references. Diagnostics should avoid transcript
|
||||
and prompt content unless a deliberate, bounded inspection mechanism requires
|
||||
it.
|
||||
documentation, or examples. Pipeline configuration names environment variables;
|
||||
command-owned startup may populate previously unset variables from ConfigKit's
|
||||
explicit or conventional secrets directory. This bootstrap occurs once per
|
||||
invocation before any typed configuration load or adapter construction and is
|
||||
not hidden inside library loaders. Safe bootstrap diagnostics may identify
|
||||
files and statuses but never values. Diagnostics should avoid transcript and
|
||||
prompt content unless a deliberate, bounded inspection mechanism requires it.
|
||||
|
||||
Logs, reports, generated invocation files, generated configuration, and render
|
||||
debug files are diagnostics, not canonical pipeline products. They should be
|
||||
|
||||
@@ -58,14 +58,14 @@ scripts/check-release-candidate.sh vMAJOR.MINOR.PATCH
|
||||
|
||||
It validates the version and matching note, module hygiene, formatting,
|
||||
whitespace, uncached tests, race tests, static checks, documentation, examples,
|
||||
and six official cross-build assets. It uses `GOWORK=off`, does not contact
|
||||
and three official cross-build assets. It uses `GOWORK=off`, does not contact
|
||||
application services, CI, or Gitea, and does not create tags or modify tracked
|
||||
source. Fix any failure on `main`, commit it, push it normally, and rerun the
|
||||
checker.
|
||||
|
||||
The checker builds Linux, macOS, and Windows assets for `amd64` and `arm64`.
|
||||
Cross-builds prove compilation; they are not native macOS or Windows runtime
|
||||
evidence.
|
||||
The checker builds Linux assets for `amd64` and `arm64` and a macOS asset for
|
||||
`arm64`. The macOS cross-build proves compilation; it is not native macOS
|
||||
runtime evidence.
|
||||
|
||||
## Publish the tag
|
||||
|
||||
|
||||
@@ -22,5 +22,7 @@ See the [release procedure](../release.md) for creating a candidate and tag.
|
||||
When asynchronous publication succeeds, the corresponding Gitea release is the
|
||||
canonical source for downloadable binaries and checksums.
|
||||
|
||||
- [v2.0.0](v2.0.0.md)
|
||||
- [v1.7.0](v1.7.0.md)
|
||||
- [v1.6.0](v1.6.0.md)
|
||||
- [v1.5.0](v1.5.0.md)
|
||||
|
||||
94
docs/releases/v2.0.0.md
Normal file
94
docs/releases/v2.0.0.md
Normal file
@@ -0,0 +1,94 @@
|
||||
# Narratio v2.0.0
|
||||
|
||||
Narratio v2.0.0 replaces its configuration infrastructure with ConfigKit,
|
||||
simplifies the supported release platforms, and makes artifact definitions and
|
||||
their effective dependency graph directly inspectable.
|
||||
|
||||
## Summary
|
||||
|
||||
Pipeline roots now use ConfigKit v0.1.2 for strict YAML loading, explicit
|
||||
imports, fixed-path named profiles, provenance, defaults, and typed credential
|
||||
references. The recommended production layout keeps production values in the
|
||||
base, defines each ordinary artifact and artifact family in its own imported
|
||||
file, and keeps testing changes in `profiles/testing.yml`.
|
||||
|
||||
Configured-artifact inputs now create their dependency edges automatically.
|
||||
`narratio config artifacts` reports the finalized concrete catalog, including
|
||||
generated party-family members, without loading a session or initializing
|
||||
runtime adapters.
|
||||
|
||||
## Compatibility
|
||||
|
||||
This is a breaking configuration and platform release. There is no runtime
|
||||
fallback for the retired Narratio composition/profile catalog, `pipeline.secrets`
|
||||
or `secrets.env_dir`, credential `_env` fields, or the previous YAML
|
||||
presence/default behavior. Defaults apply only to absent values; explicit empty,
|
||||
zero, false, and null values retain their typed meaning and may fail Narratio
|
||||
domain validation.
|
||||
|
||||
The maintained binaries are Linux amd64, Linux arm64, and macOS arm64. Windows
|
||||
and macOS amd64 builds are no longer produced or supported. Existing v1 release
|
||||
notes remain historical descriptions of their releases.
|
||||
|
||||
Inspection commands still avoid session, manifest, workspace, adapter, and
|
||||
remote-service activity, but they are not credential-free loaders: every
|
||||
explicit typed credential reference in the selected effective configuration must
|
||||
resolve; references replaced or removed by that profile need not resolve.
|
||||
Pipelines containing artifact families also require a campaign so the canonical
|
||||
party can be expanded.
|
||||
`config show` is a secret-safe review projection, not a promised reloadable
|
||||
authoring file.
|
||||
|
||||
## Upgrade
|
||||
|
||||
Migrate each operational bundle before running v2:
|
||||
|
||||
1. Replace old composition syntax with root-owned
|
||||
[`configkit.imports`](../config.md#pipeline-imports-and-profiles). Imports are
|
||||
additive base sources; they do not override one another.
|
||||
2. Fold the old production overlay into the base pipeline. Move testing changes
|
||||
to `profiles/testing.yml`; compare the two independent resolutions with
|
||||
`narratio config diff default testing`.
|
||||
3. Follow the maintained
|
||||
[`examples/production-testing`](../../examples/production-testing/pipeline.yml)
|
||||
layout: one explicit artifact entry per `artifacts/<key>.yml` and one family
|
||||
entry per `artifact-families/<key>.yml`. The map key, not the filename, is the
|
||||
stable artifact identity.
|
||||
4. Move credential files to an untracked `secrets/` directory beside
|
||||
`pipeline.yml`, or pass `--secrets-dir`. Rename S3 fields to `access_key_id`
|
||||
and `secret_access_key`, and rename the Audita field to `llm_api_key`.
|
||||
5. For static S3 credentials, configure both S3 references (commonly
|
||||
`OBJECT_STORAGE_KEY_ID` and `OBJECT_STORAGE_KEY`) and make them resolvable.
|
||||
To use instance roles, workload identity, shared SDK files, or another AWS SDK
|
||||
discovery provider, omit both references. A partial pair is invalid.
|
||||
6. Remove a `depends_on` entry when it only duplicates a
|
||||
`narratio.artifact.<key>` input. Keep explicit entries that express ordering
|
||||
without supplying the upstream output as an input.
|
||||
7. Review ConfigKit's stricter explicit-value/null semantics and correct values
|
||||
that previously acquired defaults after being explicitly empty. Do not use
|
||||
`config show` output as a source migration mechanism.
|
||||
8. Run `config validate`, `config show`, `config sources`, and `config artifacts`
|
||||
with the same profile, campaign, and secrets selection intended for runtime.
|
||||
Resolve the campaign-owned party whenever families are present.
|
||||
9. Update deployment and download automation to one of the three maintained
|
||||
platform assets before removing the v1 binary.
|
||||
|
||||
The [configuration reference](../config.md), [CLI reference](../cli.md), and
|
||||
[maintained examples](../../examples/README.md) are the canonical detailed
|
||||
guides.
|
||||
|
||||
## Changes
|
||||
|
||||
- Adopted ConfigKit v0.1.2 across pipeline, campaign, session, and party loading
|
||||
while keeping Narratio domain validation and post-campaign expansion explicit.
|
||||
- Added ConfigKit imports and fixed `profiles/<name>.yml` pipeline overlays,
|
||||
unified safe effective show/diff/digest projection, and source provenance.
|
||||
- Added invocation-scoped ConfigKit secrets-directory bootstrap and typed S3 and
|
||||
Audita credential references without exposing resolved values.
|
||||
- Inferred, normalized, and validated configured-artifact input dependencies as
|
||||
part of the one concrete graph consumed by analyze planning and resume logic.
|
||||
- Added `config artifacts` with stable concrete identities, definition sources,
|
||||
effective dependencies, and generated family/party provenance.
|
||||
- Added the maintained production-base/testing-overlay artifact authoring layout.
|
||||
- Reduced release assets and active platform support to Linux amd64/arm64 and
|
||||
macOS arm64.
|
||||
375
docs/roadmap/configkit-adoption.md
Normal file
375
docs/roadmap/configkit-adoption.md
Normal file
@@ -0,0 +1,375 @@
|
||||
# ConfigKit Adoption and Platform Simplification
|
||||
|
||||
**Status:** Accepted; implementation pending
|
||||
|
||||
## Purpose
|
||||
|
||||
Narratio currently owns a substantial YAML configuration subsystem for strict
|
||||
decoding, explicit imports, additive composition, profile overlays, presence
|
||||
tracking, provenance, and effective-value inspection. ConfigKit was extracted
|
||||
from this implementation and now provides those generic responsibilities as a
|
||||
maintained library.
|
||||
|
||||
Narratio 2.0 will adopt ConfigKit as its configuration foundation, remove the
|
||||
superseded generic infrastructure, and retain only Narratio-specific configuration
|
||||
policy and domain resolution. The same major release will narrow the supported
|
||||
platform set to the systems that Narratio is expected to run on in practice.
|
||||
|
||||
This roadmap defines the desired contracts and end state. The
|
||||
[implementation plan](implementation.md) owns sequencing and completion gates.
|
||||
|
||||
## Goals
|
||||
|
||||
- Reduce Narratio-owned configuration code and tests by delegating generic YAML
|
||||
mechanics to ConfigKit.
|
||||
- Establish one strict and explainable configuration lifecycle rather than
|
||||
retaining parallel or compatibility loaders.
|
||||
- Preserve Narratio's application-specific defaults, validation, campaign/session
|
||||
resolution, party handling, artifact-family expansion, digests, and CLI policy.
|
||||
- Make individual artifact definitions easy to organize, identify, inspect, and
|
||||
reuse as dependencies of later artifacts.
|
||||
- Keep raw credentials out of YAML and adopt a predictable secrets-directory
|
||||
convention that works before configuration loading.
|
||||
- Remove unsupported platform code and release obligations that provide no known
|
||||
user value.
|
||||
- Deliver the resulting contract changes as Narratio v2.0.0 with a clear migration
|
||||
guide.
|
||||
|
||||
## Platform Contract
|
||||
|
||||
Narratio's v2 platform matrix is:
|
||||
|
||||
| Platform | Status | Required evidence and deliverables |
|
||||
| --- | --- | --- |
|
||||
| Linux amd64 | Primary, supported | Default tests, race checks, static analysis, build validation, and a release asset |
|
||||
| Linux arm64 | Primary, supported | Cross-build validation and a release asset; native validation is used when an appropriate runner is available |
|
||||
| macOS arm64 | Best effort | Cross-build validation and a release asset; native CI and platform-equivalent runtime guarantees are not required release gates |
|
||||
| macOS amd64 | Unsupported | No build gate, release asset, compatibility promise, or platform-specific maintenance |
|
||||
| Windows | Unsupported | No build gate, release asset, compatibility promise, or platform-specific maintenance |
|
||||
|
||||
Windows- and Intel-Mac-specific code, tests, CI entries, release packaging, and
|
||||
documentation are removed when they no longer serve a supported build. Portable
|
||||
code should remain portable where doing so is natural, but accidental compilation
|
||||
on an unsupported target does not create a support commitment.
|
||||
|
||||
Best-effort macOS arm64 support means Narratio intends to keep the target buildable
|
||||
and will consider reported defects when practical. Lack of a native runner or an
|
||||
unverified platform-specific behavior does not block a release. A target for which
|
||||
Narratio publishes an asset must still pass its configured cross-build gate.
|
||||
|
||||
## ConfigKit Dependency and Ownership
|
||||
|
||||
Narratio will depend on the tagged ConfigKit module at an exact reviewed version,
|
||||
initially `gitea.maximumdirect.net/eric/ConfigKit@v0.1.1`. ConfigKit is pre-1.0 and
|
||||
may make compatibility changes in later minor versions, so dependency upgrades are
|
||||
intentional changes accompanied by contract review and focused Narratio integration
|
||||
tests; Narratio does not follow an unpinned branch or automatically accept a newer
|
||||
minor release.
|
||||
|
||||
ConfigKit owns:
|
||||
|
||||
- bounded file, byte, and reader input;
|
||||
- the accepted YAML syntax and strict typed decoding;
|
||||
- import declarations, traversal, containment, and resource limits;
|
||||
- additive composition and conflict attribution;
|
||||
- selected-profile overlay semantics;
|
||||
- explicit input presence and source provenance;
|
||||
- application-default and recorded-resolution coordination;
|
||||
- structured configuration diagnostics;
|
||||
- secret-reference parsing and resolution;
|
||||
- secrets-directory loading and its safe report; and
|
||||
- detached effective configuration inspection supplied by its public result.
|
||||
|
||||
Narratio owns:
|
||||
|
||||
- pipeline, campaign, session, party, and runtime domain types;
|
||||
- configuration-file discovery and CLI selection;
|
||||
- concrete defaults, normalization, and semantic validation;
|
||||
- campaign/session identity and stable-input resolution;
|
||||
- canonical party parsing requirements and authored-byte retention;
|
||||
- character artifact-family expansion and generated ownership attribution;
|
||||
- root-relative interpretation of Narratio-owned runtime paths;
|
||||
- effective pipeline digests and semantic profile diffs;
|
||||
- command output and exit behavior; and
|
||||
- adapter and runtime initialization after configuration succeeds.
|
||||
|
||||
Narratio must use ConfigKit through its public package. It must not import
|
||||
ConfigKit's private YAML implementation or reproduce ConfigKit's parser,
|
||||
composition, profile, provenance, or secret-bootstrap mechanics locally.
|
||||
|
||||
## Configuration Format
|
||||
|
||||
Narratio adopts ConfigKit's configuration conventions directly. The old Narratio
|
||||
`composition:` envelope, declared profile catalog, arbitrary overlay paths, and
|
||||
`composition.default_profile` are removed without a compatibility parser.
|
||||
|
||||
The root pipeline uses `configkit.imports` declarations. Import paths, optional
|
||||
imports, globs, recursion, ordering, composition conflicts, containment, and limits
|
||||
have the meanings defined by the pinned ConfigKit contract. Narratio does not add a
|
||||
second import syntax or narrower application-specific traversal implementation.
|
||||
|
||||
The unoverlaid root and its imports are the production configuration. A named
|
||||
profile is a ConfigKit overlay at `profiles/<name>.yml`; in particular,
|
||||
`profiles/testing.yml` contains inexpensive-model overrides and testing-only
|
||||
artifacts. Omitting `--profile`, or selecting ConfigKit's `default` profile, uses the
|
||||
production base. `--profile testing` applies the testing overlay. Narratio does not
|
||||
maintain a separate configured default-profile mechanism or validate unselected
|
||||
profile files.
|
||||
|
||||
ConfigKit's rules for explicit false, zero, empty, and null values replace
|
||||
Narratio's custom presence tracking. Application defaults and normalization use a
|
||||
ConfigKit definition and recorded resolution where the final value depends on
|
||||
whether input was present. `NormalizeConfig.UnmarshalYAML` and equivalent
|
||||
parser-coupled application hooks are removed.
|
||||
|
||||
Pipeline, campaign, session, and party inputs use the common ConfigKit loading
|
||||
boundary where its schema model can represent their types. Narratio may perform
|
||||
narrow pre- or post-load domain work, such as retaining authored party bytes,
|
||||
rejecting unresolved session-template placeholders, resolving stable inputs, and
|
||||
expanding artifact families. Such work must not recreate a second generic YAML
|
||||
loader.
|
||||
|
||||
## Artifact Definition Ergonomics
|
||||
|
||||
The canonical v2 configuration layout defines each ordinary Scriptorium artifact
|
||||
in its own file under `artifacts/`. The root pipeline imports the directory through
|
||||
ConfigKit, for example:
|
||||
|
||||
```yaml
|
||||
configkit:
|
||||
imports:
|
||||
- glob: artifacts/*.yml
|
||||
```
|
||||
|
||||
Each file contributes one ordinary entry at
|
||||
`scriptorium.artifacts.<artifact_key>`. The map key is the authoritative Narratio
|
||||
artifact key and therefore makes the artifact's internal identity explicit in the
|
||||
definition itself:
|
||||
|
||||
```yaml
|
||||
# artifacts/session_recap.yml
|
||||
scriptorium:
|
||||
artifacts:
|
||||
session_recap:
|
||||
enabled: true
|
||||
prompt_id: dnd.session_recap
|
||||
output_path: artifacts/session_recap.md
|
||||
```
|
||||
|
||||
The canonical reusable source ID is derived without another configured identifier:
|
||||
`session_recap` becomes `narratio.artifact.session_recap`. The recommended filename
|
||||
is `<artifact_key>.yml`, but the filename is organizational rather than an identity
|
||||
source. Renaming or moving a file without changing its map key does not change the
|
||||
artifact contract. ConfigKit additive composition rejects duplicate artifact keys.
|
||||
|
||||
The one-artifact-per-file layout is the documented happy path and is used by
|
||||
maintained examples. Narratio does not prohibit an inline artifact or another
|
||||
ConfigKit-compatible source organization merely because it differs from that
|
||||
layout. Artifact identity always comes from the effective map key, never implicitly
|
||||
from a filename or import position.
|
||||
|
||||
Artifact families use the corresponding `artifact-families/*.yml` convention, with
|
||||
one `scriptorium.artifact_families.<family_key>` entry per file. The family key is
|
||||
explicit. Concrete member keys and source IDs continue to use Narratio's existing
|
||||
deterministic family/member naming rule after canonical party resolution.
|
||||
|
||||
Narratio adds a read-only `narratio config artifacts` command using the ordinary
|
||||
pipeline, profile, and campaign selectors. It reports every effective ordinary
|
||||
artifact's key, canonical `narratio.artifact.<key>` source ID, defining source,
|
||||
enabled state, output path, and effective dependencies. Generated family members
|
||||
also report their family origin. Output ordering is deterministic by artifact key,
|
||||
and the command does not require a session or initialize runtime adapters.
|
||||
|
||||
A configured-artifact input creates an implicit dependency on the referenced
|
||||
artifact. For example:
|
||||
|
||||
```yaml
|
||||
inputs:
|
||||
recap:
|
||||
source: narratio.artifact.session_recap
|
||||
```
|
||||
|
||||
The effective dependency graph is the union of explicit `depends_on` entries and
|
||||
dependencies inferred from `narratio.artifact.<key>` inputs. Inferred and explicit
|
||||
duplicates collapse to one dependency. `depends_on` remains available for an
|
||||
ordering requirement when the upstream output is not supplied as an input.
|
||||
Validation applies the same existence, self-reference, and cycle rules to both
|
||||
kinds of dependency, and diagnostics identify the declaration or input that
|
||||
introduced an invalid edge. Operators no longer need to repeat an artifact input in
|
||||
`depends_on` merely to make that input valid.
|
||||
|
||||
## Effective Inspection, Digests, and Profile Diff
|
||||
|
||||
ConfigKit results are the authority for composed input presence, base/profile
|
||||
origins, diagnostics, and the initial typed effective configuration. Narratio's
|
||||
configuration commands adapt those results to existing application-facing command
|
||||
formats rather than exposing ConfigKit implementation types directly.
|
||||
|
||||
Party-dependent artifact families are expanded only after the selected campaign and
|
||||
canonical party are known. That derived state is intentionally outside ConfigKit's
|
||||
completed load snapshot. Narratio retains a narrow application-owned layer that:
|
||||
|
||||
- records generated family/member and publish-rule attribution;
|
||||
- projects the final post-expansion pipeline for operator inspection;
|
||||
- recomputes the effective pipeline digest after expansion; and
|
||||
- ensures stages and adapters receive only the final concrete configuration.
|
||||
|
||||
Narratio continues to own semantic `config diff` behavior because ConfigKit does
|
||||
not provide configuration diffs or fingerprints. Each side of a profile comparison
|
||||
may be loaded independently through ConfigKit. The former guarantee that both sides
|
||||
come from one parsed source snapshot is removed; operators must not rely on a
|
||||
meaningful comparison while concurrently modifying its source files. Equality is
|
||||
still based on normalized effective values rather than YAML formatting or source
|
||||
layout.
|
||||
|
||||
## Credential and Secrets Convention
|
||||
|
||||
Raw API keys and other credential values remain prohibited in pipeline, campaign,
|
||||
session, generated configuration, manifests, diagnostics, examples, and logs.
|
||||
Credential-bearing configuration fields use ConfigKit secret references: YAML
|
||||
contains an environment-variable name, while the resolved value remains available
|
||||
only through ConfigKit's trusted typed boundary. Fields should use credential names
|
||||
rather than legacy names ending in `_env` when the v2 schema is updated.
|
||||
|
||||
The `pipeline.secrets.env_dir` stanza is removed. Narratio selects a secrets
|
||||
directory before parsing the pipeline using this precedence:
|
||||
|
||||
1. an explicitly supplied `--secrets-dir PATH`; otherwise
|
||||
2. the `secrets` subdirectory beside the selected root pipeline file.
|
||||
|
||||
The conventional directory is derived from the resolved root pipeline path, never
|
||||
from the current working directory, an imported file, or a selected profile. A
|
||||
relative explicit CLI path is resolved from the caller's current working directory,
|
||||
consistent with an ordinary command-line path argument.
|
||||
|
||||
If an explicitly selected directory is missing or invalid, the command fails with a
|
||||
clear bootstrap diagnostic. If the conventional directory is absent, Narratio
|
||||
performs no directory bootstrap and continues; pre-existing environment variables
|
||||
may satisfy all references. If the conventional directory exists but ConfigKit
|
||||
rejects or cannot read it, the command fails. Already-set environment variables,
|
||||
including empty values, retain ConfigKit's documented precedence.
|
||||
|
||||
Narratio explicitly calls `configkit.LoadSecretsDir` at most once for a command,
|
||||
before the first ConfigKit load that may resolve secret references and before any
|
||||
concurrent work or runtime client initialization. This application-level convention
|
||||
does not make ConfigKit's ordinary `Load` mutate process-global state. The bootstrap
|
||||
report is rendered or summarized through a safe Narratio command boundary without
|
||||
secret values.
|
||||
|
||||
Narratio adopts ConfigKit's secret-directory rules and limits without restoring its
|
||||
old permission-mode, selected-directory-symlink, invalid-entry, byte-limit, or
|
||||
newline-trimming behavior through compatibility wrappers. Configuration inspection
|
||||
may require the referenced environment variables or the selected secrets directory
|
||||
because an ordinary successful ConfigKit load resolves effective typed secret
|
||||
references. Inspection output may show non-secret reference names where appropriate,
|
||||
but must never expose resolved values.
|
||||
|
||||
## Validation and Diagnostics
|
||||
|
||||
Narratio's semantic rules remain recognizable application code registered with the
|
||||
appropriate ConfigKit definition. Validation continues to reject invalid pipeline,
|
||||
campaign, session, party, adapter, publish, and artifact relationships before
|
||||
runtime work where practical. Validation order and exact prose are not compatibility
|
||||
contracts.
|
||||
|
||||
ConfigKit diagnostic codes, paths, and source locations should be preserved through
|
||||
the application error boundary where useful. Narratio adds command context and exit
|
||||
classification but does not parse ConfigKit error strings or expose raw parser
|
||||
errors. Secret values must remain absent from success and failure diagnostics.
|
||||
|
||||
## Compatibility and Migration
|
||||
|
||||
This work intentionally changes supported platforms, pipeline composition syntax,
|
||||
profile layout and selection, secret configuration, YAML semantics, diagnostics,
|
||||
and some inspection prerequisites. It is released as Narratio v2.0.0.
|
||||
|
||||
There is no runtime compatibility layer for the old `composition` or
|
||||
`pipeline.secrets` syntax. Maintained examples are converted to the ConfigKit
|
||||
format, and the v2 release documentation provides a concise migration guide that
|
||||
covers:
|
||||
|
||||
- converting imports to `configkit.imports` declarations;
|
||||
- folding the production overlay into the base configuration;
|
||||
- placing testing overrides in `profiles/testing.yml`;
|
||||
- moving ordinary artifacts and artifact families into the recommended individual
|
||||
definition files and using their map keys as canonical identities;
|
||||
- removing redundant `depends_on` entries when a configured-artifact input now
|
||||
supplies the same dependency;
|
||||
- moving secret files to the conventional sibling `secrets/` directory or using
|
||||
`--secrets-dir`;
|
||||
- renaming credential-reference fields where applicable;
|
||||
- ConfigKit's stricter YAML and explicit-value semantics;
|
||||
- the independent-load behavior of `config diff`; and
|
||||
- the removal of Windows and macOS amd64 artifacts.
|
||||
|
||||
Obsolete application implementation, tests, documentation, examples, CI paths, and
|
||||
release machinery are removed rather than left deprecated inside the v2 codebase.
|
||||
|
||||
## Required Changes
|
||||
|
||||
- Add and pin the ConfigKit module dependency and establish a small application
|
||||
integration boundary for definitions, result adaptation, and diagnostics.
|
||||
- Replace Narratio's generic YAML parsing, composition, import, profile, presence,
|
||||
and provenance implementation with ConfigKit public APIs.
|
||||
- Express existing defaults, presence-sensitive normalization, and semantic
|
||||
validation through ConfigKit definitions without moving Narratio domain policy
|
||||
into the library.
|
||||
- Adapt pipeline/campaign/session/party loading and preserve the existing
|
||||
application resolution steps that occur after their independent typed loads.
|
||||
- Rework effective configuration, sources, and diff commands around ConfigKit
|
||||
results plus Narratio's post-party generated-attribution layer.
|
||||
- Add the maintained per-artifact and per-family file layouts, the read-only
|
||||
`config artifacts` catalog, and effective dependency inference from configured
|
||||
artifact inputs.
|
||||
- Convert credential configuration to typed references, add the common
|
||||
`--secrets-dir` option, implement conventional-directory selection, and replace
|
||||
the custom API-key directory loader with `configkit.LoadSecretsDir`.
|
||||
- Update adapters to consume resolved credentials or safe environment reference
|
||||
names from typed configuration without independently rediscovering secret values.
|
||||
- Remove Windows and macOS amd64 build targets, release assets, CI entries,
|
||||
platform-specific implementation, and obsolete tests and documentation.
|
||||
- Convert maintained examples and add v2 migration and release documentation.
|
||||
- Retain focused Narratio tests for domain policy and assembled application
|
||||
behavior while deleting parser/composer/bootstrap mechanism tests now owned by
|
||||
ConfigKit. Tests remain deterministic, offline, credential-free, and aligned with
|
||||
the repository testing policy.
|
||||
- Update canonical architecture, configuration, CLI, operations, development,
|
||||
internal, troubleshooting, and release documentation only as the corresponding
|
||||
behavior is implemented.
|
||||
|
||||
## Target End State
|
||||
|
||||
Narratio v2.0.0 has one configuration infrastructure dependency and no duplicate
|
||||
YAML composition subsystem. Operators maintain a production base configuration,
|
||||
optional explicit imports, and fixed named overlays such as `testing`. ConfigKit
|
||||
provides strict parsing, composition, typed loading, provenance, diagnostics, and
|
||||
credential resolution; Narratio supplies domain meaning and its post-campaign
|
||||
artifact expansion.
|
||||
|
||||
Operators can add an artifact by creating one discoverable YAML file whose map key
|
||||
is its stable Narratio identity. `config artifacts` exposes that key, its canonical
|
||||
source ID, origin, output, and effective dependency set. A later artifact can reuse
|
||||
the output through `narratio.artifact.<key>` without repeating the same edge in
|
||||
`depends_on`, while explicit ordering-only dependencies remain supported.
|
||||
|
||||
Secrets normally live in a predictable `secrets/` directory beside the selected
|
||||
pipeline, with `--secrets-dir` available for deployment-specific placement. Raw
|
||||
credential values never enter YAML or inspection output. The bootstrap is explicit
|
||||
in application startup and ordinary ConfigKit loading remains free of process-global
|
||||
mutation.
|
||||
|
||||
Release and CI maintenance covers Linux amd64 and arm64 as the primary platforms
|
||||
and macOS arm64 on a best-effort basis. Windows and macOS amd64 carry no remaining
|
||||
support or release burden. The repository contains no legacy configuration parser,
|
||||
profile compatibility layer, obsolete platform implementation, or redundant tests
|
||||
for behavior owned by ConfigKit.
|
||||
|
||||
## Deferred Work
|
||||
|
||||
- A ConfigKit reference-only inspection or typed prepare/finalize lifecycle may be
|
||||
considered upstream if credential-free inspection becomes a demonstrated shared
|
||||
requirement. Narratio does not implement a private substitute in this migration.
|
||||
- ConfigKit hot reload, remote providers, automatic configuration migration, and
|
||||
application command frameworks remain outside Narratio's scope.
|
||||
- Native macOS arm64 CI may be added later when a reliable runner is available; it
|
||||
is not required for v2.0.0.
|
||||
522
docs/roadmap/implementation.md
Normal file
522
docs/roadmap/implementation.md
Normal file
@@ -0,0 +1,522 @@
|
||||
# ConfigKit Adoption Implementation Plan
|
||||
|
||||
Status: Complete
|
||||
|
||||
Implement the [ConfigKit adoption roadmap](configkit-adoption.md) in the order
|
||||
below. The target is Narratio v2.0.0: ConfigKit v0.1.1, the reduced platform
|
||||
matrix, and the artifact-authoring workflow described there. Each stage is a
|
||||
bounded implementation assignment for a gpt-5.6-terra coding agent. Mark a stage
|
||||
complete only after its changes and acceptance checks succeed; record a short
|
||||
completion and verification note beside its status.
|
||||
|
||||
## Instructions for every stage
|
||||
|
||||
- Read `docs/development.md` and all three policies under `docs/policy/`.
|
||||
Read the relevant roadmap section and the stage-specific sources listed below.
|
||||
Paths identify starting points, not an exhaustive edit allowlist; inspect callers
|
||||
and tests before changing a package boundary.
|
||||
- Use the tagged ConfigKit public API. The local `../ConfigKit` checkout is useful
|
||||
reference material, but verify its version before relying on it. The module
|
||||
cache for v0.1.1 is an alternative. Do not require a sibling checkout, a local
|
||||
`replace`, a Go workspace, an upstream patch, or an unpinned revision to build.
|
||||
- Keep Narratio domain policy in `internal/config`, startup and selectors in
|
||||
`internal/app`, and credential delivery in the owning adapters. Do not build
|
||||
a general configuration framework around ConfigKit.
|
||||
- Keep each stage buildable and run focused behavioral tests for its changed
|
||||
boundary. Temporary migration helpers may exist between stages only where
|
||||
named here; they must not introduce runtime format auto-detection or fallback.
|
||||
- Update each canonical current-behavior document when its behavior changes.
|
||||
The roadmap remains the owner of unimplemented work. Preserve historical
|
||||
release notes; their descriptions of old platforms and formats are history.
|
||||
- Tests use real ConfigKit and temporary files, with fake external adapters or
|
||||
storage where needed. Use synthetic credentials and isolate environment
|
||||
mutation; such tests must not run in parallel. Retain domain and security
|
||||
coverage, remove superseded mechanism tests, and avoid exact diagnostic prose
|
||||
or private callback choreography assertions.
|
||||
- Do not tag or publish v2.0.0 as part of this plan. Release preparation belongs
|
||||
here; release authorization and tag publication are a later task. The existing
|
||||
release completion boundary remains successful upstream tag push, with no CI
|
||||
polling or waiting.
|
||||
|
||||
## Decisions that apply across stages
|
||||
|
||||
The roadmap is authoritative for product scope. These implementation choices
|
||||
make that scope concrete:
|
||||
|
||||
- Enable ConfigKit imports and profile selection for pipeline files only.
|
||||
Campaign, session, and party are single-source definitions with imports and
|
||||
profiles disabled. Preserve their domain schemas and source-relative inputs.
|
||||
Omitted or empty profile and the exact name `default` select the base; other
|
||||
names use ConfigKit's fixed `profiles/<name>.yml` convention.
|
||||
- Defaults apply to absent fields. Explicit empty/zero/false/null values follow
|
||||
ConfigKit decoding and then domain validation; do not refill invalid explicit
|
||||
values with legacy defaults. Optional null stage sections remain absent;
|
||||
required sections fail domain validation rather than being resurrected.
|
||||
- Keep ConfigKit metadata/result snapshots separate from runtime-only Narratio
|
||||
state. ConfigKit discards ignored and unexported fields when detaching values;
|
||||
attach root paths, retained party bytes, family catalogs, and digest metadata
|
||||
after loading. No validator may mutate bound configuration.
|
||||
- Use one Narratio-owned safe effective projection for the final expanded
|
||||
pipeline. It contains configuration values and credential reference names,
|
||||
never resolved credential bytes. It feeds show, semantic diff, digest, and
|
||||
catalog data. ConfigKit metadata remains the authority for input provenance;
|
||||
Narratio adds only domain-generated attribution.
|
||||
- Rename `storage.s3.access_key_id_env` to `access_key_id`,
|
||||
`storage.s3.secret_access_key_env` to `secret_access_key`, and
|
||||
`audita.llm_api_key_env` to `llm_api_key`. Each is an optional `*configkit.Secret`.
|
||||
An explicit reference always resolves during loading, even if the associated
|
||||
stage will not execute. Require both S3 fields together or neither. With neither,
|
||||
retain the AWS SDK default credential chain; do not synthesize mandatory secret
|
||||
references that would break role-based authentication or local-only inspection.
|
||||
With no Audita reference, preserve its existing downstream credential discovery.
|
||||
Explicit references replace Narratio's old per-adapter environment lookup.
|
||||
- Keep absent S3 credentials out of defaults; migration examples explicitly name
|
||||
the existing AWS variables when static credentials are intended. A non-nil
|
||||
missing/empty reference fails instead of silently falling back. This preserves
|
||||
provider-managed credentials while making explicit configuration trustworthy.
|
||||
- `config artifacts` uses the existing configuration command selectors and text
|
||||
output convention. No new JSON mode, artifact creation wizard, inferred filename
|
||||
identity, or profile catalog is added.
|
||||
- Preserve the roadmap's unique base artifact definition rule explicitly:
|
||||
ConfigKit rejects conflicting leaves but can compose disjoint children of the
|
||||
same mapping. Use its public source metadata to reject an ordinary artifact key
|
||||
declared by multiple base files, even if their fields are disjoint. Apply the
|
||||
same rule to family keys. A selected profile may override or extend a definition.
|
||||
This is a narrow Narratio declaration rule, not a different merge algorithm.
|
||||
- Relax only the obsolete configuration/secrets filesystem restrictions identified
|
||||
by the roadmap. Do not remove session restore, cleanup, publication, workspace
|
||||
confinement, or ordinary shared-workspace mode guarantees as collateral work.
|
||||
|
||||
## Stage 1 — Reduce the platform and release matrix
|
||||
|
||||
Status: Complete — Reduced CI and release output to Linux amd64/arm64 and macOS
|
||||
arm64, removed Windows-only runtime implementations and accommodations, and
|
||||
verified focused tests, all three cross-builds, exact asset names, and embedded
|
||||
version reporting.
|
||||
|
||||
Read: roadmap “Platform Contract”; `scripts/build-release-assets.sh`,
|
||||
`scripts/check-release-candidate.sh`, `.woodpecker/`, `internal/releasecheck`,
|
||||
`docs/release.md`, and platform-specific files in application, file operations,
|
||||
and subprocess packages.
|
||||
|
||||
1. Make Linux amd64, Linux arm64, and macOS arm64 the only maintained build and
|
||||
release targets. Remove Windows suffix/packaging branches and macOS amd64
|
||||
targets from scripts, workflows, tests, and current documentation.
|
||||
2. Delete Windows-only implementations and tests; simplify paired build-tag
|
||||
files where the remaining implementations can be shared. Preserve portable
|
||||
code when retaining it adds no special support machinery. Preserve functioning
|
||||
Unix subprocess cancellation, locking, and file operations on both supported OSes.
|
||||
3. Retain the local Linux verification gates and cross-build gates for all three
|
||||
published assets. Do not introduce a native macOS runner requirement or claim
|
||||
that cross-compilation verifies runtime behavior.
|
||||
4. Update architecture/operations platform statements and contributor/release
|
||||
guidance. Keep existing release protections and the no-CI-wait boundary intact.
|
||||
|
||||
Acceptance: focused platform/subprocess/fileops/releasecheck tests pass; build all
|
||||
three targets with `CGO_ENABLED=0`; verify the asset builder produces exactly the
|
||||
three expected versioned binaries and the host binary reports the supplied version.
|
||||
Existing release tests still prove guarded publication. No current support promise
|
||||
or asset list retains Windows or macOS amd64.
|
||||
|
||||
## Stage 2 — Separate configuration normalization from validation
|
||||
|
||||
Status: Complete — Added explicit deterministic pipeline normalization before
|
||||
validation, made Notarius validation read-only on accepted and rejected inputs,
|
||||
documented the boundary, and verified config race tests plus app/stage consumers.
|
||||
|
||||
Read: `internal/config/load.go`, `defaults.go`, `normalize.go`, `validate*.go`,
|
||||
`artifact_families.go`, and ConfigKit's “Recorded application resolution”,
|
||||
“Semantic validation”, and “Defaults and typed decoding” contracts.
|
||||
|
||||
1. Separate value-producing normalization from read-only validation, including
|
||||
Notarius reference/output normalization currently performed by
|
||||
`validateNotarius`. Audit all validators for map, slice, pointer, and scalar
|
||||
mutations, including changes that would occur only on error paths.
|
||||
2. Keep reusable domain normalization ordinary package code. Validation checks the
|
||||
already normalized candidate and must work for both loaded and programmatically
|
||||
constructed values without silently fixing them.
|
||||
3. Inventory defaults in `load.go`: static defaults, per-map-entry/list-element
|
||||
defaults, and inherited or presence-sensitive defaults. Retain named constants
|
||||
in `defaults.go`. This inventory should guide code and concise comments, not
|
||||
become a permanent parallel defaults document.
|
||||
4. Keep the current loader calling normalization explicitly before validation
|
||||
during this preparatory stage. The ConfigKit resolver replaces that loading
|
||||
choreography in Stage 4. Preserve family expansion as an explicit domain step.
|
||||
|
||||
Acceptance: focused configuration tests prove normalization is deterministic and
|
||||
validation does not mutate accepted or rejected inputs. Cover representative
|
||||
Notarius normalization/collision handling and existing artifact/publish rules.
|
||||
Do not add a separate mutation test for every trivial validator.
|
||||
|
||||
## Stage 3 — Replace composition-dependent effective projection
|
||||
|
||||
Status: Complete — Introduced one typed safe projection for effective YAML,
|
||||
semantic values, source paths, and the versioned v3 digest; removed composition
|
||||
parsing from those consumers and verified ordering, expansion, relocation,
|
||||
nil/empty, exact-integer, CLI diff, and semantic-resume behavior.
|
||||
|
||||
Read: `internal/config/effective_pipeline.go`, `effective_digest.go`,
|
||||
`pipeline_composition.go`, `internal/app/config_commands.go`, the config-diff
|
||||
implementation, and `docs/internal/configuration.md`.
|
||||
|
||||
1. Introduce one package-owned, schema-specific safe projection of the effective
|
||||
pipeline. Use explicit typed projections with stable configuration field names;
|
||||
preserve null versus empty containers and exact integers. Ordinary nested
|
||||
configuration types may be reused where they contain no runtime state.
|
||||
Credential fields must be projected explicitly as reference names when Stage 7
|
||||
changes their types. Do not introduce a generic reflection schema engine.
|
||||
2. Render this projection as deterministic YAML using the existing YAML encoder;
|
||||
encoding output is not a replacement YAML loader. Produce semantic records
|
||||
directly from the safe projection using standard JSON encoding/decoding with
|
||||
number preservation if a plain value tree is useful. Do not reparse YAML or
|
||||
reconstruct `compositionNode` trees to compare or hash effective values.
|
||||
3. Make show, diff values, and digest use that common projection. Mapping order is
|
||||
immaterial; sequences remain atomic for diff; nil and empty remain distinct.
|
||||
Exclude family declarations once expanded, all runtime metadata, profile names,
|
||||
source locations, and raw credentials. Preserve logical Notarius path handling
|
||||
so moving an equivalent bundle does not change its digest.
|
||||
4. Version the changed digest encoding explicitly using the package's digest
|
||||
version constant. Digest/profile metadata remains provenance, never a blanket
|
||||
resume cache key. Keep stage/artifact semantic evidence behavior intact.
|
||||
5. Decouple enumeration of effective field paths from the legacy composition
|
||||
renderer. Input ownership can still come from the current loader until Stage 4.
|
||||
|
||||
Acceptance: show/diff/digest tests cover equivalent monolithic/split values,
|
||||
different map insertion order, null/empty distinction, family-expanded outputs,
|
||||
and bundle relocation. A meaningful artifact change changes the digest; formatting
|
||||
and profile naming alone do not. Existing semantic resume tests pass. No effective
|
||||
projection consumer needs the custom composition parser/renderer.
|
||||
|
||||
## Stage 4 — Cut pipeline loading over to ConfigKit
|
||||
|
||||
Status: Complete — Replaced pipeline composition with ConfigKit v0.1.1 imports,
|
||||
fixed-path profile selection, fresh typed defaults, recorded resolution, source
|
||||
metadata adaptation, and duplicate base artifact/family enforcement; removed the
|
||||
legacy parser and verified all tests plus the configuration race suite.
|
||||
|
||||
Read: ConfigKit v0.1.1 `docs/config.md`, `docs/consumers/README.md`, and its
|
||||
Narratio-shaped compiled example; Narratio `internal/config/load.go`,
|
||||
`composition.go`, `pipeline_composition.go`, and profile/composition tests.
|
||||
|
||||
1. Pin `gitea.maximumdirect.net/eric/ConfigKit v0.1.1` in `go.mod`/`go.sum`.
|
||||
Implement a small pipeline definition using `Load`, `File`, enabled imports,
|
||||
`Options.Profile`, defaults, recorded resolution, and read-only validation.
|
||||
2. Migrate static defaults to fresh default values. Apply map-entry, list-element,
|
||||
and inherited defaults with `Resolution.Default`/`IsExplicit`; use recorded
|
||||
normalization for transformations from Stage 2. Do not call old post-decode
|
||||
zero-filling defaults. Remove `NormalizeConfig.UnmarshalYAML` and its private
|
||||
presence bit. Include publication list-entry defaults, optional section
|
||||
allocation, and trim's Scriptorium requirement in this conversion.
|
||||
3. Perform file-local semantic checks in the definition. Defer checks needing
|
||||
canonical party expansion or campaign/session context to the existing explicit
|
||||
resolution boundary; do not reject references to future concrete family members
|
||||
merely because the campaign has not loaded yet. Run full checks after expansion.
|
||||
4. Replace pipeline composition and profile loaders. `configkit.imports` is the
|
||||
only directive; no `composition` or `_configkit` alias is accepted. Pass profile
|
||||
selection directly under the agreed convention; unselected overlays are not read.
|
||||
`LoadPipelineProfilePair` may remain a thin pair of independent loads, with no
|
||||
shared parse cache or paired source-snapshot promise.
|
||||
5. Adapt ConfigKit sources and metadata to Narratio's existing reporting records.
|
||||
Translate paths structurally using public path APIs, not error-message parsing.
|
||||
Preserve multiple contributing locations for composite/profile-owned mappings
|
||||
and distinguish defaults/resolution from authored values.
|
||||
Check artifact/family map-entry ownership for duplicate base declarations using
|
||||
those locations; ignore synthetic defaults and allow the selected overlay.
|
||||
6. Attach runtime metadata only after load succeeds. Preserve root-relative runtime
|
||||
path interpretation and capture logical paths before making them absolute.
|
||||
Retain family declarations and original ConfigKit metadata for later expansion.
|
||||
7. Replace generic mechanism tests with a small integration set proving actual
|
||||
Narratio wiring. Convert fixtures affected by this stage. Delete
|
||||
`composition.go`, obsolete pipeline traversal/overlay/presence machinery, and
|
||||
their redundant tests once the callers are migrated.
|
||||
|
||||
Acceptance: a real root with imports and a testing overlay loads; duplicate claims
|
||||
fail with useful source attribution, including two base files contributing disjoint
|
||||
fields to one artifact or family key; base/default selection works; malformed
|
||||
unselected profiles do not fail base loading. Omitted defaults and representative
|
||||
explicit false/zero/empty/null inputs obey the new semantics. Invalid domain values
|
||||
still fail. Commands use the new pipeline loader with no format fallback. Update
|
||||
`docs/config.md` and configuration internals for the implemented boundary.
|
||||
|
||||
## Stage 5 — Migrate campaign, session, and canonical party loading
|
||||
|
||||
Status: Complete — Migrated campaign, session, and canonical-party documents to
|
||||
independent ConfigKit definitions, preserved bounded session inspection, exact
|
||||
party bytes and metadata-derived authored order, and retained identity, source,
|
||||
family-expansion, and validation behavior; verified all tests and focused races.
|
||||
|
||||
Read: `internal/config/load.go`, `party.go`, `party_resolution.go`,
|
||||
`artifact_families.go`, session-template tests, `internal/app/config_loader.go`,
|
||||
and ConfigKit source/schema/version contracts.
|
||||
|
||||
1. Use separate ConfigKit definitions for campaign, session, and party. Replace
|
||||
`decodeStrictYAML*` and the custom party YAML-node parser. Model the party input
|
||||
as the existing string schema version and character-keyed typed map, with the
|
||||
existing `alias` list, ordered class list, and optional positive class level.
|
||||
2. Keep party domain checks: valid IDs, required display names and classes,
|
||||
trimmed/control-free names, case-insensitive name/alias ambiguity, and duplicate
|
||||
class checks. Validate the required version through `VersionRule`. Keep an
|
||||
exact independent copy of authored bytes outside ConfigKit's schema object.
|
||||
3. Preserve authored character order using successful metadata source coordinates
|
||||
for the single party input when constructing `CanonicalParty.Characters`.
|
||||
Preserve class/alias sequence order naturally. Family expansion and players
|
||||
projection continue using their existing deterministic ID sorting. Do not keep
|
||||
a second YAML parse just to recover mapping order.
|
||||
4. Preserve the narrow session placeholder rejection before typed load, expected
|
||||
session/previous-session identity checks, campaign selection, stable-input
|
||||
precedence, and local versus remotely downloaded session provenance.
|
||||
Retain the initially loaded pipeline/campaign during remote session acquisition.
|
||||
5. Where bytes must be retained or inspected before `ConfigKit.Bytes`, bound the
|
||||
initial read too, using the existing fileops bounded-read helper and a named
|
||||
package constant of 8 MiB matching the pinned source limit. Otherwise prefer
|
||||
`ConfigKit.File`. Do not read an unbounded file and rely on later rejection.
|
||||
6. Complete post-party family and publish-rule expansion, then full cross-config
|
||||
validation and digest refresh. Keep generated origins outside the library
|
||||
snapshot. Remove unused decoder plumbing and obsolete party parsing tests.
|
||||
|
||||
Acceptance: retained authored party bytes are identical; domain errors, aliases,
|
||||
class order, family origins, and derived players are correct. Local and remote
|
||||
session loads preserve identity, input path bases, and the original pipeline.
|
||||
Unknown fields and disabled directives fail through ConfigKit. Focused config,
|
||||
prepare/extract, and application loading tests pass.
|
||||
|
||||
## Stage 6 — Move secrets bootstrap before all configuration loads
|
||||
|
||||
Status: Complete — Added invocation-scoped ConfigKit secrets bootstrap before
|
||||
all pipeline loads and adapter construction, exposed consistent explicit and
|
||||
conventional directory selection across the CLI, removed the retired pipeline
|
||||
schema and custom loader, and verified focused, full, and race test suites.
|
||||
|
||||
Read: roadmap “Credential and Secrets Convention”; ConfigKit's bootstrap
|
||||
contract; `internal/app/secrets_env*.go`, `config_loader.go`, `config_commands.go`,
|
||||
`object_store.go`, command dependencies, and command argument owners.
|
||||
|
||||
1. Add a shared `--secrets-dir` selector to every CLI path that loads a pipeline,
|
||||
including configuration inspection/diff, execution aliases, planning, publish,
|
||||
and session/storage operations. Route all through one command-owned bootstrap
|
||||
context. Help/version and commands that do not load configuration do not bootstrap.
|
||||
2. Discover the root pipeline first. An explicit relative directory is based on
|
||||
caller CWD; otherwise choose `secrets/` beside the resolved root pipeline.
|
||||
An explicitly empty flag is invalid. Missing conventional directories are
|
||||
skipped; missing explicit directories and unreadable/invalid existing directories
|
||||
fail. Use ConfigKit's selected-directory symlink behavior without custom rejection.
|
||||
3. Call `LoadSecretsDir` at most once per command before any ConfigKit load or
|
||||
adapter creation, including both loads of config diff and remote session lookup.
|
||||
Use invocation state, not a package-global `sync.Once`. A new command gets its
|
||||
own bootstrap; library config loaders themselves never mutate the environment.
|
||||
4. Present safe bootstrap warnings/failures through stderr or the existing safe
|
||||
diagnostic boundary, never into YAML show output. Preserve the partial report
|
||||
on bootstrap failure. Do not add rollback or reinterpret ConfigKit file rules.
|
||||
5. Remove late bootstrap calls in object-store and runner composition, the custom
|
||||
directory reader and permission platform files, `pipeline.secrets`, its defaults
|
||||
and validators, and their obsolete tests. Convert affected fixtures immediately.
|
||||
|
||||
Acceptance: test absent conventional directory, explicit override from another
|
||||
CWD, invalid explicit directory, already-set empty/nonempty environment values,
|
||||
and bootstrap failure before adapter construction. A diff invocation and a remote
|
||||
session acquisition each bootstrap once. Config inspection requires no session,
|
||||
workspace writes, manifests, or runtime clients. Update CLI, config, operations,
|
||||
architecture, and troubleshooting to the new bootstrap/permission contract.
|
||||
|
||||
## Stage 7 — Resolve typed credentials and protect all output boundaries
|
||||
|
||||
Status: Complete — Migrated S3 and Audita credentials to optional ConfigKit
|
||||
typed references, passed resolved values directly to runtime adapters, preserved
|
||||
the AWS default provider chain when unconfigured, and verified reference-only
|
||||
inspection, provenance, generated configuration, manifest, and diagnostic
|
||||
boundaries with focused, full, and race test suites.
|
||||
|
||||
Read: ConfigKit “Typed secret references” and rendering contracts;
|
||||
`internal/config/config.go`, `defaults.go`, `validate_adapters.go`, the safe
|
||||
projection from Stage 3, `internal/adapters/storage/s3_backend.go`,
|
||||
`internal/adapters/audita/subprocess.go`, and their composition callers.
|
||||
|
||||
1. Implement the three renamed optional typed fields and S3 pair policy above.
|
||||
Reject old `_env` schema keys; remove their default-name and lookup helpers.
|
||||
Convert maintained fixtures with synthetic environment setup where references
|
||||
are present. Do not add credentials to unrelated local pipeline fixtures.
|
||||
2. Pass resolved S3 values to the SDK adapter's explicit static provider only when
|
||||
the pair is configured. With neither, preserve its default provider chain.
|
||||
Pass Audita the resolved value and safe reference name; deliver the value only
|
||||
through the child environment's existing `AUDITA_LLM_API_KEY` override.
|
||||
Do not re-read the environment for an explicitly resolved reference.
|
||||
3. Adapt every serialization, digest, source/diff output, diagnostic, generated
|
||||
invocation config, manifest, and provenance caller to safe reference names.
|
||||
Never YAML-marshal ConfigKit's opaque secret directly or call `Value()` in an
|
||||
inspection path. Secret changes alone must not alter effective digests or
|
||||
invalidate artifact evidence; reference-name changes may change provenance.
|
||||
4. Keep ConfigKit errors structured and safe. Domain validation messages must
|
||||
avoid raw credential values; do not unwrap private parser errors or interpolate
|
||||
whole trusted configuration structs into error messages.
|
||||
5. Preserve downstream tool ownership for Notarius/Scriptorium credentials that
|
||||
Narratio does not configure. Do not add a parallel credential schema for them.
|
||||
|
||||
Acceptance: use distinctive synthetic secrets to prove no leakage in show, sources,
|
||||
diff, catalog-ready projections, success/failure diagnostics, manifests, or generated
|
||||
Audita configuration. Verify the child receives the credential and an environment
|
||||
change after loading does not change explicit delivery. Test S3 paired references,
|
||||
partial-pair rejection, missing effective references, and nil SDK fallback offline.
|
||||
Update adapter and credential documentation and remove obsolete lookup tests.
|
||||
|
||||
## Stage 8 — Infer and normalize artifact dependencies once
|
||||
|
||||
Status: Complete — Finalized one normalized artifact graph from explicit ordering
|
||||
edges and configured-artifact inputs (including rewritten family members), retained
|
||||
edge origins for diagnostics, and verified focused/full tests plus the full race
|
||||
suite and an assembled input-only analyze workflow.
|
||||
|
||||
Read: `internal/config/validate_scriptorium.go`, `artifact_families.go`,
|
||||
`internal/artifactpolicy`, `internal/artifacts`, analyze planning/execution, and
|
||||
their artifact dependency and semantic-evidence tests.
|
||||
|
||||
1. At the final configuration resolution boundary, union explicit `depends_on`
|
||||
with configured-artifact input references. Use the existing source-ID parser;
|
||||
do not infer edges from filenames, arbitrary strings, or non-artifact inputs.
|
||||
2. Infer after family member inputs become concrete `narratio.artifact.<key>`
|
||||
sources. Normalize, deduplicate, and sort the effective dependency list once.
|
||||
Keep explicit ordering-only dependencies valid. Repeated finalization must not
|
||||
accumulate edges or alter ordering. Store source-edge attribution separately
|
||||
from the effective graph for diagnostics and generated provenance.
|
||||
3. Apply existing existence, self-reference, enabled-dependency, output, and cycle
|
||||
validation to the union. Remove the error requiring an input to be repeated in
|
||||
`depends_on`. Retain diagnostics pointing to the explicit list entry or the
|
||||
input that contributed an invalid edge, including family origin where relevant.
|
||||
4. Ensure execution, targeted analysis, planning, prerequisite closure, reuse,
|
||||
forced regeneration, and semantic evidence consume this same concrete graph.
|
||||
No adapter or command may infer a second graph. Refresh final projection/digest
|
||||
after inference; equivalent explicit and inferred declarations compare equally.
|
||||
|
||||
Acceptance: cover input-only dependency ordering, duplicate explicit/inferred edges,
|
||||
ordering-only edges, missing/self/cyclic references, and rewritten family inputs.
|
||||
Use an assembled fake-artifact workflow to prove upstream output is supplied to a
|
||||
dependent artifact and failure/resume retains successful predecessors. Existing
|
||||
force/selection and invalidation tests remain valid. Update config and analyze
|
||||
internals with the implemented inference rule.
|
||||
|
||||
## Stage 9 — Add the effective artifact catalog command
|
||||
|
||||
Status: Complete — Added the read-only `config artifacts` catalog over finalized
|
||||
configuration, including effective dependencies and definition/family/party
|
||||
provenance; documented it and verified focused, full, and race test suites.
|
||||
|
||||
Read: `internal/app/config_commands.go`, config diff/source rendering, the safe
|
||||
projection and family-origin catalogs, and `docs/cli.md`.
|
||||
|
||||
1. Add `narratio config artifacts` to dispatch and help. Reuse ordinary pipeline,
|
||||
profile, campaign, campaign-file, and secrets-directory parsing/resolution.
|
||||
Resolve a campaign when families require it or when explicitly selected; a
|
||||
pipeline with only ordinary artifacts can be inspected without a campaign.
|
||||
2. Render one stable record per effective concrete artifact in key order with
|
||||
key, `narratio.artifact.<key>` source ID, defining source location(s), enabled
|
||||
state, output path, and sorted effective dependencies. Family members additionally
|
||||
identify family key, character ID, defining family source, and party source.
|
||||
3. Derive identity from the map key. A profile may contribute to an existing
|
||||
artifact, so report contributing definition locations instead of inventing
|
||||
a single filename owner. Use a deterministic escaped tabular text rendering;
|
||||
include a header and no records for an empty catalog. Propagate writer errors.
|
||||
4. Use the same finalized state as show/sources/diff and runtime configuration.
|
||||
No session download, adapter initialization, manifest access, or workspace
|
||||
materialization is allowed. Secret bootstrap remains the agreed startup exception.
|
||||
|
||||
Acceptance: CLI tests cover ordinary and generated artifacts, disabled entries,
|
||||
profile-added/overridden artifacts, empty catalog, no-campaign ordinary inspection,
|
||||
required-campaign failure for families, stable ordering, and no runtime side effects.
|
||||
Check identity stays the same after moving an artifact file without changing its key.
|
||||
Document the command in its canonical CLI owner and link to config layout examples.
|
||||
|
||||
## Stage 10 — Finish the operator layout and v2 migration documentation
|
||||
|
||||
Status: Complete
|
||||
|
||||
Completion: Reworked the maintained bundle around an unprofiled production base,
|
||||
one-entry artifact and family imports, and a testing overlay; updated the example
|
||||
harness and inspection workflows; consolidated the v2 configuration guidance; and
|
||||
added the v2.0.0 operator migration checklist. Focused documentation/example tests,
|
||||
the offline inspection walkthrough, and the full normal/race suites pass.
|
||||
|
||||
Read: roadmap “Artifact Definition Ergonomics” and “Compatibility and Migration”;
|
||||
`examples/README.md`, existing example bundles and their loader test,
|
||||
`docs/config.md`, `docs/operations.md`, `docs/release.md`, and internal configuration
|
||||
and artifact documentation.
|
||||
|
||||
1. Convert maintained pipeline examples to a production base with explicit
|
||||
`configkit.imports`, `artifacts/<key>.yml`, `artifact-families/<key>.yml`, and
|
||||
`profiles/testing.yml`. Each recommended artifact/family file defines exactly
|
||||
one explicit map entry. Fold old production overlays into the base.
|
||||
2. Include an input-only dependency example and at least two character-family
|
||||
examples sharing a campaign-owned party roster. Keep supported inline artifacts
|
||||
legal; file layout is guidance, not a filename-based validation requirement.
|
||||
3. Update the example validation harness to load root configurations and selected
|
||||
testing profiles, not attempt to load each imported fragment as a full pipeline.
|
||||
Use synthetic environment setup for explicit references. No example includes
|
||||
real credentials or depends on local tools/cloud services to validate.
|
||||
4. Consolidate canonical documentation updates from earlier stages. Remove current
|
||||
references to old composition/profile catalogs, default-profile configuration,
|
||||
secrets.env_dir, credential `_env` fields, credential-free inspection guarantees,
|
||||
paired diff snapshots, and removed platform support. Explain explicit-value and
|
||||
null changes without promising reloadable inspection output.
|
||||
5. Add `docs/releases/v2.0.0.md` using the release-note structure, with a concrete
|
||||
migration checklist linking to the canonical config/CLI docs and examples.
|
||||
Include static S3-reference migration versus SDK credential discovery, new
|
||||
inspection prerequisites, artifact input dependency inference, and platform
|
||||
removals. Do not claim publication or edit older immutable release notes.
|
||||
|
||||
Acceptance: `go test ./internal/doccheck` and
|
||||
`go test ./internal/config -run '^TestExamplesLoadAndValidate$'` pass. Walk through
|
||||
the new operator examples with config validate/show/sources/diff/artifacts offline.
|
||||
Their identities, family members, origins, and testing overrides agree.
|
||||
|
||||
## Stage 11 — Verify the complete migration and remove remaining scaffolding
|
||||
|
||||
Status: Complete
|
||||
|
||||
Completion: Audited the accepted roadmap and the full change set from `1558c9c`;
|
||||
no legacy configuration loader, profile/composition traversal, presence store,
|
||||
custom secrets loader, or production YAML decoder remains. The migration removes a
|
||||
net 1,235 production and 843 test Go lines; remaining configuration code owns domain
|
||||
defaults, normalization, validation, expansion, provenance, and safe projections.
|
||||
Go 1.26.7 module hygiene, tests, race tests, vet, build, documentation, examples,
|
||||
and the v2.0.0 release-candidate checker pass. Linux amd64 was tested natively;
|
||||
Linux arm64 and macOS arm64 release assets were cross-built, with native macOS
|
||||
arm64 validation remaining best effort and outside the release gate.
|
||||
|
||||
Read: the whole feature roadmap, completed stage notes, and the current test and
|
||||
release procedures. Review the diff from the commit introducing this plan.
|
||||
|
||||
1. Audit every roadmap requirement against current code and executable evidence.
|
||||
Remove temporary migration adapters and any residual generic YAML parser,
|
||||
composer, profile traversal, explicit-presence store, old secrets loader,
|
||||
decoder hooks, or unused tests. `gopkg.in/yaml.v3` may remain for output encoding
|
||||
and unrelated external formats; it must not supply a second configuration loader.
|
||||
2. Confirm result-owned snapshots versus final expanded configuration are handled
|
||||
consistently; every inspection/runtime path sees normalized inferred dependencies
|
||||
and generated publish rules. Recheck secret safety and semantic resume boundaries.
|
||||
3. Inspect current platform conditions and asset lists. Retain historical notes but
|
||||
no active Windows/macOS amd64 obligation. Compare owned production/test code
|
||||
before and after; report removals and any remaining complexity with reasons.
|
||||
Do not chase a LOC quota by removing useful domain checks or regressions.
|
||||
4. Run the repository gates with the maintained Go 1.26 toolchain:
|
||||
|
||||
```sh
|
||||
go test ./...
|
||||
go test -race ./...
|
||||
go vet ./...
|
||||
go build ./...
|
||||
go test ./internal/doccheck
|
||||
go test ./internal/config -run '^TestExamplesLoadAndValidate$'
|
||||
```
|
||||
|
||||
5. Run `scripts/check-release-candidate.sh v2.0.0` after the release note and all
|
||||
candidate changes exist. This also verifies module hygiene and all three
|
||||
release assets with `GOWORK=off`. Resolve failures; do not bypass checks or
|
||||
publish a tag. Record native versus cross-build evidence accurately.
|
||||
|
||||
Acceptance: all gates pass and every roadmap outcome has direct evidence. No
|
||||
dependency on the sibling ConfigKit checkout remains. Record completion and any
|
||||
best-effort platform limitations, then hand off for a separate release decision.
|
||||
|
||||
## Open Questions
|
||||
|
||||
None required before implementation. The agreed roadmap and the explicit choices
|
||||
above resolve the migration decisions. Upstream credential-free inspection and
|
||||
native macOS arm64 CI remain deferred, not prerequisites for this plan.
|
||||
331
docs/roadmap/remediation.md
Normal file
331
docs/roadmap/remediation.md
Normal file
@@ -0,0 +1,331 @@
|
||||
# ConfigKit Migration Review — Required Remediations
|
||||
|
||||
Reviewed 2026-09-07 at `f7a40af`, against the changes after `1558c9c`, the
|
||||
[feature roadmap](configkit-adoption.md), and the completed
|
||||
[implementation plan](implementation.md).
|
||||
|
||||
The main migration is implemented. The following findings are reproducible gaps
|
||||
in the intended behavior, not proposals to restore retired compatibility or
|
||||
filesystem restrictions. P2 denotes a correctness issue to fix before release;
|
||||
P3 denotes an operator-facing completeness issue also required by the accepted
|
||||
plan. No application code was changed during this review.
|
||||
|
||||
## 1. Derive explicit publish destinations after family expansion
|
||||
|
||||
Status: Resolved — P2
|
||||
|
||||
Resolution: Missing destinations are now derived through one shared domain helper
|
||||
both during ConfigKit resolution and again after concrete family expansion. Explicit
|
||||
destinations remain unchanged, while unknown sources still fail read-only publish
|
||||
validation. Regression coverage verifies resolved runtime and effective inspection
|
||||
output for generated members, explicit overrides, and unknown sources.
|
||||
|
||||
**Evidence:** `internal/config/pipeline_configkit.go:208` derives missing publish
|
||||
destinations during the ConfigKit resolver. A concrete family member does not
|
||||
exist yet, so the resolver ignores the failed derivation. After family expansion,
|
||||
`internal/config/artifact_families.go:141` calls the now-read-only publish
|
||||
validator, which rejects the still-empty destination. The previous implementation
|
||||
could derive it during that later validation.
|
||||
|
||||
**Reproduction:** In an otherwise valid pipeline, define a family `meta` over
|
||||
`party.characters`, with output pattern `artifacts/{character_id}.md`. Select a
|
||||
valid campaign whose party contains `alice`, and add:
|
||||
|
||||
```yaml
|
||||
publish:
|
||||
outputs:
|
||||
- source: narratio.artifact.meta_alice
|
||||
```
|
||||
|
||||
`config validate --config <pipeline> --campaign-file <campaign>` fails with
|
||||
`pipeline.publish.outputs[0].dest is required when destination cannot be derived
|
||||
from "narratio.artifact.meta_alice"`. Adding `dest: artifacts/alice.md` makes the
|
||||
same configuration pass. The destination is therefore available and derivable;
|
||||
the lifecycle omits the required final normalization step.
|
||||
|
||||
**Required change:** Complete publish destination resolution after concrete
|
||||
artifacts exist, before publish validation and final projection/digest generation.
|
||||
Share the domain derivation logic and keep validation read-only. Preserve
|
||||
explicit destinations and generated family publish-rule collision checks. Do not
|
||||
require redundant `dest` fields or revive mutation inside validators.
|
||||
|
||||
**Acceptance:** Test an explicit publish source naming a generated member with
|
||||
omitted destination, an explicit destination override, and a genuinely unknown
|
||||
source. Validate both effective inspection and resolved runtime configuration.
|
||||
|
||||
## 2. Stop re-defaulting an explicitly empty Notarius working directory
|
||||
|
||||
Status: Resolved — P2
|
||||
|
||||
Resolution: Removed both legacy blank-value fallbacks after ConfigKit resolution.
|
||||
An omitted working directory is still derived by the presence-aware resolver, an
|
||||
explicit relative directory is still rooted at the pipeline, and explicit empty or
|
||||
whitespace-only values remain empty and fail enabled-Notarius validation. Existing
|
||||
relocation-invariant digest coverage remains green.
|
||||
|
||||
**Evidence:** ConfigKit's resolver correctly uses presence-aware defaulting at
|
||||
`internal/config/pipeline_configkit.go:180`. However,
|
||||
`internal/config/load.go:454` subsequently replaces any blank working directory
|
||||
with the config file's directory, irrespective of input presence.
|
||||
`internal/config/effective_digest.go` also applies a blank-value fallback while
|
||||
capturing logical paths. These retained legacy fallbacks bypass the new rule that
|
||||
defaults apply only to absent fields.
|
||||
|
||||
**Reproduction:** Set `notarius.enabled: true`, a valid config path, pipeline ID
|
||||
and output map, and `notarius.working_directory: ""`. With
|
||||
`config_path: notarius/config.yml`, `config show` succeeds and reports
|
||||
`working_directory: notarius`; validation never sees the explicit invalid blank.
|
||||
|
||||
**Required change:** Let the ConfigKit resolver own the absent-field default.
|
||||
The subsequent runtime and logical-path conversions must preserve explicit
|
||||
emptiness so enabled-Notarius validation can reject it. Remove duplicate
|
||||
value-based defaulting rather than adding another private presence mechanism.
|
||||
|
||||
**Acceptance:** Omission still derives the expected directory; explicit empty
|
||||
and whitespace-only values fail when Notarius is enabled; a configured relative
|
||||
directory resolves from the pipeline root. Preserve digest relocation invariance.
|
||||
|
||||
## 3. Enforce unique base definitions even when a profile removes them
|
||||
|
||||
Status: Resolved — P2
|
||||
|
||||
Resolution: Loading now discovers base declarations through ConfigKit's public
|
||||
historical-path enumeration and metadata history from the selected result,
|
||||
without separately resolving the base (see finding 7). Duplicate ordinary or
|
||||
family definitions therefore fail even when an overlay nulls their container or
|
||||
all of Scriptorium. A single base owner
|
||||
with a legitimate profile override remains supported.
|
||||
|
||||
**Evidence:** `rejectDuplicateBaseDefinitions` at
|
||||
`internal/config/pipeline_configkit.go:354` enumerates only artifact/family keys
|
||||
remaining in the final typed configuration. It inspects history for those keys
|
||||
but never examines base declarations removed by an overlay.
|
||||
|
||||
**Reproduction:** Import `first.yml` defining
|
||||
`scriptorium.artifacts.recap.prompt_id` and `second.yml` defining
|
||||
`scriptorium.artifacts.recap.output_path`. Base validation correctly rejects the
|
||||
two-file declaration. Add `profiles/clear.yml` containing:
|
||||
|
||||
```yaml
|
||||
scriptorium:
|
||||
artifacts: null
|
||||
```
|
||||
|
||||
The same root with `--profile clear` passes. A selected profile can thus hide the
|
||||
base ownership conflict, contrary to the plan's unique base-definition rule.
|
||||
This concerns disjoint contributions to one artifact, not ConfigKit's generic
|
||||
conflicting-leaf checks, which still work.
|
||||
|
||||
**Required change:** Discover the complete base declaration set through public
|
||||
ConfigKit metadata/history, including declarations displaced by an overlay, and
|
||||
validate ownership independently of final effective-key survival. Keep legitimate
|
||||
profile overrides and profile-only definitions supported. Do not reparse source
|
||||
YAML or add a second composer.
|
||||
|
||||
**Acceptance:** Cover duplicate ordinary and family definitions hidden by a null
|
||||
artifact/family container or null Scriptorium section. Base and selected-profile
|
||||
loads must reject the same base conflict, while a single base declaration with a
|
||||
valid overlay remains accepted.
|
||||
|
||||
## 4. Preserve null versus empty collections in effective YAML
|
||||
|
||||
Status: Resolved — P2
|
||||
|
||||
Resolution: Effective YAML is now emitted from the credential-safe JSON value
|
||||
projection through an output-only YAML-node conversion. Explicit nulls, empty maps,
|
||||
empty slices, booleans, strings, and exact JSON numbers retain their semantic types.
|
||||
Public show/diff tests cover null and empty maps and slices, while digest,
|
||||
credential-redaction, deterministic output, and exact-integer coverage remain intact.
|
||||
|
||||
**Evidence:** `internal/config/effective_pipeline.go:154` marshals the typed
|
||||
projection directly with `yaml.Marshal`. The encoder emits nil slices as `[]`
|
||||
and nil maps as `{}`, while the JSON projection used by diff and digest preserves
|
||||
them as `null`. The shared input type does not guarantee shared output semantics.
|
||||
|
||||
**Reproduction:** Set `audita.modules: null` in a valid base and
|
||||
`audita.modules: []` in `profiles/empty.yml`. Both `config show` invocations emit
|
||||
`modules: []`, but `config diff default empty` reports:
|
||||
|
||||
```text
|
||||
changed audita.modules null []
|
||||
```
|
||||
|
||||
The commands disagree about the effective value. The same renderer also supplies
|
||||
the resolved pipeline diagnostic written by prepare.
|
||||
|
||||
**Required change:** Render YAML from a credential-safe projection that explicitly
|
||||
preserves nulls and empty collections, consistently with the semantic projection.
|
||||
Keep exact integers and safe credential reference names. An output-only conversion
|
||||
is appropriate; do not restore YAML decoding or a configuration parser. This
|
||||
requires truthful inspection, not a guarantee that inspection output is reloadable.
|
||||
|
||||
**Acceptance:** Verify null versus empty maps and slices through the public show
|
||||
boundary, and agreement with diff/digest distinctions. Retain credential-leakage
|
||||
and exact-integer regression coverage.
|
||||
|
||||
## 5. Complete provenance for generated artifacts and inferred dependencies
|
||||
|
||||
Status: Resolved — P3
|
||||
|
||||
Resolution: Generated catalog records now collect the complete base/profile family
|
||||
definition set from ConfigKit history. Effective source ownership for materialized
|
||||
dependency lists is rebuilt narrowly from retained explicit-list and input-edge
|
||||
origins, including mixed and generated-family dependencies. CLI tests verify a
|
||||
profiled family catalog agrees with generated source attribution and that inferred
|
||||
ordinary edges name their input contributor instead of the central default.
|
||||
|
||||
Two new reporting paths lose available declaration information:
|
||||
|
||||
- **Profile contributions to family members:**
|
||||
`internal/config/artifact_catalog.go:52` fills a generated member's definitions
|
||||
from the single `member.Source`. That source is chosen by the first matching
|
||||
non-default family field in `artifactFamilySource`, not the full set of effective
|
||||
contributors. Define family `meta` in the base and override its `prompt_id` in
|
||||
`profiles/testing.yml`. `config artifacts --profile testing` reports only the
|
||||
base file for `meta_alice`, even though `config sources` correctly attributes
|
||||
that member's prompt to the testing overlay.
|
||||
- **Inferred ordinary dependencies:** With `second.inputs.prior.source` set to
|
||||
`narratio.artifact.first` and no explicit dependency list, `config sources`
|
||||
reports `scriptorium.artifacts.second.depends_on` as `default / default`.
|
||||
`finalizeArtifactDependencies` stores the input-edge origin, but the effective
|
||||
ownership projection retains the pre-inference default attribution from
|
||||
`internal/config/effective_digest.go:55` and does not consult that origin.
|
||||
|
||||
**Impact:** Operators inspecting a testing profile or following a dependency can
|
||||
be directed to the wrong file or told a user-derived value is a central default.
|
||||
Runtime ordering and generated artifact values were correct in these reproductions.
|
||||
|
||||
**Required change:** Use ConfigKit's contributing locations for family definition
|
||||
sets, as for ordinary artifacts, and add narrow generated attribution for inferred
|
||||
dependency lists using their retained edge origins. Preserve family/character/party
|
||||
identity and deterministic ordering. Avoid introducing a second general provenance
|
||||
store or choosing a file based on the first alphabetically visited field.
|
||||
|
||||
**Acceptance:** A profiled family catalog exposes both relevant files; an inferred
|
||||
edge points back to its input declaration; mixed explicit/inferred dependencies
|
||||
retain their relevant contributors. Verify catalog and sources agree after expansion.
|
||||
|
||||
## 6. Surface successful secrets bootstrap reports, including skipped empty variables
|
||||
|
||||
Status: Resolved — P3
|
||||
|
||||
Resolution: Successful bootstrap reports now flow once through the command
|
||||
diagnostics channel, while direct internal command calls default that channel to
|
||||
discard. ConfigKit's value-free report makes skipped existing empty variables
|
||||
explicit and remains available before later typed-load failures; structured output
|
||||
stays isolated on stdout. Command-level coverage verifies successful reporting,
|
||||
skipped-empty reporting, value redaction, clean config-show output, and one report
|
||||
for a two-profile diff.
|
||||
|
||||
**Evidence:** `internal/app/secrets_bootstrap.go:36` discards the report whenever
|
||||
`LoadSecretsDir` returns no error. Reports are only rendered when directory
|
||||
bootstrap itself fails. This does not implement the roadmap's safe bootstrap
|
||||
reporting boundary or Stage 6's warning presentation requirement.
|
||||
|
||||
**Reproduction:** Place a synthetic key in `secrets/NARRATIO_REVIEW_KEY`, reference
|
||||
it from `audita.llm_api_key`, and invoke config validation with
|
||||
`NARRATIO_REVIEW_KEY=` in the environment. ConfigKit correctly skips the file
|
||||
because the variable already exists, then typed loading fails with `secret.empty`.
|
||||
Narratio shows only that load failure, omitting the bootstrap report's
|
||||
`skipped_existing` and `ExistingEmpty` information. A successful load likewise
|
||||
discards all bootstrap results.
|
||||
|
||||
**Required change:** Retain and render or summarize the safe report once per
|
||||
invocation through a diagnostic output channel, including when a later typed load
|
||||
fails. Make skipped-existing-empty outcomes explicit. Preserve process environment
|
||||
precedence and the existing partial report on failure; keep values out of all
|
||||
diagnostics and keep structured/YAML stdout uncontaminated. Do not add bootstrap
|
||||
rollback, retries, or repeated directory loading.
|
||||
|
||||
**Acceptance:** Command-level tests demonstrate safe success/report output and a
|
||||
useful skipped-empty diagnostic, without leaking a synthetic secret or polluting
|
||||
show output. A two-profile diff still bootstraps and reports only once.
|
||||
|
||||
## 7. Do not resolve unused base credentials when loading a profile
|
||||
|
||||
Status: Resolved — P2
|
||||
|
||||
Resolution: Upgraded to released ConfigKit v0.1.2 and replaced the extra base load
|
||||
with `Metadata.HistoricalPaths()` enumeration of declaration paths in the selected
|
||||
result. Ownership checks still inspect base-layer history for both ordinary
|
||||
artifacts and families, even when overlays remove them. Regression tests verify
|
||||
that a profile can replace an unset production credential with a resolvable testing
|
||||
reference or remove it with null. Existing hidden-duplicate and legitimate-override
|
||||
tests remain intact. No local dependency replacement or second composer is needed.
|
||||
|
||||
Follow-up review: 2026-09-07 at `3ce1981`. The original six reproductions are
|
||||
addressed, but the fix for finding 3 introduces this regression.
|
||||
|
||||
**Evidence:** `internal/config/pipeline_configkit.go:49-57` loads the base again
|
||||
without a profile to check duplicate base definitions. This calls the complete
|
||||
typed ConfigKit loader, including secret resolution, domain resolution, and
|
||||
validation—not just declaration ownership inspection. Consequently, loading a
|
||||
valid testing profile also requires credentials from the unselected base.
|
||||
|
||||
**Reproduction:** In an otherwise valid base, set `audita.llm_api_key` to
|
||||
`NARRATIO_REVIEW_PROD_KEY`. In `profiles/testing.yml`, replace it with
|
||||
`NARRATIO_REVIEW_TEST_KEY`. Leave the production variable unset and supply only
|
||||
the testing variable with a synthetic value. `config validate --config <root>
|
||||
--profile testing` fails with `secret.unset` at the base credential declaration.
|
||||
Supplying both variables makes it pass. A profile that instead sets
|
||||
`audita.llm_api_key: null` also fails when the unused production variable is
|
||||
unset. Both failures were reproduced through the current CLI.
|
||||
|
||||
**Impact:** Operators cannot use an independent testing credential environment
|
||||
or remove a production credential requirement through a profile. Commands fail
|
||||
despite having all credentials required by their effective configuration. The
|
||||
extra full load also unnecessarily repeats configuration processing.
|
||||
|
||||
**Required change:** Enforce unique base ownership without requiring the
|
||||
unselected base to succeed as a separately resolved runtime configuration.
|
||||
Inspect complete declaration history through ConfigKit's public metadata,
|
||||
including declarations displaced by overlays. If additional upstream metadata
|
||||
support is needed, address that explicitly rather than requiring unused secrets,
|
||||
suppressing arbitrary load failures, reparsing YAML, or restoring a second
|
||||
composer. Preserve the duplicate-definition protection added for finding 3.
|
||||
|
||||
**Acceptance:** Add public-boundary regression tests for a profile replacing an
|
||||
unset production credential with an available testing credential, and for a
|
||||
profile removing that credential with null. Both must load successfully without
|
||||
the production variable. Retain failures for duplicate base ordinary/family
|
||||
definitions hidden by null overlays, along with legitimate profile override and
|
||||
profile-only definition coverage. Resolve secrets only for the selected effective
|
||||
configuration.
|
||||
|
||||
## Follow-up verification
|
||||
|
||||
After adopting ConfigKit v0.1.2 and completing finding 7, the full
|
||||
`scripts/check-release-candidate.sh v2.0.0` check passed: module hygiene,
|
||||
formatting/whitespace, uncached tests and race tests, vet, build, documentation,
|
||||
maintained examples, and Linux amd64/arm64 plus macOS arm64 release cross-builds.
|
||||
`go mod verify` also passed. The new credential regressions failed before the
|
||||
ownership-check fix and passed afterward. No local module replacements, release
|
||||
tags, or pushes were used.
|
||||
|
||||
At `3ce1981`, `go test ./...`, `go test -race ./...`, `go vet ./...`, and
|
||||
`go build ./...` passed. The new finding was reproduced with a freshly built CLI,
|
||||
temporary configuration files, and synthetic credentials; no external adapters
|
||||
or cloud services were invoked. That review pass changed only this review record.
|
||||
|
||||
## Original verification and scope
|
||||
|
||||
The following checks passed at the reviewed commit:
|
||||
|
||||
- `go test ./...`
|
||||
- `go test -race ./...`
|
||||
- `go vet ./...`
|
||||
- `go build ./...`
|
||||
- The release asset builder for `v2.0.0`, producing exactly Linux amd64, Linux
|
||||
arm64, and macOS arm64 assets; the Linux amd64 binary reported the expected version.
|
||||
|
||||
The existing full suites include documentation and maintained-example checks.
|
||||
The findings above were additionally reproduced through a freshly built CLI with
|
||||
isolated temporary YAML fixtures and synthetic credentials. No live downstream
|
||||
tools or cloud services were invoked. Linux arm64 and macOS arm64 were cross-built,
|
||||
not tested natively. No tag was created or pushed.
|
||||
|
||||
The migration removes a net 1,235 production Go lines and 843 test Go lines versus
|
||||
`1558c9c` (physical diff lines, including comments and whitespace). ConfigKit owns
|
||||
generic loading/composition; Narratio retains domain resolution, orchestration,
|
||||
safe inspection, and adapter delivery. These findings require completing that
|
||||
design, not reversing the simplification.
|
||||
@@ -717,7 +717,8 @@ Symptom:
|
||||
|
||||
Likely causes:
|
||||
|
||||
- invalid `pipeline.secrets.env_dir`;
|
||||
- missing, unreadable, or invalid explicit `--secrets-dir`;
|
||||
- invalid entries in the conventional `secrets/` directory beside the root pipeline;
|
||||
- missing credential environment variables;
|
||||
- invalid S3 endpoint/bucket settings.
|
||||
|
||||
@@ -730,7 +731,7 @@ env | sed 's/=.*//' | grep -E 'OBJECT_STORAGE|AWS|AUDITA|SCRIPTORIUM'
|
||||
|
||||
Safe fix:
|
||||
|
||||
- correct secret-file path and permissions;
|
||||
- correct `--secrets-dir`, secret-file type/name, and operator-managed permissions;
|
||||
- provide required env vars;
|
||||
- keep secret values out of YAML.
|
||||
|
||||
|
||||
@@ -11,18 +11,26 @@ in the [configuration reference](../docs/config.md).
|
||||
WhisperX URL.
|
||||
- [Production-shaped pipeline](pipeline.production.yml): S3 storage, publish,
|
||||
external tools, and configured Scriptorium artifacts.
|
||||
- [Production/testing split bundle](production-testing/pipeline.yml): explicit
|
||||
`conf.d` imports, a production default, and selectable production/testing
|
||||
overlays. It also demonstrates canonical-party artifact families and a
|
||||
testing-only disabled artifact. Validate it with:
|
||||
- [Production-base/testing-overlay bundle](production-testing/pipeline.yml):
|
||||
explicit `conf.d` imports, one ordinary artifact per `artifacts/<key>.yml`,
|
||||
one family per `artifact-families/<key>.yml`, and the fixed
|
||||
`profiles/testing.yml` overlay. Production values live in the base. The
|
||||
bundle demonstrates an input-only artifact dependency, two character families
|
||||
sharing the campaign party, and a testing-only disabled artifact. Inspect it
|
||||
offline with:
|
||||
|
||||
```sh
|
||||
narratio config validate --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml
|
||||
narratio config diff production testing --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml
|
||||
narratio config show --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml --profile testing
|
||||
narratio config sources --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml --profile testing
|
||||
narratio config artifacts --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml --profile testing
|
||||
narratio config diff default testing --config examples/production-testing/pipeline.yml --campaign-file examples/campaigns/sample-campaign/campaign.yml
|
||||
```
|
||||
|
||||
`config show` and `config sources` accept the same selectors and remain
|
||||
read-only.
|
||||
These inspection commands load configuration and resolve explicit credential
|
||||
references, but do not load a session or create runtime state. This bundle is
|
||||
intentionally local-storage and credential-reference free so the walkthrough
|
||||
needs no tools or cloud services.
|
||||
- [Full annotated pipeline](pipeline.full.annotated.yml): every implemented
|
||||
pipeline section with explanatory comments.
|
||||
- [Extraction subset pipeline](pipeline.extraction-subset.yml): a focused
|
||||
@@ -58,5 +66,14 @@ with the sample campaign and a compatible local- or S3-audio session.
|
||||
reserves the expected filename and directory shape. Replace it with a real
|
||||
FLAC file before running transcription.
|
||||
|
||||
The examples contain environment-variable names but no credential values. They
|
||||
use fictional campaign content and reserved example domains.
|
||||
The examples contain environment-variable names but no credential values. Set
|
||||
`OBJECT_STORAGE_KEY_ID`, `OBJECT_STORAGE_KEY`, and `AUDITA_LLM_API_KEY` to
|
||||
nonempty synthetic or real values before loading the production-shaped and full
|
||||
annotated pipelines. They use fictional campaign content and reserved example
|
||||
domains.
|
||||
|
||||
For a deployed bundle, keep credential files out of source control in a
|
||||
`secrets/` directory beside the root `pipeline.yml`, one file per referenced
|
||||
environment-variable name, or select another directory with `--secrets-dir`.
|
||||
Static S3 credentials are optional: omit both S3 credential references to use
|
||||
the AWS SDK discovery chain.
|
||||
|
||||
@@ -7,9 +7,8 @@ workspace:
|
||||
# Optional: remove run-scoped workdir after successful publish commit.
|
||||
cleanup_after_publish: false
|
||||
|
||||
# Optional: local secret file loader (directory of ENV_VAR_NAME files).
|
||||
# secrets:
|
||||
# env_dir: ./secrets
|
||||
# Credential files, when used, belong in a sibling secrets/ directory or in the
|
||||
# directory selected with --secrets-dir; they are not pipeline configuration.
|
||||
|
||||
storage:
|
||||
# Defaults to "local". Use "s3" explicitly for publish + S3 audio workflows.
|
||||
@@ -24,8 +23,8 @@ storage:
|
||||
endpoint: ""
|
||||
force_path_style: false
|
||||
# Optional; defaults shown explicitly.
|
||||
access_key_id_env: OBJECT_STORAGE_KEY_ID
|
||||
secret_access_key_env: OBJECT_STORAGE_KEY
|
||||
access_key_id: OBJECT_STORAGE_KEY_ID
|
||||
secret_access_key: OBJECT_STORAGE_KEY
|
||||
|
||||
campaigns:
|
||||
# Optional; defaults to /usr/local/share/narratio/campaigns.
|
||||
@@ -94,7 +93,7 @@ audita:
|
||||
# Optional overrides; defaults shown explicitly where applicable.
|
||||
binary: audita
|
||||
timeout: 3h
|
||||
llm_api_key_env: AUDITA_LLM_API_KEY
|
||||
llm_api_key: AUDITA_LLM_API_KEY
|
||||
modules: [glossary, homophones, spoken_word, grammar]
|
||||
base_url: ""
|
||||
model: ""
|
||||
|
||||
@@ -8,8 +8,8 @@ storage:
|
||||
bucket: my-dnd-archive
|
||||
root_prefix: dnd
|
||||
region: us-east-1
|
||||
access_key_id_env: OBJECT_STORAGE_KEY_ID
|
||||
secret_access_key_env: OBJECT_STORAGE_KEY
|
||||
access_key_id: OBJECT_STORAGE_KEY_ID
|
||||
secret_access_key: OBJECT_STORAGE_KEY
|
||||
|
||||
campaigns:
|
||||
root: /usr/local/share/narratio/campaigns
|
||||
@@ -57,7 +57,7 @@ seriatim:
|
||||
audita:
|
||||
binary: audita
|
||||
timeout: 3h
|
||||
llm_api_key_env: AUDITA_LLM_API_KEY
|
||||
llm_api_key: AUDITA_LLM_API_KEY
|
||||
modules: [glossary, homophones, spoken_word, grammar]
|
||||
output_schema: audita-v1
|
||||
work_dir_retention: auto
|
||||
|
||||
@@ -1,23 +1,11 @@
|
||||
scriptorium:
|
||||
binary: scriptorium
|
||||
config_path: ./scriptorium/config.yml
|
||||
artifact_families:
|
||||
character_meta:
|
||||
enabled: true
|
||||
for_each: party.characters
|
||||
prompt_id: dnd.character_meta
|
||||
output_path_pattern: artifacts/characters/{character_id}/meta.md
|
||||
member_vars:
|
||||
character_id: character_id
|
||||
character_name: character.name
|
||||
player_name: player.name
|
||||
class_summary: character.class_summary
|
||||
character_items:
|
||||
enabled: true
|
||||
for_each: party.characters
|
||||
prompt_id: dnd.character_items
|
||||
profile_id: production-placeholder
|
||||
output_path_pattern: artifacts/characters/{character_id}/items.md
|
||||
member_dependencies: [character_meta]
|
||||
inputs:
|
||||
character_meta:
|
||||
source: narratio.member_artifact.character_meta
|
||||
@@ -0,0 +1,13 @@
|
||||
scriptorium:
|
||||
artifact_families:
|
||||
character_meta:
|
||||
enabled: true
|
||||
for_each: party.characters
|
||||
prompt_id: dnd.character_meta
|
||||
profile_id: production-placeholder
|
||||
output_path_pattern: artifacts/characters/{character_id}/meta.md
|
||||
member_vars:
|
||||
character_id: character_id
|
||||
character_name: character.name
|
||||
player_name: player.name
|
||||
class_summary: character.class_summary
|
||||
11
examples/production-testing/artifacts/player_handout.yml
Normal file
11
examples/production-testing/artifacts/player_handout.yml
Normal file
@@ -0,0 +1,11 @@
|
||||
scriptorium:
|
||||
artifacts:
|
||||
player_handout:
|
||||
enabled: true
|
||||
prompt_id: dnd.player_handout
|
||||
profile_id: production-placeholder
|
||||
output_path: artifacts/player-handout.md
|
||||
inputs:
|
||||
recap:
|
||||
source: narratio.artifact.session_recap
|
||||
required: true
|
||||
11
examples/production-testing/artifacts/session_recap.yml
Normal file
11
examples/production-testing/artifacts/session_recap.yml
Normal file
@@ -0,0 +1,11 @@
|
||||
scriptorium:
|
||||
artifacts:
|
||||
session_recap:
|
||||
enabled: true
|
||||
prompt_id: dnd.session_recap
|
||||
profile_id: production-placeholder
|
||||
output_path: artifacts/session-recap.md
|
||||
inputs:
|
||||
transcript:
|
||||
source: narratio.transcript.final_trimmed
|
||||
required: true
|
||||
@@ -10,6 +10,8 @@ audita:
|
||||
binary: audita
|
||||
modules: [glossary, grammar]
|
||||
output_schema: audita-v1
|
||||
model: narratio-production-model-placeholder
|
||||
validation_model: narratio-production-validator-placeholder
|
||||
|
||||
normalize:
|
||||
output_path: transcripts/final.json
|
||||
|
||||
@@ -1,15 +1,14 @@
|
||||
# Copyable production/testing pipeline entry point. Every fragment is named
|
||||
# explicitly; Narratio never scans conf.d automatically.
|
||||
composition:
|
||||
# Copyable production-base/testing-overlay entry point. ConfigKit expands only
|
||||
# these explicit path and glob imports; Narratio never scans directories itself.
|
||||
configkit:
|
||||
imports:
|
||||
- conf.d/platform.yml
|
||||
- conf.d/storage.yml
|
||||
- conf.d/transcript.yml
|
||||
- conf.d/artifacts.yml
|
||||
- conf.d/publish.yml
|
||||
default_profile: production
|
||||
profiles:
|
||||
production:
|
||||
overlay: profiles/production.yml
|
||||
testing:
|
||||
overlay: profiles/testing.yml
|
||||
- path: conf.d/platform.yml
|
||||
- path: conf.d/storage.yml
|
||||
- path: conf.d/transcript.yml
|
||||
- path: conf.d/publish.yml
|
||||
- glob: artifacts/*.yml
|
||||
- glob: artifact-families/*.yml
|
||||
|
||||
scriptorium:
|
||||
binary: scriptorium
|
||||
config_path: ./scriptorium/config.yml
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
audita:
|
||||
model: narratio-production-model-placeholder
|
||||
validation_model: narratio-production-validator-placeholder
|
||||
|
||||
scriptorium:
|
||||
artifact_families:
|
||||
character_meta:
|
||||
profile_id: production-placeholder
|
||||
character_items:
|
||||
profile_id: production-placeholder
|
||||
@@ -4,6 +4,10 @@ audita:
|
||||
|
||||
scriptorium:
|
||||
artifacts:
|
||||
session_recap:
|
||||
profile_id: testing-placeholder
|
||||
player_handout:
|
||||
profile_id: testing-placeholder
|
||||
testing_notes:
|
||||
enabled: false
|
||||
output_path: artifacts/testing-notes.md
|
||||
|
||||
2
go.mod
2
go.mod
@@ -3,6 +3,7 @@ module gitea.maximumdirect.net/eric/narratio
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.1.2
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.17
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.16
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
|
||||
@@ -28,4 +29,5 @@ require (
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
|
||||
)
|
||||
|
||||
4
go.sum
4
go.sum
@@ -1,3 +1,5 @@
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.1.2 h1:uZ5Mvf271SLItXEvUpQYfAvCOVYj9RaoG0auCcbCcSU=
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.1.2/go.mod h1:tYWpsrbLyu05r8+1MapWp5fN6ZrX9I8hU2xEPedYmYk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho=
|
||||
@@ -36,6 +38,8 @@ github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
|
||||
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -24,7 +23,8 @@ const MaxReportOutputBytes int64 = 16 * 1024 * 1024
|
||||
type SubprocessRunnerConfig struct {
|
||||
Binary string
|
||||
Timeout time.Duration
|
||||
LLMAPIKeyEnv string
|
||||
LLMAPIKey string
|
||||
LLMAPIKeyReference string
|
||||
Modules []string
|
||||
BaseURL string
|
||||
Model string
|
||||
@@ -43,7 +43,8 @@ type SubprocessRunnerConfig struct {
|
||||
type SubprocessRunner struct {
|
||||
binary string
|
||||
timeout time.Duration
|
||||
llmAPIKeyEnv string
|
||||
llmAPIKey string
|
||||
llmAPIKeyReference string
|
||||
modules []string
|
||||
baseURL string
|
||||
model string
|
||||
@@ -62,7 +63,8 @@ type SubprocessRunner struct {
|
||||
func NewSubprocessRunnerFromConfigValues(
|
||||
binary string,
|
||||
timeout string,
|
||||
llmAPIKeyEnv string,
|
||||
llmAPIKey string,
|
||||
llmAPIKeyReference string,
|
||||
modules []string,
|
||||
baseURL string,
|
||||
model string,
|
||||
@@ -86,7 +88,8 @@ func NewSubprocessRunnerFromConfigValues(
|
||||
return NewSubprocessRunner(SubprocessRunnerConfig{
|
||||
Binary: binary,
|
||||
Timeout: parsedTimeout,
|
||||
LLMAPIKeyEnv: llmAPIKeyEnv,
|
||||
LLMAPIKey: llmAPIKey,
|
||||
LLMAPIKeyReference: llmAPIKeyReference,
|
||||
Modules: modules,
|
||||
BaseURL: baseURL,
|
||||
Model: model,
|
||||
@@ -110,6 +113,9 @@ func NewSubprocessRunner(cfg SubprocessRunnerConfig) (*SubprocessRunner, error)
|
||||
if cfg.Timeout <= 0 {
|
||||
return nil, fmt.Errorf("audita timeout must be > 0")
|
||||
}
|
||||
if (cfg.LLMAPIKey == "") != (strings.TrimSpace(cfg.LLMAPIKeyReference) == "") {
|
||||
return nil, fmt.Errorf("audita credential value and reference must be provided together")
|
||||
}
|
||||
for i, module := range cfg.Modules {
|
||||
if strings.TrimSpace(module) == "" {
|
||||
return nil, fmt.Errorf("audita module at index %d is empty", i)
|
||||
@@ -152,7 +158,8 @@ func NewSubprocessRunner(cfg SubprocessRunnerConfig) (*SubprocessRunner, error)
|
||||
return &SubprocessRunner{
|
||||
binary: strings.TrimSpace(cfg.Binary),
|
||||
timeout: cfg.Timeout,
|
||||
llmAPIKeyEnv: strings.TrimSpace(cfg.LLMAPIKeyEnv),
|
||||
llmAPIKey: cfg.LLMAPIKey,
|
||||
llmAPIKeyReference: strings.TrimSpace(cfg.LLMAPIKeyReference),
|
||||
modules: modules,
|
||||
baseURL: strings.TrimSpace(cfg.BaseURL),
|
||||
model: strings.TrimSpace(cfg.Model),
|
||||
@@ -195,16 +202,10 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
|
||||
}
|
||||
args := r.buildArgs(req, reqModules)
|
||||
|
||||
credentialEnvVar := strings.TrimSpace(r.llmAPIKeyEnv)
|
||||
credentialPresent := false
|
||||
credentialPresent := r.llmAPIKeyReference != ""
|
||||
env := map[string]string{}
|
||||
if credentialEnvVar != "" {
|
||||
credential, ok := os.LookupEnv(credentialEnvVar)
|
||||
if !ok || strings.TrimSpace(credential) == "" {
|
||||
return PolishResult{}, fmt.Errorf("audita: required credential environment variable %s is not set", credentialEnvVar)
|
||||
}
|
||||
env["AUDITA_LLM_API_KEY"] = credential
|
||||
credentialPresent = true
|
||||
if credentialPresent {
|
||||
env["AUDITA_LLM_API_KEY"] = r.llmAPIKey
|
||||
}
|
||||
|
||||
if req.GeneratedConfigPath != "" {
|
||||
@@ -269,7 +270,7 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
|
||||
"total_llm_concurrency": r.totalLLMConcurrency,
|
||||
"proposal_llm_concurrency": r.proposalLLMConcurrency,
|
||||
"validation_llm_concurrency": r.validationLLMConcurrency,
|
||||
"credential_env_var": r.llmAPIKeyEnv,
|
||||
"credential_env_var": r.llmAPIKeyReference,
|
||||
"credential_present": credentialPresent,
|
||||
},
|
||||
}, nil
|
||||
@@ -299,7 +300,7 @@ func (r *SubprocessRunner) failureResult(req PolishRequest, modules []string, ru
|
||||
"total_llm_concurrency": r.totalLLMConcurrency,
|
||||
"proposal_llm_concurrency": r.proposalLLMConcurrency,
|
||||
"validation_llm_concurrency": r.validationLLMConcurrency,
|
||||
"credential_env_var": r.llmAPIKeyEnv,
|
||||
"credential_env_var": r.llmAPIKeyReference,
|
||||
"credential_present": credentialPresent,
|
||||
},
|
||||
}
|
||||
@@ -375,7 +376,7 @@ func (r *SubprocessRunner) writeInvocationConfig(req PolishRequest, args []strin
|
||||
"output_path": req.OutputProcessedPath,
|
||||
"report_path": req.ReportPath,
|
||||
"work_dir": req.WorkDir,
|
||||
"credential_env_var": r.llmAPIKeyEnv,
|
||||
"credential_env_var": r.llmAPIKeyReference,
|
||||
"credential_present": credentialPresent,
|
||||
}
|
||||
return writeGeneratedYAML(req.GeneratedConfigPath, payload)
|
||||
|
||||
@@ -6,16 +6,12 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "success")
|
||||
@@ -31,7 +27,8 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
||||
runner, err := NewSubprocessRunner(SubprocessRunnerConfig{
|
||||
Binary: wrapper,
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||
LLMAPIKey: "super-secret",
|
||||
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary", "homophones", "glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
@@ -48,6 +45,7 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("NewSubprocessRunner() error = %v", err)
|
||||
}
|
||||
t.Setenv("OPENAI_KEY_SOURCE", "changed-after-resolution")
|
||||
|
||||
dir := t.TempDir()
|
||||
req := PolishRequest{
|
||||
@@ -132,47 +130,31 @@ func TestSubprocessRunnerSuccessArgsEnvAndValidation(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerMissingConfiguredCredentialFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "MISSING_AUDITA_KEY",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
func TestSubprocessRunnerRejectsIncompleteResolvedCredential(t *testing.T) {
|
||||
_, err := NewSubprocessRunner(SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyReference: "MISSING_AUDITA_KEY",
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "required credential environment variable MISSING_AUDITA_KEY is not set") {
|
||||
t.Fatalf("error = %q, want missing credential guidance", err.Error())
|
||||
if err == nil || !strings.Contains(err.Error(), "credential value and reference must be provided together") {
|
||||
t.Fatalf("NewSubprocessRunner() error = %v, want credential pair error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerUnconfiguredCredentialEnvOmitsCredential(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "success")
|
||||
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
|
||||
@@ -190,9 +172,6 @@ func TestSubprocessRunnerUnconfiguredCredentialEnvOmitsCredential(t *testing.T)
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerOmitsUnspecifiedParentEnvironment(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "success")
|
||||
t.Setenv("AUDITA_INHERITED_MARKER", "inherited-from-parent")
|
||||
@@ -200,13 +179,13 @@ func TestSubprocessRunnerOmitsUnspecifiedParentEnvironment(t *testing.T) {
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
LLMAPIKeyEnv: "",
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
LLMAPIKey: "",
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
if _, err := runner.Run(context.Background(), req); err != nil {
|
||||
@@ -220,21 +199,18 @@ func TestSubprocessRunnerOmitsUnspecifiedParentEnvironment(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerOmitsModulesFlagWhenNotConfigured(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "success")
|
||||
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "",
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "",
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
if _, err := runner.Run(context.Background(), req); err != nil {
|
||||
@@ -249,19 +225,16 @@ func TestSubprocessRunnerOmitsModulesFlagWhenNotConfigured(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerOmitsBaseURLAndModelFlagsWhenNotConfigured(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "success")
|
||||
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", recordPath)
|
||||
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "",
|
||||
Report: false,
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "",
|
||||
Report: false,
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
if _, err := runner.Run(context.Background(), req); err != nil {
|
||||
@@ -279,22 +252,20 @@ func TestSubprocessRunnerOmitsBaseURLAndModelFlagsWhenNotConfigured(t *testing.T
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "fail")
|
||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: true,
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "super-secret",
|
||||
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: true,
|
||||
})
|
||||
req := auditaReqForTest(t, true)
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
@@ -310,25 +281,26 @@ func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
|
||||
if !strings.Contains(err.Error(), req.StdoutLogPath) || !strings.Contains(err.Error(), req.StderrLogPath) {
|
||||
t.Fatalf("error = %q, want stdout/stderr log paths", err.Error())
|
||||
}
|
||||
if strings.Contains(err.Error(), "super-secret") {
|
||||
t.Fatalf("failure diagnostic exposed credential value: %q", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerSubprocessFailureAddsStderrDescriptorHint(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "fail_bad_fd")
|
||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: true,
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "super-secret",
|
||||
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: true,
|
||||
})
|
||||
req := auditaReqForTest(t, true)
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
@@ -341,22 +313,20 @@ func TestSubprocessRunnerSubprocessFailureAddsStderrDescriptorHint(t *testing.T)
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerMissingOutputFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "missing_output")
|
||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "super-secret",
|
||||
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
@@ -369,22 +339,20 @@ func TestSubprocessRunnerMissingOutputFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "invalid_output")
|
||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "super-secret",
|
||||
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
@@ -397,22 +365,20 @@ func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerSegmentsMissingFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "segments_missing")
|
||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "super-secret",
|
||||
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: false,
|
||||
})
|
||||
req := auditaReqForTest(t, false)
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
@@ -425,22 +391,20 @@ func TestSubprocessRunnerSegmentsMissingFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_HELPER_MODE", "invalid_report")
|
||||
t.Setenv("OPENAI_KEY_SOURCE", "super-secret")
|
||||
t.Setenv("AUDITA_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
|
||||
runner := mustAuditaRunner(t, SubprocessRunnerConfig{
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKeyEnv: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: true,
|
||||
Binary: writeAuditaHelperWrapper(t),
|
||||
Timeout: mustParseAuditaDuration(t, "2s"),
|
||||
LLMAPIKey: "super-secret",
|
||||
LLMAPIKeyReference: "OPENAI_KEY_SOURCE",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "https://openrouter.ai/api/v1",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Report: true,
|
||||
})
|
||||
req := auditaReqForTest(t, true)
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
@@ -453,21 +417,22 @@ func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerConstructorValidation(t *testing.T) {
|
||||
_, err := NewSubprocessRunnerFromConfigValues("", "3h", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", "", "", "", "", nil, nil, "", nil, true)
|
||||
_, err := NewSubprocessRunnerFromConfigValues("", "3h", "synthetic-secret", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", "", "", "", "", nil, nil, "", nil, true)
|
||||
if err == nil {
|
||||
t.Fatal("expected binary validation error")
|
||||
}
|
||||
_, err = NewSubprocessRunnerFromConfigValues("audita", "bad", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", "", "", "", "", nil, nil, "", nil, true)
|
||||
_, err = NewSubprocessRunnerFromConfigValues("audita", "bad", "synthetic-secret", "AUDITA_LLM_API_KEY", []string{"glossary"}, "https://openrouter.ai/api/v1", "openrouter/google/gemma-4-31b-it", "", "", "", "", nil, nil, "", nil, true)
|
||||
if err == nil {
|
||||
t.Fatal("expected timeout parse error")
|
||||
}
|
||||
_, err = NewSubprocessRunner(SubprocessRunnerConfig{
|
||||
Binary: "audita",
|
||||
Timeout: mustParseAuditaDuration(t, "1s"),
|
||||
LLMAPIKeyEnv: "AUDITA_LLM_API_KEY",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "://",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
Binary: "audita",
|
||||
Timeout: mustParseAuditaDuration(t, "1s"),
|
||||
LLMAPIKey: "synthetic-secret",
|
||||
LLMAPIKeyReference: "AUDITA_LLM_API_KEY",
|
||||
Modules: []string{"glossary"},
|
||||
BaseURL: "://",
|
||||
Model: "openrouter/google/gemma-4-31b-it",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected base url validation error")
|
||||
|
||||
@@ -6,16 +6,12 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestSubprocessRunnerRunSuccessBuildsDeterministicArgsAndCapturesLogs(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
|
||||
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
|
||||
t.Setenv("SCRIPTORIUM_HELPER_MODE", "run_success")
|
||||
@@ -95,9 +91,6 @@ func TestSubprocessRunnerRunSuccessBuildsDeterministicArgsAndCapturesLogs(t *tes
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerRunExitCodeOneFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
|
||||
t.Setenv("SCRIPTORIUM_HELPER_MODE", "fail_exit1")
|
||||
t.Setenv("SCRIPTORIUM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -118,9 +111,6 @@ func TestSubprocessRunnerRunExitCodeOneFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerRunExitCodeTwoReturnsValidationFailureAndPreservesOutput(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
|
||||
t.Setenv("SCRIPTORIUM_HELPER_MODE", "fail_exit2_with_output")
|
||||
t.Setenv("SCRIPTORIUM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -144,9 +134,6 @@ func TestSubprocessRunnerRunExitCodeTwoReturnsValidationFailureAndPreservesOutpu
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerRunMissingOutputOnSuccessFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
|
||||
t.Setenv("SCRIPTORIUM_HELPER_MODE", "run_success_missing_output")
|
||||
t.Setenv("SCRIPTORIUM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -166,9 +153,6 @@ func TestSubprocessRunnerRunMissingOutputOnSuccessFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerRenderSuccess(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SCRIPTORIUM_HELPER", "1")
|
||||
t.Setenv("SCRIPTORIUM_HELPER_MODE", "render_success")
|
||||
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||
|
||||
@@ -6,16 +6,12 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestSubprocessRunnerSuccessWithReportArgsAndEnv(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "success")
|
||||
@@ -131,9 +127,6 @@ func TestSubprocessRunnerSuccessWithReportArgsAndEnv(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "fail")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -153,9 +146,6 @@ func TestSubprocessRunnerSubprocessFailure(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerMissingOutputFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "missing_output")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -172,9 +162,6 @@ func TestSubprocessRunnerMissingOutputFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "invalid_output")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -191,9 +178,6 @@ func TestSubprocessRunnerInvalidOutputJSONFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "invalid_report")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -210,9 +194,6 @@ func TestSubprocessRunnerInvalidReportJSONFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerTrimSuccessInvocationAndProvenance(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "trim_success")
|
||||
@@ -297,9 +278,6 @@ func TestSubprocessRunnerTrimSuccessInvocationAndProvenance(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerTrimSubprocessFailure(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "fail")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -319,9 +297,6 @@ func TestSubprocessRunnerTrimSubprocessFailure(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerTrimMissingOutputFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "missing_output")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -338,9 +313,6 @@ func TestSubprocessRunnerTrimMissingOutputFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerTrimInvalidOutputJSONFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "invalid_output")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -357,9 +329,6 @@ func TestSubprocessRunnerTrimInvalidOutputJSONFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerTrimOutputMissingSegmentsFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "trim_missing_segments")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -376,9 +345,6 @@ func TestSubprocessRunnerTrimOutputMissingSegmentsFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerNormalizeSuccessInvocationAndProvenance(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "normalize_success")
|
||||
@@ -453,9 +419,6 @@ func TestSubprocessRunnerNormalizeSuccessInvocationAndProvenance(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerNormalizeSubprocessFailure(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "fail")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -475,9 +438,6 @@ func TestSubprocessRunnerNormalizeSubprocessFailure(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerNormalizeMissingOutputFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "normalize_missing_output")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -494,9 +454,6 @@ func TestSubprocessRunnerNormalizeMissingOutputFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerNormalizeInvalidOutputJSONFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "normalize_invalid_output")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -513,9 +470,6 @@ func TestSubprocessRunnerNormalizeInvalidOutputJSONFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerNormalizeOutputMissingSegmentsFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "normalize_missing_segments")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -532,9 +486,6 @@ func TestSubprocessRunnerNormalizeOutputMissingSegmentsFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerNormalizeReportMissingFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "normalize_report_missing")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -551,9 +502,6 @@ func TestSubprocessRunnerNormalizeReportMissingFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerNormalizeInvalidReportJSONFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "normalize_invalid_report")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -570,9 +518,6 @@ func TestSubprocessRunnerNormalizeInvalidReportJSONFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerRenderSuccessInvocationAndProvenance(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "render_success")
|
||||
@@ -639,9 +584,6 @@ func TestSubprocessRunnerRenderSuccessInvocationAndProvenance(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerRenderWithoutTitleOmitsTitleArg(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "render_success")
|
||||
recordPath := filepath.Join(t.TempDir(), "record.json")
|
||||
@@ -663,9 +605,6 @@ func TestSubprocessRunnerRenderWithoutTitleOmitsTitleArg(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerRenderSubprocessFailure(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "fail")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -682,9 +621,6 @@ func TestSubprocessRunnerRenderSubprocessFailure(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerRenderMissingOutputFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "missing_output")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
@@ -701,9 +637,6 @@ func TestSubprocessRunnerRenderMissingOutputFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerRenderEmptyOutputFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
t.Setenv("GO_WANT_SERIATIM_HELPER", "1")
|
||||
t.Setenv("SERIATIM_HELPER_MODE", "render_empty_output")
|
||||
t.Setenv("SERIATIM_HELPER_RECORD_PATH", filepath.Join(t.TempDir(), "record.json"))
|
||||
|
||||
@@ -37,8 +37,12 @@ type s3ClientOptions struct {
|
||||
Region string
|
||||
Endpoint string
|
||||
ForcePathStyle bool
|
||||
AccessKeyID string
|
||||
SecretKey string
|
||||
Credentials *s3StaticCredentials
|
||||
}
|
||||
|
||||
type s3StaticCredentials struct {
|
||||
AccessKeyID string
|
||||
SecretAccessKey string
|
||||
}
|
||||
|
||||
var newS3Client = func(ctx context.Context, opts s3ClientOptions) (s3API, error) {
|
||||
@@ -46,11 +50,11 @@ var newS3Client = func(ctx context.Context, opts s3ClientOptions) (s3API, error)
|
||||
if strings.TrimSpace(opts.Region) != "" {
|
||||
loadOpts = append(loadOpts, awsconfig.WithRegion(strings.TrimSpace(opts.Region)))
|
||||
}
|
||||
if strings.TrimSpace(opts.AccessKeyID) != "" && strings.TrimSpace(opts.SecretKey) != "" {
|
||||
if opts.Credentials != nil {
|
||||
loadOpts = append(loadOpts, awsconfig.WithCredentialsProvider(
|
||||
credentials.NewStaticCredentialsProvider(
|
||||
strings.TrimSpace(opts.AccessKeyID),
|
||||
strings.TrimSpace(opts.SecretKey),
|
||||
opts.Credentials.AccessKeyID,
|
||||
opts.Credentials.SecretAccessKey,
|
||||
"",
|
||||
),
|
||||
))
|
||||
@@ -75,14 +79,24 @@ func NewS3BackendFromConfig(ctx context.Context, cfg config.StorageS3Config) (*S
|
||||
if bucket == "" {
|
||||
return nil, fmt.Errorf("storage.s3.bucket is required")
|
||||
}
|
||||
if (cfg.AccessKeyID == nil) != (cfg.SecretAccessKey == nil) {
|
||||
return nil, fmt.Errorf("storage.s3 access key ID and secret access key references must be configured together")
|
||||
}
|
||||
if cfg.AccessKeyID != nil && (!cfg.AccessKeyID.IsResolved() || !cfg.SecretAccessKey.IsResolved()) {
|
||||
return nil, fmt.Errorf("storage.s3 configured credential references must be resolved")
|
||||
}
|
||||
|
||||
client, err := newS3Client(ctx, s3ClientOptions{
|
||||
clientOptions := s3ClientOptions{
|
||||
Region: cfg.Region,
|
||||
Endpoint: cfg.Endpoint,
|
||||
ForcePathStyle: cfg.ForcePathStyle,
|
||||
AccessKeyID: s3CredentialFromEnv(orDefaultEnvName(cfg.AccessKeyIDEnv, config.DefaultS3AccessKeyIDEnv)),
|
||||
SecretKey: s3CredentialFromEnv(orDefaultEnvName(cfg.SecretKeyEnv, config.DefaultS3SecretAccessKeyEnv)),
|
||||
})
|
||||
}
|
||||
if cfg.AccessKeyID != nil && cfg.SecretAccessKey != nil {
|
||||
clientOptions.Credentials = &s3StaticCredentials{
|
||||
AccessKeyID: cfg.AccessKeyID.Value(), SecretAccessKey: cfg.SecretAccessKey.Value(),
|
||||
}
|
||||
}
|
||||
client, err := newS3Client(ctx, clientOptions)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build s3 client: %w", err)
|
||||
}
|
||||
@@ -93,26 +107,6 @@ func NewS3BackendFromConfig(ctx context.Context, cfg config.StorageS3Config) (*S
|
||||
}, nil
|
||||
}
|
||||
|
||||
func s3CredentialFromEnv(envVarName string) string {
|
||||
name := strings.TrimSpace(envVarName)
|
||||
if name == "" {
|
||||
return ""
|
||||
}
|
||||
value, ok := os.LookupEnv(name)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
|
||||
func orDefaultEnvName(name, fallback string) string {
|
||||
trimmed := strings.TrimSpace(name)
|
||||
if trimmed == "" {
|
||||
return fallback
|
||||
}
|
||||
return trimmed
|
||||
}
|
||||
|
||||
// List returns objects under prefix.
|
||||
func (b *S3Backend) List(ctx context.Context, prefix string) ([]ObjectInfo, error) {
|
||||
normalizedPrefix := normalizeObjectKey(prefix)
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/aws/smithy-go"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
@@ -319,8 +320,10 @@ func TestS3BackendDeleteHonorsCancellation(t *testing.T) {
|
||||
func TestNewS3BackendFromConfigUsesClientOptions(t *testing.T) {
|
||||
original := newS3Client
|
||||
t.Cleanup(func() { newS3Client = original })
|
||||
t.Setenv("OBJECT_STORAGE_KEY_ID", "id-123")
|
||||
t.Setenv("OBJECT_STORAGE_KEY", "secret-abc")
|
||||
accessKey := resolvedTestSecret(t, "NARRATIO_S3_TEST_ACCESS", "id-123")
|
||||
secretKey := resolvedTestSecret(t, "NARRATIO_S3_TEST_SECRET", "secret-abc")
|
||||
t.Setenv("NARRATIO_S3_TEST_ACCESS", "changed-id")
|
||||
t.Setenv("NARRATIO_S3_TEST_SECRET", "changed-secret")
|
||||
|
||||
var got s3ClientOptions
|
||||
newS3Client = func(_ context.Context, opts s3ClientOptions) (s3API, error) {
|
||||
@@ -329,10 +332,12 @@ func TestNewS3BackendFromConfigUsesClientOptions(t *testing.T) {
|
||||
}
|
||||
|
||||
backend, err := NewS3BackendFromConfig(context.Background(), config.StorageS3Config{
|
||||
Bucket: "my-archive",
|
||||
Region: "us-east-1",
|
||||
Endpoint: "http://localhost:9000",
|
||||
ForcePathStyle: true,
|
||||
Bucket: "my-archive",
|
||||
Region: "us-east-1",
|
||||
Endpoint: "http://localhost:9000",
|
||||
ForcePathStyle: true,
|
||||
AccessKeyID: accessKey,
|
||||
SecretAccessKey: secretKey,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewS3BackendFromConfig() error = %v", err)
|
||||
@@ -343,7 +348,7 @@ func TestNewS3BackendFromConfigUsesClientOptions(t *testing.T) {
|
||||
if got.Region != "us-east-1" || got.Endpoint != "http://localhost:9000" || !got.ForcePathStyle {
|
||||
t.Fatalf("client options = %#v, want region/endpoint/path-style values", got)
|
||||
}
|
||||
if got.AccessKeyID != "id-123" || got.SecretKey != "secret-abc" {
|
||||
if got.Credentials == nil || got.Credentials.AccessKeyID != "id-123" || got.Credentials.SecretAccessKey != "secret-abc" {
|
||||
t.Fatalf("client options credentials = %#v, want env-resolved static credentials", got)
|
||||
}
|
||||
}
|
||||
@@ -355,7 +360,21 @@ func TestNewS3BackendFromConfigRequiresBucket(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewS3BackendFromConfigFallsBackWhenCredentialEnvMissing(t *testing.T) {
|
||||
func TestNewS3BackendFromConfigRejectsIncompleteOrUnresolvedCredentials(t *testing.T) {
|
||||
unresolved := configkit.SecretEnv("NARRATIO_UNRESOLVED_S3_KEY")
|
||||
for name, cfg := range map[string]config.StorageS3Config{
|
||||
"partial": {Bucket: "my-archive", AccessKeyID: &unresolved},
|
||||
"unresolved": {Bucket: "my-archive", AccessKeyID: &unresolved, SecretAccessKey: &unresolved},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if _, err := NewS3BackendFromConfig(context.Background(), cfg); err == nil {
|
||||
t.Fatal("NewS3BackendFromConfig() succeeded, want credential error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewS3BackendFromConfigUsesDefaultProviderWithoutCredentialReferences(t *testing.T) {
|
||||
original := newS3Client
|
||||
t.Cleanup(func() { newS3Client = original })
|
||||
|
||||
@@ -366,19 +385,30 @@ func TestNewS3BackendFromConfigFallsBackWhenCredentialEnvMissing(t *testing.T) {
|
||||
}
|
||||
|
||||
_, err := NewS3BackendFromConfig(context.Background(), config.StorageS3Config{
|
||||
Bucket: "my-archive",
|
||||
Region: "us-east-1",
|
||||
AccessKeyIDEnv: "MISSING_ACCESS_KEY_ID",
|
||||
SecretKeyEnv: "MISSING_SECRET_KEY",
|
||||
Bucket: "my-archive",
|
||||
Region: "us-east-1",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewS3BackendFromConfig() error = %v", err)
|
||||
}
|
||||
if got.AccessKeyID != "" || got.SecretKey != "" {
|
||||
t.Fatalf("client options credentials = %#v, want empty fallback values", got)
|
||||
if got.Credentials != nil {
|
||||
t.Fatalf("client options credentials = %#v, want default provider chain", got.Credentials)
|
||||
}
|
||||
}
|
||||
|
||||
func resolvedTestSecret(t *testing.T, name, value string) *configkit.Secret {
|
||||
t.Helper()
|
||||
t.Setenv(name, value)
|
||||
type document struct {
|
||||
Secret *configkit.Secret `yaml:"secret"`
|
||||
}
|
||||
result, err := configkit.Load(configkit.Bytes("test.yml", []byte("secret: "+name+"\n")), configkit.Options[document]{})
|
||||
if err != nil {
|
||||
t.Fatalf("resolve test secret: %v", err)
|
||||
}
|
||||
return result.Config.Secret
|
||||
}
|
||||
|
||||
func strPtr(v string) *string { return &v }
|
||||
func int64Ptr(v int64) *int64 { return &v }
|
||||
func boolPtr(v bool) *bool { return &v }
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//go:build linux || darwin || windows
|
||||
//go:build linux || darwin
|
||||
|
||||
package subprocess
|
||||
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
//go:build !linux && !darwin && !windows
|
||||
//go:build !linux && !darwin
|
||||
|
||||
package subprocess
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
)
|
||||
|
||||
func newOwnedProcessTree() (ownedProcessTree, error) {
|
||||
|
||||
@@ -1,145 +0,0 @@
|
||||
//go:build windows
|
||||
|
||||
package subprocess
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"syscall"
|
||||
"unsafe"
|
||||
|
||||
"golang.org/x/sys/windows"
|
||||
)
|
||||
|
||||
type windowsProcessTree struct {
|
||||
job windows.Handle
|
||||
}
|
||||
|
||||
func newOwnedProcessTree() (ownedProcessTree, error) {
|
||||
return &windowsProcessTree{}, nil
|
||||
}
|
||||
|
||||
func (tree *windowsProcessTree) Start(cmd *exec.Cmd) error {
|
||||
job, err := windows.CreateJobObject(nil, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create job object: %w", err)
|
||||
}
|
||||
|
||||
limits := windows.JOBOBJECT_EXTENDED_LIMIT_INFORMATION{}
|
||||
limits.BasicLimitInformation.LimitFlags = windows.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
|
||||
if _, err := windows.SetInformationJobObject(job, windows.JobObjectExtendedLimitInformation, uintptr(unsafe.Pointer(&limits)), uint32(unsafe.Sizeof(limits))); err != nil {
|
||||
_ = windows.CloseHandle(job)
|
||||
return fmt.Errorf("configure job object: %w", err)
|
||||
}
|
||||
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{CreationFlags: windows.CREATE_SUSPENDED}
|
||||
if err := cmd.Start(); err != nil {
|
||||
_ = windows.CloseHandle(job)
|
||||
return err
|
||||
}
|
||||
|
||||
var assignErr error
|
||||
withHandleErr := cmd.Process.WithHandle(func(handle uintptr) {
|
||||
assignErr = windows.AssignProcessToJobObject(job, windows.Handle(handle))
|
||||
})
|
||||
if withHandleErr != nil {
|
||||
return failWindowsProcessStart(cmd, job, fmt.Errorf("acquire stable process handle for job assignment: %w", withHandleErr))
|
||||
}
|
||||
if assignErr != nil {
|
||||
return failWindowsProcessStart(cmd, job, fmt.Errorf("assign process to job object: %w", assignErr))
|
||||
}
|
||||
if err := resumeInitialThread(uint32(cmd.Process.Pid)); err != nil {
|
||||
return failWindowsProcessStart(cmd, job, fmt.Errorf("resume initial process thread: %w", err))
|
||||
}
|
||||
|
||||
tree.job = job
|
||||
return nil
|
||||
}
|
||||
|
||||
func failWindowsProcessStart(cmd *exec.Cmd, job windows.Handle, setupErr error) error {
|
||||
killErr := cmd.Process.Kill()
|
||||
waitErr := cmd.Wait()
|
||||
var exitErr *exec.ExitError
|
||||
if killErr == nil && errors.As(waitErr, &exitErr) {
|
||||
// A killed suspended child normally reports a non-zero exit from Wait;
|
||||
// successful reaping is not an additional setup failure.
|
||||
waitErr = nil
|
||||
}
|
||||
return joinErrors(
|
||||
setupErr,
|
||||
wrapWindowsProcessCleanupError("kill suspended child", killErr),
|
||||
wrapWindowsProcessCleanupError("reap suspended child", waitErr),
|
||||
wrapWindowsProcessCleanupError("close unowned job object", windows.CloseHandle(job)),
|
||||
)
|
||||
}
|
||||
|
||||
func wrapWindowsProcessCleanupError(operation string, err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%s: %w", operation, err)
|
||||
}
|
||||
|
||||
func resumeInitialThread(processID uint32) error {
|
||||
snapshot, err := windows.CreateToolhelp32Snapshot(windows.TH32CS_SNAPTHREAD, 0)
|
||||
if err != nil {
|
||||
return fmt.Errorf("snapshot initial thread: %w", err)
|
||||
}
|
||||
defer func() { _ = windows.CloseHandle(snapshot) }()
|
||||
|
||||
entry := windows.ThreadEntry32{Size: uint32(unsafe.Sizeof(windows.ThreadEntry32{}))}
|
||||
if err := windows.Thread32First(snapshot, &entry); err != nil {
|
||||
return fmt.Errorf("find initial thread: %w", err)
|
||||
}
|
||||
for {
|
||||
if entry.OwnerProcessID != processID {
|
||||
// Keep enumerating until the suspended process's only initial thread
|
||||
// is found.
|
||||
} else {
|
||||
thread, openErr := windows.OpenThread(windows.THREAD_SUSPEND_RESUME, false, entry.ThreadID)
|
||||
if openErr != nil {
|
||||
return fmt.Errorf("open initial thread: %w", openErr)
|
||||
}
|
||||
defer func() { _ = windows.CloseHandle(thread) }()
|
||||
if _, resumeErr := windows.ResumeThread(thread); resumeErr != nil {
|
||||
return fmt.Errorf("resume initial thread: %w", resumeErr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := windows.Thread32Next(snapshot, &entry); err != nil {
|
||||
if errors.Is(err, windows.ERROR_NO_MORE_FILES) {
|
||||
break
|
||||
}
|
||||
return fmt.Errorf("find initial thread: %w", err)
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("find initial thread: no thread found for process %d", processID)
|
||||
}
|
||||
|
||||
func (tree *windowsProcessTree) TerminateGracefully() error {
|
||||
// Windows jobs have no portable graceful signal. Terminating the owned job
|
||||
// is the safe fallback and prevents a descendant from escaping cleanup.
|
||||
return tree.terminate()
|
||||
}
|
||||
|
||||
func (tree *windowsProcessTree) TerminateForcefully() error {
|
||||
return tree.terminate()
|
||||
}
|
||||
|
||||
func (tree *windowsProcessTree) Dispose() error {
|
||||
if tree.job == 0 {
|
||||
return nil
|
||||
}
|
||||
err := windows.CloseHandle(tree.job)
|
||||
tree.job = 0
|
||||
return err
|
||||
}
|
||||
|
||||
func (tree *windowsProcessTree) terminate() error {
|
||||
if tree.job == 0 {
|
||||
return nil
|
||||
}
|
||||
return windows.TerminateJobObject(tree.job, 1)
|
||||
}
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
@@ -137,9 +136,6 @@ func TestRunFailureRedactsSensitiveTail(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunRejectsSymlinkDiagnosticWithoutTruncatingTarget(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("creating symlinks requires privileges that are not available on every Windows runner")
|
||||
}
|
||||
exe, err := os.Executable()
|
||||
if err != nil {
|
||||
t.Fatalf("os.Executable() error = %v", err)
|
||||
|
||||
@@ -254,10 +254,10 @@ func TestAssembledSplitBundleCommandsUseOneEffectiveFamilyConfiguration(t *testi
|
||||
|
||||
var validated bytes.Buffer
|
||||
if err := ConfigValidate(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath}, &validated); err != nil {
|
||||
t.Fatalf("validate production default: %v", err)
|
||||
t.Fatalf("validate production base: %v", err)
|
||||
}
|
||||
if !strings.Contains(validated.String(), "profile=production") {
|
||||
t.Fatalf("validate output = %q, want production profile", validated.String())
|
||||
if !strings.Contains(validated.String(), "profile=none") {
|
||||
t.Fatalf("validate output = %q, want unprofiled production base", validated.String())
|
||||
}
|
||||
for _, command := range []func(context.Context, []string, io.Writer) error{ConfigShow, ConfigSources} {
|
||||
if err := command(context.Background(), []string{
|
||||
@@ -268,7 +268,7 @@ func TestAssembledSplitBundleCommandsUseOneEffectiveFamilyConfiguration(t *testi
|
||||
}
|
||||
var diff bytes.Buffer
|
||||
if err := ConfigDiff(context.Background(), []string{
|
||||
"production", "testing", "--config", pipelinePath, "--campaign-file", campaignPath,
|
||||
"default", "testing", "--config", pipelinePath, "--campaign-file", campaignPath,
|
||||
}, &diff); err != nil {
|
||||
t.Fatalf("compare profiles: %v", err)
|
||||
}
|
||||
@@ -365,7 +365,7 @@ func TestAssembledSplitBundleProfileSwitchRecordsProvenanceAndLimitsReuse(t *tes
|
||||
if !reflect.DeepEqual(productionRuns, make([]int, len(names))) {
|
||||
t.Fatalf("production runs = %v, want complete reuse", productionRuns)
|
||||
}
|
||||
assertAssembledProvenance(t, store, productionSummary, "production", config.EffectivePipelineDigest(production.Pipeline))
|
||||
assertAssembledProvenance(t, store, productionSummary, "", config.EffectivePipelineDigest(production.Pipeline))
|
||||
|
||||
testingRuns := make([]int, len(names))
|
||||
testingStages := assembledSemanticStages(names, providers, testingRuns)
|
||||
@@ -415,7 +415,9 @@ func assertAssembledProvenance(t *testing.T, store *manifest.LocalStore, summary
|
||||
t.Fatal(err)
|
||||
}
|
||||
value := session.EffectiveConfig
|
||||
if value == nil || value.SelectedProfile == nil || value.SelectedProfile.Name != profile || value.EffectiveConfigDigest != digest {
|
||||
profileMatches := value != nil && ((profile == "" && value.SelectedProfile == nil) ||
|
||||
(profile != "" && value.SelectedProfile != nil && value.SelectedProfile.Name == profile))
|
||||
if !profileMatches || value.EffectiveConfigDigest != digest {
|
||||
t.Fatalf("effective configuration provenance = %#v, want profile=%q digest=%q", value, profile, digest)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -45,7 +45,7 @@ func cleanSessionWithDependencies(ctx context.Context, flags commonConfigFlags,
|
||||
if strings.TrimSpace(flags.sessionID) == "" {
|
||||
return fmt.Errorf("clean: session_id is required unless --all is set")
|
||||
}
|
||||
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), deps)
|
||||
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), flags.secretsDir.pointer(), deps)
|
||||
if err != nil {
|
||||
return fmt.Errorf("clean: %w", err)
|
||||
}
|
||||
@@ -93,7 +93,7 @@ func cleanAllLocalWithDependencies(flags commonConfigFlags, dryRun, clearCache b
|
||||
strings.TrimSpace(flags.previousSessionID) != "" {
|
||||
return fmt.Errorf("clean: --all cannot be combined with --campaign, --campaign-file, --session, a session_id, or --previous-session-id")
|
||||
}
|
||||
_, pipelineCfg, err := loadPipelineConfigWithDependencies(flags.pipelinePath, config.PipelineLoadOptions{Profile: flags.profile.pointer()}, deps)
|
||||
_, pipelineCfg, err := loadPipelineConfigWithDependencies(flags.pipelinePath, config.PipelineLoadOptions{Profile: flags.profile.pointer()}, flags.secretsDir.pointer(), deps)
|
||||
if err != nil {
|
||||
return fmt.Errorf("clean: %w", err)
|
||||
}
|
||||
|
||||
@@ -2,16 +2,19 @@ package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"io"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
type commandDependencies struct {
|
||||
diagnostics io.Writer
|
||||
loadPipelineConfig func(string, config.PipelineLoadOptions) (*config.PipelineConfig, error)
|
||||
downloadObjectToTemp func(context.Context, storage.ObjectStore, string, string) (string, error)
|
||||
readSecretsDirectory func(string) ([]os.DirEntry, error)
|
||||
loadSecretsDir func(string) (*configkit.SecretsReport, error)
|
||||
secretsBootstrap *secretsBootstrap
|
||||
newObjectStore func(context.Context, *config.Config) (storage.ObjectStore, error)
|
||||
discoverRemoteCurrentState func(context.Context, *config.Config, storage.ObjectStore) (*RemoteCurrentState, error)
|
||||
buildRestorePlan func(context.Context, *config.Config, *RemoteCurrentState, storage.ObjectStore, RestorePlanOptions) (*RestorePlan, error)
|
||||
@@ -19,14 +22,20 @@ type commandDependencies struct {
|
||||
}
|
||||
|
||||
func normalizeCommandDependencies(deps commandDependencies) commandDependencies {
|
||||
if deps.diagnostics == nil {
|
||||
deps.diagnostics = io.Discard
|
||||
}
|
||||
if deps.loadPipelineConfig == nil {
|
||||
deps.loadPipelineConfig = config.LoadPipelineWithOptions
|
||||
}
|
||||
if deps.downloadObjectToTemp == nil {
|
||||
deps.downloadObjectToTemp = storage.DownloadObjectToTemp
|
||||
}
|
||||
if deps.readSecretsDirectory == nil {
|
||||
deps.readSecretsDirectory = os.ReadDir
|
||||
if deps.loadSecretsDir == nil {
|
||||
deps.loadSecretsDir = configkit.LoadSecretsDir
|
||||
}
|
||||
if deps.secretsBootstrap == nil {
|
||||
deps.secretsBootstrap = &secretsBootstrap{}
|
||||
}
|
||||
if deps.newObjectStore == nil {
|
||||
deps.newObjectStore = storage.NewObjectStoreFromConfig
|
||||
|
||||
@@ -19,6 +19,9 @@ func executeWithPipelineExecutor(args []string, stdout, stderr io.Writer, execut
|
||||
}
|
||||
|
||||
func executeWithDependencies(args []string, stdout, stderr io.Writer, deps commandDependencies, executor pipelineExecutor) int {
|
||||
if deps.diagnostics == nil {
|
||||
deps.diagnostics = stderr
|
||||
}
|
||||
deps = normalizeCommandDependencies(deps)
|
||||
if len(args) == 0 {
|
||||
printUsage(stderr)
|
||||
|
||||
@@ -192,7 +192,7 @@ func TestExecuteRunStagePolishLoadsCredentialFromSecretsDir(t *testing.T) {
|
||||
configDir := t.TempDir()
|
||||
sessionID := "2026-05-03"
|
||||
secretsDir := filepath.Join(configDir, "secrets")
|
||||
if err := os.MkdirAll(secretsDir, secretDirectoryPrivateMode); err != nil {
|
||||
if err := os.MkdirAll(secretsDir, 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll(%q): %v", secretsDir, err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(secretsDir, "OPENROUTER_API_KEY"), []byte("from-secret-file\n"), 0o600); err != nil {
|
||||
@@ -211,15 +211,13 @@ func TestExecuteRunStagePolishLoadsCredentialFromSecretsDir(t *testing.T) {
|
||||
root: ` + workspaceRoot + `
|
||||
storage:
|
||||
backend: local
|
||||
secrets:
|
||||
env_dir: ./secrets
|
||||
whisperx:
|
||||
transcribe_url: https://example.com/transcribe
|
||||
seriatim:
|
||||
binary: ` + seriatimBinary + `
|
||||
audita:
|
||||
binary: ` + auditaBinary + `
|
||||
llm_api_key_env: OPENROUTER_API_KEY
|
||||
llm_api_key: OPENROUTER_API_KEY
|
||||
notification:
|
||||
mode: noop
|
||||
`
|
||||
@@ -264,9 +262,31 @@ inputs:
|
||||
if !strings.Contains(stdout.String(), "stage=polish executed=1 skipped=0") {
|
||||
t.Fatalf("stdout = %q, want polish execution", stdout.String())
|
||||
}
|
||||
foundReference := false
|
||||
if err := filepath.WalkDir(workRoot, func(path string, entry os.DirEntry, walkErr error) error {
|
||||
if walkErr != nil || entry.IsDir() {
|
||||
return walkErr
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if bytes.Contains(data, []byte("from-secret-file")) {
|
||||
t.Fatalf("workspace output %q exposed credential value", path)
|
||||
}
|
||||
if bytes.Contains(data, []byte("OPENROUTER_API_KEY")) {
|
||||
foundReference = true
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatalf("inspect polish outputs: %v", err)
|
||||
}
|
||||
if !foundReference {
|
||||
t.Fatal("polish outputs omitted safe credential reference")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRunFailsWhenConfiguredSecretsDirMissing(t *testing.T) {
|
||||
func TestExecuteRunFailsWhenExplicitSecretsDirMissing(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
configDir := t.TempDir()
|
||||
pipelinePath := filepath.Join(configDir, "pipeline.yml")
|
||||
@@ -277,8 +297,6 @@ func TestExecuteRunFailsWhenConfiguredSecretsDirMissing(t *testing.T) {
|
||||
root: ` + workspaceRoot + `
|
||||
storage:
|
||||
backend: local
|
||||
secrets:
|
||||
env_dir: ./missing-secrets
|
||||
whisperx:
|
||||
transcribe_url: https://example.com/transcribe
|
||||
seriatim:
|
||||
@@ -305,11 +323,11 @@ inputs:
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"run", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||
code := Execute([]string{"run", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--secrets-dir", filepath.Join(configDir, "missing-secrets")}, &stdout, &stderr)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "validate secrets env_dir") {
|
||||
if !strings.Contains(stderr.String(), "bootstrap secrets directory") {
|
||||
t.Fatalf("stderr = %q, want secrets validation error context", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
211
internal/app/config_artifacts_test.go
Normal file
211
internal/app/config_artifacts_test.go
Normal file
@@ -0,0 +1,211 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const artifactCatalogHeader = "key\tsource\tdefinitions\tenabled\toutput\tdependencies\tfamily\tcharacter\tfamily_source\tparty_source\n"
|
||||
|
||||
func TestConfigArtifactsReportsOrdinaryCatalogWithoutCampaign(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||
mustWriteTestFile(t, pipelinePath, `workspace:
|
||||
root: `+filepath.Join(dir, "workspace")+`
|
||||
whisperx:
|
||||
transcribe_url: https://transcription.example.com/transcribe
|
||||
scriptorium:
|
||||
binary: scriptorium
|
||||
artifacts:
|
||||
zeta:
|
||||
enabled: false
|
||||
output_path: artifacts/zeta.md
|
||||
alpha:
|
||||
enabled: false
|
||||
output_path: artifacts/alpha.md
|
||||
inputs:
|
||||
upstream: {source: narratio.artifact.zeta, required: true}
|
||||
`)
|
||||
|
||||
var out bytes.Buffer
|
||||
if err := ConfigArtifacts(context.Background(), []string{"--config", pipelinePath}, &out); err != nil {
|
||||
t.Fatalf("ConfigArtifacts() error = %v", err)
|
||||
}
|
||||
lines := strings.Split(strings.TrimSuffix(out.String(), "\n"), "\n")
|
||||
if len(lines) != 3 || !strings.HasPrefix(lines[1], "alpha\t") || !strings.HasPrefix(lines[2], "zeta\t") {
|
||||
t.Fatalf("catalog order/output = %q", out.String())
|
||||
}
|
||||
alpha := strings.Split(lines[1], "\t")
|
||||
if len(alpha) != 10 {
|
||||
t.Fatalf("alpha fields = %#v", alpha)
|
||||
}
|
||||
if alpha[1] != "narratio.artifact.alpha" || alpha[2] != pipelinePath || alpha[3] != "false" || alpha[4] != "artifacts/alpha.md" || alpha[5] != "zeta" {
|
||||
t.Fatalf("alpha record = %#v", alpha)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigArtifactsEmptyCatalogWritesOnlyHeader(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||
mustWriteTestFile(t, pipelinePath, "workspace:\n root: "+filepath.Join(dir, "workspace")+"\nwhisperx:\n transcribe_url: https://transcription.example.com/transcribe\n")
|
||||
var out bytes.Buffer
|
||||
if err := ConfigArtifacts(context.Background(), []string{"--config", pipelinePath}, &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.String() != artifactCatalogHeader {
|
||||
t.Fatalf("empty catalog = %q, want header only", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigArtifactsReportsGeneratedFamilyProvenance(t *testing.T) {
|
||||
pipelinePath, campaignPath := writeInspectionFamilyConfig(t)
|
||||
var out bytes.Buffer
|
||||
if err := ConfigArtifacts(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath}, &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
records := catalogRecordsByKey(t, out.String())
|
||||
meta := records["character_meta_arannis"]
|
||||
note := records["character_note_arannis"]
|
||||
partyPath := filepath.Join(filepath.Dir(campaignPath), "party.yml")
|
||||
if len(meta) != 10 || meta[6] != "character_meta" || meta[7] != "arannis" || meta[8] != pipelinePath || meta[9] != partyPath {
|
||||
t.Fatalf("generated meta record = %#v", meta)
|
||||
}
|
||||
if len(note) != 10 || note[5] != "character_meta_arannis" || note[6] != "character_note" {
|
||||
t.Fatalf("generated note record = %#v", note)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigArtifactsReportsProfiledFamilyContributors(t *testing.T) {
|
||||
pipelinePath, campaignPath := writeInspectionFamilyConfig(t)
|
||||
profilePath := filepath.Join(filepath.Dir(pipelinePath), "profiles", "testing.yml")
|
||||
mustWriteTestFile(t, profilePath, `scriptorium:
|
||||
artifact_families:
|
||||
character_note:
|
||||
prompt_id: dnd.character_note.testing
|
||||
`)
|
||||
|
||||
args := []string{"--config", pipelinePath, "--campaign-file", campaignPath, "--profile", "testing"}
|
||||
var catalog bytes.Buffer
|
||||
if err := ConfigArtifacts(context.Background(), args, &catalog); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
records := catalogRecordsByKey(t, catalog.String())
|
||||
if got := strings.Split(records["character_note_arannis"][2], ","); !reflect.DeepEqual(got, []string{pipelinePath, profilePath}) {
|
||||
t.Fatalf("generated family definitions = %#v", got)
|
||||
}
|
||||
|
||||
var sources bytes.Buffer
|
||||
if err := ConfigSources(context.Background(), args, &sources); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := "scriptorium.artifacts.character_note_arannis.prompt_id\tfamily\t" + profilePath
|
||||
if !strings.Contains(sources.String(), want) {
|
||||
t.Fatalf("sources output missing profiled family contributor %q:\n%s", want, sources.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigArtifactsReportsBaseAndProfileContributors(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||
profilePath := filepath.Join(dir, "profiles", "testing.yml")
|
||||
mustWriteTestFile(t, pipelinePath, `workspace:
|
||||
root: /srv/narratio
|
||||
whisperx:
|
||||
transcribe_url: https://transcription.example.com/transcribe
|
||||
scriptorium:
|
||||
binary: scriptorium
|
||||
artifacts:
|
||||
notes:
|
||||
enabled: false
|
||||
output_path: artifacts/notes.md
|
||||
`)
|
||||
mustWriteTestFile(t, profilePath, `scriptorium:
|
||||
artifacts:
|
||||
notes:
|
||||
enabled: true
|
||||
prompt_id: notes
|
||||
profile_only:
|
||||
enabled: false
|
||||
output_path: artifacts/profile-only.md
|
||||
`)
|
||||
|
||||
var out bytes.Buffer
|
||||
if err := ConfigArtifacts(context.Background(), []string{"--config", pipelinePath, "--profile", "testing"}, &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
records := catalogRecordsByKey(t, out.String())
|
||||
if got := strings.Split(records["notes"][2], ","); !reflect.DeepEqual(got, []string{pipelinePath, profilePath}) {
|
||||
t.Fatalf("notes definitions = %#v", got)
|
||||
}
|
||||
if got := records["profile_only"][2]; got != profilePath {
|
||||
t.Fatalf("profile-only definition = %q, want %q", got, profilePath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigArtifactsIdentityDoesNotDependOnDefinitionFilename(t *testing.T) {
|
||||
inlineDir := t.TempDir()
|
||||
inlinePath := filepath.Join(inlineDir, "pipeline.yml")
|
||||
mustWriteTestFile(t, inlinePath, artifactIdentityPipeline("", "recap"))
|
||||
importDir := t.TempDir()
|
||||
importPath := filepath.Join(importDir, "pipeline.yml")
|
||||
mustWriteTestFile(t, importPath, artifactIdentityPipeline("artifacts.yml", ""))
|
||||
mustWriteTestFile(t, filepath.Join(importDir, "artifacts.yml"), "scriptorium:\n artifacts:\n recap:\n enabled: false\n output_path: artifacts/recap.md\n")
|
||||
|
||||
for _, path := range []string{inlinePath, importPath} {
|
||||
var out bytes.Buffer
|
||||
if err := ConfigArtifacts(context.Background(), []string{"--config", path}, &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := catalogRecordsByKey(t, out.String())["recap"][1]; got != "narratio.artifact.recap" {
|
||||
t.Fatalf("source identity for %s = %q", path, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigArtifactsPropagatesWriterErrorsAndEscapesFields(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||
mustWriteTestFile(t, pipelinePath, "workspace:\n root: /srv/narratio\nwhisperx:\n transcribe_url: https://transcription.example.com/transcribe\n")
|
||||
want := errors.New("write rejected")
|
||||
err := ConfigArtifacts(context.Background(), []string{"--config", pipelinePath}, rejectingCatalogWriter{err: want})
|
||||
if !errors.Is(err, want) {
|
||||
t.Fatalf("ConfigArtifacts() error = %v, want writer error", err)
|
||||
}
|
||||
if got := escapeCatalogField("a\\b\tc\nd\r"); got != `a\\b\tc\nd\r` {
|
||||
t.Fatalf("escaped field = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func catalogRecordsByKey(t *testing.T, output string) map[string][]string {
|
||||
t.Helper()
|
||||
lines := strings.Split(strings.TrimSuffix(output, "\n"), "\n")
|
||||
if len(lines) == 0 || lines[0]+"\n" != artifactCatalogHeader {
|
||||
t.Fatalf("catalog header = %q", output)
|
||||
}
|
||||
records := make(map[string][]string, len(lines)-1)
|
||||
for _, line := range lines[1:] {
|
||||
fields := strings.Split(line, "\t")
|
||||
records[fields[0]] = fields
|
||||
}
|
||||
return records
|
||||
}
|
||||
|
||||
func artifactIdentityPipeline(importPath, inlineKey string) string {
|
||||
document := "workspace:\n root: /srv/narratio\nwhisperx:\n transcribe_url: https://transcription.example.com/transcribe\nscriptorium:\n binary: scriptorium\n"
|
||||
if importPath != "" {
|
||||
document = "configkit:\n imports:\n - path: " + importPath + "\n" + document
|
||||
}
|
||||
if inlineKey != "" {
|
||||
document += " artifacts:\n " + inlineKey + ":\n enabled: false\n output_path: artifacts/recap.md\n"
|
||||
}
|
||||
return document
|
||||
}
|
||||
|
||||
type rejectingCatalogWriter struct{ err error }
|
||||
|
||||
func (w rejectingCatalogWriter) Write([]byte) (int, error) { return 0, w.err }
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
@@ -17,6 +18,7 @@ type inspectionFlags struct {
|
||||
campaignPath string
|
||||
campaignFilePath string
|
||||
profile singletonStringFlag
|
||||
secretsDir singletonStringFlag
|
||||
}
|
||||
|
||||
type inspectionConfig struct {
|
||||
@@ -35,7 +37,7 @@ func Config(ctx context.Context, args []string, out io.Writer) error {
|
||||
func configWithDependencies(ctx context.Context, args []string, out io.Writer, deps commandDependencies) error {
|
||||
deps = normalizeCommandDependencies(deps)
|
||||
if len(args) == 0 {
|
||||
return fmt.Errorf("config: expected subcommand: validate|show|sources|diff")
|
||||
return fmt.Errorf("config: expected subcommand: validate|show|sources|artifacts|diff")
|
||||
}
|
||||
if args[0] == "--help" || args[0] == "-h" {
|
||||
writeConfigCommandUsage(out)
|
||||
@@ -48,8 +50,10 @@ func configWithDependencies(ctx context.Context, args []string, out io.Writer, d
|
||||
return configShowWithDependencies(ctx, args[1:], out, deps)
|
||||
case "sources":
|
||||
return configSourcesWithDependencies(ctx, args[1:], out, deps)
|
||||
case "artifacts":
|
||||
return configArtifactsWithDependencies(ctx, args[1:], out, deps)
|
||||
case "diff":
|
||||
return ConfigDiff(ctx, args[1:], out)
|
||||
return configDiffWithDependencies(ctx, args[1:], out, deps)
|
||||
default:
|
||||
return fmt.Errorf("config: unknown subcommand %q", args[0])
|
||||
}
|
||||
@@ -139,6 +143,70 @@ func configSourcesWithDependencies(_ context.Context, args []string, out io.Writ
|
||||
return nil
|
||||
}
|
||||
|
||||
// ConfigArtifacts validates and reports the effective concrete artifact catalog
|
||||
// without loading session state or runtime collaborators.
|
||||
func ConfigArtifacts(ctx context.Context, args []string, out io.Writer) error {
|
||||
return configArtifactsWithDependencies(ctx, args, out, commandDependencies{})
|
||||
}
|
||||
|
||||
func configArtifactsWithDependencies(_ context.Context, args []string, out io.Writer, deps commandDependencies) error {
|
||||
flags, err := parseInspectionFlags("config artifacts", args, out)
|
||||
if err != nil {
|
||||
if errors.Is(err, flag.ErrHelp) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
resolved, err := resolveInspectionConfigWithDependencies(flags, deps)
|
||||
if err != nil {
|
||||
return fmt.Errorf("config artifacts: %w", err)
|
||||
}
|
||||
records, err := config.EffectiveArtifactCatalog(resolved.Pipeline, resolved.Party)
|
||||
if err != nil {
|
||||
return fmt.Errorf("config artifacts: %w", err)
|
||||
}
|
||||
if _, err := fmt.Fprintln(out, "key\tsource\tdefinitions\tenabled\toutput\tdependencies\tfamily\tcharacter\tfamily_source\tparty_source"); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, record := range records {
|
||||
fields := []string{
|
||||
record.Key,
|
||||
record.SourceID,
|
||||
strings.Join(normalizedCatalogPaths(record.DefinitionSources), ","),
|
||||
strconv.FormatBool(record.Enabled),
|
||||
record.OutputPath,
|
||||
strings.Join(record.Dependencies, ","),
|
||||
record.FamilyKey,
|
||||
record.CharacterID,
|
||||
config.NormalizedConfigurationPath(record.FamilySource),
|
||||
config.NormalizedConfigurationPath(record.PartySource),
|
||||
}
|
||||
for index := range fields {
|
||||
fields[index] = escapeCatalogField(fields[index])
|
||||
}
|
||||
if _, err := fmt.Fprintln(out, strings.Join(fields, "\t")); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizedCatalogPaths(paths []string) []string {
|
||||
result := make([]string, len(paths))
|
||||
for index, path := range paths {
|
||||
result[index] = config.NormalizedConfigurationPath(path)
|
||||
}
|
||||
sort.Strings(result)
|
||||
return result
|
||||
}
|
||||
|
||||
func escapeCatalogField(value string) string {
|
||||
value = strings.ReplaceAll(value, "\\", "\\\\")
|
||||
value = strings.ReplaceAll(value, "\t", "\\t")
|
||||
value = strings.ReplaceAll(value, "\r", "\\r")
|
||||
return strings.ReplaceAll(value, "\n", "\\n")
|
||||
}
|
||||
|
||||
func parseInspectionFlags(command string, args []string, out io.Writer) (inspectionFlags, error) {
|
||||
fs := flag.NewFlagSet(command, flag.ContinueOnError)
|
||||
fs.SetOutput(out)
|
||||
@@ -148,8 +216,10 @@ func parseInspectionFlags(command string, args []string, out io.Writer) (inspect
|
||||
fs.StringVar(&flags.campaignFilePath, "campaign-file", "", "path to campaign.yml")
|
||||
flags.profile.name = "profile"
|
||||
fs.Var(&flags.profile, "profile", "named pipeline profile")
|
||||
flags.secretsDir.name = "secrets-dir"
|
||||
fs.Var(&flags.secretsDir, "secrets-dir", "secrets directory (optional; defaults beside pipeline.yml)")
|
||||
fs.Usage = func() {
|
||||
fmt.Fprintf(out, "Usage: narratio %s [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>]\n\n", command)
|
||||
fmt.Fprintf(out, "Usage: narratio %s [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--profile <name>] [--secrets-dir <path>]\n\n", command)
|
||||
fmt.Fprintln(out, "Resolves configuration without a session, workspace, manifest, adapters, or external services.")
|
||||
fmt.Fprintln(out)
|
||||
fmt.Fprintln(out, "Flags:")
|
||||
@@ -168,7 +238,7 @@ func parseInspectionFlags(command string, args []string, out io.Writer) (inspect
|
||||
}
|
||||
|
||||
func resolveInspectionConfigWithDependencies(flags inspectionFlags, deps commandDependencies) (*inspectionConfig, error) {
|
||||
pipelinePath, pipeline, err := loadPipelineConfigWithDependencies(flags.pipelinePath, config.PipelineLoadOptions{Profile: flags.profile.pointer()}, deps)
|
||||
pipelinePath, pipeline, err := loadPipelineConfigWithDependencies(flags.pipelinePath, config.PipelineLoadOptions{Profile: flags.profile.pointer()}, flags.secretsDir.pointer(), deps)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -253,9 +323,9 @@ func inspectionProfile(pipeline *config.PipelineConfig) string {
|
||||
}
|
||||
|
||||
func writeConfigCommandUsage(out io.Writer) {
|
||||
fmt.Fprintln(out, "Usage: narratio config <validate|show|sources|diff>")
|
||||
fmt.Fprintln(out, "Usage: narratio config <validate|show|sources|artifacts|diff>")
|
||||
fmt.Fprintln(out)
|
||||
fmt.Fprintln(out, "Use config validate to check an effective pipeline, config show to print normalized YAML, config sources to report ownership, or config diff to compare two profiles.")
|
||||
fmt.Fprintln(out, "Use config validate to check an effective pipeline, config show to print normalized YAML, config sources to report ownership, config artifacts to list concrete artifacts, or config diff to compare two profiles.")
|
||||
}
|
||||
|
||||
func writeInspectionSourceHeader(out io.Writer, resolved *inspectionConfig) error {
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
@@ -21,13 +22,14 @@ func TestConfigCommandHelpAndDispatch(t *testing.T) {
|
||||
want string
|
||||
code int
|
||||
}{
|
||||
{name: "top level help", args: []string{"config", "--help"}, want: "Usage: narratio config <validate|show|sources|diff>"},
|
||||
{name: "top level help", args: []string{"config", "--help"}, want: "Usage: narratio config <validate|show|sources|artifacts|diff>"},
|
||||
{name: "validate help", args: []string{"config", "validate", "--help"}, want: "Usage: narratio config validate"},
|
||||
{name: "show help", args: []string{"config", "show", "--help"}, want: "Usage: narratio config show"},
|
||||
{name: "sources help", args: []string{"config", "sources", "--help"}, want: "Usage: narratio config sources"},
|
||||
{name: "artifacts help", args: []string{"config", "artifacts", "--help"}, want: "Usage: narratio config artifacts"},
|
||||
{name: "diff help", args: []string{"config", "diff", "--help"}, want: "Usage: narratio config diff"},
|
||||
{name: "unknown", args: []string{"config", "unknown"}, want: `config: unknown subcommand "unknown"`, code: 1},
|
||||
{name: "missing", args: []string{"config"}, want: "config: expected subcommand: validate|show|sources|diff", code: 1},
|
||||
{name: "missing", args: []string{"config"}, want: "config: expected subcommand: validate|show|sources|artifacts|diff", code: 1},
|
||||
{name: "session flag", args: []string{"config", "validate", "--session", "session.yml"}, want: "flag provided but not defined", code: 1},
|
||||
{name: "stage flag", args: []string{"config", "show", "--from", "prepare"}, want: "flag provided but not defined", code: 1},
|
||||
{name: "force flag", args: []string{"config", "show", "--force"}, want: "flag provided but not defined", code: 1},
|
||||
@@ -52,7 +54,7 @@ func TestExecuteConfigUsesInvocationPipelineLoader(t *testing.T) {
|
||||
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||
mustWriteTestFile(t, pipelinePath, "workspace:\n root: "+filepath.Join(dir, "workspace")+"\nwhisperx:\n transcribe_url: https://transcription.example.com/transcribe\n")
|
||||
|
||||
for _, subcommand := range []string{"validate", "show", "sources"} {
|
||||
for _, subcommand := range []string{"validate", "show", "sources", "artifacts"} {
|
||||
t.Run(subcommand, func(t *testing.T) {
|
||||
calls := 0
|
||||
deps := commandDependencies{
|
||||
@@ -91,8 +93,6 @@ func TestConfigValidateAndShowPipelineOnlyAreSideEffectFree(t *testing.T) {
|
||||
defer server.Close()
|
||||
mustWriteTestFile(t, pipelinePath, `workspace:
|
||||
root: `+workspaceRoot+`
|
||||
secrets:
|
||||
env_dir: `+secretsDir+`
|
||||
whisperx:
|
||||
transcribe_url: `+server.URL+`
|
||||
`)
|
||||
@@ -207,6 +207,56 @@ scriptorium:
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigShowPreservesNullAndEmptyCollections(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||
mustWriteTestFile(t, pipelinePath, `workspace:
|
||||
root: /tmp/narratio-work
|
||||
whisperx:
|
||||
transcribe_url: https://transcription.example.test/transcribe
|
||||
audita:
|
||||
modules: null
|
||||
scriptorium:
|
||||
artifacts: null
|
||||
`)
|
||||
mustWriteTestFile(t, filepath.Join(dir, "profiles", "empty.yml"), `audita:
|
||||
modules: []
|
||||
scriptorium:
|
||||
artifacts: {}
|
||||
`)
|
||||
|
||||
var base, empty bytes.Buffer
|
||||
if err := ConfigShow(context.Background(), []string{"--config", pipelinePath}, &base); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ConfigShow(context.Background(), []string{"--config", pipelinePath, "--profile", "empty"}, &empty); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []string{"modules: null\n", "artifacts: null\n"} {
|
||||
if !strings.Contains(base.String(), want) {
|
||||
t.Fatalf("base show output missing %q:\n%s", want, base.String())
|
||||
}
|
||||
}
|
||||
for _, want := range []string{"modules: []\n", "artifacts: {}\n"} {
|
||||
if !strings.Contains(empty.String(), want) {
|
||||
t.Fatalf("empty-profile show output missing %q:\n%s", want, empty.String())
|
||||
}
|
||||
}
|
||||
|
||||
var diff bytes.Buffer
|
||||
if err := ConfigDiff(context.Background(), []string{"default", "empty", "--config", pipelinePath}, &diff); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"changed\taudita.modules\tnull\t[]\n",
|
||||
"changed\tscriptorium.artifacts\tnull\t{}\n",
|
||||
} {
|
||||
if !strings.Contains(diff.String(), want) {
|
||||
t.Fatalf("diff output missing %q:\n%s", want, diff.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigDiffComparesExpandedFamiliesAndPublishRules(t *testing.T) {
|
||||
pipelinePath, campaignPath := writeInspectionFamilyConfig(t)
|
||||
writeInspectionProfiles(t, pipelinePath, `scriptorium:
|
||||
@@ -276,6 +326,103 @@ func TestConfigDiffReportsEqualityAndRejectsInvalidInput(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigDiffBootstrapsSecretsOnceForBothProfiles(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, _, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||
writeInspectionProfiles(t, pipelinePath, "whisperx:\n language: en\n", "whisperx:\n language: fr\n")
|
||||
secretsDir := t.TempDir()
|
||||
const secretName = "NARRATIO_DIFF_BOOTSTRAP_REPORT"
|
||||
restoreEnvAfterTest(t, secretName)
|
||||
if err := os.Unsetenv(secretName); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mustWriteSecretFile(t, filepath.Join(secretsDir, secretName), "diff-secret-must-not-appear\n")
|
||||
bootstrapCalls := 0
|
||||
var diagnostics bytes.Buffer
|
||||
deps := commandDependencies{loadSecretsDir: func(got string) (*configkit.SecretsReport, error) {
|
||||
bootstrapCalls++
|
||||
if got != secretsDir {
|
||||
t.Fatalf("secrets directory = %q, want %q", got, secretsDir)
|
||||
}
|
||||
return configkit.LoadSecretsDir(got)
|
||||
}, diagnostics: &diagnostics}
|
||||
|
||||
if err := configDiffWithDependencies(context.Background(), []string{
|
||||
"production", "testing", "--config", pipelinePath, "--secrets-dir", secretsDir,
|
||||
}, io.Discard, deps); err != nil {
|
||||
t.Fatalf("config diff: %v", err)
|
||||
}
|
||||
if bootstrapCalls != 1 {
|
||||
t.Fatalf("bootstrap calls = %d, want 1", bootstrapCalls)
|
||||
}
|
||||
if got := diagnostics.String(); strings.Count(got, `name="`+secretName+`"`) != 1 {
|
||||
t.Fatalf("bootstrap report count = %d, want 1; diagnostics = %q", strings.Count(got, `name="`+secretName+`"`), got)
|
||||
}
|
||||
if strings.Contains(diagnostics.String(), "diff-secret-must-not-appear") {
|
||||
t.Fatalf("bootstrap diagnostics exposed secret value: %q", diagnostics.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigInspectionOutputsNeverExposeResolvedCredentialValues(t *testing.T) {
|
||||
const firstCanary = "inspection-credential-canary-one"
|
||||
const secondCanary = "inspection-credential-canary-two"
|
||||
for name, value := range map[string]string{
|
||||
"NARRATIO_INSPECT_ACCESS": firstCanary,
|
||||
"NARRATIO_INSPECT_SECRET": firstCanary,
|
||||
"NARRATIO_INSPECT_LLM_ONE": firstCanary,
|
||||
"NARRATIO_INSPECT_LLM_TWO": secondCanary,
|
||||
} {
|
||||
t.Setenv(name, value)
|
||||
}
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, _, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||
pipelineData, err := os.ReadFile(pipelinePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pipelineYAML := strings.Replace(string(pipelineData), " bucket: test-bucket\n", ` bucket: test-bucket
|
||||
access_key_id: NARRATIO_INSPECT_ACCESS
|
||||
secret_access_key: NARRATIO_INSPECT_SECRET
|
||||
`, 1)
|
||||
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writeInspectionProfiles(t, pipelinePath,
|
||||
"audita:\n model: production\n llm_api_key: NARRATIO_INSPECT_LLM_ONE\n",
|
||||
"audita:\n model: testing\n llm_api_key: NARRATIO_INSPECT_LLM_TWO\n",
|
||||
)
|
||||
|
||||
var show bytes.Buffer
|
||||
if err := ConfigShow(context.Background(), []string{"--config", pipelinePath, "--profile", "production"}, &show); err != nil {
|
||||
t.Fatalf("config show: %v", err)
|
||||
}
|
||||
var sources bytes.Buffer
|
||||
if err := ConfigSources(context.Background(), []string{"--config", pipelinePath, "--profile", "production"}, &sources); err != nil {
|
||||
t.Fatalf("config sources: %v", err)
|
||||
}
|
||||
var catalog bytes.Buffer
|
||||
if err := ConfigArtifacts(context.Background(), []string{"--config", pipelinePath, "--profile", "production"}, &catalog); err != nil {
|
||||
t.Fatalf("config artifacts: %v", err)
|
||||
}
|
||||
var diff bytes.Buffer
|
||||
if err := ConfigDiff(context.Background(), []string{"production", "testing", "--config", pipelinePath}, &diff); err != nil {
|
||||
t.Fatalf("config diff: %v", err)
|
||||
}
|
||||
for label, output := range map[string]string{"show": show.String(), "sources": sources.String(), "artifacts": catalog.String(), "diff": diff.String()} {
|
||||
if strings.Contains(output, firstCanary) || strings.Contains(output, secondCanary) {
|
||||
t.Fatalf("%s output exposed credential value: %q", label, output)
|
||||
}
|
||||
}
|
||||
for _, reference := range []string{"NARRATIO_INSPECT_ACCESS", "NARRATIO_INSPECT_SECRET", "NARRATIO_INSPECT_LLM_ONE"} {
|
||||
if !strings.Contains(show.String(), reference) {
|
||||
t.Fatalf("show omitted safe credential reference %q: %q", reference, show.String())
|
||||
}
|
||||
}
|
||||
if !strings.Contains(diff.String(), "NARRATIO_INSPECT_LLM_ONE") || !strings.Contains(diff.String(), "NARRATIO_INSPECT_LLM_TWO") {
|
||||
t.Fatalf("diff omitted safe credential references: %q", diff.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigCommandsInspectMaintainedSplitBundleWithoutRuntimeState(t *testing.T) {
|
||||
examplesDir := filepath.Join("..", "..", "examples")
|
||||
pipelinePath := filepath.Join(examplesDir, "production-testing", "pipeline.yml")
|
||||
@@ -285,13 +432,13 @@ func TestConfigCommandsInspectMaintainedSplitBundleWithoutRuntimeState(t *testin
|
||||
t.Fatalf("example workspace stat = %v, want absent", err)
|
||||
}
|
||||
|
||||
for _, command := range []func(context.Context, []string, io.Writer) error{ConfigValidate, ConfigShow, ConfigSources} {
|
||||
if err := command(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath, "--profile", "production"}, io.Discard); err != nil {
|
||||
for _, command := range []func(context.Context, []string, io.Writer) error{ConfigValidate, ConfigShow, ConfigSources, ConfigArtifacts} {
|
||||
if err := command(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath}, io.Discard); err != nil {
|
||||
t.Fatalf("inspection command %T error = %v", command, err)
|
||||
}
|
||||
}
|
||||
var diff bytes.Buffer
|
||||
if err := ConfigDiff(context.Background(), []string{"production", "testing", "--config", pipelinePath, "--campaign-file", campaignPath}, &diff); err != nil {
|
||||
if err := ConfigDiff(context.Background(), []string{"default", "testing", "--config", pipelinePath, "--campaign-file", campaignPath}, &diff); err != nil {
|
||||
t.Fatalf("ConfigDiff() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(diff.String(), "audita.model") {
|
||||
@@ -304,17 +451,9 @@ func TestConfigCommandsInspectMaintainedSplitBundleWithoutRuntimeState(t *testin
|
||||
|
||||
func writeInspectionProfiles(t *testing.T, pipelinePath, production, testing string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(pipelinePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
composition := "composition:\n default_profile: production\n profiles:\n production:\n overlay: production.yml\n testing:\n overlay: testing.yml\n"
|
||||
if err := os.WriteFile(pipelinePath, append([]byte(composition), data...), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dir := filepath.Dir(pipelinePath)
|
||||
mustWriteTestFile(t, filepath.Join(dir, "production.yml"), production)
|
||||
mustWriteTestFile(t, filepath.Join(dir, "testing.yml"), testing)
|
||||
profilesDir := filepath.Join(filepath.Dir(pipelinePath), "profiles")
|
||||
mustWriteTestFile(t, filepath.Join(profilesDir, "production.yml"), production)
|
||||
mustWriteTestFile(t, filepath.Join(profilesDir, "testing.yml"), testing)
|
||||
}
|
||||
|
||||
func TestConfigCommandsSelectProfilesAndCampaigns(t *testing.T) {
|
||||
@@ -326,7 +465,7 @@ func TestConfigCommandsSelectProfilesAndCampaigns(t *testing.T) {
|
||||
if err := ConfigValidate(context.Background(), []string{"--config", pipelinePath}, &defaultOut); err != nil {
|
||||
t.Fatalf("default ConfigValidate() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(defaultOut.String(), "profile=production") {
|
||||
if !strings.Contains(defaultOut.String(), "profile=none") {
|
||||
t.Fatalf("default output = %q", defaultOut.String())
|
||||
}
|
||||
|
||||
@@ -352,7 +491,7 @@ func TestConfigCommandsSelectProfilesAndCampaigns(t *testing.T) {
|
||||
func TestConfigCommandsExpandFamiliesAndRequireCampaign(t *testing.T) {
|
||||
pipelinePath, campaignPath := writeInspectionFamilyConfig(t)
|
||||
|
||||
for _, command := range []func(context.Context, []string, io.Writer) error{ConfigValidate, ConfigShow} {
|
||||
for _, command := range []func(context.Context, []string, io.Writer) error{ConfigValidate, ConfigShow, ConfigArtifacts} {
|
||||
var out bytes.Buffer
|
||||
err := command(context.Background(), []string{"--config", pipelinePath}, &out)
|
||||
if err == nil || !strings.Contains(err.Error(), "artifact_families requires a campaign") {
|
||||
@@ -398,8 +537,9 @@ func TestConfigValidateReportsCanonicalPartyAndImportErrors(t *testing.T) {
|
||||
|
||||
dir := t.TempDir()
|
||||
rootPath := filepath.Join(dir, "pipeline.yml")
|
||||
mustWriteTestFile(t, rootPath, `composition:
|
||||
imports: [conf.yml]
|
||||
mustWriteTestFile(t, rootPath, `configkit:
|
||||
imports:
|
||||
- path: conf.yml
|
||||
workspace:
|
||||
root: /one
|
||||
whisperx:
|
||||
@@ -407,7 +547,7 @@ whisperx:
|
||||
`)
|
||||
mustWriteTestFile(t, filepath.Join(dir, "conf.yml"), "workspace:\n root: /two\n")
|
||||
err := ConfigValidate(context.Background(), []string{"--config", rootPath}, io.Discard)
|
||||
if err == nil || !strings.Contains(err.Error(), "workspace.root") || !strings.Contains(err.Error(), "pipeline.yml") || !strings.Contains(err.Error(), "conf.yml") {
|
||||
if err == nil || !strings.Contains(err.Error(), "/workspace/root") || !strings.Contains(err.Error(), "pipeline.yml") || !strings.Contains(err.Error(), "conf.yml") {
|
||||
t.Fatalf("import diagnostic = %v", err)
|
||||
}
|
||||
}
|
||||
@@ -432,7 +572,7 @@ func TestConfigSourcesReportsStableOwnership(t *testing.T) {
|
||||
for _, want := range []string{
|
||||
"root: ",
|
||||
"imports: none",
|
||||
"profile: name=testing selection=cli overlay=",
|
||||
"profile: name=testing selection=explicit overlay=",
|
||||
"campaign: id=sample-campaign source=",
|
||||
"party: source=",
|
||||
"digest: ",
|
||||
@@ -475,15 +615,7 @@ func TestConfigSourcesReportsCanonicalFamilyAndPublishOrigins(t *testing.T) {
|
||||
|
||||
func TestConfigSourcesPreservesProfileOwnershipThroughFamilyExpansion(t *testing.T) {
|
||||
pipelinePath, campaignPath := writeInspectionFamilyConfig(t)
|
||||
data, err := os.ReadFile(pipelinePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profilePath := filepath.Join(filepath.Dir(pipelinePath), "testing.yml")
|
||||
composition := "composition:\n default_profile: testing\n profiles:\n testing:\n overlay: testing.yml\n"
|
||||
if err := os.WriteFile(pipelinePath, append([]byte(composition), data...), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profilePath := filepath.Join(filepath.Dir(pipelinePath), "profiles", "testing.yml")
|
||||
mustWriteTestFile(t, profilePath, `scriptorium:
|
||||
artifact_families:
|
||||
character_note:
|
||||
@@ -492,7 +624,7 @@ func TestConfigSourcesPreservesProfileOwnershipThroughFamilyExpansion(t *testing
|
||||
`)
|
||||
|
||||
var out bytes.Buffer
|
||||
if err := ConfigSources(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath}, &out); err != nil {
|
||||
if err := ConfigSources(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath, "--profile", "testing"}, &out); err != nil {
|
||||
t.Fatalf("ConfigSources() error = %v", err)
|
||||
}
|
||||
want := "scriptorium.artifacts.character_note_arannis.enabled\tfamily\t" + profilePath
|
||||
@@ -501,17 +633,74 @@ func TestConfigSourcesPreservesProfileOwnershipThroughFamilyExpansion(t *testing
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigSourcesAttributesInferredArtifactDependencies(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||
inputPath := filepath.Join(dir, "input-only.yml")
|
||||
profilePath := filepath.Join(dir, "profiles", "testing.yml")
|
||||
mustWriteTestFile(t, pipelinePath, `configkit:
|
||||
imports:
|
||||
- path: input-only.yml
|
||||
workspace:
|
||||
root: /srv/narratio
|
||||
whisperx:
|
||||
transcribe_url: https://transcription.example.test/transcribe
|
||||
scriptorium:
|
||||
artifacts:
|
||||
first:
|
||||
enabled: false
|
||||
output_path: artifacts/first.md
|
||||
second:
|
||||
enabled: false
|
||||
output_path: artifacts/second.md
|
||||
mixed:
|
||||
enabled: false
|
||||
output_path: artifacts/mixed.md
|
||||
depends_on: [first]
|
||||
`)
|
||||
mustWriteTestFile(t, inputPath, `scriptorium:
|
||||
artifacts:
|
||||
input_only:
|
||||
enabled: false
|
||||
output_path: artifacts/input-only.md
|
||||
inputs:
|
||||
prior:
|
||||
source: narratio.artifact.first
|
||||
`)
|
||||
mustWriteTestFile(t, profilePath, `scriptorium:
|
||||
artifacts:
|
||||
mixed:
|
||||
inputs:
|
||||
prior:
|
||||
source: narratio.artifact.second
|
||||
`)
|
||||
|
||||
var out bytes.Buffer
|
||||
if err := ConfigSources(context.Background(), []string{"--config", pipelinePath, "--profile", "testing"}, &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"scriptorium.artifacts.input_only.depends_on\timport\t" + inputPath,
|
||||
"scriptorium.artifacts.mixed.depends_on\troot\t" + pipelinePath,
|
||||
"scriptorium.artifacts.mixed.depends_on\tprofile\t" + profilePath,
|
||||
} {
|
||||
if !strings.Contains(out.String(), want) {
|
||||
t.Fatalf("sources output missing dependency contributor %q:\n%s", want, out.String())
|
||||
}
|
||||
}
|
||||
if strings.Contains(out.String(), "scriptorium.artifacts.input_only.depends_on\tdefault\tdefault") {
|
||||
t.Fatalf("inferred dependency retained default attribution:\n%s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigSourcesReportsRootImportProfileAndDefaultOwnership(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
rootPath := filepath.Join(dir, "pipeline.yml")
|
||||
importPath := filepath.Join(dir, "base.yml")
|
||||
profilePath := filepath.Join(dir, "testing.yml")
|
||||
mustWriteTestFile(t, rootPath, `composition:
|
||||
imports: [base.yml]
|
||||
default_profile: testing
|
||||
profiles:
|
||||
testing:
|
||||
overlay: testing.yml
|
||||
profilePath := filepath.Join(dir, "profiles", "testing.yml")
|
||||
mustWriteTestFile(t, rootPath, `configkit:
|
||||
imports:
|
||||
- path: base.yml
|
||||
workspace:
|
||||
root: /srv/narratio
|
||||
whisperx:
|
||||
@@ -521,13 +710,13 @@ whisperx:
|
||||
mustWriteTestFile(t, profilePath, "whisperx:\n language: fr\n")
|
||||
|
||||
var out bytes.Buffer
|
||||
if err := ConfigSources(context.Background(), []string{"--config", rootPath}, &out); err != nil {
|
||||
if err := ConfigSources(context.Background(), []string{"--config", rootPath, "--profile", "testing"}, &out); err != nil {
|
||||
t.Fatalf("ConfigSources() error = %v", err)
|
||||
}
|
||||
output := out.String()
|
||||
for _, want := range []string{
|
||||
"import: " + importPath,
|
||||
"profile: name=testing selection=default overlay=" + profilePath,
|
||||
"profile: name=testing selection=explicit overlay=" + profilePath,
|
||||
"workspace.root\troot\t" + rootPath,
|
||||
"storage.backend\timport\t" + importPath,
|
||||
"whisperx.language\tprofile\t" + profilePath,
|
||||
|
||||
@@ -16,6 +16,7 @@ type configDiffFlags struct {
|
||||
pipelinePath singletonStringFlag
|
||||
campaignPath singletonStringFlag
|
||||
campaignFilePath singletonStringFlag
|
||||
secretsDir singletonStringFlag
|
||||
}
|
||||
|
||||
type configDiffRequest struct {
|
||||
@@ -40,7 +41,11 @@ type configDiffRecord struct {
|
||||
// read-only configuration resolution boundary used by config validate, show,
|
||||
// and sources. Differences describe normalized effective values rather than
|
||||
// the layout of root, import, or overlay files.
|
||||
func ConfigDiff(_ context.Context, args []string, out io.Writer) error {
|
||||
func ConfigDiff(ctx context.Context, args []string, out io.Writer) error {
|
||||
return configDiffWithDependencies(ctx, args, out, commandDependencies{})
|
||||
}
|
||||
|
||||
func configDiffWithDependencies(_ context.Context, args []string, out io.Writer, deps commandDependencies) error {
|
||||
request, err := parseConfigDiffRequest(args, out)
|
||||
if err != nil {
|
||||
if errors.Is(err, flag.ErrHelp) {
|
||||
@@ -48,7 +53,7 @@ func ConfigDiff(_ context.Context, args []string, out io.Writer) error {
|
||||
}
|
||||
return err
|
||||
}
|
||||
resolved, err := resolveConfigDiff(request)
|
||||
resolved, err := resolveConfigDiff(request, deps)
|
||||
if err != nil {
|
||||
return fmt.Errorf("config diff: %w", err)
|
||||
}
|
||||
@@ -102,11 +107,13 @@ func parseConfigDiffRequest(args []string, out io.Writer) (configDiffRequest, er
|
||||
request.flags.pipelinePath.name = "config"
|
||||
request.flags.campaignPath.name = "campaign"
|
||||
request.flags.campaignFilePath.name = "campaign-file"
|
||||
request.flags.secretsDir.name = "secrets-dir"
|
||||
fs.Var(&request.flags.pipelinePath, "config", "path to pipeline.yml (optional; defaults searched)")
|
||||
fs.Var(&request.flags.campaignPath, "campaign", "campaign ID")
|
||||
fs.Var(&request.flags.campaignFilePath, "campaign-file", "path to campaign.yml")
|
||||
fs.Var(&request.flags.secretsDir, "secrets-dir", "secrets directory (optional; defaults beside pipeline.yml)")
|
||||
fs.Usage = func() {
|
||||
fmt.Fprintln(out, "Usage: narratio config diff <left-profile> <right-profile> [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>]")
|
||||
fmt.Fprintln(out, "Usage: narratio config diff <left-profile> <right-profile> [--config <pipeline.yml>] [--campaign <id> | --campaign-file <campaign.yml>] [--secrets-dir <path>]")
|
||||
fmt.Fprintln(out)
|
||||
fmt.Fprintln(out, "Compares fully resolved profiles without a session, workspace, manifest, adapters, or external services.")
|
||||
fmt.Fprintln(out)
|
||||
@@ -120,6 +127,9 @@ func parseConfigDiffRequest(args []string, out io.Writer) (configDiffRequest, er
|
||||
if len(profiles) != 2 || strings.TrimSpace(profiles[0]) == "" || strings.TrimSpace(profiles[1]) == "" {
|
||||
return configDiffRequest{}, fmt.Errorf("config diff: exactly two non-empty profile names are required")
|
||||
}
|
||||
if profiles[0] == profiles[1] {
|
||||
return configDiffRequest{}, fmt.Errorf("config diff: profile names must differ")
|
||||
}
|
||||
if request.flags.campaignPath.set && request.flags.campaignFilePath.set {
|
||||
return configDiffRequest{}, fmt.Errorf("config diff: --campaign and --campaign-file are mutually exclusive")
|
||||
}
|
||||
@@ -156,21 +166,22 @@ func splitConfigDiffArguments(args []string) (profiles, flagArgs []string) {
|
||||
|
||||
func configDiffStringFlag(name string) bool {
|
||||
switch name {
|
||||
case "config", "campaign", "campaign-file", "profile":
|
||||
case "config", "campaign", "campaign-file", "profile", "secrets-dir":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func resolveConfigDiff(request configDiffRequest) (*configDiffResolved, error) {
|
||||
pipelinePath, err := resolvePipelineConfigPath(request.flags.pipelinePath.value)
|
||||
func resolveConfigDiff(request configDiffRequest, deps commandDependencies) (*configDiffResolved, error) {
|
||||
deps = normalizeCommandDependencies(deps)
|
||||
pipelinePath, leftPipeline, err := loadPipelineConfigWithDependencies(request.flags.pipelinePath.value, config.PipelineLoadOptions{Profile: &request.leftProfile}, request.flags.secretsDir.pointer(), deps)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("load left pipeline profile: %w", err)
|
||||
}
|
||||
leftPipeline, rightPipeline, err := config.LoadPipelineProfilePair(pipelinePath, request.leftProfile, request.rightProfile)
|
||||
_, rightPipeline, err := loadPipelineConfigWithDependencies(request.flags.pipelinePath.value, config.PipelineLoadOptions{Profile: &request.rightProfile}, request.flags.secretsDir.pointer(), deps)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("load right pipeline profile: %w", err)
|
||||
}
|
||||
needsCampaign := pipelineHasArtifactFamilies(leftPipeline) || pipelineHasArtifactFamilies(rightPipeline)
|
||||
hasCampaignSelection := request.flags.campaignPath.set || request.flags.campaignFilePath.set
|
||||
|
||||
@@ -34,7 +34,7 @@ func retainedCommandConfig(cfg *config.Config) *commandConfig {
|
||||
return &commandConfig{Config: cfg}
|
||||
}
|
||||
|
||||
func loadCommandConfigWithDependencies(ctx context.Context, pipelineFlag, campaignFlag, campaignFileFlag, sessionFlag string, sessionOpts config.SessionLoadOptions, deps commandDependencies) (loaded *commandConfig, err error) {
|
||||
func loadCommandConfigWithDependencies(ctx context.Context, pipelineFlag, campaignFlag, campaignFileFlag, sessionFlag string, sessionOpts config.SessionLoadOptions, secretsDir *string, deps commandDependencies) (loaded *commandConfig, err error) {
|
||||
deps = normalizeCommandDependencies(deps)
|
||||
var cleanup func() error
|
||||
defer func() {
|
||||
@@ -46,7 +46,7 @@ func loadCommandConfigWithDependencies(ctx context.Context, pipelineFlag, campai
|
||||
}
|
||||
}()
|
||||
|
||||
base, err := loadPipelineCampaignConfigWithDependencies(pipelineFlag, campaignFlag, campaignFileFlag, config.PipelineLoadOptions{Profile: sessionOpts.Profile}, deps)
|
||||
base, err := loadPipelineCampaignConfigWithDependencies(pipelineFlag, campaignFlag, campaignFileFlag, config.PipelineLoadOptions{Profile: sessionOpts.Profile}, secretsDir, deps)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -135,9 +135,9 @@ func loadCommandConfigWithDependencies(ctx context.Context, pipelineFlag, campai
|
||||
return loaded, nil
|
||||
}
|
||||
|
||||
func loadPipelineCampaignConfigWithDependencies(pipelineFlag, campaignFlag, campaignFileFlag string, pipelineOpts config.PipelineLoadOptions, deps commandDependencies) (*pipelineCampaignConfig, error) {
|
||||
func loadPipelineCampaignConfigWithDependencies(pipelineFlag, campaignFlag, campaignFileFlag string, pipelineOpts config.PipelineLoadOptions, secretsDir *string, deps commandDependencies) (*pipelineCampaignConfig, error) {
|
||||
deps = normalizeCommandDependencies(deps)
|
||||
loadedPipelinePath, pipelineCfg, err := loadPipelineConfigWithDependencies(pipelineFlag, pipelineOpts, deps)
|
||||
loadedPipelinePath, pipelineCfg, err := loadPipelineConfigWithDependencies(pipelineFlag, pipelineOpts, secretsDir, deps)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -161,12 +161,19 @@ func loadPipelineCampaignConfigWithDependencies(pipelineFlag, campaignFlag, camp
|
||||
return &loaded, nil
|
||||
}
|
||||
|
||||
func loadPipelineConfigWithDependencies(pipelineFlag string, opts config.PipelineLoadOptions, deps commandDependencies) (string, *config.PipelineConfig, error) {
|
||||
func loadPipelineConfigWithDependencies(pipelineFlag string, opts config.PipelineLoadOptions, secretsDir *string, deps commandDependencies) (string, *config.PipelineConfig, error) {
|
||||
deps = normalizeCommandDependencies(deps)
|
||||
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelineFlag)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if !deps.secretsBootstrap.selectionSet {
|
||||
deps.secretsBootstrap.selection = secretsDir
|
||||
deps.secretsBootstrap.selectionSet = true
|
||||
}
|
||||
if err := deps.secretsBootstrap.load(resolvedPipelinePath, deps.loadSecretsDir, deps.diagnostics); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
pipelineCfg, err := deps.loadPipelineConfig(resolvedPipelinePath, opts)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
|
||||
@@ -35,7 +35,7 @@ func TestLoadCommandConfigRetainsInitiallyLoadedPipeline(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
loaded, err := loadCommandConfigWithDependencies(context.Background(), pipelinePath, "", campaignPath, sessionPath, config.SessionLoadOptions{SessionID: "2026-05-03"}, deps)
|
||||
loaded, err := loadCommandConfigWithDependencies(context.Background(), pipelinePath, "", campaignPath, sessionPath, config.SessionLoadOptions{SessionID: "2026-05-03"}, nil, deps)
|
||||
if err != nil {
|
||||
t.Fatalf("loadCommandConfig() error = %v", err)
|
||||
}
|
||||
|
||||
@@ -11,9 +11,6 @@ import (
|
||||
|
||||
func newCommandObjectStoreWithDependencies(ctx context.Context, cfg *config.Config, logger *slog.Logger, deps commandDependencies) (storage.ObjectStore, error) {
|
||||
deps = normalizeCommandDependencies(deps)
|
||||
if _, err := loadSecretsFromConfigWithDependencies(cfg, logger, deps); err != nil {
|
||||
return nil, fmt.Errorf("load secrets from files: %w", err)
|
||||
}
|
||||
return initializeObjectStoreBackend(ctx, cfg, deps.newObjectStore)
|
||||
}
|
||||
|
||||
|
||||
@@ -3,8 +3,6 @@ package app
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -12,89 +10,8 @@ import (
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
func TestNewCommandObjectStoreLoadsSecretsBeforeFactory(t *testing.T) {
|
||||
accessKeyEnv := "NARRATIO_TEST_COMMAND_STORE_KEY_ID"
|
||||
secretKeyEnv := "NARRATIO_TEST_COMMAND_STORE_SECRET"
|
||||
restoreEnvAfterTest(t, accessKeyEnv, secretKeyEnv)
|
||||
|
||||
secretsDir := t.TempDir()
|
||||
mustWriteSecretFile(t, filepath.Join(secretsDir, accessKeyEnv), "loaded-key-id\n")
|
||||
mustWriteSecretFile(t, filepath.Join(secretsDir, secretKeyEnv), "loaded-secret\n")
|
||||
|
||||
cfg := commandObjectStoreTestConfig(secretsDir)
|
||||
fake := &storage.FakeBackend{}
|
||||
called := false
|
||||
deps := commandDependencies{newObjectStore: func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
called = true
|
||||
if got := os.Getenv(accessKeyEnv); got != "loaded-key-id" {
|
||||
return nil, errors.New("access key was not loaded before object store init")
|
||||
}
|
||||
if got := os.Getenv(secretKeyEnv); got != "loaded-secret" {
|
||||
return nil, errors.New("secret key was not loaded before object store init")
|
||||
}
|
||||
return fake, nil
|
||||
}}
|
||||
|
||||
store, err := newCommandObjectStoreWithDependencies(context.Background(), cfg, nil, deps)
|
||||
if err != nil {
|
||||
t.Fatalf("newCommandObjectStore() error = %v", err)
|
||||
}
|
||||
if store != fake {
|
||||
t.Fatalf("store = %#v, want fake backend", store)
|
||||
}
|
||||
if !called {
|
||||
t.Fatal("object store factory was not called")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewCommandObjectStorePreservesExistingEnv(t *testing.T) {
|
||||
accessKeyEnv := "NARRATIO_TEST_COMMAND_STORE_EXISTING_KEY_ID"
|
||||
secretKeyEnv := "NARRATIO_TEST_COMMAND_STORE_EXISTING_SECRET"
|
||||
t.Setenv(accessKeyEnv, "existing-key-id")
|
||||
t.Setenv(secretKeyEnv, "existing-secret")
|
||||
|
||||
secretsDir := t.TempDir()
|
||||
mustWriteSecretFile(t, filepath.Join(secretsDir, accessKeyEnv), "file-key-id\n")
|
||||
mustWriteSecretFile(t, filepath.Join(secretsDir, secretKeyEnv), "file-secret\n")
|
||||
|
||||
cfg := commandObjectStoreTestConfig(secretsDir)
|
||||
deps := commandDependencies{newObjectStore: func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
if got := os.Getenv(accessKeyEnv); got != "existing-key-id" {
|
||||
return nil, errors.New("existing access key was overwritten")
|
||||
}
|
||||
if got := os.Getenv(secretKeyEnv); got != "existing-secret" {
|
||||
return nil, errors.New("existing secret key was overwritten")
|
||||
}
|
||||
return &storage.FakeBackend{}, nil
|
||||
}}
|
||||
|
||||
if _, err := newCommandObjectStoreWithDependencies(context.Background(), cfg, nil, deps); err != nil {
|
||||
t.Fatalf("newCommandObjectStore() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewCommandObjectStoreSecretErrorStopsFactory(t *testing.T) {
|
||||
cfg := commandObjectStoreTestConfig(filepath.Join(t.TempDir(), "missing"))
|
||||
called := false
|
||||
deps := commandDependencies{newObjectStore: func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
called = true
|
||||
return &storage.FakeBackend{}, nil
|
||||
}}
|
||||
|
||||
_, err := newCommandObjectStoreWithDependencies(context.Background(), cfg, nil, deps)
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
if called {
|
||||
t.Fatal("object store factory was called after secret load failure")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "load secrets from files") {
|
||||
t.Fatalf("error = %q, want secret loading context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewCommandObjectStoreFactoryErrorIsContextual(t *testing.T) {
|
||||
cfg := commandObjectStoreTestConfig("")
|
||||
cfg := commandObjectStoreTestConfig()
|
||||
deps := commandDependencies{newObjectStore: func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
return nil, errors.New("factory boom")
|
||||
}}
|
||||
@@ -108,21 +25,17 @@ func TestNewCommandObjectStoreFactoryErrorIsContextual(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func commandObjectStoreTestConfig(secretsDir string) *config.Config {
|
||||
cfg := &config.Config{
|
||||
func commandObjectStoreTestConfig() *config.Config {
|
||||
return &config.Config{
|
||||
Pipeline: &config.PipelineConfig{
|
||||
Storage: config.StorageConfig{
|
||||
Backend: "s3",
|
||||
S3: &config.StorageS3Config{
|
||||
Bucket: "test-bucket",
|
||||
AccessKeyIDEnv: "NARRATIO_TEST_COMMAND_STORE_KEY_ID",
|
||||
SecretKeyEnv: "NARRATIO_TEST_COMMAND_STORE_SECRET",
|
||||
Bucket: "test-bucket",
|
||||
AccessKeyID: testSecretReference("NARRATIO_TEST_COMMAND_STORE_KEY_ID"),
|
||||
SecretAccessKey: testSecretReference("NARRATIO_TEST_COMMAND_STORE_SECRET"),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
if strings.TrimSpace(secretsDir) != "" {
|
||||
cfg.Pipeline.Secrets = &config.SecretsConfig{EnvDir: secretsDir}
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
@@ -24,6 +24,7 @@ type commonConfigFlags struct {
|
||||
sessionID string
|
||||
previousSessionID string
|
||||
profile singletonStringFlag
|
||||
secretsDir singletonStringFlag
|
||||
}
|
||||
|
||||
func addCommonConfigFlags(fs *flag.FlagSet, flags *commonConfigFlags) {
|
||||
@@ -35,6 +36,8 @@ func addCommonConfigFlags(fs *flag.FlagSet, flags *commonConfigFlags) {
|
||||
fs.StringVar(&flags.previousSessionID, "previous-session-id", "", "expected previous session identifier")
|
||||
flags.profile.name = "profile"
|
||||
fs.Var(&flags.profile, "profile", "named pipeline profile")
|
||||
flags.secretsDir.name = "secrets-dir"
|
||||
fs.Var(&flags.secretsDir, "secrets-dir", "secrets directory (optional; defaults beside pipeline.yml)")
|
||||
}
|
||||
|
||||
func (f commonConfigFlags) sessionOptions() config.SessionLoadOptions {
|
||||
@@ -112,7 +115,7 @@ func artifactsWithDependencies(ctx context.Context, args []string, out io.Writer
|
||||
|
||||
func loadHelperContextWithDependencies(ctx context.Context, flags commonConfigFlags, needStore bool, deps commandDependencies) (*config.Config, storage.ObjectStore, []artifacts.RemoteLock, *manifest.Manifest, func(), error) {
|
||||
deps = normalizeCommandDependencies(deps)
|
||||
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), deps)
|
||||
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), flags.secretsDir.pointer(), deps)
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, nil, err
|
||||
}
|
||||
|
||||
@@ -131,7 +131,7 @@ func locksRemoveWithDependencies(ctx context.Context, args []string, out io.Writ
|
||||
|
||||
func loadLockCommandContextWithDependencies(ctx context.Context, flags commonConfigFlags, deps commandDependencies) (*config.Config, storage.ObjectStore, string, func(), error) {
|
||||
deps = normalizeCommandDependencies(deps)
|
||||
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), deps)
|
||||
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), flags.secretsDir.pointer(), deps)
|
||||
if err != nil {
|
||||
return nil, nil, "", nil, err
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ func sessionInitWithDependencies(ctx context.Context, args []string, out io.Writ
|
||||
fs.SetOutput(io.Discard)
|
||||
var pipelinePath, campaignPath, campaignFilePath, sessionID, previousSessionID, date, title, output, audioS3Prefix, audioDir string
|
||||
var profile singletonStringFlag
|
||||
var secretsDir singletonStringFlag
|
||||
var remote, force bool
|
||||
fs.StringVar(&pipelinePath, "config", "", "path to pipeline.yml (optional; defaults searched)")
|
||||
fs.StringVar(&campaignPath, "campaign", "", "campaign ID")
|
||||
@@ -41,6 +42,8 @@ func sessionInitWithDependencies(ctx context.Context, args []string, out io.Writ
|
||||
fs.StringVar(&audioDir, "audio-dir", "", "local audio directory")
|
||||
profile.name = "profile"
|
||||
fs.Var(&profile, "profile", "named pipeline profile")
|
||||
secretsDir.name = "secrets-dir"
|
||||
fs.Var(&secretsDir, "secrets-dir", "secrets directory (optional; defaults beside pipeline.yml)")
|
||||
fs.BoolVar(&remote, "remote", false, "write session.yml to S3 session prefix")
|
||||
fs.BoolVar(&force, "force", false, "overwrite existing target")
|
||||
if err := parseSessionAwareFlags("session init", fs, args, &sessionID); err != nil {
|
||||
@@ -56,7 +59,7 @@ func sessionInitWithDependencies(ctx context.Context, args []string, out io.Writ
|
||||
return fmt.Errorf("session init: --audio-dir and --audio-s3-prefix are mutually exclusive")
|
||||
}
|
||||
|
||||
base, err := loadPipelineCampaignConfigWithDependencies(pipelinePath, campaignPath, campaignFilePath, config.PipelineLoadOptions{Profile: profile.pointer()}, deps)
|
||||
base, err := loadPipelineCampaignConfigWithDependencies(pipelinePath, campaignPath, campaignFilePath, config.PipelineLoadOptions{Profile: profile.pointer()}, secretsDir.pointer(), deps)
|
||||
if err != nil {
|
||||
return fmt.Errorf("session init: %w", err)
|
||||
}
|
||||
|
||||
@@ -30,7 +30,7 @@ func sessionValidateWithDependencies(ctx context.Context, args []string, out io.
|
||||
}
|
||||
|
||||
findings := []finding{}
|
||||
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), deps)
|
||||
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), flags.secretsDir.pointer(), deps)
|
||||
if err != nil {
|
||||
findings = append(findings, errorFinding("config", err.Error()))
|
||||
return renderFindings(out, "", "", findings)
|
||||
|
||||
@@ -31,7 +31,7 @@ func statusWithDependencies(ctx context.Context, args []string, out io.Writer, d
|
||||
if strings.TrimSpace(flags.sessionID) == "" {
|
||||
return fmt.Errorf("status: session_id is required")
|
||||
}
|
||||
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), deps)
|
||||
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), flags.secretsDir.pointer(), deps)
|
||||
if err != nil {
|
||||
return fmt.Errorf("status: %w", err)
|
||||
}
|
||||
|
||||
@@ -31,7 +31,7 @@ func planWithDependencies(ctx context.Context, args []string, out io.Writer, dep
|
||||
return err
|
||||
}
|
||||
flags := request.Config
|
||||
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), deps)
|
||||
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), flags.secretsDir.pointer(), deps)
|
||||
if err != nil {
|
||||
return fmt.Errorf("plan: %w", err)
|
||||
}
|
||||
|
||||
@@ -88,7 +88,7 @@ func TestPlanShowsRunAndSkipFromManifest(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanDoesNotLoadConfiguredSecrets(t *testing.T) {
|
||||
func TestPlanSkipsAbsentConventionalSecretsDirectory(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
configDir := t.TempDir()
|
||||
pipelinePath := filepath.Join(configDir, "pipeline.yml")
|
||||
@@ -99,8 +99,6 @@ func TestPlanDoesNotLoadConfiguredSecrets(t *testing.T) {
|
||||
root: ` + workspaceRoot + `
|
||||
storage:
|
||||
backend: local
|
||||
secrets:
|
||||
env_dir: ./missing-secrets
|
||||
whisperx:
|
||||
transcribe_url: https://example.com/transcribe
|
||||
seriatim:
|
||||
@@ -128,7 +126,7 @@ inputs:
|
||||
var out bytes.Buffer
|
||||
err := Plan(context.Background(), []string{"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("Plan() error = %v, want missing runtime secrets ignored", err)
|
||||
t.Fatalf("Plan() error = %v, want absent conventional secrets directory skipped", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -37,10 +37,10 @@ func TestRunProfileSelectionFlowsThroughSharedLoader(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if language != "en" {
|
||||
t.Fatalf("default profile language = %q, want en", language)
|
||||
t.Fatalf("base configuration language = %q, want en", language)
|
||||
}
|
||||
|
||||
for _, profile := range []string{"unknown", ""} {
|
||||
for _, profile := range []string{"unknown"} {
|
||||
err := runWithPipelineExecutor(context.Background(), []string{
|
||||
"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--profile", profile,
|
||||
}, io.Discard, executor)
|
||||
@@ -48,19 +48,20 @@ func TestRunProfileSelectionFlowsThroughSharedLoader(t *testing.T) {
|
||||
t.Fatalf("profile %q error = %v, want selection rejection", profile, err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := runWithPipelineExecutor(context.Background(), []string{
|
||||
"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--profile", "",
|
||||
}, io.Discard, executor); err != nil {
|
||||
t.Fatalf("explicit empty profile should select base: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func enableCommandTestProfiles(t *testing.T, pipelinePath string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(pipelinePath)
|
||||
if err != nil {
|
||||
dir := filepath.Join(filepath.Dir(pipelinePath), "profiles")
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
composition := "composition:\n default_profile: production\n profiles:\n production:\n overlay: production.yml\n testing:\n overlay: testing.yml\n"
|
||||
if err := os.WriteFile(pipelinePath, append([]byte(composition), data...), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dir := filepath.Dir(pipelinePath)
|
||||
if err := os.WriteFile(filepath.Join(dir, "production.yml"), []byte("whisperx:\n language: en\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
@@ -78,7 +79,7 @@ inputs:
|
||||
setSessionConfigSearchPaths(t, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||
deps, storeInitCalls := remoteSessionLoaderDependencies(fake)
|
||||
|
||||
loaded, err := loadCommandConfigWithDependencies(context.Background(), pipelinePath, "", campaignPath, "", config.SessionLoadOptions{SessionID: "2026-05-03"}, deps)
|
||||
loaded, err := loadCommandConfigWithDependencies(context.Background(), pipelinePath, "", campaignPath, "", config.SessionLoadOptions{SessionID: "2026-05-03"}, nil, deps)
|
||||
if err != nil {
|
||||
t.Fatalf("loadCommandConfig() error = %v", err)
|
||||
}
|
||||
@@ -200,7 +201,7 @@ inputs:
|
||||
|
||||
var downloadedPath string
|
||||
deps.downloadObjectToTemp = captureRemoteSessionTempPath(&downloadedPath)
|
||||
loaded, err := loadCommandConfigWithDependencies(context.Background(), pipelinePath, "", campaignPath, "", config.SessionLoadOptions{SessionID: "2026-05-03"}, deps)
|
||||
loaded, err := loadCommandConfigWithDependencies(context.Background(), pipelinePath, "", campaignPath, "", config.SessionLoadOptions{SessionID: "2026-05-03"}, nil, deps)
|
||||
if err != nil {
|
||||
t.Fatalf("loadCommandConfig() error = %v", err)
|
||||
}
|
||||
@@ -246,12 +247,18 @@ inputs:
|
||||
prefix: audio/
|
||||
`)
|
||||
setSessionConfigSearchPaths(t, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||
deps := commandDependencies{newObjectStore: func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
if os.Getenv(accessKeyEnv) != "remote-session-key-id" || os.Getenv(secretKeyEnv) != "remote-session-secret" {
|
||||
return nil, fmt.Errorf("secrets were not loaded before remote session object store init")
|
||||
}
|
||||
return fake, nil
|
||||
}}
|
||||
bootstrapCalls := 0
|
||||
deps := commandDependencies{
|
||||
loadSecretsDir: func(directory string) (*configkit.SecretsReport, error) {
|
||||
bootstrapCalls++
|
||||
return configkit.LoadSecretsDir(directory)
|
||||
},
|
||||
newObjectStore: func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
if os.Getenv(accessKeyEnv) != "remote-session-key-id" || os.Getenv(secretKeyEnv) != "remote-session-secret" {
|
||||
return nil, fmt.Errorf("secrets were not loaded before remote session object store init")
|
||||
}
|
||||
return fake, nil
|
||||
}}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
@@ -259,6 +266,9 @@ inputs:
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
if bootstrapCalls != 1 {
|
||||
t.Fatalf("bootstrap calls = %d, want 1", bootstrapCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteExplicitLocalSessionPrecedenceSkipsRemote(t *testing.T) {
|
||||
@@ -468,10 +478,13 @@ func addSecretsToPipelineConfig(t *testing.T, pipelinePath, secretsDir, accessKe
|
||||
pipelineYAML := strings.Replace(
|
||||
string(pipelineData),
|
||||
"storage:\n backend: s3\n s3:\n bucket: test-bucket\n",
|
||||
"storage:\n backend: s3\n s3:\n bucket: test-bucket\n access_key_id_env: "+accessKeyEnv+"\n secret_access_key_env: "+secretKeyEnv+"\nsecrets:\n env_dir: "+secretsDir+"\n",
|
||||
"storage:\n backend: s3\n s3:\n bucket: test-bucket\n access_key_id: "+accessKeyEnv+"\n secret_access_key: "+secretKeyEnv+"\n",
|
||||
1,
|
||||
)
|
||||
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||
t.Fatalf("write pipeline: %v", err)
|
||||
}
|
||||
if err := os.Symlink(secretsDir, filepath.Join(filepath.Dir(pipelinePath), "secrets")); err != nil {
|
||||
t.Fatalf("link conventional secrets directory: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -53,7 +53,7 @@ func restoreWithDependencies(ctx context.Context, args []string, out io.Writer,
|
||||
if strings.TrimSpace(flags.sessionID) == "" {
|
||||
return fmt.Errorf("restore: session_id is required")
|
||||
}
|
||||
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), deps)
|
||||
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), flags.secretsDir.pointer(), deps)
|
||||
if err != nil {
|
||||
return fmt.Errorf("restore: %w", err)
|
||||
}
|
||||
|
||||
@@ -179,13 +179,8 @@ func TestExecuteRestoreLoadsSecretsBeforeObjectStoreInit(t *testing.T) {
|
||||
secretsDir := filepath.Join(t.TempDir(), "secrets")
|
||||
mustWriteSecretFile(t, filepath.Join(secretsDir, accessKeyEnv), "test-access-key-id\n")
|
||||
mustWriteSecretFile(t, filepath.Join(secretsDir, secretKeyEnv), "test-secret-key\n")
|
||||
f, err := os.OpenFile(pipelinePath, os.O_APPEND|os.O_WRONLY, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("open pipeline config for append: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
if _, err := f.WriteString("\nsecrets:\n env_dir: " + secretsDir + "\n"); err != nil {
|
||||
t.Fatalf("append secrets config: %v", err)
|
||||
if err := os.Symlink(secretsDir, filepath.Join(filepath.Dir(pipelinePath), "secrets")); err != nil {
|
||||
t.Fatalf("link conventional secrets directory: %v", err)
|
||||
}
|
||||
|
||||
storeInitCalled := false
|
||||
|
||||
@@ -29,7 +29,7 @@ func runWithDependencies(ctx context.Context, args []string, out io.Writer, exec
|
||||
return err
|
||||
}
|
||||
flags := request.Config
|
||||
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), deps)
|
||||
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), flags.secretsDir.pointer(), deps)
|
||||
if err != nil {
|
||||
return fmt.Errorf("run: %w", err)
|
||||
}
|
||||
|
||||
@@ -80,6 +80,8 @@ func runStageWithDependencies(ctx context.Context, args []string, out io.Writer,
|
||||
SessionPath: flags.sessionPath,
|
||||
SessionID: flags.sessionID,
|
||||
PreviousSessionID: flags.previousSessionID,
|
||||
Profile: flags.profile.pointer(),
|
||||
SecretsDir: flags.secretsDir.pointer(),
|
||||
Force: force,
|
||||
SelectedArtifacts: normalizedArtifacts,
|
||||
}, executor, deps)
|
||||
@@ -137,6 +139,8 @@ func analyzeWithDependencies(ctx context.Context, args []string, out io.Writer,
|
||||
SessionPath: flags.sessionPath,
|
||||
SessionID: flags.sessionID,
|
||||
PreviousSessionID: flags.previousSessionID,
|
||||
Profile: flags.profile.pointer(),
|
||||
SecretsDir: flags.secretsDir.pointer(),
|
||||
Force: true,
|
||||
SelectedArtifacts: normalizedArtifacts,
|
||||
}, executor, deps)
|
||||
@@ -192,6 +196,8 @@ func publishWithDependencies(ctx context.Context, args []string, out io.Writer,
|
||||
SessionPath: flags.sessionPath,
|
||||
SessionID: flags.sessionID,
|
||||
PreviousSessionID: flags.previousSessionID,
|
||||
Profile: flags.profile.pointer(),
|
||||
SecretsDir: flags.secretsDir.pointer(),
|
||||
Force: true,
|
||||
SelectedArtifacts: normalizedArtifacts,
|
||||
}, executor, deps)
|
||||
@@ -218,6 +224,8 @@ type singleStageCommand struct {
|
||||
SessionPath string
|
||||
SessionID string
|
||||
PreviousSessionID string
|
||||
Profile *string
|
||||
SecretsDir *string
|
||||
Force bool
|
||||
SelectedArtifacts []string
|
||||
}
|
||||
@@ -232,7 +240,8 @@ func runSingleStageCommand(ctx context.Context, req singleStageCommand, executor
|
||||
loaded, err := loadCommandConfigWithDependencies(ctx, req.PipelinePath, req.CampaignPath, req.CampaignFilePath, req.SessionPath, config.SessionLoadOptions{
|
||||
SessionID: req.SessionID,
|
||||
PreviousSessionID: req.PreviousSessionID,
|
||||
}, deps)
|
||||
Profile: req.Profile,
|
||||
}, req.SecretsDir, deps)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", req.CommandName, err)
|
||||
}
|
||||
|
||||
@@ -211,11 +211,18 @@ func buildDefaultAuditaRunner(cfg *config.Config) (audita.Runner, error) {
|
||||
if a.Report != nil {
|
||||
report = *a.Report
|
||||
}
|
||||
llmAPIKey := ""
|
||||
llmAPIKeyReference := ""
|
||||
if a.LLMAPIKey != nil {
|
||||
llmAPIKey = a.LLMAPIKey.Value()
|
||||
llmAPIKeyReference = a.LLMAPIKey.EnvName()
|
||||
}
|
||||
|
||||
runner, err := audita.NewSubprocessRunnerFromConfigValues(
|
||||
a.Binary,
|
||||
a.Timeout,
|
||||
a.LLMAPIKeyEnv,
|
||||
llmAPIKey,
|
||||
llmAPIKeyReference,
|
||||
append([]string(nil), a.Modules...),
|
||||
a.BaseURL,
|
||||
a.Model,
|
||||
|
||||
@@ -157,9 +157,6 @@ func initializeLockedInvocation(runtime *invocationRuntime) error {
|
||||
if err := runtime.manifestStore.Save(runtime.ctx, runtime.manifestPath, m); err != nil {
|
||||
return fmt.Errorf("save manifest identity %q: %w", runtime.manifestPath, err)
|
||||
}
|
||||
if _, err := loadSecretsFromConfig(runtime.env.Config, runtime.env.Logger); err != nil {
|
||||
return runtime.persistTerminalFailure(fmt.Errorf("load secrets from files: %w", err))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
72
internal/app/secrets_bootstrap.go
Normal file
72
internal/app/secrets_bootstrap.go
Normal file
@@ -0,0 +1,72 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
)
|
||||
|
||||
type secretsBootstrap struct {
|
||||
selection *string
|
||||
selectionSet bool
|
||||
called bool
|
||||
}
|
||||
|
||||
func (b *secretsBootstrap) load(pipelinePath string, loader func(string) (*configkit.SecretsReport, error), diagnostics io.Writer) error {
|
||||
if b == nil || b.called {
|
||||
return nil
|
||||
}
|
||||
b.called = true
|
||||
directory, required, err := selectedSecretsDirectory(pipelinePath, b.selection)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !required {
|
||||
if _, err := os.Stat(directory); errors.Is(err, os.ErrNotExist) {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("inspect conventional secrets directory %q: %w", directory, err)
|
||||
}
|
||||
}
|
||||
report, loadErr := loader(directory)
|
||||
if loadErr == nil {
|
||||
if report != nil {
|
||||
if _, err := report.WriteTo(diagnostics); err != nil {
|
||||
return fmt.Errorf("write secrets bootstrap report: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
var rendered bytes.Buffer
|
||||
if report != nil {
|
||||
_, _ = report.WriteTo(&rendered)
|
||||
}
|
||||
if rendered.Len() != 0 {
|
||||
return fmt.Errorf("bootstrap secrets directory: %w\n%s", loadErr, strings.TrimSpace(rendered.String()))
|
||||
}
|
||||
return fmt.Errorf("bootstrap secrets directory: %w", loadErr)
|
||||
}
|
||||
|
||||
func selectedSecretsDirectory(pipelinePath string, selection *string) (string, bool, error) {
|
||||
if selection != nil {
|
||||
if strings.TrimSpace(*selection) == "" {
|
||||
return "", true, fmt.Errorf("--secrets-dir requires a non-empty path")
|
||||
}
|
||||
path, err := filepath.Abs(*selection)
|
||||
if err != nil {
|
||||
return "", true, fmt.Errorf("resolve --secrets-dir %q: %w", *selection, err)
|
||||
}
|
||||
return filepath.Clean(path), true, nil
|
||||
}
|
||||
root, err := filepath.Abs(pipelinePath)
|
||||
if err != nil {
|
||||
return "", false, fmt.Errorf("resolve pipeline path for secrets discovery: %w", err)
|
||||
}
|
||||
return filepath.Join(filepath.Dir(root), "secrets"), false, nil
|
||||
}
|
||||
200
internal/app/secrets_bootstrap_test.go
Normal file
200
internal/app/secrets_bootstrap_test.go
Normal file
@@ -0,0 +1,200 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
func TestPipelineLoadSkipsAbsentConventionalSecretsDirectory(t *testing.T) {
|
||||
pipelinePath := filepath.Join(t.TempDir(), "pipeline.yml")
|
||||
if err := os.WriteFile(pipelinePath, []byte("version: 1\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secretCalls := 0
|
||||
pipelineCalls := 0
|
||||
deps := commandDependencies{
|
||||
loadSecretsDir: func(string) (*configkit.SecretsReport, error) {
|
||||
secretCalls++
|
||||
return nil, nil
|
||||
},
|
||||
loadPipelineConfig: func(string, config.PipelineLoadOptions) (*config.PipelineConfig, error) {
|
||||
pipelineCalls++
|
||||
return &config.PipelineConfig{}, nil
|
||||
},
|
||||
}
|
||||
|
||||
if _, _, err := loadPipelineConfigWithDependencies(pipelinePath, config.PipelineLoadOptions{}, nil, deps); err != nil {
|
||||
t.Fatalf("load pipeline: %v", err)
|
||||
}
|
||||
if secretCalls != 0 || pipelineCalls != 1 {
|
||||
t.Fatalf("calls = secrets %d, pipeline %d; want 0, 1", secretCalls, pipelineCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExplicitSecretsDirectoryIsResolvedFromCallerWorkingDirectory(t *testing.T) {
|
||||
callerDir := t.TempDir()
|
||||
selected := filepath.Join(callerDir, "credentials")
|
||||
if err := os.Mkdir(selected, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
original, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Chdir(callerDir); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = os.Chdir(original) })
|
||||
|
||||
got, required, err := selectedSecretsDirectory(filepath.Join(t.TempDir(), "pipeline.yml"), stringPointer("credentials"))
|
||||
if err != nil {
|
||||
t.Fatalf("select secrets: %v", err)
|
||||
}
|
||||
if !required || got != selected {
|
||||
t.Fatalf("selection = %q, required=%t; want %q, true", got, required, selected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExplicitSecretsFailurePrecedesPipelineLoad(t *testing.T) {
|
||||
pipelinePath := filepath.Join(t.TempDir(), "pipeline.yml")
|
||||
if err := os.WriteFile(pipelinePath, []byte("version: 1\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for name, selection := range map[string]string{"empty": "", "missing": filepath.Join(t.TempDir(), "missing")} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
pipelineCalls := 0
|
||||
deps := commandDependencies{loadPipelineConfig: func(string, config.PipelineLoadOptions) (*config.PipelineConfig, error) {
|
||||
pipelineCalls++
|
||||
return &config.PipelineConfig{}, nil
|
||||
}}
|
||||
if _, _, err := loadPipelineConfigWithDependencies(pipelinePath, config.PipelineLoadOptions{}, &selection, deps); err == nil {
|
||||
t.Fatal("load succeeded, want bootstrap error")
|
||||
}
|
||||
if pipelineCalls != 0 {
|
||||
t.Fatalf("pipeline load calls = %d, want 0", pipelineCalls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSecretsBootstrapPreservesExistingEnvironmentValues(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
const emptyName = "NARRATIO_BOOTSTRAP_EXISTING_EMPTY"
|
||||
const valueName = "NARRATIO_BOOTSTRAP_EXISTING_VALUE"
|
||||
const loadedName = "NARRATIO_BOOTSTRAP_NEW_VALUE"
|
||||
t.Setenv(emptyName, "")
|
||||
t.Setenv(valueName, "from-environment")
|
||||
restoreEnvAfterTest(t, loadedName)
|
||||
for name, value := range map[string]string{
|
||||
emptyName: "from-file-empty\n",
|
||||
valueName: "from-file-value\n",
|
||||
loadedName: "new-value\n",
|
||||
} {
|
||||
mustWriteSecretFile(t, filepath.Join(directory, name), value)
|
||||
}
|
||||
|
||||
bootstrap := &secretsBootstrap{selection: &directory, selectionSet: true}
|
||||
var diagnostics bytes.Buffer
|
||||
if err := bootstrap.load(filepath.Join(t.TempDir(), "pipeline.yml"), configkit.LoadSecretsDir, &diagnostics); err != nil {
|
||||
t.Fatalf("bootstrap: %v", err)
|
||||
}
|
||||
if got := diagnostics.String(); !strings.Contains(got, `status="skipped_existing"`) || !strings.Contains(got, `existing_empty=true`) {
|
||||
t.Fatalf("diagnostics lack explicit skipped-empty result: %q", got)
|
||||
}
|
||||
if got := os.Getenv(emptyName); got != "" {
|
||||
t.Fatalf("existing empty value = %q, want empty", got)
|
||||
}
|
||||
if got := os.Getenv(valueName); got != "from-environment" {
|
||||
t.Fatalf("existing value = %q, want environment value", got)
|
||||
}
|
||||
if got := os.Getenv(loadedName); got != "new-value" {
|
||||
t.Fatalf("new value = %q, want file value", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSecretsBootstrapPreservesSafePartialReportOnFailure(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
mustWriteSecretFile(t, filepath.Join(directory, "INVALID-NAME"), "must-not-appear\n")
|
||||
bootstrap := &secretsBootstrap{selection: &directory, selectionSet: true}
|
||||
err := bootstrap.load(filepath.Join(t.TempDir(), "pipeline.yml"), configkit.LoadSecretsDir, io.Discard)
|
||||
if err == nil {
|
||||
t.Fatal("bootstrap succeeded, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), `status="failed"`) || !strings.Contains(err.Error(), `name="INVALID-NAME"`) {
|
||||
t.Fatalf("error lacks partial report: %v", err)
|
||||
}
|
||||
if strings.Contains(err.Error(), "must-not-appear") {
|
||||
t.Fatalf("error exposed secret value: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteConfigShowReportsSuccessfulSecretsBootstrapOnStderr(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, _, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||
secretsDir := t.TempDir()
|
||||
const name = "NARRATIO_BOOTSTRAP_REPORT_SECRET"
|
||||
const canary = "synthetic-secret-must-not-appear"
|
||||
restoreEnvAfterTest(t, name)
|
||||
if err := os.Unsetenv(name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mustWriteSecretFile(t, filepath.Join(secretsDir, name), canary+"\n")
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
if code := Execute([]string{"config", "show", "--config", pipelinePath, "--secrets-dir", secretsDir}, &stdout, &stderr); code != 0 {
|
||||
t.Fatalf("config show exit = %d, stderr = %q", code, stderr.String())
|
||||
}
|
||||
if got := stderr.String(); !strings.Contains(got, `status="loaded"`) || !strings.Contains(got, `name="`+name+`"`) {
|
||||
t.Fatalf("stderr lacks safe bootstrap report: %q", got)
|
||||
}
|
||||
if strings.Contains(stderr.String(), canary) || strings.Contains(stdout.String(), canary) {
|
||||
t.Fatal("command output exposed secret value")
|
||||
}
|
||||
if strings.Contains(stdout.String(), `status="loaded"`) || strings.Contains(stdout.String(), name) {
|
||||
t.Fatalf("show stdout was polluted by bootstrap report: %q", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteConfigValidateReportsSkippedEmptyBeforeTypedLoadFailure(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, _, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||
const name = "NARRATIO_REVIEW_KEY"
|
||||
const canary = "synthetic-skipped-secret-must-not-appear"
|
||||
t.Setenv(name, "")
|
||||
pipelineYAML, err := os.ReadFile(pipelinePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pipelineYAML = []byte(strings.Replace(string(pipelineYAML), "audita:\n binary:", "audita:\n llm_api_key: "+name+"\n binary:", 1))
|
||||
if err := os.WriteFile(pipelinePath, pipelineYAML, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secretsDir := t.TempDir()
|
||||
mustWriteSecretFile(t, filepath.Join(secretsDir, name), canary+"\n")
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
if code := Execute([]string{"config", "validate", "--config", pipelinePath, "--secrets-dir", secretsDir}, &stdout, &stderr); code == 0 {
|
||||
t.Fatal("config validate succeeded, want empty-secret failure")
|
||||
}
|
||||
got := stderr.String()
|
||||
if !strings.Contains(got, `status="skipped_existing"`) || !strings.Contains(got, `existing_empty=true`) {
|
||||
t.Fatalf("stderr lacks useful skipped-empty diagnostic: %q", got)
|
||||
}
|
||||
if !strings.Contains(got, "secret.empty") {
|
||||
t.Fatalf("stderr lacks typed load failure: %q", got)
|
||||
}
|
||||
if strings.Contains(got, canary) || strings.Contains(stdout.String(), canary) {
|
||||
t.Fatal("command output exposed skipped secret value")
|
||||
}
|
||||
}
|
||||
|
||||
func stringPointer(value string) *string { return &value }
|
||||
@@ -1,147 +0,0 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||
)
|
||||
|
||||
var envVarNamePattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||
|
||||
// APIKeyFileByteLimit bounds one credential value read from the configured
|
||||
// secrets directory. API keys are expected to be short single-line values.
|
||||
const APIKeyFileByteLimit int64 = 8 * 1024
|
||||
|
||||
const secretDirectoryPrivateMode os.FileMode = 0o700
|
||||
const secretFilePrivateMode os.FileMode = 0o600
|
||||
|
||||
type secretsLoadStats struct {
|
||||
Dir string
|
||||
Loaded int
|
||||
PreservedExisting int
|
||||
Skipped int
|
||||
}
|
||||
|
||||
func loadSecretsFromConfig(cfg *config.Config, logger *slog.Logger) (*secretsLoadStats, error) {
|
||||
return loadSecretsFromConfigWithDependencies(cfg, logger, commandDependencies{})
|
||||
}
|
||||
|
||||
func loadSecretsFromConfigWithDependencies(cfg *config.Config, logger *slog.Logger, deps commandDependencies) (*secretsLoadStats, error) {
|
||||
deps = normalizeCommandDependencies(deps)
|
||||
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Secrets == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
rawDir := strings.TrimSpace(cfg.Pipeline.Secrets.EnvDir)
|
||||
if rawDir == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
resolvedDir := rawDir
|
||||
if !filepath.IsAbs(resolvedDir) {
|
||||
cwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve secrets env_dir %q from current working directory: %w", rawDir, err)
|
||||
}
|
||||
resolvedDir = filepath.Join(cwd, resolvedDir)
|
||||
}
|
||||
resolvedDir = filepath.Clean(resolvedDir)
|
||||
|
||||
directoryInfo, err := os.Lstat(resolvedDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("API-key loader: validate secrets env_dir: %w", err)
|
||||
}
|
||||
if directoryInfo.Mode()&os.ModeSymlink != 0 {
|
||||
return nil, fmt.Errorf("API-key loader: secrets env_dir must not be a symbolic link")
|
||||
}
|
||||
if err := validateSecretDirectory(directoryInfo); err != nil {
|
||||
return nil, fmt.Errorf("API-key loader: validate secrets env_dir: %w", err)
|
||||
}
|
||||
|
||||
entries, err := deps.readSecretsDirectory(resolvedDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("API-key loader: read secrets env_dir %q: %w", resolvedDir, err)
|
||||
}
|
||||
directoryInfo, err = os.Lstat(resolvedDir)
|
||||
if err != nil || directoryInfo.Mode()&os.ModeSymlink != 0 {
|
||||
if err == nil {
|
||||
err = fmt.Errorf("symbolic links are not allowed")
|
||||
}
|
||||
return nil, fmt.Errorf("API-key loader: validate secrets env_dir: %w", err)
|
||||
}
|
||||
|
||||
stats := &secretsLoadStats{Dir: resolvedDir}
|
||||
for _, entry := range entries {
|
||||
name := entry.Name()
|
||||
if !envVarNamePattern.MatchString(name) {
|
||||
stats.Skipped++
|
||||
continue
|
||||
}
|
||||
if _, exists := os.LookupEnv(name); exists {
|
||||
stats.PreservedExisting++
|
||||
continue
|
||||
}
|
||||
value, err := readAPIKeyFile(resolvedDir, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.Setenv(name, value); err != nil {
|
||||
return nil, fmt.Errorf("API-key loader: set environment variable %q: %w", name, err)
|
||||
}
|
||||
stats.Loaded++
|
||||
}
|
||||
|
||||
if logger != nil {
|
||||
logger.Info(
|
||||
"loaded secret environment variables from filesystem",
|
||||
"secrets_env_dir", stats.Dir,
|
||||
"loaded", stats.Loaded,
|
||||
"preserved_existing", stats.PreservedExisting,
|
||||
"skipped", stats.Skipped,
|
||||
)
|
||||
}
|
||||
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func validateSecretDirectory(info os.FileInfo) error {
|
||||
if !info.IsDir() {
|
||||
return fmt.Errorf("secrets env_dir is not a directory")
|
||||
}
|
||||
return validateSecretDirectoryPrivacy(info)
|
||||
}
|
||||
|
||||
func validateSecretFile(info os.FileInfo) error {
|
||||
if !info.Mode().IsRegular() {
|
||||
return fmt.Errorf("secret entry is not a regular file")
|
||||
}
|
||||
return validateSecretFilePrivacy(info)
|
||||
}
|
||||
|
||||
func readAPIKeyFile(directory, name string) (string, error) {
|
||||
path := filepath.Join(directory, name)
|
||||
info, err := os.Lstat(path)
|
||||
if err == nil && info.Mode()&os.ModeSymlink != 0 {
|
||||
return "", fmt.Errorf("API-key loader: read secret file %q with %d-byte limit: symbolic links are not allowed", name, APIKeyFileByteLimit)
|
||||
}
|
||||
if err == nil {
|
||||
err = validateSecretFile(info)
|
||||
}
|
||||
if err == nil {
|
||||
content, readErr := fileops.ReadFileBounded(path, APIKeyFileByteLimit)
|
||||
if readErr == nil {
|
||||
return strings.TrimRight(string(content), "\r\n"), nil
|
||||
}
|
||||
err = readErr
|
||||
}
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("API-key loader: read secret file %q with %d-byte limit: %w", name, APIKeyFileByteLimit, err)
|
||||
}
|
||||
return "", fmt.Errorf("API-key loader: read secret file %q with %d-byte limit", name, APIKeyFileByteLimit)
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
//go:build !windows
|
||||
|
||||
package app
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
func validateSecretDirectoryPrivacy(info os.FileInfo) error {
|
||||
if info.Mode().Perm()&^secretDirectoryPrivateMode != 0 {
|
||||
return fmt.Errorf("secret directory mode %04o grants group or other access; require %04o", info.Mode().Perm(), secretDirectoryPrivateMode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateSecretFilePrivacy(info os.FileInfo) error {
|
||||
if info.Mode().Perm()&^secretFilePrivateMode != 0 {
|
||||
return fmt.Errorf("secret file mode %04o grants group or other access; require %04o", info.Mode().Perm(), secretFilePrivateMode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,289 +0,0 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
func TestLoadSecretsFromConfigLoadsValidFiles(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
restoreEnvAfterTest(t, "NARRATIO_TEST_SECRET_A", "NARRATIO_TEST_SECRET_B")
|
||||
mustWriteSecretFile(t, filepath.Join(dir, "NARRATIO_TEST_SECRET_A"), "value-1\n")
|
||||
mustWriteSecretFile(t, filepath.Join(dir, "NARRATIO_TEST_SECRET_B"), "value-2\r\n")
|
||||
mustWriteSecretFile(t, filepath.Join(dir, "not-valid-name.txt"), "ignored")
|
||||
|
||||
cfg := &config.Config{
|
||||
Pipeline: &config.PipelineConfig{
|
||||
Secrets: &config.SecretsConfig{EnvDir: dir},
|
||||
},
|
||||
}
|
||||
|
||||
stats, err := loadSecretsFromConfig(cfg, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("loadSecretsFromConfig() error = %v", err)
|
||||
}
|
||||
if stats == nil {
|
||||
t.Fatal("stats = nil, want non-nil")
|
||||
}
|
||||
if stats.Loaded != 2 {
|
||||
t.Fatalf("Loaded = %d, want 2", stats.Loaded)
|
||||
}
|
||||
if stats.PreservedExisting != 0 {
|
||||
t.Fatalf("PreservedExisting = %d, want 0", stats.PreservedExisting)
|
||||
}
|
||||
if stats.Skipped == 0 {
|
||||
t.Fatalf("Skipped = %d, want > 0 for invalid filename", stats.Skipped)
|
||||
}
|
||||
if got := os.Getenv("NARRATIO_TEST_SECRET_A"); got != "value-1" {
|
||||
t.Fatalf("NARRATIO_TEST_SECRET_A = %q, want value-1", got)
|
||||
}
|
||||
if got := os.Getenv("NARRATIO_TEST_SECRET_B"); got != "value-2" {
|
||||
t.Fatalf("NARRATIO_TEST_SECRET_B = %q, want value-2", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadSecretsFromConfigPreservesExistingEnv(t *testing.T) {
|
||||
t.Setenv("OBJECT_STORAGE_KEY", "existing")
|
||||
|
||||
dir := t.TempDir()
|
||||
mustWriteSecretFile(t, filepath.Join(dir, "OBJECT_STORAGE_KEY"), "from-file\n")
|
||||
|
||||
cfg := &config.Config{
|
||||
Pipeline: &config.PipelineConfig{
|
||||
Secrets: &config.SecretsConfig{EnvDir: dir},
|
||||
},
|
||||
}
|
||||
|
||||
stats, err := loadSecretsFromConfig(cfg, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("loadSecretsFromConfig() error = %v", err)
|
||||
}
|
||||
if stats.PreservedExisting != 1 {
|
||||
t.Fatalf("PreservedExisting = %d, want 1", stats.PreservedExisting)
|
||||
}
|
||||
if got := os.Getenv("OBJECT_STORAGE_KEY"); got != "existing" {
|
||||
t.Fatalf("OBJECT_STORAGE_KEY = %q, want existing", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadSecretsFromConfigRelativeDirUsesCWD(t *testing.T) {
|
||||
cwd := t.TempDir()
|
||||
secretsDir := filepath.Join(cwd, "secrets")
|
||||
if err := os.MkdirAll(secretsDir, secretDirectoryPrivateMode); err != nil {
|
||||
t.Fatalf("MkdirAll(%q): %v", secretsDir, err)
|
||||
}
|
||||
mustWriteSecretFile(t, filepath.Join(secretsDir, "OBJECT_STORAGE_KEY_ID"), "id-123\n")
|
||||
|
||||
originalWD, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatalf("Getwd(): %v", err)
|
||||
}
|
||||
if err := os.Chdir(cwd); err != nil {
|
||||
t.Fatalf("Chdir(%q): %v", cwd, err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = os.Chdir(originalWD)
|
||||
})
|
||||
|
||||
cfg := &config.Config{
|
||||
Pipeline: &config.PipelineConfig{
|
||||
Secrets: &config.SecretsConfig{EnvDir: "./secrets"},
|
||||
},
|
||||
}
|
||||
if _, err := loadSecretsFromConfig(cfg, nil); err != nil {
|
||||
t.Fatalf("loadSecretsFromConfig() error = %v", err)
|
||||
}
|
||||
if got := os.Getenv("OBJECT_STORAGE_KEY_ID"); got != "id-123" {
|
||||
t.Fatalf("OBJECT_STORAGE_KEY_ID = %q, want id-123", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadSecretsFromConfigMissingDirFails(t *testing.T) {
|
||||
cfg := &config.Config{
|
||||
Pipeline: &config.PipelineConfig{
|
||||
Secrets: &config.SecretsConfig{EnvDir: filepath.Join(t.TempDir(), "missing")},
|
||||
},
|
||||
}
|
||||
|
||||
_, err := loadSecretsFromConfig(cfg, nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "API-key loader") {
|
||||
t.Fatalf("error = %q, want API-key loader context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadSecretsFromConfigUnreadableValidEntryFails(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("symlink behavior differs on windows")
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
ensureSecretDirectory(t, dir)
|
||||
const secretName = "NARRATIO_TEST_BROKEN_SECRET"
|
||||
restoreEnvAfterTest(t, secretName)
|
||||
if err := os.Unsetenv(secretName); err != nil {
|
||||
t.Fatalf("Unsetenv(%q): %v", secretName, err)
|
||||
}
|
||||
broken := filepath.Join(dir, secretName)
|
||||
if err := os.Symlink(filepath.Join(dir, "does-not-exist"), broken); err != nil {
|
||||
t.Fatalf("Symlink(%q): %v", broken, err)
|
||||
}
|
||||
|
||||
cfg := &config.Config{
|
||||
Pipeline: &config.PipelineConfig{
|
||||
Secrets: &config.SecretsConfig{EnvDir: dir},
|
||||
},
|
||||
}
|
||||
|
||||
_, err := loadSecretsFromConfig(cfg, nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "API-key loader") {
|
||||
t.Fatalf("error = %q, want API-key loader context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadSecretsFromConfigRejectsUnsafeModes(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("Windows access control is verified with ACLs")
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
directory os.FileMode
|
||||
file os.FileMode
|
||||
wantSecret string
|
||||
}{
|
||||
{name: "directory", directory: 0o750, file: secretFilePrivateMode, wantSecret: "directory-secret"},
|
||||
{name: "file", directory: secretDirectoryPrivateMode, file: 0o640, wantSecret: "file-secret"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
restoreEnvAfterTest(t, "OPENROUTER_API_KEY")
|
||||
path := filepath.Join(dir, "OPENROUTER_API_KEY")
|
||||
mustWriteSecretFile(t, path, tc.wantSecret)
|
||||
if err := os.Chmod(dir, tc.directory); err != nil {
|
||||
t.Fatalf("Chmod(directory) error = %v", err)
|
||||
}
|
||||
if err := os.Chmod(path, tc.file); err != nil {
|
||||
t.Fatalf("Chmod(file) error = %v", err)
|
||||
}
|
||||
|
||||
_, err := loadSecretsFromConfig(secretConfig(dir), nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "API-key loader") {
|
||||
t.Fatalf("loadSecretsFromConfig() error = %v, want API-key loader failure", err)
|
||||
}
|
||||
if strings.Contains(err.Error(), tc.wantSecret) {
|
||||
t.Fatalf("error leaked secret content: %q", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadSecretsFromConfigRejectsNonRegularAndOversizedEntries(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
ensureSecretDirectory(t, dir)
|
||||
restoreEnvAfterTest(t, "OPENROUTER_API_KEY")
|
||||
if err := os.Mkdir(filepath.Join(dir, "OPENROUTER_API_KEY"), secretDirectoryPrivateMode); err != nil {
|
||||
t.Fatalf("Mkdir(non-regular entry) error = %v", err)
|
||||
}
|
||||
if _, err := loadSecretsFromConfig(secretConfig(dir), nil); err == nil || !strings.Contains(err.Error(), "not a regular file") {
|
||||
t.Fatalf("loadSecretsFromConfig(non-regular) error = %v, want regular-file failure", err)
|
||||
}
|
||||
|
||||
if err := os.Remove(filepath.Join(dir, "OPENROUTER_API_KEY")); err != nil {
|
||||
t.Fatalf("Remove(non-regular entry) error = %v", err)
|
||||
}
|
||||
overLimit := strings.Repeat("credential-value-", int(APIKeyFileByteLimit/17)+1)
|
||||
mustWriteSecretFile(t, filepath.Join(dir, "OPENROUTER_API_KEY"), overLimit)
|
||||
_, err := loadSecretsFromConfig(secretConfig(dir), nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "API-key loader") || !strings.Contains(err.Error(), "8192-byte limit") {
|
||||
t.Fatalf("loadSecretsFromConfig(over-limit) error = %v, want bounded API-key loader failure", err)
|
||||
}
|
||||
if strings.Contains(err.Error(), overLimit[:32]) {
|
||||
t.Fatalf("error leaked secret content: %q", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadSecretsFromConfigRejectsSymlinkAndAncestorReplacement(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("symlink behavior differs on windows")
|
||||
}
|
||||
dir := t.TempDir()
|
||||
outside := t.TempDir()
|
||||
ensureSecretDirectory(t, dir)
|
||||
restoreEnvAfterTest(t, "OPENROUTER_API_KEY")
|
||||
outsideValue := "outside-secret-value"
|
||||
mustWriteSecretFile(t, filepath.Join(outside, "OPENROUTER_API_KEY"), outsideValue)
|
||||
if err := os.Symlink(filepath.Join(outside, "OPENROUTER_API_KEY"), filepath.Join(dir, "OPENROUTER_API_KEY")); err != nil {
|
||||
t.Fatalf("Symlink(leaf) error = %v", err)
|
||||
}
|
||||
_, err := loadSecretsFromConfig(secretConfig(dir), nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "API-key loader") {
|
||||
t.Fatalf("loadSecretsFromConfig(symlink) error = %v, want failure", err)
|
||||
}
|
||||
if strings.Contains(err.Error(), outsideValue) {
|
||||
t.Fatalf("error leaked secret content: %q", err)
|
||||
}
|
||||
|
||||
if err := os.Remove(filepath.Join(dir, "OPENROUTER_API_KEY")); err != nil {
|
||||
t.Fatalf("Remove(leaf symlink) error = %v", err)
|
||||
}
|
||||
mustWriteSecretFile(t, filepath.Join(dir, "OPENROUTER_API_KEY"), "inside-secret")
|
||||
deps := commandDependencies{
|
||||
readSecretsDirectory: func(path string) ([]os.DirEntry, error) {
|
||||
entries, err := os.ReadDir(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.Remove(filepath.Join(dir, "OPENROUTER_API_KEY")); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.Remove(dir); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.Symlink(outside, dir); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return entries, nil
|
||||
},
|
||||
}
|
||||
_, err = loadSecretsFromConfigWithDependencies(secretConfig(dir), nil, deps)
|
||||
if err == nil || !strings.Contains(err.Error(), "API-key loader") {
|
||||
t.Fatalf("loadSecretsFromConfig(ancestor replacement) error = %v, want failure", err)
|
||||
}
|
||||
if _, exists := os.LookupEnv("OPENROUTER_API_KEY"); exists {
|
||||
t.Fatal("ancestor replacement loaded a secret value")
|
||||
}
|
||||
}
|
||||
|
||||
func mustWriteSecretFile(t *testing.T, path, contents string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), secretDirectoryPrivateMode); err != nil {
|
||||
t.Fatalf("MkdirAll(%q): %v", path, err)
|
||||
}
|
||||
if err := os.Chmod(filepath.Dir(path), secretDirectoryPrivateMode); err != nil {
|
||||
t.Fatalf("Chmod(%q): %v", filepath.Dir(path), err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(contents), 0o600); err != nil {
|
||||
t.Fatalf("WriteFile(%q): %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func secretConfig(dir string) *config.Config {
|
||||
return &config.Config{Pipeline: &config.PipelineConfig{Secrets: &config.SecretsConfig{EnvDir: dir}}}
|
||||
}
|
||||
|
||||
func ensureSecretDirectory(t *testing.T, directory string) {
|
||||
t.Helper()
|
||||
if err := os.Chmod(directory, secretDirectoryPrivateMode); err != nil {
|
||||
t.Fatalf("Chmod(%q): %v", directory, err)
|
||||
}
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
//go:build windows
|
||||
|
||||
package app
|
||||
|
||||
import "os"
|
||||
|
||||
// Windows access control is determined by ACLs rather than POSIX mode bits.
|
||||
func validateSecretDirectoryPrivacy(_ os.FileInfo) error { return nil }
|
||||
|
||||
// Windows access control is determined by ACLs rather than POSIX mode bits.
|
||||
func validateSecretFilePrivacy(_ os.FileInfo) error { return nil }
|
||||
@@ -471,8 +471,8 @@ func TestDeliverySemanticChangesInvalidateOnlyTheirFixedDependents(t *testing.T)
|
||||
cfg.Pipeline.Publish.Outputs[0].Dest = "published/alternate.json"
|
||||
}, wantRuns: [2]int{1, 1}},
|
||||
{name: "publish credentials", mutate: func(cfg *config.Config) {
|
||||
cfg.Pipeline.Storage.S3.AccessKeyIDEnv = "OTHER_ACCESS_KEY"
|
||||
cfg.Pipeline.Storage.S3.SecretKeyEnv = "OTHER_SECRET_KEY"
|
||||
cfg.Pipeline.Storage.S3.AccessKeyID = testSecretReference("OTHER_ACCESS_KEY")
|
||||
cfg.Pipeline.Storage.S3.SecretAccessKey = testSecretReference("OTHER_SECRET_KEY")
|
||||
}, wantRuns: [2]int{0, 0}},
|
||||
{name: "notify mode", mutate: func(cfg *config.Config) {
|
||||
cfg.Pipeline.Notification.Mode = "webhook"
|
||||
@@ -534,7 +534,7 @@ func configureDeliverySemanticConfig(cfg *config.Config) {
|
||||
Backend: config.StorageBackendLocal,
|
||||
S3: &config.StorageS3Config{
|
||||
Bucket: "campaign", RootPrefix: "narratio", Region: "us-east-1",
|
||||
Endpoint: "https://objects.example", AccessKeyIDEnv: "ACCESS_KEY", SecretKeyEnv: "SECRET_KEY",
|
||||
Endpoint: "https://objects.example", AccessKeyID: testSecretReference("ACCESS_KEY"), SecretAccessKey: testSecretReference("SECRET_KEY"),
|
||||
},
|
||||
}
|
||||
cfg.Pipeline.Notification.Mode = config.DefaultNotificationMode
|
||||
|
||||
@@ -2,7 +2,10 @@ package app
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
)
|
||||
|
||||
// restoreEnvAfterTest clears names for a test and restores both their prior
|
||||
@@ -29,3 +32,18 @@ func restoreEnvAfterTest(t *testing.T, names ...string) {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func testSecretReference(name string) *configkit.Secret {
|
||||
reference := configkit.SecretEnv(name)
|
||||
return &reference
|
||||
}
|
||||
|
||||
func mustWriteSecretFile(t *testing.T, path, value string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(value), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -117,9 +116,6 @@ func TestInspectExtractionEvidenceClassifiesBundleStates(t *testing.T) {
|
||||
paths.Root = t.TempDir()
|
||||
}},
|
||||
{name: "unsafe symlink", state: ExtractionEvidenceUnsafe, mutate: func(t *testing.T, _ *SessionPaths, m *manifest.Manifest) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("symlink creation requires privileges on Windows")
|
||||
}
|
||||
path := m.Stages["extract"].Outputs[0].LocalPath
|
||||
outside := filepath.Join(t.TempDir(), "outside.json")
|
||||
writeExtractionFixtureFile(t, outside, `{"outside":true}`)
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
//go:build !linux && !darwin && !windows
|
||||
//go:build !linux && !darwin
|
||||
|
||||
package artifacts
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"runtime"
|
||||
)
|
||||
|
||||
func acquireHeldFileLock(_ *os.File) error {
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
//go:build windows
|
||||
|
||||
package artifacts
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
|
||||
"golang.org/x/sys/windows"
|
||||
)
|
||||
|
||||
func acquireHeldFileLock(file *os.File) error {
|
||||
err := windows.LockFileEx(
|
||||
windows.Handle(file.Fd()),
|
||||
windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY,
|
||||
0,
|
||||
1,
|
||||
0,
|
||||
&windows.Overlapped{},
|
||||
)
|
||||
if errors.Is(err, windows.ERROR_LOCK_VIOLATION) {
|
||||
return errHeldLockConflict
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func releaseHeldFileLock(file *os.File) error {
|
||||
return windows.UnlockFileEx(windows.Handle(file.Fd()), 0, 1, 0, &windows.Overlapped{})
|
||||
}
|
||||
144
internal/config/artifact_catalog.go
Normal file
144
internal/config/artifact_catalog.go
Normal file
@@ -0,0 +1,144 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
|
||||
)
|
||||
|
||||
// EffectiveArtifactRecord describes one concrete artifact after configuration
|
||||
// resolution and family expansion.
|
||||
type EffectiveArtifactRecord struct {
|
||||
Key string
|
||||
SourceID string
|
||||
DefinitionSources []string
|
||||
Enabled bool
|
||||
OutputPath string
|
||||
Dependencies []string
|
||||
FamilyKey string
|
||||
CharacterID string
|
||||
FamilySource string
|
||||
PartySource string
|
||||
}
|
||||
|
||||
// EffectiveArtifactCatalog returns the deterministic concrete artifact catalog
|
||||
// consumed by runtime planning.
|
||||
func EffectiveArtifactCatalog(cfg *PipelineConfig, party ResolvedParty) ([]EffectiveArtifactRecord, error) {
|
||||
if cfg == nil {
|
||||
return nil, fmt.Errorf("pipeline config is required")
|
||||
}
|
||||
if cfg.Scriptorium == nil || len(cfg.Scriptorium.Artifacts) == 0 {
|
||||
return []EffectiveArtifactRecord{}, nil
|
||||
}
|
||||
|
||||
keys := make([]string, 0, len(cfg.Scriptorium.Artifacts))
|
||||
for key := range cfg.Scriptorium.Artifacts {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
records := make([]EffectiveArtifactRecord, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
artifact := cfg.Scriptorium.Artifacts[key]
|
||||
record := EffectiveArtifactRecord{
|
||||
Key: key,
|
||||
SourceID: artifactpolicy.ConfiguredSourceID(key),
|
||||
Enabled: artifact.Enabled,
|
||||
OutputPath: artifact.OutputPath,
|
||||
Dependencies: append([]string(nil), artifact.DependsOn...),
|
||||
}
|
||||
if member, ok := cfg.familyCatalog.Members[key]; ok {
|
||||
record.DefinitionSources = artifactFamilyDefinitionSources(cfg, member.Family)
|
||||
record.FamilyKey = member.Family
|
||||
record.CharacterID = member.CharacterID
|
||||
record.FamilySource = member.Source
|
||||
record.PartySource = party.Source.Path
|
||||
} else {
|
||||
record.DefinitionSources = artifactDefinitionSources(cfg, key)
|
||||
}
|
||||
record.Dependencies = nonEmptySortedStrings(record.Dependencies)
|
||||
records = append(records, record)
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func artifactDefinitionSources(cfg *PipelineConfig, key string) []string {
|
||||
return declarationDefinitionSources(cfg,
|
||||
configkit.Keys("scriptorium", "artifacts", key),
|
||||
"scriptorium.artifacts."+key,
|
||||
)
|
||||
}
|
||||
|
||||
func artifactFamilyDefinitionSources(cfg *PipelineConfig, key string) []string {
|
||||
return declarationDefinitionSources(cfg,
|
||||
configkit.Keys("scriptorium", "artifact_families", key),
|
||||
"scriptorium.artifact_families."+key,
|
||||
)
|
||||
}
|
||||
|
||||
func declarationDefinitionSources(cfg *PipelineConfig, declaration configkit.Path, prefix string) []string {
|
||||
if cfg == nil || cfg.resolution == nil {
|
||||
return nil
|
||||
}
|
||||
pathsByID := make(map[string]string, len(cfg.resolution.configkitSources))
|
||||
for _, source := range cfg.resolution.configkitSources {
|
||||
path := source.Path
|
||||
if path == "" {
|
||||
path = source.Label
|
||||
}
|
||||
pathsByID[source.ID] = path
|
||||
}
|
||||
|
||||
seen := map[string]struct{}{}
|
||||
for _, change := range cfg.resolution.configkitMetadata.History(declaration) {
|
||||
if change.Origin.Layer != configkit.LayerBase && change.Origin.Layer != configkit.LayerProfile {
|
||||
continue
|
||||
}
|
||||
for _, location := range change.Origin.Locations {
|
||||
source := pathsByID[location.SourceID]
|
||||
if source == "" {
|
||||
source = location.SourceID
|
||||
}
|
||||
seen[source] = struct{}{}
|
||||
}
|
||||
}
|
||||
if len(seen) == 0 {
|
||||
for _, ownership := range cfg.resolution.ownership {
|
||||
if ownership.path != prefix && !strings.HasPrefix(ownership.path, prefix+".") {
|
||||
continue
|
||||
}
|
||||
for _, source := range ownership.sources {
|
||||
if source != pipelineDefaultOwnershipSource && source != "resolution" {
|
||||
seen[source] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(seen) == 0 && cfg.resolution.rootPath != "" {
|
||||
seen[cfg.resolution.rootPath] = struct{}{}
|
||||
}
|
||||
result := make([]string, 0, len(seen))
|
||||
for source := range seen {
|
||||
result = append(result, source)
|
||||
}
|
||||
sort.Strings(result)
|
||||
return result
|
||||
}
|
||||
|
||||
func nonEmptySortedStrings(values []string) []string {
|
||||
seen := map[string]struct{}{}
|
||||
for _, value := range values {
|
||||
value = strings.TrimSpace(value)
|
||||
if value != "" {
|
||||
seen[value] = struct{}{}
|
||||
}
|
||||
}
|
||||
result := make([]string, 0, len(seen))
|
||||
for value := range seen {
|
||||
result = append(result, value)
|
||||
}
|
||||
sort.Strings(result)
|
||||
return result
|
||||
}
|
||||
123
internal/config/artifact_dependencies.go
Normal file
123
internal/config/artifact_dependencies.go
Normal file
@@ -0,0 +1,123 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
|
||||
)
|
||||
|
||||
type artifactDependencyOrigin struct {
|
||||
explicitIndex int
|
||||
inputName string
|
||||
family string
|
||||
}
|
||||
|
||||
func finalizeArtifactDependencies(cfg *PipelineConfig) {
|
||||
if cfg == nil || cfg.Scriptorium == nil {
|
||||
return
|
||||
}
|
||||
for artifactName, artifact := range cfg.Scriptorium.Artifacts {
|
||||
priorOrigins := artifact.dependencyOrigins
|
||||
origins := make(map[string][]artifactDependencyOrigin)
|
||||
dependencies := make(map[string]struct{})
|
||||
family := ""
|
||||
if member, ok := cfg.familyCatalog.Members[artifactName]; ok {
|
||||
family = member.Family
|
||||
}
|
||||
|
||||
for index, dependency := range artifact.DependsOn {
|
||||
dependency = strings.TrimSpace(dependency)
|
||||
if prior, finalized := priorOrigins[dependency]; finalized {
|
||||
explicit := explicitOrigins(prior)
|
||||
if len(explicit) > 0 {
|
||||
dependencies[dependency] = struct{}{}
|
||||
origins[dependency] = append(origins[dependency], explicit...)
|
||||
}
|
||||
continue
|
||||
}
|
||||
dependencies[dependency] = struct{}{}
|
||||
origins[dependency] = append(origins[dependency], artifactDependencyOrigin{
|
||||
explicitIndex: index,
|
||||
family: family,
|
||||
})
|
||||
}
|
||||
|
||||
inputNames := make([]string, 0, len(artifact.Inputs))
|
||||
for inputName := range artifact.Inputs {
|
||||
inputNames = append(inputNames, inputName)
|
||||
}
|
||||
sort.Strings(inputNames)
|
||||
for _, inputName := range inputNames {
|
||||
descriptor, err := artifactpolicy.DescribeScriptoriumInputSource(artifact.Inputs[inputName].Source)
|
||||
if err != nil || descriptor.Source.Kind != artifactpolicy.SourceKindConfiguredArtifact {
|
||||
continue
|
||||
}
|
||||
dependency := descriptor.Source.ConfiguredKey
|
||||
dependencies[dependency] = struct{}{}
|
||||
origins[dependency] = append(origins[dependency], artifactDependencyOrigin{
|
||||
explicitIndex: -1,
|
||||
inputName: inputName,
|
||||
family: family,
|
||||
})
|
||||
}
|
||||
|
||||
artifact.DependsOn = sortedDependencyKeys(dependencies)
|
||||
artifact.dependencyOrigins = origins
|
||||
cfg.Scriptorium.Artifacts[artifactName] = artifact
|
||||
}
|
||||
}
|
||||
|
||||
func explicitOrigins(origins []artifactDependencyOrigin) []artifactDependencyOrigin {
|
||||
result := make([]artifactDependencyOrigin, 0, len(origins))
|
||||
for _, origin := range origins {
|
||||
if origin.explicitIndex >= 0 {
|
||||
result = append(result, origin)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func sortedDependencyKeys(dependencies map[string]struct{}) []string {
|
||||
result := make([]string, 0, len(dependencies))
|
||||
for dependency := range dependencies {
|
||||
result = append(result, dependency)
|
||||
}
|
||||
sort.Strings(result)
|
||||
return result
|
||||
}
|
||||
|
||||
func dependencyOriginField(artifactName, dependency string, artifact ScriptoriumArtifactConfig) string {
|
||||
origins := artifact.dependencyOrigins[dependency]
|
||||
for _, origin := range origins {
|
||||
if origin.explicitIndex < 0 {
|
||||
continue
|
||||
}
|
||||
if origin.family != "" {
|
||||
return fmt.Sprintf("pipeline.scriptorium.artifact_families.%s.member_dependencies[%d]", origin.family, origin.explicitIndex)
|
||||
}
|
||||
return fmt.Sprintf("pipeline.scriptorium.artifacts.%s.depends_on[%d]", artifactName, origin.explicitIndex)
|
||||
}
|
||||
for _, origin := range origins {
|
||||
if origin.inputName == "" {
|
||||
continue
|
||||
}
|
||||
if origin.family != "" {
|
||||
return fmt.Sprintf("pipeline.scriptorium.artifact_families.%s.inputs.%s.source", origin.family, origin.inputName)
|
||||
}
|
||||
return fmt.Sprintf("pipeline.scriptorium.artifacts.%s.inputs.%s.source", artifactName, origin.inputName)
|
||||
}
|
||||
return fmt.Sprintf("pipeline.scriptorium.artifacts.%s.depends_on", artifactName)
|
||||
}
|
||||
|
||||
func cloneDependencyOrigins(in map[string][]artifactDependencyOrigin) map[string][]artifactDependencyOrigin {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string][]artifactDependencyOrigin, len(in))
|
||||
for dependency, origins := range in {
|
||||
out[dependency] = append([]artifactDependencyOrigin(nil), origins...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
112
internal/config/artifact_dependencies_test.go
Normal file
112
internal/config/artifact_dependencies_test.go
Normal file
@@ -0,0 +1,112 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFinalizeArtifactDependenciesUnionsNormalizesAndIsIdempotent(t *testing.T) {
|
||||
cfg := dependencyTestPipeline([]string{" beta ", "alpha", "beta"})
|
||||
artifact := cfg.Scriptorium.Artifacts["result"]
|
||||
artifact.Inputs["gamma"] = ScriptoriumInputConfig{Source: "narratio.artifact.gamma"}
|
||||
artifact.Inputs["alpha"] = ScriptoriumInputConfig{Source: "narratio.artifact.alpha"}
|
||||
cfg.Scriptorium.Artifacts["result"] = artifact
|
||||
cfg.Scriptorium.Artifacts["gamma"] = ScriptoriumArtifactConfig{}
|
||||
|
||||
finalizeArtifactDependencies(cfg)
|
||||
want := []string{"alpha", "beta", "gamma"}
|
||||
got := cfg.Scriptorium.Artifacts["result"]
|
||||
if !reflect.DeepEqual(got.DependsOn, want) {
|
||||
t.Fatalf("dependencies = %#v, want %#v", got.DependsOn, want)
|
||||
}
|
||||
if len(got.dependencyOrigins["alpha"]) != 2 {
|
||||
t.Fatalf("alpha origins = %#v, want explicit and input origins", got.dependencyOrigins["alpha"])
|
||||
}
|
||||
|
||||
firstOrigins := cloneDependencyOrigins(got.dependencyOrigins)
|
||||
finalizeArtifactDependencies(cfg)
|
||||
got = cfg.Scriptorium.Artifacts["result"]
|
||||
if !reflect.DeepEqual(got.DependsOn, want) {
|
||||
t.Fatalf("dependencies after second finalization = %#v, want %#v", got.DependsOn, want)
|
||||
}
|
||||
if !reflect.DeepEqual(got.dependencyOrigins, firstOrigins) {
|
||||
t.Fatalf("origins changed after second finalization: %#v != %#v", got.dependencyOrigins, firstOrigins)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInferredArtifactDependencyDiagnosticsPointToInputs(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
artifacts map[string]ScriptoriumArtifactConfig
|
||||
want []string
|
||||
}{
|
||||
{
|
||||
name: "missing",
|
||||
artifacts: map[string]ScriptoriumArtifactConfig{
|
||||
"result": {Inputs: map[string]ScriptoriumInputConfig{"missing": {Source: "narratio.artifact.unknown"}}},
|
||||
},
|
||||
want: []string{"pipeline.scriptorium.artifacts.result.inputs.missing.source", "not configured"},
|
||||
},
|
||||
{
|
||||
name: "self",
|
||||
artifacts: map[string]ScriptoriumArtifactConfig{
|
||||
"result": {Inputs: map[string]ScriptoriumInputConfig{"self": {Source: "narratio.artifact.result"}}},
|
||||
},
|
||||
want: []string{"pipeline.scriptorium.artifacts.result.inputs.self.source", "itself"},
|
||||
},
|
||||
{
|
||||
name: "cycle",
|
||||
artifacts: map[string]ScriptoriumArtifactConfig{
|
||||
"alpha": {Inputs: map[string]ScriptoriumInputConfig{"beta": {Source: "narratio.artifact.beta"}}},
|
||||
"beta": {Inputs: map[string]ScriptoriumInputConfig{"alpha": {Source: "narratio.artifact.alpha"}}},
|
||||
},
|
||||
want: []string{".inputs.", "dependencies must not contain cycles"},
|
||||
},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
cfg := &PipelineConfig{Scriptorium: &ScriptoriumConfig{Artifacts: test.artifacts}}
|
||||
finalizeArtifactDependencies(cfg)
|
||||
err := ValidateScriptoriumArtifactDependencies(cfg.Scriptorium.Artifacts)
|
||||
if err == nil {
|
||||
t.Fatal("expected dependency validation error")
|
||||
}
|
||||
for _, fragment := range test.want {
|
||||
if !strings.Contains(err.Error(), fragment) {
|
||||
t.Fatalf("error = %q, want fragment %q", err, fragment)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExplicitAndInferredDependencyGraphsHaveEqualDigests(t *testing.T) {
|
||||
explicit := dependencyTestPipeline([]string{"alpha"})
|
||||
inferred := dependencyTestPipeline(nil)
|
||||
for _, cfg := range []*PipelineConfig{explicit, inferred} {
|
||||
finalizeArtifactDependencies(cfg)
|
||||
cfg.resolution = &pipelineResolutionMetadata{}
|
||||
if err := recomputePipelineEffectiveDigest(cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if got, want := EffectivePipelineDigest(explicit), EffectivePipelineDigest(inferred); got != want {
|
||||
t.Fatalf("equivalent dependency graph digests differ: %s != %s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func dependencyTestPipeline(dependsOn []string) *PipelineConfig {
|
||||
return &PipelineConfig{Scriptorium: &ScriptoriumConfig{
|
||||
Binary: "scriptorium",
|
||||
Artifacts: map[string]ScriptoriumArtifactConfig{
|
||||
"alpha": {},
|
||||
"beta": {},
|
||||
"result": {
|
||||
DependsOn: dependsOn,
|
||||
Inputs: map[string]ScriptoriumInputConfig{
|
||||
"alpha": {Source: "narratio.artifact.alpha"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}}
|
||||
}
|
||||
@@ -124,9 +124,15 @@ func expandPipelineArtifactFamilies(cfg *PipelineConfig, party ResolvedParty) er
|
||||
if err := expandFamilyPublishRules(cfg, catalog); err != nil {
|
||||
return err
|
||||
}
|
||||
applyMissingPublishDestinations(cfg.Publish, cfg.Scriptorium, cfg.Notarius)
|
||||
cfg.Scriptorium.ArtifactFamilies = nil
|
||||
cfg.familyCatalog = catalog
|
||||
cfg.resolution.artifactFamiliesExpanded = true
|
||||
finalizeArtifactDependencies(cfg)
|
||||
for key, member := range cfg.familyCatalog.Members {
|
||||
member.Dependencies = append([]string(nil), cfg.Scriptorium.Artifacts[key].DependsOn...)
|
||||
cfg.familyCatalog.Members[key] = member
|
||||
}
|
||||
if err := rejectConcreteMemberArtifactSources(expanded); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -261,9 +267,6 @@ func expandFamilyInputs(prefix string, family ScriptoriumArtifactFamilyConfig, c
|
||||
if !matched {
|
||||
continue
|
||||
}
|
||||
if !familyDependencyContains(family.MemberDependencies, familyKey) {
|
||||
return nil, fmt.Errorf("%s.inputs.%s.source %q requires member_dependencies entry %q", prefix, name, input.Source, familyKey)
|
||||
}
|
||||
input.Source = artifactpolicy.ConfiguredSourceID(familyKey + "_" + characterID)
|
||||
inputs[name] = input
|
||||
}
|
||||
@@ -299,15 +302,6 @@ func rejectConcreteMemberArtifactSources(artifacts map[string]ScriptoriumArtifac
|
||||
return nil
|
||||
}
|
||||
|
||||
func familyDependencyContains(values []string, want string) bool {
|
||||
for _, value := range values {
|
||||
if strings.TrimSpace(value) == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func inputSourceMap(inputs map[string]ScriptoriumInputConfig) map[string]string {
|
||||
if len(inputs) == 0 {
|
||||
return nil
|
||||
@@ -392,6 +386,7 @@ func cloneArtifactDefinitions(in map[string]ScriptoriumArtifactConfig) map[strin
|
||||
out := make(map[string]ScriptoriumArtifactConfig, len(in))
|
||||
for key, artifact := range in {
|
||||
artifact.DependsOn = append([]string(nil), artifact.DependsOn...)
|
||||
artifact.dependencyOrigins = cloneDependencyOrigins(artifact.dependencyOrigins)
|
||||
artifact.Inputs = cloneArtifactInputs(artifact.Inputs)
|
||||
artifact.Vars = cloneArtifactVars(artifact.Vars)
|
||||
out[key] = artifact
|
||||
|
||||
@@ -8,6 +8,15 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func absolutePath(t *testing.T, path string) string {
|
||||
t.Helper()
|
||||
absolute, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return absolute
|
||||
}
|
||||
|
||||
func TestArtifactFamiliesExpandCanonicalCharactersDeterministically(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writePartyResolutionFile(t, dir+"/party.yml", `schema_version: narratio.party.v1
|
||||
@@ -148,7 +157,7 @@ func TestArtifactFamilyStrictDecodeRejectsUnknownFields(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
pipelinePath, _, _ := writeArtifactFamilyConfig(t, dir, artifactFamilyPipelineYAML(validFamilyYAML+" unsupported: true\n"))
|
||||
_, err := LoadPipeline(pipelinePath)
|
||||
if err == nil || !strings.Contains(err.Error(), "field unsupported not found") {
|
||||
if err == nil || !strings.Contains(err.Error(), "decode.unknown_field") {
|
||||
t.Fatalf("LoadPipeline() error = %v", err)
|
||||
}
|
||||
}
|
||||
@@ -214,7 +223,6 @@ characters:
|
||||
prompt_id: dnd.character_items
|
||||
output_path_pattern: artifacts/characters/{character_id}/items.md
|
||||
depends_on: [session_recap]
|
||||
member_dependencies: [character_meta]
|
||||
inputs:
|
||||
transcript: {source: narratio.transcript.final_trimmed, required: true}
|
||||
prior_meta: {source: narratio.member_artifact.character_meta, required: true}
|
||||
@@ -225,7 +233,7 @@ characters:
|
||||
t.Fatal(err)
|
||||
}
|
||||
items := cfg.Pipeline.Scriptorium.Artifacts["character_items_arannis"]
|
||||
if !reflect.DeepEqual(items.DependsOn, []string{"session_recap", "character_meta_arannis"}) {
|
||||
if !reflect.DeepEqual(items.DependsOn, []string{"character_meta_arannis", "session_recap"}) {
|
||||
t.Fatalf("member dependencies = %#v", items.DependsOn)
|
||||
}
|
||||
if got := items.Inputs["prior_meta"].Source; got != "narratio.artifact.character_meta_arannis" {
|
||||
@@ -253,11 +261,8 @@ func TestArtifactFamiliesRejectInvalidMemberDependenciesAndSources(t *testing.T)
|
||||
for_each: party.characters
|
||||
output_path_pattern: artifacts/characters/{character_id}/other.md
|
||||
`, want: "duplicates family"},
|
||||
{name: "undeclared source dependency", families: strings.Replace(validFamilyYAML, "member_vars:", "inputs:\n prior: {source: narratio.member_artifact.other, required: true}\n member_vars:", 1) + `
|
||||
other:
|
||||
for_each: party.characters
|
||||
output_path_pattern: artifacts/characters/{character_id}/other.md
|
||||
`, want: "requires member_dependencies"},
|
||||
{name: "missing inferred family dependency", families: strings.Replace(validFamilyYAML, "member_vars:", "inputs:\n prior: {source: narratio.member_artifact.missing, required: true}\n member_vars:", 1), want: "pipeline.scriptorium.artifact_families.character_meta.inputs.prior.source"},
|
||||
{name: "self inferred family dependency", families: strings.Replace(validFamilyYAML, "member_vars:", "inputs:\n prior: {source: narratio.member_artifact.character_meta, required: true}\n member_vars:", 1), want: "pipeline.scriptorium.artifact_families.character_meta.inputs.prior.source"},
|
||||
{name: "transitive cycle", families: strings.Replace(validFamilyYAML, "member_vars:", "member_dependencies: [other]\n member_vars:", 1) + `
|
||||
other:
|
||||
for_each: party.characters
|
||||
@@ -329,6 +334,82 @@ func TestFamilyPublishRulesExpandToConcreteArtifacts(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExplicitFamilyMemberPublishDestinationResolution(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writePartyResolutionFile(t, filepath.Join(dir, "party.yml"), `schema_version: narratio.party.v1
|
||||
characters:
|
||||
alice:
|
||||
player: {name: Alice Player}
|
||||
character:
|
||||
name: Alice
|
||||
classes: [{name: wizard}]
|
||||
`)
|
||||
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
source string
|
||||
dest string
|
||||
wantDest string
|
||||
wantErr string
|
||||
}{
|
||||
{
|
||||
name: "derive generated member output",
|
||||
source: "narratio.artifact.meta_alice",
|
||||
wantDest: "artifacts/alice.md",
|
||||
},
|
||||
{
|
||||
name: "preserve explicit override",
|
||||
source: "narratio.artifact.meta_alice",
|
||||
dest: "published/alice-summary.md",
|
||||
wantDest: "published/alice-summary.md",
|
||||
},
|
||||
{
|
||||
name: "reject unknown source",
|
||||
source: "narratio.artifact.missing",
|
||||
wantErr: "is unsupported",
|
||||
},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
publishDestination := ""
|
||||
if test.dest != "" {
|
||||
publishDestination = " dest: " + test.dest + "\n"
|
||||
}
|
||||
pipeline := artifactFamilyPipelineYAML(`
|
||||
meta:
|
||||
enabled: true
|
||||
for_each: party.characters
|
||||
prompt_id: dnd.meta
|
||||
profile_id: production
|
||||
output_path_pattern: artifacts/{character_id}.md
|
||||
`) + `
|
||||
publish:
|
||||
outputs:
|
||||
- source: ` + test.source + "\n" + publishDestination
|
||||
pipelinePath, campaignPath, sessionPath := writeArtifactFamilyConfig(t, dir, pipeline)
|
||||
cfg, err := LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, SessionLoadOptions{})
|
||||
if test.wantErr != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), test.wantErr) {
|
||||
t.Fatalf("LoadWithSessionOptions() error = %v, want %q", err, test.wantErr)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("LoadWithSessionOptions() error = %v", err)
|
||||
}
|
||||
if got := cfg.Pipeline.Publish.Outputs[0].Dest; got != test.wantDest {
|
||||
t.Fatalf("resolved runtime destination = %q, want %q", got, test.wantDest)
|
||||
}
|
||||
effective, err := MarshalEffectivePipeline(cfg.Pipeline)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(effective), "dest: "+test.wantDest+"\n") {
|
||||
t.Fatalf("effective pipeline omitted resolved destination:\n%s", effective)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const validFamilyYAML = `
|
||||
character_meta:
|
||||
enabled: true
|
||||
|
||||
@@ -60,7 +60,7 @@ func TestCampaignStrictDecodeRejectsLegacyCampaignField(t *testing.T) {
|
||||
if err == nil {
|
||||
t.Fatal("expected load error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "campaign file") || !strings.Contains(err.Error(), "strict decode failed") {
|
||||
if !strings.Contains(err.Error(), "load campaign config") || !strings.Contains(err.Error(), "decode.unknown_field") {
|
||||
t.Fatalf("error = %q, want campaign strict decode context", err.Error())
|
||||
}
|
||||
}
|
||||
@@ -75,7 +75,7 @@ func TestCampaignStrictDecodeRejectsUnknownFields(t *testing.T) {
|
||||
if err == nil {
|
||||
t.Fatal("expected load error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "campaign file") || !strings.Contains(err.Error(), "strict decode failed") {
|
||||
if !strings.Contains(err.Error(), "load campaign config") || !strings.Contains(err.Error(), "decode.unknown_field") {
|
||||
t.Fatalf("error = %q, want campaign strict decode context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,783 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// compositionDocument is the presence-aware representation used while
|
||||
// assembling pipeline configuration sources. It deliberately models YAML
|
||||
// mechanics rather than duplicating PipelineConfig's field schema.
|
||||
type compositionDocument struct {
|
||||
root *compositionNode
|
||||
sources []string
|
||||
}
|
||||
|
||||
type compositionNode struct {
|
||||
kind yaml.Kind
|
||||
tag string
|
||||
value string
|
||||
path string
|
||||
sources []string
|
||||
fields []compositionField
|
||||
items []*compositionNode
|
||||
}
|
||||
|
||||
type compositionField struct {
|
||||
key string
|
||||
value *compositionNode
|
||||
}
|
||||
|
||||
// compositionValueRecord is a deterministic semantic leaf projection. Lists
|
||||
// are atomic configuration values, while mappings are traversed recursively.
|
||||
type compositionValueRecord struct {
|
||||
Path string
|
||||
Kind yaml.Kind
|
||||
Value string
|
||||
Sources []string
|
||||
}
|
||||
|
||||
// parseCompositionDocument parses exactly one YAML mapping while retaining
|
||||
// source ownership, explicit zero values, and declaration order.
|
||||
func parseCompositionDocument(source string, reader io.Reader) (*compositionDocument, error) {
|
||||
if strings.TrimSpace(source) == "" {
|
||||
return nil, fmt.Errorf("configuration source identity is required")
|
||||
}
|
||||
if reader == nil {
|
||||
return nil, fmt.Errorf("configuration source %q: reader is nil", source)
|
||||
}
|
||||
|
||||
decoder := yaml.NewDecoder(reader)
|
||||
var document yaml.Node
|
||||
if err := decoder.Decode(&document); err != nil {
|
||||
if err == io.EOF {
|
||||
return nil, fmt.Errorf("configuration source %q: document is empty", source)
|
||||
}
|
||||
return nil, fmt.Errorf("configuration source %q: decode YAML: %w", source, err)
|
||||
}
|
||||
var trailing yaml.Node
|
||||
if err := decoder.Decode(&trailing); err == nil {
|
||||
return nil, fmt.Errorf("configuration source %q: must contain exactly one YAML document", source)
|
||||
} else if err != io.EOF {
|
||||
return nil, fmt.Errorf("configuration source %q: decode trailing YAML: %w", source, err)
|
||||
}
|
||||
if document.Kind != yaml.DocumentNode || len(document.Content) != 1 {
|
||||
return nil, fmt.Errorf("configuration source %q: must contain exactly one YAML document", source)
|
||||
}
|
||||
if document.Content[0].Kind != yaml.MappingNode {
|
||||
return nil, fmt.Errorf(
|
||||
"configuration source %q: top-level document must be a mapping, got %s",
|
||||
source,
|
||||
yamlKindName(document.Content[0].Kind),
|
||||
)
|
||||
}
|
||||
|
||||
root, err := buildCompositionNode(document.Content[0], source, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &compositionDocument{root: root, sources: []string{source}}, nil
|
||||
}
|
||||
|
||||
func parseCompositionBytes(source string, data []byte) (*compositionDocument, error) {
|
||||
return parseCompositionDocument(source, bytes.NewReader(data))
|
||||
}
|
||||
|
||||
func buildCompositionNode(node *yaml.Node, source, path string) (*compositionNode, error) {
|
||||
if node == nil {
|
||||
return nil, fmt.Errorf("configuration source %q at %s: YAML node is nil", source, displayCompositionPath(path))
|
||||
}
|
||||
if node.Kind == yaml.AliasNode {
|
||||
return nil, compositionNodeError(source, path, node, "YAML aliases are not supported because source ownership would be ambiguous")
|
||||
}
|
||||
result := &compositionNode{
|
||||
kind: node.Kind, tag: node.Tag, value: node.Value, path: path,
|
||||
sources: []string{source},
|
||||
}
|
||||
|
||||
switch node.Kind {
|
||||
case yaml.MappingNode:
|
||||
if len(node.Content)%2 != 0 {
|
||||
return nil, compositionNodeError(source, path, node, "mapping has an incomplete key/value pair")
|
||||
}
|
||||
seen := make(map[string]*yaml.Node, len(node.Content)/2)
|
||||
for index := 0; index < len(node.Content); index += 2 {
|
||||
keyNode := node.Content[index]
|
||||
valueNode := node.Content[index+1]
|
||||
if keyNode.Kind == yaml.AliasNode {
|
||||
return nil, compositionNodeError(source, path, keyNode, "YAML aliases are not supported because source ownership would be ambiguous")
|
||||
}
|
||||
if valueNode.Kind == yaml.AliasNode {
|
||||
return nil, compositionNodeError(source, appendCompositionPath(path, keyNode.Value), valueNode, "YAML aliases are not supported because source ownership would be ambiguous")
|
||||
}
|
||||
if keyNode.Kind != yaml.ScalarNode || keyNode.Tag != "!!str" {
|
||||
return nil, compositionNodeError(source, path, keyNode, "mapping keys must be strings")
|
||||
}
|
||||
key := keyNode.Value
|
||||
fieldPath := appendCompositionPath(path, key)
|
||||
if prior, duplicate := seen[key]; duplicate {
|
||||
return nil, fmt.Errorf(
|
||||
"configuration source %q at %s: duplicate YAML key %q (first declared at line %d, column %d; repeated at line %d, column %d)",
|
||||
source, displayCompositionPath(fieldPath), key,
|
||||
prior.Line, prior.Column, keyNode.Line, keyNode.Column,
|
||||
)
|
||||
}
|
||||
seen[key] = keyNode
|
||||
child, err := buildCompositionNode(valueNode, source, fieldPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result.fields = append(result.fields, compositionField{key: key, value: child})
|
||||
}
|
||||
case yaml.SequenceNode:
|
||||
for index, childNode := range node.Content {
|
||||
childPath := fmt.Sprintf("%s[%d]", path, index)
|
||||
child, err := buildCompositionNode(childNode, source, childPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result.items = append(result.items, child)
|
||||
}
|
||||
case yaml.ScalarNode:
|
||||
// Scalar tag and lexical value retain distinctions such as explicit
|
||||
// false, zero, an empty string, and null until final strict decoding.
|
||||
default:
|
||||
return nil, compositionNodeError(source, path, node, fmt.Sprintf("unsupported YAML node kind %s", yamlKindName(node.Kind)))
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// mergeAdditiveComposition recursively combines disjoint mappings. A scalar,
|
||||
// list, or final keyed value may have only one base owner, regardless of
|
||||
// whether duplicate values happen to be equal.
|
||||
func mergeAdditiveComposition(base, incoming *compositionDocument) (*compositionDocument, error) {
|
||||
return mergeAdditiveCompositions(base, incoming)
|
||||
}
|
||||
|
||||
// mergeAdditiveCompositions validates the complete base source set before
|
||||
// merging so a conflict names every source that claims the same final path.
|
||||
func mergeAdditiveCompositions(documents ...*compositionDocument) (*compositionDocument, error) {
|
||||
if len(documents) == 0 {
|
||||
return nil, fmt.Errorf("configuration additive base merge requires at least one document")
|
||||
}
|
||||
for index, document := range documents {
|
||||
if err := validateCompositionDocument(document, fmt.Sprintf("base[%d]", index)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if err := validateAdditiveClaims(documents); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
result := &compositionDocument{
|
||||
root: cloneCompositionNode(documents[0].root),
|
||||
sources: append([]string(nil), documents[0].sources...),
|
||||
}
|
||||
for _, incoming := range documents[1:] {
|
||||
merged, err := mergeAdditiveNodes(result.root, incoming.root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result.root = merged
|
||||
result.sources = appendUniqueStrings(result.sources, incoming.sources...)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func validateAdditiveClaims(documents []*compositionDocument) error {
|
||||
claims := make(map[string][]*compositionNode)
|
||||
for _, document := range documents {
|
||||
appendCompositionClaims(document.root, claims)
|
||||
}
|
||||
paths := make([]string, 0, len(claims))
|
||||
for path := range claims {
|
||||
paths = append(paths, path)
|
||||
}
|
||||
sort.Slice(paths, func(i, j int) bool {
|
||||
leftDepth := compositionPathDepth(paths[i])
|
||||
rightDepth := compositionPathDepth(paths[j])
|
||||
if leftDepth != rightDepth {
|
||||
return leftDepth < rightDepth
|
||||
}
|
||||
return paths[i] < paths[j]
|
||||
})
|
||||
for _, path := range paths {
|
||||
values := claims[path]
|
||||
if len(values) < 2 {
|
||||
continue
|
||||
}
|
||||
allPopulatedMappings := true
|
||||
for _, value := range values {
|
||||
if value.kind != yaml.MappingNode || len(value.fields) == 0 {
|
||||
allPopulatedMappings = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if !allPopulatedMappings {
|
||||
return newCompositionConflict("additive base merge", path, values...)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func appendCompositionClaims(node *compositionNode, claims map[string][]*compositionNode) {
|
||||
if node == nil {
|
||||
return
|
||||
}
|
||||
if node.path != "" {
|
||||
claims[node.path] = append(claims[node.path], node)
|
||||
}
|
||||
for _, field := range node.fields {
|
||||
appendCompositionClaims(field.value, claims)
|
||||
}
|
||||
}
|
||||
|
||||
func compositionPathDepth(path string) int {
|
||||
if path == "" {
|
||||
return 0
|
||||
}
|
||||
return strings.Count(path, ".") + strings.Count(path, "[") + 1
|
||||
}
|
||||
|
||||
func mergeAdditiveNodes(base, incoming *compositionNode) (*compositionNode, error) {
|
||||
if base.kind != yaml.MappingNode || incoming.kind != yaml.MappingNode {
|
||||
return nil, newCompositionConflict("additive base merge", base.path, base, incoming)
|
||||
}
|
||||
base.sources = appendUniqueStrings(base.sources, incoming.sources...)
|
||||
for _, incomingField := range incoming.fields {
|
||||
index := compositionFieldIndex(base.fields, incomingField.key)
|
||||
if index < 0 {
|
||||
field := cloneCompositionField(incomingField)
|
||||
base.fields = append(base.fields, field)
|
||||
continue
|
||||
}
|
||||
baseValue := base.fields[index].value
|
||||
incomingValue := incomingField.value
|
||||
if baseValue.kind == yaml.MappingNode && incomingValue.kind == yaml.MappingNode {
|
||||
if len(baseValue.fields) == 0 || len(incomingValue.fields) == 0 {
|
||||
return nil, newCompositionConflict("additive base merge", incomingValue.path, baseValue, incomingValue)
|
||||
}
|
||||
merged, err := mergeAdditiveNodes(baseValue, incomingValue)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
base.fields[index].value = merged
|
||||
continue
|
||||
}
|
||||
return nil, newCompositionConflict("additive base merge", incomingValue.path, baseValue, incomingValue)
|
||||
}
|
||||
return base, nil
|
||||
}
|
||||
|
||||
// mergeOverlayComposition applies the sole overwrite layer. Mappings merge
|
||||
// recursively; same-kind scalars and lists replace; null and kind changes are
|
||||
// rejected.
|
||||
func mergeOverlayComposition(base, overlay *compositionDocument) (*compositionDocument, error) {
|
||||
if err := validateCompositionDocument(base, "base"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := validateCompositionDocument(overlay, "overlay"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if null := firstNullCompositionNode(overlay.root); null != nil {
|
||||
return nil, fmt.Errorf(
|
||||
"configuration overlay at %s from %s: null cannot delete an effective value",
|
||||
displayCompositionPath(null.path), formatCompositionSources(null.sources),
|
||||
)
|
||||
}
|
||||
merged, err := mergeOverlayNodes(cloneCompositionNode(base.root), overlay.root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &compositionDocument{
|
||||
root: merged,
|
||||
sources: appendUniqueStrings(
|
||||
append([]string(nil), base.sources...), overlay.sources...,
|
||||
),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func mergeOverlayNodes(base, overlay *compositionNode) (*compositionNode, error) {
|
||||
if base.kind != yaml.MappingNode || overlay.kind != yaml.MappingNode {
|
||||
return nil, newCompositionConflict("profile overlay", base.path, base, overlay)
|
||||
}
|
||||
base.sources = appendUniqueStrings(base.sources, overlay.sources...)
|
||||
for _, overlayField := range overlay.fields {
|
||||
index := compositionFieldIndex(base.fields, overlayField.key)
|
||||
if index < 0 {
|
||||
field := cloneCompositionField(overlayField)
|
||||
base.fields = append(base.fields, field)
|
||||
continue
|
||||
}
|
||||
baseValue := base.fields[index].value
|
||||
overlayValue := overlayField.value
|
||||
if baseValue.kind != overlayValue.kind {
|
||||
return nil, newCompositionConflict("profile overlay kind change", overlayValue.path, baseValue, overlayValue)
|
||||
}
|
||||
if baseValue.kind == yaml.MappingNode {
|
||||
merged, err := mergeOverlayNodes(baseValue, overlayValue)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
base.fields[index].value = merged
|
||||
continue
|
||||
}
|
||||
base.fields[index].value = cloneCompositionNode(overlayValue)
|
||||
}
|
||||
return base, nil
|
||||
}
|
||||
|
||||
// canonicalYAML renders the effective mapping with sorted keys and normalized
|
||||
// presentation while retaining sequence order and scalar YAML types.
|
||||
func (document *compositionDocument) canonicalYAML() ([]byte, error) {
|
||||
if err := validateCompositionDocument(document, "document"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
root, err := compositionYAMLNode(document.root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var buffer bytes.Buffer
|
||||
encoder := yaml.NewEncoder(&buffer)
|
||||
encoder.SetIndent(2)
|
||||
if err := encoder.Encode(root); err != nil {
|
||||
return nil, fmt.Errorf("render effective configuration YAML: %w", err)
|
||||
}
|
||||
if err := encoder.Close(); err != nil {
|
||||
return nil, fmt.Errorf("finish effective configuration YAML: %w", err)
|
||||
}
|
||||
return buffer.Bytes(), nil
|
||||
}
|
||||
|
||||
// canonicalDigestInput provides a deterministic, formatting-independent byte
|
||||
// representation for later secret-free effective configuration digesting.
|
||||
func (document *compositionDocument) canonicalDigestInput() ([]byte, error) {
|
||||
if err := validateCompositionDocument(document, "document"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
value, err := canonicalCompositionValue(document.root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("serialize canonical configuration digest input: %w", err)
|
||||
}
|
||||
return append(data, '\n'), nil
|
||||
}
|
||||
|
||||
// semanticRecords returns sorted atomic values for future effective diff and
|
||||
// source-report projections. A caller receives copies of all source slices.
|
||||
func (document *compositionDocument) semanticRecords() ([]compositionValueRecord, error) {
|
||||
if err := validateCompositionDocument(document, "document"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var records []compositionValueRecord
|
||||
if err := appendCompositionRecords(document.root, &records); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Slice(records, func(i, j int) bool { return records[i].Path < records[j].Path })
|
||||
return records, nil
|
||||
}
|
||||
|
||||
// compactSemanticRecords returns the same logical atomic paths as
|
||||
// semanticRecords, but represents values as compact JSON-compatible YAML
|
||||
// values rather than the typed structural form used for digesting. It is the
|
||||
// stable human-facing projection for semantic comparisons.
|
||||
func (document *compositionDocument) compactSemanticRecords() ([]compositionValueRecord, error) {
|
||||
if err := validateCompositionDocument(document, "document"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var records []compositionValueRecord
|
||||
if err := appendCompactCompositionRecords(document.root, &records); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Slice(records, func(i, j int) bool { return records[i].Path < records[j].Path })
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func appendCompositionRecords(node *compositionNode, records *[]compositionValueRecord) error {
|
||||
if node.kind == yaml.MappingNode && len(node.fields) > 0 {
|
||||
for _, field := range node.fields {
|
||||
if err := appendCompositionRecords(field.value, records); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
value, err := canonicalCompositionValue(node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
encoded, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return fmt.Errorf("serialize configuration value at %s: %w", displayCompositionPath(node.path), err)
|
||||
}
|
||||
*records = append(*records, compositionValueRecord{
|
||||
Path: node.path, Kind: node.kind, Value: string(encoded),
|
||||
Sources: append([]string(nil), node.sources...),
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
func appendCompactCompositionRecords(node *compositionNode, records *[]compositionValueRecord) error {
|
||||
if node.kind == yaml.MappingNode && len(node.fields) > 0 {
|
||||
for _, field := range node.fields {
|
||||
if err := appendCompactCompositionRecords(field.value, records); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
value, err := compactCompositionValue(node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
encoded, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return fmt.Errorf("serialize configuration value at %s: %w", displayCompositionPath(node.path), err)
|
||||
}
|
||||
*records = append(*records, compositionValueRecord{
|
||||
Path: node.path, Kind: node.kind, Value: string(encoded),
|
||||
Sources: append([]string(nil), node.sources...),
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
func compactCompositionValue(node *compositionNode) (any, error) {
|
||||
switch node.kind {
|
||||
case yaml.MappingNode:
|
||||
values := make(map[string]any, len(node.fields))
|
||||
for _, field := range node.fields {
|
||||
value, err := compactCompositionValue(field.value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
values[field.key] = value
|
||||
}
|
||||
return values, nil
|
||||
case yaml.SequenceNode:
|
||||
values := make([]any, 0, len(node.items))
|
||||
for _, item := range node.items {
|
||||
value, err := compactCompositionValue(item)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
values = append(values, value)
|
||||
}
|
||||
return values, nil
|
||||
case yaml.ScalarNode:
|
||||
return canonicalScalarValue(node)
|
||||
default:
|
||||
return nil, fmt.Errorf("configuration at %s has unsupported YAML kind %s", displayCompositionPath(node.path), yamlKindName(node.kind))
|
||||
}
|
||||
}
|
||||
|
||||
type canonicalCompositionField struct {
|
||||
Key string `json:"key"`
|
||||
Value any `json:"value"`
|
||||
}
|
||||
|
||||
type canonicalCompositionNode struct {
|
||||
Kind string `json:"kind"`
|
||||
Tag string `json:"tag,omitempty"`
|
||||
Value any `json:"value,omitempty"`
|
||||
Fields []canonicalCompositionField `json:"fields,omitempty"`
|
||||
Items []any `json:"items,omitempty"`
|
||||
}
|
||||
|
||||
func canonicalCompositionValue(node *compositionNode) (any, error) {
|
||||
switch node.kind {
|
||||
case yaml.MappingNode:
|
||||
fields := append([]compositionField(nil), node.fields...)
|
||||
sort.Slice(fields, func(i, j int) bool { return fields[i].key < fields[j].key })
|
||||
result := canonicalCompositionNode{Kind: "mapping"}
|
||||
if len(fields) == 0 {
|
||||
result.Fields = []canonicalCompositionField{}
|
||||
}
|
||||
for _, field := range fields {
|
||||
value, err := canonicalCompositionValue(field.value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result.Fields = append(result.Fields, canonicalCompositionField{Key: field.key, Value: value})
|
||||
}
|
||||
return result, nil
|
||||
case yaml.SequenceNode:
|
||||
result := canonicalCompositionNode{Kind: "sequence", Items: make([]any, 0, len(node.items))}
|
||||
for _, item := range node.items {
|
||||
value, err := canonicalCompositionValue(item)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result.Items = append(result.Items, value)
|
||||
}
|
||||
return result, nil
|
||||
case yaml.ScalarNode:
|
||||
value, err := canonicalScalarValue(node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return canonicalCompositionNode{Kind: "scalar", Tag: node.tag, Value: value}, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("configuration at %s has unsupported YAML kind %s", displayCompositionPath(node.path), yamlKindName(node.kind))
|
||||
}
|
||||
}
|
||||
|
||||
func canonicalScalarValue(node *compositionNode) (any, error) {
|
||||
raw := &yaml.Node{Kind: yaml.ScalarNode, Tag: node.tag, Value: node.value}
|
||||
var value any
|
||||
if err := raw.Decode(&value); err != nil {
|
||||
return nil, fmt.Errorf("decode scalar at %s: %w", displayCompositionPath(node.path), err)
|
||||
}
|
||||
switch typed := value.(type) {
|
||||
case nil, bool, string, int, int64, uint64, float64:
|
||||
return typed, nil
|
||||
default:
|
||||
// yaml.v3 may decode timestamps or uncommon scalar tags into types that
|
||||
// encoding/json can serialize deterministically. Preserve the resolved
|
||||
// tag alongside the value in the containing canonical node.
|
||||
return typed, nil
|
||||
}
|
||||
}
|
||||
|
||||
func compositionYAMLNode(node *compositionNode) (*yaml.Node, error) {
|
||||
switch node.kind {
|
||||
case yaml.MappingNode:
|
||||
result := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
|
||||
fields := append([]compositionField(nil), node.fields...)
|
||||
sort.Slice(fields, func(i, j int) bool { return fields[i].key < fields[j].key })
|
||||
for _, field := range fields {
|
||||
value, err := compositionYAMLNode(field.value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result.Content = append(result.Content,
|
||||
&yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: field.key},
|
||||
value,
|
||||
)
|
||||
}
|
||||
return result, nil
|
||||
case yaml.SequenceNode:
|
||||
result := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
|
||||
for _, item := range node.items {
|
||||
value, err := compositionYAMLNode(item)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result.Content = append(result.Content, value)
|
||||
}
|
||||
return result, nil
|
||||
case yaml.ScalarNode:
|
||||
return normalizedCompositionScalarNode(node)
|
||||
default:
|
||||
return nil, fmt.Errorf("configuration at %s has unsupported YAML kind %s", displayCompositionPath(node.path), yamlKindName(node.kind))
|
||||
}
|
||||
}
|
||||
|
||||
func normalizedCompositionScalarNode(node *compositionNode) (*yaml.Node, error) {
|
||||
raw := &yaml.Node{Kind: yaml.ScalarNode, Tag: node.tag, Value: node.value}
|
||||
var value any
|
||||
if err := raw.Decode(&value); err != nil {
|
||||
return nil, fmt.Errorf("normalize scalar at %s: %w", displayCompositionPath(node.path), err)
|
||||
}
|
||||
normalized := &yaml.Node{}
|
||||
if err := normalized.Encode(value); err != nil {
|
||||
return nil, fmt.Errorf("encode normalized scalar at %s: %w", displayCompositionPath(node.path), err)
|
||||
}
|
||||
if normalized.Kind != yaml.ScalarNode {
|
||||
return nil, fmt.Errorf("normalize scalar at %s produced YAML kind %s", displayCompositionPath(node.path), yamlKindName(normalized.Kind))
|
||||
}
|
||||
return normalized, nil
|
||||
}
|
||||
|
||||
func validateCompositionDocument(document *compositionDocument, role string) error {
|
||||
if document == nil || document.root == nil {
|
||||
return fmt.Errorf("configuration composition %s document is nil", role)
|
||||
}
|
||||
if document.root.kind != yaml.MappingNode {
|
||||
return fmt.Errorf("configuration composition %s root must be a mapping", role)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func firstNullCompositionNode(node *compositionNode) *compositionNode {
|
||||
if node == nil {
|
||||
return nil
|
||||
}
|
||||
if node.kind == yaml.ScalarNode && node.tag == "!!null" {
|
||||
return node
|
||||
}
|
||||
for _, field := range node.fields {
|
||||
if found := firstNullCompositionNode(field.value); found != nil {
|
||||
return found
|
||||
}
|
||||
}
|
||||
for _, item := range node.items {
|
||||
if found := firstNullCompositionNode(item); found != nil {
|
||||
return found
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func cloneCompositionNode(node *compositionNode) *compositionNode {
|
||||
if node == nil {
|
||||
return nil
|
||||
}
|
||||
clone := &compositionNode{
|
||||
kind: node.kind, tag: node.tag, value: node.value, path: node.path,
|
||||
sources: append([]string(nil), node.sources...),
|
||||
}
|
||||
for _, field := range node.fields {
|
||||
clone.fields = append(clone.fields, cloneCompositionField(field))
|
||||
}
|
||||
for _, item := range node.items {
|
||||
clone.items = append(clone.items, cloneCompositionNode(item))
|
||||
}
|
||||
return clone
|
||||
}
|
||||
|
||||
func cloneCompositionField(field compositionField) compositionField {
|
||||
return compositionField{key: field.key, value: cloneCompositionNode(field.value)}
|
||||
}
|
||||
|
||||
func compositionFieldIndex(fields []compositionField, key string) int {
|
||||
for index := range fields {
|
||||
if fields[index].key == key {
|
||||
return index
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func appendCompositionPath(parent, key string) string {
|
||||
if isSimpleCompositionPathSegment(key) {
|
||||
if parent == "" {
|
||||
return key
|
||||
}
|
||||
return parent + "." + key
|
||||
}
|
||||
if parent == "" {
|
||||
return "[" + strconv.Quote(key) + "]"
|
||||
}
|
||||
return parent + "[" + strconv.Quote(key) + "]"
|
||||
}
|
||||
|
||||
func isSimpleCompositionPathSegment(value string) bool {
|
||||
if value == "" {
|
||||
return false
|
||||
}
|
||||
for index, char := range value {
|
||||
if (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || char == '_' || (index > 0 && char >= '0' && char <= '9') || (index > 0 && char == '-') {
|
||||
continue
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func displayCompositionPath(path string) string {
|
||||
if path == "" {
|
||||
return "<root>"
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func compositionNodeError(source, path string, node *yaml.Node, message string) error {
|
||||
line, column := 0, 0
|
||||
if node != nil {
|
||||
line, column = node.Line, node.Column
|
||||
}
|
||||
return fmt.Errorf(
|
||||
"configuration source %q at %s (line %d, column %d): %s",
|
||||
source, displayCompositionPath(path), line, column, message,
|
||||
)
|
||||
}
|
||||
|
||||
func newCompositionConflict(operation, path string, values ...*compositionNode) error {
|
||||
var sources []string
|
||||
var kinds []string
|
||||
for _, value := range values {
|
||||
if value == nil {
|
||||
continue
|
||||
}
|
||||
sources = appendUniqueStrings(sources, compositionClaimSources(value)...)
|
||||
kind := yamlKindName(value.kind)
|
||||
if !containsString(kinds, kind) {
|
||||
kinds = append(kinds, kind)
|
||||
}
|
||||
}
|
||||
return fmt.Errorf(
|
||||
"configuration %s conflict at %s: claimed by %s (YAML kinds: %s)",
|
||||
operation, displayCompositionPath(path), formatCompositionSources(sources), strings.Join(kinds, ", "),
|
||||
)
|
||||
}
|
||||
|
||||
func compositionClaimSources(node *compositionNode) []string {
|
||||
if node == nil {
|
||||
return nil
|
||||
}
|
||||
sources := append([]string(nil), node.sources...)
|
||||
for _, field := range node.fields {
|
||||
sources = appendUniqueStrings(sources, compositionClaimSources(field.value)...)
|
||||
}
|
||||
for _, item := range node.items {
|
||||
sources = appendUniqueStrings(sources, compositionClaimSources(item)...)
|
||||
}
|
||||
return sources
|
||||
}
|
||||
|
||||
func appendUniqueStrings(values []string, additions ...string) []string {
|
||||
seen := make(map[string]struct{}, len(values)+len(additions))
|
||||
result := make([]string, 0, len(values)+len(additions))
|
||||
for _, value := range append(append([]string(nil), values...), additions...) {
|
||||
if _, exists := seen[value]; exists {
|
||||
continue
|
||||
}
|
||||
seen[value] = struct{}{}
|
||||
result = append(result, value)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func containsString(values []string, target string) bool {
|
||||
for _, value := range values {
|
||||
if value == target {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func formatCompositionSources(sources []string) string {
|
||||
quoted := make([]string, 0, len(sources))
|
||||
for _, source := range sources {
|
||||
quoted = append(quoted, strconv.Quote(source))
|
||||
}
|
||||
return strings.Join(quoted, ", ")
|
||||
}
|
||||
|
||||
func yamlKindName(kind yaml.Kind) string {
|
||||
switch kind {
|
||||
case yaml.DocumentNode:
|
||||
return "document"
|
||||
case yaml.MappingNode:
|
||||
return "mapping"
|
||||
case yaml.SequenceNode:
|
||||
return "sequence"
|
||||
case yaml.ScalarNode:
|
||||
return "scalar"
|
||||
case yaml.AliasNode:
|
||||
return "alias"
|
||||
default:
|
||||
return fmt.Sprintf("kind(%d)", kind)
|
||||
}
|
||||
}
|
||||
@@ -1,353 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func TestParseCompositionDocumentRetainsPresenceOwnershipAndDeclarationOrder(t *testing.T) {
|
||||
document := mustParseComposition(t, "root.yml", `zeta: false
|
||||
zero: 0
|
||||
empty_map: {}
|
||||
empty_list: []
|
||||
nested:
|
||||
value: ""
|
||||
`)
|
||||
|
||||
wantOrder := []string{"zeta", "zero", "empty_map", "empty_list", "nested"}
|
||||
gotOrder := make([]string, 0, len(document.root.fields))
|
||||
for _, field := range document.root.fields {
|
||||
gotOrder = append(gotOrder, field.key)
|
||||
}
|
||||
if !reflect.DeepEqual(gotOrder, wantOrder) {
|
||||
t.Fatalf("declaration order = %#v, want %#v", gotOrder, wantOrder)
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
path string
|
||||
kind yaml.Kind
|
||||
tag string
|
||||
value string
|
||||
}{
|
||||
{path: "zeta", kind: yaml.ScalarNode, tag: "!!bool", value: "false"},
|
||||
{path: "zero", kind: yaml.ScalarNode, tag: "!!int", value: "0"},
|
||||
{path: "empty_map", kind: yaml.MappingNode, tag: "!!map"},
|
||||
{path: "empty_list", kind: yaml.SequenceNode, tag: "!!seq"},
|
||||
{path: "nested.value", kind: yaml.ScalarNode, tag: "!!str", value: ""},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
node := compositionNodeAtPath(t, document.root, tt.path)
|
||||
if node.kind != tt.kind || node.tag != tt.tag || node.value != tt.value || !reflect.DeepEqual(node.sources, []string{"root.yml"}) {
|
||||
t.Fatalf("node %s = kind=%v tag=%q value=%q sources=%#v", tt.path, node.kind, node.tag, node.value, node.sources)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCompositionDocumentRejectsAmbiguousOrMalformedYAML(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
yaml string
|
||||
want string
|
||||
}{
|
||||
{name: "duplicate top-level key", yaml: "value: 1\nvalue: 2\n", want: "duplicate YAML key"},
|
||||
{name: "duplicate nested key", yaml: "outer:\n value: 1\n value: 2\n", want: "outer.value"},
|
||||
{name: "alias", yaml: "base: &base\n value: 1\ncopy: *base\n", want: "aliases are not supported"},
|
||||
{name: "trailing document", yaml: "value: 1\n---\nvalue: 2\n", want: "exactly one YAML document"},
|
||||
{name: "top-level sequence", yaml: "- value\n", want: "top-level document must be a mapping"},
|
||||
{name: "non-string key", yaml: "1: value\n", want: "mapping keys must be strings"},
|
||||
{name: "malformed", yaml: "outer: [\n", want: "decode YAML"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, err := parseCompositionDocument("broken.yml", strings.NewReader(tt.yaml))
|
||||
if err == nil {
|
||||
t.Fatal("parseCompositionDocument() error = nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "broken.yml") || !strings.Contains(err.Error(), tt.want) {
|
||||
t.Fatalf("error = %q, want source and %q", err, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeAdditiveCompositionJoinsOnlyDisjointMappings(t *testing.T) {
|
||||
root := mustParseComposition(t, "pipeline.yml", `scriptorium:
|
||||
artifacts:
|
||||
recap:
|
||||
enabled: true
|
||||
zero: 0
|
||||
`)
|
||||
imports := mustParseComposition(t, "conf.d/artifacts.yml", `scriptorium:
|
||||
artifacts:
|
||||
handout:
|
||||
enabled: false
|
||||
empty_list: []
|
||||
`)
|
||||
|
||||
merged, err := mergeAdditiveComposition(root, imports)
|
||||
if err != nil {
|
||||
t.Fatalf("mergeAdditiveComposition() error = %v", err)
|
||||
}
|
||||
for _, path := range []string{
|
||||
"empty_list", "scriptorium.artifacts.handout.enabled",
|
||||
"scriptorium.artifacts.recap.enabled", "zero",
|
||||
} {
|
||||
_ = compositionNodeAtPath(t, merged.root, path)
|
||||
}
|
||||
if got := compositionNodeAtPath(t, merged.root, "scriptorium.artifacts.handout.enabled").sources; !reflect.DeepEqual(got, []string{"conf.d/artifacts.yml"}) {
|
||||
t.Fatalf("handout sources = %#v", got)
|
||||
}
|
||||
if got := compositionNodeAtPath(t, merged.root, "scriptorium.artifacts.recap.enabled").sources; !reflect.DeepEqual(got, []string{"pipeline.yml"}) {
|
||||
t.Fatalf("recap sources = %#v", got)
|
||||
}
|
||||
|
||||
// Merge operations return a new document and retain the declared order in
|
||||
// each input for source-aware diagnostics.
|
||||
if len(root.root.fields) != 2 || len(imports.root.fields) != 2 {
|
||||
t.Fatalf("merge mutated inputs: root=%d import=%d", len(root.root.fields), len(imports.root.fields))
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeAdditiveCompositionRejectsEveryDuplicateClass(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
baseYAML string
|
||||
nextYAML string
|
||||
path string
|
||||
}{
|
||||
{name: "equal scalar", baseYAML: "value: true\n", nextYAML: "value: true\n", path: "value"},
|
||||
{name: "different scalar", baseYAML: "value: true\n", nextYAML: "value: false\n", path: "value"},
|
||||
{name: "atomic list", baseYAML: "values: [one]\n", nextYAML: "values: [two]\n", path: "values"},
|
||||
{name: "keyed entry", baseYAML: "items:\n shared:\n left: 1\n", nextYAML: "items:\n shared:\n left: 2\n", path: "items.shared.left"},
|
||||
{name: "kind conflict", baseYAML: "value:\n nested: true\n", nextYAML: "value: scalar\n", path: "value"},
|
||||
{name: "duplicate empty map", baseYAML: "value: {}\n", nextYAML: "value: {}\n", path: "value"},
|
||||
{name: "empty map then populated map", baseYAML: "value: {}\n", nextYAML: "value: {nested: true}\n", path: "value"},
|
||||
{name: "populated map then empty map", baseYAML: "value: {nested: true}\n", nextYAML: "value: {}\n", path: "value"},
|
||||
{name: "duplicate empty list", baseYAML: "value: []\n", nextYAML: "value: []\n", path: "value"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
base := mustParseComposition(t, "base.yml", tt.baseYAML)
|
||||
next := mustParseComposition(t, "next.yml", tt.nextYAML)
|
||||
_, err := mergeAdditiveComposition(base, next)
|
||||
if err == nil {
|
||||
t.Fatal("mergeAdditiveComposition() error = nil")
|
||||
}
|
||||
for _, want := range []string{tt.path, "base.yml", "next.yml"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Fatalf("error = %q, want %q", err, want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeAdditiveCompositionReportsAllClaimingSources(t *testing.T) {
|
||||
left := mustParseComposition(t, "left.yml", "group:\n left: 1\n")
|
||||
right := mustParseComposition(t, "right.yml", "group:\n right: 2\n")
|
||||
base, err := mergeAdditiveComposition(left, right)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
overlap := mustParseComposition(t, "overlap.yml", "group: scalar\n")
|
||||
_, err = mergeAdditiveComposition(base, overlap)
|
||||
if err == nil {
|
||||
t.Fatal("mergeAdditiveComposition() error = nil")
|
||||
}
|
||||
for _, want := range []string{"group", "left.yml", "right.yml", "overlap.yml"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Fatalf("error = %q, want %q", err, want)
|
||||
}
|
||||
}
|
||||
|
||||
_, err = mergeAdditiveCompositions(
|
||||
mustParseComposition(t, "first.yml", "value: 1\n"),
|
||||
mustParseComposition(t, "second.yml", "value: 2\n"),
|
||||
mustParseComposition(t, "third.yml", "value: 3\n"),
|
||||
)
|
||||
if err == nil {
|
||||
t.Fatal("mergeAdditiveCompositions() error = nil")
|
||||
}
|
||||
for _, want := range []string{"value", "first.yml", "second.yml", "third.yml"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Fatalf("error = %q, want %q", err, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeOverlayCompositionRecursesMapsAndReplacesAtomicValues(t *testing.T) {
|
||||
base := mustParseComposition(t, "base.yml", `feature:
|
||||
enabled: true
|
||||
retries: 3
|
||||
values: [one, two]
|
||||
inherited: kept
|
||||
artifacts:
|
||||
recap:
|
||||
enabled: true
|
||||
`)
|
||||
overlay := mustParseComposition(t, "testing.yml", `feature:
|
||||
enabled: false
|
||||
retries: 0
|
||||
values: []
|
||||
added: present
|
||||
artifacts:
|
||||
handout:
|
||||
enabled: false
|
||||
`)
|
||||
|
||||
merged, err := mergeOverlayComposition(base, overlay)
|
||||
if err != nil {
|
||||
t.Fatalf("mergeOverlayComposition() error = %v", err)
|
||||
}
|
||||
assertCompositionScalar(t, merged.root, "feature.enabled", "!!bool", "false", "testing.yml")
|
||||
assertCompositionScalar(t, merged.root, "feature.retries", "!!int", "0", "testing.yml")
|
||||
assertCompositionScalar(t, merged.root, "feature.inherited", "!!str", "kept", "base.yml")
|
||||
assertCompositionScalar(t, merged.root, "feature.added", "!!str", "present", "testing.yml")
|
||||
if values := compositionNodeAtPath(t, merged.root, "feature.values"); values.kind != yaml.SequenceNode || len(values.items) != 0 || !reflect.DeepEqual(values.sources, []string{"testing.yml"}) {
|
||||
t.Fatalf("replaced list = %#v", values)
|
||||
}
|
||||
_ = compositionNodeAtPath(t, merged.root, "artifacts.recap.enabled")
|
||||
_ = compositionNodeAtPath(t, merged.root, "artifacts.handout.enabled")
|
||||
}
|
||||
|
||||
func TestMergeOverlayCompositionRejectsKindChangesAndNullDeletion(t *testing.T) {
|
||||
kindTests := []struct {
|
||||
name string
|
||||
baseYAML string
|
||||
overlay string
|
||||
}{
|
||||
{name: "map to scalar", baseYAML: "value: {nested: true}\n", overlay: "value: replacement\n"},
|
||||
{name: "scalar to map", baseYAML: "value: original\n", overlay: "value: {nested: true}\n"},
|
||||
{name: "list to scalar", baseYAML: "value: [one]\n", overlay: "value: replacement\n"},
|
||||
{name: "scalar to list", baseYAML: "value: original\n", overlay: "value: [one]\n"},
|
||||
}
|
||||
for _, tt := range kindTests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, err := mergeOverlayComposition(
|
||||
mustParseComposition(t, "base.yml", tt.baseYAML),
|
||||
mustParseComposition(t, "overlay.yml", tt.overlay),
|
||||
)
|
||||
if err == nil || !strings.Contains(err.Error(), "value") || !strings.Contains(err.Error(), "kind change") {
|
||||
t.Fatalf("error = %v, want value kind change", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
for _, overlayYAML := range []string{"value: null\n", "value: ~\n", "nested:\n value:\n"} {
|
||||
_, err := mergeOverlayComposition(
|
||||
mustParseComposition(t, "base.yml", "value: original\nnested:\n value: original\n"),
|
||||
mustParseComposition(t, "overlay.yml", overlayYAML),
|
||||
)
|
||||
if err == nil || !strings.Contains(err.Error(), "null cannot delete") || !strings.Contains(err.Error(), "overlay.yml") {
|
||||
t.Fatalf("error = %v, want source-qualified null rejection", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompositionCanonicalOutputsAreDeterministic(t *testing.T) {
|
||||
first := mustParseComposition(t, "first.yml", `zeta: 01
|
||||
alpha:
|
||||
list: [true, false]
|
||||
empty: {}
|
||||
`)
|
||||
second := mustParseComposition(t, "second.yml", `alpha:
|
||||
empty: {}
|
||||
list:
|
||||
- true
|
||||
- false
|
||||
zeta: 1
|
||||
`)
|
||||
|
||||
firstYAML, err := first.canonicalYAML()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secondYAML, err := second.canonicalYAML()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(firstYAML, secondYAML) {
|
||||
t.Fatalf("canonical YAML differs:\n%s\n---\n%s", firstYAML, secondYAML)
|
||||
}
|
||||
firstDigest, err := first.canonicalDigestInput()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secondDigest, err := second.canonicalDigestInput()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(firstDigest, secondDigest) {
|
||||
t.Fatalf("digest input differs:\n%s\n---\n%s", firstDigest, secondDigest)
|
||||
}
|
||||
|
||||
left := mustParseComposition(t, "left.yml", "zeta: 1\n")
|
||||
right := mustParseComposition(t, "right.yml", "alpha: 2\n")
|
||||
leftRight, err := mergeAdditiveComposition(left, right)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rightLeft, err := mergeAdditiveComposition(right, left)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want, _ := leftRight.canonicalDigestInput()
|
||||
got, _ := rightLeft.canonicalDigestInput()
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("source traversal changed semantic digest input: got %s want %s", got, want)
|
||||
}
|
||||
wantYAML, _ := leftRight.canonicalYAML()
|
||||
gotYAML, _ := rightLeft.canonicalYAML()
|
||||
if !reflect.DeepEqual(gotYAML, wantYAML) {
|
||||
t.Fatalf("source traversal changed canonical YAML: got %s want %s", gotYAML, wantYAML)
|
||||
}
|
||||
|
||||
records, err := first.semanticRecords()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
paths := make([]string, 0, len(records))
|
||||
for _, record := range records {
|
||||
paths = append(paths, record.Path)
|
||||
}
|
||||
if wantPaths := []string{"alpha.empty", "alpha.list", "zeta"}; !reflect.DeepEqual(paths, wantPaths) {
|
||||
t.Fatalf("semantic record paths = %#v, want %#v", paths, wantPaths)
|
||||
}
|
||||
}
|
||||
|
||||
func mustParseComposition(t *testing.T, source, input string) *compositionDocument {
|
||||
t.Helper()
|
||||
document, err := parseCompositionBytes(source, []byte(input))
|
||||
if err != nil {
|
||||
t.Fatalf("parseCompositionBytes(%q) error = %v", source, err)
|
||||
}
|
||||
return document
|
||||
}
|
||||
|
||||
func compositionNodeAtPath(t *testing.T, root *compositionNode, path string) *compositionNode {
|
||||
t.Helper()
|
||||
node := root
|
||||
for _, segment := range strings.Split(path, ".") {
|
||||
if node == nil || node.kind != yaml.MappingNode {
|
||||
t.Fatalf("path %q reached non-mapping at %q", path, segment)
|
||||
}
|
||||
index := compositionFieldIndex(node.fields, segment)
|
||||
if index < 0 {
|
||||
t.Fatalf("path %q missing segment %q", path, segment)
|
||||
}
|
||||
node = node.fields[index].value
|
||||
}
|
||||
return node
|
||||
}
|
||||
|
||||
func assertCompositionScalar(t *testing.T, root *compositionNode, path, tag, value, source string) {
|
||||
t.Helper()
|
||||
node := compositionNodeAtPath(t, root, path)
|
||||
if node.kind != yaml.ScalarNode || node.tag != tag || node.value != value || !reflect.DeepEqual(node.sources, []string{source}) {
|
||||
t.Fatalf("%s = kind=%s tag=%q value=%q sources=%#v", path, yamlKindName(node.kind), node.tag, node.value, node.sources)
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,7 @@
|
||||
package config
|
||||
|
||||
import configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
|
||||
// Config is the resolved combined configuration from pipeline.yml,
|
||||
// campaign.yml, and session.yml.
|
||||
type Config struct {
|
||||
@@ -17,25 +19,24 @@ type Config struct {
|
||||
|
||||
// PipelineConfig contains durable pipeline-level settings.
|
||||
type PipelineConfig struct {
|
||||
Workspace WorkspaceConfig `yaml:"workspace"`
|
||||
Campaigns CampaignsConfig `yaml:"campaigns"`
|
||||
Storage StorageConfig `yaml:"storage"`
|
||||
Spool SpoolConfig `yaml:"spool"`
|
||||
Cache CacheConfig `yaml:"cache"`
|
||||
Publish *PublishConfig `yaml:"publish"`
|
||||
Secrets *SecretsConfig `yaml:"secrets"`
|
||||
WhisperX WhisperXConfig `yaml:"whisperx"`
|
||||
Seriatim SeriatimConfig `yaml:"seriatim"`
|
||||
Audita AuditaConfig `yaml:"audita"`
|
||||
Normalize *NormalizeConfig `yaml:"normalize"`
|
||||
Trim *TrimConfig `yaml:"trim"`
|
||||
Render *RenderConfig `yaml:"render"`
|
||||
Scriptorium *ScriptoriumConfig `yaml:"scriptorium"`
|
||||
Notarius *NotariusConfig `yaml:"notarius"`
|
||||
Notification NotificationConfig `yaml:"notification"`
|
||||
Workspace WorkspaceConfig `yaml:"workspace" json:"workspace"`
|
||||
Campaigns CampaignsConfig `yaml:"campaigns" json:"campaigns"`
|
||||
Storage StorageConfig `yaml:"storage" json:"storage"`
|
||||
Spool SpoolConfig `yaml:"spool" json:"spool"`
|
||||
Cache CacheConfig `yaml:"cache" json:"cache"`
|
||||
Publish *PublishConfig `yaml:"publish" json:"publish"`
|
||||
WhisperX WhisperXConfig `yaml:"whisperx" json:"whisperx"`
|
||||
Seriatim SeriatimConfig `yaml:"seriatim" json:"seriatim"`
|
||||
Audita AuditaConfig `yaml:"audita" json:"audita"`
|
||||
Normalize *NormalizeConfig `yaml:"normalize" json:"normalize"`
|
||||
Trim *TrimConfig `yaml:"trim" json:"trim"`
|
||||
Render *RenderConfig `yaml:"render" json:"render"`
|
||||
Scriptorium *ScriptoriumConfig `yaml:"scriptorium" json:"scriptorium"`
|
||||
Notarius *NotariusConfig `yaml:"notarius" json:"notarius"`
|
||||
Notification NotificationConfig `yaml:"notification" json:"notification"`
|
||||
|
||||
resolution *pipelineResolutionMetadata `yaml:"-"`
|
||||
familyCatalog ArtifactFamilyCatalog `yaml:"-"`
|
||||
resolution *pipelineResolutionMetadata `yaml:"-" json:"-"`
|
||||
familyCatalog ArtifactFamilyCatalog `yaml:"-" json:"-"`
|
||||
}
|
||||
|
||||
// ArtifactFamilies returns a copy of runtime-only expansion provenance.
|
||||
@@ -48,51 +49,46 @@ func ArtifactFamilies(cfg *PipelineConfig) ArtifactFamilyCatalog {
|
||||
|
||||
// CampaignsConfig configures the local campaign registry.
|
||||
type CampaignsConfig struct {
|
||||
Root string `yaml:"root"`
|
||||
DefaultCampaignID string `yaml:"default_campaign_id"`
|
||||
Root string `yaml:"root" json:"root"`
|
||||
DefaultCampaignID string `yaml:"default_campaign_id" json:"default_campaign_id"`
|
||||
}
|
||||
|
||||
// CampaignConfig contains stable campaign-level identity and input defaults.
|
||||
type CampaignConfig struct {
|
||||
CampaignID string `yaml:"campaign_id"`
|
||||
SessionTemplateFile string `yaml:"session_template_file"`
|
||||
Inputs CampaignInputsConfig `yaml:"inputs"`
|
||||
CampaignID string `yaml:"campaign_id" json:"campaign_id"`
|
||||
SessionTemplateFile string `yaml:"session_template_file" json:"session_template_file"`
|
||||
Inputs CampaignInputsConfig `yaml:"inputs" json:"inputs"`
|
||||
}
|
||||
|
||||
// CampaignInputsConfig contains stable campaign-level input file references.
|
||||
type CampaignInputsConfig struct {
|
||||
SpeakersFile string `yaml:"speakers_file"`
|
||||
AutocorrectFile string `yaml:"autocorrect_file"`
|
||||
GlossaryFile string `yaml:"glossary_file"`
|
||||
PartyFile string `yaml:"party_file"`
|
||||
SpellCatalogFile string `yaml:"spell_catalog_file"`
|
||||
SpeakersFile string `yaml:"speakers_file" json:"speakers_file"`
|
||||
AutocorrectFile string `yaml:"autocorrect_file" json:"autocorrect_file"`
|
||||
GlossaryFile string `yaml:"glossary_file" json:"glossary_file"`
|
||||
PartyFile string `yaml:"party_file" json:"party_file"`
|
||||
SpellCatalogFile string `yaml:"spell_catalog_file" json:"spell_catalog_file"`
|
||||
}
|
||||
|
||||
// SessionConfig contains per-session inputs and metadata.
|
||||
type SessionConfig struct {
|
||||
SessionID string `yaml:"session_id"`
|
||||
PreviousSessionID string `yaml:"previous_session_id"`
|
||||
Campaign string `yaml:"campaign"`
|
||||
Date string `yaml:"date"`
|
||||
Title string `yaml:"title"`
|
||||
Inputs SessionInputsConfig `yaml:"inputs"`
|
||||
SessionID string `yaml:"session_id" json:"session_id"`
|
||||
PreviousSessionID string `yaml:"previous_session_id" json:"previous_session_id"`
|
||||
Campaign string `yaml:"campaign" json:"campaign"`
|
||||
Date string `yaml:"date" json:"date"`
|
||||
Title string `yaml:"title" json:"title"`
|
||||
Inputs SessionInputsConfig `yaml:"inputs" json:"inputs"`
|
||||
}
|
||||
|
||||
// WorkspaceConfig configures local workspace behavior.
|
||||
type WorkspaceConfig struct {
|
||||
Root string `yaml:"root"`
|
||||
CleanupAfterPublish bool `yaml:"cleanup_after_publish"`
|
||||
}
|
||||
|
||||
// SecretsConfig configures optional local filesystem secret loading.
|
||||
type SecretsConfig struct {
|
||||
EnvDir string `yaml:"env_dir"`
|
||||
Root string `yaml:"root" json:"root"`
|
||||
CleanupAfterPublish bool `yaml:"cleanup_after_publish" json:"cleanup_after_publish"`
|
||||
}
|
||||
|
||||
// StorageConfig configures storage backends and related parameters.
|
||||
type StorageConfig struct {
|
||||
Backend string `yaml:"backend"`
|
||||
S3 *StorageS3Config `yaml:"s3"`
|
||||
Backend string `yaml:"backend" json:"backend"`
|
||||
S3 *StorageS3Config `yaml:"s3" json:"s3"`
|
||||
}
|
||||
|
||||
const (
|
||||
@@ -104,178 +100,178 @@ const (
|
||||
|
||||
// StorageS3Config configures S3 storage coordinates.
|
||||
type StorageS3Config struct {
|
||||
Bucket string `yaml:"bucket"`
|
||||
RootPrefix string `yaml:"root_prefix"`
|
||||
Region string `yaml:"region"`
|
||||
Endpoint string `yaml:"endpoint"`
|
||||
ForcePathStyle bool `yaml:"force_path_style"`
|
||||
AccessKeyIDEnv string `yaml:"access_key_id_env"`
|
||||
SecretKeyEnv string `yaml:"secret_access_key_env"`
|
||||
Bucket string `yaml:"bucket" json:"bucket"`
|
||||
RootPrefix string `yaml:"root_prefix" json:"root_prefix"`
|
||||
Region string `yaml:"region" json:"region"`
|
||||
Endpoint string `yaml:"endpoint" json:"endpoint"`
|
||||
ForcePathStyle bool `yaml:"force_path_style" json:"force_path_style"`
|
||||
AccessKeyID *configkit.Secret `yaml:"access_key_id,omitempty" json:"access_key_id,omitempty"`
|
||||
SecretAccessKey *configkit.Secret `yaml:"secret_access_key,omitempty" json:"secret_access_key,omitempty"`
|
||||
}
|
||||
|
||||
// SpoolConfig configures local spool storage for staged data.
|
||||
type SpoolConfig struct {
|
||||
Root string `yaml:"root"`
|
||||
DeleteAudioAfterPublish bool `yaml:"delete_audio_after_publish"`
|
||||
Root string `yaml:"root" json:"root"`
|
||||
DeleteAudioAfterPublish bool `yaml:"delete_audio_after_publish" json:"delete_audio_after_publish"`
|
||||
}
|
||||
|
||||
// CacheConfig configures durable local caches for reusable remote inputs.
|
||||
type CacheConfig struct {
|
||||
Root string `yaml:"root"`
|
||||
S3Audio *bool `yaml:"s3_audio"`
|
||||
Root string `yaml:"root" json:"root"`
|
||||
S3Audio *bool `yaml:"s3_audio" json:"s3_audio"`
|
||||
}
|
||||
|
||||
// PublishConfig configures publish behavior and source-based output uploads.
|
||||
type PublishConfig struct {
|
||||
Enabled *bool `yaml:"enabled"`
|
||||
UploadRun *bool `yaml:"upload_run"`
|
||||
Outputs []PublishOutputRule `yaml:"outputs"`
|
||||
Enabled *bool `yaml:"enabled" json:"enabled"`
|
||||
UploadRun *bool `yaml:"upload_run" json:"upload_run"`
|
||||
Outputs []PublishOutputRule `yaml:"outputs" json:"outputs"`
|
||||
}
|
||||
|
||||
// PublishOutputRule configures one source-to-destination publish mapping.
|
||||
type PublishOutputRule struct {
|
||||
Source string `yaml:"source"`
|
||||
Dest string `yaml:"dest"`
|
||||
Required *bool `yaml:"required"`
|
||||
Source string `yaml:"source" json:"source"`
|
||||
Dest string `yaml:"dest" json:"dest"`
|
||||
Required *bool `yaml:"required" json:"required"`
|
||||
}
|
||||
|
||||
// WhisperXConfig configures WhisperX adapter settings.
|
||||
type WhisperXConfig struct {
|
||||
TranscribeURL string `yaml:"transcribe_url"`
|
||||
Language string `yaml:"language"`
|
||||
Timeout string `yaml:"timeout"`
|
||||
Retries *int `yaml:"retries"`
|
||||
RetryDelay string `yaml:"retry_delay"`
|
||||
Concurrency *int `yaml:"concurrency"`
|
||||
TranscribeURL string `yaml:"transcribe_url" json:"transcribe_url"`
|
||||
Language string `yaml:"language" json:"language"`
|
||||
Timeout string `yaml:"timeout" json:"timeout"`
|
||||
Retries *int `yaml:"retries" json:"retries"`
|
||||
RetryDelay string `yaml:"retry_delay" json:"retry_delay"`
|
||||
Concurrency *int `yaml:"concurrency" json:"concurrency"`
|
||||
}
|
||||
|
||||
// SeriatimConfig configures seriatim adapter settings.
|
||||
type SeriatimConfig struct {
|
||||
Binary string `yaml:"binary"`
|
||||
Timeout string `yaml:"timeout"`
|
||||
OutputSchema string `yaml:"output_schema"`
|
||||
CoalesceGap *float64 `yaml:"coalesce_gap"`
|
||||
Report *bool `yaml:"report"`
|
||||
Env SeriatimEnvConfig `yaml:"env"`
|
||||
Binary string `yaml:"binary" json:"binary"`
|
||||
Timeout string `yaml:"timeout" json:"timeout"`
|
||||
OutputSchema string `yaml:"output_schema" json:"output_schema"`
|
||||
CoalesceGap *float64 `yaml:"coalesce_gap" json:"coalesce_gap"`
|
||||
Report *bool `yaml:"report" json:"report"`
|
||||
Env SeriatimEnvConfig `yaml:"env" json:"env"`
|
||||
}
|
||||
|
||||
// SeriatimEnvConfig configures optional advanced seriatim environment tuning.
|
||||
type SeriatimEnvConfig struct {
|
||||
OverlapWordRunGap *float64 `yaml:"overlap_word_run_gap"`
|
||||
OverlapWordRunReorderWindow *float64 `yaml:"overlap_word_run_reorder_window"`
|
||||
BackchannelMaxDuration *float64 `yaml:"backchannel_max_duration"`
|
||||
FillerMaxDuration *float64 `yaml:"filler_max_duration"`
|
||||
OverlapWordRunGap *float64 `yaml:"overlap_word_run_gap" json:"overlap_word_run_gap"`
|
||||
OverlapWordRunReorderWindow *float64 `yaml:"overlap_word_run_reorder_window" json:"overlap_word_run_reorder_window"`
|
||||
BackchannelMaxDuration *float64 `yaml:"backchannel_max_duration" json:"backchannel_max_duration"`
|
||||
FillerMaxDuration *float64 `yaml:"filler_max_duration" json:"filler_max_duration"`
|
||||
}
|
||||
|
||||
// AuditaConfig configures audita adapter settings.
|
||||
type AuditaConfig struct {
|
||||
Binary string `yaml:"binary"`
|
||||
Timeout string `yaml:"timeout"`
|
||||
LLMAPIKeyEnv string `yaml:"llm_api_key_env"`
|
||||
Modules []string `yaml:"modules"`
|
||||
BaseURL string `yaml:"base_url"`
|
||||
Model string `yaml:"model"`
|
||||
TotalLLMConcurrency *int `yaml:"total_llm_concurrency"`
|
||||
ProposalLLMConcurrency *int `yaml:"proposal_llm_concurrency"`
|
||||
ValidationModel string `yaml:"validation_model"`
|
||||
ValidationLLMConcurrency *int `yaml:"validation_llm_concurrency"`
|
||||
TranscriptDescription string `yaml:"transcript_description"`
|
||||
ConfigPath string `yaml:"config_path"`
|
||||
OutputSchema string `yaml:"output_schema"`
|
||||
WorkDirRetention string `yaml:"work_dir_retention"`
|
||||
Report *bool `yaml:"report"`
|
||||
Binary string `yaml:"binary" json:"binary"`
|
||||
Timeout string `yaml:"timeout" json:"timeout"`
|
||||
LLMAPIKey *configkit.Secret `yaml:"llm_api_key,omitempty" json:"llm_api_key,omitempty"`
|
||||
Modules []string `yaml:"modules" json:"modules"`
|
||||
BaseURL string `yaml:"base_url" json:"base_url"`
|
||||
Model string `yaml:"model" json:"model"`
|
||||
TotalLLMConcurrency *int `yaml:"total_llm_concurrency" json:"total_llm_concurrency"`
|
||||
ProposalLLMConcurrency *int `yaml:"proposal_llm_concurrency" json:"proposal_llm_concurrency"`
|
||||
ValidationModel string `yaml:"validation_model" json:"validation_model"`
|
||||
ValidationLLMConcurrency *int `yaml:"validation_llm_concurrency" json:"validation_llm_concurrency"`
|
||||
TranscriptDescription string `yaml:"transcript_description" json:"transcript_description"`
|
||||
ConfigPath string `yaml:"config_path" json:"config_path"`
|
||||
OutputSchema string `yaml:"output_schema" json:"output_schema"`
|
||||
WorkDirRetention string `yaml:"work_dir_retention" json:"work_dir_retention"`
|
||||
Report *bool `yaml:"report" json:"report"`
|
||||
}
|
||||
|
||||
// NormalizeConfig configures normalize-stage transcript schema/output behavior.
|
||||
type NormalizeConfig struct {
|
||||
OutputPath string `yaml:"output_path"`
|
||||
OutputSchema string `yaml:"output_schema"`
|
||||
Report *bool `yaml:"report"`
|
||||
|
||||
outputPathSet bool `yaml:"-"`
|
||||
OutputPath string `yaml:"output_path" json:"output_path"`
|
||||
OutputSchema string `yaml:"output_schema" json:"output_schema"`
|
||||
Report *bool `yaml:"report" json:"report"`
|
||||
}
|
||||
|
||||
// TrimConfig configures trim-stage transcript boundary behavior.
|
||||
type TrimConfig struct {
|
||||
Enabled *bool `yaml:"enabled"`
|
||||
OutputPath string `yaml:"output_path"`
|
||||
Bounds TrimBoundsConfig `yaml:"bounds"`
|
||||
Seriatim TrimSeriatimConfig `yaml:"seriatim"`
|
||||
Enabled *bool `yaml:"enabled" json:"enabled"`
|
||||
OutputPath string `yaml:"output_path" json:"output_path"`
|
||||
Bounds TrimBoundsConfig `yaml:"bounds" json:"bounds"`
|
||||
Seriatim TrimSeriatimConfig `yaml:"seriatim" json:"seriatim"`
|
||||
}
|
||||
|
||||
// TrimBoundsConfig configures Scriptorium bounds prompt execution for trim.
|
||||
type TrimBoundsConfig struct {
|
||||
PromptID string `yaml:"prompt_id"`
|
||||
ProfileID string `yaml:"profile_id"`
|
||||
TranscriptInputName string `yaml:"transcript_input_name"`
|
||||
OutputPath string `yaml:"output_path"`
|
||||
Timeout string `yaml:"timeout"`
|
||||
RenderDebug bool `yaml:"render_debug"`
|
||||
RenderOutputPath string `yaml:"render_output_path"`
|
||||
PromptID string `yaml:"prompt_id" json:"prompt_id"`
|
||||
ProfileID string `yaml:"profile_id" json:"profile_id"`
|
||||
TranscriptInputName string `yaml:"transcript_input_name" json:"transcript_input_name"`
|
||||
OutputPath string `yaml:"output_path" json:"output_path"`
|
||||
Timeout string `yaml:"timeout" json:"timeout"`
|
||||
RenderDebug bool `yaml:"render_debug" json:"render_debug"`
|
||||
RenderOutputPath string `yaml:"render_output_path" json:"render_output_path"`
|
||||
}
|
||||
|
||||
// TrimSeriatimConfig configures Seriatim options for trim output construction.
|
||||
type TrimSeriatimConfig struct {
|
||||
Report *bool `yaml:"report"`
|
||||
Report *bool `yaml:"report" json:"report"`
|
||||
}
|
||||
|
||||
// RenderConfig configures render-stage output formatting behavior.
|
||||
type RenderConfig struct {
|
||||
Enabled *bool `yaml:"enabled"`
|
||||
Format string `yaml:"format"`
|
||||
Title string `yaml:"title"`
|
||||
IncludeTimestamps *bool `yaml:"include_timestamps"`
|
||||
IncludeSegmentIDs *bool `yaml:"include_segment_ids"`
|
||||
IncludeMetadata bool `yaml:"include_metadata"`
|
||||
Enabled *bool `yaml:"enabled" json:"enabled"`
|
||||
Format string `yaml:"format" json:"format"`
|
||||
Title string `yaml:"title" json:"title"`
|
||||
IncludeTimestamps *bool `yaml:"include_timestamps" json:"include_timestamps"`
|
||||
IncludeSegmentIDs *bool `yaml:"include_segment_ids" json:"include_segment_ids"`
|
||||
IncludeMetadata bool `yaml:"include_metadata" json:"include_metadata"`
|
||||
}
|
||||
|
||||
// ScriptoriumConfig configures Scriptorium-backed artifact generation.
|
||||
type ScriptoriumConfig struct {
|
||||
Binary string `yaml:"binary"`
|
||||
ConfigPath string `yaml:"config_path"`
|
||||
Timeout string `yaml:"timeout"`
|
||||
RenderDebug bool `yaml:"render_debug"`
|
||||
Artifacts map[string]ScriptoriumArtifactConfig `yaml:"artifacts"`
|
||||
ArtifactFamilies map[string]ScriptoriumArtifactFamilyConfig `yaml:"artifact_families"`
|
||||
Binary string `yaml:"binary" json:"binary"`
|
||||
ConfigPath string `yaml:"config_path" json:"config_path"`
|
||||
Timeout string `yaml:"timeout" json:"timeout"`
|
||||
RenderDebug bool `yaml:"render_debug" json:"render_debug"`
|
||||
Artifacts map[string]ScriptoriumArtifactConfig `yaml:"artifacts" json:"artifacts"`
|
||||
ArtifactFamilies map[string]ScriptoriumArtifactFamilyConfig `yaml:"artifact_families" json:"artifact_families"`
|
||||
}
|
||||
|
||||
// ScriptoriumArtifactConfig configures one named output artifact workflow.
|
||||
type ScriptoriumArtifactConfig struct {
|
||||
Enabled bool `yaml:"enabled"`
|
||||
DependsOn []string `yaml:"depends_on"`
|
||||
RenderDebug *bool `yaml:"render_debug"`
|
||||
PromptID string `yaml:"prompt_id"`
|
||||
ProfileID string `yaml:"profile_id"`
|
||||
OutputPath string `yaml:"output_path"`
|
||||
Timeout string `yaml:"timeout"`
|
||||
Inputs map[string]ScriptoriumInputConfig `yaml:"inputs"`
|
||||
Vars map[string]any `yaml:"vars"`
|
||||
Enabled bool `yaml:"enabled" json:"enabled"`
|
||||
DependsOn []string `yaml:"depends_on" json:"depends_on"`
|
||||
RenderDebug *bool `yaml:"render_debug" json:"render_debug"`
|
||||
PromptID string `yaml:"prompt_id" json:"prompt_id"`
|
||||
ProfileID string `yaml:"profile_id" json:"profile_id"`
|
||||
OutputPath string `yaml:"output_path" json:"output_path"`
|
||||
Timeout string `yaml:"timeout" json:"timeout"`
|
||||
Inputs map[string]ScriptoriumInputConfig `yaml:"inputs" json:"inputs"`
|
||||
Vars map[string]any `yaml:"vars" json:"vars"`
|
||||
|
||||
dependencyOrigins map[string][]artifactDependencyOrigin
|
||||
}
|
||||
|
||||
// ScriptoriumArtifactFamilyConfig declares the shared configuration expanded
|
||||
// into one ordinary artifact for each canonical party character.
|
||||
type ScriptoriumArtifactFamilyConfig struct {
|
||||
Enabled bool `yaml:"enabled"`
|
||||
ForEach string `yaml:"for_each"`
|
||||
DependsOn []string `yaml:"depends_on"`
|
||||
RenderDebug *bool `yaml:"render_debug"`
|
||||
PromptID string `yaml:"prompt_id"`
|
||||
ProfileID string `yaml:"profile_id"`
|
||||
OutputPathPattern string `yaml:"output_path_pattern"`
|
||||
Timeout string `yaml:"timeout"`
|
||||
Inputs map[string]ScriptoriumInputConfig `yaml:"inputs"`
|
||||
Vars map[string]any `yaml:"vars"`
|
||||
MemberVars map[string]string `yaml:"member_vars"`
|
||||
MemberDependencies []string `yaml:"member_dependencies"`
|
||||
Publish *ScriptoriumArtifactFamilyPublishConfig `yaml:"publish"`
|
||||
Enabled bool `yaml:"enabled" json:"enabled"`
|
||||
ForEach string `yaml:"for_each" json:"for_each"`
|
||||
DependsOn []string `yaml:"depends_on" json:"depends_on"`
|
||||
RenderDebug *bool `yaml:"render_debug" json:"render_debug"`
|
||||
PromptID string `yaml:"prompt_id" json:"prompt_id"`
|
||||
ProfileID string `yaml:"profile_id" json:"profile_id"`
|
||||
OutputPathPattern string `yaml:"output_path_pattern" json:"output_path_pattern"`
|
||||
Timeout string `yaml:"timeout" json:"timeout"`
|
||||
Inputs map[string]ScriptoriumInputConfig `yaml:"inputs" json:"inputs"`
|
||||
Vars map[string]any `yaml:"vars" json:"vars"`
|
||||
MemberVars map[string]string `yaml:"member_vars" json:"member_vars"`
|
||||
MemberDependencies []string `yaml:"member_dependencies" json:"member_dependencies"`
|
||||
Publish *ScriptoriumArtifactFamilyPublishConfig `yaml:"publish" json:"publish"`
|
||||
}
|
||||
|
||||
// ScriptoriumArtifactFamilyPublishConfig retains the typed declaration for
|
||||
// family publishing. It is resolved by the publish owner.
|
||||
type ScriptoriumArtifactFamilyPublishConfig struct {
|
||||
Enabled bool `yaml:"enabled"`
|
||||
Required bool `yaml:"required"`
|
||||
DestPattern string `yaml:"dest_pattern"`
|
||||
Enabled bool `yaml:"enabled" json:"enabled"`
|
||||
Required bool `yaml:"required" json:"required"`
|
||||
DestPattern string `yaml:"dest_pattern" json:"dest_pattern"`
|
||||
}
|
||||
|
||||
// ArtifactFamilyCatalog records the runtime-only origin of expanded artifacts.
|
||||
@@ -305,50 +301,50 @@ type ArtifactFamilyMemberOrigin struct {
|
||||
|
||||
// ScriptoriumInputConfig configures one named prompt input source.
|
||||
type ScriptoriumInputConfig struct {
|
||||
Source string `yaml:"source"`
|
||||
Required bool `yaml:"required"`
|
||||
Source string `yaml:"source" json:"source"`
|
||||
Required bool `yaml:"required" json:"required"`
|
||||
}
|
||||
|
||||
// NotariusConfig configures structured artifact extraction by Notarius.
|
||||
type NotariusConfig struct {
|
||||
Enabled bool `yaml:"enabled"`
|
||||
Binary string `yaml:"binary"`
|
||||
ConfigPath string `yaml:"config_path"`
|
||||
PipelineID string `yaml:"pipeline_id"`
|
||||
Timeout string `yaml:"timeout"`
|
||||
WorkingDirectory string `yaml:"working_directory"`
|
||||
References map[string]string `yaml:"references"`
|
||||
Outputs map[string]NotariusOutputConfig `yaml:"outputs"`
|
||||
Enabled bool `yaml:"enabled" json:"enabled"`
|
||||
Binary string `yaml:"binary" json:"binary"`
|
||||
ConfigPath string `yaml:"config_path" json:"config_path"`
|
||||
PipelineID string `yaml:"pipeline_id" json:"pipeline_id"`
|
||||
Timeout string `yaml:"timeout" json:"timeout"`
|
||||
WorkingDirectory string `yaml:"working_directory" json:"working_directory"`
|
||||
References map[string]string `yaml:"references" json:"references"`
|
||||
Outputs map[string]NotariusOutputConfig `yaml:"outputs" json:"outputs"`
|
||||
}
|
||||
|
||||
// NotariusOutputConfig declares one required extraction lane contract.
|
||||
type NotariusOutputConfig struct {
|
||||
LaneID string `yaml:"lane_id"`
|
||||
MediaType string `yaml:"media_type"`
|
||||
SchemaID string `yaml:"schema_id"`
|
||||
SchemaVersion string `yaml:"schema_version"`
|
||||
ModuleKey string `yaml:"module_key"`
|
||||
LaneID string `yaml:"lane_id" json:"lane_id"`
|
||||
MediaType string `yaml:"media_type" json:"media_type"`
|
||||
SchemaID string `yaml:"schema_id" json:"schema_id"`
|
||||
SchemaVersion string `yaml:"schema_version" json:"schema_version"`
|
||||
ModuleKey string `yaml:"module_key" json:"module_key"`
|
||||
}
|
||||
|
||||
// NotificationConfig configures the supported notification behavior.
|
||||
type NotificationConfig struct {
|
||||
Mode string `yaml:"mode"`
|
||||
Mode string `yaml:"mode" json:"mode"`
|
||||
}
|
||||
|
||||
// SessionInputsConfig contains per-session input references.
|
||||
type SessionInputsConfig struct {
|
||||
AudioDir string `yaml:"audio_dir"`
|
||||
AudioFiles []string `yaml:"audio_files"`
|
||||
AudioS3 *SessionAudioS3Input `yaml:"audio_s3"`
|
||||
SpeakersFile string `yaml:"speakers_file"`
|
||||
AutocorrectFile string `yaml:"autocorrect_file"`
|
||||
GlossaryFile string `yaml:"glossary_file"`
|
||||
SpellCatalogFile string `yaml:"spell_catalog_file"`
|
||||
AudioDir string `yaml:"audio_dir" json:"audio_dir"`
|
||||
AudioFiles []string `yaml:"audio_files" json:"audio_files"`
|
||||
AudioS3 *SessionAudioS3Input `yaml:"audio_s3" json:"audio_s3"`
|
||||
SpeakersFile string `yaml:"speakers_file" json:"speakers_file"`
|
||||
AutocorrectFile string `yaml:"autocorrect_file" json:"autocorrect_file"`
|
||||
GlossaryFile string `yaml:"glossary_file" json:"glossary_file"`
|
||||
SpellCatalogFile string `yaml:"spell_catalog_file" json:"spell_catalog_file"`
|
||||
}
|
||||
|
||||
// SessionAudioS3Input configures S3 session-audio input discovery.
|
||||
type SessionAudioS3Input struct {
|
||||
Prefix string `yaml:"prefix"`
|
||||
Prefix string `yaml:"prefix" json:"prefix"`
|
||||
}
|
||||
|
||||
// ResolvedStableInputs records where stable input file paths came from after
|
||||
|
||||
59
internal/config/credential_projection.go
Normal file
59
internal/config/credential_projection.go
Normal file
@@ -0,0 +1,59 @@
|
||||
package config
|
||||
|
||||
import configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
|
||||
// MarshalYAML exposes only public credential reference names. Resolved values
|
||||
// remain available solely through ConfigKit's explicit trusted accessor.
|
||||
func (cfg StorageS3Config) MarshalYAML() (any, error) {
|
||||
type safeStorageS3 struct {
|
||||
Bucket string `yaml:"bucket"`
|
||||
RootPrefix string `yaml:"root_prefix"`
|
||||
Region string `yaml:"region"`
|
||||
Endpoint string `yaml:"endpoint"`
|
||||
ForcePathStyle bool `yaml:"force_path_style"`
|
||||
AccessKeyID *string `yaml:"access_key_id,omitempty"`
|
||||
SecretAccessKey *string `yaml:"secret_access_key,omitempty"`
|
||||
}
|
||||
return safeStorageS3{
|
||||
Bucket: cfg.Bucket, RootPrefix: cfg.RootPrefix, Region: cfg.Region,
|
||||
Endpoint: cfg.Endpoint, ForcePathStyle: cfg.ForcePathStyle,
|
||||
AccessKeyID: secretReferenceName(cfg.AccessKeyID), SecretAccessKey: secretReferenceName(cfg.SecretAccessKey),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// MarshalYAML exposes only the public Audita credential reference name.
|
||||
func (cfg AuditaConfig) MarshalYAML() (any, error) {
|
||||
type safeAudita struct {
|
||||
Binary string `yaml:"binary"`
|
||||
Timeout string `yaml:"timeout"`
|
||||
LLMAPIKey *string `yaml:"llm_api_key,omitempty"`
|
||||
Modules []string `yaml:"modules"`
|
||||
BaseURL string `yaml:"base_url"`
|
||||
Model string `yaml:"model"`
|
||||
TotalLLMConcurrency *int `yaml:"total_llm_concurrency"`
|
||||
ProposalLLMConcurrency *int `yaml:"proposal_llm_concurrency"`
|
||||
ValidationModel string `yaml:"validation_model"`
|
||||
ValidationLLMConcurrency *int `yaml:"validation_llm_concurrency"`
|
||||
TranscriptDescription string `yaml:"transcript_description"`
|
||||
ConfigPath string `yaml:"config_path"`
|
||||
OutputSchema string `yaml:"output_schema"`
|
||||
WorkDirRetention string `yaml:"work_dir_retention"`
|
||||
Report *bool `yaml:"report"`
|
||||
}
|
||||
return safeAudita{
|
||||
Binary: cfg.Binary, Timeout: cfg.Timeout, LLMAPIKey: secretReferenceName(cfg.LLMAPIKey),
|
||||
Modules: cfg.Modules, BaseURL: cfg.BaseURL, Model: cfg.Model,
|
||||
TotalLLMConcurrency: cfg.TotalLLMConcurrency, ProposalLLMConcurrency: cfg.ProposalLLMConcurrency,
|
||||
ValidationModel: cfg.ValidationModel, ValidationLLMConcurrency: cfg.ValidationLLMConcurrency,
|
||||
TranscriptDescription: cfg.TranscriptDescription, ConfigPath: cfg.ConfigPath,
|
||||
OutputSchema: cfg.OutputSchema, WorkDirRetention: cfg.WorkDirRetention, Report: cfg.Report,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func secretReferenceName(secret *configkit.Secret) *string {
|
||||
if secret == nil {
|
||||
return nil
|
||||
}
|
||||
name := secret.EnvName()
|
||||
return &name
|
||||
}
|
||||
@@ -9,8 +9,6 @@ const (
|
||||
DefaultPipelineConfigPathEtc = "/etc/narratio/pipeline.yml"
|
||||
DefaultSessionConfigPathUsrLocal = "/usr/local/etc/narratio/session.yml"
|
||||
DefaultSessionConfigPathEtc = "/etc/narratio/session.yml"
|
||||
DefaultS3AccessKeyIDEnv = "OBJECT_STORAGE_KEY_ID"
|
||||
DefaultS3SecretAccessKeyEnv = "OBJECT_STORAGE_KEY"
|
||||
DefaultStorageS3RootPrefix = "dnd"
|
||||
DefaultWorkspaceRoot = "/var/lib/narratio"
|
||||
DefaultCampaignsRoot = "/usr/local/share/narratio/campaigns"
|
||||
|
||||
32
internal/config/document_configkit.go
Normal file
32
internal/config/document_configkit.go
Normal file
@@ -0,0 +1,32 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
)
|
||||
|
||||
const maxConfigDocumentBytes int64 = 8 << 20
|
||||
|
||||
func loadCampaignWithConfigKit(path string) (*CampaignConfig, error) {
|
||||
result, err := configkit.Load(configkit.File(path), configkit.Options[CampaignConfig]{
|
||||
Definition: configkit.Definition[CampaignConfig]{},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, describeConfigKitLoadError("campaign", err)
|
||||
}
|
||||
return &result.Config, nil
|
||||
}
|
||||
|
||||
func loadSessionWithConfigKit(label string, data []byte) (*SessionConfig, error) {
|
||||
if int64(len(data)) > maxConfigDocumentBytes {
|
||||
return nil, fmt.Errorf("load session config: session file %q exceeds %d-byte limit", label, maxConfigDocumentBytes)
|
||||
}
|
||||
result, err := configkit.Load(configkit.Bytes(label, data), configkit.Options[SessionConfig]{
|
||||
Definition: configkit.Definition[SessionConfig]{},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, describeConfigKitLoadError("session", err)
|
||||
}
|
||||
return &result.Config, nil
|
||||
}
|
||||
56
internal/config/document_configkit_test.go
Normal file
56
internal/config/document_configkit_test.go
Normal file
@@ -0,0 +1,56 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSingleSourceDocumentsRejectConfigKitDirectives(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
campaignPath := filepath.Join(dir, "campaign.yml")
|
||||
if err := os.WriteFile(campaignPath, []byte("configkit:\n imports: []\ncampaign_id: sample\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
load func() error
|
||||
}{
|
||||
{name: "campaign", load: func() error { _, err := LoadCampaign(campaignPath); return err }},
|
||||
{name: "session", load: func() error {
|
||||
_, err := LoadSessionBytesWithOptions("session.yml", []byte("configkit:\n imports: []\nsession_id: sample\n"), SessionLoadOptions{})
|
||||
return err
|
||||
}},
|
||||
{name: "party", load: func() error {
|
||||
_, err := ParseParty([]byte("configkit:\n imports: []\nschema_version: narratio.party.v1\ncharacters: {}\n"))
|
||||
return err
|
||||
}},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if err := test.load(); err == nil || !strings.Contains(err.Error(), "import.disabled") {
|
||||
t.Fatalf("load error = %v, want disabled-import diagnostic", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectedDocumentsEnforceConfigKitSizeLimit(t *testing.T) {
|
||||
tooLarge := bytes.Repeat([]byte{' '}, int(maxConfigDocumentBytes)+1)
|
||||
if _, err := LoadSessionBytesWithOptions("session.yml", tooLarge, SessionLoadOptions{}); err == nil || !strings.Contains(err.Error(), "exceeds") {
|
||||
t.Fatalf("oversized session error = %v", err)
|
||||
}
|
||||
if _, err := ParseParty(tooLarge); err == nil || !strings.Contains(err.Error(), "exceeds") {
|
||||
t.Fatalf("oversized party error = %v", err)
|
||||
}
|
||||
|
||||
path := filepath.Join(t.TempDir(), "session.yml")
|
||||
if err := os.WriteFile(path, tooLarge, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := LoadSession(path); err == nil || !strings.Contains(err.Error(), "limit") {
|
||||
t.Fatalf("oversized local session error = %v", err)
|
||||
}
|
||||
}
|
||||
@@ -5,14 +5,13 @@ import (
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
const (
|
||||
pipelineDefaultOwnershipSource = "default"
|
||||
effectivePipelineDigestVersion = "narratio-effective-pipeline-v2\n"
|
||||
effectivePipelineDigestVersion = "narratio-effective-pipeline-v3\n"
|
||||
)
|
||||
|
||||
// PipelineProfileProvenance identifies the profile selected while resolving a
|
||||
@@ -49,15 +48,7 @@ func finalizePipelineResolution(cfg *PipelineConfig) error {
|
||||
for _, ownership := range cfg.resolution.ownership {
|
||||
declared[ownership.path] = append([]string(nil), ownership.sources...)
|
||||
}
|
||||
data, err := yaml.Marshal(cfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("serialize normalized effective pipeline: %w", err)
|
||||
}
|
||||
document, err := parseCompositionBytes("normalized effective pipeline", data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
records, err := document.semanticRecords()
|
||||
records, err := projectionValueRecords(projectEffectivePipeline(cfg, true))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -71,33 +62,24 @@ func finalizePipelineResolution(cfg *PipelineConfig) error {
|
||||
path: record.Path, sources: append([]string(nil), sources...),
|
||||
})
|
||||
}
|
||||
for path, sources := range declared {
|
||||
if strings.HasPrefix(path, "scriptorium.artifact_families.") {
|
||||
ownership = append(ownership, pipelineFieldOwnership{path: path, sources: append([]string(nil), sources...)})
|
||||
}
|
||||
}
|
||||
sort.Slice(ownership, func(i, j int) bool { return ownership[i].path < ownership[j].path })
|
||||
cfg.resolution.ownership = ownership
|
||||
return recomputePipelineEffectiveDigest(cfg)
|
||||
}
|
||||
|
||||
// recomputePipelineEffectiveDigest is the single package-owned hook for
|
||||
// refreshing provenance after later resolution expands concrete pipeline
|
||||
// values. Composition declarations and runtime provenance are not serialized.
|
||||
// values. Loader directives and runtime provenance are not serialized.
|
||||
func recomputePipelineEffectiveDigest(cfg *PipelineConfig) error {
|
||||
if cfg == nil || cfg.resolution == nil {
|
||||
return fmt.Errorf("pipeline resolution metadata is required")
|
||||
}
|
||||
digestConfig := *cfg
|
||||
if cfg.Notarius != nil && cfg.resolution.logicalNotariusCaptured {
|
||||
notarius := *cfg.Notarius
|
||||
notarius.ConfigPath = cfg.resolution.logicalNotariusConfig
|
||||
notarius.WorkingDirectory = cfg.resolution.logicalNotariusWorking
|
||||
digestConfig.Notarius = ¬arius
|
||||
}
|
||||
data, err := yaml.Marshal(&digestConfig)
|
||||
if err != nil {
|
||||
return fmt.Errorf("serialize normalized effective pipeline: %w", err)
|
||||
}
|
||||
document, err := parseCompositionBytes("normalized effective pipeline", data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
canonical, err := document.canonicalDigestInput()
|
||||
canonical, err := projectionJSON(projectEffectivePipeline(cfg, true))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -116,9 +98,6 @@ func captureLogicalNotariusPaths(cfg *PipelineConfig) {
|
||||
}
|
||||
configPath := normalizeLogicalFilesystemPath(cfg.Notarius.ConfigPath)
|
||||
workingDirectory := normalizeLogicalFilesystemPath(cfg.Notarius.WorkingDirectory)
|
||||
if cfg.Notarius.Enabled && workingDirectory == "" && configPath != "" {
|
||||
workingDirectory = normalizeLogicalFilesystemPath(filepath.Dir(configPath))
|
||||
}
|
||||
cfg.resolution.logicalNotariusConfig = configPath
|
||||
cfg.resolution.logicalNotariusWorking = workingDirectory
|
||||
cfg.resolution.logicalNotariusCaptured = true
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
@@ -55,28 +57,12 @@ type EffectivePipelineValueRecord struct {
|
||||
// EffectivePipelineValues projects a fully resolved pipeline into sorted
|
||||
// logical configuration values. Mappings are flattened, while sequences remain
|
||||
// atomic values. The projection uses the same normalized effective mapping as
|
||||
// config show and therefore excludes composition and family declarations.
|
||||
// config show and therefore excludes loader directives and family declarations.
|
||||
func EffectivePipelineValues(cfg *PipelineConfig) ([]EffectivePipelineValueRecord, error) {
|
||||
if cfg == nil {
|
||||
return nil, fmt.Errorf("pipeline config is required")
|
||||
}
|
||||
data, err := MarshalEffectivePipeline(cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
document, err := parseCompositionBytes("effective pipeline", data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
records, err := document.compactSemanticRecords()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
values := make([]EffectivePipelineValueRecord, 0, len(records))
|
||||
for _, record := range records {
|
||||
values = append(values, EffectivePipelineValueRecord{Path: record.Path, Value: record.Value})
|
||||
}
|
||||
return values, nil
|
||||
return projectionValueRecords(projectEffectivePipeline(cfg, true))
|
||||
}
|
||||
|
||||
// EffectivePipelineSources projects pipeline ownership after defaults and
|
||||
@@ -86,15 +72,14 @@ func EffectivePipelineSources(cfg *PipelineConfig, party ResolvedParty) ([]Effec
|
||||
if cfg == nil {
|
||||
return nil, fmt.Errorf("pipeline config is required")
|
||||
}
|
||||
data, err := MarshalEffectivePipeline(cfg)
|
||||
values, err := EffectivePipelineValues(cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
document, err := parseCompositionBytes("effective pipeline", data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
paths := make([]string, 0, len(values))
|
||||
for _, value := range values {
|
||||
paths = append(paths, value.Path)
|
||||
}
|
||||
paths := effectivePipelineFieldPaths(document.root)
|
||||
owners := effectivePipelineOwners(cfg)
|
||||
families := ArtifactFamilies(cfg)
|
||||
publishCount := 0
|
||||
@@ -168,54 +153,70 @@ func MarshalEffectivePipeline(cfg *PipelineConfig) ([]byte, error) {
|
||||
return nil, fmt.Errorf("pipeline config is required")
|
||||
}
|
||||
|
||||
data, err := yaml.Marshal(cfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("serialize effective pipeline: %w", err)
|
||||
}
|
||||
document, err := parseCompositionBytes("effective pipeline", data)
|
||||
value, err := projectionValueTree(projectEffectivePipeline(cfg, true))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
removeResolutionOnlyPipelineFields(document)
|
||||
return document.canonicalYAML()
|
||||
node, err := effectiveYAMLNode(value)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("serialize effective pipeline: %w", err)
|
||||
}
|
||||
data, err := yaml.Marshal(node)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("serialize effective pipeline: %w", err)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func removeResolutionOnlyPipelineFields(document *compositionDocument) {
|
||||
if document == nil || document.root == nil {
|
||||
return
|
||||
}
|
||||
scriptoriumIndex := compositionFieldIndex(document.root.fields, "scriptorium")
|
||||
if scriptoriumIndex < 0 {
|
||||
return
|
||||
}
|
||||
scriptorium := document.root.fields[scriptoriumIndex].value
|
||||
if scriptorium == nil || scriptorium.kind != yaml.MappingNode {
|
||||
return
|
||||
}
|
||||
familyIndex := compositionFieldIndex(scriptorium.fields, "artifact_families")
|
||||
if familyIndex < 0 {
|
||||
return
|
||||
}
|
||||
scriptorium.fields = append(scriptorium.fields[:familyIndex], scriptorium.fields[familyIndex+1:]...)
|
||||
}
|
||||
|
||||
func effectivePipelineFieldPaths(node *compositionNode) []string {
|
||||
var paths []string
|
||||
var visit func(*compositionNode)
|
||||
visit = func(current *compositionNode) {
|
||||
if current == nil {
|
||||
return
|
||||
func effectiveYAMLNode(value any) (*yaml.Node, error) {
|
||||
switch value := value.(type) {
|
||||
case nil:
|
||||
return &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!null", Value: "null"}, nil
|
||||
case bool:
|
||||
return &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!bool", Value: strconv.FormatBool(value)}, nil
|
||||
case string:
|
||||
return &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: value}, nil
|
||||
case json.Number:
|
||||
tag := "!!int"
|
||||
if strings.ContainsAny(value.String(), ".eE") {
|
||||
tag = "!!float"
|
||||
}
|
||||
if current.kind == yaml.MappingNode && len(current.fields) > 0 {
|
||||
for _, field := range current.fields {
|
||||
visit(field.value)
|
||||
return &yaml.Node{Kind: yaml.ScalarNode, Tag: tag, Value: value.String()}, nil
|
||||
case []any:
|
||||
node := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
|
||||
if len(value) == 0 {
|
||||
node.Style = yaml.FlowStyle
|
||||
}
|
||||
for _, item := range value {
|
||||
child, err := effectiveYAMLNode(item)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return
|
||||
node.Content = append(node.Content, child)
|
||||
}
|
||||
paths = append(paths, current.path)
|
||||
return node, nil
|
||||
case map[string]any:
|
||||
node := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
|
||||
if len(value) == 0 {
|
||||
node.Style = yaml.FlowStyle
|
||||
}
|
||||
keys := make([]string, 0, len(value))
|
||||
for key := range value {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
child, err := effectiveYAMLNode(value[key])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
node.Content = append(node.Content,
|
||||
&yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: key}, child)
|
||||
}
|
||||
return node, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported projected value type %T", value)
|
||||
}
|
||||
visit(node)
|
||||
return paths
|
||||
}
|
||||
|
||||
func effectivePipelineOwners(cfg *PipelineConfig) map[string][]string {
|
||||
@@ -226,9 +227,39 @@ func effectivePipelineOwners(cfg *PipelineConfig) map[string][]string {
|
||||
for _, ownership := range cfg.resolution.ownership {
|
||||
owners[ownership.path] = append([]string(nil), ownership.sources...)
|
||||
}
|
||||
if cfg.Scriptorium != nil {
|
||||
for key, artifact := range cfg.Scriptorium.Artifacts {
|
||||
paths := artifactDependencyOwnershipPaths(key, artifact)
|
||||
if sources := sourcesForPipelinePaths(owners, paths); len(sources) != 0 {
|
||||
sort.Strings(sources)
|
||||
owners["scriptorium.artifacts."+key+".depends_on"] = sources
|
||||
}
|
||||
}
|
||||
}
|
||||
return owners
|
||||
}
|
||||
|
||||
func artifactDependencyOwnershipPaths(artifactKey string, artifact ScriptoriumArtifactConfig) []string {
|
||||
var paths []string
|
||||
for _, dependency := range artifact.DependsOn {
|
||||
for _, origin := range artifact.dependencyOrigins[dependency] {
|
||||
prefix := "scriptorium.artifacts." + artifactKey
|
||||
if origin.family != "" {
|
||||
prefix = "scriptorium.artifact_families." + origin.family
|
||||
}
|
||||
switch {
|
||||
case origin.inputName != "":
|
||||
paths = append(paths, prefix+".inputs."+origin.inputName+".source")
|
||||
case origin.family != "":
|
||||
paths = append(paths, prefix+".member_dependencies")
|
||||
default:
|
||||
paths = append(paths, prefix+".depends_on")
|
||||
}
|
||||
}
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
func generatedArtifactFamilyForPath(path string, families ArtifactFamilyCatalog) (string, bool) {
|
||||
const prefix = "scriptorium.artifacts."
|
||||
if !strings.HasPrefix(path, prefix) {
|
||||
@@ -268,7 +299,7 @@ func generatedArtifactSources(path, familyKey string, families ArtifactFamilyCat
|
||||
case suffix == "output_path":
|
||||
candidates = []string{familyPrefix + "output_path_pattern"}
|
||||
case suffix == "depends_on":
|
||||
candidates = []string{familyPrefix + "depends_on", familyPrefix + "member_dependencies"}
|
||||
candidates = []string{path, familyPrefix + "depends_on", familyPrefix + "member_dependencies"}
|
||||
case strings.HasPrefix(suffix, "vars."):
|
||||
variable := strings.TrimPrefix(suffix, "vars.")
|
||||
candidates = []string{familyPrefix + "vars." + variable, familyPrefix + "member_vars." + variable}
|
||||
@@ -361,6 +392,9 @@ func pipelineSourceRole(cfg *PipelineConfig, source string) string {
|
||||
if source == pipelineDefaultOwnershipSource {
|
||||
return "default"
|
||||
}
|
||||
if source == "resolution" {
|
||||
return "resolution"
|
||||
}
|
||||
if cfg == nil || cfg.resolution == nil {
|
||||
return "pipeline"
|
||||
}
|
||||
@@ -379,7 +413,7 @@ func pipelineSourceRole(cfg *PipelineConfig, source string) string {
|
||||
}
|
||||
|
||||
func normalizedProvenanceSource(source string) string {
|
||||
if source == pipelineDefaultOwnershipSource {
|
||||
if source == pipelineDefaultOwnershipSource || source == "resolution" {
|
||||
return source
|
||||
}
|
||||
return normalizedProvenancePath(source)
|
||||
|
||||
137
internal/config/effective_projection.go
Normal file
137
internal/config/effective_projection.go
Normal file
@@ -0,0 +1,137 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// effectivePipelineProjection is the single safe, schema-specific view used
|
||||
// for inspection, comparison, and identity. Runtime metadata and unresolved
|
||||
// family declarations have no slot in this type.
|
||||
type effectivePipelineProjection struct {
|
||||
Workspace WorkspaceConfig `yaml:"workspace" json:"workspace"`
|
||||
Campaigns CampaignsConfig `yaml:"campaigns" json:"campaigns"`
|
||||
Storage StorageConfig `yaml:"storage" json:"storage"`
|
||||
Spool SpoolConfig `yaml:"spool" json:"spool"`
|
||||
Cache CacheConfig `yaml:"cache" json:"cache"`
|
||||
Publish *PublishConfig `yaml:"publish" json:"publish"`
|
||||
WhisperX WhisperXConfig `yaml:"whisperx" json:"whisperx"`
|
||||
Seriatim SeriatimConfig `yaml:"seriatim" json:"seriatim"`
|
||||
Audita AuditaConfig `yaml:"audita" json:"audita"`
|
||||
Normalize *NormalizeConfig `yaml:"normalize" json:"normalize"`
|
||||
Trim *TrimConfig `yaml:"trim" json:"trim"`
|
||||
Render *RenderConfig `yaml:"render" json:"render"`
|
||||
Scriptorium *effectiveScriptoriumConfig `yaml:"scriptorium" json:"scriptorium"`
|
||||
Notarius *NotariusConfig `yaml:"notarius" json:"notarius"`
|
||||
Notification NotificationConfig `yaml:"notification" json:"notification"`
|
||||
}
|
||||
|
||||
type effectiveScriptoriumConfig struct {
|
||||
Binary string `yaml:"binary" json:"binary"`
|
||||
ConfigPath string `yaml:"config_path" json:"config_path"`
|
||||
Timeout string `yaml:"timeout" json:"timeout"`
|
||||
RenderDebug bool `yaml:"render_debug" json:"render_debug"`
|
||||
Artifacts map[string]ScriptoriumArtifactConfig `yaml:"artifacts" json:"artifacts"`
|
||||
}
|
||||
|
||||
type projectedPipelineValue struct {
|
||||
record EffectivePipelineValueRecord
|
||||
segments []string
|
||||
}
|
||||
|
||||
func projectEffectivePipeline(cfg *PipelineConfig, logicalNotarius bool) effectivePipelineProjection {
|
||||
projection := effectivePipelineProjection{
|
||||
Workspace: cfg.Workspace, Campaigns: cfg.Campaigns, Storage: cfg.Storage,
|
||||
Spool: cfg.Spool, Cache: cfg.Cache, Publish: cfg.Publish,
|
||||
WhisperX: cfg.WhisperX, Seriatim: cfg.Seriatim, Audita: cfg.Audita,
|
||||
Normalize: cfg.Normalize, Trim: cfg.Trim, Render: cfg.Render,
|
||||
Notarius: cfg.Notarius, Notification: cfg.Notification,
|
||||
}
|
||||
if cfg.Scriptorium != nil {
|
||||
projection.Scriptorium = &effectiveScriptoriumConfig{
|
||||
Binary: cfg.Scriptorium.Binary, ConfigPath: cfg.Scriptorium.ConfigPath,
|
||||
Timeout: cfg.Scriptorium.Timeout, RenderDebug: cfg.Scriptorium.RenderDebug,
|
||||
Artifacts: cfg.Scriptorium.Artifacts,
|
||||
}
|
||||
}
|
||||
if logicalNotarius && cfg.Notarius != nil && cfg.resolution != nil && cfg.resolution.logicalNotariusCaptured {
|
||||
notarius := *cfg.Notarius
|
||||
notarius.ConfigPath = cfg.resolution.logicalNotariusConfig
|
||||
notarius.WorkingDirectory = cfg.resolution.logicalNotariusWorking
|
||||
projection.Notarius = ¬arius
|
||||
}
|
||||
return projection
|
||||
}
|
||||
|
||||
func projectionJSON(projection effectivePipelineProjection) ([]byte, error) {
|
||||
data, err := json.Marshal(projection)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("encode effective pipeline projection: %w", err)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func projectionValueTree(value any) (any, error) {
|
||||
data, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("encode typed pipeline projection: %w", err)
|
||||
}
|
||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||
decoder.UseNumber()
|
||||
var root any
|
||||
if err := decoder.Decode(&root); err != nil {
|
||||
return nil, fmt.Errorf("decode effective pipeline projection: %w", err)
|
||||
}
|
||||
return root, nil
|
||||
}
|
||||
|
||||
func projectionValueRecords(projection effectivePipelineProjection) ([]EffectivePipelineValueRecord, error) {
|
||||
detailed, err := projectionDetailedRecords(projection)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
records := make([]EffectivePipelineValueRecord, 0, len(detailed))
|
||||
for _, value := range detailed {
|
||||
records = append(records, value.record)
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func projectionDetailedRecords(value any) ([]projectedPipelineValue, error) {
|
||||
decoded, err := projectionValueTree(value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
root, ok := decoded.(map[string]any)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("effective pipeline projection must be a mapping")
|
||||
}
|
||||
var records []projectedPipelineValue
|
||||
appendProjectionRecords(&records, nil, root)
|
||||
sort.Slice(records, func(i, j int) bool { return records[i].record.Path < records[j].record.Path })
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func appendProjectionRecords(records *[]projectedPipelineValue, segments []string, value any) {
|
||||
if mapping, ok := value.(map[string]any); ok && len(mapping) > 0 {
|
||||
keys := make([]string, 0, len(mapping))
|
||||
for key := range mapping {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
child := append(append([]string(nil), segments...), key)
|
||||
appendProjectionRecords(records, child, mapping[key])
|
||||
}
|
||||
return
|
||||
}
|
||||
encoded, _ := json.Marshal(value)
|
||||
path := strings.Join(segments, ".")
|
||||
*records = append(*records, projectedPipelineValue{
|
||||
record: EffectivePipelineValueRecord{Path: path, Value: string(encoded)},
|
||||
segments: append([]string(nil), segments...),
|
||||
})
|
||||
}
|
||||
@@ -2,14 +2,12 @@ package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||
)
|
||||
|
||||
// LoadPipeline loads pipeline configuration from a YAML file with strict field checking.
|
||||
@@ -17,9 +15,8 @@ func LoadPipeline(path string) (*PipelineConfig, error) {
|
||||
return LoadPipelineWithOptions(path, PipelineLoadOptions{})
|
||||
}
|
||||
|
||||
// PipelineLoadOptions carries an optional explicit profile selection. A nil
|
||||
// Profile means the caller omitted selection; a non-nil empty value is an
|
||||
// explicit invalid selection.
|
||||
// PipelineLoadOptions carries an optional explicit profile selection. Nil,
|
||||
// empty, and the exact name "default" select the base configuration.
|
||||
type PipelineLoadOptions struct {
|
||||
Profile *string
|
||||
}
|
||||
@@ -27,78 +24,37 @@ type PipelineLoadOptions struct {
|
||||
// LoadPipelineWithOptions loads pipeline configuration with strict field
|
||||
// checking and optional named-profile selection.
|
||||
func LoadPipelineWithOptions(path string, opts PipelineLoadOptions) (*PipelineConfig, error) {
|
||||
sources, err := loadPipelineCompositionSources(path)
|
||||
cfg, err := loadPipelineWithConfigKit(path, opts)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load pipeline config: %w", err)
|
||||
}
|
||||
if _, err := selectPipelineProfile(sources.envelope, opts); err != nil {
|
||||
return nil, fmt.Errorf("load pipeline config: %w", err)
|
||||
}
|
||||
if err := sources.loadOverlays(); err != nil {
|
||||
return nil, fmt.Errorf("load pipeline config: %w", err)
|
||||
}
|
||||
cfg, err := sources.resolve(opts)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load pipeline config: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
return finalizeLoadedPipeline(path, cfg)
|
||||
}
|
||||
|
||||
// LoadPipelineProfilePair resolves two explicit named profiles from one parsed
|
||||
// pipeline root and its declared source set. Each result is independently
|
||||
// decoded, defaulted, and finalized so later resolution can safely mutate one
|
||||
// effective pipeline without affecting the other.
|
||||
// LoadPipelineProfilePair resolves two profiles as independent ConfigKit loads.
|
||||
func LoadPipelineProfilePair(path, leftProfile, rightProfile string) (*PipelineConfig, *PipelineConfig, error) {
|
||||
if _, err := normalizePipelineProfileName(leftProfile, "left profile selection"); err != nil {
|
||||
return nil, nil, fmt.Errorf("load left pipeline profile: %w", err)
|
||||
}
|
||||
if _, err := normalizePipelineProfileName(rightProfile, "right profile selection"); err != nil {
|
||||
return nil, nil, fmt.Errorf("load right pipeline profile: %w", err)
|
||||
}
|
||||
if leftProfile == rightProfile {
|
||||
return nil, nil, fmt.Errorf("load pipeline profiles: left and right profile selections must differ")
|
||||
}
|
||||
sources, err := loadPipelineCompositionSources(path)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("load pipeline config: %w", err)
|
||||
}
|
||||
leftOptions := PipelineLoadOptions{Profile: &leftProfile}
|
||||
rightOptions := PipelineLoadOptions{Profile: &rightProfile}
|
||||
if _, err := selectPipelineProfile(sources.envelope, leftOptions); err != nil {
|
||||
return nil, nil, fmt.Errorf("load left pipeline profile: %w", err)
|
||||
}
|
||||
if _, err := selectPipelineProfile(sources.envelope, rightOptions); err != nil {
|
||||
return nil, nil, fmt.Errorf("load right pipeline profile: %w", err)
|
||||
}
|
||||
if err := sources.loadOverlays(); err != nil {
|
||||
return nil, nil, fmt.Errorf("load pipeline config: %w", err)
|
||||
}
|
||||
left, err := sources.resolve(leftOptions)
|
||||
left, err := LoadPipelineWithOptions(path, leftOptions)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("load left pipeline profile: %w", err)
|
||||
}
|
||||
right, err := sources.resolve(rightOptions)
|
||||
right, err := LoadPipelineWithOptions(path, rightOptions)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("load right pipeline profile: %w", err)
|
||||
}
|
||||
left, err = finalizeLoadedPipeline(path, left)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
right, err = finalizeLoadedPipeline(path, right)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return left, right, nil
|
||||
}
|
||||
|
||||
func finalizeLoadedPipeline(path string, cfg *PipelineConfig) (*PipelineConfig, error) {
|
||||
cfg.resolution.publishDeclared = cfg.Publish != nil
|
||||
applyPipelineDefaults(cfg)
|
||||
captureLogicalNotariusPaths(cfg)
|
||||
if err := resolveNotariusPaths(cfg, path); err != nil {
|
||||
return nil, fmt.Errorf("load pipeline config: %w", err)
|
||||
}
|
||||
finalizeArtifactDependencies(cfg)
|
||||
if err := finalizePipelineResolution(cfg); err != nil {
|
||||
return nil, fmt.Errorf("load pipeline config: %w", err)
|
||||
}
|
||||
@@ -108,11 +64,7 @@ func finalizeLoadedPipeline(path string, cfg *PipelineConfig) (*PipelineConfig,
|
||||
|
||||
// LoadCampaign loads campaign configuration from a YAML file with strict field checking.
|
||||
func LoadCampaign(path string) (*CampaignConfig, error) {
|
||||
var cfg CampaignConfig
|
||||
if err := decodeStrictYAML("campaign", path, &cfg); err != nil {
|
||||
return nil, fmt.Errorf("load campaign config: %w", err)
|
||||
}
|
||||
return &cfg, nil
|
||||
return loadCampaignWithConfigKit(path)
|
||||
}
|
||||
|
||||
// LoadSession loads session configuration from a YAML file with strict field checking.
|
||||
@@ -130,7 +82,7 @@ type SessionLoadOptions struct {
|
||||
// LoadSessionWithOptions loads session configuration from a YAML file with
|
||||
// strict field checking.
|
||||
func LoadSessionWithOptions(path string, opts SessionLoadOptions) (*SessionConfig, error) {
|
||||
sessionBytes, err := os.ReadFile(path)
|
||||
sessionBytes, err := fileops.ReadFileBounded(path, maxConfigDocumentBytes)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load session config: session file %q: open: %w", path, err)
|
||||
}
|
||||
@@ -144,9 +96,9 @@ func LoadSessionBytesWithOptions(label string, data []byte, opts SessionLoadOpti
|
||||
return nil, fmt.Errorf("load session config: %w", err)
|
||||
}
|
||||
|
||||
var cfg SessionConfig
|
||||
if err := decodeStrictYAMLFromReader("session", label, strings.NewReader(string(data)), &cfg); err != nil {
|
||||
return nil, fmt.Errorf("load session config: %w", err)
|
||||
cfg, err := loadSessionWithConfigKit(label, data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if strings.TrimSpace(opts.SessionID) != "" && strings.TrimSpace(cfg.SessionID) != "" && strings.TrimSpace(cfg.SessionID) != strings.TrimSpace(opts.SessionID) {
|
||||
return nil, fmt.Errorf(
|
||||
@@ -174,7 +126,7 @@ func LoadSessionBytesWithOptions(label string, data []byte, opts SessionLoadOpti
|
||||
)
|
||||
}
|
||||
}
|
||||
return &cfg, nil
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// Load loads and resolves combined pipeline, campaign, and session configuration.
|
||||
@@ -437,33 +389,6 @@ func selectStableInput(campaignValue, sessionValue, campaignPath, sessionPath st
|
||||
}
|
||||
}
|
||||
|
||||
func decodeStrictYAML(kind, path string, out any) error {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s file %q: open: %w", kind, path, err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
return decodeStrictYAMLFromReader(kind, path, f, out)
|
||||
}
|
||||
|
||||
func decodeStrictYAMLFromReader(kind, path string, r io.Reader, out any) error {
|
||||
dec := yaml.NewDecoder(r)
|
||||
dec.KnownFields(true)
|
||||
if err := dec.Decode(out); err != nil {
|
||||
return fmt.Errorf("%s file %q: strict decode failed: %w", kind, path, err)
|
||||
}
|
||||
|
||||
var extra yaml.Node
|
||||
if err := dec.Decode(&extra); err == nil {
|
||||
return fmt.Errorf("%s file %q: must contain exactly one YAML document", kind, path)
|
||||
} else if err != io.EOF {
|
||||
return fmt.Errorf("%s file %q: trailing content decode failed: %w", kind, path, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
var sessionTemplatePlaceholderPattern = regexp.MustCompile(`\{\{[^}]*\}\}`)
|
||||
var sessionTemplateVariablePattern = regexp.MustCompile(`\{\{\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\}\}`)
|
||||
|
||||
@@ -501,217 +426,6 @@ func shortName(path, fallback string) string {
|
||||
return base
|
||||
}
|
||||
|
||||
func applyPipelineDefaults(cfg *PipelineConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
applyWorkspaceDefaults(&cfg.Workspace)
|
||||
applyCampaignsDefaults(&cfg.Campaigns)
|
||||
applyStorageDefaults(&cfg.Storage)
|
||||
applySpoolDefaults(&cfg.Spool)
|
||||
applyCacheDefaults(&cfg.Cache)
|
||||
applyPublishDefaults(&cfg.Publish)
|
||||
applyWhisperXDefaults(&cfg.WhisperX)
|
||||
applySeriatimDefaults(&cfg.Seriatim)
|
||||
applyAuditaDefaults(&cfg.Audita)
|
||||
if cfg.Normalize == nil {
|
||||
cfg.Normalize = &NormalizeConfig{}
|
||||
}
|
||||
applyNormalizeDefaults(cfg.Normalize)
|
||||
if cfg.Trim == nil {
|
||||
cfg.Trim = &TrimConfig{}
|
||||
}
|
||||
applyTrimDefaults(cfg.Trim)
|
||||
if trimEnabled(cfg.Trim) && cfg.Scriptorium == nil {
|
||||
cfg.Scriptorium = &ScriptoriumConfig{}
|
||||
}
|
||||
applyRenderDefaults(&cfg.Render)
|
||||
applyScriptoriumDefaults(cfg.Scriptorium)
|
||||
applyNotariusDefaults(cfg.Notarius)
|
||||
applyNotificationDefaults(&cfg.Notification)
|
||||
}
|
||||
|
||||
func applyNotificationDefaults(cfg *NotificationConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
if strings.TrimSpace(cfg.Mode) == "" {
|
||||
cfg.Mode = DefaultNotificationMode
|
||||
}
|
||||
}
|
||||
|
||||
func applyCampaignsDefaults(cfg *CampaignsConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
if cfg.Root == "" {
|
||||
cfg.Root = DefaultCampaignsRoot
|
||||
}
|
||||
}
|
||||
|
||||
func applyWorkspaceDefaults(cfg *WorkspaceConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
if cfg.Root == "" {
|
||||
cfg.Root = DefaultWorkspaceRoot
|
||||
}
|
||||
}
|
||||
|
||||
func applyStorageDefaults(cfg *StorageConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
backend := strings.ToLower(strings.TrimSpace(cfg.Backend))
|
||||
if backend == "" {
|
||||
backend = StorageBackendLocal
|
||||
}
|
||||
cfg.Backend = backend
|
||||
if cfg.S3 == nil {
|
||||
return
|
||||
}
|
||||
if cfg.S3.RootPrefix == "" {
|
||||
cfg.S3.RootPrefix = DefaultStorageS3RootPrefix
|
||||
}
|
||||
if cfg.S3.AccessKeyIDEnv == "" {
|
||||
cfg.S3.AccessKeyIDEnv = DefaultS3AccessKeyIDEnv
|
||||
}
|
||||
if cfg.S3.SecretKeyEnv == "" {
|
||||
cfg.S3.SecretKeyEnv = DefaultS3SecretAccessKeyEnv
|
||||
}
|
||||
}
|
||||
|
||||
func applySpoolDefaults(cfg *SpoolConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
if cfg.Root == "" {
|
||||
cfg.Root = DefaultSpoolRoot
|
||||
}
|
||||
}
|
||||
|
||||
func applyCacheDefaults(cfg *CacheConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
if cfg.Root == "" {
|
||||
cfg.Root = DefaultCacheRoot
|
||||
}
|
||||
if cfg.S3Audio == nil {
|
||||
cfg.S3Audio = boolPtr(DefaultCacheS3Audio)
|
||||
}
|
||||
}
|
||||
|
||||
func applyPublishDefaults(cfg **PublishConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
if *cfg == nil {
|
||||
*cfg = &PublishConfig{}
|
||||
}
|
||||
|
||||
if (*cfg).Enabled == nil {
|
||||
(*cfg).Enabled = boolPtr(DefaultArchiveEnabled)
|
||||
}
|
||||
if (*cfg).UploadRun == nil {
|
||||
(*cfg).UploadRun = boolPtr(DefaultArchiveUploadRun)
|
||||
}
|
||||
if len((*cfg).Outputs) == 0 {
|
||||
(*cfg).Outputs = append([]PublishOutputRule(nil), DefaultPublishOutputs...)
|
||||
}
|
||||
for i := range (*cfg).Outputs {
|
||||
if (*cfg).Outputs[i].Required == nil {
|
||||
(*cfg).Outputs[i].Required = boolPtr(true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func applyWhisperXDefaults(cfg *WhisperXConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
if cfg.Language == "" {
|
||||
cfg.Language = DefaultWhisperXLanguage
|
||||
}
|
||||
if cfg.Timeout == "" {
|
||||
cfg.Timeout = DefaultWhisperXTimeout
|
||||
}
|
||||
if cfg.RetryDelay == "" {
|
||||
cfg.RetryDelay = DefaultWhisperXRetryDelay
|
||||
}
|
||||
if cfg.Concurrency == nil {
|
||||
cfg.Concurrency = intPtr(DefaultWhisperXConcurrency)
|
||||
}
|
||||
if cfg.Retries == nil {
|
||||
cfg.Retries = intPtr(DefaultWhisperXRetries)
|
||||
}
|
||||
}
|
||||
|
||||
func intPtr(v int) *int {
|
||||
p := v
|
||||
return &p
|
||||
}
|
||||
|
||||
func applySeriatimDefaults(cfg *SeriatimConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
if cfg.Binary == "" {
|
||||
cfg.Binary = DefaultSeriatimBinary
|
||||
}
|
||||
if cfg.Timeout == "" {
|
||||
cfg.Timeout = DefaultSeriatimTimeout
|
||||
}
|
||||
if cfg.OutputSchema == "" {
|
||||
cfg.OutputSchema = DefaultSeriatimOutputSchema
|
||||
}
|
||||
if cfg.CoalesceGap == nil {
|
||||
cfg.CoalesceGap = float64Ptr(DefaultSeriatimCoalesceGap)
|
||||
}
|
||||
if cfg.Report == nil {
|
||||
cfg.Report = boolPtr(DefaultSeriatimReport)
|
||||
}
|
||||
}
|
||||
|
||||
func applyAuditaDefaults(cfg *AuditaConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
if cfg.Binary == "" {
|
||||
cfg.Binary = DefaultAuditaBinary
|
||||
}
|
||||
if cfg.Timeout == "" {
|
||||
cfg.Timeout = DefaultAuditaTimeout
|
||||
}
|
||||
if cfg.Report == nil {
|
||||
cfg.Report = boolPtr(DefaultAuditaReport)
|
||||
}
|
||||
}
|
||||
|
||||
func applyScriptoriumDefaults(cfg *ScriptoriumConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
if cfg.Binary == "" {
|
||||
cfg.Binary = DefaultScriptoriumBinary
|
||||
}
|
||||
if cfg.Timeout == "" {
|
||||
cfg.Timeout = DefaultScriptoriumTimeout
|
||||
}
|
||||
}
|
||||
|
||||
func applyNotariusDefaults(cfg *NotariusConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
if cfg.Binary == "" {
|
||||
cfg.Binary = DefaultNotariusBinary
|
||||
}
|
||||
if cfg.Timeout == "" {
|
||||
cfg.Timeout = DefaultNotariusTimeout
|
||||
}
|
||||
}
|
||||
|
||||
func resolveNotariusPaths(cfg *PipelineConfig, pipelinePath string) error {
|
||||
if cfg == nil || cfg.Notarius == nil || !cfg.Notarius.Enabled {
|
||||
return nil
|
||||
@@ -737,12 +451,6 @@ func resolveNotariusPaths(cfg *PipelineConfig, pipelinePath string) error {
|
||||
return fmt.Errorf("resolve pipeline.notarius.config_path: %w", err)
|
||||
}
|
||||
cfg.Notarius.ConfigPath = resolvedConfigPath
|
||||
if strings.TrimSpace(cfg.Notarius.WorkingDirectory) == "" {
|
||||
if resolvedConfigPath != "" {
|
||||
cfg.Notarius.WorkingDirectory = filepath.Dir(resolvedConfigPath)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
workingDirectory, err := resolve(cfg.Notarius.WorkingDirectory)
|
||||
if err != nil {
|
||||
return fmt.Errorf("resolve pipeline.notarius.working_directory: %w", err)
|
||||
@@ -750,83 +458,3 @@ func resolveNotariusPaths(cfg *PipelineConfig, pipelinePath string) error {
|
||||
cfg.Notarius.WorkingDirectory = workingDirectory
|
||||
return nil
|
||||
}
|
||||
|
||||
func applyTrimDefaults(cfg *TrimConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
if cfg.Enabled == nil {
|
||||
cfg.Enabled = boolPtr(DefaultTrimEnabled)
|
||||
}
|
||||
if strings.TrimSpace(cfg.OutputPath) == "" {
|
||||
cfg.OutputPath = DefaultTrimOutputPath
|
||||
}
|
||||
if strings.TrimSpace(cfg.Bounds.PromptID) == "" {
|
||||
cfg.Bounds.PromptID = DefaultTrimBoundsPromptID
|
||||
}
|
||||
if strings.TrimSpace(cfg.Bounds.TranscriptInputName) == "" {
|
||||
cfg.Bounds.TranscriptInputName = DefaultTrimBoundsTranscriptInputName
|
||||
}
|
||||
if strings.TrimSpace(cfg.Bounds.OutputPath) == "" {
|
||||
cfg.Bounds.OutputPath = DefaultTrimBoundsOutputPath
|
||||
}
|
||||
if cfg.Bounds.Timeout == "" {
|
||||
cfg.Bounds.Timeout = DefaultTrimBoundsTimeout
|
||||
}
|
||||
if cfg.Seriatim.Report == nil {
|
||||
cfg.Seriatim.Report = boolPtr(DefaultTrimSeriatimReport)
|
||||
}
|
||||
}
|
||||
|
||||
func trimEnabled(cfg *TrimConfig) bool {
|
||||
return cfg != nil && cfg.Enabled != nil && *cfg.Enabled
|
||||
}
|
||||
|
||||
func applyRenderDefaults(cfg **RenderConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
if *cfg == nil {
|
||||
*cfg = &RenderConfig{}
|
||||
}
|
||||
if (*cfg).Enabled == nil {
|
||||
(*cfg).Enabled = boolPtr(DefaultRenderEnabled)
|
||||
}
|
||||
if strings.TrimSpace((*cfg).Format) == "" {
|
||||
(*cfg).Format = DefaultRenderFormat
|
||||
}
|
||||
if strings.TrimSpace((*cfg).Title) == "" {
|
||||
(*cfg).Title = DefaultRenderTitle
|
||||
}
|
||||
if (*cfg).IncludeTimestamps == nil {
|
||||
(*cfg).IncludeTimestamps = boolPtr(DefaultRenderTimestamps)
|
||||
}
|
||||
if (*cfg).IncludeSegmentIDs == nil {
|
||||
(*cfg).IncludeSegmentIDs = boolPtr(DefaultRenderSegmentIDs)
|
||||
}
|
||||
}
|
||||
|
||||
func applyNormalizeDefaults(cfg *NormalizeConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
if cfg.OutputSchema == "" {
|
||||
cfg.OutputSchema = DefaultNormalizeOutputSchema
|
||||
}
|
||||
if cfg.OutputPath == "" && !cfg.outputPathWasSet() {
|
||||
cfg.OutputPath = DefaultNormalizeOutputPath
|
||||
}
|
||||
if cfg.Report == nil {
|
||||
cfg.Report = boolPtr(DefaultNormalizeReport)
|
||||
}
|
||||
}
|
||||
|
||||
func float64Ptr(v float64) *float64 {
|
||||
p := v
|
||||
return &p
|
||||
}
|
||||
|
||||
func boolPtr(v bool) *bool {
|
||||
p := v
|
||||
return &p
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -8,6 +9,8 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func TestRepresentativeLoadDefaultsAndValidation(t *testing.T) {
|
||||
@@ -83,6 +86,150 @@ whisperx:
|
||||
}
|
||||
}
|
||||
|
||||
func TestCredentialValuesNeverEnterEffectiveProjectionOrDigest(t *testing.T) {
|
||||
const firstCanary = "credential-canary-first"
|
||||
const secondCanary = "credential-canary-second"
|
||||
t.Setenv("NARRATIO_TEST_ACCESS_KEY", firstCanary)
|
||||
t.Setenv("NARRATIO_TEST_SECRET_KEY", firstCanary)
|
||||
t.Setenv("NARRATIO_TEST_AUDITA_KEY", firstCanary)
|
||||
pipelineYAML := strings.Replace(testPipelineBaseYAML, "audita:\n binary: audita\n", "audita:\n binary: audita\n llm_api_key: NARRATIO_TEST_AUDITA_KEY\n", 1) + `
|
||||
storage:
|
||||
backend: s3
|
||||
s3:
|
||||
bucket: test-bucket
|
||||
access_key_id: NARRATIO_TEST_ACCESS_KEY
|
||||
secret_access_key: NARRATIO_TEST_SECRET_KEY
|
||||
`
|
||||
pipelinePath, _ := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
|
||||
|
||||
first, err := LoadPipeline(pipelinePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
firstYAML, err := MarshalEffectivePipeline(first)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
firstValues, err := EffectivePipelineValues(first)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
serializedValues := ""
|
||||
for _, value := range firstValues {
|
||||
serializedValues += value.Path + "=" + value.Value + "\n"
|
||||
}
|
||||
directYAML, err := yaml.Marshal(first)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
directJSON, err := json.Marshal(first)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, output := range []string{string(firstYAML), serializedValues, string(directYAML), string(directJSON)} {
|
||||
if strings.Contains(output, firstCanary) {
|
||||
t.Fatalf("effective output exposed credential value: %q", output)
|
||||
}
|
||||
for _, reference := range []string{"NARRATIO_TEST_ACCESS_KEY", "NARRATIO_TEST_SECRET_KEY", "NARRATIO_TEST_AUDITA_KEY"} {
|
||||
if !strings.Contains(output, reference) {
|
||||
t.Fatalf("effective output omitted safe reference %q: %q", reference, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
t.Setenv("NARRATIO_TEST_ACCESS_KEY", secondCanary)
|
||||
t.Setenv("NARRATIO_TEST_SECRET_KEY", secondCanary)
|
||||
t.Setenv("NARRATIO_TEST_AUDITA_KEY", secondCanary)
|
||||
second, err := LoadPipeline(pipelinePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if EffectivePipelineDigest(first) != EffectivePipelineDigest(second) {
|
||||
t.Fatal("credential value change altered effective digest")
|
||||
}
|
||||
t.Setenv("NARRATIO_TEST_AUDITA_KEY_ALTERNATE", secondCanary)
|
||||
alternatePath, _ := writeConfigFiles(t, strings.Replace(pipelineYAML, "NARRATIO_TEST_AUDITA_KEY", "NARRATIO_TEST_AUDITA_KEY_ALTERNATE", 1), testSessionBaseYAML)
|
||||
alternate, err := LoadPipeline(alternatePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if EffectivePipelineDigest(first) == EffectivePipelineDigest(alternate) {
|
||||
t.Fatal("credential reference-name change did not alter effective digest")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectivePipelineValuesPreserveNilEmptyAndExactIntegers(t *testing.T) {
|
||||
base := &PipelineConfig{
|
||||
Audita: AuditaConfig{Modules: nil},
|
||||
Scriptorium: &ScriptoriumConfig{Artifacts: nil},
|
||||
resolution: &pipelineResolutionMetadata{},
|
||||
}
|
||||
nilValues, err := EffectivePipelineValues(base)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
nilYAML, err := MarshalEffectivePipeline(base)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(nilYAML), "modules: null\n") || !strings.Contains(string(nilYAML), "artifacts: null\n") {
|
||||
t.Fatalf("effective YAML lost null collections: %q", nilYAML)
|
||||
}
|
||||
if err := recomputePipelineEffectiveDigest(base); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
nilDigest := EffectivePipelineDigest(base)
|
||||
base.Audita.Modules = []string{}
|
||||
base.Scriptorium.Artifacts = map[string]ScriptoriumArtifactConfig{}
|
||||
emptyYAML, err := MarshalEffectivePipeline(base)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(emptyYAML), "modules: []\n") || !strings.Contains(string(emptyYAML), "artifacts: {}\n") {
|
||||
t.Fatalf("effective YAML lost empty collections: %q", emptyYAML)
|
||||
}
|
||||
if err := recomputePipelineEffectiveDigest(base); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if emptyDigest := EffectivePipelineDigest(base); emptyDigest == nilDigest {
|
||||
t.Fatalf("null and empty collections have equal digest %q", emptyDigest)
|
||||
}
|
||||
base.Scriptorium.Artifacts = map[string]ScriptoriumArtifactConfig{
|
||||
"recap": {Vars: map[string]any{"exact": int64(9007199254740993)}},
|
||||
}
|
||||
emptyValues, err := EffectivePipelineValues(base)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := effectiveValueForTest(t, nilValues, "audita.modules"); got != "null" {
|
||||
t.Fatalf("nil sequence value = %q, want null", got)
|
||||
}
|
||||
if got := effectiveValueForTest(t, emptyValues, "audita.modules"); got != "[]" {
|
||||
t.Fatalf("empty sequence value = %q, want []", got)
|
||||
}
|
||||
if got := effectiveValueForTest(t, emptyValues, "scriptorium.artifacts.recap.vars.exact"); got != "9007199254740993" {
|
||||
t.Fatalf("exact integer value = %q", got)
|
||||
}
|
||||
encoded, err := MarshalEffectivePipeline(base)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(encoded), "9007199254740993") {
|
||||
t.Fatalf("effective YAML lost exact integer: %q", encoded)
|
||||
}
|
||||
}
|
||||
|
||||
func effectiveValueForTest(t *testing.T, values []EffectivePipelineValueRecord, path string) string {
|
||||
t.Helper()
|
||||
for _, value := range values {
|
||||
if value.Path == path {
|
||||
return value.Value
|
||||
}
|
||||
}
|
||||
t.Fatalf("effective value %q not found", path)
|
||||
return ""
|
||||
}
|
||||
|
||||
func TestValidateMissingAudioSource(t *testing.T) {
|
||||
cfg := loadedValidConfig(t)
|
||||
cfg.Session.Inputs.AudioDir = ""
|
||||
@@ -102,6 +249,9 @@ func TestValidateMissingAudioSource(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestExamplesLoadAndValidate(t *testing.T) {
|
||||
t.Setenv("OBJECT_STORAGE_KEY_ID", "synthetic-example-access-key")
|
||||
t.Setenv("OBJECT_STORAGE_KEY", "synthetic-example-secret-key")
|
||||
t.Setenv("AUDITA_LLM_API_KEY", "synthetic-example-audita-key")
|
||||
examplesDir := filepath.Join("..", "..", "examples")
|
||||
if got, want := maintainedPipelineRoots(t, examplesDir), []string{
|
||||
"pipeline.extraction-subset.yml",
|
||||
@@ -164,15 +314,19 @@ func TestExamplesLoadAndValidate(t *testing.T) {
|
||||
t.Fatalf("load split bundle campaign: %v", err)
|
||||
}
|
||||
for _, profile := range []struct {
|
||||
name string
|
||||
model string
|
||||
name string
|
||||
selection string
|
||||
model string
|
||||
}{
|
||||
{name: "production", model: "narratio-production-model-placeholder"},
|
||||
{name: "testing", model: "narratio-testing-model-placeholder"},
|
||||
{name: "production base", model: "narratio-production-model-placeholder"},
|
||||
{name: "testing", selection: "testing", model: "narratio-testing-model-placeholder"},
|
||||
} {
|
||||
t.Run(profile.name, func(t *testing.T) {
|
||||
selected := profile.name
|
||||
pipeline, err := LoadPipelineWithOptions(pipelinePath, PipelineLoadOptions{Profile: &selected})
|
||||
options := PipelineLoadOptions{}
|
||||
if profile.selection != "" {
|
||||
options.Profile = &profile.selection
|
||||
}
|
||||
pipeline, err := LoadPipelineWithOptions(pipelinePath, options)
|
||||
if err != nil {
|
||||
t.Fatalf("load split bundle pipeline: %v", err)
|
||||
}
|
||||
@@ -190,13 +344,19 @@ func TestExamplesLoadAndValidate(t *testing.T) {
|
||||
if cfg.Pipeline.Audita.Model != profile.model {
|
||||
t.Fatalf("audita model = %q, want %q", cfg.Pipeline.Audita.Model, profile.model)
|
||||
}
|
||||
if cfg.Pipeline.Secrets != nil || cfg.Pipeline.Audita.LLMAPIKeyEnv != "" || cfg.Pipeline.Storage.Backend != StorageBackendLocal || cfg.Pipeline.Storage.S3 != nil {
|
||||
t.Fatalf("split bundle must remain offline and secret-free: secrets=%#v audita=%#v storage=%#v", cfg.Pipeline.Secrets, cfg.Pipeline.Audita, cfg.Pipeline.Storage)
|
||||
if cfg.Pipeline.Audita.LLMAPIKey != nil || cfg.Pipeline.Storage.Backend != StorageBackendLocal || cfg.Pipeline.Storage.S3 != nil {
|
||||
t.Fatalf("split bundle must remain offline and secret-free: audita=%#v storage=%#v", cfg.Pipeline.Audita, cfg.Pipeline.Storage)
|
||||
}
|
||||
if selectedProfile, ok := SelectedPipelineProfile(cfg.Pipeline); !ok || selectedProfile.Name != profile.name {
|
||||
t.Fatalf("selected profile = %#v, want %q", selectedProfile, profile.name)
|
||||
selectedProfile, selected := SelectedPipelineProfile(cfg.Pipeline)
|
||||
if profile.selection == "" && selected {
|
||||
t.Fatalf("production base unexpectedly selected profile %#v", selectedProfile)
|
||||
}
|
||||
if profile.selection != "" && (!selected || selectedProfile.Name != profile.selection) {
|
||||
t.Fatalf("selected profile = %#v, want %q", selectedProfile, profile.selection)
|
||||
}
|
||||
for _, key := range []string{
|
||||
"session_recap",
|
||||
"player_handout",
|
||||
"character_meta_arannis",
|
||||
"character_meta_brenna",
|
||||
"character_items_arannis",
|
||||
@@ -209,6 +369,12 @@ func TestExamplesLoadAndValidate(t *testing.T) {
|
||||
if got := cfg.Pipeline.Scriptorium.Artifacts["character_items_arannis"].Inputs["character_meta"].Source; got != "narratio.artifact.character_meta_arannis" {
|
||||
t.Fatalf("same-member input source = %q", got)
|
||||
}
|
||||
if got := cfg.Pipeline.Scriptorium.Artifacts["player_handout"].DependsOn; !reflect.DeepEqual(got, []string{"session_recap"}) {
|
||||
t.Fatalf("input-only artifact dependency = %#v", got)
|
||||
}
|
||||
if got := cfg.Pipeline.Scriptorium.Artifacts["character_items_arannis"].DependsOn; !reflect.DeepEqual(got, []string{"character_meta_arannis"}) {
|
||||
t.Fatalf("family input dependency = %#v", got)
|
||||
}
|
||||
if cfg.Party.Canonical == nil || len(cfg.Party.Canonical.Characters) != 2 {
|
||||
t.Fatalf("canonical party = %#v", cfg.Party)
|
||||
}
|
||||
@@ -219,7 +385,18 @@ func TestExamplesLoadAndValidate(t *testing.T) {
|
||||
if len(cfg.Pipeline.Publish.Outputs) != 3 {
|
||||
t.Fatalf("publish outputs = %#v, want transcript plus two family outputs", cfg.Pipeline.Publish.Outputs)
|
||||
}
|
||||
if profile.name == "testing" {
|
||||
catalog, err := EffectiveArtifactCatalog(cfg.Pipeline, cfg.Party)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantCatalogSize := 6
|
||||
if profile.selection == "testing" {
|
||||
wantCatalogSize++
|
||||
}
|
||||
if len(catalog) != wantCatalogSize {
|
||||
t.Fatalf("effective artifact catalog has %d records, want %d", len(catalog), wantCatalogSize)
|
||||
}
|
||||
if profile.selection == "testing" {
|
||||
artifact, exists := cfg.Pipeline.Scriptorium.Artifacts["testing_notes"]
|
||||
if !exists || artifact.Enabled {
|
||||
t.Fatalf("testing-only disabled artifact = %#v", artifact)
|
||||
@@ -259,6 +436,9 @@ func maintainedPipelineRoots(t *testing.T, examplesDir string) []string {
|
||||
}
|
||||
|
||||
func TestMaintainedExtractionExamplesPreservePublishedContracts(t *testing.T) {
|
||||
t.Setenv("OBJECT_STORAGE_KEY_ID", "synthetic-example-access-key")
|
||||
t.Setenv("OBJECT_STORAGE_KEY", "synthetic-example-secret-key")
|
||||
t.Setenv("AUDITA_LLM_API_KEY", "synthetic-example-audita-key")
|
||||
examplesDir := filepath.Join("..", "..", "examples")
|
||||
full, err := LoadPipeline(filepath.Join(examplesDir, "pipeline.full.annotated.yml"))
|
||||
if err != nil {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user