Adopt ConfigKit for pipeline loading

This commit is contained in:
2026-09-07 03:54:41 +00:00
parent 0e6ba414b3
commit 9e40c729e1
33 changed files with 683 additions and 3162 deletions

View File

@@ -53,10 +53,10 @@ Rules:
- if both positional `<session_id>` and `--session-id` are provided, values must match. - if both positional `<session_id>` and `--session-id` are provided, values must match.
- `--previous-session-id` is a strict expectation: the selected session file - `--previous-session-id` is a strict expectation: the selected session file
must contain the same `previous_session_id`. must contain the same `previous_session_id`.
- `--profile` selects a declared pipeline profile. It may be supplied once; - `--profile` selects `profiles/<name>.yml` relative to the pipeline root. It
an explicit empty or unknown value fails configuration resolution. When it is may be supplied once; omitted, empty, and the exact name `default` select the
omitted, a declared `default_profile` is used. The same selection applies to base configuration, while an unknown named profile fails. The same selection
all common-flag commands, including `regenerate-artifacts`. applies to all common-flag commands, including `regenerate-artifacts`.
- `clean --all` cannot be combined with campaign/session selectors. - `clean --all` cannot be combined with campaign/session selectors.
- notification delivery is currently limited to the configured `noop` mode; see - notification delivery is currently limited to the configured `noop` mode; see
the [configuration reference](./config.md#notifications). the [configuration reference](./config.md#notifications).
@@ -101,7 +101,7 @@ Session, range, force, and artifact-execution flags are not accepted.
`config validate` writes a concise root-path, selected-profile (or `none`), and `config validate` writes a concise root-path, selected-profile (or `none`), and
effective-digest summary after successful complete validation. `config show` effective-digest summary after successful complete validation. `config show`
writes one deterministic, secret-free YAML document containing defaulted and writes one deterministic, secret-free YAML document containing defaulted and
expanded concrete configuration. It omits composition declarations, artifact expanded concrete configuration. It omits ConfigKit directives, artifact
family declarations, and runtime provenance. family declarations, and runtime provenance.
`config sources` reports the same fully validated resolution without printing `config sources` reports the same fully validated resolution without printing
@@ -113,8 +113,8 @@ and generated family ownership. A generated party member has
one family record and one party record at the same logical path. The output one family record and one party record at the same logical path. The output
never reads or prints secret values. never reads or prints secret values.
`config diff` resolves both supplied profile names from one parsed root source `config diff` resolves both supplied profile names independently and compares
set and compares their fully resolved, secret-free effective mappings. It does their fully resolved, secret-free effective mappings. It does
not accept `--profile`; the two positional names must be distinct, declared not accept `--profile`; the two positional names must be distinct, declared
profiles. When party-driven families are present, both profiles must resolve to profiles. When party-driven families are present, both profiles must resolve to
the same selected campaign and party. Use `--campaign-file` if profile-specific the same selected campaign and party. Use `--campaign-file` if profile-specific

View File

@@ -49,8 +49,8 @@ sources` use the same `--config`, `--campaign`, `--campaign-file`, and
`--profile` selection rules as pipeline commands, but do not select, discover, `--profile` selection rules as pipeline commands, but do not select, discover,
or load a session. They do not read credential values or create runtime state. or load a session. They do not read credential values or create runtime state.
`narratio config diff <left-profile> <right-profile>` uses the same pipeline and `narratio config diff <left-profile> <right-profile>` uses the same pipeline and
campaign selectors, resolves each named profile independently from one parsed campaign selectors, resolves each named profile with an independent ConfigKit
root source set, and does not accept a separate `--profile` flag. load, and does not accept a separate `--profile` flag.
Campaign selection is optional only when the resolved pipeline has no Campaign selection is optional only when the resolved pipeline has no
`scriptorium.artifact_families`. When families are declared, Narratio selects a `scriptorium.artifact_families`. When families are declared, Narratio selects a
@@ -59,7 +59,7 @@ then parses the campaign-owned party and expands concrete artifacts and any
family publish rules before validation. `config validate` prints the resulting family publish rules before validation. `config validate` prints the resulting
root, profile, and effective digest. `config show` emits the normalized root, profile, and effective digest. `config show` emits the normalized
effective pipeline YAML, with defaults and concrete expansion included but effective pipeline YAML, with defaults and concrete expansion included but
composition and family declarations omitted. `config sources` prints a stable ConfigKit directives and family declarations omitted. `config sources` prints a stable
source projection instead of effective values: root/import/profile/default source projection instead of effective values: root/import/profile/default
ownership plus campaign/party and generated-family records. Canonical derived ownership plus campaign/party and generated-family records. Canonical derived
players trace to the party. The [CLI reference](cli.md#config-validate-config-show-config-sources-and-config-diff) players trace to the party. The [CLI reference](cli.md#config-validate-config-show-config-sources-and-config-diff)
@@ -77,8 +77,9 @@ difference.
Use the [production/testing bundle](../examples/production-testing/pipeline.yml) Use the [production/testing bundle](../examples/production-testing/pipeline.yml)
as the complete copyable migration reference. Split stable pipeline settings as the complete copyable migration reference. Split stable pipeline settings
into explicit additive imports, place production/testing differences in one into explicit additive imports and place production/testing differences in the
selected overlay, and retain a production `default_profile`. Convert campaign fixed `profiles/production.yml` and `profiles/testing.yml` overlays. Convert
campaign
rosters to [canonical party input](integrations/party.md), then express rosters to [canonical party input](integrations/party.md), then express
character work as families. Inspect the result character work as families. Inspect the result
with `config validate`, `config show`, and `config sources`; use `config diff` with `config validate`, `config show`, and `config sources`; use `config diff`
@@ -96,16 +97,15 @@ remote state with an unsafe legacy identity must be migrated before use.
## Validation and Merge Rules ## Validation and Merge Rules
- YAML decode is strict (`KnownFields(true)`) and accepts exactly one document: - ConfigKit performs strict typed YAML decoding and accepts exactly one document;
unknown fields or trailing documents fail load. unknown fields, duplicate keys, and trailing documents fail load.
- A pipeline file may explicitly import additive YAML fragments through the - A pipeline file may explicitly import additive YAML fragments through the
root-only `composition.imports` list. Imported files contribute fields to one root-only `configkit.imports` list. Imported files contribute fields to one
logical pipeline document; they do not override fields supplied by the root logical pipeline document; they do not override fields supplied by the root
or another import. or another import.
- A root pipeline may declare named profiles. Exactly one profile is selected - An omitted profile, an empty profile option, and the exact name `default`
by an option-aware caller or by `composition.default_profile`; a caller's select the base configuration. Any other name selects
explicit selection takes precedence. Declaring profiles without either form `profiles/<name>.yml` relative to the pipeline root.
of selection is an error.
- Configured timeout and retry-delay durations must be positive. An omitted - Configured timeout and retry-delay durations must be positive. An omitted
artifact timeout continues to inherit its configured Scriptorium timeout. artifact timeout continues to inherit its configured Scriptorium timeout.
- Session files must be concrete; unresolved `{{ ... }}` placeholders fail load. - Session files must be concrete; unresolved `{{ ... }}` placeholders fail load.
@@ -123,22 +123,16 @@ remote state with an unsafe legacy identity must be migrated before use.
- local (`audio_dir` or `audio_files`), or - local (`audio_dir` or `audio_files`), or
- S3 (`audio_s3.prefix`). - S3 (`audio_s3.prefix`).
### Pipeline composition ### Pipeline imports and profiles
Large pipeline configurations may be split into explicitly named fragments and Large pipeline configurations may be split into explicitly named fragments and
may declare one overlay per selectable profile: may use fixed-path profile overlays:
```yaml ```yaml
composition: configkit:
imports: imports:
- config/storage.yml - path: config/storage.yml
- config/integrations.yaml - path: config/integrations.yaml
default_profile: production
profiles:
production:
overlay: profiles/production.yml
testing:
overlay: profiles/testing.yml
campaigns: campaigns:
root: /usr/local/share/narratio/campaigns root: /usr/local/share/narratio/campaigns
@@ -152,36 +146,29 @@ Imports are resolved relative to the directory containing the root pipeline
file and are loaded in declaration order. Narratio does not scan directories or file and are loaded in declaration order. Narratio does not scan directories or
infer fragments. Each import must be a confined regular `.yml` or `.yaml` file: infer fragments. Each import must be a confined regular `.yml` or `.yaml` file:
absolute paths, traversal, symlinks, directories, duplicate files, and an absolute paths, traversal, symlinks, directories, duplicate files, and an
import of the root pipeline itself are rejected. Only the root pipeline may import of the root pipeline itself are rejected. Imported files cannot declare
contain `composition`; nested composition is rejected. another `configkit` control section.
Composition is additive. A map may be extended by multiple files when every Base merging is additive. A map may be extended by multiple files when every
leaf is distinct, but a scalar, list, or map/list/scalar kind cannot be claimed leaf is distinct, but a scalar, list, or map/list/scalar kind cannot be claimed
more than once, even when the repeated values are identical. Conflict errors more than once, even when the repeated values are identical. Conflict errors
name the full field path and every source that claimed it. The assembled YAML is name the full field path and every source that claimed it. Narratio additionally
then decoded against the normal strict pipeline schema and defaults are applied rejects an artifact or artifact-family key declared by more than one base file,
once. even when those files contribute disjoint children.
Profile names are case-sensitive, non-empty, trimmed, and cannot contain Profile names are case-sensitive and use ConfigKit's safe fixed-path naming
control characters. If `profiles` is present, it must contain at least one rules. Only the selected overlay is opened; a malformed unselected overlay does
entry and every entry must contain only an `overlay` path. An explicit profile not affect a base or different-profile load. Maps merge recursively, overlay
selection overrides `default_profile`; unknown and explicitly empty selections scalars replace base scalars, and overlay lists replace base lists completely.
fail. Narratio never selects the first profile implicitly and does not read a Explicit `false`, zero, empty strings, empty containers, and null values remain
profile selection from the environment. explicit inputs and are not refilled by defaults. Profiles cannot inherit,
stack, import files, or declare another ConfigKit control section.
Every declared overlay is resolved relative to the root pipeline directory and During ConfigKit loading, Narratio applies fresh typed defaults, recorded
must satisfy the same confined regular-YAML-file rules as an import. Narratio entry-level defaults and normalization, read-only file-local validation, and
parses every declared overlay even when it is not selected, then applies only ordinary path resolution before computing a deterministic effective
the selected one. Maps merge recursively, overlay scalars replace base scalars,
and overlay lists replace base lists completely. Explicit `false`, zero, empty
lists, and empty maps remain meaningful. YAML null cannot delete a value, and
kind changes are rejected. Profiles cannot inherit from or stack with other
profiles, and overlays cannot import files or declare profiles.
After composition, Narratio strictly decodes the result, applies centralized
defaults once, resolves ordinary paths, and computes a deterministic effective
configuration digest. The digest represents the normalized, secret-free configuration digest. The digest represents the normalized, secret-free
runtime pipeline mapping; it excludes composition declarations, source runtime pipeline mapping; it excludes ConfigKit directives, source
provenance, profile identity, and raw environment secret values. Equivalent provenance, profile identity, and raw environment secret values. Equivalent
effective mappings therefore have the same digest regardless of how fields are effective mappings therefore have the same digest regardless of how fields are
split among the root and imports. split among the root and imports.
@@ -274,9 +261,7 @@ Rules:
| Field | Type | Required | Default / Rule | | Field | Type | Required | Default / Rule |
| --- | --- | --- | --- | | --- | --- | --- | --- |
| `composition.imports[]` | list of strings | No | explicit additive pipeline fragments relative to the root pipeline directory; `.yml` or `.yaml` regular files only | | `configkit.imports[].path` | string | No | explicit additive pipeline fragment relative to the root pipeline directory; `.yml` or `.yaml` regular files only |
| `composition.default_profile` | string | Conditional | selected when profiles exist and no caller explicitly selects one; must name a declared profile |
| `composition.profiles.<name>.overlay` | string | Conditional | required for every declared profile; one confined `.yml` or `.yaml` overlay relative to the root pipeline directory |
| `pipeline.workspace.root` | string | No | `/var/lib/narratio` | | `pipeline.workspace.root` | string | No | `/var/lib/narratio` |
| `pipeline.workspace.cleanup_after_publish` | bool | No | `false`; selects the published run's workspace target for final cleanup and requires effective `pipeline.publish.enabled=true` and `pipeline.publish.upload_run=true` | | `pipeline.workspace.cleanup_after_publish` | bool | No | `false`; selects the published run's workspace target for final cleanup and requires effective `pipeline.publish.enabled=true` and `pipeline.publish.upload_run=true` |
| `pipeline.campaigns.root` | string | No | `/usr/local/share/narratio/campaigns` | | `pipeline.campaigns.root` | string | No | `/usr/local/share/narratio/campaigns` |

View File

@@ -8,31 +8,21 @@ pipeline-loading boundary implemented by `internal/config`.
`LoadPipeline` assembles and validates a pipeline in this order: `LoadPipeline` assembles and validates a pipeline in this order:
1. Parse the root YAML into a presence-aware composition tree. The tree retains 1. Give ConfigKit a fresh typed `PipelineConfig` default value, the root file,
source names, full field paths, node kinds, declaration order, and explicit enabled imports, and the caller's profile selection. Omitted, empty, and
zero, false, empty-map, and empty-list values. exact `default` selections use the base; named profiles use the fixed
2. Decode the root-only `composition` envelope, validate its declaration shapes `profiles/<name>.yml` convention.
and cross-field profile semantics, then clone the root without that envelope 2. ConfigKit parses the root and declared imports, additively merges base
as the ordinary pipeline body. The parsed source document is not mutated. A sources, applies only the selected overlay, and strictly decodes the result.
load option retains the difference between omitted and explicitly empty 3. The definition resolver applies recorded entry-level and inherited defaults,
profile selection. runs deterministic domain normalization, records derived publish destinations,
3. Open each import relative to the root pipeline directory through the and resolves ordinary paths relative to the root pipeline file.
confined regular-file boundary. Imports must use a `.yml` or `.yaml` 4. The definition validator performs read-only checks that require only the
extension and cannot traverse, use symlinks, repeat a file, import the root, pipeline file. Checks requiring canonical-party family expansion remain at
or contain another composition envelope. the existing explicit pipeline/campaign resolution boundary.
4. Resolve and structurally parse every declared profile overlay through the 5. Narratio inspects public ConfigKit metadata to reject artifact and family keys
same confined regular-file boundary. Missing or malformed unselected declared by multiple base files, adapts source ownership records, attaches
overlays fail the load. Overlays cannot contain a composition envelope. runtime-only state, and computes the normalized effective digest.
5. Additively merge the root body and imports. Distinct map leaves compose;
repeated scalar or list paths and node-kind disagreements are conflicts.
6. Select exactly one declared profile from an explicit option or the default,
then recursively merge its overlay. Overlay leaves replace base leaves,
lists are atomic replacements, and null or kind changes fail.
7. Emit deterministic canonical YAML and strictly decode it into
`PipelineConfig`.
8. Apply pipeline defaults once, explicitly normalize reusable domain values,
resolve ordinary relative pipeline paths from the root pipeline file, and
digest the normalized effective mapping.
This ordering preserves monolithic configuration behavior. Moving a field to This ordering preserves monolithic configuration behavior. Moving a field to
an imported fragment changes its source ownership, not its path base, default, an imported fragment changes its source ownership, not its path base, default,
@@ -71,7 +61,7 @@ commands do not marshal runtime objects directly.
`EffectivePipelineSources` and `EffectiveCampaignSources` provide the separate `EffectivePipelineSources` and `EffectiveCampaignSources` provide the separate
safe provenance projection for `config sources`. Pipeline ownership begins with safe provenance projection for `config sources`. Pipeline ownership begins with
the complete logical field paths retained during composition and classifies the complete logical field paths projected from the schema and classifies
each contributor as root, import, profile, or centralized default. The each contributor as root, import, profile, or centralized default. The
projection replaces generated concrete member paths with paired family and projection replaces generated concrete member paths with paired family and
canonical-party records, and does the same for generated publish rules. canonical-party records, and does the same for generated publish rules.
@@ -79,9 +69,8 @@ Campaign records identify campaign-owned fields and party inputs; canonical
derived players point to the party source. The application command only joins these sorted derived players point to the party source. The application command only joins these sorted
records with selection metadata and never reparses configuration files. records with selection metadata and never reparses configuration files.
`config diff` uses a paired profile loader that parses the root, imports, and `config diff` uses a thin paired loader that performs two independent ConfigKit
declared overlays once, then clones the additive base before independently loads. When campaign
selecting, decoding, defaulting, and finalizing each profile. When campaign
resolution is needed, the command loads one selected campaign and party and resolution is needed, the command loads one selected campaign and party and
expands both effective pipelines from that same party value. The configuration expands both effective pipelines from that same party value. The configuration
owner projects each normalized effective mapping directly from that same typed owner projects each normalized effective mapping directly from that same typed
@@ -128,21 +117,22 @@ that candidate without changing it. Scriptorium validates artifact declarations
and outputs, dependencies and inputs, variables, referenced outputs, and the final and outputs, dependencies and inputs, variables, referenced outputs, and the final
dependency graph in deterministic artifact order. dependency graph in deterministic artifact order.
Diagnostics identify the relevant semantic field. Composition conflicts also Diagnostics identify the relevant semantic field. Merge conflicts also
retain every claiming source needed to correct an additive ownership conflict. retain every claiming source needed to correct an additive ownership conflict.
Complete prose, punctuation, formatting, and private validation choreography are Complete prose, punctuation, formatting, and private validation choreography are
not compatibility contracts. not compatibility contracts.
## Diagnostics And Runtime Metadata ## Diagnostics And Runtime Metadata
Syntax, duplicate-key, composition, conflict, and schema failures include the Syntax, duplicate-key, merge, profile, and schema failures include the
relevant source name and full field path. Additive conflicts report every relevant source name and full field path. Additive conflicts report every
claiming source so operators can repair the split without repeatedly claiming source so operators can repair the split without repeatedly
rediscovering additional conflicts. rediscovering additional conflicts.
The loaded pipeline retains private runtime metadata for the absolute root The loaded pipeline retains private runtime metadata for the absolute root
path, ordered imports, selected profile name and selection source, selected path, ordered imports, selected profile name and selection source, selected
overlay, contributing sources, effective digest, and leaf ownership. Base overlay, original ConfigKit metadata and source snapshots, effective digest,
and leaf ownership. Base
leaves retain their root/import owners, replaced leaves belong to the selected leaves retain their root/import owners, replaced leaves belong to the selected
overlay, and centrally supplied values use the synthetic `default` owner. This overlay, and centrally supplied values use the synthetic `default` owner. This
metadata does not participate in YAML decoding or alter the public metadata does not participate in YAML decoding or alter the public
@@ -152,7 +142,7 @@ The effective digest is SHA-256 over a versioned deterministic JSON encoding of
the same typed safe projection used by show and diff. Runtime Notarius paths remain absolute for the same typed safe projection used by show and diff. Runtime Notarius paths remain absolute for
execution, but the digest substitutes their normalized logical values captured execution, but the digest substitutes their normalized logical values captured
before root-relative resolution, so relocating an equivalent configuration before root-relative resolution, so relocating an equivalent configuration
bundle does not change provenance. Because composition and resolution metadata bundle does not change provenance. Because ConfigKit and resolution metadata
are private, the digest excludes source layout, profile name, and ownership. are private, the digest excludes source layout, profile name, and ownership.
Configuration stores environment variable names rather than resolving raw Configuration stores environment variable names rather than resolving raw
credentials, so raw secret values are neither loaded nor hashed. credentials, so raw secret values are neither loaded nor hashed.
@@ -161,12 +151,11 @@ for later runtime expansion.
## Test Surfaces ## Test Surfaces
`composition_test.go` protects the presence and merge algebra independently of `pipeline_configkit_test.go` exercises real imports and fixed-path profiles,
the public schema. `pipeline_composition_test.go` exercises explicit imports, base selection, unselected-overlay isolation, source attribution, and Narratio's
confinement, conflicts, strict decoding, metadata, and root-relative path unique artifact/family base-declaration rule. Existing focused configuration
behavior through `LoadPipeline`. `pipeline_profiles_test.go` covers selection, tests protect defaults, explicit-value semantics, strict decoding, validation,
all-overlay validation, overlay behavior, provenance, option propagation, and metadata, and root-relative path behavior. Application configuration-loader tests protect the
effective-digest stability. Application configuration-loader tests protect the
single-read boundary by changing the pipeline file after its initial load and single-read boundary by changing the pipeline file after its initial load and
confirming local session resolution retains the original pipeline. Other confirming local session resolution retains the original pipeline. Other
configuration tests continue to protect defaults and validation after assembly. configuration tests continue to protect defaults and validation after assembly.

View File

@@ -37,7 +37,7 @@ failed stage or required manifest save prevents later stages from starting.
| --- | --- | --- | | --- | --- | --- |
| Executable | `cmd/narratio` | Process entry, standard stream wiring, argument handoff, and exit status. | | Executable | `cmd/narratio` | Process entry, standard stream wiring, argument handoff, and exit status. |
| Application orchestration | `internal/app` | Command dispatch, configuration selection, secret-file environment loading, production composition, session locking, planning, execution, restore, and user-facing reporting. | | Application orchestration | `internal/app` | Command dispatch, configuration selection, secret-file environment loading, production composition, session locking, planning, execution, restore, and user-facing reporting. |
| Configuration | [`internal/config`](configuration.md) | Presence-aware root/import/profile composition, canonical party and family expansion, strict YAML loading, defaults, normalization, session templating, and validation. | | Configuration | [`internal/config`](configuration.md) | ConfigKit-backed pipeline imports and profiles, canonical party and family expansion, strict YAML loading, defaults, normalization, session templating, and validation. |
| Pipeline stages | `internal/stage` | Canonical stage registry, shared stage contract, execution dependencies, and implemented stage behavior. | | Pipeline stages | `internal/stage` | Canonical stage registry, shared stage contract, execution dependencies, and implemented stage behavior. |
| External boundaries | `internal/adapters`, `internal/audio` | WhisperX HTTP, downstream subprocesses, notification, object storage, and S3 audio materialization behind Narratio contracts. | | External boundaries | `internal/adapters`, `internal/audio` | WhisperX HTTP, downstream subprocesses, notification, object storage, and S3 audio materialization behind Narratio contracts. |
| Manifests | `internal/manifest` | Durable session progress, stage transitions, validation, and atomic persistence. | | Manifests | `internal/manifest` | Durable session progress, stage transitions, validation, and atomic persistence. |
@@ -105,8 +105,8 @@ so any earlier replacement can stale a prior cleanup result.
## Focused Documentation ## Focused Documentation
- [Configuration Internals](configuration.md): pipeline composition, import - [Configuration Internals](configuration.md): pipeline imports and profiles,
confinement, field ownership, decoding, and root-relative path semantics. field ownership, decoding, and root-relative path semantics.
- [Adapter Internals](adapters.md): external adapter boundaries, composition, - [Adapter Internals](adapters.md): external adapter boundaries, composition,
failure behavior, and test surfaces. failure behavior, and test surfaces.
- [Artifact Internals](artifacts.md): source identities, runtime catalog, - [Artifact Internals](artifacts.md): source identities, runtime catalog,

View File

@@ -146,7 +146,7 @@ Configuration is strict, explicit, centralized, and operator-oriented.
- YAML decoding rejects unknown fields. - YAML decoding rejects unknown fields.
- Defaults are centralized and testable. - Defaults are centralized and testable.
- Empty configured values do not silently replace meaningful defaults. - Empty configured values do not silently replace meaningful defaults.
- Validation rejects invalid composition before stage execution where - Validation rejects invalid configuration before stage execution where
practical. practical.
- Root-owned imports and one selected profile resolve deterministically through - Root-owned imports and one selected profile resolve deterministically through
the configuration owner; commands do not implement their own merge rules. the configuration owner; commands do not implement their own merge rules.

View File

@@ -189,7 +189,10 @@ projection consumer needs the custom composition parser/renderer.
## Stage 4 — Cut pipeline loading over to ConfigKit ## Stage 4 — Cut pipeline loading over to ConfigKit
Status: Pending Status: Complete — Replaced pipeline composition with ConfigKit v0.1.1 imports,
fixed-path profile selection, fresh typed defaults, recorded resolution, source
metadata adaptation, and duplicate base artifact/family enforcement; removed the
legacy parser and verified all tests plus the configuration race suite.
Read: ConfigKit v0.1.1 `docs/config.md`, `docs/consumers/README.md`, and its Read: ConfigKit v0.1.1 `docs/config.md`, `docs/consumers/README.md`, and its
Narratio-shaped compiled example; Narratio `internal/config/load.go`, Narratio-shaped compiled example; Narratio `internal/config/load.go`,

View File

@@ -1,15 +1,9 @@
# Copyable production/testing pipeline entry point. Every fragment is named # Copyable production/testing pipeline entry point. Every fragment is named
# explicitly; Narratio never scans conf.d automatically. # explicitly; Narratio never scans conf.d automatically.
composition: configkit:
imports: imports:
- conf.d/platform.yml - path: conf.d/platform.yml
- conf.d/storage.yml - path: conf.d/storage.yml
- conf.d/transcript.yml - path: conf.d/transcript.yml
- conf.d/artifacts.yml - path: conf.d/artifacts.yml
- conf.d/publish.yml - path: conf.d/publish.yml
default_profile: production
profiles:
production:
overlay: profiles/production.yml
testing:
overlay: profiles/testing.yml

2
go.mod
View File

@@ -3,6 +3,7 @@ module gitea.maximumdirect.net/eric/narratio
go 1.26.0 go 1.26.0
require ( require (
gitea.maximumdirect.net/eric/ConfigKit v0.1.1
github.com/aws/aws-sdk-go-v2/config v1.32.17 github.com/aws/aws-sdk-go-v2/config v1.32.17
github.com/aws/aws-sdk-go-v2/credentials v1.19.16 github.com/aws/aws-sdk-go-v2/credentials v1.19.16
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
@@ -28,4 +29,5 @@ require (
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
) )

4
go.sum
View File

@@ -1,3 +1,5 @@
gitea.maximumdirect.net/eric/ConfigKit v0.1.1 h1:/JXi95QDhgRtiQN7R/+6cGFcfc0+gfl+D44nKXPd52k=
gitea.maximumdirect.net/eric/ConfigKit v0.1.1/go.mod h1:tYWpsrbLyu05r8+1MapWp5fN6ZrX9I8hU2xEPedYmYk=
github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=
github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho=
@@ -36,6 +38,8 @@ github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=

View File

@@ -253,8 +253,8 @@ func TestAssembledSplitBundleCommandsUseOneEffectiveFamilyConfiguration(t *testi
} }
var validated bytes.Buffer var validated bytes.Buffer
if err := ConfigValidate(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath}, &validated); err != nil { if err := ConfigValidate(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath, "--profile", "production"}, &validated); err != nil {
t.Fatalf("validate production default: %v", err) t.Fatalf("validate production profile: %v", err)
} }
if !strings.Contains(validated.String(), "profile=production") { if !strings.Contains(validated.String(), "profile=production") {
t.Fatalf("validate output = %q, want production profile", validated.String()) t.Fatalf("validate output = %q, want production profile", validated.String())
@@ -330,7 +330,7 @@ func TestAssembledSplitBundleCommandsUseOneEffectiveFamilyConfiguration(t *testi
} }
func TestAssembledSplitBundleProfileSwitchRecordsProvenanceAndLimitsReuse(t *testing.T) { func TestAssembledSplitBundleProfileSwitchRecordsProvenanceAndLimitsReuse(t *testing.T) {
production := loadAssembledSplitBundleConfig(t, "") production := loadAssembledSplitBundleConfig(t, "production")
testingCfg := loadAssembledSplitBundleConfig(t, "testing") testingCfg := loadAssembledSplitBundleConfig(t, "testing")
workspace := t.TempDir() workspace := t.TempDir()
production.Pipeline.Workspace.Root = workspace production.Pipeline.Workspace.Root = workspace

View File

@@ -304,17 +304,9 @@ func TestConfigCommandsInspectMaintainedSplitBundleWithoutRuntimeState(t *testin
func writeInspectionProfiles(t *testing.T, pipelinePath, production, testing string) { func writeInspectionProfiles(t *testing.T, pipelinePath, production, testing string) {
t.Helper() t.Helper()
data, err := os.ReadFile(pipelinePath) profilesDir := filepath.Join(filepath.Dir(pipelinePath), "profiles")
if err != nil { mustWriteTestFile(t, filepath.Join(profilesDir, "production.yml"), production)
t.Fatal(err) mustWriteTestFile(t, filepath.Join(profilesDir, "testing.yml"), testing)
}
composition := "composition:\n default_profile: production\n profiles:\n production:\n overlay: production.yml\n testing:\n overlay: testing.yml\n"
if err := os.WriteFile(pipelinePath, append([]byte(composition), data...), 0o644); err != nil {
t.Fatal(err)
}
dir := filepath.Dir(pipelinePath)
mustWriteTestFile(t, filepath.Join(dir, "production.yml"), production)
mustWriteTestFile(t, filepath.Join(dir, "testing.yml"), testing)
} }
func TestConfigCommandsSelectProfilesAndCampaigns(t *testing.T) { func TestConfigCommandsSelectProfilesAndCampaigns(t *testing.T) {
@@ -326,7 +318,7 @@ func TestConfigCommandsSelectProfilesAndCampaigns(t *testing.T) {
if err := ConfigValidate(context.Background(), []string{"--config", pipelinePath}, &defaultOut); err != nil { if err := ConfigValidate(context.Background(), []string{"--config", pipelinePath}, &defaultOut); err != nil {
t.Fatalf("default ConfigValidate() error = %v", err) t.Fatalf("default ConfigValidate() error = %v", err)
} }
if !strings.Contains(defaultOut.String(), "profile=production") { if !strings.Contains(defaultOut.String(), "profile=none") {
t.Fatalf("default output = %q", defaultOut.String()) t.Fatalf("default output = %q", defaultOut.String())
} }
@@ -398,8 +390,9 @@ func TestConfigValidateReportsCanonicalPartyAndImportErrors(t *testing.T) {
dir := t.TempDir() dir := t.TempDir()
rootPath := filepath.Join(dir, "pipeline.yml") rootPath := filepath.Join(dir, "pipeline.yml")
mustWriteTestFile(t, rootPath, `composition: mustWriteTestFile(t, rootPath, `configkit:
imports: [conf.yml] imports:
- path: conf.yml
workspace: workspace:
root: /one root: /one
whisperx: whisperx:
@@ -407,7 +400,7 @@ whisperx:
`) `)
mustWriteTestFile(t, filepath.Join(dir, "conf.yml"), "workspace:\n root: /two\n") mustWriteTestFile(t, filepath.Join(dir, "conf.yml"), "workspace:\n root: /two\n")
err := ConfigValidate(context.Background(), []string{"--config", rootPath}, io.Discard) err := ConfigValidate(context.Background(), []string{"--config", rootPath}, io.Discard)
if err == nil || !strings.Contains(err.Error(), "workspace.root") || !strings.Contains(err.Error(), "pipeline.yml") || !strings.Contains(err.Error(), "conf.yml") { if err == nil || !strings.Contains(err.Error(), "/workspace/root") || !strings.Contains(err.Error(), "pipeline.yml") || !strings.Contains(err.Error(), "conf.yml") {
t.Fatalf("import diagnostic = %v", err) t.Fatalf("import diagnostic = %v", err)
} }
} }
@@ -432,7 +425,7 @@ func TestConfigSourcesReportsStableOwnership(t *testing.T) {
for _, want := range []string{ for _, want := range []string{
"root: ", "root: ",
"imports: none", "imports: none",
"profile: name=testing selection=cli overlay=", "profile: name=testing selection=explicit overlay=",
"campaign: id=sample-campaign source=", "campaign: id=sample-campaign source=",
"party: source=", "party: source=",
"digest: ", "digest: ",
@@ -475,15 +468,7 @@ func TestConfigSourcesReportsCanonicalFamilyAndPublishOrigins(t *testing.T) {
func TestConfigSourcesPreservesProfileOwnershipThroughFamilyExpansion(t *testing.T) { func TestConfigSourcesPreservesProfileOwnershipThroughFamilyExpansion(t *testing.T) {
pipelinePath, campaignPath := writeInspectionFamilyConfig(t) pipelinePath, campaignPath := writeInspectionFamilyConfig(t)
data, err := os.ReadFile(pipelinePath) profilePath := filepath.Join(filepath.Dir(pipelinePath), "profiles", "testing.yml")
if err != nil {
t.Fatal(err)
}
profilePath := filepath.Join(filepath.Dir(pipelinePath), "testing.yml")
composition := "composition:\n default_profile: testing\n profiles:\n testing:\n overlay: testing.yml\n"
if err := os.WriteFile(pipelinePath, append([]byte(composition), data...), 0o644); err != nil {
t.Fatal(err)
}
mustWriteTestFile(t, profilePath, `scriptorium: mustWriteTestFile(t, profilePath, `scriptorium:
artifact_families: artifact_families:
character_note: character_note:
@@ -492,7 +477,7 @@ func TestConfigSourcesPreservesProfileOwnershipThroughFamilyExpansion(t *testing
`) `)
var out bytes.Buffer var out bytes.Buffer
if err := ConfigSources(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath}, &out); err != nil { if err := ConfigSources(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath, "--profile", "testing"}, &out); err != nil {
t.Fatalf("ConfigSources() error = %v", err) t.Fatalf("ConfigSources() error = %v", err)
} }
want := "scriptorium.artifacts.character_note_arannis.enabled\tfamily\t" + profilePath want := "scriptorium.artifacts.character_note_arannis.enabled\tfamily\t" + profilePath
@@ -505,13 +490,10 @@ func TestConfigSourcesReportsRootImportProfileAndDefaultOwnership(t *testing.T)
dir := t.TempDir() dir := t.TempDir()
rootPath := filepath.Join(dir, "pipeline.yml") rootPath := filepath.Join(dir, "pipeline.yml")
importPath := filepath.Join(dir, "base.yml") importPath := filepath.Join(dir, "base.yml")
profilePath := filepath.Join(dir, "testing.yml") profilePath := filepath.Join(dir, "profiles", "testing.yml")
mustWriteTestFile(t, rootPath, `composition: mustWriteTestFile(t, rootPath, `configkit:
imports: [base.yml] imports:
default_profile: testing - path: base.yml
profiles:
testing:
overlay: testing.yml
workspace: workspace:
root: /srv/narratio root: /srv/narratio
whisperx: whisperx:
@@ -521,13 +503,13 @@ whisperx:
mustWriteTestFile(t, profilePath, "whisperx:\n language: fr\n") mustWriteTestFile(t, profilePath, "whisperx:\n language: fr\n")
var out bytes.Buffer var out bytes.Buffer
if err := ConfigSources(context.Background(), []string{"--config", rootPath}, &out); err != nil { if err := ConfigSources(context.Background(), []string{"--config", rootPath, "--profile", "testing"}, &out); err != nil {
t.Fatalf("ConfigSources() error = %v", err) t.Fatalf("ConfigSources() error = %v", err)
} }
output := out.String() output := out.String()
for _, want := range []string{ for _, want := range []string{
"import: " + importPath, "import: " + importPath,
"profile: name=testing selection=default overlay=" + profilePath, "profile: name=testing selection=explicit overlay=" + profilePath,
"workspace.root\troot\t" + rootPath, "workspace.root\troot\t" + rootPath,
"storage.backend\timport\t" + importPath, "storage.backend\timport\t" + importPath,
"whisperx.language\tprofile\t" + profilePath, "whisperx.language\tprofile\t" + profilePath,

View File

@@ -37,10 +37,10 @@ func TestRunProfileSelectionFlowsThroughSharedLoader(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
if language != "en" { if language != "en" {
t.Fatalf("default profile language = %q, want en", language) t.Fatalf("base configuration language = %q, want en", language)
} }
for _, profile := range []string{"unknown", ""} { for _, profile := range []string{"unknown"} {
err := runWithPipelineExecutor(context.Background(), []string{ err := runWithPipelineExecutor(context.Background(), []string{
"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--profile", profile, "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--profile", profile,
}, io.Discard, executor) }, io.Discard, executor)
@@ -48,19 +48,20 @@ func TestRunProfileSelectionFlowsThroughSharedLoader(t *testing.T) {
t.Fatalf("profile %q error = %v, want selection rejection", profile, err) t.Fatalf("profile %q error = %v, want selection rejection", profile, err)
} }
} }
if err := runWithPipelineExecutor(context.Background(), []string{
"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--profile", "",
}, io.Discard, executor); err != nil {
t.Fatalf("explicit empty profile should select base: %v", err)
}
} }
func enableCommandTestProfiles(t *testing.T, pipelinePath string) { func enableCommandTestProfiles(t *testing.T, pipelinePath string) {
t.Helper() t.Helper()
data, err := os.ReadFile(pipelinePath) dir := filepath.Join(filepath.Dir(pipelinePath), "profiles")
if err != nil { if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err) t.Fatal(err)
} }
composition := "composition:\n default_profile: production\n profiles:\n production:\n overlay: production.yml\n testing:\n overlay: testing.yml\n"
if err := os.WriteFile(pipelinePath, append([]byte(composition), data...), 0o644); err != nil {
t.Fatal(err)
}
dir := filepath.Dir(pipelinePath)
if err := os.WriteFile(filepath.Join(dir, "production.yml"), []byte("whisperx:\n language: en\n"), 0o644); err != nil { if err := os.WriteFile(filepath.Join(dir, "production.yml"), []byte("whisperx:\n language: en\n"), 0o644); err != nil {
t.Fatal(err) t.Fatal(err)
} }

View File

@@ -8,6 +8,15 @@ import (
"testing" "testing"
) )
func absolutePath(t *testing.T, path string) string {
t.Helper()
absolute, err := filepath.Abs(path)
if err != nil {
t.Fatal(err)
}
return absolute
}
func TestArtifactFamiliesExpandCanonicalCharactersDeterministically(t *testing.T) { func TestArtifactFamiliesExpandCanonicalCharactersDeterministically(t *testing.T) {
dir := t.TempDir() dir := t.TempDir()
writePartyResolutionFile(t, dir+"/party.yml", `schema_version: narratio.party.v1 writePartyResolutionFile(t, dir+"/party.yml", `schema_version: narratio.party.v1
@@ -148,7 +157,7 @@ func TestArtifactFamilyStrictDecodeRejectsUnknownFields(t *testing.T) {
dir := t.TempDir() dir := t.TempDir()
pipelinePath, _, _ := writeArtifactFamilyConfig(t, dir, artifactFamilyPipelineYAML(validFamilyYAML+" unsupported: true\n")) pipelinePath, _, _ := writeArtifactFamilyConfig(t, dir, artifactFamilyPipelineYAML(validFamilyYAML+" unsupported: true\n"))
_, err := LoadPipeline(pipelinePath) _, err := LoadPipeline(pipelinePath)
if err == nil || !strings.Contains(err.Error(), "field unsupported not found") { if err == nil || !strings.Contains(err.Error(), "decode.unknown_field") {
t.Fatalf("LoadPipeline() error = %v", err) t.Fatalf("LoadPipeline() error = %v", err)
} }
} }

View File

@@ -1,783 +0,0 @@
package config
import (
"bytes"
"encoding/json"
"fmt"
"io"
"sort"
"strconv"
"strings"
"gopkg.in/yaml.v3"
)
// compositionDocument is the presence-aware representation used while
// assembling pipeline configuration sources. It deliberately models YAML
// mechanics rather than duplicating PipelineConfig's field schema.
type compositionDocument struct {
root *compositionNode
sources []string
}
type compositionNode struct {
kind yaml.Kind
tag string
value string
path string
sources []string
fields []compositionField
items []*compositionNode
}
type compositionField struct {
key string
value *compositionNode
}
// compositionValueRecord is a deterministic semantic leaf projection. Lists
// are atomic configuration values, while mappings are traversed recursively.
type compositionValueRecord struct {
Path string
Kind yaml.Kind
Value string
Sources []string
}
// parseCompositionDocument parses exactly one YAML mapping while retaining
// source ownership, explicit zero values, and declaration order.
func parseCompositionDocument(source string, reader io.Reader) (*compositionDocument, error) {
if strings.TrimSpace(source) == "" {
return nil, fmt.Errorf("configuration source identity is required")
}
if reader == nil {
return nil, fmt.Errorf("configuration source %q: reader is nil", source)
}
decoder := yaml.NewDecoder(reader)
var document yaml.Node
if err := decoder.Decode(&document); err != nil {
if err == io.EOF {
return nil, fmt.Errorf("configuration source %q: document is empty", source)
}
return nil, fmt.Errorf("configuration source %q: decode YAML: %w", source, err)
}
var trailing yaml.Node
if err := decoder.Decode(&trailing); err == nil {
return nil, fmt.Errorf("configuration source %q: must contain exactly one YAML document", source)
} else if err != io.EOF {
return nil, fmt.Errorf("configuration source %q: decode trailing YAML: %w", source, err)
}
if document.Kind != yaml.DocumentNode || len(document.Content) != 1 {
return nil, fmt.Errorf("configuration source %q: must contain exactly one YAML document", source)
}
if document.Content[0].Kind != yaml.MappingNode {
return nil, fmt.Errorf(
"configuration source %q: top-level document must be a mapping, got %s",
source,
yamlKindName(document.Content[0].Kind),
)
}
root, err := buildCompositionNode(document.Content[0], source, "")
if err != nil {
return nil, err
}
return &compositionDocument{root: root, sources: []string{source}}, nil
}
func parseCompositionBytes(source string, data []byte) (*compositionDocument, error) {
return parseCompositionDocument(source, bytes.NewReader(data))
}
func buildCompositionNode(node *yaml.Node, source, path string) (*compositionNode, error) {
if node == nil {
return nil, fmt.Errorf("configuration source %q at %s: YAML node is nil", source, displayCompositionPath(path))
}
if node.Kind == yaml.AliasNode {
return nil, compositionNodeError(source, path, node, "YAML aliases are not supported because source ownership would be ambiguous")
}
result := &compositionNode{
kind: node.Kind, tag: node.Tag, value: node.Value, path: path,
sources: []string{source},
}
switch node.Kind {
case yaml.MappingNode:
if len(node.Content)%2 != 0 {
return nil, compositionNodeError(source, path, node, "mapping has an incomplete key/value pair")
}
seen := make(map[string]*yaml.Node, len(node.Content)/2)
for index := 0; index < len(node.Content); index += 2 {
keyNode := node.Content[index]
valueNode := node.Content[index+1]
if keyNode.Kind == yaml.AliasNode {
return nil, compositionNodeError(source, path, keyNode, "YAML aliases are not supported because source ownership would be ambiguous")
}
if valueNode.Kind == yaml.AliasNode {
return nil, compositionNodeError(source, appendCompositionPath(path, keyNode.Value), valueNode, "YAML aliases are not supported because source ownership would be ambiguous")
}
if keyNode.Kind != yaml.ScalarNode || keyNode.Tag != "!!str" {
return nil, compositionNodeError(source, path, keyNode, "mapping keys must be strings")
}
key := keyNode.Value
fieldPath := appendCompositionPath(path, key)
if prior, duplicate := seen[key]; duplicate {
return nil, fmt.Errorf(
"configuration source %q at %s: duplicate YAML key %q (first declared at line %d, column %d; repeated at line %d, column %d)",
source, displayCompositionPath(fieldPath), key,
prior.Line, prior.Column, keyNode.Line, keyNode.Column,
)
}
seen[key] = keyNode
child, err := buildCompositionNode(valueNode, source, fieldPath)
if err != nil {
return nil, err
}
result.fields = append(result.fields, compositionField{key: key, value: child})
}
case yaml.SequenceNode:
for index, childNode := range node.Content {
childPath := fmt.Sprintf("%s[%d]", path, index)
child, err := buildCompositionNode(childNode, source, childPath)
if err != nil {
return nil, err
}
result.items = append(result.items, child)
}
case yaml.ScalarNode:
// Scalar tag and lexical value retain distinctions such as explicit
// false, zero, an empty string, and null until final strict decoding.
default:
return nil, compositionNodeError(source, path, node, fmt.Sprintf("unsupported YAML node kind %s", yamlKindName(node.Kind)))
}
return result, nil
}
// mergeAdditiveComposition recursively combines disjoint mappings. A scalar,
// list, or final keyed value may have only one base owner, regardless of
// whether duplicate values happen to be equal.
func mergeAdditiveComposition(base, incoming *compositionDocument) (*compositionDocument, error) {
return mergeAdditiveCompositions(base, incoming)
}
// mergeAdditiveCompositions validates the complete base source set before
// merging so a conflict names every source that claims the same final path.
func mergeAdditiveCompositions(documents ...*compositionDocument) (*compositionDocument, error) {
if len(documents) == 0 {
return nil, fmt.Errorf("configuration additive base merge requires at least one document")
}
for index, document := range documents {
if err := validateCompositionDocument(document, fmt.Sprintf("base[%d]", index)); err != nil {
return nil, err
}
}
if err := validateAdditiveClaims(documents); err != nil {
return nil, err
}
result := &compositionDocument{
root: cloneCompositionNode(documents[0].root),
sources: append([]string(nil), documents[0].sources...),
}
for _, incoming := range documents[1:] {
merged, err := mergeAdditiveNodes(result.root, incoming.root)
if err != nil {
return nil, err
}
result.root = merged
result.sources = appendUniqueStrings(result.sources, incoming.sources...)
}
return result, nil
}
func validateAdditiveClaims(documents []*compositionDocument) error {
claims := make(map[string][]*compositionNode)
for _, document := range documents {
appendCompositionClaims(document.root, claims)
}
paths := make([]string, 0, len(claims))
for path := range claims {
paths = append(paths, path)
}
sort.Slice(paths, func(i, j int) bool {
leftDepth := compositionPathDepth(paths[i])
rightDepth := compositionPathDepth(paths[j])
if leftDepth != rightDepth {
return leftDepth < rightDepth
}
return paths[i] < paths[j]
})
for _, path := range paths {
values := claims[path]
if len(values) < 2 {
continue
}
allPopulatedMappings := true
for _, value := range values {
if value.kind != yaml.MappingNode || len(value.fields) == 0 {
allPopulatedMappings = false
break
}
}
if !allPopulatedMappings {
return newCompositionConflict("additive base merge", path, values...)
}
}
return nil
}
func appendCompositionClaims(node *compositionNode, claims map[string][]*compositionNode) {
if node == nil {
return
}
if node.path != "" {
claims[node.path] = append(claims[node.path], node)
}
for _, field := range node.fields {
appendCompositionClaims(field.value, claims)
}
}
func compositionPathDepth(path string) int {
if path == "" {
return 0
}
return strings.Count(path, ".") + strings.Count(path, "[") + 1
}
func mergeAdditiveNodes(base, incoming *compositionNode) (*compositionNode, error) {
if base.kind != yaml.MappingNode || incoming.kind != yaml.MappingNode {
return nil, newCompositionConflict("additive base merge", base.path, base, incoming)
}
base.sources = appendUniqueStrings(base.sources, incoming.sources...)
for _, incomingField := range incoming.fields {
index := compositionFieldIndex(base.fields, incomingField.key)
if index < 0 {
field := cloneCompositionField(incomingField)
base.fields = append(base.fields, field)
continue
}
baseValue := base.fields[index].value
incomingValue := incomingField.value
if baseValue.kind == yaml.MappingNode && incomingValue.kind == yaml.MappingNode {
if len(baseValue.fields) == 0 || len(incomingValue.fields) == 0 {
return nil, newCompositionConflict("additive base merge", incomingValue.path, baseValue, incomingValue)
}
merged, err := mergeAdditiveNodes(baseValue, incomingValue)
if err != nil {
return nil, err
}
base.fields[index].value = merged
continue
}
return nil, newCompositionConflict("additive base merge", incomingValue.path, baseValue, incomingValue)
}
return base, nil
}
// mergeOverlayComposition applies the sole overwrite layer. Mappings merge
// recursively; same-kind scalars and lists replace; null and kind changes are
// rejected.
func mergeOverlayComposition(base, overlay *compositionDocument) (*compositionDocument, error) {
if err := validateCompositionDocument(base, "base"); err != nil {
return nil, err
}
if err := validateCompositionDocument(overlay, "overlay"); err != nil {
return nil, err
}
if null := firstNullCompositionNode(overlay.root); null != nil {
return nil, fmt.Errorf(
"configuration overlay at %s from %s: null cannot delete an effective value",
displayCompositionPath(null.path), formatCompositionSources(null.sources),
)
}
merged, err := mergeOverlayNodes(cloneCompositionNode(base.root), overlay.root)
if err != nil {
return nil, err
}
return &compositionDocument{
root: merged,
sources: appendUniqueStrings(
append([]string(nil), base.sources...), overlay.sources...,
),
}, nil
}
func mergeOverlayNodes(base, overlay *compositionNode) (*compositionNode, error) {
if base.kind != yaml.MappingNode || overlay.kind != yaml.MappingNode {
return nil, newCompositionConflict("profile overlay", base.path, base, overlay)
}
base.sources = appendUniqueStrings(base.sources, overlay.sources...)
for _, overlayField := range overlay.fields {
index := compositionFieldIndex(base.fields, overlayField.key)
if index < 0 {
field := cloneCompositionField(overlayField)
base.fields = append(base.fields, field)
continue
}
baseValue := base.fields[index].value
overlayValue := overlayField.value
if baseValue.kind != overlayValue.kind {
return nil, newCompositionConflict("profile overlay kind change", overlayValue.path, baseValue, overlayValue)
}
if baseValue.kind == yaml.MappingNode {
merged, err := mergeOverlayNodes(baseValue, overlayValue)
if err != nil {
return nil, err
}
base.fields[index].value = merged
continue
}
base.fields[index].value = cloneCompositionNode(overlayValue)
}
return base, nil
}
// canonicalYAML renders the effective mapping with sorted keys and normalized
// presentation while retaining sequence order and scalar YAML types.
func (document *compositionDocument) canonicalYAML() ([]byte, error) {
if err := validateCompositionDocument(document, "document"); err != nil {
return nil, err
}
root, err := compositionYAMLNode(document.root)
if err != nil {
return nil, err
}
var buffer bytes.Buffer
encoder := yaml.NewEncoder(&buffer)
encoder.SetIndent(2)
if err := encoder.Encode(root); err != nil {
return nil, fmt.Errorf("render effective configuration YAML: %w", err)
}
if err := encoder.Close(); err != nil {
return nil, fmt.Errorf("finish effective configuration YAML: %w", err)
}
return buffer.Bytes(), nil
}
// canonicalDigestInput provides a deterministic, formatting-independent byte
// representation for later secret-free effective configuration digesting.
func (document *compositionDocument) canonicalDigestInput() ([]byte, error) {
if err := validateCompositionDocument(document, "document"); err != nil {
return nil, err
}
value, err := canonicalCompositionValue(document.root)
if err != nil {
return nil, err
}
data, err := json.Marshal(value)
if err != nil {
return nil, fmt.Errorf("serialize canonical configuration digest input: %w", err)
}
return append(data, '\n'), nil
}
// semanticRecords returns sorted atomic values for future effective diff and
// source-report projections. A caller receives copies of all source slices.
func (document *compositionDocument) semanticRecords() ([]compositionValueRecord, error) {
if err := validateCompositionDocument(document, "document"); err != nil {
return nil, err
}
var records []compositionValueRecord
if err := appendCompositionRecords(document.root, &records); err != nil {
return nil, err
}
sort.Slice(records, func(i, j int) bool { return records[i].Path < records[j].Path })
return records, nil
}
// compactSemanticRecords returns the same logical atomic paths as
// semanticRecords, but represents values as compact JSON-compatible YAML
// values rather than the typed structural form used for digesting. It is the
// stable human-facing projection for semantic comparisons.
func (document *compositionDocument) compactSemanticRecords() ([]compositionValueRecord, error) {
if err := validateCompositionDocument(document, "document"); err != nil {
return nil, err
}
var records []compositionValueRecord
if err := appendCompactCompositionRecords(document.root, &records); err != nil {
return nil, err
}
sort.Slice(records, func(i, j int) bool { return records[i].Path < records[j].Path })
return records, nil
}
func appendCompositionRecords(node *compositionNode, records *[]compositionValueRecord) error {
if node.kind == yaml.MappingNode && len(node.fields) > 0 {
for _, field := range node.fields {
if err := appendCompositionRecords(field.value, records); err != nil {
return err
}
}
return nil
}
value, err := canonicalCompositionValue(node)
if err != nil {
return err
}
encoded, err := json.Marshal(value)
if err != nil {
return fmt.Errorf("serialize configuration value at %s: %w", displayCompositionPath(node.path), err)
}
*records = append(*records, compositionValueRecord{
Path: node.path, Kind: node.kind, Value: string(encoded),
Sources: append([]string(nil), node.sources...),
})
return nil
}
func appendCompactCompositionRecords(node *compositionNode, records *[]compositionValueRecord) error {
if node.kind == yaml.MappingNode && len(node.fields) > 0 {
for _, field := range node.fields {
if err := appendCompactCompositionRecords(field.value, records); err != nil {
return err
}
}
return nil
}
value, err := compactCompositionValue(node)
if err != nil {
return err
}
encoded, err := json.Marshal(value)
if err != nil {
return fmt.Errorf("serialize configuration value at %s: %w", displayCompositionPath(node.path), err)
}
*records = append(*records, compositionValueRecord{
Path: node.path, Kind: node.kind, Value: string(encoded),
Sources: append([]string(nil), node.sources...),
})
return nil
}
func compactCompositionValue(node *compositionNode) (any, error) {
switch node.kind {
case yaml.MappingNode:
values := make(map[string]any, len(node.fields))
for _, field := range node.fields {
value, err := compactCompositionValue(field.value)
if err != nil {
return nil, err
}
values[field.key] = value
}
return values, nil
case yaml.SequenceNode:
values := make([]any, 0, len(node.items))
for _, item := range node.items {
value, err := compactCompositionValue(item)
if err != nil {
return nil, err
}
values = append(values, value)
}
return values, nil
case yaml.ScalarNode:
return canonicalScalarValue(node)
default:
return nil, fmt.Errorf("configuration at %s has unsupported YAML kind %s", displayCompositionPath(node.path), yamlKindName(node.kind))
}
}
type canonicalCompositionField struct {
Key string `json:"key"`
Value any `json:"value"`
}
type canonicalCompositionNode struct {
Kind string `json:"kind"`
Tag string `json:"tag,omitempty"`
Value any `json:"value,omitempty"`
Fields []canonicalCompositionField `json:"fields,omitempty"`
Items []any `json:"items,omitempty"`
}
func canonicalCompositionValue(node *compositionNode) (any, error) {
switch node.kind {
case yaml.MappingNode:
fields := append([]compositionField(nil), node.fields...)
sort.Slice(fields, func(i, j int) bool { return fields[i].key < fields[j].key })
result := canonicalCompositionNode{Kind: "mapping"}
if len(fields) == 0 {
result.Fields = []canonicalCompositionField{}
}
for _, field := range fields {
value, err := canonicalCompositionValue(field.value)
if err != nil {
return nil, err
}
result.Fields = append(result.Fields, canonicalCompositionField{Key: field.key, Value: value})
}
return result, nil
case yaml.SequenceNode:
result := canonicalCompositionNode{Kind: "sequence", Items: make([]any, 0, len(node.items))}
for _, item := range node.items {
value, err := canonicalCompositionValue(item)
if err != nil {
return nil, err
}
result.Items = append(result.Items, value)
}
return result, nil
case yaml.ScalarNode:
value, err := canonicalScalarValue(node)
if err != nil {
return nil, err
}
return canonicalCompositionNode{Kind: "scalar", Tag: node.tag, Value: value}, nil
default:
return nil, fmt.Errorf("configuration at %s has unsupported YAML kind %s", displayCompositionPath(node.path), yamlKindName(node.kind))
}
}
func canonicalScalarValue(node *compositionNode) (any, error) {
raw := &yaml.Node{Kind: yaml.ScalarNode, Tag: node.tag, Value: node.value}
var value any
if err := raw.Decode(&value); err != nil {
return nil, fmt.Errorf("decode scalar at %s: %w", displayCompositionPath(node.path), err)
}
switch typed := value.(type) {
case nil, bool, string, int, int64, uint64, float64:
return typed, nil
default:
// yaml.v3 may decode timestamps or uncommon scalar tags into types that
// encoding/json can serialize deterministically. Preserve the resolved
// tag alongside the value in the containing canonical node.
return typed, nil
}
}
func compositionYAMLNode(node *compositionNode) (*yaml.Node, error) {
switch node.kind {
case yaml.MappingNode:
result := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
fields := append([]compositionField(nil), node.fields...)
sort.Slice(fields, func(i, j int) bool { return fields[i].key < fields[j].key })
for _, field := range fields {
value, err := compositionYAMLNode(field.value)
if err != nil {
return nil, err
}
result.Content = append(result.Content,
&yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: field.key},
value,
)
}
return result, nil
case yaml.SequenceNode:
result := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
for _, item := range node.items {
value, err := compositionYAMLNode(item)
if err != nil {
return nil, err
}
result.Content = append(result.Content, value)
}
return result, nil
case yaml.ScalarNode:
return normalizedCompositionScalarNode(node)
default:
return nil, fmt.Errorf("configuration at %s has unsupported YAML kind %s", displayCompositionPath(node.path), yamlKindName(node.kind))
}
}
func normalizedCompositionScalarNode(node *compositionNode) (*yaml.Node, error) {
raw := &yaml.Node{Kind: yaml.ScalarNode, Tag: node.tag, Value: node.value}
var value any
if err := raw.Decode(&value); err != nil {
return nil, fmt.Errorf("normalize scalar at %s: %w", displayCompositionPath(node.path), err)
}
normalized := &yaml.Node{}
if err := normalized.Encode(value); err != nil {
return nil, fmt.Errorf("encode normalized scalar at %s: %w", displayCompositionPath(node.path), err)
}
if normalized.Kind != yaml.ScalarNode {
return nil, fmt.Errorf("normalize scalar at %s produced YAML kind %s", displayCompositionPath(node.path), yamlKindName(normalized.Kind))
}
return normalized, nil
}
func validateCompositionDocument(document *compositionDocument, role string) error {
if document == nil || document.root == nil {
return fmt.Errorf("configuration composition %s document is nil", role)
}
if document.root.kind != yaml.MappingNode {
return fmt.Errorf("configuration composition %s root must be a mapping", role)
}
return nil
}
func firstNullCompositionNode(node *compositionNode) *compositionNode {
if node == nil {
return nil
}
if node.kind == yaml.ScalarNode && node.tag == "!!null" {
return node
}
for _, field := range node.fields {
if found := firstNullCompositionNode(field.value); found != nil {
return found
}
}
for _, item := range node.items {
if found := firstNullCompositionNode(item); found != nil {
return found
}
}
return nil
}
func cloneCompositionNode(node *compositionNode) *compositionNode {
if node == nil {
return nil
}
clone := &compositionNode{
kind: node.kind, tag: node.tag, value: node.value, path: node.path,
sources: append([]string(nil), node.sources...),
}
for _, field := range node.fields {
clone.fields = append(clone.fields, cloneCompositionField(field))
}
for _, item := range node.items {
clone.items = append(clone.items, cloneCompositionNode(item))
}
return clone
}
func cloneCompositionField(field compositionField) compositionField {
return compositionField{key: field.key, value: cloneCompositionNode(field.value)}
}
func compositionFieldIndex(fields []compositionField, key string) int {
for index := range fields {
if fields[index].key == key {
return index
}
}
return -1
}
func appendCompositionPath(parent, key string) string {
if isSimpleCompositionPathSegment(key) {
if parent == "" {
return key
}
return parent + "." + key
}
if parent == "" {
return "[" + strconv.Quote(key) + "]"
}
return parent + "[" + strconv.Quote(key) + "]"
}
func isSimpleCompositionPathSegment(value string) bool {
if value == "" {
return false
}
for index, char := range value {
if (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || char == '_' || (index > 0 && char >= '0' && char <= '9') || (index > 0 && char == '-') {
continue
}
return false
}
return true
}
func displayCompositionPath(path string) string {
if path == "" {
return "<root>"
}
return path
}
func compositionNodeError(source, path string, node *yaml.Node, message string) error {
line, column := 0, 0
if node != nil {
line, column = node.Line, node.Column
}
return fmt.Errorf(
"configuration source %q at %s (line %d, column %d): %s",
source, displayCompositionPath(path), line, column, message,
)
}
func newCompositionConflict(operation, path string, values ...*compositionNode) error {
var sources []string
var kinds []string
for _, value := range values {
if value == nil {
continue
}
sources = appendUniqueStrings(sources, compositionClaimSources(value)...)
kind := yamlKindName(value.kind)
if !containsString(kinds, kind) {
kinds = append(kinds, kind)
}
}
return fmt.Errorf(
"configuration %s conflict at %s: claimed by %s (YAML kinds: %s)",
operation, displayCompositionPath(path), formatCompositionSources(sources), strings.Join(kinds, ", "),
)
}
func compositionClaimSources(node *compositionNode) []string {
if node == nil {
return nil
}
sources := append([]string(nil), node.sources...)
for _, field := range node.fields {
sources = appendUniqueStrings(sources, compositionClaimSources(field.value)...)
}
for _, item := range node.items {
sources = appendUniqueStrings(sources, compositionClaimSources(item)...)
}
return sources
}
func appendUniqueStrings(values []string, additions ...string) []string {
seen := make(map[string]struct{}, len(values)+len(additions))
result := make([]string, 0, len(values)+len(additions))
for _, value := range append(append([]string(nil), values...), additions...) {
if _, exists := seen[value]; exists {
continue
}
seen[value] = struct{}{}
result = append(result, value)
}
return result
}
func containsString(values []string, target string) bool {
for _, value := range values {
if value == target {
return true
}
}
return false
}
func formatCompositionSources(sources []string) string {
quoted := make([]string, 0, len(sources))
for _, source := range sources {
quoted = append(quoted, strconv.Quote(source))
}
return strings.Join(quoted, ", ")
}
func yamlKindName(kind yaml.Kind) string {
switch kind {
case yaml.DocumentNode:
return "document"
case yaml.MappingNode:
return "mapping"
case yaml.SequenceNode:
return "sequence"
case yaml.ScalarNode:
return "scalar"
case yaml.AliasNode:
return "alias"
default:
return fmt.Sprintf("kind(%d)", kind)
}
}

View File

@@ -1,353 +0,0 @@
package config
import (
"reflect"
"strings"
"testing"
"gopkg.in/yaml.v3"
)
func TestParseCompositionDocumentRetainsPresenceOwnershipAndDeclarationOrder(t *testing.T) {
document := mustParseComposition(t, "root.yml", `zeta: false
zero: 0
empty_map: {}
empty_list: []
nested:
value: ""
`)
wantOrder := []string{"zeta", "zero", "empty_map", "empty_list", "nested"}
gotOrder := make([]string, 0, len(document.root.fields))
for _, field := range document.root.fields {
gotOrder = append(gotOrder, field.key)
}
if !reflect.DeepEqual(gotOrder, wantOrder) {
t.Fatalf("declaration order = %#v, want %#v", gotOrder, wantOrder)
}
tests := []struct {
path string
kind yaml.Kind
tag string
value string
}{
{path: "zeta", kind: yaml.ScalarNode, tag: "!!bool", value: "false"},
{path: "zero", kind: yaml.ScalarNode, tag: "!!int", value: "0"},
{path: "empty_map", kind: yaml.MappingNode, tag: "!!map"},
{path: "empty_list", kind: yaml.SequenceNode, tag: "!!seq"},
{path: "nested.value", kind: yaml.ScalarNode, tag: "!!str", value: ""},
}
for _, tt := range tests {
node := compositionNodeAtPath(t, document.root, tt.path)
if node.kind != tt.kind || node.tag != tt.tag || node.value != tt.value || !reflect.DeepEqual(node.sources, []string{"root.yml"}) {
t.Fatalf("node %s = kind=%v tag=%q value=%q sources=%#v", tt.path, node.kind, node.tag, node.value, node.sources)
}
}
}
func TestParseCompositionDocumentRejectsAmbiguousOrMalformedYAML(t *testing.T) {
tests := []struct {
name string
yaml string
want string
}{
{name: "duplicate top-level key", yaml: "value: 1\nvalue: 2\n", want: "duplicate YAML key"},
{name: "duplicate nested key", yaml: "outer:\n value: 1\n value: 2\n", want: "outer.value"},
{name: "alias", yaml: "base: &base\n value: 1\ncopy: *base\n", want: "aliases are not supported"},
{name: "trailing document", yaml: "value: 1\n---\nvalue: 2\n", want: "exactly one YAML document"},
{name: "top-level sequence", yaml: "- value\n", want: "top-level document must be a mapping"},
{name: "non-string key", yaml: "1: value\n", want: "mapping keys must be strings"},
{name: "malformed", yaml: "outer: [\n", want: "decode YAML"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := parseCompositionDocument("broken.yml", strings.NewReader(tt.yaml))
if err == nil {
t.Fatal("parseCompositionDocument() error = nil")
}
if !strings.Contains(err.Error(), "broken.yml") || !strings.Contains(err.Error(), tt.want) {
t.Fatalf("error = %q, want source and %q", err, tt.want)
}
})
}
}
func TestMergeAdditiveCompositionJoinsOnlyDisjointMappings(t *testing.T) {
root := mustParseComposition(t, "pipeline.yml", `scriptorium:
artifacts:
recap:
enabled: true
zero: 0
`)
imports := mustParseComposition(t, "conf.d/artifacts.yml", `scriptorium:
artifacts:
handout:
enabled: false
empty_list: []
`)
merged, err := mergeAdditiveComposition(root, imports)
if err != nil {
t.Fatalf("mergeAdditiveComposition() error = %v", err)
}
for _, path := range []string{
"empty_list", "scriptorium.artifacts.handout.enabled",
"scriptorium.artifacts.recap.enabled", "zero",
} {
_ = compositionNodeAtPath(t, merged.root, path)
}
if got := compositionNodeAtPath(t, merged.root, "scriptorium.artifacts.handout.enabled").sources; !reflect.DeepEqual(got, []string{"conf.d/artifacts.yml"}) {
t.Fatalf("handout sources = %#v", got)
}
if got := compositionNodeAtPath(t, merged.root, "scriptorium.artifacts.recap.enabled").sources; !reflect.DeepEqual(got, []string{"pipeline.yml"}) {
t.Fatalf("recap sources = %#v", got)
}
// Merge operations return a new document and retain the declared order in
// each input for source-aware diagnostics.
if len(root.root.fields) != 2 || len(imports.root.fields) != 2 {
t.Fatalf("merge mutated inputs: root=%d import=%d", len(root.root.fields), len(imports.root.fields))
}
}
func TestMergeAdditiveCompositionRejectsEveryDuplicateClass(t *testing.T) {
tests := []struct {
name string
baseYAML string
nextYAML string
path string
}{
{name: "equal scalar", baseYAML: "value: true\n", nextYAML: "value: true\n", path: "value"},
{name: "different scalar", baseYAML: "value: true\n", nextYAML: "value: false\n", path: "value"},
{name: "atomic list", baseYAML: "values: [one]\n", nextYAML: "values: [two]\n", path: "values"},
{name: "keyed entry", baseYAML: "items:\n shared:\n left: 1\n", nextYAML: "items:\n shared:\n left: 2\n", path: "items.shared.left"},
{name: "kind conflict", baseYAML: "value:\n nested: true\n", nextYAML: "value: scalar\n", path: "value"},
{name: "duplicate empty map", baseYAML: "value: {}\n", nextYAML: "value: {}\n", path: "value"},
{name: "empty map then populated map", baseYAML: "value: {}\n", nextYAML: "value: {nested: true}\n", path: "value"},
{name: "populated map then empty map", baseYAML: "value: {nested: true}\n", nextYAML: "value: {}\n", path: "value"},
{name: "duplicate empty list", baseYAML: "value: []\n", nextYAML: "value: []\n", path: "value"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
base := mustParseComposition(t, "base.yml", tt.baseYAML)
next := mustParseComposition(t, "next.yml", tt.nextYAML)
_, err := mergeAdditiveComposition(base, next)
if err == nil {
t.Fatal("mergeAdditiveComposition() error = nil")
}
for _, want := range []string{tt.path, "base.yml", "next.yml"} {
if !strings.Contains(err.Error(), want) {
t.Fatalf("error = %q, want %q", err, want)
}
}
})
}
}
func TestMergeAdditiveCompositionReportsAllClaimingSources(t *testing.T) {
left := mustParseComposition(t, "left.yml", "group:\n left: 1\n")
right := mustParseComposition(t, "right.yml", "group:\n right: 2\n")
base, err := mergeAdditiveComposition(left, right)
if err != nil {
t.Fatal(err)
}
overlap := mustParseComposition(t, "overlap.yml", "group: scalar\n")
_, err = mergeAdditiveComposition(base, overlap)
if err == nil {
t.Fatal("mergeAdditiveComposition() error = nil")
}
for _, want := range []string{"group", "left.yml", "right.yml", "overlap.yml"} {
if !strings.Contains(err.Error(), want) {
t.Fatalf("error = %q, want %q", err, want)
}
}
_, err = mergeAdditiveCompositions(
mustParseComposition(t, "first.yml", "value: 1\n"),
mustParseComposition(t, "second.yml", "value: 2\n"),
mustParseComposition(t, "third.yml", "value: 3\n"),
)
if err == nil {
t.Fatal("mergeAdditiveCompositions() error = nil")
}
for _, want := range []string{"value", "first.yml", "second.yml", "third.yml"} {
if !strings.Contains(err.Error(), want) {
t.Fatalf("error = %q, want %q", err, want)
}
}
}
func TestMergeOverlayCompositionRecursesMapsAndReplacesAtomicValues(t *testing.T) {
base := mustParseComposition(t, "base.yml", `feature:
enabled: true
retries: 3
values: [one, two]
inherited: kept
artifacts:
recap:
enabled: true
`)
overlay := mustParseComposition(t, "testing.yml", `feature:
enabled: false
retries: 0
values: []
added: present
artifacts:
handout:
enabled: false
`)
merged, err := mergeOverlayComposition(base, overlay)
if err != nil {
t.Fatalf("mergeOverlayComposition() error = %v", err)
}
assertCompositionScalar(t, merged.root, "feature.enabled", "!!bool", "false", "testing.yml")
assertCompositionScalar(t, merged.root, "feature.retries", "!!int", "0", "testing.yml")
assertCompositionScalar(t, merged.root, "feature.inherited", "!!str", "kept", "base.yml")
assertCompositionScalar(t, merged.root, "feature.added", "!!str", "present", "testing.yml")
if values := compositionNodeAtPath(t, merged.root, "feature.values"); values.kind != yaml.SequenceNode || len(values.items) != 0 || !reflect.DeepEqual(values.sources, []string{"testing.yml"}) {
t.Fatalf("replaced list = %#v", values)
}
_ = compositionNodeAtPath(t, merged.root, "artifacts.recap.enabled")
_ = compositionNodeAtPath(t, merged.root, "artifacts.handout.enabled")
}
func TestMergeOverlayCompositionRejectsKindChangesAndNullDeletion(t *testing.T) {
kindTests := []struct {
name string
baseYAML string
overlay string
}{
{name: "map to scalar", baseYAML: "value: {nested: true}\n", overlay: "value: replacement\n"},
{name: "scalar to map", baseYAML: "value: original\n", overlay: "value: {nested: true}\n"},
{name: "list to scalar", baseYAML: "value: [one]\n", overlay: "value: replacement\n"},
{name: "scalar to list", baseYAML: "value: original\n", overlay: "value: [one]\n"},
}
for _, tt := range kindTests {
t.Run(tt.name, func(t *testing.T) {
_, err := mergeOverlayComposition(
mustParseComposition(t, "base.yml", tt.baseYAML),
mustParseComposition(t, "overlay.yml", tt.overlay),
)
if err == nil || !strings.Contains(err.Error(), "value") || !strings.Contains(err.Error(), "kind change") {
t.Fatalf("error = %v, want value kind change", err)
}
})
}
for _, overlayYAML := range []string{"value: null\n", "value: ~\n", "nested:\n value:\n"} {
_, err := mergeOverlayComposition(
mustParseComposition(t, "base.yml", "value: original\nnested:\n value: original\n"),
mustParseComposition(t, "overlay.yml", overlayYAML),
)
if err == nil || !strings.Contains(err.Error(), "null cannot delete") || !strings.Contains(err.Error(), "overlay.yml") {
t.Fatalf("error = %v, want source-qualified null rejection", err)
}
}
}
func TestCompositionCanonicalOutputsAreDeterministic(t *testing.T) {
first := mustParseComposition(t, "first.yml", `zeta: 01
alpha:
list: [true, false]
empty: {}
`)
second := mustParseComposition(t, "second.yml", `alpha:
empty: {}
list:
- true
- false
zeta: 1
`)
firstYAML, err := first.canonicalYAML()
if err != nil {
t.Fatal(err)
}
secondYAML, err := second.canonicalYAML()
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(firstYAML, secondYAML) {
t.Fatalf("canonical YAML differs:\n%s\n---\n%s", firstYAML, secondYAML)
}
firstDigest, err := first.canonicalDigestInput()
if err != nil {
t.Fatal(err)
}
secondDigest, err := second.canonicalDigestInput()
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(firstDigest, secondDigest) {
t.Fatalf("digest input differs:\n%s\n---\n%s", firstDigest, secondDigest)
}
left := mustParseComposition(t, "left.yml", "zeta: 1\n")
right := mustParseComposition(t, "right.yml", "alpha: 2\n")
leftRight, err := mergeAdditiveComposition(left, right)
if err != nil {
t.Fatal(err)
}
rightLeft, err := mergeAdditiveComposition(right, left)
if err != nil {
t.Fatal(err)
}
want, _ := leftRight.canonicalDigestInput()
got, _ := rightLeft.canonicalDigestInput()
if !reflect.DeepEqual(got, want) {
t.Fatalf("source traversal changed semantic digest input: got %s want %s", got, want)
}
wantYAML, _ := leftRight.canonicalYAML()
gotYAML, _ := rightLeft.canonicalYAML()
if !reflect.DeepEqual(gotYAML, wantYAML) {
t.Fatalf("source traversal changed canonical YAML: got %s want %s", gotYAML, wantYAML)
}
records, err := first.semanticRecords()
if err != nil {
t.Fatal(err)
}
paths := make([]string, 0, len(records))
for _, record := range records {
paths = append(paths, record.Path)
}
if wantPaths := []string{"alpha.empty", "alpha.list", "zeta"}; !reflect.DeepEqual(paths, wantPaths) {
t.Fatalf("semantic record paths = %#v, want %#v", paths, wantPaths)
}
}
func mustParseComposition(t *testing.T, source, input string) *compositionDocument {
t.Helper()
document, err := parseCompositionBytes(source, []byte(input))
if err != nil {
t.Fatalf("parseCompositionBytes(%q) error = %v", source, err)
}
return document
}
func compositionNodeAtPath(t *testing.T, root *compositionNode, path string) *compositionNode {
t.Helper()
node := root
for _, segment := range strings.Split(path, ".") {
if node == nil || node.kind != yaml.MappingNode {
t.Fatalf("path %q reached non-mapping at %q", path, segment)
}
index := compositionFieldIndex(node.fields, segment)
if index < 0 {
t.Fatalf("path %q missing segment %q", path, segment)
}
node = node.fields[index].value
}
return node
}
func assertCompositionScalar(t *testing.T, root *compositionNode, path, tag, value, source string) {
t.Helper()
node := compositionNodeAtPath(t, root, path)
if node.kind != yaml.ScalarNode || node.tag != tag || node.value != value || !reflect.DeepEqual(node.sources, []string{source}) {
t.Fatalf("%s = kind=%s tag=%q value=%q sources=%#v", path, yamlKindName(node.kind), node.tag, node.value, node.sources)
}
}

View File

@@ -191,8 +191,6 @@ type NormalizeConfig struct {
OutputPath string `yaml:"output_path" json:"output_path"` OutputPath string `yaml:"output_path" json:"output_path"`
OutputSchema string `yaml:"output_schema" json:"output_schema"` OutputSchema string `yaml:"output_schema" json:"output_schema"`
Report *bool `yaml:"report" json:"report"` Report *bool `yaml:"report" json:"report"`
outputPathSet bool `yaml:"-" json:"-"`
} }
// TrimConfig configures trim-stage transcript boundary behavior. // TrimConfig configures trim-stage transcript boundary behavior.

View File

@@ -74,7 +74,7 @@ func finalizePipelineResolution(cfg *PipelineConfig) error {
// recomputePipelineEffectiveDigest is the single package-owned hook for // recomputePipelineEffectiveDigest is the single package-owned hook for
// refreshing provenance after later resolution expands concrete pipeline // refreshing provenance after later resolution expands concrete pipeline
// values. Composition declarations and runtime provenance are not serialized. // values. Loader directives and runtime provenance are not serialized.
func recomputePipelineEffectiveDigest(cfg *PipelineConfig) error { func recomputePipelineEffectiveDigest(cfg *PipelineConfig) error {
if cfg == nil || cfg.resolution == nil { if cfg == nil || cfg.resolution == nil {
return fmt.Errorf("pipeline resolution metadata is required") return fmt.Errorf("pipeline resolution metadata is required")

View File

@@ -55,7 +55,7 @@ type EffectivePipelineValueRecord struct {
// EffectivePipelineValues projects a fully resolved pipeline into sorted // EffectivePipelineValues projects a fully resolved pipeline into sorted
// logical configuration values. Mappings are flattened, while sequences remain // logical configuration values. Mappings are flattened, while sequences remain
// atomic values. The projection uses the same normalized effective mapping as // atomic values. The projection uses the same normalized effective mapping as
// config show and therefore excludes composition and family declarations. // config show and therefore excludes loader directives and family declarations.
func EffectivePipelineValues(cfg *PipelineConfig) ([]EffectivePipelineValueRecord, error) { func EffectivePipelineValues(cfg *PipelineConfig) ([]EffectivePipelineValueRecord, error) {
if cfg == nil { if cfg == nil {
return nil, fmt.Errorf("pipeline config is required") return nil, fmt.Errorf("pipeline config is required")
@@ -301,6 +301,9 @@ func pipelineSourceRole(cfg *PipelineConfig, source string) string {
if source == pipelineDefaultOwnershipSource { if source == pipelineDefaultOwnershipSource {
return "default" return "default"
} }
if source == "resolution" {
return "resolution"
}
if cfg == nil || cfg.resolution == nil { if cfg == nil || cfg.resolution == nil {
return "pipeline" return "pipeline"
} }
@@ -319,7 +322,7 @@ func pipelineSourceRole(cfg *PipelineConfig, source string) string {
} }
func normalizedProvenanceSource(source string) string { func normalizedProvenanceSource(source string) string {
if source == pipelineDefaultOwnershipSource { if source == pipelineDefaultOwnershipSource || source == "resolution" {
return source return source
} }
return normalizedProvenancePath(source) return normalizedProvenancePath(source)

View File

@@ -5,6 +5,7 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"sort" "sort"
"strings"
) )
// effectivePipelineProjection is the single safe, schema-specific view used // effectivePipelineProjection is the single safe, schema-specific view used
@@ -37,6 +38,11 @@ type effectiveScriptoriumConfig struct {
Artifacts map[string]ScriptoriumArtifactConfig `yaml:"artifacts" json:"artifacts"` Artifacts map[string]ScriptoriumArtifactConfig `yaml:"artifacts" json:"artifacts"`
} }
type projectedPipelineValue struct {
record EffectivePipelineValueRecord
segments []string
}
func projectEffectivePipeline(cfg *PipelineConfig, logicalNotarius bool) effectivePipelineProjection { func projectEffectivePipeline(cfg *PipelineConfig, logicalNotarius bool) effectivePipelineProjection {
projection := effectivePipelineProjection{ projection := effectivePipelineProjection{
Workspace: cfg.Workspace, Campaigns: cfg.Campaigns, Storage: cfg.Storage, Workspace: cfg.Workspace, Campaigns: cfg.Campaigns, Storage: cfg.Storage,
@@ -70,23 +76,35 @@ func projectionJSON(projection effectivePipelineProjection) ([]byte, error) {
} }
func projectionValueRecords(projection effectivePipelineProjection) ([]EffectivePipelineValueRecord, error) { func projectionValueRecords(projection effectivePipelineProjection) ([]EffectivePipelineValueRecord, error) {
data, err := projectionJSON(projection) detailed, err := projectionDetailedRecords(projection)
if err != nil { if err != nil {
return nil, err return nil, err
} }
records := make([]EffectivePipelineValueRecord, 0, len(detailed))
for _, value := range detailed {
records = append(records, value.record)
}
return records, nil
}
func projectionDetailedRecords(value any) ([]projectedPipelineValue, error) {
data, err := json.Marshal(value)
if err != nil {
return nil, fmt.Errorf("encode typed pipeline projection: %w", err)
}
decoder := json.NewDecoder(bytes.NewReader(data)) decoder := json.NewDecoder(bytes.NewReader(data))
decoder.UseNumber() decoder.UseNumber()
var root map[string]any var root map[string]any
if err := decoder.Decode(&root); err != nil { if err := decoder.Decode(&root); err != nil {
return nil, fmt.Errorf("decode effective pipeline projection: %w", err) return nil, fmt.Errorf("decode effective pipeline projection: %w", err)
} }
var records []EffectivePipelineValueRecord var records []projectedPipelineValue
appendProjectionRecords(&records, "", root) appendProjectionRecords(&records, nil, root)
sort.Slice(records, func(i, j int) bool { return records[i].Path < records[j].Path }) sort.Slice(records, func(i, j int) bool { return records[i].record.Path < records[j].record.Path })
return records, nil return records, nil
} }
func appendProjectionRecords(records *[]EffectivePipelineValueRecord, path string, value any) { func appendProjectionRecords(records *[]projectedPipelineValue, segments []string, value any) {
if mapping, ok := value.(map[string]any); ok && len(mapping) > 0 { if mapping, ok := value.(map[string]any); ok && len(mapping) > 0 {
keys := make([]string, 0, len(mapping)) keys := make([]string, 0, len(mapping))
for key := range mapping { for key := range mapping {
@@ -94,14 +112,15 @@ func appendProjectionRecords(records *[]EffectivePipelineValueRecord, path strin
} }
sort.Strings(keys) sort.Strings(keys)
for _, key := range keys { for _, key := range keys {
childPath := key child := append(append([]string(nil), segments...), key)
if path != "" { appendProjectionRecords(records, child, mapping[key])
childPath = path + "." + key
}
appendProjectionRecords(records, childPath, mapping[key])
} }
return return
} }
encoded, _ := json.Marshal(value) encoded, _ := json.Marshal(value)
*records = append(*records, EffectivePipelineValueRecord{Path: path, Value: string(encoded)}) path := strings.Join(segments, ".")
*records = append(*records, projectedPipelineValue{
record: EffectivePipelineValueRecord{Path: path, Value: string(encoded)},
segments: append([]string(nil), segments...),
})
} }

View File

@@ -17,9 +17,8 @@ func LoadPipeline(path string) (*PipelineConfig, error) {
return LoadPipelineWithOptions(path, PipelineLoadOptions{}) return LoadPipelineWithOptions(path, PipelineLoadOptions{})
} }
// PipelineLoadOptions carries an optional explicit profile selection. A nil // PipelineLoadOptions carries an optional explicit profile selection. Nil,
// Profile means the caller omitted selection; a non-nil empty value is an // empty, and the exact name "default" select the base configuration.
// explicit invalid selection.
type PipelineLoadOptions struct { type PipelineLoadOptions struct {
Profile *string Profile *string
} }
@@ -27,77 +26,32 @@ type PipelineLoadOptions struct {
// LoadPipelineWithOptions loads pipeline configuration with strict field // LoadPipelineWithOptions loads pipeline configuration with strict field
// checking and optional named-profile selection. // checking and optional named-profile selection.
func LoadPipelineWithOptions(path string, opts PipelineLoadOptions) (*PipelineConfig, error) { func LoadPipelineWithOptions(path string, opts PipelineLoadOptions) (*PipelineConfig, error) {
sources, err := loadPipelineCompositionSources(path) cfg, err := loadPipelineWithConfigKit(path, opts)
if err != nil { if err != nil {
return nil, fmt.Errorf("load pipeline config: %w", err) return nil, err
}
if _, err := selectPipelineProfile(sources.envelope, opts); err != nil {
return nil, fmt.Errorf("load pipeline config: %w", err)
}
if err := sources.loadOverlays(); err != nil {
return nil, fmt.Errorf("load pipeline config: %w", err)
}
cfg, err := sources.resolve(opts)
if err != nil {
return nil, fmt.Errorf("load pipeline config: %w", err)
} }
return finalizeLoadedPipeline(path, cfg) return finalizeLoadedPipeline(path, cfg)
} }
// LoadPipelineProfilePair resolves two explicit named profiles from one parsed // LoadPipelineProfilePair resolves two profiles as independent ConfigKit loads.
// pipeline root and its declared source set. Each result is independently
// decoded, defaulted, and finalized so later resolution can safely mutate one
// effective pipeline without affecting the other.
func LoadPipelineProfilePair(path, leftProfile, rightProfile string) (*PipelineConfig, *PipelineConfig, error) { func LoadPipelineProfilePair(path, leftProfile, rightProfile string) (*PipelineConfig, *PipelineConfig, error) {
if _, err := normalizePipelineProfileName(leftProfile, "left profile selection"); err != nil {
return nil, nil, fmt.Errorf("load left pipeline profile: %w", err)
}
if _, err := normalizePipelineProfileName(rightProfile, "right profile selection"); err != nil {
return nil, nil, fmt.Errorf("load right pipeline profile: %w", err)
}
if leftProfile == rightProfile { if leftProfile == rightProfile {
return nil, nil, fmt.Errorf("load pipeline profiles: left and right profile selections must differ") return nil, nil, fmt.Errorf("load pipeline profiles: left and right profile selections must differ")
} }
sources, err := loadPipelineCompositionSources(path)
if err != nil {
return nil, nil, fmt.Errorf("load pipeline config: %w", err)
}
leftOptions := PipelineLoadOptions{Profile: &leftProfile} leftOptions := PipelineLoadOptions{Profile: &leftProfile}
rightOptions := PipelineLoadOptions{Profile: &rightProfile} rightOptions := PipelineLoadOptions{Profile: &rightProfile}
if _, err := selectPipelineProfile(sources.envelope, leftOptions); err != nil { left, err := LoadPipelineWithOptions(path, leftOptions)
return nil, nil, fmt.Errorf("load left pipeline profile: %w", err)
}
if _, err := selectPipelineProfile(sources.envelope, rightOptions); err != nil {
return nil, nil, fmt.Errorf("load right pipeline profile: %w", err)
}
if err := sources.loadOverlays(); err != nil {
return nil, nil, fmt.Errorf("load pipeline config: %w", err)
}
left, err := sources.resolve(leftOptions)
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("load left pipeline profile: %w", err) return nil, nil, fmt.Errorf("load left pipeline profile: %w", err)
} }
right, err := sources.resolve(rightOptions) right, err := LoadPipelineWithOptions(path, rightOptions)
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("load right pipeline profile: %w", err) return nil, nil, fmt.Errorf("load right pipeline profile: %w", err)
} }
left, err = finalizeLoadedPipeline(path, left)
if err != nil {
return nil, nil, err
}
right, err = finalizeLoadedPipeline(path, right)
if err != nil {
return nil, nil, err
}
return left, right, nil return left, right, nil
} }
func finalizeLoadedPipeline(path string, cfg *PipelineConfig) (*PipelineConfig, error) { func finalizeLoadedPipeline(path string, cfg *PipelineConfig) (*PipelineConfig, error) {
cfg.resolution.publishDeclared = cfg.Publish != nil
applyPipelineDefaults(cfg)
if err := NormalizePipelineConfig(cfg); err != nil {
return nil, fmt.Errorf("load pipeline config: %w", err)
}
captureLogicalNotariusPaths(cfg) captureLogicalNotariusPaths(cfg)
if err := resolveNotariusPaths(cfg, path); err != nil { if err := resolveNotariusPaths(cfg, path); err != nil {
return nil, fmt.Errorf("load pipeline config: %w", err) return nil, fmt.Errorf("load pipeline config: %w", err)
@@ -504,217 +458,6 @@ func shortName(path, fallback string) string {
return base return base
} }
func applyPipelineDefaults(cfg *PipelineConfig) {
if cfg == nil {
return
}
applyWorkspaceDefaults(&cfg.Workspace)
applyCampaignsDefaults(&cfg.Campaigns)
applyStorageDefaults(&cfg.Storage)
applySpoolDefaults(&cfg.Spool)
applyCacheDefaults(&cfg.Cache)
applyPublishDefaults(&cfg.Publish)
applyWhisperXDefaults(&cfg.WhisperX)
applySeriatimDefaults(&cfg.Seriatim)
applyAuditaDefaults(&cfg.Audita)
if cfg.Normalize == nil {
cfg.Normalize = &NormalizeConfig{}
}
applyNormalizeDefaults(cfg.Normalize)
if cfg.Trim == nil {
cfg.Trim = &TrimConfig{}
}
applyTrimDefaults(cfg.Trim)
if trimEnabled(cfg.Trim) && cfg.Scriptorium == nil {
cfg.Scriptorium = &ScriptoriumConfig{}
}
applyRenderDefaults(&cfg.Render)
applyScriptoriumDefaults(cfg.Scriptorium)
applyNotariusDefaults(cfg.Notarius)
applyNotificationDefaults(&cfg.Notification)
}
func applyNotificationDefaults(cfg *NotificationConfig) {
if cfg == nil {
return
}
if strings.TrimSpace(cfg.Mode) == "" {
cfg.Mode = DefaultNotificationMode
}
}
func applyCampaignsDefaults(cfg *CampaignsConfig) {
if cfg == nil {
return
}
if cfg.Root == "" {
cfg.Root = DefaultCampaignsRoot
}
}
func applyWorkspaceDefaults(cfg *WorkspaceConfig) {
if cfg == nil {
return
}
if cfg.Root == "" {
cfg.Root = DefaultWorkspaceRoot
}
}
func applyStorageDefaults(cfg *StorageConfig) {
if cfg == nil {
return
}
backend := strings.ToLower(strings.TrimSpace(cfg.Backend))
if backend == "" {
backend = StorageBackendLocal
}
cfg.Backend = backend
if cfg.S3 == nil {
return
}
if cfg.S3.RootPrefix == "" {
cfg.S3.RootPrefix = DefaultStorageS3RootPrefix
}
if cfg.S3.AccessKeyIDEnv == "" {
cfg.S3.AccessKeyIDEnv = DefaultS3AccessKeyIDEnv
}
if cfg.S3.SecretKeyEnv == "" {
cfg.S3.SecretKeyEnv = DefaultS3SecretAccessKeyEnv
}
}
func applySpoolDefaults(cfg *SpoolConfig) {
if cfg == nil {
return
}
if cfg.Root == "" {
cfg.Root = DefaultSpoolRoot
}
}
func applyCacheDefaults(cfg *CacheConfig) {
if cfg == nil {
return
}
if cfg.Root == "" {
cfg.Root = DefaultCacheRoot
}
if cfg.S3Audio == nil {
cfg.S3Audio = boolPtr(DefaultCacheS3Audio)
}
}
func applyPublishDefaults(cfg **PublishConfig) {
if cfg == nil {
return
}
if *cfg == nil {
*cfg = &PublishConfig{}
}
if (*cfg).Enabled == nil {
(*cfg).Enabled = boolPtr(DefaultArchiveEnabled)
}
if (*cfg).UploadRun == nil {
(*cfg).UploadRun = boolPtr(DefaultArchiveUploadRun)
}
if len((*cfg).Outputs) == 0 {
(*cfg).Outputs = append([]PublishOutputRule(nil), DefaultPublishOutputs...)
}
for i := range (*cfg).Outputs {
if (*cfg).Outputs[i].Required == nil {
(*cfg).Outputs[i].Required = boolPtr(true)
}
}
}
func applyWhisperXDefaults(cfg *WhisperXConfig) {
if cfg == nil {
return
}
if cfg.Language == "" {
cfg.Language = DefaultWhisperXLanguage
}
if cfg.Timeout == "" {
cfg.Timeout = DefaultWhisperXTimeout
}
if cfg.RetryDelay == "" {
cfg.RetryDelay = DefaultWhisperXRetryDelay
}
if cfg.Concurrency == nil {
cfg.Concurrency = intPtr(DefaultWhisperXConcurrency)
}
if cfg.Retries == nil {
cfg.Retries = intPtr(DefaultWhisperXRetries)
}
}
func intPtr(v int) *int {
p := v
return &p
}
func applySeriatimDefaults(cfg *SeriatimConfig) {
if cfg == nil {
return
}
if cfg.Binary == "" {
cfg.Binary = DefaultSeriatimBinary
}
if cfg.Timeout == "" {
cfg.Timeout = DefaultSeriatimTimeout
}
if cfg.OutputSchema == "" {
cfg.OutputSchema = DefaultSeriatimOutputSchema
}
if cfg.CoalesceGap == nil {
cfg.CoalesceGap = float64Ptr(DefaultSeriatimCoalesceGap)
}
if cfg.Report == nil {
cfg.Report = boolPtr(DefaultSeriatimReport)
}
}
func applyAuditaDefaults(cfg *AuditaConfig) {
if cfg == nil {
return
}
if cfg.Binary == "" {
cfg.Binary = DefaultAuditaBinary
}
if cfg.Timeout == "" {
cfg.Timeout = DefaultAuditaTimeout
}
if cfg.Report == nil {
cfg.Report = boolPtr(DefaultAuditaReport)
}
}
func applyScriptoriumDefaults(cfg *ScriptoriumConfig) {
if cfg == nil {
return
}
if cfg.Binary == "" {
cfg.Binary = DefaultScriptoriumBinary
}
if cfg.Timeout == "" {
cfg.Timeout = DefaultScriptoriumTimeout
}
}
func applyNotariusDefaults(cfg *NotariusConfig) {
if cfg == nil {
return
}
if cfg.Binary == "" {
cfg.Binary = DefaultNotariusBinary
}
if cfg.Timeout == "" {
cfg.Timeout = DefaultNotariusTimeout
}
}
func resolveNotariusPaths(cfg *PipelineConfig, pipelinePath string) error { func resolveNotariusPaths(cfg *PipelineConfig, pipelinePath string) error {
if cfg == nil || cfg.Notarius == nil || !cfg.Notarius.Enabled { if cfg == nil || cfg.Notarius == nil || !cfg.Notarius.Enabled {
return nil return nil
@@ -753,83 +496,3 @@ func resolveNotariusPaths(cfg *PipelineConfig, pipelinePath string) error {
cfg.Notarius.WorkingDirectory = workingDirectory cfg.Notarius.WorkingDirectory = workingDirectory
return nil return nil
} }
func applyTrimDefaults(cfg *TrimConfig) {
if cfg == nil {
return
}
if cfg.Enabled == nil {
cfg.Enabled = boolPtr(DefaultTrimEnabled)
}
if strings.TrimSpace(cfg.OutputPath) == "" {
cfg.OutputPath = DefaultTrimOutputPath
}
if strings.TrimSpace(cfg.Bounds.PromptID) == "" {
cfg.Bounds.PromptID = DefaultTrimBoundsPromptID
}
if strings.TrimSpace(cfg.Bounds.TranscriptInputName) == "" {
cfg.Bounds.TranscriptInputName = DefaultTrimBoundsTranscriptInputName
}
if strings.TrimSpace(cfg.Bounds.OutputPath) == "" {
cfg.Bounds.OutputPath = DefaultTrimBoundsOutputPath
}
if cfg.Bounds.Timeout == "" {
cfg.Bounds.Timeout = DefaultTrimBoundsTimeout
}
if cfg.Seriatim.Report == nil {
cfg.Seriatim.Report = boolPtr(DefaultTrimSeriatimReport)
}
}
func trimEnabled(cfg *TrimConfig) bool {
return cfg != nil && cfg.Enabled != nil && *cfg.Enabled
}
func applyRenderDefaults(cfg **RenderConfig) {
if cfg == nil {
return
}
if *cfg == nil {
*cfg = &RenderConfig{}
}
if (*cfg).Enabled == nil {
(*cfg).Enabled = boolPtr(DefaultRenderEnabled)
}
if strings.TrimSpace((*cfg).Format) == "" {
(*cfg).Format = DefaultRenderFormat
}
if strings.TrimSpace((*cfg).Title) == "" {
(*cfg).Title = DefaultRenderTitle
}
if (*cfg).IncludeTimestamps == nil {
(*cfg).IncludeTimestamps = boolPtr(DefaultRenderTimestamps)
}
if (*cfg).IncludeSegmentIDs == nil {
(*cfg).IncludeSegmentIDs = boolPtr(DefaultRenderSegmentIDs)
}
}
func applyNormalizeDefaults(cfg *NormalizeConfig) {
if cfg == nil {
return
}
if cfg.OutputSchema == "" {
cfg.OutputSchema = DefaultNormalizeOutputSchema
}
if cfg.OutputPath == "" && !cfg.outputPathWasSet() {
cfg.OutputPath = DefaultNormalizeOutputPath
}
if cfg.Report == nil {
cfg.Report = boolPtr(DefaultNormalizeReport)
}
}
func float64Ptr(v float64) *float64 {
p := v
return &p
}
func boolPtr(v bool) *bool {
p := v
return &p
}

View File

@@ -48,7 +48,7 @@ func TestLoadersRejectTrailingYAMLDocuments(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
err := tt.load() err := tt.load()
if err == nil || !strings.Contains(err.Error(), "exactly one YAML document") { if err == nil || (!strings.Contains(err.Error(), "exactly one YAML document") && !strings.Contains(err.Error(), "yaml.multiple_documents")) {
t.Fatalf("load error = %v, want exactly-one-document error", err) t.Fatalf("load error = %v, want exactly-one-document error", err)
} }
}) })

View File

@@ -1,49 +0,0 @@
package config
import (
"fmt"
"gopkg.in/yaml.v3"
)
// UnmarshalYAML tracks explicit normalize.output_path presence so validation can
// distinguish omitted vs explicitly empty values.
func (cfg *NormalizeConfig) UnmarshalYAML(node *yaml.Node) error {
if node.Kind != yaml.MappingNode {
return fmt.Errorf("normalize must be a mapping")
}
allowedKeys := map[string]struct{}{
"output_path": {},
"output_schema": {},
"report": {},
}
for i := 0; i+1 < len(node.Content); i += 2 {
key := node.Content[i].Value
if _, ok := allowedKeys[key]; !ok {
return fmt.Errorf("field %q not found in type config.NormalizeConfig", key)
}
}
type rawNormalize NormalizeConfig
var raw rawNormalize
if err := node.Decode(&raw); err != nil {
return err
}
*cfg = NormalizeConfig(raw)
for i := 0; i+1 < len(node.Content); i += 2 {
if node.Content[i].Value == "output_path" {
cfg.outputPathSet = true
break
}
}
return nil
}
// outputPathSet records whether normalize.output_path appeared in YAML.
// This is intentionally unexported and not serialized.
func (cfg *NormalizeConfig) outputPathWasSet() bool {
if cfg == nil {
return false
}
return cfg.outputPathSet
}

View File

@@ -107,6 +107,12 @@ notarius:
enabled: true enabled: true
config_path: notarius/config.yml config_path: notarius/config.yml
pipeline_id: dnd-session pipeline_id: dnd-session
outputs:
npc_registry:
lane_id: npc-registry
media_type: application/json
schema_id: notarius.dnd.npc_registry
schema_version: v1
` `
paths := make([]string, 2) paths := make([]string, 2)
configs := make([]*PipelineConfig, 2) configs := make([]*PipelineConfig, 2)
@@ -151,7 +157,7 @@ func TestNotariusStrictYAML(t *testing.T) {
_, err := LoadPipeline(path) _, err := LoadPipeline(path)
want := "strict decode failed" want := "strict decode failed"
if tt.name == "duplicate reference selector" { if tt.name == "duplicate reference selector" {
want = "duplicate YAML key" want = "yaml.duplicate_key"
} }
if err == nil || !strings.Contains(err.Error(), want) { if err == nil || !strings.Contains(err.Error(), want) {
t.Fatalf("LoadPipeline() error = %v, want containing %q", err, want) t.Fatalf("LoadPipeline() error = %v, want containing %q", err, want)

View File

@@ -23,12 +23,12 @@ func TestNotificationConfigSupportsOnlyNoopMode(t *testing.T) {
{ {
name: "backend is rejected", name: "backend is rejected",
section: "notification:\n backend: email\n", section: "notification:\n backend: email\n",
wantLoad: "field backend not found", wantLoad: "decode.unknown_field",
}, },
{ {
name: "recipient is rejected", name: "recipient is rejected",
section: "notification:\n recipient: party@example.com\n", section: "notification:\n recipient: party@example.com\n",
wantLoad: "field recipient not found", wantLoad: "decode.unknown_field",
}, },
{ {
name: "provider mode is rejected", name: "provider mode is rejected",

View File

@@ -1,532 +0,0 @@
package config
import (
"fmt"
"os"
"path/filepath"
"strings"
"unicode"
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
"gopkg.in/yaml.v3"
)
type pipelineResolutionMetadata struct {
rootPath string
imports []string
sources []string
selectedProfile *pipelineProfileSelection
effectiveDigest string
logicalNotariusConfig string
logicalNotariusWorking string
logicalNotariusCaptured bool
ownership []pipelineFieldOwnership
artifactFamilies map[string]ScriptoriumArtifactFamilyConfig
artifactFamiliesExpanded bool
publishDeclared bool
}
type pipelineProfileSelection struct {
name string
source string
overlayPath string
}
type pipelineFieldOwnership struct {
path string
sources []string
}
type pipelineCompositionEnvelope struct {
imports []string
defaultProfile *string
profiles []pipelineProfileDeclaration
}
type pipelineProfileDeclaration struct {
name string
overlay string
}
// pipelineCompositionSources retains one validated root source set. Each
// selected profile is resolved from a cloned base document so callers can
// safely compare or otherwise resolve multiple profiles without rereading or
// mutating the source set.
type pipelineCompositionSources struct {
rootPath string
envelope pipelineCompositionEnvelope
imports []loadedPipelineImport
overlays []loadedPipelineProfileOverlay
overlaysLoaded bool
base *compositionDocument
}
func loadPipelineCompositionSources(path string) (*pipelineCompositionSources, error) {
rootPath, err := filepath.Abs(path)
if err != nil {
return nil, fmt.Errorf("resolve root pipeline path %q: %w", path, err)
}
rootFile, err := os.Open(rootPath)
if err != nil {
return nil, fmt.Errorf("pipeline file %q: open: %w", path, err)
}
rootDocument, parseErr := parseCompositionDocument(rootPath, rootFile)
closeErr := rootFile.Close()
if parseErr != nil {
return nil, parseErr
}
if closeErr != nil {
return nil, fmt.Errorf("pipeline file %q: close: %w", rootPath, closeErr)
}
baseRoot, envelope, err := splitPipelineCompositionEnvelope(rootDocument)
if err != nil {
return nil, err
}
imports, err := loadPipelineImports(rootPath, envelope.imports)
if err != nil {
return nil, err
}
documents := make([]*compositionDocument, 0, len(imports)+1)
documents = append(documents, baseRoot)
for _, imported := range imports {
documents = append(documents, imported.document)
}
merged, err := mergeAdditiveCompositions(documents...)
if err != nil {
return nil, err
}
return &pipelineCompositionSources{
rootPath: rootPath,
envelope: envelope,
imports: imports,
base: merged,
}, nil
}
func (sources *pipelineCompositionSources) loadOverlays() error {
if sources == nil {
return fmt.Errorf("pipeline composition sources are required")
}
if sources.overlaysLoaded {
return nil
}
overlays, err := loadPipelineProfileOverlays(sources.rootPath, sources.envelope.profiles, sources.imports)
if err != nil {
return err
}
sources.overlays = overlays
sources.overlaysLoaded = true
return nil
}
func (sources *pipelineCompositionSources) resolve(opts PipelineLoadOptions) (*PipelineConfig, error) {
if sources == nil || sources.base == nil {
return nil, fmt.Errorf("pipeline composition sources are required")
}
if !sources.overlaysLoaded {
return nil, fmt.Errorf("pipeline profile overlays have not been loaded")
}
selection, err := selectPipelineProfile(sources.envelope, opts)
if err != nil {
return nil, err
}
merged := &compositionDocument{
root: cloneCompositionNode(sources.base.root),
sources: append([]string(nil), sources.base.sources...),
}
if selection != nil {
overlay, ok := loadedProfileOverlay(sources.overlays, selection.name)
if !ok {
return nil, fmt.Errorf("selected profile %q overlay was not loaded", selection.name)
}
merged, err = mergeOverlayComposition(merged, overlay.document)
if err != nil {
return nil, err
}
selection.overlayPath = overlay.path
}
rendered, err := merged.canonicalYAML()
if err != nil {
return nil, err
}
var cfg PipelineConfig
if err := decodeStrictYAMLFromReader("pipeline", sources.rootPath, strings.NewReader(string(rendered)), &cfg); err != nil {
return nil, fmt.Errorf("assembled pipeline sources %s: %w", formatCompositionSources(merged.sources), err)
}
records, err := merged.semanticRecords()
if err != nil {
return nil, err
}
metadata := &pipelineResolutionMetadata{
rootPath: sources.rootPath,
sources: append([]string(nil), merged.sources...),
selectedProfile: selection,
}
for _, imported := range sources.imports {
metadata.imports = append(metadata.imports, imported.path)
}
for _, record := range records {
metadata.ownership = append(metadata.ownership, pipelineFieldOwnership{
path: record.Path, sources: append([]string(nil), record.Sources...),
})
}
cfg.resolution = metadata
return &cfg, nil
}
func splitPipelineCompositionEnvelope(document *compositionDocument) (*compositionDocument, pipelineCompositionEnvelope, error) {
if err := validateCompositionDocument(document, "root pipeline"); err != nil {
return nil, pipelineCompositionEnvelope{}, err
}
envelope, fieldIndex, err := decodePipelineCompositionEnvelope(document.root)
if err != nil {
return nil, pipelineCompositionEnvelope{}, err
}
if err := validatePipelineCompositionEnvelope(envelope); err != nil {
return nil, pipelineCompositionEnvelope{}, err
}
return pipelineBodyWithoutComposition(document, fieldIndex), envelope, nil
}
func decodePipelineCompositionEnvelope(root *compositionNode) (pipelineCompositionEnvelope, int, error) {
index := compositionFieldIndex(root.fields, "composition")
if index < 0 {
return pipelineCompositionEnvelope{}, -1, nil
}
envelopeNode := root.fields[index].value
if envelopeNode.kind != yaml.MappingNode {
return pipelineCompositionEnvelope{}, -1, fmt.Errorf(
"configuration source %s at composition: expected a mapping, got %s",
formatCompositionSources(envelopeNode.sources), yamlKindName(envelopeNode.kind),
)
}
var envelope pipelineCompositionEnvelope
for _, field := range envelopeNode.fields {
switch field.key {
case "imports":
if field.value.kind != yaml.SequenceNode {
return pipelineCompositionEnvelope{}, -1, fmt.Errorf(
"configuration source %s at composition.imports: expected a list, got %s",
formatCompositionSources(field.value.sources), yamlKindName(field.value.kind),
)
}
for itemIndex, item := range field.value.items {
if item.kind != yaml.ScalarNode || item.tag != "!!str" {
return pipelineCompositionEnvelope{}, -1, fmt.Errorf(
"configuration source %s at composition.imports[%d]: expected a string path",
formatCompositionSources(item.sources), itemIndex,
)
}
envelope.imports = append(envelope.imports, item.value)
}
case "default_profile":
if field.value.kind != yaml.ScalarNode || field.value.tag != "!!str" {
return pipelineCompositionEnvelope{}, -1, fmt.Errorf(
"configuration source %s at composition.default_profile: expected a string profile name",
formatCompositionSources(field.value.sources),
)
}
name, err := normalizePipelineProfileName(field.value.value, "composition.default_profile")
if err != nil {
return pipelineCompositionEnvelope{}, -1, err
}
envelope.defaultProfile = &name
case "profiles":
if field.value.kind != yaml.MappingNode {
return pipelineCompositionEnvelope{}, -1, fmt.Errorf(
"configuration source %s at composition.profiles: expected a mapping, got %s",
formatCompositionSources(field.value.sources), yamlKindName(field.value.kind),
)
}
if len(field.value.fields) == 0 {
return pipelineCompositionEnvelope{}, -1, fmt.Errorf("composition.profiles must declare at least one named profile")
}
for _, profileField := range field.value.fields {
name, err := normalizePipelineProfileName(profileField.key, "composition.profiles profile name")
if err != nil {
return pipelineCompositionEnvelope{}, -1, err
}
profile, err := parsePipelineProfileDeclaration(name, profileField.value)
if err != nil {
return pipelineCompositionEnvelope{}, -1, err
}
envelope.profiles = append(envelope.profiles, profile)
}
default:
return pipelineCompositionEnvelope{}, -1, fmt.Errorf(
"configuration source %s at composition.%s: unknown composition field %q",
formatCompositionSources(field.value.sources), field.key, field.key,
)
}
}
return envelope, index, nil
}
func validatePipelineCompositionEnvelope(envelope pipelineCompositionEnvelope) error {
if envelope.defaultProfile != nil && !pipelineProfileDeclared(envelope.profiles, *envelope.defaultProfile) {
return fmt.Errorf(
"composition.default_profile %q does not name a declared profile", *envelope.defaultProfile,
)
}
return nil
}
func pipelineBodyWithoutComposition(document *compositionDocument, compositionIndex int) *compositionDocument {
root := cloneCompositionNode(document.root)
if compositionIndex >= 0 {
root.fields = append(root.fields[:compositionIndex], root.fields[compositionIndex+1:]...)
}
return &compositionDocument{root: root, sources: append([]string(nil), document.sources...)}
}
func parsePipelineProfileDeclaration(name string, node *compositionNode) (pipelineProfileDeclaration, error) {
path := "composition.profiles." + name
if node.kind != yaml.MappingNode {
return pipelineProfileDeclaration{}, fmt.Errorf(
"configuration source %s at %s: expected a mapping, got %s",
formatCompositionSources(node.sources), path, yamlKindName(node.kind),
)
}
profile := pipelineProfileDeclaration{name: name}
for _, field := range node.fields {
if field.key != "overlay" {
return pipelineProfileDeclaration{}, fmt.Errorf(
"configuration source %s at %s.%s: unknown profile field %q; only overlay is supported",
formatCompositionSources(field.value.sources), path, field.key, field.key,
)
}
if field.value.kind != yaml.ScalarNode || field.value.tag != "!!str" {
return pipelineProfileDeclaration{}, fmt.Errorf(
"configuration source %s at %s.overlay: expected a string path",
formatCompositionSources(field.value.sources), path,
)
}
profile.overlay = field.value.value
}
if profile.overlay == "" {
return pipelineProfileDeclaration{}, fmt.Errorf("%s.overlay is required", path)
}
return profile, nil
}
func normalizePipelineProfileName(value, label string) (string, error) {
if value == "" || strings.TrimSpace(value) != value {
return "", fmt.Errorf("%s must be non-empty without surrounding whitespace", label)
}
for _, character := range value {
if unicode.IsControl(character) {
return "", fmt.Errorf("%s %q must not contain control characters", label, value)
}
}
return value, nil
}
func pipelineProfileDeclared(profiles []pipelineProfileDeclaration, name string) bool {
for _, profile := range profiles {
if profile.name == name {
return true
}
}
return false
}
func selectPipelineProfile(envelope pipelineCompositionEnvelope, opts PipelineLoadOptions) (*pipelineProfileSelection, error) {
if opts.Profile != nil {
name, err := normalizePipelineProfileName(*opts.Profile, "explicit profile selection")
if err != nil {
return nil, err
}
if len(envelope.profiles) == 0 {
return nil, fmt.Errorf("explicit profile %q was selected but the pipeline declares no profiles", name)
}
if !pipelineProfileDeclared(envelope.profiles, name) {
return nil, fmt.Errorf("explicit profile %q is not declared by the pipeline", name)
}
return &pipelineProfileSelection{name: name, source: "cli"}, nil
}
if len(envelope.profiles) == 0 {
return nil, nil
}
if envelope.defaultProfile == nil {
return nil, fmt.Errorf("pipeline declares profiles but composition.default_profile is omitted and no profile was explicitly selected")
}
return &pipelineProfileSelection{name: *envelope.defaultProfile, source: "default"}, nil
}
type loadedPipelineImport struct {
path string
document *compositionDocument
info os.FileInfo
}
type loadedPipelineProfileOverlay struct {
profile string
path string
document *compositionDocument
info os.FileInfo
}
func loadPipelineProfileOverlays(
rootPath string,
declared []pipelineProfileDeclaration,
imports []loadedPipelineImport,
) ([]loadedPipelineProfileOverlay, error) {
if len(declared) == 0 {
return nil, nil
}
rootDir := filepath.Dir(rootPath)
rootInfo, err := os.Stat(rootPath)
if err != nil {
return nil, fmt.Errorf("inspect root pipeline file %q: %w", rootPath, err)
}
seenPaths := make(map[string]string, len(declared))
loaded := make([]loadedPipelineProfileOverlay, 0, len(declared))
for _, profile := range declared {
label := "composition.profiles." + profile.name + ".overlay"
raw := profile.overlay
if strings.TrimSpace(raw) != raw || raw == "" {
return nil, fmt.Errorf("%s must be a non-empty path without surrounding whitespace", label)
}
normalized, err := pathsafe.NormalizeRelativeDestination(raw)
if err != nil {
return nil, fmt.Errorf("%s path %q is invalid: %w", label, raw, err)
}
extension := filepath.Ext(filepath.FromSlash(normalized))
if extension != ".yml" && extension != ".yaml" {
return nil, fmt.Errorf("%s path %q must use .yml or .yaml", label, raw)
}
if prior, duplicate := seenPaths[normalized]; duplicate {
return nil, fmt.Errorf("%s path %q duplicates profile %q overlay after normalization", label, raw, prior)
}
seenPaths[normalized] = profile.name
resolved := filepath.Join(rootDir, filepath.FromSlash(normalized))
file, err := os.Open(resolved)
if err != nil {
return nil, fmt.Errorf("open %s path %q beneath root pipeline directory: %w", label, raw, err)
}
info, statErr := file.Stat()
if statErr != nil {
_ = file.Close()
return nil, fmt.Errorf("inspect %s path %q: %w", label, raw, statErr)
}
if os.SameFile(rootInfo, info) {
_ = file.Close()
return nil, fmt.Errorf("%s path %q references the root pipeline itself", label, raw)
}
for _, imported := range imports {
if os.SameFile(imported.info, info) {
_ = file.Close()
return nil, fmt.Errorf("%s path %q references the same file as imported source %q", label, raw, imported.path)
}
}
for _, prior := range loaded {
if os.SameFile(prior.info, info) {
_ = file.Close()
return nil, fmt.Errorf("%s path %q references the same file as profile %q overlay", label, raw, prior.profile)
}
}
document, parseErr := parseCompositionDocument(resolved, file)
closeErr := file.Close()
if parseErr != nil {
return nil, parseErr
}
if closeErr != nil {
return nil, fmt.Errorf("close %s path %q: %w", label, raw, closeErr)
}
if compositionFieldIndex(document.root.fields, "composition") >= 0 {
return nil, fmt.Errorf("profile overlay source %q declares composition; only the root pipeline may declare composition", resolved)
}
loaded = append(loaded, loadedPipelineProfileOverlay{
profile: profile.name, path: resolved, document: document, info: info,
})
}
return loaded, nil
}
func loadedProfileOverlay(overlays []loadedPipelineProfileOverlay, name string) (loadedPipelineProfileOverlay, bool) {
for _, overlay := range overlays {
if overlay.profile == name {
return overlay, true
}
}
return loadedPipelineProfileOverlay{}, false
}
func loadPipelineImports(rootPath string, declared []string) ([]loadedPipelineImport, error) {
if len(declared) == 0 {
return nil, nil
}
rootDir := filepath.Dir(rootPath)
rootInfo, err := os.Stat(rootPath)
if err != nil {
return nil, fmt.Errorf("inspect root pipeline file %q: %w", rootPath, err)
}
seenPaths := make(map[string]int, len(declared))
loaded := make([]loadedPipelineImport, 0, len(declared))
for index, raw := range declared {
if strings.TrimSpace(raw) != raw || raw == "" {
return nil, fmt.Errorf("composition.imports[%d] must be a non-empty path without surrounding whitespace", index)
}
normalized, err := pathsafe.NormalizeRelativeDestination(raw)
if err != nil {
return nil, fmt.Errorf("composition.imports[%d] path %q is invalid: %w", index, raw, err)
}
extension := filepath.Ext(filepath.FromSlash(normalized))
if extension != ".yml" && extension != ".yaml" {
return nil, fmt.Errorf("composition.imports[%d] path %q must use .yml or .yaml", index, raw)
}
if prior, duplicate := seenPaths[normalized]; duplicate {
return nil, fmt.Errorf(
"composition.imports[%d] path %q duplicates composition.imports[%d] after normalization",
index, raw, prior,
)
}
seenPaths[normalized] = index
resolved := filepath.Join(rootDir, filepath.FromSlash(normalized))
if resolved == rootPath {
return nil, fmt.Errorf("composition.imports[%d] path %q imports the root pipeline itself", index, raw)
}
file, err := os.Open(resolved)
if err != nil {
return nil, fmt.Errorf("open composition.imports[%d] path %q beneath root pipeline directory: %w", index, raw, err)
}
info, statErr := file.Stat()
if statErr != nil {
_ = file.Close()
return nil, fmt.Errorf("inspect composition.imports[%d] path %q: %w", index, raw, statErr)
}
if os.SameFile(rootInfo, info) {
_ = file.Close()
return nil, fmt.Errorf("composition.imports[%d] path %q imports the root pipeline itself", index, raw)
}
for priorIndex, prior := range loaded {
if os.SameFile(prior.info, info) {
_ = file.Close()
return nil, fmt.Errorf(
"composition.imports[%d] path %q references the same file as composition.imports[%d] %q",
index, raw, priorIndex, declared[priorIndex],
)
}
}
document, parseErr := parseCompositionDocument(resolved, file)
closeErr := file.Close()
if parseErr != nil {
return nil, parseErr
}
if closeErr != nil {
return nil, fmt.Errorf("close composition.imports[%d] path %q: %w", index, raw, closeErr)
}
if compositionFieldIndex(document.root.fields, "composition") >= 0 {
return nil, fmt.Errorf(
"imported configuration source %q declares composition; only the root pipeline may declare composition",
resolved,
)
}
loaded = append(loaded, loadedPipelineImport{path: resolved, document: document, info: info})
}
return loaded, nil
}

View File

@@ -1,431 +0,0 @@
package config
import (
"os"
"path/filepath"
"reflect"
"slices"
"strings"
"testing"
)
func TestSplitPipelineCompositionEnvelopePreservesInputAndDeclarationOrder(t *testing.T) {
document, err := parseCompositionBytes("pipeline.yml", []byte(`composition:
imports: [first.yml, second.yml]
default_profile: production
profiles:
production:
overlay: production.yml
testing:
overlay: testing.yml
workspace:
root: /srv/narratio
`))
if err != nil {
t.Fatal(err)
}
originalRoot := cloneCompositionNode(document.root)
originalSources := append([]string(nil), document.sources...)
body, envelope, err := splitPipelineCompositionEnvelope(document)
if err != nil {
t.Fatalf("splitPipelineCompositionEnvelope() error = %v", err)
}
if !reflect.DeepEqual(document.root, originalRoot) || !slices.Equal(document.sources, originalSources) {
t.Fatalf("splitPipelineCompositionEnvelope() mutated its input")
}
if compositionFieldIndex(body.root.fields, "composition") >= 0 {
t.Fatalf("pipeline body still contains composition envelope")
}
if compositionFieldIndex(body.root.fields, "workspace") < 0 {
t.Fatalf("pipeline body lost ordinary root fields")
}
if want := []string{"first.yml", "second.yml"}; !slices.Equal(envelope.imports, want) {
t.Fatalf("imports = %#v, want %#v", envelope.imports, want)
}
if len(envelope.profiles) != 2 || envelope.profiles[0].name != "production" || envelope.profiles[1].name != "testing" {
t.Fatalf("profiles = %#v, want declaration order", envelope.profiles)
}
}
func TestSplitPipelineCompositionEnvelopeRejectsMalformedShapesWithoutMutation(t *testing.T) {
tests := []struct {
name string
yaml string
want string
}{
{name: "envelope", yaml: "composition: []\n", want: "expected a mapping"},
{name: "imports mapping", yaml: "composition:\n imports: {}\n", want: "expected a list"},
{name: "import value", yaml: "composition:\n imports: [1]\n", want: "expected a string path"},
{name: "default profile", yaml: "composition:\n default_profile: {}\n", want: "expected a string profile name"},
{name: "profiles sequence", yaml: "composition:\n profiles: []\n", want: "expected a mapping"},
{name: "empty profiles", yaml: "composition:\n profiles: {}\n", want: "at least one named profile"},
{name: "profile declaration", yaml: "composition:\n profiles:\n production: production.yml\n", want: "expected a mapping"},
{name: "invalid profile name", yaml: "composition:\n profiles:\n ' production ':\n overlay: production.yml\n", want: "surrounding whitespace"},
{name: "unknown field", yaml: "composition:\n unexpected: true\n", want: "unknown composition field"},
{name: "unknown default", yaml: "composition:\n default_profile: testing\n profiles:\n production:\n overlay: production.yml\n", want: "does not name a declared profile"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
document, err := parseCompositionBytes("pipeline.yml", []byte(test.yaml))
if err != nil {
t.Fatal(err)
}
originalRoot := cloneCompositionNode(document.root)
_, _, err = splitPipelineCompositionEnvelope(document)
if err == nil || !strings.Contains(err.Error(), test.want) {
t.Fatalf("splitPipelineCompositionEnvelope() error = %v, want containing %q", err, test.want)
}
if !reflect.DeepEqual(document.root, originalRoot) {
t.Fatalf("splitPipelineCompositionEnvelope() mutated invalid input")
}
})
}
}
func TestLoadPipelineCompositionImportsDisjointFieldsAndTracksOwnership(t *testing.T) {
dir := t.TempDir()
rootPath := writePipelineSource(t, dir, "pipeline.yml", `composition:
imports:
- conf.d/platform.yml
- conf.d/artifacts.yml
whisperx:
transcribe_url: https://transcription.example.com/transcribe
`)
platformPath := writePipelineSource(t, dir, "conf.d/platform.yml", `workspace:
root: /srv/narratio/work
storage:
backend: local
`)
artifactsPath := writePipelineSource(t, dir, "conf.d/artifacts.yml", `scriptorium:
artifacts:
player_handout:
enabled: false
session_recap:
enabled: true
prompt_id: dnd.session_recap
output_path: artifacts/session_recap.md
`)
cfg, err := LoadPipeline(rootPath)
if err != nil {
t.Fatalf("LoadPipeline() error = %v", err)
}
if cfg.Workspace.Root != "/srv/narratio/work" || cfg.Storage.Backend != StorageBackendLocal {
t.Fatalf("imported platform config = workspace=%q storage=%#v", cfg.Workspace.Root, cfg.Storage)
}
if cfg.Scriptorium == nil || len(cfg.Scriptorium.Artifacts) != 2 || cfg.Scriptorium.Artifacts["player_handout"].Enabled {
t.Fatalf("imported artifacts = %#v", cfg.Scriptorium)
}
if cfg.WhisperX.TranscribeURL != "https://transcription.example.com/transcribe" {
t.Fatalf("root field = %q", cfg.WhisperX.TranscribeURL)
}
if cfg.resolution == nil {
t.Fatal("pipeline resolution metadata = nil")
}
wantSources := []string{absolutePath(t, rootPath), absolutePath(t, platformPath), absolutePath(t, artifactsPath)}
if !slices.Equal(cfg.resolution.sources, wantSources) || !slices.Equal(cfg.resolution.imports, wantSources[1:]) {
t.Fatalf("resolution sources=%#v imports=%#v, want %#v / %#v", cfg.resolution.sources, cfg.resolution.imports, wantSources, wantSources[1:])
}
assertPipelineFieldOwner(t, cfg, "whisperx.transcribe_url", absolutePath(t, rootPath))
assertPipelineFieldOwner(t, cfg, "workspace.root", absolutePath(t, platformPath))
assertPipelineFieldOwner(t, cfg, "scriptorium.artifacts.session_recap.prompt_id", absolutePath(t, artifactsPath))
}
func TestLoadPipelineCompositionMergesDisjointKeyedEntries(t *testing.T) {
dir := t.TempDir()
rootPath := writePipelineSource(t, dir, "pipeline.yml", `composition:
imports: [first.yml, second.yml]
whisperx:
transcribe_url: https://transcription.example.com/transcribe
scriptorium:
artifacts:
root_artifact:
enabled: false
`)
writePipelineSource(t, dir, "first.yml", `scriptorium:
artifacts:
first_artifact:
enabled: false
`)
writePipelineSource(t, dir, "second.yml", `scriptorium:
artifacts:
second_artifact:
enabled: false
`)
cfg, err := LoadPipeline(rootPath)
if err != nil {
t.Fatal(err)
}
if got := len(cfg.Scriptorium.Artifacts); got != 3 {
t.Fatalf("artifact count = %d, want 3: %#v", got, cfg.Scriptorium.Artifacts)
}
}
func TestLoadPipelineCompositionRejectsBaseConflictsWithAllSources(t *testing.T) {
tests := []struct {
name string
root string
imports map[string]string
path string
sources []string
}{
{
name: "root and import identical scalar",
root: "whisperx:\n language: en\n",
imports: map[string]string{"one.yml": "whisperx:\n language: en\n"},
path: "whisperx.language", sources: []string{"pipeline.yml", "one.yml"},
},
{
name: "all import claimants",
imports: map[string]string{
"one.yml": "workspace:\n root: /one\n",
"two.yml": "workspace:\n root: /two\n",
"three.yml": "workspace:\n root: /three\n",
},
path: "workspace.root", sources: []string{"one.yml", "two.yml", "three.yml"},
},
{
name: "atomic list",
root: "audita:\n modules: [one]\n",
imports: map[string]string{"one.yml": "audita:\n modules: [two]\n"},
path: "audita.modules", sources: []string{"pipeline.yml", "one.yml"},
},
{
name: "kind conflict",
root: "workspace:\n root: /work\n",
imports: map[string]string{"one.yml": "workspace: invalid\n"},
path: "workspace", sources: []string{"pipeline.yml", "one.yml"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
dir := t.TempDir()
order := make([]string, 0, len(tt.imports))
for _, name := range []string{"one.yml", "two.yml", "three.yml"} {
if _, ok := tt.imports[name]; ok {
order = append(order, name)
}
}
root := "composition:\n imports:\n"
for _, name := range order {
root += " - " + name + "\n"
}
root += tt.root
rootPath := writePipelineSource(t, dir, "pipeline.yml", root)
for name, content := range tt.imports {
writePipelineSource(t, dir, name, content)
}
_, err := LoadPipeline(rootPath)
if err == nil {
t.Fatal("LoadPipeline() error = nil")
}
if !strings.Contains(err.Error(), tt.path) {
t.Fatalf("error = %q, want path %q", err, tt.path)
}
for _, source := range tt.sources {
if !strings.Contains(err.Error(), source) {
t.Fatalf("error = %q, want source %q", err, source)
}
}
})
}
}
func TestLoadPipelineCompositionRejectsUnsafeOrInvalidImports(t *testing.T) {
tests := []struct {
name string
imports []string
setup func(*testing.T, string)
want string
}{
{name: "empty", imports: []string{""}, want: "non-empty path"},
{name: "surrounding whitespace", imports: []string{" one.yml "}, want: "surrounding whitespace"},
{name: "absolute", imports: []string{"/tmp/one.yml"}, want: "invalid"},
{name: "traversal", imports: []string{"../one.yml"}, want: "invalid"},
{name: "unsupported extension", imports: []string{"one.json"}, want: ".yml or .yaml"},
{name: "missing", imports: []string{"missing.yml"}, want: "open composition.imports"},
{name: "duplicate normalized", imports: []string{"one.yml", "./one.yml"}, setup: func(t *testing.T, dir string) {
writePipelineSource(t, dir, "one.yml", "workspace:\n root: /work\n")
}, want: "duplicates composition.imports"},
{name: "root self import", imports: []string{"pipeline.yml"}, want: "root pipeline itself"},
{name: "directory", imports: []string{"directory.yml"}, setup: func(t *testing.T, dir string) {
if err := os.Mkdir(filepath.Join(dir, "directory.yml"), 0o755); err != nil {
t.Fatal(err)
}
}, want: "is a directory"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
dir := t.TempDir()
if tt.setup != nil {
tt.setup(t, dir)
}
rootPath := writeImportRoot(t, dir, tt.imports)
_, err := LoadPipeline(rootPath)
if err == nil || !strings.Contains(strings.ToLower(err.Error()), strings.ToLower(tt.want)) {
t.Fatalf("LoadPipeline() error = %v, want containing %q", err, tt.want)
}
})
}
}
func TestLoadPipelineCompositionRejectsSameFileAliasesAndImportedComposition(t *testing.T) {
t.Run("same file through hard link", func(t *testing.T) {
dir := t.TempDir()
writePipelineSource(t, dir, "one.yml", "workspace:\n root: /work\n")
if err := os.Link(filepath.Join(dir, "one.yml"), filepath.Join(dir, "two.yml")); err != nil {
t.Skipf("hard links unavailable: %v", err)
}
rootPath := writeImportRoot(t, dir, []string{"one.yml", "two.yml"})
_, err := LoadPipeline(rootPath)
if err == nil || !strings.Contains(err.Error(), "same file") {
t.Fatalf("LoadPipeline() error = %v, want same-file rejection", err)
}
})
t.Run("imported composition", func(t *testing.T) {
dir := t.TempDir()
rootPath := writeImportRoot(t, dir, []string{"nested.yml"})
writePipelineSource(t, dir, "nested.yml", "composition:\n imports: []\n")
_, err := LoadPipeline(rootPath)
if err == nil || !strings.Contains(err.Error(), "only the root pipeline") || !strings.Contains(err.Error(), "nested.yml") {
t.Fatalf("LoadPipeline() error = %v, want imported composition rejection", err)
}
})
t.Run("empty profiles", func(t *testing.T) {
dir := t.TempDir()
rootPath := writePipelineSource(t, dir, "pipeline.yml", "composition:\n profiles: {}\n")
_, err := LoadPipeline(rootPath)
if err == nil || !strings.Contains(err.Error(), "must declare at least one named profile") {
t.Fatalf("LoadPipeline() error = %v, want empty profiles rejection", err)
}
})
}
func TestLoadPipelineCompositionReportsImportedParseAndSchemaSources(t *testing.T) {
t.Run("malformed imported YAML", func(t *testing.T) {
dir := t.TempDir()
rootPath := writeImportRoot(t, dir, []string{"broken.yml"})
brokenPath := writePipelineSource(t, dir, "broken.yml", "workspace: [\n")
_, err := LoadPipeline(rootPath)
if err == nil || !strings.Contains(err.Error(), absolutePath(t, brokenPath)) || !strings.Contains(err.Error(), "decode YAML") {
t.Fatalf("LoadPipeline() error = %v, want imported parse source", err)
}
})
t.Run("unknown imported field", func(t *testing.T) {
dir := t.TempDir()
rootPath := writeImportRoot(t, dir, []string{"unknown.yml"})
unknownPath := writePipelineSource(t, dir, "unknown.yml", "unknown_field: true\n")
_, err := LoadPipeline(rootPath)
if err == nil || !strings.Contains(err.Error(), absolutePath(t, unknownPath)) || !strings.Contains(err.Error(), "strict decode failed") {
t.Fatalf("LoadPipeline() error = %v, want assembled source-aware strict error", err)
}
})
}
func TestLoadPipelineCompositionKeepsRelativePathsRootBased(t *testing.T) {
rootDir := t.TempDir()
monolithicPath := writePipelineSource(t, rootDir, "monolithic.yml", testPipelineBaseYAML+`
notarius:
enabled: true
config_path: tool/notarius.yml
pipeline_id: dnd-session
outputs:
npc_registry:
lane_id: npc-registry
media_type: application/json
schema_id: notarius.dnd.npc_registry
schema_version: v1
`)
composedPath := writePipelineSource(t, rootDir, "pipeline.yml", `composition:
imports: [conf.d/extraction.yml]
`+testPipelineBaseYAML)
writePipelineSource(t, rootDir, "conf.d/extraction.yml", `notarius:
enabled: true
config_path: tool/notarius.yml
pipeline_id: dnd-session
outputs:
npc_registry:
lane_id: npc-registry
media_type: application/json
schema_id: notarius.dnd.npc_registry
schema_version: v1
`)
monolithic, err := LoadPipeline(monolithicPath)
if err != nil {
t.Fatal(err)
}
composed, err := LoadPipeline(composedPath)
if err != nil {
t.Fatal(err)
}
want := filepath.Join(rootDir, "tool", "notarius.yml")
if monolithic.Notarius.ConfigPath != want || composed.Notarius.ConfigPath != want {
t.Fatalf("config paths = monolithic %q composed %q, want %q", monolithic.Notarius.ConfigPath, composed.Notarius.ConfigPath, want)
}
if monolithic.Notarius.WorkingDirectory != filepath.Dir(want) || composed.Notarius.WorkingDirectory != filepath.Dir(want) {
t.Fatalf("working directories = %q / %q", monolithic.Notarius.WorkingDirectory, composed.Notarius.WorkingDirectory)
}
}
func writeImportRoot(t *testing.T, dir string, imports []string) string {
t.Helper()
var builder strings.Builder
builder.WriteString("composition:\n imports:\n")
for _, imported := range imports {
builder.WriteString(" - ")
if imported == "" {
builder.WriteString(`""`)
} else {
builder.WriteString(`"` + imported + `"`)
}
builder.WriteByte('\n')
}
builder.WriteString("whisperx:\n transcribe_url: https://transcription.example.com/transcribe\n")
return writePipelineSource(t, dir, "pipeline.yml", builder.String())
}
func writePipelineSource(t *testing.T, root, relative, content string) string {
t.Helper()
path := filepath.Join(root, filepath.FromSlash(relative))
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
t.Fatal(err)
}
return path
}
func absolutePath(t *testing.T, path string) string {
t.Helper()
absolute, err := filepath.Abs(path)
if err != nil {
t.Fatal(err)
}
return absolute
}
func assertPipelineFieldOwner(t *testing.T, cfg *PipelineConfig, path, source string) {
t.Helper()
if cfg == nil || cfg.resolution == nil {
t.Fatal("pipeline resolution metadata is absent")
}
for _, ownership := range cfg.resolution.ownership {
if ownership.path == path {
if !slices.Equal(ownership.sources, []string{source}) {
t.Fatalf("owner of %s = %#v, want %q", path, ownership.sources, source)
}
return
}
}
t.Fatalf("ownership path %q not found: %#v", path, cfg.resolution.ownership)
}

View File

@@ -0,0 +1,391 @@
package config
import (
"bytes"
"errors"
"fmt"
"path/filepath"
"reflect"
"sort"
"strings"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
)
type pipelineResolutionMetadata struct {
rootPath string
imports []string
sources []string
selectedProfile *pipelineProfileSelection
effectiveDigest string
logicalNotariusConfig string
logicalNotariusWorking string
logicalNotariusCaptured bool
ownership []pipelineFieldOwnership
artifactFamilies map[string]ScriptoriumArtifactFamilyConfig
artifactFamiliesExpanded bool
publishDeclared bool
configkitMetadata configkit.Metadata
configkitSources []configkit.SourceInfo
}
type pipelineProfileSelection struct {
name string
source string
overlayPath string
}
type pipelineFieldOwnership struct {
path string
sources []string
}
func loadPipelineWithConfigKit(path string, opts PipelineLoadOptions) (*PipelineConfig, error) {
profile := selectedPipelineProfileName(opts)
result, err := configkit.Load(configkit.File(path), configkit.Options[PipelineConfig]{
Imports: true,
Profile: profile,
Definition: configkit.Definition[PipelineConfig]{
Defaults: pipelineConfigDefaults,
Resolve: resolvePipelineConfig,
Validate: validatePipelineConfigKit,
},
})
if err != nil {
return nil, describePipelineLoadError(err)
}
cfg := result.Config
metadata, err := pipelineMetadataFromConfigKit(&cfg, result, opts)
if err != nil {
return nil, fmt.Errorf("load pipeline config: %w", err)
}
cfg.resolution = metadata
return &cfg, nil
}
func describePipelineLoadError(err error) error {
var loadError *configkit.LoadError
if !errors.As(err, &loadError) {
return fmt.Errorf("load pipeline config: %w", err)
}
var diagnostics bytes.Buffer
resolved := loadError.Diagnostics()
sourcePaths := make(map[string]string, len(loadError.Sources()))
for _, source := range loadError.Sources() {
if source.Path != "" {
sourcePaths[source.ID] = source.Path
} else if source.Label != "" {
sourcePaths[source.ID] = source.Label
}
}
for index := range resolved {
if resolved[index].Primary != nil {
if name := sourcePaths[resolved[index].Primary.SourceID]; name != "" {
resolved[index].Primary.SourceID = name
}
}
for relatedIndex := range resolved[index].Related {
if name := sourcePaths[resolved[index].Related[relatedIndex].SourceID]; name != "" {
resolved[index].Related[relatedIndex].SourceID = name
}
}
}
if writeErr := configkit.WriteDiagnostics(&diagnostics, resolved); writeErr != nil {
return fmt.Errorf("load pipeline config: %w", err)
}
return fmt.Errorf("load pipeline config: strict decode failed or validation failed: %w: %s", err, strings.TrimSpace(diagnostics.String()))
}
func selectedPipelineProfileName(opts PipelineLoadOptions) string {
if opts.Profile == nil || *opts.Profile == "" || *opts.Profile == "default" {
return ""
}
return *opts.Profile
}
func pipelineConfigDefaults() PipelineConfig {
return PipelineConfig{
Workspace: WorkspaceConfig{Root: DefaultWorkspaceRoot},
Campaigns: CampaignsConfig{Root: DefaultCampaignsRoot},
Storage: StorageConfig{Backend: StorageBackendLocal},
Spool: SpoolConfig{Root: DefaultSpoolRoot},
Cache: CacheConfig{Root: DefaultCacheRoot, S3Audio: boolPtr(DefaultCacheS3Audio)},
Publish: &PublishConfig{
Enabled: boolPtr(DefaultArchiveEnabled), UploadRun: boolPtr(DefaultArchiveUploadRun),
Outputs: defaultPublishOutputs(),
},
WhisperX: WhisperXConfig{
Language: DefaultWhisperXLanguage, Timeout: DefaultWhisperXTimeout,
RetryDelay: DefaultWhisperXRetryDelay, Concurrency: intPtr(DefaultWhisperXConcurrency),
Retries: intPtr(DefaultWhisperXRetries),
},
Seriatim: SeriatimConfig{
Binary: DefaultSeriatimBinary, Timeout: DefaultSeriatimTimeout,
OutputSchema: DefaultSeriatimOutputSchema, CoalesceGap: float64Ptr(DefaultSeriatimCoalesceGap),
Report: boolPtr(DefaultSeriatimReport),
},
Audita: AuditaConfig{Binary: DefaultAuditaBinary, Timeout: DefaultAuditaTimeout, Report: boolPtr(DefaultAuditaReport)},
Normalize: &NormalizeConfig{OutputPath: DefaultNormalizeOutputPath, OutputSchema: DefaultNormalizeOutputSchema, Report: boolPtr(DefaultNormalizeReport)},
Trim: &TrimConfig{
Enabled: boolPtr(DefaultTrimEnabled), OutputPath: DefaultTrimOutputPath,
Bounds: TrimBoundsConfig{
PromptID: DefaultTrimBoundsPromptID, TranscriptInputName: DefaultTrimBoundsTranscriptInputName,
OutputPath: DefaultTrimBoundsOutputPath, Timeout: DefaultTrimBoundsTimeout,
},
Seriatim: TrimSeriatimConfig{Report: boolPtr(DefaultTrimSeriatimReport)},
},
Render: &RenderConfig{
Enabled: boolPtr(DefaultRenderEnabled), Format: DefaultRenderFormat, Title: DefaultRenderTitle,
IncludeTimestamps: boolPtr(DefaultRenderTimestamps), IncludeSegmentIDs: boolPtr(DefaultRenderSegmentIDs),
},
Notification: NotificationConfig{Mode: DefaultNotificationMode},
}
}
func defaultPublishOutputs() []PublishOutputRule {
outputs := append([]PublishOutputRule(nil), DefaultPublishOutputs...)
for index := range outputs {
outputs[index].Required = boolPtr(true)
}
return outputs
}
func resolvePipelineConfig(cfg *PipelineConfig, resolution *configkit.Resolution) error {
backend := strings.ToLower(strings.TrimSpace(cfg.Storage.Backend))
if backend != cfg.Storage.Backend {
configkit.Normalize(resolution, configkit.Keys("storage", "backend"), &cfg.Storage.Backend, backend, "canonical storage backend")
}
if cfg.Storage.S3 != nil {
path := configkit.Keys("storage", "s3")
configkit.Default(resolution, path.Key("root_prefix"), &cfg.Storage.S3.RootPrefix, DefaultStorageS3RootPrefix, "default S3 root prefix")
configkit.Default(resolution, path.Key("access_key_id_env"), &cfg.Storage.S3.AccessKeyIDEnv, DefaultS3AccessKeyIDEnv, "default S3 access-key reference")
configkit.Default(resolution, path.Key("secret_access_key_env"), &cfg.Storage.S3.SecretKeyEnv, DefaultS3SecretAccessKeyEnv, "default S3 secret-key reference")
}
if cfg.Trim != nil && cfg.Trim.Enabled != nil && *cfg.Trim.Enabled && cfg.Scriptorium == nil {
value := &ScriptoriumConfig{Binary: DefaultScriptoriumBinary, Timeout: DefaultScriptoriumTimeout}
configkit.Default(resolution, configkit.Keys("scriptorium"), &cfg.Scriptorium, value, "trim requires Scriptorium")
}
if cfg.Scriptorium != nil {
path := configkit.Keys("scriptorium")
configkit.Default(resolution, path.Key("binary"), &cfg.Scriptorium.Binary, DefaultScriptoriumBinary, "default Scriptorium binary")
configkit.Default(resolution, path.Key("timeout"), &cfg.Scriptorium.Timeout, DefaultScriptoriumTimeout, "default Scriptorium timeout")
}
if cfg.Notarius != nil {
path := configkit.Keys("notarius")
configkit.Default(resolution, path.Key("binary"), &cfg.Notarius.Binary, DefaultNotariusBinary, "default Notarius binary")
configkit.Default(resolution, path.Key("timeout"), &cfg.Notarius.Timeout, DefaultNotariusTimeout, "default Notarius timeout")
if cfg.Notarius.Enabled && cfg.Notarius.ConfigPath != "" {
configkit.Default(resolution, path.Key("working_directory"), &cfg.Notarius.WorkingDirectory,
filepath.Dir(cfg.Notarius.ConfigPath), "derived from Notarius config path", path.Key("config_path"))
}
}
if cfg.Publish != nil {
defaultedOutput := false
for index := range cfg.Publish.Outputs {
output := cfg.Publish.Outputs[index]
path := configkit.Keys("publish", "outputs").Index(index).Key("required")
if !resolution.IsExplicit(path) {
defaultedOutput = true
}
configkit.Default(resolution, path, &output.Required, boolPtr(true), "publish outputs are required by default")
cfg.Publish.Outputs[index] = output
}
if defaultedOutput {
resolution.Record(configkit.Keys("publish", "outputs"), configkit.ChangeDerivedDefault, "publish output entry defaults")
}
}
before := cloneNotariusForResolution(cfg.Notarius)
if err := NormalizePipelineConfig(cfg); err != nil {
resolution.Error(configkit.Keys("notarius"), err.Error())
return nil
}
if !reflect.DeepEqual(before, cfg.Notarius) {
resolution.Record(configkit.Keys("notarius"), configkit.ChangeNormalization, "canonical Notarius configuration")
}
if cfg.Publish != nil {
configuredOutputs := scriptoriumOutputPathMap(cfg.Scriptorium)
extractionOutputs := notariusOutputKeySet(cfg.Notarius)
for index := range cfg.Publish.Outputs {
output := cfg.Publish.Outputs[index]
if strings.TrimSpace(output.Dest) != "" {
continue
}
destination, err := artifactpolicy.ResolvePublishedDestinationWithExtractions(strings.TrimSpace(output.Source), "", configuredOutputs, extractionOutputs)
if err != nil {
continue
}
configkit.Normalize(resolution, configkit.Keys("publish", "outputs").Index(index).Key("dest"), &output.Dest, destination, "derived from publish source")
cfg.Publish.Outputs[index] = output
}
}
return nil
}
func cloneNotariusForResolution(value *NotariusConfig) *NotariusConfig {
if value == nil {
return nil
}
clone := *value
clone.References = cloneStringMap(value.References)
if value.Outputs != nil {
clone.Outputs = make(map[string]NotariusOutputConfig, len(value.Outputs))
for key, output := range value.Outputs {
clone.Outputs[key] = output
}
}
return &clone
}
func cloneStringMap(value map[string]string) map[string]string {
if value == nil {
return nil
}
clone := make(map[string]string, len(value))
for key, item := range value {
clone[key] = item
}
return clone
}
func validatePipelineConfigKit(cfg *PipelineConfig, validation *configkit.Validation) {
// Checks that depend on family expansion or campaign/session context remain
// at Narratio's explicit resolution boundary. This early definition check is
// deliberately limited to an invariant that is wholly file-local.
if strings.TrimSpace(cfg.Workspace.Root) == "" {
validation.Error(configkit.Keys("workspace", "root"), "workspace root is required")
}
}
func pipelineMetadataFromConfigKit(cfg *PipelineConfig, result *configkit.Result[PipelineConfig], opts PipelineLoadOptions) (*pipelineResolutionMetadata, error) {
sources := result.Sources()
metadata := result.Metadata()
if err := rejectDuplicateBaseDefinitions(cfg, metadata, sources); err != nil {
return nil, err
}
resolved := &pipelineResolutionMetadata{
configkitMetadata: metadata, configkitSources: sources,
publishDeclared: metadata.IsExplicit(configkit.Keys("publish")),
}
pathsByID := make(map[string]string, len(sources))
for index, source := range sources {
path := source.Path
if path == "" {
path = source.Label
}
pathsByID[source.ID] = path
resolved.sources = append(resolved.sources, path)
if index == 0 {
resolved.rootPath = path
} else if source.Layer == configkit.LayerBase {
resolved.imports = append(resolved.imports, path)
}
if source.Layer == configkit.LayerProfile {
resolved.selectedProfile = &pipelineProfileSelection{name: selectedPipelineProfileName(opts), source: "explicit", overlayPath: path}
}
}
detailed, err := pipelineMetadataPaths(cfg)
if err != nil {
return nil, err
}
for _, value := range detailed {
info, ok := metadata.Lookup(configkit.Keys(value.segments...))
sourceNames := []string{pipelineDefaultOwnershipSource}
if ok {
sourceNames = ownershipSources(info.Origin, pathsByID)
}
resolved.ownership = append(resolved.ownership, pipelineFieldOwnership{path: value.record.Path, sources: sourceNames})
}
return resolved, nil
}
func pipelineMetadataPaths(cfg *PipelineConfig) ([]projectedPipelineValue, error) {
effective, err := projectionDetailedRecords(projectEffectivePipeline(cfg, false))
if err != nil {
return nil, err
}
if cfg.Scriptorium == nil || len(cfg.Scriptorium.ArtifactFamilies) == 0 {
return effective, nil
}
families, err := projectionDetailedRecords(map[string]any{"scriptorium": map[string]any{"artifact_families": cfg.Scriptorium.ArtifactFamilies}})
if err != nil {
return nil, err
}
return append(effective, families...), nil
}
func ownershipSources(origin configkit.Origin, pathsByID map[string]string) []string {
if origin.Layer == configkit.LayerDefault || (origin.Layer == configkit.LayerResolution && origin.Operation == configkit.ChangeDerivedDefault) {
return []string{pipelineDefaultOwnershipSource}
}
if origin.Layer == configkit.LayerResolution {
return []string{"resolution"}
}
seen := map[string]struct{}{}
var sources []string
for _, location := range origin.Locations {
value := pathsByID[location.SourceID]
if value == "" {
value = location.SourceID
}
if _, ok := seen[value]; !ok {
seen[value] = struct{}{}
sources = append(sources, value)
}
}
if len(sources) == 0 {
return []string{pipelineDefaultOwnershipSource}
}
return sources
}
func rejectDuplicateBaseDefinitions(cfg *PipelineConfig, metadata configkit.Metadata, sources []configkit.SourceInfo) error {
pathsByID := make(map[string]string, len(sources))
for _, source := range sources {
pathsByID[source.ID] = source.Path
}
var declarations []struct {
kind string
key string
path configkit.Path
}
if cfg.Scriptorium != nil {
for key := range cfg.Scriptorium.Artifacts {
declarations = append(declarations, struct {
kind, key string
path configkit.Path
}{"artifact", key, configkit.Keys("scriptorium", "artifacts", key)})
}
for key := range cfg.Scriptorium.ArtifactFamilies {
declarations = append(declarations, struct {
kind, key string
path configkit.Path
}{"artifact family", key, configkit.Keys("scriptorium", "artifact_families", key)})
}
}
sort.Slice(declarations, func(i, j int) bool { return declarations[i].path.String() < declarations[j].path.String() })
for _, declaration := range declarations {
seen := map[string]struct{}{}
for _, change := range metadata.History(declaration.path) {
if change.Origin.Layer != configkit.LayerBase {
continue
}
for _, location := range change.Origin.Locations {
seen[location.SourceID] = struct{}{}
}
}
if len(seen) > 1 {
claims := make([]string, 0, len(seen))
for sourceID := range seen {
claims = append(claims, pathsByID[sourceID])
}
sort.Strings(claims)
return fmt.Errorf("pipeline Scriptorium %s %q is declared by multiple base files: %s", declaration.kind, declaration.key, strings.Join(claims, ", "))
}
}
return nil
}
func intPtr(value int) *int { return &value }
func float64Ptr(value float64) *float64 { return &value }
func boolPtr(value bool) *bool { return &value }

View File

@@ -0,0 +1,102 @@
package config
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestPipelineConfigKitImportsProfileAndBaseSelection(t *testing.T) {
dir := t.TempDir()
writePipelineFixture(t, dir, "pipeline.yml", `configkit:
imports:
- path: config/workspace.yml
- path: config/artifact.yml
whisperx:
transcribe_url: https://transcription.example.test/transcribe
`)
writePipelineFixture(t, dir, "config/workspace.yml", "workspace:\n root: /srv/base\n")
writePipelineFixture(t, dir, "config/artifact.yml", `scriptorium:
artifacts:
recap:
enabled: false
output_path: artifacts/recap.md
`)
writePipelineFixture(t, dir, "profiles/testing.yml", "workspace:\n root: /srv/testing\n")
writePipelineFixture(t, dir, "profiles/unselected.yml", "unknown: [\n")
root := filepath.Join(dir, "pipeline.yml")
for _, profile := range []*string{nil, stringPointer(""), stringPointer("default")} {
cfg, err := LoadPipelineWithOptions(root, PipelineLoadOptions{Profile: profile})
if err != nil {
t.Fatalf("load base selection %#v: %v", profile, err)
}
if cfg.Workspace.Root != "/srv/base" {
t.Fatalf("base workspace = %q", cfg.Workspace.Root)
}
}
selected := "testing"
cfg, err := LoadPipelineWithOptions(root, PipelineLoadOptions{Profile: &selected})
if err != nil {
t.Fatal(err)
}
if cfg.Workspace.Root != "/srv/testing" || len(EffectivePipelineImports(cfg)) != 2 {
t.Fatalf("profiled pipeline = %#v, imports=%#v", cfg.Workspace, EffectivePipelineImports(cfg))
}
profile, ok := SelectedPipelineProfile(cfg)
if !ok || profile.Name != "testing" || profile.OverlayPath != filepath.Join(dir, "profiles", "testing.yml") {
t.Fatalf("selected profile = %#v, %v", profile, ok)
}
}
func TestPipelineConfigKitRejectsDuplicateBaseDeclarations(t *testing.T) {
for _, test := range []struct {
name, first, second string
}{
{name: "artifact", first: "scriptorium:\n artifacts:\n recap:\n prompt_id: recap\n", second: "scriptorium:\n artifacts:\n recap:\n output_path: artifacts/recap.md\n"},
{name: "artifact family", first: "scriptorium:\n artifact_families:\n recap:\n for_each: party.characters\n", second: "scriptorium:\n artifact_families:\n recap:\n output_path_pattern: artifacts/{character_id}.md\n"},
} {
t.Run(test.name, func(t *testing.T) {
dir := t.TempDir()
writePipelineFixture(t, dir, "pipeline.yml", `configkit:
imports:
- path: first.yml
- path: second.yml
whisperx:
transcribe_url: https://transcription.example.test/transcribe
`)
writePipelineFixture(t, dir, "first.yml", test.first)
writePipelineFixture(t, dir, "second.yml", test.second)
_, err := LoadPipeline(filepath.Join(dir, "pipeline.yml"))
if err == nil || !strings.Contains(err.Error(), "declared by multiple base files") ||
!strings.Contains(err.Error(), "first.yml") || !strings.Contains(err.Error(), "second.yml") {
t.Fatalf("duplicate declaration error = %v", err)
}
})
}
}
func TestPipelineConfigKitRejectsUnknownDirectiveAlias(t *testing.T) {
dir := t.TempDir()
path := writePipelineFixture(t, dir, "pipeline.yml", "composition:\n imports: []\n")
_, err := LoadPipeline(path)
if err == nil || !strings.Contains(err.Error(), "decode.unknown_field") {
t.Fatalf("composition alias error = %v", err)
}
}
func writePipelineFixture(t *testing.T, root, relative, contents string) string {
t.Helper()
path := filepath.Join(root, filepath.FromSlash(relative))
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte(contents), 0o644); err != nil {
t.Fatal(err)
}
return path
}
func stringPointer(value string) *string { return &value }

View File

@@ -1,479 +0,0 @@
package config
import (
"os"
"path/filepath"
"slices"
"strings"
"testing"
"github.com/google/go-cmp/cmp"
)
func TestLoadPipelineProfilesSelectDefaultAndExplicitOverlay(t *testing.T) {
dir := t.TempDir()
rootPath := writeProfilePipeline(t, dir, `composition:
imports: [conf.d/base.yml]
default_profile: production
profiles:
production:
overlay: profiles/production.yml
testing:
overlay: profiles/testing.yml
whisperx:
transcribe_url: https://transcription.example.com/transcribe
`)
importPath := writePipelineSource(t, dir, "conf.d/base.yml", `workspace:
root: /srv/base
audita:
modules: [base, shared]
`)
productionPath := writePipelineSource(t, dir, "profiles/production.yml", `workspace:
root: /srv/production
whisperx:
language: en
`)
testingPath := writePipelineSource(t, dir, "profiles/testing.yml", `workspace:
root: /srv/testing
whisperx:
language: fr
`)
production, err := LoadPipeline(rootPath)
if err != nil {
t.Fatal(err)
}
if production.Workspace.Root != "/srv/production" || production.WhisperX.Language != "en" {
t.Fatalf("default profile result = workspace=%q language=%q", production.Workspace.Root, production.WhisperX.Language)
}
assertSelectedPipelineProfile(t, production, "production", "default", productionPath)
if want := []string{absolutePath(t, rootPath), absolutePath(t, importPath), absolutePath(t, productionPath)}; !slices.Equal(production.resolution.sources, want) {
t.Fatalf("default sources = %#v, want %#v", production.resolution.sources, want)
}
assertPipelineFieldOwner(t, production, "workspace.root", absolutePath(t, productionPath))
assertPipelineFieldOwner(t, production, "audita.modules", absolutePath(t, importPath))
assertPipelineFieldOwner(t, production, "storage.backend", pipelineDefaultOwnershipSource)
profile := "testing"
testingCfg, err := LoadPipelineWithOptions(rootPath, PipelineLoadOptions{Profile: &profile})
if err != nil {
t.Fatal(err)
}
if testingCfg.Workspace.Root != "/srv/testing" || testingCfg.WhisperX.Language != "fr" {
t.Fatalf("explicit profile result = workspace=%q language=%q", testingCfg.Workspace.Root, testingCfg.WhisperX.Language)
}
assertSelectedPipelineProfile(t, testingCfg, "testing", "cli", testingPath)
}
func TestLoadPipelineProfileSelectionErrors(t *testing.T) {
for _, test := range []struct {
name string
root string
profile *string
want string
}{
{name: "profiles require selection", root: profileComposition("", "production"), want: "default_profile is omitted"},
{name: "unknown explicit", root: profileComposition("production", "production"), profile: profilePointer("unknown"), want: "not declared"},
{name: "stacked explicit", root: profileComposition("production", "production", "testing"), profile: profilePointer("production,testing"), want: "not declared"},
{name: "explicit empty", root: profileComposition("production", "production"), profile: profilePointer(""), want: "must be non-empty"},
{name: "explicit whitespace", root: profileComposition("production", "production"), profile: profilePointer(" production "), want: "surrounding whitespace"},
{name: "invalid default", root: profileComposition("unknown", "production"), want: "does not name a declared profile"},
{name: "empty default", root: profileComposition("EMPTY", "production"), want: "must be non-empty"},
{name: "empty profile name", root: `composition:
default_profile: production
profiles:
"":
overlay: production.yml
whisperx:
transcribe_url: https://transcription.example.com/transcribe
`, want: "profile name must be non-empty"},
} {
t.Run(test.name, func(t *testing.T) {
dir := t.TempDir()
root := test.root
if test.name == "empty default" {
root = strings.ReplaceAll(root, "default_profile: EMPTY", `default_profile: ""`)
}
rootPath := writeProfilePipeline(t, dir, root)
writePipelineSource(t, dir, "production.yml", "whisperx:\n language: en\n")
_, err := LoadPipelineWithOptions(rootPath, PipelineLoadOptions{Profile: test.profile})
if err == nil || !strings.Contains(strings.ToLower(err.Error()), strings.ToLower(test.want)) {
t.Fatalf("LoadPipelineWithOptions() error = %v, want containing %q", err, test.want)
}
})
}
t.Run("explicit against profile-free pipeline", func(t *testing.T) {
dir := t.TempDir()
rootPath := writeProfilePipeline(t, dir, testPipelineBaseYAML)
selected := "testing"
_, err := LoadPipelineWithOptions(rootPath, PipelineLoadOptions{Profile: &selected})
if err == nil || !strings.Contains(err.Error(), "declares no profiles") {
t.Fatalf("error = %v, want profile-free rejection", err)
}
})
}
func TestPipelineProfileNamesRejectControlCharacters(t *testing.T) {
if _, err := normalizePipelineProfileName("production\x00testing", "profile"); err == nil || !strings.Contains(err.Error(), "control characters") {
t.Fatalf("normalizePipelineProfileName() error = %v, want control-character rejection", err)
}
}
func TestLoadPipelineProfilesValidateEveryDeclaredOverlay(t *testing.T) {
for _, test := range []struct {
name string
path string
content string
setup func(*testing.T, string)
want string
}{
{name: "missing", path: "profiles/missing.yml", want: "open composition.profiles.testing.overlay"},
{name: "malformed", path: "profiles/testing.yml", content: "workspace: [\n", want: "decode yaml"},
{name: "duplicate keys", path: "profiles/testing.yml", content: "workspace:\n root: /one\n root: /two\n", want: "duplicate yaml key"},
{name: "trailing document", path: "profiles/testing.yml", content: "workspace:\n root: /one\n---\nworkspace:\n root: /two\n", want: "exactly one yaml document"},
{name: "nested composition", path: "profiles/testing.yml", content: "composition:\n imports: [nested.yml]\n", want: "only the root pipeline"},
{name: "inheritance", path: "profiles/testing.yml", content: "workspace:\n root: /testing\n", setup: func(t *testing.T, dir string) {
rootPath := filepath.Join(dir, "pipeline.yml")
data, err := os.ReadFile(rootPath)
if err != nil {
t.Fatal(err)
}
updated := strings.Replace(string(data), "testing:\n overlay:", "testing:\n extends: production\n overlay:", 1)
if err := os.WriteFile(rootPath, []byte(updated), 0o644); err != nil {
t.Fatal(err)
}
}, want: "only overlay is supported"},
{name: "traversal", path: "../testing.yml", want: "invalid"},
{name: "extension", path: "profiles/testing.json", content: "{}\n", want: ".yml or .yaml"},
{name: "directory", path: "profiles/testing.yml", setup: func(t *testing.T, dir string) {
if err := os.MkdirAll(filepath.Join(dir, "profiles/testing.yml"), 0o755); err != nil {
t.Fatal(err)
}
}, want: "is a directory"},
} {
t.Run(test.name, func(t *testing.T) {
dir := t.TempDir()
rootPath := writeProfilePipeline(t, dir, `composition:
default_profile: production
profiles:
production:
overlay: profiles/production.yml
testing:
overlay: `+test.path+`
whisperx:
transcribe_url: https://transcription.example.com/transcribe
`)
writePipelineSource(t, dir, "profiles/production.yml", "whisperx:\n language: en\n")
if test.content != "" {
writePipelineSource(t, dir, test.path, test.content)
}
if test.setup != nil {
test.setup(t, dir)
}
_, err := LoadPipeline(rootPath)
if err == nil || !strings.Contains(strings.ToLower(err.Error()), strings.ToLower(test.want)) {
t.Fatalf("LoadPipeline() error = %v, want containing %q", err, test.want)
}
})
}
}
func TestLoadPipelineProfileOverlaySemantics(t *testing.T) {
dir := t.TempDir()
rootPath := writeProfilePipeline(t, dir, `composition:
default_profile: testing
profiles:
testing:
overlay: testing.yml
workspace:
root: /base
cleanup_after_publish: true
whisperx:
transcribe_url: https://transcription.example.com/transcribe
retries: 3
audita:
modules: [base, shared]
scriptorium:
artifacts:
session_recap:
enabled: true
prompt_id: dnd.session_recap
output_path: artifacts/session_recap.md
`)
writePipelineSource(t, dir, "testing.yml", `workspace:
cleanup_after_publish: false
whisperx:
retries: 0
audita:
modules: []
scriptorium:
artifacts:
session_recap:
enabled: false
experimental:
enabled: false
`)
cfg, err := LoadPipeline(rootPath)
if err != nil {
t.Fatal(err)
}
if cfg.Workspace.Root != "/base" || cfg.Workspace.CleanupAfterPublish || cfg.WhisperX.Retries == nil || *cfg.WhisperX.Retries != 0 {
t.Fatalf("recursive/zero overlay = workspace=%#v retries=%#v", cfg.Workspace, cfg.WhisperX.Retries)
}
if cfg.Audita.Modules == nil || len(cfg.Audita.Modules) != 0 {
t.Fatalf("list replacement = %#v, want explicit empty list", cfg.Audita.Modules)
}
if cfg.Scriptorium == nil || cfg.Scriptorium.Artifacts["session_recap"].Enabled || len(cfg.Scriptorium.Artifacts) != 2 {
t.Fatalf("keyed artifact overlay = %#v", cfg.Scriptorium)
}
}
func TestLoadPipelineProfileRejectsNullAndKindChanges(t *testing.T) {
for _, test := range []struct {
name string
overlay string
want string
}{
{name: "null deletion", overlay: "workspace:\n root: null\n", want: "null cannot delete"},
{name: "mapping scalar", overlay: "workspace: /other\n", want: "kind change"},
{name: "list mapping", overlay: "audita:\n modules:\n testing: true\n", want: "kind change"},
} {
t.Run(test.name, func(t *testing.T) {
dir := t.TempDir()
rootPath := writeProfilePipeline(t, dir, `composition:
default_profile: testing
profiles:
testing:
overlay: testing.yml
workspace:
root: /base
audita:
modules: [base]
whisperx:
transcribe_url: https://transcription.example.com/transcribe
`)
writePipelineSource(t, dir, "testing.yml", test.overlay)
_, err := LoadPipeline(rootPath)
if err == nil || !strings.Contains(strings.ToLower(err.Error()), strings.ToLower(test.want)) {
t.Fatalf("LoadPipeline() error = %v, want containing %q", err, test.want)
}
})
}
}
func TestPipelineEffectiveDigestTracksNormalizedMeaning(t *testing.T) {
dir := t.TempDir()
monolithicPath := writePipelineSource(t, dir, "monolithic.yml", `workspace:
root: /srv/narratio
whisperx:
transcribe_url: https://transcription.example.com/transcribe
language: en
`)
composedPath := writePipelineSource(t, dir, "pipeline.yml", `composition:
imports: [workspace.yml]
whisperx:
transcribe_url: https://transcription.example.com/transcribe
language: en
`)
writePipelineSource(t, dir, "workspace.yml", "workspace:\n root: /srv/narratio\n")
monolithic, err := LoadPipeline(monolithicPath)
if err != nil {
t.Fatal(err)
}
composed, err := LoadPipeline(composedPath)
if err != nil {
t.Fatal(err)
}
if monolithic.resolution.effectiveDigest == "" || monolithic.resolution.effectiveDigest != composed.resolution.effectiveDigest {
t.Fatalf("equal effective results have digests %q and %q", monolithic.resolution.effectiveDigest, composed.resolution.effectiveDigest)
}
changedPath := writePipelineSource(t, dir, "changed.yml", `workspace:
root: /srv/narratio
whisperx:
transcribe_url: https://transcription.example.com/transcribe
language: fr
`)
changed, err := LoadPipeline(changedPath)
if err != nil {
t.Fatal(err)
}
if changed.resolution.effectiveDigest == monolithic.resolution.effectiveDigest {
t.Fatal("semantic value change retained effective digest")
}
t.Setenv("AWS_SECRET_ACCESS_KEY", "secret-one")
first, err := LoadPipeline(monolithicPath)
if err != nil {
t.Fatal(err)
}
t.Setenv("AWS_SECRET_ACCESS_KEY", "secret-two")
second, err := LoadPipeline(monolithicPath)
if err != nil {
t.Fatal(err)
}
if first.resolution.effectiveDigest != second.resolution.effectiveDigest || strings.Contains(first.resolution.effectiveDigest, "secret") {
t.Fatalf("environment secret affected or appeared in digest: %q / %q", first.resolution.effectiveDigest, second.resolution.effectiveDigest)
}
}
func TestEffectivePipelineValuesIgnoreEquivalentSourceLayout(t *testing.T) {
dir := t.TempDir()
monolithicPath := writePipelineSource(t, dir, "monolithic.yml", `workspace:
root: /srv/narratio
whisperx:
transcribe_url: https://transcription.example.com/transcribe
language: en
`)
composedPath := writePipelineSource(t, dir, "pipeline.yml", `composition:
imports: [workspace.yml]
whisperx:
language: en
transcribe_url: https://transcription.example.com/transcribe
`)
writePipelineSource(t, dir, "workspace.yml", "workspace:\n root: /srv/narratio\n")
monolithic, err := LoadPipeline(monolithicPath)
if err != nil {
t.Fatal(err)
}
composed, err := LoadPipeline(composedPath)
if err != nil {
t.Fatal(err)
}
monolithicValues, err := EffectivePipelineValues(monolithic)
if err != nil {
t.Fatal(err)
}
composedValues, err := EffectivePipelineValues(composed)
if err != nil {
t.Fatal(err)
}
if diff := cmp.Diff(monolithicValues, composedValues); diff != "" {
t.Fatalf("effective values differ (-monolithic +composed):\n%s", diff)
}
}
func TestPipelineEffectiveDigestExcludesProfileIdentity(t *testing.T) {
dir := t.TempDir()
rootPath := writeProfilePipeline(t, dir, profileComposition("production", "production", "testing"))
overlay := "workspace:\n root: /srv/narratio\n"
writePipelineSource(t, dir, "production.yml", overlay)
writePipelineSource(t, dir, "testing.yml", overlay)
production := "production"
productionCfg, err := LoadPipelineWithOptions(rootPath, PipelineLoadOptions{Profile: &production})
if err != nil {
t.Fatal(err)
}
testing := "testing"
testingCfg, err := LoadPipelineWithOptions(rootPath, PipelineLoadOptions{Profile: &testing})
if err != nil {
t.Fatal(err)
}
if productionCfg.resolution.effectiveDigest != testingCfg.resolution.effectiveDigest {
t.Fatalf("profile identity changed equal effective digests: %q / %q", productionCfg.resolution.effectiveDigest, testingCfg.resolution.effectiveDigest)
}
}
func TestLoadPipelineProfilePairResolvesIndependentEffectivePipelines(t *testing.T) {
dir := t.TempDir()
rootPath := writeProfilePipeline(t, dir, `composition:
imports: [base.yml]
default_profile: production
profiles:
production:
overlay: production.yml
testing:
overlay: testing.yml
whisperx:
transcribe_url: https://transcription.example.com/transcribe
`)
writePipelineSource(t, dir, "base.yml", "workspace:\n root: /srv/base\n")
writePipelineSource(t, dir, "production.yml", "workspace:\n root: /srv/production\nwhisperx:\n language: en\n")
writePipelineSource(t, dir, "testing.yml", "workspace:\n root: /srv/testing\nwhisperx:\n language: fr\n")
production, testing, err := LoadPipelineProfilePair(rootPath, "production", "testing")
if err != nil {
t.Fatal(err)
}
if production.Workspace.Root != "/srv/production" || production.WhisperX.Language != "en" {
t.Fatalf("production pair result = %#v", production)
}
if testing.Workspace.Root != "/srv/testing" || testing.WhisperX.Language != "fr" {
t.Fatalf("testing pair result = %#v", testing)
}
production.Workspace.Root = "/mutated-left"
if testing.Workspace.Root != "/srv/testing" {
t.Fatalf("right profile shared mutable state with left: %q", testing.Workspace.Root)
}
testingFirst, productionSecond, err := LoadPipelineProfilePair(rootPath, "testing", "production")
if err != nil {
t.Fatal(err)
}
if testingFirst.Workspace.Root != "/srv/testing" || productionSecond.Workspace.Root != "/srv/production" {
t.Fatalf("reversed profile pair = testing=%q production=%q", testingFirst.Workspace.Root, productionSecond.Workspace.Root)
}
if _, _, err := LoadPipelineProfilePair(rootPath, "production", "production"); err == nil || !strings.Contains(err.Error(), "must differ") {
t.Fatalf("equal profile pair error = %v, want selection rejection", err)
}
if _, _, err := LoadPipelineProfilePair(rootPath, "unknown", "testing"); err == nil || !strings.Contains(err.Error(), "not declared") {
t.Fatalf("unknown profile pair error = %v, want selection rejection", err)
}
}
func TestLoadWithSessionOptionsCarriesExplicitProfilePresence(t *testing.T) {
dir := t.TempDir()
rootPath := writeProfilePipeline(t, dir, profileComposition("production", "production", "testing"))
writePipelineSource(t, dir, "production.yml", "workspace:\n root: /production\n")
writePipelineSource(t, dir, "testing.yml", "workspace:\n root: /testing\n")
campaignPath := writePipelineSource(t, dir, "campaign.yml", "campaign_id: campaign\ninputs:\n speakers_file: speakers.yml\n autocorrect_file: autocorrect.yml\n glossary_file: glossary.yml\n party_file: party.yml\n")
writePipelineSource(t, dir, "party.yml", canonicalPartyFixture)
sessionPath := writePipelineSource(t, dir, "session.yml", "session_id: session\ncampaign: campaign\n")
selected := "testing"
cfg, err := LoadWithSessionOptions(rootPath, campaignPath, sessionPath, SessionLoadOptions{Profile: &selected})
if err != nil {
t.Fatal(err)
}
if cfg.Pipeline.Workspace.Root != "/testing" || cfg.Pipeline.resolution.selectedProfile.source != "cli" {
t.Fatalf("combined profile result = workspace=%q metadata=%#v", cfg.Pipeline.Workspace.Root, cfg.Pipeline.resolution.selectedProfile)
}
}
func writeProfilePipeline(t *testing.T, dir, content string) string {
t.Helper()
return writePipelineSource(t, dir, "pipeline.yml", content)
}
func profileComposition(defaultProfile string, profiles ...string) string {
var builder strings.Builder
builder.WriteString("composition:\n")
if defaultProfile != "" {
builder.WriteString(" default_profile: " + defaultProfile + "\n")
}
builder.WriteString(" profiles:\n")
for _, profile := range profiles {
builder.WriteString(" " + profile + ":\n overlay: " + profile + ".yml\n")
}
builder.WriteString("whisperx:\n transcribe_url: https://transcription.example.com/transcribe\n")
return builder.String()
}
func profilePointer(value string) *string {
return &value
}
func assertSelectedPipelineProfile(t *testing.T, cfg *PipelineConfig, name, source, overlayPath string) {
t.Helper()
if cfg == nil || cfg.resolution == nil || cfg.resolution.selectedProfile == nil {
t.Fatal("selected profile provenance is absent")
}
selection := cfg.resolution.selectedProfile
if selection.name != name || selection.source != source || selection.overlayPath != absolutePath(t, overlayPath) {
t.Fatalf("selected profile = %#v, want name=%q source=%q overlay=%q", selection, name, source, absolutePath(t, overlayPath))
}
}

View File

@@ -129,7 +129,7 @@ func TestScriptoriumLoadAndValidate(t *testing.T) {
source: narratio.transcript.polished source: narratio.transcript.polished
artifact: session_recap artifact: session_recap
`, `,
wantLoadErr: "field artifact not found", wantLoadErr: "decode.unknown_field",
}, },
{ {
name: "canonical previous-session source is accepted", name: "canonical previous-session source is accepted",

View File

@@ -540,7 +540,7 @@ publish:
` `
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML) pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
_, err := Load(pipelinePath, sessionPath) _, err := Load(pipelinePath, sessionPath)
if err == nil || !strings.Contains(err.Error(), "strict decode failed") || !strings.Contains(err.Error(), "field locks not found") { if err == nil || !strings.Contains(err.Error(), "decode.unknown_field") {
t.Fatalf("Load() error = %v, want retired locks field rejected", err) t.Fatalf("Load() error = %v, want retired locks field rejected", err)
} }
} }

View File

@@ -57,12 +57,14 @@ func TestTrimLoadAndValidate(t *testing.T) {
}, },
}, },
{ {
name: "enabled omitted defaults enabled", name: "explicit null remains absent",
trimYAML: `trim: trimYAML: `trim:
`, `,
assert: func(t *testing.T, cfg *Config) { assert: func(t *testing.T, cfg *Config) {
t.Helper() t.Helper()
assertDefaultTrimConfig(t, cfg) if cfg.Pipeline.Trim != nil {
t.Fatalf("trim config = %#v, want nil", cfg.Pipeline.Trim)
}
}, },
}, },
{ {

View File

@@ -60,12 +60,7 @@ func validatePublish(cfg *PublishConfig, scriptorium *ScriptoriumConfig, notariu
} }
dest := strings.TrimSpace(item.Dest) dest := strings.TrimSpace(item.Dest)
if dest == "" { if dest == "" {
derivedDest, err := artifactpolicy.ResolvePublishedDestinationWithExtractions(source, "", configuredOutputs, extractionOutputs) return fmt.Errorf("%s.dest is required when destination cannot be derived from %q", prefix, source)
if err != nil {
return fmt.Errorf("%s.dest is required when destination cannot be derived from %q: %w", prefix, source, err)
}
dest = derivedDest
cfg.Outputs[i].Dest = derivedDest
} }
normalizedDest, err := pathsafe.NormalizeRelativeDestination(dest) normalizedDest, err := pathsafe.NormalizeRelativeDestination(dest)
if err != nil { if err != nil {