Adopt ConfigKit for pipeline loading

This commit is contained in:
2026-09-07 03:54:41 +00:00
parent 0e6ba414b3
commit 9e40c729e1
33 changed files with 683 additions and 3162 deletions

View File

@@ -53,10 +53,10 @@ Rules:
- if both positional `<session_id>` and `--session-id` are provided, values must match.
- `--previous-session-id` is a strict expectation: the selected session file
must contain the same `previous_session_id`.
- `--profile` selects a declared pipeline profile. It may be supplied once;
an explicit empty or unknown value fails configuration resolution. When it is
omitted, a declared `default_profile` is used. The same selection applies to
all common-flag commands, including `regenerate-artifacts`.
- `--profile` selects `profiles/<name>.yml` relative to the pipeline root. It
may be supplied once; omitted, empty, and the exact name `default` select the
base configuration, while an unknown named profile fails. The same selection
applies to all common-flag commands, including `regenerate-artifacts`.
- `clean --all` cannot be combined with campaign/session selectors.
- notification delivery is currently limited to the configured `noop` mode; see
the [configuration reference](./config.md#notifications).
@@ -101,7 +101,7 @@ Session, range, force, and artifact-execution flags are not accepted.
`config validate` writes a concise root-path, selected-profile (or `none`), and
effective-digest summary after successful complete validation. `config show`
writes one deterministic, secret-free YAML document containing defaulted and
expanded concrete configuration. It omits composition declarations, artifact
expanded concrete configuration. It omits ConfigKit directives, artifact
family declarations, and runtime provenance.
`config sources` reports the same fully validated resolution without printing
@@ -113,8 +113,8 @@ and generated family ownership. A generated party member has
one family record and one party record at the same logical path. The output
never reads or prints secret values.
`config diff` resolves both supplied profile names from one parsed root source
set and compares their fully resolved, secret-free effective mappings. It does
`config diff` resolves both supplied profile names independently and compares
their fully resolved, secret-free effective mappings. It does
not accept `--profile`; the two positional names must be distinct, declared
profiles. When party-driven families are present, both profiles must resolve to
the same selected campaign and party. Use `--campaign-file` if profile-specific

View File

@@ -49,8 +49,8 @@ sources` use the same `--config`, `--campaign`, `--campaign-file`, and
`--profile` selection rules as pipeline commands, but do not select, discover,
or load a session. They do not read credential values or create runtime state.
`narratio config diff <left-profile> <right-profile>` uses the same pipeline and
campaign selectors, resolves each named profile independently from one parsed
root source set, and does not accept a separate `--profile` flag.
campaign selectors, resolves each named profile with an independent ConfigKit
load, and does not accept a separate `--profile` flag.
Campaign selection is optional only when the resolved pipeline has no
`scriptorium.artifact_families`. When families are declared, Narratio selects a
@@ -59,7 +59,7 @@ then parses the campaign-owned party and expands concrete artifacts and any
family publish rules before validation. `config validate` prints the resulting
root, profile, and effective digest. `config show` emits the normalized
effective pipeline YAML, with defaults and concrete expansion included but
composition and family declarations omitted. `config sources` prints a stable
ConfigKit directives and family declarations omitted. `config sources` prints a stable
source projection instead of effective values: root/import/profile/default
ownership plus campaign/party and generated-family records. Canonical derived
players trace to the party. The [CLI reference](cli.md#config-validate-config-show-config-sources-and-config-diff)
@@ -77,8 +77,9 @@ difference.
Use the [production/testing bundle](../examples/production-testing/pipeline.yml)
as the complete copyable migration reference. Split stable pipeline settings
into explicit additive imports, place production/testing differences in one
selected overlay, and retain a production `default_profile`. Convert campaign
into explicit additive imports and place production/testing differences in the
fixed `profiles/production.yml` and `profiles/testing.yml` overlays. Convert
campaign
rosters to [canonical party input](integrations/party.md), then express
character work as families. Inspect the result
with `config validate`, `config show`, and `config sources`; use `config diff`
@@ -96,16 +97,15 @@ remote state with an unsafe legacy identity must be migrated before use.
## Validation and Merge Rules
- YAML decode is strict (`KnownFields(true)`) and accepts exactly one document:
unknown fields or trailing documents fail load.
- ConfigKit performs strict typed YAML decoding and accepts exactly one document;
unknown fields, duplicate keys, and trailing documents fail load.
- A pipeline file may explicitly import additive YAML fragments through the
root-only `composition.imports` list. Imported files contribute fields to one
root-only `configkit.imports` list. Imported files contribute fields to one
logical pipeline document; they do not override fields supplied by the root
or another import.
- A root pipeline may declare named profiles. Exactly one profile is selected
by an option-aware caller or by `composition.default_profile`; a caller's
explicit selection takes precedence. Declaring profiles without either form
of selection is an error.
- An omitted profile, an empty profile option, and the exact name `default`
select the base configuration. Any other name selects
`profiles/<name>.yml` relative to the pipeline root.
- Configured timeout and retry-delay durations must be positive. An omitted
artifact timeout continues to inherit its configured Scriptorium timeout.
- Session files must be concrete; unresolved `{{ ... }}` placeholders fail load.
@@ -123,22 +123,16 @@ remote state with an unsafe legacy identity must be migrated before use.
- local (`audio_dir` or `audio_files`), or
- S3 (`audio_s3.prefix`).
### Pipeline composition
### Pipeline imports and profiles
Large pipeline configurations may be split into explicitly named fragments and
may declare one overlay per selectable profile:
may use fixed-path profile overlays:
```yaml
composition:
configkit:
imports:
- config/storage.yml
- config/integrations.yaml
default_profile: production
profiles:
production:
overlay: profiles/production.yml
testing:
overlay: profiles/testing.yml
- path: config/storage.yml
- path: config/integrations.yaml
campaigns:
root: /usr/local/share/narratio/campaigns
@@ -152,36 +146,29 @@ Imports are resolved relative to the directory containing the root pipeline
file and are loaded in declaration order. Narratio does not scan directories or
infer fragments. Each import must be a confined regular `.yml` or `.yaml` file:
absolute paths, traversal, symlinks, directories, duplicate files, and an
import of the root pipeline itself are rejected. Only the root pipeline may
contain `composition`; nested composition is rejected.
import of the root pipeline itself are rejected. Imported files cannot declare
another `configkit` control section.
Composition is additive. A map may be extended by multiple files when every
Base merging is additive. A map may be extended by multiple files when every
leaf is distinct, but a scalar, list, or map/list/scalar kind cannot be claimed
more than once, even when the repeated values are identical. Conflict errors
name the full field path and every source that claimed it. The assembled YAML is
then decoded against the normal strict pipeline schema and defaults are applied
once.
name the full field path and every source that claimed it. Narratio additionally
rejects an artifact or artifact-family key declared by more than one base file,
even when those files contribute disjoint children.
Profile names are case-sensitive, non-empty, trimmed, and cannot contain
control characters. If `profiles` is present, it must contain at least one
entry and every entry must contain only an `overlay` path. An explicit profile
selection overrides `default_profile`; unknown and explicitly empty selections
fail. Narratio never selects the first profile implicitly and does not read a
profile selection from the environment.
Profile names are case-sensitive and use ConfigKit's safe fixed-path naming
rules. Only the selected overlay is opened; a malformed unselected overlay does
not affect a base or different-profile load. Maps merge recursively, overlay
scalars replace base scalars, and overlay lists replace base lists completely.
Explicit `false`, zero, empty strings, empty containers, and null values remain
explicit inputs and are not refilled by defaults. Profiles cannot inherit,
stack, import files, or declare another ConfigKit control section.
Every declared overlay is resolved relative to the root pipeline directory and
must satisfy the same confined regular-YAML-file rules as an import. Narratio
parses every declared overlay even when it is not selected, then applies only
the selected one. Maps merge recursively, overlay scalars replace base scalars,
and overlay lists replace base lists completely. Explicit `false`, zero, empty
lists, and empty maps remain meaningful. YAML null cannot delete a value, and
kind changes are rejected. Profiles cannot inherit from or stack with other
profiles, and overlays cannot import files or declare profiles.
After composition, Narratio strictly decodes the result, applies centralized
defaults once, resolves ordinary paths, and computes a deterministic effective
During ConfigKit loading, Narratio applies fresh typed defaults, recorded
entry-level defaults and normalization, read-only file-local validation, and
ordinary path resolution before computing a deterministic effective
configuration digest. The digest represents the normalized, secret-free
runtime pipeline mapping; it excludes composition declarations, source
runtime pipeline mapping; it excludes ConfigKit directives, source
provenance, profile identity, and raw environment secret values. Equivalent
effective mappings therefore have the same digest regardless of how fields are
split among the root and imports.
@@ -274,9 +261,7 @@ Rules:
| Field | Type | Required | Default / Rule |
| --- | --- | --- | --- |
| `composition.imports[]` | list of strings | No | explicit additive pipeline fragments relative to the root pipeline directory; `.yml` or `.yaml` regular files only |
| `composition.default_profile` | string | Conditional | selected when profiles exist and no caller explicitly selects one; must name a declared profile |
| `composition.profiles.<name>.overlay` | string | Conditional | required for every declared profile; one confined `.yml` or `.yaml` overlay relative to the root pipeline directory |
| `configkit.imports[].path` | string | No | explicit additive pipeline fragment relative to the root pipeline directory; `.yml` or `.yaml` regular files only |
| `pipeline.workspace.root` | string | No | `/var/lib/narratio` |
| `pipeline.workspace.cleanup_after_publish` | bool | No | `false`; selects the published run's workspace target for final cleanup and requires effective `pipeline.publish.enabled=true` and `pipeline.publish.upload_run=true` |
| `pipeline.campaigns.root` | string | No | `/usr/local/share/narratio/campaigns` |

View File

@@ -8,31 +8,21 @@ pipeline-loading boundary implemented by `internal/config`.
`LoadPipeline` assembles and validates a pipeline in this order:
1. Parse the root YAML into a presence-aware composition tree. The tree retains
source names, full field paths, node kinds, declaration order, and explicit
zero, false, empty-map, and empty-list values.
2. Decode the root-only `composition` envelope, validate its declaration shapes
and cross-field profile semantics, then clone the root without that envelope
as the ordinary pipeline body. The parsed source document is not mutated. A
load option retains the difference between omitted and explicitly empty
profile selection.
3. Open each import relative to the root pipeline directory through the
confined regular-file boundary. Imports must use a `.yml` or `.yaml`
extension and cannot traverse, use symlinks, repeat a file, import the root,
or contain another composition envelope.
4. Resolve and structurally parse every declared profile overlay through the
same confined regular-file boundary. Missing or malformed unselected
overlays fail the load. Overlays cannot contain a composition envelope.
5. Additively merge the root body and imports. Distinct map leaves compose;
repeated scalar or list paths and node-kind disagreements are conflicts.
6. Select exactly one declared profile from an explicit option or the default,
then recursively merge its overlay. Overlay leaves replace base leaves,
lists are atomic replacements, and null or kind changes fail.
7. Emit deterministic canonical YAML and strictly decode it into
`PipelineConfig`.
8. Apply pipeline defaults once, explicitly normalize reusable domain values,
resolve ordinary relative pipeline paths from the root pipeline file, and
digest the normalized effective mapping.
1. Give ConfigKit a fresh typed `PipelineConfig` default value, the root file,
enabled imports, and the caller's profile selection. Omitted, empty, and
exact `default` selections use the base; named profiles use the fixed
`profiles/<name>.yml` convention.
2. ConfigKit parses the root and declared imports, additively merges base
sources, applies only the selected overlay, and strictly decodes the result.
3. The definition resolver applies recorded entry-level and inherited defaults,
runs deterministic domain normalization, records derived publish destinations,
and resolves ordinary paths relative to the root pipeline file.
4. The definition validator performs read-only checks that require only the
pipeline file. Checks requiring canonical-party family expansion remain at
the existing explicit pipeline/campaign resolution boundary.
5. Narratio inspects public ConfigKit metadata to reject artifact and family keys
declared by multiple base files, adapts source ownership records, attaches
runtime-only state, and computes the normalized effective digest.
This ordering preserves monolithic configuration behavior. Moving a field to
an imported fragment changes its source ownership, not its path base, default,
@@ -71,7 +61,7 @@ commands do not marshal runtime objects directly.
`EffectivePipelineSources` and `EffectiveCampaignSources` provide the separate
safe provenance projection for `config sources`. Pipeline ownership begins with
the complete logical field paths retained during composition and classifies
the complete logical field paths projected from the schema and classifies
each contributor as root, import, profile, or centralized default. The
projection replaces generated concrete member paths with paired family and
canonical-party records, and does the same for generated publish rules.
@@ -79,9 +69,8 @@ Campaign records identify campaign-owned fields and party inputs; canonical
derived players point to the party source. The application command only joins these sorted
records with selection metadata and never reparses configuration files.
`config diff` uses a paired profile loader that parses the root, imports, and
declared overlays once, then clones the additive base before independently
selecting, decoding, defaulting, and finalizing each profile. When campaign
`config diff` uses a thin paired loader that performs two independent ConfigKit
loads. When campaign
resolution is needed, the command loads one selected campaign and party and
expands both effective pipelines from that same party value. The configuration
owner projects each normalized effective mapping directly from that same typed
@@ -128,21 +117,22 @@ that candidate without changing it. Scriptorium validates artifact declarations
and outputs, dependencies and inputs, variables, referenced outputs, and the final
dependency graph in deterministic artifact order.
Diagnostics identify the relevant semantic field. Composition conflicts also
Diagnostics identify the relevant semantic field. Merge conflicts also
retain every claiming source needed to correct an additive ownership conflict.
Complete prose, punctuation, formatting, and private validation choreography are
not compatibility contracts.
## Diagnostics And Runtime Metadata
Syntax, duplicate-key, composition, conflict, and schema failures include the
Syntax, duplicate-key, merge, profile, and schema failures include the
relevant source name and full field path. Additive conflicts report every
claiming source so operators can repair the split without repeatedly
rediscovering additional conflicts.
The loaded pipeline retains private runtime metadata for the absolute root
path, ordered imports, selected profile name and selection source, selected
overlay, contributing sources, effective digest, and leaf ownership. Base
overlay, original ConfigKit metadata and source snapshots, effective digest,
and leaf ownership. Base
leaves retain their root/import owners, replaced leaves belong to the selected
overlay, and centrally supplied values use the synthetic `default` owner. This
metadata does not participate in YAML decoding or alter the public
@@ -152,7 +142,7 @@ The effective digest is SHA-256 over a versioned deterministic JSON encoding of
the same typed safe projection used by show and diff. Runtime Notarius paths remain absolute for
execution, but the digest substitutes their normalized logical values captured
before root-relative resolution, so relocating an equivalent configuration
bundle does not change provenance. Because composition and resolution metadata
bundle does not change provenance. Because ConfigKit and resolution metadata
are private, the digest excludes source layout, profile name, and ownership.
Configuration stores environment variable names rather than resolving raw
credentials, so raw secret values are neither loaded nor hashed.
@@ -161,12 +151,11 @@ for later runtime expansion.
## Test Surfaces
`composition_test.go` protects the presence and merge algebra independently of
the public schema. `pipeline_composition_test.go` exercises explicit imports,
confinement, conflicts, strict decoding, metadata, and root-relative path
behavior through `LoadPipeline`. `pipeline_profiles_test.go` covers selection,
all-overlay validation, overlay behavior, provenance, option propagation, and
effective-digest stability. Application configuration-loader tests protect the
`pipeline_configkit_test.go` exercises real imports and fixed-path profiles,
base selection, unselected-overlay isolation, source attribution, and Narratio's
unique artifact/family base-declaration rule. Existing focused configuration
tests protect defaults, explicit-value semantics, strict decoding, validation,
metadata, and root-relative path behavior. Application configuration-loader tests protect the
single-read boundary by changing the pipeline file after its initial load and
confirming local session resolution retains the original pipeline. Other
configuration tests continue to protect defaults and validation after assembly.

View File

@@ -37,7 +37,7 @@ failed stage or required manifest save prevents later stages from starting.
| --- | --- | --- |
| Executable | `cmd/narratio` | Process entry, standard stream wiring, argument handoff, and exit status. |
| Application orchestration | `internal/app` | Command dispatch, configuration selection, secret-file environment loading, production composition, session locking, planning, execution, restore, and user-facing reporting. |
| Configuration | [`internal/config`](configuration.md) | Presence-aware root/import/profile composition, canonical party and family expansion, strict YAML loading, defaults, normalization, session templating, and validation. |
| Configuration | [`internal/config`](configuration.md) | ConfigKit-backed pipeline imports and profiles, canonical party and family expansion, strict YAML loading, defaults, normalization, session templating, and validation. |
| Pipeline stages | `internal/stage` | Canonical stage registry, shared stage contract, execution dependencies, and implemented stage behavior. |
| External boundaries | `internal/adapters`, `internal/audio` | WhisperX HTTP, downstream subprocesses, notification, object storage, and S3 audio materialization behind Narratio contracts. |
| Manifests | `internal/manifest` | Durable session progress, stage transitions, validation, and atomic persistence. |
@@ -105,8 +105,8 @@ so any earlier replacement can stale a prior cleanup result.
## Focused Documentation
- [Configuration Internals](configuration.md): pipeline composition, import
confinement, field ownership, decoding, and root-relative path semantics.
- [Configuration Internals](configuration.md): pipeline imports and profiles,
field ownership, decoding, and root-relative path semantics.
- [Adapter Internals](adapters.md): external adapter boundaries, composition,
failure behavior, and test surfaces.
- [Artifact Internals](artifacts.md): source identities, runtime catalog,

View File

@@ -146,7 +146,7 @@ Configuration is strict, explicit, centralized, and operator-oriented.
- YAML decoding rejects unknown fields.
- Defaults are centralized and testable.
- Empty configured values do not silently replace meaningful defaults.
- Validation rejects invalid composition before stage execution where
- Validation rejects invalid configuration before stage execution where
practical.
- Root-owned imports and one selected profile resolve deterministically through
the configuration owner; commands do not implement their own merge rules.

View File

@@ -189,7 +189,10 @@ projection consumer needs the custom composition parser/renderer.
## Stage 4 — Cut pipeline loading over to ConfigKit
Status: Pending
Status: Complete — Replaced pipeline composition with ConfigKit v0.1.1 imports,
fixed-path profile selection, fresh typed defaults, recorded resolution, source
metadata adaptation, and duplicate base artifact/family enforcement; removed the
legacy parser and verified all tests plus the configuration race suite.
Read: ConfigKit v0.1.1 `docs/config.md`, `docs/consumers/README.md`, and its
Narratio-shaped compiled example; Narratio `internal/config/load.go`,

View File

@@ -1,15 +1,9 @@
# Copyable production/testing pipeline entry point. Every fragment is named
# explicitly; Narratio never scans conf.d automatically.
composition:
configkit:
imports:
- conf.d/platform.yml
- conf.d/storage.yml
- conf.d/transcript.yml
- conf.d/artifacts.yml
- conf.d/publish.yml
default_profile: production
profiles:
production:
overlay: profiles/production.yml
testing:
overlay: profiles/testing.yml
- path: conf.d/platform.yml
- path: conf.d/storage.yml
- path: conf.d/transcript.yml
- path: conf.d/artifacts.yml
- path: conf.d/publish.yml

2
go.mod
View File

@@ -3,6 +3,7 @@ module gitea.maximumdirect.net/eric/narratio
go 1.26.0
require (
gitea.maximumdirect.net/eric/ConfigKit v0.1.1
github.com/aws/aws-sdk-go-v2/config v1.32.17
github.com/aws/aws-sdk-go-v2/credentials v1.19.16
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
@@ -28,4 +29,5 @@ require (
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
)

4
go.sum
View File

@@ -1,3 +1,5 @@
gitea.maximumdirect.net/eric/ConfigKit v0.1.1 h1:/JXi95QDhgRtiQN7R/+6cGFcfc0+gfl+D44nKXPd52k=
gitea.maximumdirect.net/eric/ConfigKit v0.1.1/go.mod h1:tYWpsrbLyu05r8+1MapWp5fN6ZrX9I8hU2xEPedYmYk=
github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=
github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho=
@@ -36,6 +38,8 @@ github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=

View File

@@ -253,8 +253,8 @@ func TestAssembledSplitBundleCommandsUseOneEffectiveFamilyConfiguration(t *testi
}
var validated bytes.Buffer
if err := ConfigValidate(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath}, &validated); err != nil {
t.Fatalf("validate production default: %v", err)
if err := ConfigValidate(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath, "--profile", "production"}, &validated); err != nil {
t.Fatalf("validate production profile: %v", err)
}
if !strings.Contains(validated.String(), "profile=production") {
t.Fatalf("validate output = %q, want production profile", validated.String())
@@ -330,7 +330,7 @@ func TestAssembledSplitBundleCommandsUseOneEffectiveFamilyConfiguration(t *testi
}
func TestAssembledSplitBundleProfileSwitchRecordsProvenanceAndLimitsReuse(t *testing.T) {
production := loadAssembledSplitBundleConfig(t, "")
production := loadAssembledSplitBundleConfig(t, "production")
testingCfg := loadAssembledSplitBundleConfig(t, "testing")
workspace := t.TempDir()
production.Pipeline.Workspace.Root = workspace

View File

@@ -304,17 +304,9 @@ func TestConfigCommandsInspectMaintainedSplitBundleWithoutRuntimeState(t *testin
func writeInspectionProfiles(t *testing.T, pipelinePath, production, testing string) {
t.Helper()
data, err := os.ReadFile(pipelinePath)
if err != nil {
t.Fatal(err)
}
composition := "composition:\n default_profile: production\n profiles:\n production:\n overlay: production.yml\n testing:\n overlay: testing.yml\n"
if err := os.WriteFile(pipelinePath, append([]byte(composition), data...), 0o644); err != nil {
t.Fatal(err)
}
dir := filepath.Dir(pipelinePath)
mustWriteTestFile(t, filepath.Join(dir, "production.yml"), production)
mustWriteTestFile(t, filepath.Join(dir, "testing.yml"), testing)
profilesDir := filepath.Join(filepath.Dir(pipelinePath), "profiles")
mustWriteTestFile(t, filepath.Join(profilesDir, "production.yml"), production)
mustWriteTestFile(t, filepath.Join(profilesDir, "testing.yml"), testing)
}
func TestConfigCommandsSelectProfilesAndCampaigns(t *testing.T) {
@@ -326,7 +318,7 @@ func TestConfigCommandsSelectProfilesAndCampaigns(t *testing.T) {
if err := ConfigValidate(context.Background(), []string{"--config", pipelinePath}, &defaultOut); err != nil {
t.Fatalf("default ConfigValidate() error = %v", err)
}
if !strings.Contains(defaultOut.String(), "profile=production") {
if !strings.Contains(defaultOut.String(), "profile=none") {
t.Fatalf("default output = %q", defaultOut.String())
}
@@ -398,8 +390,9 @@ func TestConfigValidateReportsCanonicalPartyAndImportErrors(t *testing.T) {
dir := t.TempDir()
rootPath := filepath.Join(dir, "pipeline.yml")
mustWriteTestFile(t, rootPath, `composition:
imports: [conf.yml]
mustWriteTestFile(t, rootPath, `configkit:
imports:
- path: conf.yml
workspace:
root: /one
whisperx:
@@ -407,7 +400,7 @@ whisperx:
`)
mustWriteTestFile(t, filepath.Join(dir, "conf.yml"), "workspace:\n root: /two\n")
err := ConfigValidate(context.Background(), []string{"--config", rootPath}, io.Discard)
if err == nil || !strings.Contains(err.Error(), "workspace.root") || !strings.Contains(err.Error(), "pipeline.yml") || !strings.Contains(err.Error(), "conf.yml") {
if err == nil || !strings.Contains(err.Error(), "/workspace/root") || !strings.Contains(err.Error(), "pipeline.yml") || !strings.Contains(err.Error(), "conf.yml") {
t.Fatalf("import diagnostic = %v", err)
}
}
@@ -432,7 +425,7 @@ func TestConfigSourcesReportsStableOwnership(t *testing.T) {
for _, want := range []string{
"root: ",
"imports: none",
"profile: name=testing selection=cli overlay=",
"profile: name=testing selection=explicit overlay=",
"campaign: id=sample-campaign source=",
"party: source=",
"digest: ",
@@ -475,15 +468,7 @@ func TestConfigSourcesReportsCanonicalFamilyAndPublishOrigins(t *testing.T) {
func TestConfigSourcesPreservesProfileOwnershipThroughFamilyExpansion(t *testing.T) {
pipelinePath, campaignPath := writeInspectionFamilyConfig(t)
data, err := os.ReadFile(pipelinePath)
if err != nil {
t.Fatal(err)
}
profilePath := filepath.Join(filepath.Dir(pipelinePath), "testing.yml")
composition := "composition:\n default_profile: testing\n profiles:\n testing:\n overlay: testing.yml\n"
if err := os.WriteFile(pipelinePath, append([]byte(composition), data...), 0o644); err != nil {
t.Fatal(err)
}
profilePath := filepath.Join(filepath.Dir(pipelinePath), "profiles", "testing.yml")
mustWriteTestFile(t, profilePath, `scriptorium:
artifact_families:
character_note:
@@ -492,7 +477,7 @@ func TestConfigSourcesPreservesProfileOwnershipThroughFamilyExpansion(t *testing
`)
var out bytes.Buffer
if err := ConfigSources(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath}, &out); err != nil {
if err := ConfigSources(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath, "--profile", "testing"}, &out); err != nil {
t.Fatalf("ConfigSources() error = %v", err)
}
want := "scriptorium.artifacts.character_note_arannis.enabled\tfamily\t" + profilePath
@@ -505,13 +490,10 @@ func TestConfigSourcesReportsRootImportProfileAndDefaultOwnership(t *testing.T)
dir := t.TempDir()
rootPath := filepath.Join(dir, "pipeline.yml")
importPath := filepath.Join(dir, "base.yml")
profilePath := filepath.Join(dir, "testing.yml")
mustWriteTestFile(t, rootPath, `composition:
imports: [base.yml]
default_profile: testing
profiles:
testing:
overlay: testing.yml
profilePath := filepath.Join(dir, "profiles", "testing.yml")
mustWriteTestFile(t, rootPath, `configkit:
imports:
- path: base.yml
workspace:
root: /srv/narratio
whisperx:
@@ -521,13 +503,13 @@ whisperx:
mustWriteTestFile(t, profilePath, "whisperx:\n language: fr\n")
var out bytes.Buffer
if err := ConfigSources(context.Background(), []string{"--config", rootPath}, &out); err != nil {
if err := ConfigSources(context.Background(), []string{"--config", rootPath, "--profile", "testing"}, &out); err != nil {
t.Fatalf("ConfigSources() error = %v", err)
}
output := out.String()
for _, want := range []string{
"import: " + importPath,
"profile: name=testing selection=default overlay=" + profilePath,
"profile: name=testing selection=explicit overlay=" + profilePath,
"workspace.root\troot\t" + rootPath,
"storage.backend\timport\t" + importPath,
"whisperx.language\tprofile\t" + profilePath,

View File

@@ -37,10 +37,10 @@ func TestRunProfileSelectionFlowsThroughSharedLoader(t *testing.T) {
t.Fatal(err)
}
if language != "en" {
t.Fatalf("default profile language = %q, want en", language)
t.Fatalf("base configuration language = %q, want en", language)
}
for _, profile := range []string{"unknown", ""} {
for _, profile := range []string{"unknown"} {
err := runWithPipelineExecutor(context.Background(), []string{
"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--profile", profile,
}, io.Discard, executor)
@@ -48,19 +48,20 @@ func TestRunProfileSelectionFlowsThroughSharedLoader(t *testing.T) {
t.Fatalf("profile %q error = %v, want selection rejection", profile, err)
}
}
if err := runWithPipelineExecutor(context.Background(), []string{
"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--profile", "",
}, io.Discard, executor); err != nil {
t.Fatalf("explicit empty profile should select base: %v", err)
}
}
func enableCommandTestProfiles(t *testing.T, pipelinePath string) {
t.Helper()
data, err := os.ReadFile(pipelinePath)
if err != nil {
dir := filepath.Join(filepath.Dir(pipelinePath), "profiles")
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
composition := "composition:\n default_profile: production\n profiles:\n production:\n overlay: production.yml\n testing:\n overlay: testing.yml\n"
if err := os.WriteFile(pipelinePath, append([]byte(composition), data...), 0o644); err != nil {
t.Fatal(err)
}
dir := filepath.Dir(pipelinePath)
if err := os.WriteFile(filepath.Join(dir, "production.yml"), []byte("whisperx:\n language: en\n"), 0o644); err != nil {
t.Fatal(err)
}

View File

@@ -8,6 +8,15 @@ import (
"testing"
)
func absolutePath(t *testing.T, path string) string {
t.Helper()
absolute, err := filepath.Abs(path)
if err != nil {
t.Fatal(err)
}
return absolute
}
func TestArtifactFamiliesExpandCanonicalCharactersDeterministically(t *testing.T) {
dir := t.TempDir()
writePartyResolutionFile(t, dir+"/party.yml", `schema_version: narratio.party.v1
@@ -148,7 +157,7 @@ func TestArtifactFamilyStrictDecodeRejectsUnknownFields(t *testing.T) {
dir := t.TempDir()
pipelinePath, _, _ := writeArtifactFamilyConfig(t, dir, artifactFamilyPipelineYAML(validFamilyYAML+" unsupported: true\n"))
_, err := LoadPipeline(pipelinePath)
if err == nil || !strings.Contains(err.Error(), "field unsupported not found") {
if err == nil || !strings.Contains(err.Error(), "decode.unknown_field") {
t.Fatalf("LoadPipeline() error = %v", err)
}
}

View File

@@ -1,783 +0,0 @@
package config
import (
"bytes"
"encoding/json"
"fmt"
"io"
"sort"
"strconv"
"strings"
"gopkg.in/yaml.v3"
)
// compositionDocument is the presence-aware representation used while
// assembling pipeline configuration sources. It deliberately models YAML
// mechanics rather than duplicating PipelineConfig's field schema.
type compositionDocument struct {
root *compositionNode
sources []string
}
type compositionNode struct {
kind yaml.Kind
tag string
value string
path string
sources []string
fields []compositionField
items []*compositionNode
}
type compositionField struct {
key string
value *compositionNode
}
// compositionValueRecord is a deterministic semantic leaf projection. Lists
// are atomic configuration values, while mappings are traversed recursively.
type compositionValueRecord struct {
Path string
Kind yaml.Kind
Value string
Sources []string
}
// parseCompositionDocument parses exactly one YAML mapping while retaining
// source ownership, explicit zero values, and declaration order.
func parseCompositionDocument(source string, reader io.Reader) (*compositionDocument, error) {
if strings.TrimSpace(source) == "" {
return nil, fmt.Errorf("configuration source identity is required")
}
if reader == nil {
return nil, fmt.Errorf("configuration source %q: reader is nil", source)
}
decoder := yaml.NewDecoder(reader)
var document yaml.Node
if err := decoder.Decode(&document); err != nil {
if err == io.EOF {
return nil, fmt.Errorf("configuration source %q: document is empty", source)
}
return nil, fmt.Errorf("configuration source %q: decode YAML: %w", source, err)
}
var trailing yaml.Node
if err := decoder.Decode(&trailing); err == nil {
return nil, fmt.Errorf("configuration source %q: must contain exactly one YAML document", source)
} else if err != io.EOF {
return nil, fmt.Errorf("configuration source %q: decode trailing YAML: %w", source, err)
}
if document.Kind != yaml.DocumentNode || len(document.Content) != 1 {
return nil, fmt.Errorf("configuration source %q: must contain exactly one YAML document", source)
}
if document.Content[0].Kind != yaml.MappingNode {
return nil, fmt.Errorf(
"configuration source %q: top-level document must be a mapping, got %s",
source,
yamlKindName(document.Content[0].Kind),
)
}
root, err := buildCompositionNode(document.Content[0], source, "")
if err != nil {
return nil, err
}
return &compositionDocument{root: root, sources: []string{source}}, nil
}
func parseCompositionBytes(source string, data []byte) (*compositionDocument, error) {
return parseCompositionDocument(source, bytes.NewReader(data))
}
func buildCompositionNode(node *yaml.Node, source, path string) (*compositionNode, error) {
if node == nil {
return nil, fmt.Errorf("configuration source %q at %s: YAML node is nil", source, displayCompositionPath(path))
}
if node.Kind == yaml.AliasNode {
return nil, compositionNodeError(source, path, node, "YAML aliases are not supported because source ownership would be ambiguous")
}
result := &compositionNode{
kind: node.Kind, tag: node.Tag, value: node.Value, path: path,
sources: []string{source},
}
switch node.Kind {
case yaml.MappingNode:
if len(node.Content)%2 != 0 {
return nil, compositionNodeError(source, path, node, "mapping has an incomplete key/value pair")
}
seen := make(map[string]*yaml.Node, len(node.Content)/2)
for index := 0; index < len(node.Content); index += 2 {
keyNode := node.Content[index]
valueNode := node.Content[index+1]
if keyNode.Kind == yaml.AliasNode {
return nil, compositionNodeError(source, path, keyNode, "YAML aliases are not supported because source ownership would be ambiguous")
}
if valueNode.Kind == yaml.AliasNode {
return nil, compositionNodeError(source, appendCompositionPath(path, keyNode.Value), valueNode, "YAML aliases are not supported because source ownership would be ambiguous")
}
if keyNode.Kind != yaml.ScalarNode || keyNode.Tag != "!!str" {
return nil, compositionNodeError(source, path, keyNode, "mapping keys must be strings")
}
key := keyNode.Value
fieldPath := appendCompositionPath(path, key)
if prior, duplicate := seen[key]; duplicate {
return nil, fmt.Errorf(
"configuration source %q at %s: duplicate YAML key %q (first declared at line %d, column %d; repeated at line %d, column %d)",
source, displayCompositionPath(fieldPath), key,
prior.Line, prior.Column, keyNode.Line, keyNode.Column,
)
}
seen[key] = keyNode
child, err := buildCompositionNode(valueNode, source, fieldPath)
if err != nil {
return nil, err
}
result.fields = append(result.fields, compositionField{key: key, value: child})
}
case yaml.SequenceNode:
for index, childNode := range node.Content {
childPath := fmt.Sprintf("%s[%d]", path, index)
child, err := buildCompositionNode(childNode, source, childPath)
if err != nil {
return nil, err
}
result.items = append(result.items, child)
}
case yaml.ScalarNode:
// Scalar tag and lexical value retain distinctions such as explicit
// false, zero, an empty string, and null until final strict decoding.
default:
return nil, compositionNodeError(source, path, node, fmt.Sprintf("unsupported YAML node kind %s", yamlKindName(node.Kind)))
}
return result, nil
}
// mergeAdditiveComposition recursively combines disjoint mappings. A scalar,
// list, or final keyed value may have only one base owner, regardless of
// whether duplicate values happen to be equal.
func mergeAdditiveComposition(base, incoming *compositionDocument) (*compositionDocument, error) {
return mergeAdditiveCompositions(base, incoming)
}
// mergeAdditiveCompositions validates the complete base source set before
// merging so a conflict names every source that claims the same final path.
func mergeAdditiveCompositions(documents ...*compositionDocument) (*compositionDocument, error) {
if len(documents) == 0 {
return nil, fmt.Errorf("configuration additive base merge requires at least one document")
}
for index, document := range documents {
if err := validateCompositionDocument(document, fmt.Sprintf("base[%d]", index)); err != nil {
return nil, err
}
}
if err := validateAdditiveClaims(documents); err != nil {
return nil, err
}
result := &compositionDocument{
root: cloneCompositionNode(documents[0].root),
sources: append([]string(nil), documents[0].sources...),
}
for _, incoming := range documents[1:] {
merged, err := mergeAdditiveNodes(result.root, incoming.root)
if err != nil {
return nil, err
}
result.root = merged
result.sources = appendUniqueStrings(result.sources, incoming.sources...)
}
return result, nil
}
func validateAdditiveClaims(documents []*compositionDocument) error {
claims := make(map[string][]*compositionNode)
for _, document := range documents {
appendCompositionClaims(document.root, claims)
}
paths := make([]string, 0, len(claims))
for path := range claims {
paths = append(paths, path)
}
sort.Slice(paths, func(i, j int) bool {
leftDepth := compositionPathDepth(paths[i])
rightDepth := compositionPathDepth(paths[j])
if leftDepth != rightDepth {
return leftDepth < rightDepth
}
return paths[i] < paths[j]
})
for _, path := range paths {
values := claims[path]
if len(values) < 2 {
continue
}
allPopulatedMappings := true
for _, value := range values {
if value.kind != yaml.MappingNode || len(value.fields) == 0 {
allPopulatedMappings = false
break
}
}
if !allPopulatedMappings {
return newCompositionConflict("additive base merge", path, values...)
}
}
return nil
}
func appendCompositionClaims(node *compositionNode, claims map[string][]*compositionNode) {
if node == nil {
return
}
if node.path != "" {
claims[node.path] = append(claims[node.path], node)
}
for _, field := range node.fields {
appendCompositionClaims(field.value, claims)
}
}
func compositionPathDepth(path string) int {
if path == "" {
return 0
}
return strings.Count(path, ".") + strings.Count(path, "[") + 1
}
func mergeAdditiveNodes(base, incoming *compositionNode) (*compositionNode, error) {
if base.kind != yaml.MappingNode || incoming.kind != yaml.MappingNode {
return nil, newCompositionConflict("additive base merge", base.path, base, incoming)
}
base.sources = appendUniqueStrings(base.sources, incoming.sources...)
for _, incomingField := range incoming.fields {
index := compositionFieldIndex(base.fields, incomingField.key)
if index < 0 {
field := cloneCompositionField(incomingField)
base.fields = append(base.fields, field)
continue
}
baseValue := base.fields[index].value
incomingValue := incomingField.value
if baseValue.kind == yaml.MappingNode && incomingValue.kind == yaml.MappingNode {
if len(baseValue.fields) == 0 || len(incomingValue.fields) == 0 {
return nil, newCompositionConflict("additive base merge", incomingValue.path, baseValue, incomingValue)
}
merged, err := mergeAdditiveNodes(baseValue, incomingValue)
if err != nil {
return nil, err
}
base.fields[index].value = merged
continue
}
return nil, newCompositionConflict("additive base merge", incomingValue.path, baseValue, incomingValue)
}
return base, nil
}
// mergeOverlayComposition applies the sole overwrite layer. Mappings merge
// recursively; same-kind scalars and lists replace; null and kind changes are
// rejected.
func mergeOverlayComposition(base, overlay *compositionDocument) (*compositionDocument, error) {
if err := validateCompositionDocument(base, "base"); err != nil {
return nil, err
}
if err := validateCompositionDocument(overlay, "overlay"); err != nil {
return nil, err
}
if null := firstNullCompositionNode(overlay.root); null != nil {
return nil, fmt.Errorf(
"configuration overlay at %s from %s: null cannot delete an effective value",
displayCompositionPath(null.path), formatCompositionSources(null.sources),
)
}
merged, err := mergeOverlayNodes(cloneCompositionNode(base.root), overlay.root)
if err != nil {
return nil, err
}
return &compositionDocument{
root: merged,
sources: appendUniqueStrings(
append([]string(nil), base.sources...), overlay.sources...,
),
}, nil
}
func mergeOverlayNodes(base, overlay *compositionNode) (*compositionNode, error) {
if base.kind != yaml.MappingNode || overlay.kind != yaml.MappingNode {
return nil, newCompositionConflict("profile overlay", base.path, base, overlay)
}
base.sources = appendUniqueStrings(base.sources, overlay.sources...)
for _, overlayField := range overlay.fields {
index := compositionFieldIndex(base.fields, overlayField.key)
if index < 0 {
field := cloneCompositionField(overlayField)
base.fields = append(base.fields, field)
continue
}
baseValue := base.fields[index].value
overlayValue := overlayField.value
if baseValue.kind != overlayValue.kind {
return nil, newCompositionConflict("profile overlay kind change", overlayValue.path, baseValue, overlayValue)
}
if baseValue.kind == yaml.MappingNode {
merged, err := mergeOverlayNodes(baseValue, overlayValue)
if err != nil {
return nil, err
}
base.fields[index].value = merged
continue
}
base.fields[index].value = cloneCompositionNode(overlayValue)
}
return base, nil
}
// canonicalYAML renders the effective mapping with sorted keys and normalized
// presentation while retaining sequence order and scalar YAML types.
func (document *compositionDocument) canonicalYAML() ([]byte, error) {
if err := validateCompositionDocument(document, "document"); err != nil {
return nil, err
}
root, err := compositionYAMLNode(document.root)
if err != nil {
return nil, err
}
var buffer bytes.Buffer
encoder := yaml.NewEncoder(&buffer)
encoder.SetIndent(2)
if err := encoder.Encode(root); err != nil {
return nil, fmt.Errorf("render effective configuration YAML: %w", err)
}
if err := encoder.Close(); err != nil {
return nil, fmt.Errorf("finish effective configuration YAML: %w", err)
}
return buffer.Bytes(), nil
}
// canonicalDigestInput provides a deterministic, formatting-independent byte
// representation for later secret-free effective configuration digesting.
func (document *compositionDocument) canonicalDigestInput() ([]byte, error) {
if err := validateCompositionDocument(document, "document"); err != nil {
return nil, err
}
value, err := canonicalCompositionValue(document.root)
if err != nil {
return nil, err
}
data, err := json.Marshal(value)
if err != nil {
return nil, fmt.Errorf("serialize canonical configuration digest input: %w", err)
}
return append(data, '\n'), nil
}
// semanticRecords returns sorted atomic values for future effective diff and
// source-report projections. A caller receives copies of all source slices.
func (document *compositionDocument) semanticRecords() ([]compositionValueRecord, error) {
if err := validateCompositionDocument(document, "document"); err != nil {
return nil, err
}
var records []compositionValueRecord
if err := appendCompositionRecords(document.root, &records); err != nil {
return nil, err
}
sort.Slice(records, func(i, j int) bool { return records[i].Path < records[j].Path })
return records, nil
}
// compactSemanticRecords returns the same logical atomic paths as
// semanticRecords, but represents values as compact JSON-compatible YAML
// values rather than the typed structural form used for digesting. It is the
// stable human-facing projection for semantic comparisons.
func (document *compositionDocument) compactSemanticRecords() ([]compositionValueRecord, error) {
if err := validateCompositionDocument(document, "document"); err != nil {
return nil, err
}
var records []compositionValueRecord
if err := appendCompactCompositionRecords(document.root, &records); err != nil {
return nil, err
}
sort.Slice(records, func(i, j int) bool { return records[i].Path < records[j].Path })
return records, nil
}
func appendCompositionRecords(node *compositionNode, records *[]compositionValueRecord) error {
if node.kind == yaml.MappingNode && len(node.fields) > 0 {
for _, field := range node.fields {
if err := appendCompositionRecords(field.value, records); err != nil {
return err
}
}
return nil
}
value, err := canonicalCompositionValue(node)
if err != nil {
return err
}
encoded, err := json.Marshal(value)
if err != nil {
return fmt.Errorf("serialize configuration value at %s: %w", displayCompositionPath(node.path), err)
}
*records = append(*records, compositionValueRecord{
Path: node.path, Kind: node.kind, Value: string(encoded),
Sources: append([]string(nil), node.sources...),
})
return nil
}
func appendCompactCompositionRecords(node *compositionNode, records *[]compositionValueRecord) error {
if node.kind == yaml.MappingNode && len(node.fields) > 0 {
for _, field := range node.fields {
if err := appendCompactCompositionRecords(field.value, records); err != nil {
return err
}
}
return nil
}
value, err := compactCompositionValue(node)
if err != nil {
return err
}
encoded, err := json.Marshal(value)
if err != nil {
return fmt.Errorf("serialize configuration value at %s: %w", displayCompositionPath(node.path), err)
}
*records = append(*records, compositionValueRecord{
Path: node.path, Kind: node.kind, Value: string(encoded),
Sources: append([]string(nil), node.sources...),
})
return nil
}
func compactCompositionValue(node *compositionNode) (any, error) {
switch node.kind {
case yaml.MappingNode:
values := make(map[string]any, len(node.fields))
for _, field := range node.fields {
value, err := compactCompositionValue(field.value)
if err != nil {
return nil, err
}
values[field.key] = value
}
return values, nil
case yaml.SequenceNode:
values := make([]any, 0, len(node.items))
for _, item := range node.items {
value, err := compactCompositionValue(item)
if err != nil {
return nil, err
}
values = append(values, value)
}
return values, nil
case yaml.ScalarNode:
return canonicalScalarValue(node)
default:
return nil, fmt.Errorf("configuration at %s has unsupported YAML kind %s", displayCompositionPath(node.path), yamlKindName(node.kind))
}
}
type canonicalCompositionField struct {
Key string `json:"key"`
Value any `json:"value"`
}
type canonicalCompositionNode struct {
Kind string `json:"kind"`
Tag string `json:"tag,omitempty"`
Value any `json:"value,omitempty"`
Fields []canonicalCompositionField `json:"fields,omitempty"`
Items []any `json:"items,omitempty"`
}
func canonicalCompositionValue(node *compositionNode) (any, error) {
switch node.kind {
case yaml.MappingNode:
fields := append([]compositionField(nil), node.fields...)
sort.Slice(fields, func(i, j int) bool { return fields[i].key < fields[j].key })
result := canonicalCompositionNode{Kind: "mapping"}
if len(fields) == 0 {
result.Fields = []canonicalCompositionField{}
}
for _, field := range fields {
value, err := canonicalCompositionValue(field.value)
if err != nil {
return nil, err
}
result.Fields = append(result.Fields, canonicalCompositionField{Key: field.key, Value: value})
}
return result, nil
case yaml.SequenceNode:
result := canonicalCompositionNode{Kind: "sequence", Items: make([]any, 0, len(node.items))}
for _, item := range node.items {
value, err := canonicalCompositionValue(item)
if err != nil {
return nil, err
}
result.Items = append(result.Items, value)
}
return result, nil
case yaml.ScalarNode:
value, err := canonicalScalarValue(node)
if err != nil {
return nil, err
}
return canonicalCompositionNode{Kind: "scalar", Tag: node.tag, Value: value}, nil
default:
return nil, fmt.Errorf("configuration at %s has unsupported YAML kind %s", displayCompositionPath(node.path), yamlKindName(node.kind))
}
}
func canonicalScalarValue(node *compositionNode) (any, error) {
raw := &yaml.Node{Kind: yaml.ScalarNode, Tag: node.tag, Value: node.value}
var value any
if err := raw.Decode(&value); err != nil {
return nil, fmt.Errorf("decode scalar at %s: %w", displayCompositionPath(node.path), err)
}
switch typed := value.(type) {
case nil, bool, string, int, int64, uint64, float64:
return typed, nil
default:
// yaml.v3 may decode timestamps or uncommon scalar tags into types that
// encoding/json can serialize deterministically. Preserve the resolved
// tag alongside the value in the containing canonical node.
return typed, nil
}
}
func compositionYAMLNode(node *compositionNode) (*yaml.Node, error) {
switch node.kind {
case yaml.MappingNode:
result := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
fields := append([]compositionField(nil), node.fields...)
sort.Slice(fields, func(i, j int) bool { return fields[i].key < fields[j].key })
for _, field := range fields {
value, err := compositionYAMLNode(field.value)
if err != nil {
return nil, err
}
result.Content = append(result.Content,
&yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: field.key},
value,
)
}
return result, nil
case yaml.SequenceNode:
result := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
for _, item := range node.items {
value, err := compositionYAMLNode(item)
if err != nil {
return nil, err
}
result.Content = append(result.Content, value)
}
return result, nil
case yaml.ScalarNode:
return normalizedCompositionScalarNode(node)
default:
return nil, fmt.Errorf("configuration at %s has unsupported YAML kind %s", displayCompositionPath(node.path), yamlKindName(node.kind))
}
}
func normalizedCompositionScalarNode(node *compositionNode) (*yaml.Node, error) {
raw := &yaml.Node{Kind: yaml.ScalarNode, Tag: node.tag, Value: node.value}
var value any
if err := raw.Decode(&value); err != nil {
return nil, fmt.Errorf("normalize scalar at %s: %w", displayCompositionPath(node.path), err)
}
normalized := &yaml.Node{}
if err := normalized.Encode(value); err != nil {
return nil, fmt.Errorf("encode normalized scalar at %s: %w", displayCompositionPath(node.path), err)
}
if normalized.Kind != yaml.ScalarNode {
return nil, fmt.Errorf("normalize scalar at %s produced YAML kind %s", displayCompositionPath(node.path), yamlKindName(normalized.Kind))
}
return normalized, nil
}
func validateCompositionDocument(document *compositionDocument, role string) error {
if document == nil || document.root == nil {
return fmt.Errorf("configuration composition %s document is nil", role)
}
if document.root.kind != yaml.MappingNode {
return fmt.Errorf("configuration composition %s root must be a mapping", role)
}
return nil
}
func firstNullCompositionNode(node *compositionNode) *compositionNode {
if node == nil {
return nil
}
if node.kind == yaml.ScalarNode && node.tag == "!!null" {
return node
}
for _, field := range node.fields {
if found := firstNullCompositionNode(field.value); found != nil {
return found
}
}
for _, item := range node.items {
if found := firstNullCompositionNode(item); found != nil {
return found
}
}
return nil
}
func cloneCompositionNode(node *compositionNode) *compositionNode {
if node == nil {
return nil
}
clone := &compositionNode{
kind: node.kind, tag: node.tag, value: node.value, path: node.path,
sources: append([]string(nil), node.sources...),
}
for _, field := range node.fields {
clone.fields = append(clone.fields, cloneCompositionField(field))
}
for _, item := range node.items {
clone.items = append(clone.items, cloneCompositionNode(item))
}
return clone
}
func cloneCompositionField(field compositionField) compositionField {
return compositionField{key: field.key, value: cloneCompositionNode(field.value)}
}
func compositionFieldIndex(fields []compositionField, key string) int {
for index := range fields {
if fields[index].key == key {
return index
}
}
return -1
}
func appendCompositionPath(parent, key string) string {
if isSimpleCompositionPathSegment(key) {
if parent == "" {
return key
}
return parent + "." + key
}
if parent == "" {
return "[" + strconv.Quote(key) + "]"
}
return parent + "[" + strconv.Quote(key) + "]"
}
func isSimpleCompositionPathSegment(value string) bool {
if value == "" {
return false
}
for index, char := range value {
if (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || char == '_' || (index > 0 && char >= '0' && char <= '9') || (index > 0 && char == '-') {
continue
}
return false
}
return true
}
func displayCompositionPath(path string) string {
if path == "" {
return "<root>"
}
return path
}
func compositionNodeError(source, path string, node *yaml.Node, message string) error {
line, column := 0, 0
if node != nil {
line, column = node.Line, node.Column
}
return fmt.Errorf(
"configuration source %q at %s (line %d, column %d): %s",
source, displayCompositionPath(path), line, column, message,
)
}
func newCompositionConflict(operation, path string, values ...*compositionNode) error {
var sources []string
var kinds []string
for _, value := range values {
if value == nil {
continue
}
sources = appendUniqueStrings(sources, compositionClaimSources(value)...)
kind := yamlKindName(value.kind)
if !containsString(kinds, kind) {
kinds = append(kinds, kind)
}
}
return fmt.Errorf(
"configuration %s conflict at %s: claimed by %s (YAML kinds: %s)",
operation, displayCompositionPath(path), formatCompositionSources(sources), strings.Join(kinds, ", "),
)
}
func compositionClaimSources(node *compositionNode) []string {
if node == nil {
return nil
}
sources := append([]string(nil), node.sources...)
for _, field := range node.fields {
sources = appendUniqueStrings(sources, compositionClaimSources(field.value)...)
}
for _, item := range node.items {
sources = appendUniqueStrings(sources, compositionClaimSources(item)...)
}
return sources
}
func appendUniqueStrings(values []string, additions ...string) []string {
seen := make(map[string]struct{}, len(values)+len(additions))
result := make([]string, 0, len(values)+len(additions))
for _, value := range append(append([]string(nil), values...), additions...) {
if _, exists := seen[value]; exists {
continue
}
seen[value] = struct{}{}
result = append(result, value)
}
return result
}
func containsString(values []string, target string) bool {
for _, value := range values {
if value == target {
return true
}
}
return false
}
func formatCompositionSources(sources []string) string {
quoted := make([]string, 0, len(sources))
for _, source := range sources {
quoted = append(quoted, strconv.Quote(source))
}
return strings.Join(quoted, ", ")
}
func yamlKindName(kind yaml.Kind) string {
switch kind {
case yaml.DocumentNode:
return "document"
case yaml.MappingNode:
return "mapping"
case yaml.SequenceNode:
return "sequence"
case yaml.ScalarNode:
return "scalar"
case yaml.AliasNode:
return "alias"
default:
return fmt.Sprintf("kind(%d)", kind)
}
}

View File

@@ -1,353 +0,0 @@
package config
import (
"reflect"
"strings"
"testing"
"gopkg.in/yaml.v3"
)
func TestParseCompositionDocumentRetainsPresenceOwnershipAndDeclarationOrder(t *testing.T) {
document := mustParseComposition(t, "root.yml", `zeta: false
zero: 0
empty_map: {}
empty_list: []
nested:
value: ""
`)
wantOrder := []string{"zeta", "zero", "empty_map", "empty_list", "nested"}
gotOrder := make([]string, 0, len(document.root.fields))
for _, field := range document.root.fields {
gotOrder = append(gotOrder, field.key)
}
if !reflect.DeepEqual(gotOrder, wantOrder) {
t.Fatalf("declaration order = %#v, want %#v", gotOrder, wantOrder)
}
tests := []struct {
path string
kind yaml.Kind
tag string
value string
}{
{path: "zeta", kind: yaml.ScalarNode, tag: "!!bool", value: "false"},
{path: "zero", kind: yaml.ScalarNode, tag: "!!int", value: "0"},
{path: "empty_map", kind: yaml.MappingNode, tag: "!!map"},
{path: "empty_list", kind: yaml.SequenceNode, tag: "!!seq"},
{path: "nested.value", kind: yaml.ScalarNode, tag: "!!str", value: ""},
}
for _, tt := range tests {
node := compositionNodeAtPath(t, document.root, tt.path)
if node.kind != tt.kind || node.tag != tt.tag || node.value != tt.value || !reflect.DeepEqual(node.sources, []string{"root.yml"}) {
t.Fatalf("node %s = kind=%v tag=%q value=%q sources=%#v", tt.path, node.kind, node.tag, node.value, node.sources)
}
}
}
func TestParseCompositionDocumentRejectsAmbiguousOrMalformedYAML(t *testing.T) {
tests := []struct {
name string
yaml string
want string
}{
{name: "duplicate top-level key", yaml: "value: 1\nvalue: 2\n", want: "duplicate YAML key"},
{name: "duplicate nested key", yaml: "outer:\n value: 1\n value: 2\n", want: "outer.value"},
{name: "alias", yaml: "base: &base\n value: 1\ncopy: *base\n", want: "aliases are not supported"},
{name: "trailing document", yaml: "value: 1\n---\nvalue: 2\n", want: "exactly one YAML document"},
{name: "top-level sequence", yaml: "- value\n", want: "top-level document must be a mapping"},
{name: "non-string key", yaml: "1: value\n", want: "mapping keys must be strings"},
{name: "malformed", yaml: "outer: [\n", want: "decode YAML"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := parseCompositionDocument("broken.yml", strings.NewReader(tt.yaml))
if err == nil {
t.Fatal("parseCompositionDocument() error = nil")
}
if !strings.Contains(err.Error(), "broken.yml") || !strings.Contains(err.Error(), tt.want) {
t.Fatalf("error = %q, want source and %q", err, tt.want)
}
})
}
}
func TestMergeAdditiveCompositionJoinsOnlyDisjointMappings(t *testing.T) {
root := mustParseComposition(t, "pipeline.yml", `scriptorium:
artifacts:
recap:
enabled: true
zero: 0
`)
imports := mustParseComposition(t, "conf.d/artifacts.yml", `scriptorium:
artifacts:
handout:
enabled: false
empty_list: []
`)
merged, err := mergeAdditiveComposition(root, imports)
if err != nil {
t.Fatalf("mergeAdditiveComposition() error = %v", err)
}
for _, path := range []string{
"empty_list", "scriptorium.artifacts.handout.enabled",
"scriptorium.artifacts.recap.enabled", "zero",
} {
_ = compositionNodeAtPath(t, merged.root, path)
}
if got := compositionNodeAtPath(t, merged.root, "scriptorium.artifacts.handout.enabled").sources; !reflect.DeepEqual(got, []string{"conf.d/artifacts.yml"}) {
t.Fatalf("handout sources = %#v", got)
}
if got := compositionNodeAtPath(t, merged.root, "scriptorium.artifacts.recap.enabled").sources; !reflect.DeepEqual(got, []string{"pipeline.yml"}) {
t.Fatalf("recap sources = %#v", got)
}
// Merge operations return a new document and retain the declared order in
// each input for source-aware diagnostics.
if len(root.root.fields) != 2 || len(imports.root.fields) != 2 {
t.Fatalf("merge mutated inputs: root=%d import=%d", len(root.root.fields), len(imports.root.fields))
}
}
func TestMergeAdditiveCompositionRejectsEveryDuplicateClass(t *testing.T) {
tests := []struct {
name string
baseYAML string
nextYAML string
path string
}{
{name: "equal scalar", baseYAML: "value: true\n", nextYAML: "value: true\n", path: "value"},
{name: "different scalar", baseYAML: "value: true\n", nextYAML: "value: false\n", path: "value"},
{name: "atomic list", baseYAML: "values: [one]\n", nextYAML: "values: [two]\n", path: "values"},
{name: "keyed entry", baseYAML: "items:\n shared:\n left: 1\n", nextYAML: "items:\n shared:\n left: 2\n", path: "items.shared.left"},
{name: "kind conflict", baseYAML: "value:\n nested: true\n", nextYAML: "value: scalar\n", path: "value"},
{name: "duplicate empty map", baseYAML: "value: {}\n", nextYAML: "value: {}\n", path: "value"},
{name: "empty map then populated map", baseYAML: "value: {}\n", nextYAML: "value: {nested: true}\n", path: "value"},
{name: "populated map then empty map", baseYAML: "value: {nested: true}\n", nextYAML: "value: {}\n", path: "value"},
{name: "duplicate empty list", baseYAML: "value: []\n", nextYAML: "value: []\n", path: "value"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
base := mustParseComposition(t, "base.yml", tt.baseYAML)
next := mustParseComposition(t, "next.yml", tt.nextYAML)
_, err := mergeAdditiveComposition(base, next)
if err == nil {
t.Fatal("mergeAdditiveComposition() error = nil")
}
for _, want := range []string{tt.path, "base.yml", "next.yml"} {
if !strings.Contains(err.Error(), want) {
t.Fatalf("error = %q, want %q", err, want)
}
}
})
}
}
func TestMergeAdditiveCompositionReportsAllClaimingSources(t *testing.T) {
left := mustParseComposition(t, "left.yml", "group:\n left: 1\n")
right := mustParseComposition(t, "right.yml", "group:\n right: 2\n")
base, err := mergeAdditiveComposition(left, right)
if err != nil {
t.Fatal(err)
}
overlap := mustParseComposition(t, "overlap.yml", "group: scalar\n")
_, err = mergeAdditiveComposition(base, overlap)
if err == nil {
t.Fatal("mergeAdditiveComposition() error = nil")
}
for _, want := range []string{"group", "left.yml", "right.yml", "overlap.yml"} {
if !strings.Contains(err.Error(), want) {
t.Fatalf("error = %q, want %q", err, want)
}
}
_, err = mergeAdditiveCompositions(
mustParseComposition(t, "first.yml", "value: 1\n"),
mustParseComposition(t, "second.yml", "value: 2\n"),
mustParseComposition(t, "third.yml", "value: 3\n"),
)
if err == nil {
t.Fatal("mergeAdditiveCompositions() error = nil")
}
for _, want := range []string{"value", "first.yml", "second.yml", "third.yml"} {
if !strings.Contains(err.Error(), want) {
t.Fatalf("error = %q, want %q", err, want)
}
}
}
func TestMergeOverlayCompositionRecursesMapsAndReplacesAtomicValues(t *testing.T) {
base := mustParseComposition(t, "base.yml", `feature:
enabled: true
retries: 3
values: [one, two]
inherited: kept
artifacts:
recap:
enabled: true
`)
overlay := mustParseComposition(t, "testing.yml", `feature:
enabled: false
retries: 0
values: []
added: present
artifacts:
handout:
enabled: false
`)
merged, err := mergeOverlayComposition(base, overlay)
if err != nil {
t.Fatalf("mergeOverlayComposition() error = %v", err)
}
assertCompositionScalar(t, merged.root, "feature.enabled", "!!bool", "false", "testing.yml")
assertCompositionScalar(t, merged.root, "feature.retries", "!!int", "0", "testing.yml")
assertCompositionScalar(t, merged.root, "feature.inherited", "!!str", "kept", "base.yml")
assertCompositionScalar(t, merged.root, "feature.added", "!!str", "present", "testing.yml")
if values := compositionNodeAtPath(t, merged.root, "feature.values"); values.kind != yaml.SequenceNode || len(values.items) != 0 || !reflect.DeepEqual(values.sources, []string{"testing.yml"}) {
t.Fatalf("replaced list = %#v", values)
}
_ = compositionNodeAtPath(t, merged.root, "artifacts.recap.enabled")
_ = compositionNodeAtPath(t, merged.root, "artifacts.handout.enabled")
}
func TestMergeOverlayCompositionRejectsKindChangesAndNullDeletion(t *testing.T) {
kindTests := []struct {
name string
baseYAML string
overlay string
}{
{name: "map to scalar", baseYAML: "value: {nested: true}\n", overlay: "value: replacement\n"},
{name: "scalar to map", baseYAML: "value: original\n", overlay: "value: {nested: true}\n"},
{name: "list to scalar", baseYAML: "value: [one]\n", overlay: "value: replacement\n"},
{name: "scalar to list", baseYAML: "value: original\n", overlay: "value: [one]\n"},
}
for _, tt := range kindTests {
t.Run(tt.name, func(t *testing.T) {
_, err := mergeOverlayComposition(
mustParseComposition(t, "base.yml", tt.baseYAML),
mustParseComposition(t, "overlay.yml", tt.overlay),
)
if err == nil || !strings.Contains(err.Error(), "value") || !strings.Contains(err.Error(), "kind change") {
t.Fatalf("error = %v, want value kind change", err)
}
})
}
for _, overlayYAML := range []string{"value: null\n", "value: ~\n", "nested:\n value:\n"} {
_, err := mergeOverlayComposition(
mustParseComposition(t, "base.yml", "value: original\nnested:\n value: original\n"),
mustParseComposition(t, "overlay.yml", overlayYAML),
)
if err == nil || !strings.Contains(err.Error(), "null cannot delete") || !strings.Contains(err.Error(), "overlay.yml") {
t.Fatalf("error = %v, want source-qualified null rejection", err)
}
}
}
func TestCompositionCanonicalOutputsAreDeterministic(t *testing.T) {
first := mustParseComposition(t, "first.yml", `zeta: 01
alpha:
list: [true, false]
empty: {}
`)
second := mustParseComposition(t, "second.yml", `alpha:
empty: {}
list:
- true
- false
zeta: 1
`)
firstYAML, err := first.canonicalYAML()
if err != nil {
t.Fatal(err)
}
secondYAML, err := second.canonicalYAML()
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(firstYAML, secondYAML) {
t.Fatalf("canonical YAML differs:\n%s\n---\n%s", firstYAML, secondYAML)
}
firstDigest, err := first.canonicalDigestInput()
if err != nil {
t.Fatal(err)
}
secondDigest, err := second.canonicalDigestInput()
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(firstDigest, secondDigest) {
t.Fatalf("digest input differs:\n%s\n---\n%s", firstDigest, secondDigest)
}
left := mustParseComposition(t, "left.yml", "zeta: 1\n")
right := mustParseComposition(t, "right.yml", "alpha: 2\n")
leftRight, err := mergeAdditiveComposition(left, right)
if err != nil {
t.Fatal(err)
}
rightLeft, err := mergeAdditiveComposition(right, left)
if err != nil {
t.Fatal(err)
}
want, _ := leftRight.canonicalDigestInput()
got, _ := rightLeft.canonicalDigestInput()
if !reflect.DeepEqual(got, want) {
t.Fatalf("source traversal changed semantic digest input: got %s want %s", got, want)
}
wantYAML, _ := leftRight.canonicalYAML()
gotYAML, _ := rightLeft.canonicalYAML()
if !reflect.DeepEqual(gotYAML, wantYAML) {
t.Fatalf("source traversal changed canonical YAML: got %s want %s", gotYAML, wantYAML)
}
records, err := first.semanticRecords()
if err != nil {
t.Fatal(err)
}
paths := make([]string, 0, len(records))
for _, record := range records {
paths = append(paths, record.Path)
}
if wantPaths := []string{"alpha.empty", "alpha.list", "zeta"}; !reflect.DeepEqual(paths, wantPaths) {
t.Fatalf("semantic record paths = %#v, want %#v", paths, wantPaths)
}
}
func mustParseComposition(t *testing.T, source, input string) *compositionDocument {
t.Helper()
document, err := parseCompositionBytes(source, []byte(input))
if err != nil {
t.Fatalf("parseCompositionBytes(%q) error = %v", source, err)
}
return document
}
func compositionNodeAtPath(t *testing.T, root *compositionNode, path string) *compositionNode {
t.Helper()
node := root
for _, segment := range strings.Split(path, ".") {
if node == nil || node.kind != yaml.MappingNode {
t.Fatalf("path %q reached non-mapping at %q", path, segment)
}
index := compositionFieldIndex(node.fields, segment)
if index < 0 {
t.Fatalf("path %q missing segment %q", path, segment)
}
node = node.fields[index].value
}
return node
}
func assertCompositionScalar(t *testing.T, root *compositionNode, path, tag, value, source string) {
t.Helper()
node := compositionNodeAtPath(t, root, path)
if node.kind != yaml.ScalarNode || node.tag != tag || node.value != value || !reflect.DeepEqual(node.sources, []string{source}) {
t.Fatalf("%s = kind=%s tag=%q value=%q sources=%#v", path, yamlKindName(node.kind), node.tag, node.value, node.sources)
}
}

View File

@@ -191,8 +191,6 @@ type NormalizeConfig struct {
OutputPath string `yaml:"output_path" json:"output_path"`
OutputSchema string `yaml:"output_schema" json:"output_schema"`
Report *bool `yaml:"report" json:"report"`
outputPathSet bool `yaml:"-" json:"-"`
}
// TrimConfig configures trim-stage transcript boundary behavior.

View File

@@ -74,7 +74,7 @@ func finalizePipelineResolution(cfg *PipelineConfig) error {
// recomputePipelineEffectiveDigest is the single package-owned hook for
// refreshing provenance after later resolution expands concrete pipeline
// values. Composition declarations and runtime provenance are not serialized.
// values. Loader directives and runtime provenance are not serialized.
func recomputePipelineEffectiveDigest(cfg *PipelineConfig) error {
if cfg == nil || cfg.resolution == nil {
return fmt.Errorf("pipeline resolution metadata is required")

View File

@@ -55,7 +55,7 @@ type EffectivePipelineValueRecord struct {
// EffectivePipelineValues projects a fully resolved pipeline into sorted
// logical configuration values. Mappings are flattened, while sequences remain
// atomic values. The projection uses the same normalized effective mapping as
// config show and therefore excludes composition and family declarations.
// config show and therefore excludes loader directives and family declarations.
func EffectivePipelineValues(cfg *PipelineConfig) ([]EffectivePipelineValueRecord, error) {
if cfg == nil {
return nil, fmt.Errorf("pipeline config is required")
@@ -301,6 +301,9 @@ func pipelineSourceRole(cfg *PipelineConfig, source string) string {
if source == pipelineDefaultOwnershipSource {
return "default"
}
if source == "resolution" {
return "resolution"
}
if cfg == nil || cfg.resolution == nil {
return "pipeline"
}
@@ -319,7 +322,7 @@ func pipelineSourceRole(cfg *PipelineConfig, source string) string {
}
func normalizedProvenanceSource(source string) string {
if source == pipelineDefaultOwnershipSource {
if source == pipelineDefaultOwnershipSource || source == "resolution" {
return source
}
return normalizedProvenancePath(source)

View File

@@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"sort"
"strings"
)
// effectivePipelineProjection is the single safe, schema-specific view used
@@ -37,6 +38,11 @@ type effectiveScriptoriumConfig struct {
Artifacts map[string]ScriptoriumArtifactConfig `yaml:"artifacts" json:"artifacts"`
}
type projectedPipelineValue struct {
record EffectivePipelineValueRecord
segments []string
}
func projectEffectivePipeline(cfg *PipelineConfig, logicalNotarius bool) effectivePipelineProjection {
projection := effectivePipelineProjection{
Workspace: cfg.Workspace, Campaigns: cfg.Campaigns, Storage: cfg.Storage,
@@ -70,23 +76,35 @@ func projectionJSON(projection effectivePipelineProjection) ([]byte, error) {
}
func projectionValueRecords(projection effectivePipelineProjection) ([]EffectivePipelineValueRecord, error) {
data, err := projectionJSON(projection)
detailed, err := projectionDetailedRecords(projection)
if err != nil {
return nil, err
}
records := make([]EffectivePipelineValueRecord, 0, len(detailed))
for _, value := range detailed {
records = append(records, value.record)
}
return records, nil
}
func projectionDetailedRecords(value any) ([]projectedPipelineValue, error) {
data, err := json.Marshal(value)
if err != nil {
return nil, fmt.Errorf("encode typed pipeline projection: %w", err)
}
decoder := json.NewDecoder(bytes.NewReader(data))
decoder.UseNumber()
var root map[string]any
if err := decoder.Decode(&root); err != nil {
return nil, fmt.Errorf("decode effective pipeline projection: %w", err)
}
var records []EffectivePipelineValueRecord
appendProjectionRecords(&records, "", root)
sort.Slice(records, func(i, j int) bool { return records[i].Path < records[j].Path })
var records []projectedPipelineValue
appendProjectionRecords(&records, nil, root)
sort.Slice(records, func(i, j int) bool { return records[i].record.Path < records[j].record.Path })
return records, nil
}
func appendProjectionRecords(records *[]EffectivePipelineValueRecord, path string, value any) {
func appendProjectionRecords(records *[]projectedPipelineValue, segments []string, value any) {
if mapping, ok := value.(map[string]any); ok && len(mapping) > 0 {
keys := make([]string, 0, len(mapping))
for key := range mapping {
@@ -94,14 +112,15 @@ func appendProjectionRecords(records *[]EffectivePipelineValueRecord, path strin
}
sort.Strings(keys)
for _, key := range keys {
childPath := key
if path != "" {
childPath = path + "." + key
}
appendProjectionRecords(records, childPath, mapping[key])
child := append(append([]string(nil), segments...), key)
appendProjectionRecords(records, child, mapping[key])
}
return
}
encoded, _ := json.Marshal(value)
*records = append(*records, EffectivePipelineValueRecord{Path: path, Value: string(encoded)})
path := strings.Join(segments, ".")
*records = append(*records, projectedPipelineValue{
record: EffectivePipelineValueRecord{Path: path, Value: string(encoded)},
segments: append([]string(nil), segments...),
})
}

View File

@@ -17,9 +17,8 @@ func LoadPipeline(path string) (*PipelineConfig, error) {
return LoadPipelineWithOptions(path, PipelineLoadOptions{})
}
// PipelineLoadOptions carries an optional explicit profile selection. A nil
// Profile means the caller omitted selection; a non-nil empty value is an
// explicit invalid selection.
// PipelineLoadOptions carries an optional explicit profile selection. Nil,
// empty, and the exact name "default" select the base configuration.
type PipelineLoadOptions struct {
Profile *string
}
@@ -27,77 +26,32 @@ type PipelineLoadOptions struct {
// LoadPipelineWithOptions loads pipeline configuration with strict field
// checking and optional named-profile selection.
func LoadPipelineWithOptions(path string, opts PipelineLoadOptions) (*PipelineConfig, error) {
sources, err := loadPipelineCompositionSources(path)
cfg, err := loadPipelineWithConfigKit(path, opts)
if err != nil {
return nil, fmt.Errorf("load pipeline config: %w", err)
}
if _, err := selectPipelineProfile(sources.envelope, opts); err != nil {
return nil, fmt.Errorf("load pipeline config: %w", err)
}
if err := sources.loadOverlays(); err != nil {
return nil, fmt.Errorf("load pipeline config: %w", err)
}
cfg, err := sources.resolve(opts)
if err != nil {
return nil, fmt.Errorf("load pipeline config: %w", err)
return nil, err
}
return finalizeLoadedPipeline(path, cfg)
}
// LoadPipelineProfilePair resolves two explicit named profiles from one parsed
// pipeline root and its declared source set. Each result is independently
// decoded, defaulted, and finalized so later resolution can safely mutate one
// effective pipeline without affecting the other.
// LoadPipelineProfilePair resolves two profiles as independent ConfigKit loads.
func LoadPipelineProfilePair(path, leftProfile, rightProfile string) (*PipelineConfig, *PipelineConfig, error) {
if _, err := normalizePipelineProfileName(leftProfile, "left profile selection"); err != nil {
return nil, nil, fmt.Errorf("load left pipeline profile: %w", err)
}
if _, err := normalizePipelineProfileName(rightProfile, "right profile selection"); err != nil {
return nil, nil, fmt.Errorf("load right pipeline profile: %w", err)
}
if leftProfile == rightProfile {
return nil, nil, fmt.Errorf("load pipeline profiles: left and right profile selections must differ")
}
sources, err := loadPipelineCompositionSources(path)
if err != nil {
return nil, nil, fmt.Errorf("load pipeline config: %w", err)
}
leftOptions := PipelineLoadOptions{Profile: &leftProfile}
rightOptions := PipelineLoadOptions{Profile: &rightProfile}
if _, err := selectPipelineProfile(sources.envelope, leftOptions); err != nil {
return nil, nil, fmt.Errorf("load left pipeline profile: %w", err)
}
if _, err := selectPipelineProfile(sources.envelope, rightOptions); err != nil {
return nil, nil, fmt.Errorf("load right pipeline profile: %w", err)
}
if err := sources.loadOverlays(); err != nil {
return nil, nil, fmt.Errorf("load pipeline config: %w", err)
}
left, err := sources.resolve(leftOptions)
left, err := LoadPipelineWithOptions(path, leftOptions)
if err != nil {
return nil, nil, fmt.Errorf("load left pipeline profile: %w", err)
}
right, err := sources.resolve(rightOptions)
right, err := LoadPipelineWithOptions(path, rightOptions)
if err != nil {
return nil, nil, fmt.Errorf("load right pipeline profile: %w", err)
}
left, err = finalizeLoadedPipeline(path, left)
if err != nil {
return nil, nil, err
}
right, err = finalizeLoadedPipeline(path, right)
if err != nil {
return nil, nil, err
}
return left, right, nil
}
func finalizeLoadedPipeline(path string, cfg *PipelineConfig) (*PipelineConfig, error) {
cfg.resolution.publishDeclared = cfg.Publish != nil
applyPipelineDefaults(cfg)
if err := NormalizePipelineConfig(cfg); err != nil {
return nil, fmt.Errorf("load pipeline config: %w", err)
}
captureLogicalNotariusPaths(cfg)
if err := resolveNotariusPaths(cfg, path); err != nil {
return nil, fmt.Errorf("load pipeline config: %w", err)
@@ -504,217 +458,6 @@ func shortName(path, fallback string) string {
return base
}
func applyPipelineDefaults(cfg *PipelineConfig) {
if cfg == nil {
return
}
applyWorkspaceDefaults(&cfg.Workspace)
applyCampaignsDefaults(&cfg.Campaigns)
applyStorageDefaults(&cfg.Storage)
applySpoolDefaults(&cfg.Spool)
applyCacheDefaults(&cfg.Cache)
applyPublishDefaults(&cfg.Publish)
applyWhisperXDefaults(&cfg.WhisperX)
applySeriatimDefaults(&cfg.Seriatim)
applyAuditaDefaults(&cfg.Audita)
if cfg.Normalize == nil {
cfg.Normalize = &NormalizeConfig{}
}
applyNormalizeDefaults(cfg.Normalize)
if cfg.Trim == nil {
cfg.Trim = &TrimConfig{}
}
applyTrimDefaults(cfg.Trim)
if trimEnabled(cfg.Trim) && cfg.Scriptorium == nil {
cfg.Scriptorium = &ScriptoriumConfig{}
}
applyRenderDefaults(&cfg.Render)
applyScriptoriumDefaults(cfg.Scriptorium)
applyNotariusDefaults(cfg.Notarius)
applyNotificationDefaults(&cfg.Notification)
}
func applyNotificationDefaults(cfg *NotificationConfig) {
if cfg == nil {
return
}
if strings.TrimSpace(cfg.Mode) == "" {
cfg.Mode = DefaultNotificationMode
}
}
func applyCampaignsDefaults(cfg *CampaignsConfig) {
if cfg == nil {
return
}
if cfg.Root == "" {
cfg.Root = DefaultCampaignsRoot
}
}
func applyWorkspaceDefaults(cfg *WorkspaceConfig) {
if cfg == nil {
return
}
if cfg.Root == "" {
cfg.Root = DefaultWorkspaceRoot
}
}
func applyStorageDefaults(cfg *StorageConfig) {
if cfg == nil {
return
}
backend := strings.ToLower(strings.TrimSpace(cfg.Backend))
if backend == "" {
backend = StorageBackendLocal
}
cfg.Backend = backend
if cfg.S3 == nil {
return
}
if cfg.S3.RootPrefix == "" {
cfg.S3.RootPrefix = DefaultStorageS3RootPrefix
}
if cfg.S3.AccessKeyIDEnv == "" {
cfg.S3.AccessKeyIDEnv = DefaultS3AccessKeyIDEnv
}
if cfg.S3.SecretKeyEnv == "" {
cfg.S3.SecretKeyEnv = DefaultS3SecretAccessKeyEnv
}
}
func applySpoolDefaults(cfg *SpoolConfig) {
if cfg == nil {
return
}
if cfg.Root == "" {
cfg.Root = DefaultSpoolRoot
}
}
func applyCacheDefaults(cfg *CacheConfig) {
if cfg == nil {
return
}
if cfg.Root == "" {
cfg.Root = DefaultCacheRoot
}
if cfg.S3Audio == nil {
cfg.S3Audio = boolPtr(DefaultCacheS3Audio)
}
}
func applyPublishDefaults(cfg **PublishConfig) {
if cfg == nil {
return
}
if *cfg == nil {
*cfg = &PublishConfig{}
}
if (*cfg).Enabled == nil {
(*cfg).Enabled = boolPtr(DefaultArchiveEnabled)
}
if (*cfg).UploadRun == nil {
(*cfg).UploadRun = boolPtr(DefaultArchiveUploadRun)
}
if len((*cfg).Outputs) == 0 {
(*cfg).Outputs = append([]PublishOutputRule(nil), DefaultPublishOutputs...)
}
for i := range (*cfg).Outputs {
if (*cfg).Outputs[i].Required == nil {
(*cfg).Outputs[i].Required = boolPtr(true)
}
}
}
func applyWhisperXDefaults(cfg *WhisperXConfig) {
if cfg == nil {
return
}
if cfg.Language == "" {
cfg.Language = DefaultWhisperXLanguage
}
if cfg.Timeout == "" {
cfg.Timeout = DefaultWhisperXTimeout
}
if cfg.RetryDelay == "" {
cfg.RetryDelay = DefaultWhisperXRetryDelay
}
if cfg.Concurrency == nil {
cfg.Concurrency = intPtr(DefaultWhisperXConcurrency)
}
if cfg.Retries == nil {
cfg.Retries = intPtr(DefaultWhisperXRetries)
}
}
func intPtr(v int) *int {
p := v
return &p
}
func applySeriatimDefaults(cfg *SeriatimConfig) {
if cfg == nil {
return
}
if cfg.Binary == "" {
cfg.Binary = DefaultSeriatimBinary
}
if cfg.Timeout == "" {
cfg.Timeout = DefaultSeriatimTimeout
}
if cfg.OutputSchema == "" {
cfg.OutputSchema = DefaultSeriatimOutputSchema
}
if cfg.CoalesceGap == nil {
cfg.CoalesceGap = float64Ptr(DefaultSeriatimCoalesceGap)
}
if cfg.Report == nil {
cfg.Report = boolPtr(DefaultSeriatimReport)
}
}
func applyAuditaDefaults(cfg *AuditaConfig) {
if cfg == nil {
return
}
if cfg.Binary == "" {
cfg.Binary = DefaultAuditaBinary
}
if cfg.Timeout == "" {
cfg.Timeout = DefaultAuditaTimeout
}
if cfg.Report == nil {
cfg.Report = boolPtr(DefaultAuditaReport)
}
}
func applyScriptoriumDefaults(cfg *ScriptoriumConfig) {
if cfg == nil {
return
}
if cfg.Binary == "" {
cfg.Binary = DefaultScriptoriumBinary
}
if cfg.Timeout == "" {
cfg.Timeout = DefaultScriptoriumTimeout
}
}
func applyNotariusDefaults(cfg *NotariusConfig) {
if cfg == nil {
return
}
if cfg.Binary == "" {
cfg.Binary = DefaultNotariusBinary
}
if cfg.Timeout == "" {
cfg.Timeout = DefaultNotariusTimeout
}
}
func resolveNotariusPaths(cfg *PipelineConfig, pipelinePath string) error {
if cfg == nil || cfg.Notarius == nil || !cfg.Notarius.Enabled {
return nil
@@ -753,83 +496,3 @@ func resolveNotariusPaths(cfg *PipelineConfig, pipelinePath string) error {
cfg.Notarius.WorkingDirectory = workingDirectory
return nil
}
func applyTrimDefaults(cfg *TrimConfig) {
if cfg == nil {
return
}
if cfg.Enabled == nil {
cfg.Enabled = boolPtr(DefaultTrimEnabled)
}
if strings.TrimSpace(cfg.OutputPath) == "" {
cfg.OutputPath = DefaultTrimOutputPath
}
if strings.TrimSpace(cfg.Bounds.PromptID) == "" {
cfg.Bounds.PromptID = DefaultTrimBoundsPromptID
}
if strings.TrimSpace(cfg.Bounds.TranscriptInputName) == "" {
cfg.Bounds.TranscriptInputName = DefaultTrimBoundsTranscriptInputName
}
if strings.TrimSpace(cfg.Bounds.OutputPath) == "" {
cfg.Bounds.OutputPath = DefaultTrimBoundsOutputPath
}
if cfg.Bounds.Timeout == "" {
cfg.Bounds.Timeout = DefaultTrimBoundsTimeout
}
if cfg.Seriatim.Report == nil {
cfg.Seriatim.Report = boolPtr(DefaultTrimSeriatimReport)
}
}
func trimEnabled(cfg *TrimConfig) bool {
return cfg != nil && cfg.Enabled != nil && *cfg.Enabled
}
func applyRenderDefaults(cfg **RenderConfig) {
if cfg == nil {
return
}
if *cfg == nil {
*cfg = &RenderConfig{}
}
if (*cfg).Enabled == nil {
(*cfg).Enabled = boolPtr(DefaultRenderEnabled)
}
if strings.TrimSpace((*cfg).Format) == "" {
(*cfg).Format = DefaultRenderFormat
}
if strings.TrimSpace((*cfg).Title) == "" {
(*cfg).Title = DefaultRenderTitle
}
if (*cfg).IncludeTimestamps == nil {
(*cfg).IncludeTimestamps = boolPtr(DefaultRenderTimestamps)
}
if (*cfg).IncludeSegmentIDs == nil {
(*cfg).IncludeSegmentIDs = boolPtr(DefaultRenderSegmentIDs)
}
}
func applyNormalizeDefaults(cfg *NormalizeConfig) {
if cfg == nil {
return
}
if cfg.OutputSchema == "" {
cfg.OutputSchema = DefaultNormalizeOutputSchema
}
if cfg.OutputPath == "" && !cfg.outputPathWasSet() {
cfg.OutputPath = DefaultNormalizeOutputPath
}
if cfg.Report == nil {
cfg.Report = boolPtr(DefaultNormalizeReport)
}
}
func float64Ptr(v float64) *float64 {
p := v
return &p
}
func boolPtr(v bool) *bool {
p := v
return &p
}

View File

@@ -48,7 +48,7 @@ func TestLoadersRejectTrailingYAMLDocuments(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := tt.load()
if err == nil || !strings.Contains(err.Error(), "exactly one YAML document") {
if err == nil || (!strings.Contains(err.Error(), "exactly one YAML document") && !strings.Contains(err.Error(), "yaml.multiple_documents")) {
t.Fatalf("load error = %v, want exactly-one-document error", err)
}
})

View File

@@ -1,49 +0,0 @@
package config
import (
"fmt"
"gopkg.in/yaml.v3"
)
// UnmarshalYAML tracks explicit normalize.output_path presence so validation can
// distinguish omitted vs explicitly empty values.
func (cfg *NormalizeConfig) UnmarshalYAML(node *yaml.Node) error {
if node.Kind != yaml.MappingNode {
return fmt.Errorf("normalize must be a mapping")
}
allowedKeys := map[string]struct{}{
"output_path": {},
"output_schema": {},
"report": {},
}
for i := 0; i+1 < len(node.Content); i += 2 {
key := node.Content[i].Value
if _, ok := allowedKeys[key]; !ok {
return fmt.Errorf("field %q not found in type config.NormalizeConfig", key)
}
}
type rawNormalize NormalizeConfig
var raw rawNormalize
if err := node.Decode(&raw); err != nil {
return err
}
*cfg = NormalizeConfig(raw)
for i := 0; i+1 < len(node.Content); i += 2 {
if node.Content[i].Value == "output_path" {
cfg.outputPathSet = true
break
}
}
return nil
}
// outputPathSet records whether normalize.output_path appeared in YAML.
// This is intentionally unexported and not serialized.
func (cfg *NormalizeConfig) outputPathWasSet() bool {
if cfg == nil {
return false
}
return cfg.outputPathSet
}

View File

@@ -107,6 +107,12 @@ notarius:
enabled: true
config_path: notarius/config.yml
pipeline_id: dnd-session
outputs:
npc_registry:
lane_id: npc-registry
media_type: application/json
schema_id: notarius.dnd.npc_registry
schema_version: v1
`
paths := make([]string, 2)
configs := make([]*PipelineConfig, 2)
@@ -151,7 +157,7 @@ func TestNotariusStrictYAML(t *testing.T) {
_, err := LoadPipeline(path)
want := "strict decode failed"
if tt.name == "duplicate reference selector" {
want = "duplicate YAML key"
want = "yaml.duplicate_key"
}
if err == nil || !strings.Contains(err.Error(), want) {
t.Fatalf("LoadPipeline() error = %v, want containing %q", err, want)

View File

@@ -23,12 +23,12 @@ func TestNotificationConfigSupportsOnlyNoopMode(t *testing.T) {
{
name: "backend is rejected",
section: "notification:\n backend: email\n",
wantLoad: "field backend not found",
wantLoad: "decode.unknown_field",
},
{
name: "recipient is rejected",
section: "notification:\n recipient: party@example.com\n",
wantLoad: "field recipient not found",
wantLoad: "decode.unknown_field",
},
{
name: "provider mode is rejected",

View File

@@ -1,532 +0,0 @@
package config
import (
"fmt"
"os"
"path/filepath"
"strings"
"unicode"
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
"gopkg.in/yaml.v3"
)
type pipelineResolutionMetadata struct {
rootPath string
imports []string
sources []string
selectedProfile *pipelineProfileSelection
effectiveDigest string
logicalNotariusConfig string
logicalNotariusWorking string
logicalNotariusCaptured bool
ownership []pipelineFieldOwnership
artifactFamilies map[string]ScriptoriumArtifactFamilyConfig
artifactFamiliesExpanded bool
publishDeclared bool
}
type pipelineProfileSelection struct {
name string
source string
overlayPath string
}
type pipelineFieldOwnership struct {
path string
sources []string
}
type pipelineCompositionEnvelope struct {
imports []string
defaultProfile *string
profiles []pipelineProfileDeclaration
}
type pipelineProfileDeclaration struct {
name string
overlay string
}
// pipelineCompositionSources retains one validated root source set. Each
// selected profile is resolved from a cloned base document so callers can
// safely compare or otherwise resolve multiple profiles without rereading or
// mutating the source set.
type pipelineCompositionSources struct {
rootPath string
envelope pipelineCompositionEnvelope
imports []loadedPipelineImport
overlays []loadedPipelineProfileOverlay
overlaysLoaded bool
base *compositionDocument
}
func loadPipelineCompositionSources(path string) (*pipelineCompositionSources, error) {
rootPath, err := filepath.Abs(path)
if err != nil {
return nil, fmt.Errorf("resolve root pipeline path %q: %w", path, err)
}
rootFile, err := os.Open(rootPath)
if err != nil {
return nil, fmt.Errorf("pipeline file %q: open: %w", path, err)
}
rootDocument, parseErr := parseCompositionDocument(rootPath, rootFile)
closeErr := rootFile.Close()
if parseErr != nil {
return nil, parseErr
}
if closeErr != nil {
return nil, fmt.Errorf("pipeline file %q: close: %w", rootPath, closeErr)
}
baseRoot, envelope, err := splitPipelineCompositionEnvelope(rootDocument)
if err != nil {
return nil, err
}
imports, err := loadPipelineImports(rootPath, envelope.imports)
if err != nil {
return nil, err
}
documents := make([]*compositionDocument, 0, len(imports)+1)
documents = append(documents, baseRoot)
for _, imported := range imports {
documents = append(documents, imported.document)
}
merged, err := mergeAdditiveCompositions(documents...)
if err != nil {
return nil, err
}
return &pipelineCompositionSources{
rootPath: rootPath,
envelope: envelope,
imports: imports,
base: merged,
}, nil
}
func (sources *pipelineCompositionSources) loadOverlays() error {
if sources == nil {
return fmt.Errorf("pipeline composition sources are required")
}
if sources.overlaysLoaded {
return nil
}
overlays, err := loadPipelineProfileOverlays(sources.rootPath, sources.envelope.profiles, sources.imports)
if err != nil {
return err
}
sources.overlays = overlays
sources.overlaysLoaded = true
return nil
}
func (sources *pipelineCompositionSources) resolve(opts PipelineLoadOptions) (*PipelineConfig, error) {
if sources == nil || sources.base == nil {
return nil, fmt.Errorf("pipeline composition sources are required")
}
if !sources.overlaysLoaded {
return nil, fmt.Errorf("pipeline profile overlays have not been loaded")
}
selection, err := selectPipelineProfile(sources.envelope, opts)
if err != nil {
return nil, err
}
merged := &compositionDocument{
root: cloneCompositionNode(sources.base.root),
sources: append([]string(nil), sources.base.sources...),
}
if selection != nil {
overlay, ok := loadedProfileOverlay(sources.overlays, selection.name)
if !ok {
return nil, fmt.Errorf("selected profile %q overlay was not loaded", selection.name)
}
merged, err = mergeOverlayComposition(merged, overlay.document)
if err != nil {
return nil, err
}
selection.overlayPath = overlay.path
}
rendered, err := merged.canonicalYAML()
if err != nil {
return nil, err
}
var cfg PipelineConfig
if err := decodeStrictYAMLFromReader("pipeline", sources.rootPath, strings.NewReader(string(rendered)), &cfg); err != nil {
return nil, fmt.Errorf("assembled pipeline sources %s: %w", formatCompositionSources(merged.sources), err)
}
records, err := merged.semanticRecords()
if err != nil {
return nil, err
}
metadata := &pipelineResolutionMetadata{
rootPath: sources.rootPath,
sources: append([]string(nil), merged.sources...),
selectedProfile: selection,
}
for _, imported := range sources.imports {
metadata.imports = append(metadata.imports, imported.path)
}
for _, record := range records {
metadata.ownership = append(metadata.ownership, pipelineFieldOwnership{
path: record.Path, sources: append([]string(nil), record.Sources...),
})
}
cfg.resolution = metadata
return &cfg, nil
}
func splitPipelineCompositionEnvelope(document *compositionDocument) (*compositionDocument, pipelineCompositionEnvelope, error) {
if err := validateCompositionDocument(document, "root pipeline"); err != nil {
return nil, pipelineCompositionEnvelope{}, err
}
envelope, fieldIndex, err := decodePipelineCompositionEnvelope(document.root)
if err != nil {
return nil, pipelineCompositionEnvelope{}, err
}
if err := validatePipelineCompositionEnvelope(envelope); err != nil {
return nil, pipelineCompositionEnvelope{}, err
}
return pipelineBodyWithoutComposition(document, fieldIndex), envelope, nil
}
func decodePipelineCompositionEnvelope(root *compositionNode) (pipelineCompositionEnvelope, int, error) {
index := compositionFieldIndex(root.fields, "composition")
if index < 0 {
return pipelineCompositionEnvelope{}, -1, nil
}
envelopeNode := root.fields[index].value
if envelopeNode.kind != yaml.MappingNode {
return pipelineCompositionEnvelope{}, -1, fmt.Errorf(
"configuration source %s at composition: expected a mapping, got %s",
formatCompositionSources(envelopeNode.sources), yamlKindName(envelopeNode.kind),
)
}
var envelope pipelineCompositionEnvelope
for _, field := range envelopeNode.fields {
switch field.key {
case "imports":
if field.value.kind != yaml.SequenceNode {
return pipelineCompositionEnvelope{}, -1, fmt.Errorf(
"configuration source %s at composition.imports: expected a list, got %s",
formatCompositionSources(field.value.sources), yamlKindName(field.value.kind),
)
}
for itemIndex, item := range field.value.items {
if item.kind != yaml.ScalarNode || item.tag != "!!str" {
return pipelineCompositionEnvelope{}, -1, fmt.Errorf(
"configuration source %s at composition.imports[%d]: expected a string path",
formatCompositionSources(item.sources), itemIndex,
)
}
envelope.imports = append(envelope.imports, item.value)
}
case "default_profile":
if field.value.kind != yaml.ScalarNode || field.value.tag != "!!str" {
return pipelineCompositionEnvelope{}, -1, fmt.Errorf(
"configuration source %s at composition.default_profile: expected a string profile name",
formatCompositionSources(field.value.sources),
)
}
name, err := normalizePipelineProfileName(field.value.value, "composition.default_profile")
if err != nil {
return pipelineCompositionEnvelope{}, -1, err
}
envelope.defaultProfile = &name
case "profiles":
if field.value.kind != yaml.MappingNode {
return pipelineCompositionEnvelope{}, -1, fmt.Errorf(
"configuration source %s at composition.profiles: expected a mapping, got %s",
formatCompositionSources(field.value.sources), yamlKindName(field.value.kind),
)
}
if len(field.value.fields) == 0 {
return pipelineCompositionEnvelope{}, -1, fmt.Errorf("composition.profiles must declare at least one named profile")
}
for _, profileField := range field.value.fields {
name, err := normalizePipelineProfileName(profileField.key, "composition.profiles profile name")
if err != nil {
return pipelineCompositionEnvelope{}, -1, err
}
profile, err := parsePipelineProfileDeclaration(name, profileField.value)
if err != nil {
return pipelineCompositionEnvelope{}, -1, err
}
envelope.profiles = append(envelope.profiles, profile)
}
default:
return pipelineCompositionEnvelope{}, -1, fmt.Errorf(
"configuration source %s at composition.%s: unknown composition field %q",
formatCompositionSources(field.value.sources), field.key, field.key,
)
}
}
return envelope, index, nil
}
func validatePipelineCompositionEnvelope(envelope pipelineCompositionEnvelope) error {
if envelope.defaultProfile != nil && !pipelineProfileDeclared(envelope.profiles, *envelope.defaultProfile) {
return fmt.Errorf(
"composition.default_profile %q does not name a declared profile", *envelope.defaultProfile,
)
}
return nil
}
func pipelineBodyWithoutComposition(document *compositionDocument, compositionIndex int) *compositionDocument {
root := cloneCompositionNode(document.root)
if compositionIndex >= 0 {
root.fields = append(root.fields[:compositionIndex], root.fields[compositionIndex+1:]...)
}
return &compositionDocument{root: root, sources: append([]string(nil), document.sources...)}
}
func parsePipelineProfileDeclaration(name string, node *compositionNode) (pipelineProfileDeclaration, error) {
path := "composition.profiles." + name
if node.kind != yaml.MappingNode {
return pipelineProfileDeclaration{}, fmt.Errorf(
"configuration source %s at %s: expected a mapping, got %s",
formatCompositionSources(node.sources), path, yamlKindName(node.kind),
)
}
profile := pipelineProfileDeclaration{name: name}
for _, field := range node.fields {
if field.key != "overlay" {
return pipelineProfileDeclaration{}, fmt.Errorf(
"configuration source %s at %s.%s: unknown profile field %q; only overlay is supported",
formatCompositionSources(field.value.sources), path, field.key, field.key,
)
}
if field.value.kind != yaml.ScalarNode || field.value.tag != "!!str" {
return pipelineProfileDeclaration{}, fmt.Errorf(
"configuration source %s at %s.overlay: expected a string path",
formatCompositionSources(field.value.sources), path,
)
}
profile.overlay = field.value.value
}
if profile.overlay == "" {
return pipelineProfileDeclaration{}, fmt.Errorf("%s.overlay is required", path)
}
return profile, nil
}
func normalizePipelineProfileName(value, label string) (string, error) {
if value == "" || strings.TrimSpace(value) != value {
return "", fmt.Errorf("%s must be non-empty without surrounding whitespace", label)
}
for _, character := range value {
if unicode.IsControl(character) {
return "", fmt.Errorf("%s %q must not contain control characters", label, value)
}
}
return value, nil
}
func pipelineProfileDeclared(profiles []pipelineProfileDeclaration, name string) bool {
for _, profile := range profiles {
if profile.name == name {
return true
}
}
return false
}
func selectPipelineProfile(envelope pipelineCompositionEnvelope, opts PipelineLoadOptions) (*pipelineProfileSelection, error) {
if opts.Profile != nil {
name, err := normalizePipelineProfileName(*opts.Profile, "explicit profile selection")
if err != nil {
return nil, err
}
if len(envelope.profiles) == 0 {
return nil, fmt.Errorf("explicit profile %q was selected but the pipeline declares no profiles", name)
}
if !pipelineProfileDeclared(envelope.profiles, name) {
return nil, fmt.Errorf("explicit profile %q is not declared by the pipeline", name)
}
return &pipelineProfileSelection{name: name, source: "cli"}, nil
}
if len(envelope.profiles) == 0 {
return nil, nil
}
if envelope.defaultProfile == nil {
return nil, fmt.Errorf("pipeline declares profiles but composition.default_profile is omitted and no profile was explicitly selected")
}
return &pipelineProfileSelection{name: *envelope.defaultProfile, source: "default"}, nil
}
type loadedPipelineImport struct {
path string
document *compositionDocument
info os.FileInfo
}
type loadedPipelineProfileOverlay struct {
profile string
path string
document *compositionDocument
info os.FileInfo
}
func loadPipelineProfileOverlays(
rootPath string,
declared []pipelineProfileDeclaration,
imports []loadedPipelineImport,
) ([]loadedPipelineProfileOverlay, error) {
if len(declared) == 0 {
return nil, nil
}
rootDir := filepath.Dir(rootPath)
rootInfo, err := os.Stat(rootPath)
if err != nil {
return nil, fmt.Errorf("inspect root pipeline file %q: %w", rootPath, err)
}
seenPaths := make(map[string]string, len(declared))
loaded := make([]loadedPipelineProfileOverlay, 0, len(declared))
for _, profile := range declared {
label := "composition.profiles." + profile.name + ".overlay"
raw := profile.overlay
if strings.TrimSpace(raw) != raw || raw == "" {
return nil, fmt.Errorf("%s must be a non-empty path without surrounding whitespace", label)
}
normalized, err := pathsafe.NormalizeRelativeDestination(raw)
if err != nil {
return nil, fmt.Errorf("%s path %q is invalid: %w", label, raw, err)
}
extension := filepath.Ext(filepath.FromSlash(normalized))
if extension != ".yml" && extension != ".yaml" {
return nil, fmt.Errorf("%s path %q must use .yml or .yaml", label, raw)
}
if prior, duplicate := seenPaths[normalized]; duplicate {
return nil, fmt.Errorf("%s path %q duplicates profile %q overlay after normalization", label, raw, prior)
}
seenPaths[normalized] = profile.name
resolved := filepath.Join(rootDir, filepath.FromSlash(normalized))
file, err := os.Open(resolved)
if err != nil {
return nil, fmt.Errorf("open %s path %q beneath root pipeline directory: %w", label, raw, err)
}
info, statErr := file.Stat()
if statErr != nil {
_ = file.Close()
return nil, fmt.Errorf("inspect %s path %q: %w", label, raw, statErr)
}
if os.SameFile(rootInfo, info) {
_ = file.Close()
return nil, fmt.Errorf("%s path %q references the root pipeline itself", label, raw)
}
for _, imported := range imports {
if os.SameFile(imported.info, info) {
_ = file.Close()
return nil, fmt.Errorf("%s path %q references the same file as imported source %q", label, raw, imported.path)
}
}
for _, prior := range loaded {
if os.SameFile(prior.info, info) {
_ = file.Close()
return nil, fmt.Errorf("%s path %q references the same file as profile %q overlay", label, raw, prior.profile)
}
}
document, parseErr := parseCompositionDocument(resolved, file)
closeErr := file.Close()
if parseErr != nil {
return nil, parseErr
}
if closeErr != nil {
return nil, fmt.Errorf("close %s path %q: %w", label, raw, closeErr)
}
if compositionFieldIndex(document.root.fields, "composition") >= 0 {
return nil, fmt.Errorf("profile overlay source %q declares composition; only the root pipeline may declare composition", resolved)
}
loaded = append(loaded, loadedPipelineProfileOverlay{
profile: profile.name, path: resolved, document: document, info: info,
})
}
return loaded, nil
}
func loadedProfileOverlay(overlays []loadedPipelineProfileOverlay, name string) (loadedPipelineProfileOverlay, bool) {
for _, overlay := range overlays {
if overlay.profile == name {
return overlay, true
}
}
return loadedPipelineProfileOverlay{}, false
}
func loadPipelineImports(rootPath string, declared []string) ([]loadedPipelineImport, error) {
if len(declared) == 0 {
return nil, nil
}
rootDir := filepath.Dir(rootPath)
rootInfo, err := os.Stat(rootPath)
if err != nil {
return nil, fmt.Errorf("inspect root pipeline file %q: %w", rootPath, err)
}
seenPaths := make(map[string]int, len(declared))
loaded := make([]loadedPipelineImport, 0, len(declared))
for index, raw := range declared {
if strings.TrimSpace(raw) != raw || raw == "" {
return nil, fmt.Errorf("composition.imports[%d] must be a non-empty path without surrounding whitespace", index)
}
normalized, err := pathsafe.NormalizeRelativeDestination(raw)
if err != nil {
return nil, fmt.Errorf("composition.imports[%d] path %q is invalid: %w", index, raw, err)
}
extension := filepath.Ext(filepath.FromSlash(normalized))
if extension != ".yml" && extension != ".yaml" {
return nil, fmt.Errorf("composition.imports[%d] path %q must use .yml or .yaml", index, raw)
}
if prior, duplicate := seenPaths[normalized]; duplicate {
return nil, fmt.Errorf(
"composition.imports[%d] path %q duplicates composition.imports[%d] after normalization",
index, raw, prior,
)
}
seenPaths[normalized] = index
resolved := filepath.Join(rootDir, filepath.FromSlash(normalized))
if resolved == rootPath {
return nil, fmt.Errorf("composition.imports[%d] path %q imports the root pipeline itself", index, raw)
}
file, err := os.Open(resolved)
if err != nil {
return nil, fmt.Errorf("open composition.imports[%d] path %q beneath root pipeline directory: %w", index, raw, err)
}
info, statErr := file.Stat()
if statErr != nil {
_ = file.Close()
return nil, fmt.Errorf("inspect composition.imports[%d] path %q: %w", index, raw, statErr)
}
if os.SameFile(rootInfo, info) {
_ = file.Close()
return nil, fmt.Errorf("composition.imports[%d] path %q imports the root pipeline itself", index, raw)
}
for priorIndex, prior := range loaded {
if os.SameFile(prior.info, info) {
_ = file.Close()
return nil, fmt.Errorf(
"composition.imports[%d] path %q references the same file as composition.imports[%d] %q",
index, raw, priorIndex, declared[priorIndex],
)
}
}
document, parseErr := parseCompositionDocument(resolved, file)
closeErr := file.Close()
if parseErr != nil {
return nil, parseErr
}
if closeErr != nil {
return nil, fmt.Errorf("close composition.imports[%d] path %q: %w", index, raw, closeErr)
}
if compositionFieldIndex(document.root.fields, "composition") >= 0 {
return nil, fmt.Errorf(
"imported configuration source %q declares composition; only the root pipeline may declare composition",
resolved,
)
}
loaded = append(loaded, loadedPipelineImport{path: resolved, document: document, info: info})
}
return loaded, nil
}

View File

@@ -1,431 +0,0 @@
package config
import (
"os"
"path/filepath"
"reflect"
"slices"
"strings"
"testing"
)
func TestSplitPipelineCompositionEnvelopePreservesInputAndDeclarationOrder(t *testing.T) {
document, err := parseCompositionBytes("pipeline.yml", []byte(`composition:
imports: [first.yml, second.yml]
default_profile: production
profiles:
production:
overlay: production.yml
testing:
overlay: testing.yml
workspace:
root: /srv/narratio
`))
if err != nil {
t.Fatal(err)
}
originalRoot := cloneCompositionNode(document.root)
originalSources := append([]string(nil), document.sources...)
body, envelope, err := splitPipelineCompositionEnvelope(document)
if err != nil {
t.Fatalf("splitPipelineCompositionEnvelope() error = %v", err)
}
if !reflect.DeepEqual(document.root, originalRoot) || !slices.Equal(document.sources, originalSources) {
t.Fatalf("splitPipelineCompositionEnvelope() mutated its input")
}
if compositionFieldIndex(body.root.fields, "composition") >= 0 {
t.Fatalf("pipeline body still contains composition envelope")
}
if compositionFieldIndex(body.root.fields, "workspace") < 0 {
t.Fatalf("pipeline body lost ordinary root fields")
}
if want := []string{"first.yml", "second.yml"}; !slices.Equal(envelope.imports, want) {
t.Fatalf("imports = %#v, want %#v", envelope.imports, want)
}
if len(envelope.profiles) != 2 || envelope.profiles[0].name != "production" || envelope.profiles[1].name != "testing" {
t.Fatalf("profiles = %#v, want declaration order", envelope.profiles)
}
}
func TestSplitPipelineCompositionEnvelopeRejectsMalformedShapesWithoutMutation(t *testing.T) {
tests := []struct {
name string
yaml string
want string
}{
{name: "envelope", yaml: "composition: []\n", want: "expected a mapping"},
{name: "imports mapping", yaml: "composition:\n imports: {}\n", want: "expected a list"},
{name: "import value", yaml: "composition:\n imports: [1]\n", want: "expected a string path"},
{name: "default profile", yaml: "composition:\n default_profile: {}\n", want: "expected a string profile name"},
{name: "profiles sequence", yaml: "composition:\n profiles: []\n", want: "expected a mapping"},
{name: "empty profiles", yaml: "composition:\n profiles: {}\n", want: "at least one named profile"},
{name: "profile declaration", yaml: "composition:\n profiles:\n production: production.yml\n", want: "expected a mapping"},
{name: "invalid profile name", yaml: "composition:\n profiles:\n ' production ':\n overlay: production.yml\n", want: "surrounding whitespace"},
{name: "unknown field", yaml: "composition:\n unexpected: true\n", want: "unknown composition field"},
{name: "unknown default", yaml: "composition:\n default_profile: testing\n profiles:\n production:\n overlay: production.yml\n", want: "does not name a declared profile"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
document, err := parseCompositionBytes("pipeline.yml", []byte(test.yaml))
if err != nil {
t.Fatal(err)
}
originalRoot := cloneCompositionNode(document.root)
_, _, err = splitPipelineCompositionEnvelope(document)
if err == nil || !strings.Contains(err.Error(), test.want) {
t.Fatalf("splitPipelineCompositionEnvelope() error = %v, want containing %q", err, test.want)
}
if !reflect.DeepEqual(document.root, originalRoot) {
t.Fatalf("splitPipelineCompositionEnvelope() mutated invalid input")
}
})
}
}
func TestLoadPipelineCompositionImportsDisjointFieldsAndTracksOwnership(t *testing.T) {
dir := t.TempDir()
rootPath := writePipelineSource(t, dir, "pipeline.yml", `composition:
imports:
- conf.d/platform.yml
- conf.d/artifacts.yml
whisperx:
transcribe_url: https://transcription.example.com/transcribe
`)
platformPath := writePipelineSource(t, dir, "conf.d/platform.yml", `workspace:
root: /srv/narratio/work
storage:
backend: local
`)
artifactsPath := writePipelineSource(t, dir, "conf.d/artifacts.yml", `scriptorium:
artifacts:
player_handout:
enabled: false
session_recap:
enabled: true
prompt_id: dnd.session_recap
output_path: artifacts/session_recap.md
`)
cfg, err := LoadPipeline(rootPath)
if err != nil {
t.Fatalf("LoadPipeline() error = %v", err)
}
if cfg.Workspace.Root != "/srv/narratio/work" || cfg.Storage.Backend != StorageBackendLocal {
t.Fatalf("imported platform config = workspace=%q storage=%#v", cfg.Workspace.Root, cfg.Storage)
}
if cfg.Scriptorium == nil || len(cfg.Scriptorium.Artifacts) != 2 || cfg.Scriptorium.Artifacts["player_handout"].Enabled {
t.Fatalf("imported artifacts = %#v", cfg.Scriptorium)
}
if cfg.WhisperX.TranscribeURL != "https://transcription.example.com/transcribe" {
t.Fatalf("root field = %q", cfg.WhisperX.TranscribeURL)
}
if cfg.resolution == nil {
t.Fatal("pipeline resolution metadata = nil")
}
wantSources := []string{absolutePath(t, rootPath), absolutePath(t, platformPath), absolutePath(t, artifactsPath)}
if !slices.Equal(cfg.resolution.sources, wantSources) || !slices.Equal(cfg.resolution.imports, wantSources[1:]) {
t.Fatalf("resolution sources=%#v imports=%#v, want %#v / %#v", cfg.resolution.sources, cfg.resolution.imports, wantSources, wantSources[1:])
}
assertPipelineFieldOwner(t, cfg, "whisperx.transcribe_url", absolutePath(t, rootPath))
assertPipelineFieldOwner(t, cfg, "workspace.root", absolutePath(t, platformPath))
assertPipelineFieldOwner(t, cfg, "scriptorium.artifacts.session_recap.prompt_id", absolutePath(t, artifactsPath))
}
func TestLoadPipelineCompositionMergesDisjointKeyedEntries(t *testing.T) {
dir := t.TempDir()
rootPath := writePipelineSource(t, dir, "pipeline.yml", `composition:
imports: [first.yml, second.yml]
whisperx:
transcribe_url: https://transcription.example.com/transcribe
scriptorium:
artifacts:
root_artifact:
enabled: false
`)
writePipelineSource(t, dir, "first.yml", `scriptorium:
artifacts:
first_artifact:
enabled: false
`)
writePipelineSource(t, dir, "second.yml", `scriptorium:
artifacts:
second_artifact:
enabled: false
`)
cfg, err := LoadPipeline(rootPath)
if err != nil {
t.Fatal(err)
}
if got := len(cfg.Scriptorium.Artifacts); got != 3 {
t.Fatalf("artifact count = %d, want 3: %#v", got, cfg.Scriptorium.Artifacts)
}
}
func TestLoadPipelineCompositionRejectsBaseConflictsWithAllSources(t *testing.T) {
tests := []struct {
name string
root string
imports map[string]string
path string
sources []string
}{
{
name: "root and import identical scalar",
root: "whisperx:\n language: en\n",
imports: map[string]string{"one.yml": "whisperx:\n language: en\n"},
path: "whisperx.language", sources: []string{"pipeline.yml", "one.yml"},
},
{
name: "all import claimants",
imports: map[string]string{
"one.yml": "workspace:\n root: /one\n",
"two.yml": "workspace:\n root: /two\n",
"three.yml": "workspace:\n root: /three\n",
},
path: "workspace.root", sources: []string{"one.yml", "two.yml", "three.yml"},
},
{
name: "atomic list",
root: "audita:\n modules: [one]\n",
imports: map[string]string{"one.yml": "audita:\n modules: [two]\n"},
path: "audita.modules", sources: []string{"pipeline.yml", "one.yml"},
},
{
name: "kind conflict",
root: "workspace:\n root: /work\n",
imports: map[string]string{"one.yml": "workspace: invalid\n"},
path: "workspace", sources: []string{"pipeline.yml", "one.yml"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
dir := t.TempDir()
order := make([]string, 0, len(tt.imports))
for _, name := range []string{"one.yml", "two.yml", "three.yml"} {
if _, ok := tt.imports[name]; ok {
order = append(order, name)
}
}
root := "composition:\n imports:\n"
for _, name := range order {
root += " - " + name + "\n"
}
root += tt.root
rootPath := writePipelineSource(t, dir, "pipeline.yml", root)
for name, content := range tt.imports {
writePipelineSource(t, dir, name, content)
}
_, err := LoadPipeline(rootPath)
if err == nil {
t.Fatal("LoadPipeline() error = nil")
}
if !strings.Contains(err.Error(), tt.path) {
t.Fatalf("error = %q, want path %q", err, tt.path)
}
for _, source := range tt.sources {
if !strings.Contains(err.Error(), source) {
t.Fatalf("error = %q, want source %q", err, source)
}
}
})
}
}
func TestLoadPipelineCompositionRejectsUnsafeOrInvalidImports(t *testing.T) {
tests := []struct {
name string
imports []string
setup func(*testing.T, string)
want string
}{
{name: "empty", imports: []string{""}, want: "non-empty path"},
{name: "surrounding whitespace", imports: []string{" one.yml "}, want: "surrounding whitespace"},
{name: "absolute", imports: []string{"/tmp/one.yml"}, want: "invalid"},
{name: "traversal", imports: []string{"../one.yml"}, want: "invalid"},
{name: "unsupported extension", imports: []string{"one.json"}, want: ".yml or .yaml"},
{name: "missing", imports: []string{"missing.yml"}, want: "open composition.imports"},
{name: "duplicate normalized", imports: []string{"one.yml", "./one.yml"}, setup: func(t *testing.T, dir string) {
writePipelineSource(t, dir, "one.yml", "workspace:\n root: /work\n")
}, want: "duplicates composition.imports"},
{name: "root self import", imports: []string{"pipeline.yml"}, want: "root pipeline itself"},
{name: "directory", imports: []string{"directory.yml"}, setup: func(t *testing.T, dir string) {
if err := os.Mkdir(filepath.Join(dir, "directory.yml"), 0o755); err != nil {
t.Fatal(err)
}
}, want: "is a directory"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
dir := t.TempDir()
if tt.setup != nil {
tt.setup(t, dir)
}
rootPath := writeImportRoot(t, dir, tt.imports)
_, err := LoadPipeline(rootPath)
if err == nil || !strings.Contains(strings.ToLower(err.Error()), strings.ToLower(tt.want)) {
t.Fatalf("LoadPipeline() error = %v, want containing %q", err, tt.want)
}
})
}
}
func TestLoadPipelineCompositionRejectsSameFileAliasesAndImportedComposition(t *testing.T) {
t.Run("same file through hard link", func(t *testing.T) {
dir := t.TempDir()
writePipelineSource(t, dir, "one.yml", "workspace:\n root: /work\n")
if err := os.Link(filepath.Join(dir, "one.yml"), filepath.Join(dir, "two.yml")); err != nil {
t.Skipf("hard links unavailable: %v", err)
}
rootPath := writeImportRoot(t, dir, []string{"one.yml", "two.yml"})
_, err := LoadPipeline(rootPath)
if err == nil || !strings.Contains(err.Error(), "same file") {
t.Fatalf("LoadPipeline() error = %v, want same-file rejection", err)
}
})
t.Run("imported composition", func(t *testing.T) {
dir := t.TempDir()
rootPath := writeImportRoot(t, dir, []string{"nested.yml"})
writePipelineSource(t, dir, "nested.yml", "composition:\n imports: []\n")
_, err := LoadPipeline(rootPath)
if err == nil || !strings.Contains(err.Error(), "only the root pipeline") || !strings.Contains(err.Error(), "nested.yml") {
t.Fatalf("LoadPipeline() error = %v, want imported composition rejection", err)
}
})
t.Run("empty profiles", func(t *testing.T) {
dir := t.TempDir()
rootPath := writePipelineSource(t, dir, "pipeline.yml", "composition:\n profiles: {}\n")
_, err := LoadPipeline(rootPath)
if err == nil || !strings.Contains(err.Error(), "must declare at least one named profile") {
t.Fatalf("LoadPipeline() error = %v, want empty profiles rejection", err)
}
})
}
func TestLoadPipelineCompositionReportsImportedParseAndSchemaSources(t *testing.T) {
t.Run("malformed imported YAML", func(t *testing.T) {
dir := t.TempDir()
rootPath := writeImportRoot(t, dir, []string{"broken.yml"})
brokenPath := writePipelineSource(t, dir, "broken.yml", "workspace: [\n")
_, err := LoadPipeline(rootPath)
if err == nil || !strings.Contains(err.Error(), absolutePath(t, brokenPath)) || !strings.Contains(err.Error(), "decode YAML") {
t.Fatalf("LoadPipeline() error = %v, want imported parse source", err)
}
})
t.Run("unknown imported field", func(t *testing.T) {
dir := t.TempDir()
rootPath := writeImportRoot(t, dir, []string{"unknown.yml"})
unknownPath := writePipelineSource(t, dir, "unknown.yml", "unknown_field: true\n")
_, err := LoadPipeline(rootPath)
if err == nil || !strings.Contains(err.Error(), absolutePath(t, unknownPath)) || !strings.Contains(err.Error(), "strict decode failed") {
t.Fatalf("LoadPipeline() error = %v, want assembled source-aware strict error", err)
}
})
}
func TestLoadPipelineCompositionKeepsRelativePathsRootBased(t *testing.T) {
rootDir := t.TempDir()
monolithicPath := writePipelineSource(t, rootDir, "monolithic.yml", testPipelineBaseYAML+`
notarius:
enabled: true
config_path: tool/notarius.yml
pipeline_id: dnd-session
outputs:
npc_registry:
lane_id: npc-registry
media_type: application/json
schema_id: notarius.dnd.npc_registry
schema_version: v1
`)
composedPath := writePipelineSource(t, rootDir, "pipeline.yml", `composition:
imports: [conf.d/extraction.yml]
`+testPipelineBaseYAML)
writePipelineSource(t, rootDir, "conf.d/extraction.yml", `notarius:
enabled: true
config_path: tool/notarius.yml
pipeline_id: dnd-session
outputs:
npc_registry:
lane_id: npc-registry
media_type: application/json
schema_id: notarius.dnd.npc_registry
schema_version: v1
`)
monolithic, err := LoadPipeline(monolithicPath)
if err != nil {
t.Fatal(err)
}
composed, err := LoadPipeline(composedPath)
if err != nil {
t.Fatal(err)
}
want := filepath.Join(rootDir, "tool", "notarius.yml")
if monolithic.Notarius.ConfigPath != want || composed.Notarius.ConfigPath != want {
t.Fatalf("config paths = monolithic %q composed %q, want %q", monolithic.Notarius.ConfigPath, composed.Notarius.ConfigPath, want)
}
if monolithic.Notarius.WorkingDirectory != filepath.Dir(want) || composed.Notarius.WorkingDirectory != filepath.Dir(want) {
t.Fatalf("working directories = %q / %q", monolithic.Notarius.WorkingDirectory, composed.Notarius.WorkingDirectory)
}
}
func writeImportRoot(t *testing.T, dir string, imports []string) string {
t.Helper()
var builder strings.Builder
builder.WriteString("composition:\n imports:\n")
for _, imported := range imports {
builder.WriteString(" - ")
if imported == "" {
builder.WriteString(`""`)
} else {
builder.WriteString(`"` + imported + `"`)
}
builder.WriteByte('\n')
}
builder.WriteString("whisperx:\n transcribe_url: https://transcription.example.com/transcribe\n")
return writePipelineSource(t, dir, "pipeline.yml", builder.String())
}
func writePipelineSource(t *testing.T, root, relative, content string) string {
t.Helper()
path := filepath.Join(root, filepath.FromSlash(relative))
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
t.Fatal(err)
}
return path
}
func absolutePath(t *testing.T, path string) string {
t.Helper()
absolute, err := filepath.Abs(path)
if err != nil {
t.Fatal(err)
}
return absolute
}
func assertPipelineFieldOwner(t *testing.T, cfg *PipelineConfig, path, source string) {
t.Helper()
if cfg == nil || cfg.resolution == nil {
t.Fatal("pipeline resolution metadata is absent")
}
for _, ownership := range cfg.resolution.ownership {
if ownership.path == path {
if !slices.Equal(ownership.sources, []string{source}) {
t.Fatalf("owner of %s = %#v, want %q", path, ownership.sources, source)
}
return
}
}
t.Fatalf("ownership path %q not found: %#v", path, cfg.resolution.ownership)
}

View File

@@ -0,0 +1,391 @@
package config
import (
"bytes"
"errors"
"fmt"
"path/filepath"
"reflect"
"sort"
"strings"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
)
type pipelineResolutionMetadata struct {
rootPath string
imports []string
sources []string
selectedProfile *pipelineProfileSelection
effectiveDigest string
logicalNotariusConfig string
logicalNotariusWorking string
logicalNotariusCaptured bool
ownership []pipelineFieldOwnership
artifactFamilies map[string]ScriptoriumArtifactFamilyConfig
artifactFamiliesExpanded bool
publishDeclared bool
configkitMetadata configkit.Metadata
configkitSources []configkit.SourceInfo
}
type pipelineProfileSelection struct {
name string
source string
overlayPath string
}
type pipelineFieldOwnership struct {
path string
sources []string
}
func loadPipelineWithConfigKit(path string, opts PipelineLoadOptions) (*PipelineConfig, error) {
profile := selectedPipelineProfileName(opts)
result, err := configkit.Load(configkit.File(path), configkit.Options[PipelineConfig]{
Imports: true,
Profile: profile,
Definition: configkit.Definition[PipelineConfig]{
Defaults: pipelineConfigDefaults,
Resolve: resolvePipelineConfig,
Validate: validatePipelineConfigKit,
},
})
if err != nil {
return nil, describePipelineLoadError(err)
}
cfg := result.Config
metadata, err := pipelineMetadataFromConfigKit(&cfg, result, opts)
if err != nil {
return nil, fmt.Errorf("load pipeline config: %w", err)
}
cfg.resolution = metadata
return &cfg, nil
}
func describePipelineLoadError(err error) error {
var loadError *configkit.LoadError
if !errors.As(err, &loadError) {
return fmt.Errorf("load pipeline config: %w", err)
}
var diagnostics bytes.Buffer
resolved := loadError.Diagnostics()
sourcePaths := make(map[string]string, len(loadError.Sources()))
for _, source := range loadError.Sources() {
if source.Path != "" {
sourcePaths[source.ID] = source.Path
} else if source.Label != "" {
sourcePaths[source.ID] = source.Label
}
}
for index := range resolved {
if resolved[index].Primary != nil {
if name := sourcePaths[resolved[index].Primary.SourceID]; name != "" {
resolved[index].Primary.SourceID = name
}
}
for relatedIndex := range resolved[index].Related {
if name := sourcePaths[resolved[index].Related[relatedIndex].SourceID]; name != "" {
resolved[index].Related[relatedIndex].SourceID = name
}
}
}
if writeErr := configkit.WriteDiagnostics(&diagnostics, resolved); writeErr != nil {
return fmt.Errorf("load pipeline config: %w", err)
}
return fmt.Errorf("load pipeline config: strict decode failed or validation failed: %w: %s", err, strings.TrimSpace(diagnostics.String()))
}
func selectedPipelineProfileName(opts PipelineLoadOptions) string {
if opts.Profile == nil || *opts.Profile == "" || *opts.Profile == "default" {
return ""
}
return *opts.Profile
}
func pipelineConfigDefaults() PipelineConfig {
return PipelineConfig{
Workspace: WorkspaceConfig{Root: DefaultWorkspaceRoot},
Campaigns: CampaignsConfig{Root: DefaultCampaignsRoot},
Storage: StorageConfig{Backend: StorageBackendLocal},
Spool: SpoolConfig{Root: DefaultSpoolRoot},
Cache: CacheConfig{Root: DefaultCacheRoot, S3Audio: boolPtr(DefaultCacheS3Audio)},
Publish: &PublishConfig{
Enabled: boolPtr(DefaultArchiveEnabled), UploadRun: boolPtr(DefaultArchiveUploadRun),
Outputs: defaultPublishOutputs(),
},
WhisperX: WhisperXConfig{
Language: DefaultWhisperXLanguage, Timeout: DefaultWhisperXTimeout,
RetryDelay: DefaultWhisperXRetryDelay, Concurrency: intPtr(DefaultWhisperXConcurrency),
Retries: intPtr(DefaultWhisperXRetries),
},
Seriatim: SeriatimConfig{
Binary: DefaultSeriatimBinary, Timeout: DefaultSeriatimTimeout,
OutputSchema: DefaultSeriatimOutputSchema, CoalesceGap: float64Ptr(DefaultSeriatimCoalesceGap),
Report: boolPtr(DefaultSeriatimReport),
},
Audita: AuditaConfig{Binary: DefaultAuditaBinary, Timeout: DefaultAuditaTimeout, Report: boolPtr(DefaultAuditaReport)},
Normalize: &NormalizeConfig{OutputPath: DefaultNormalizeOutputPath, OutputSchema: DefaultNormalizeOutputSchema, Report: boolPtr(DefaultNormalizeReport)},
Trim: &TrimConfig{
Enabled: boolPtr(DefaultTrimEnabled), OutputPath: DefaultTrimOutputPath,
Bounds: TrimBoundsConfig{
PromptID: DefaultTrimBoundsPromptID, TranscriptInputName: DefaultTrimBoundsTranscriptInputName,
OutputPath: DefaultTrimBoundsOutputPath, Timeout: DefaultTrimBoundsTimeout,
},
Seriatim: TrimSeriatimConfig{Report: boolPtr(DefaultTrimSeriatimReport)},
},
Render: &RenderConfig{
Enabled: boolPtr(DefaultRenderEnabled), Format: DefaultRenderFormat, Title: DefaultRenderTitle,
IncludeTimestamps: boolPtr(DefaultRenderTimestamps), IncludeSegmentIDs: boolPtr(DefaultRenderSegmentIDs),
},
Notification: NotificationConfig{Mode: DefaultNotificationMode},
}
}
func defaultPublishOutputs() []PublishOutputRule {
outputs := append([]PublishOutputRule(nil), DefaultPublishOutputs...)
for index := range outputs {
outputs[index].Required = boolPtr(true)
}
return outputs
}
func resolvePipelineConfig(cfg *PipelineConfig, resolution *configkit.Resolution) error {
backend := strings.ToLower(strings.TrimSpace(cfg.Storage.Backend))
if backend != cfg.Storage.Backend {
configkit.Normalize(resolution, configkit.Keys("storage", "backend"), &cfg.Storage.Backend, backend, "canonical storage backend")
}
if cfg.Storage.S3 != nil {
path := configkit.Keys("storage", "s3")
configkit.Default(resolution, path.Key("root_prefix"), &cfg.Storage.S3.RootPrefix, DefaultStorageS3RootPrefix, "default S3 root prefix")
configkit.Default(resolution, path.Key("access_key_id_env"), &cfg.Storage.S3.AccessKeyIDEnv, DefaultS3AccessKeyIDEnv, "default S3 access-key reference")
configkit.Default(resolution, path.Key("secret_access_key_env"), &cfg.Storage.S3.SecretKeyEnv, DefaultS3SecretAccessKeyEnv, "default S3 secret-key reference")
}
if cfg.Trim != nil && cfg.Trim.Enabled != nil && *cfg.Trim.Enabled && cfg.Scriptorium == nil {
value := &ScriptoriumConfig{Binary: DefaultScriptoriumBinary, Timeout: DefaultScriptoriumTimeout}
configkit.Default(resolution, configkit.Keys("scriptorium"), &cfg.Scriptorium, value, "trim requires Scriptorium")
}
if cfg.Scriptorium != nil {
path := configkit.Keys("scriptorium")
configkit.Default(resolution, path.Key("binary"), &cfg.Scriptorium.Binary, DefaultScriptoriumBinary, "default Scriptorium binary")
configkit.Default(resolution, path.Key("timeout"), &cfg.Scriptorium.Timeout, DefaultScriptoriumTimeout, "default Scriptorium timeout")
}
if cfg.Notarius != nil {
path := configkit.Keys("notarius")
configkit.Default(resolution, path.Key("binary"), &cfg.Notarius.Binary, DefaultNotariusBinary, "default Notarius binary")
configkit.Default(resolution, path.Key("timeout"), &cfg.Notarius.Timeout, DefaultNotariusTimeout, "default Notarius timeout")
if cfg.Notarius.Enabled && cfg.Notarius.ConfigPath != "" {
configkit.Default(resolution, path.Key("working_directory"), &cfg.Notarius.WorkingDirectory,
filepath.Dir(cfg.Notarius.ConfigPath), "derived from Notarius config path", path.Key("config_path"))
}
}
if cfg.Publish != nil {
defaultedOutput := false
for index := range cfg.Publish.Outputs {
output := cfg.Publish.Outputs[index]
path := configkit.Keys("publish", "outputs").Index(index).Key("required")
if !resolution.IsExplicit(path) {
defaultedOutput = true
}
configkit.Default(resolution, path, &output.Required, boolPtr(true), "publish outputs are required by default")
cfg.Publish.Outputs[index] = output
}
if defaultedOutput {
resolution.Record(configkit.Keys("publish", "outputs"), configkit.ChangeDerivedDefault, "publish output entry defaults")
}
}
before := cloneNotariusForResolution(cfg.Notarius)
if err := NormalizePipelineConfig(cfg); err != nil {
resolution.Error(configkit.Keys("notarius"), err.Error())
return nil
}
if !reflect.DeepEqual(before, cfg.Notarius) {
resolution.Record(configkit.Keys("notarius"), configkit.ChangeNormalization, "canonical Notarius configuration")
}
if cfg.Publish != nil {
configuredOutputs := scriptoriumOutputPathMap(cfg.Scriptorium)
extractionOutputs := notariusOutputKeySet(cfg.Notarius)
for index := range cfg.Publish.Outputs {
output := cfg.Publish.Outputs[index]
if strings.TrimSpace(output.Dest) != "" {
continue
}
destination, err := artifactpolicy.ResolvePublishedDestinationWithExtractions(strings.TrimSpace(output.Source), "", configuredOutputs, extractionOutputs)
if err != nil {
continue
}
configkit.Normalize(resolution, configkit.Keys("publish", "outputs").Index(index).Key("dest"), &output.Dest, destination, "derived from publish source")
cfg.Publish.Outputs[index] = output
}
}
return nil
}
func cloneNotariusForResolution(value *NotariusConfig) *NotariusConfig {
if value == nil {
return nil
}
clone := *value
clone.References = cloneStringMap(value.References)
if value.Outputs != nil {
clone.Outputs = make(map[string]NotariusOutputConfig, len(value.Outputs))
for key, output := range value.Outputs {
clone.Outputs[key] = output
}
}
return &clone
}
func cloneStringMap(value map[string]string) map[string]string {
if value == nil {
return nil
}
clone := make(map[string]string, len(value))
for key, item := range value {
clone[key] = item
}
return clone
}
func validatePipelineConfigKit(cfg *PipelineConfig, validation *configkit.Validation) {
// Checks that depend on family expansion or campaign/session context remain
// at Narratio's explicit resolution boundary. This early definition check is
// deliberately limited to an invariant that is wholly file-local.
if strings.TrimSpace(cfg.Workspace.Root) == "" {
validation.Error(configkit.Keys("workspace", "root"), "workspace root is required")
}
}
func pipelineMetadataFromConfigKit(cfg *PipelineConfig, result *configkit.Result[PipelineConfig], opts PipelineLoadOptions) (*pipelineResolutionMetadata, error) {
sources := result.Sources()
metadata := result.Metadata()
if err := rejectDuplicateBaseDefinitions(cfg, metadata, sources); err != nil {
return nil, err
}
resolved := &pipelineResolutionMetadata{
configkitMetadata: metadata, configkitSources: sources,
publishDeclared: metadata.IsExplicit(configkit.Keys("publish")),
}
pathsByID := make(map[string]string, len(sources))
for index, source := range sources {
path := source.Path
if path == "" {
path = source.Label
}
pathsByID[source.ID] = path
resolved.sources = append(resolved.sources, path)
if index == 0 {
resolved.rootPath = path
} else if source.Layer == configkit.LayerBase {
resolved.imports = append(resolved.imports, path)
}
if source.Layer == configkit.LayerProfile {
resolved.selectedProfile = &pipelineProfileSelection{name: selectedPipelineProfileName(opts), source: "explicit", overlayPath: path}
}
}
detailed, err := pipelineMetadataPaths(cfg)
if err != nil {
return nil, err
}
for _, value := range detailed {
info, ok := metadata.Lookup(configkit.Keys(value.segments...))
sourceNames := []string{pipelineDefaultOwnershipSource}
if ok {
sourceNames = ownershipSources(info.Origin, pathsByID)
}
resolved.ownership = append(resolved.ownership, pipelineFieldOwnership{path: value.record.Path, sources: sourceNames})
}
return resolved, nil
}
func pipelineMetadataPaths(cfg *PipelineConfig) ([]projectedPipelineValue, error) {
effective, err := projectionDetailedRecords(projectEffectivePipeline(cfg, false))
if err != nil {
return nil, err
}
if cfg.Scriptorium == nil || len(cfg.Scriptorium.ArtifactFamilies) == 0 {
return effective, nil
}
families, err := projectionDetailedRecords(map[string]any{"scriptorium": map[string]any{"artifact_families": cfg.Scriptorium.ArtifactFamilies}})
if err != nil {
return nil, err
}
return append(effective, families...), nil
}
func ownershipSources(origin configkit.Origin, pathsByID map[string]string) []string {
if origin.Layer == configkit.LayerDefault || (origin.Layer == configkit.LayerResolution && origin.Operation == configkit.ChangeDerivedDefault) {
return []string{pipelineDefaultOwnershipSource}
}
if origin.Layer == configkit.LayerResolution {
return []string{"resolution"}
}
seen := map[string]struct{}{}
var sources []string
for _, location := range origin.Locations {
value := pathsByID[location.SourceID]
if value == "" {
value = location.SourceID
}
if _, ok := seen[value]; !ok {
seen[value] = struct{}{}
sources = append(sources, value)
}
}
if len(sources) == 0 {
return []string{pipelineDefaultOwnershipSource}
}
return sources
}
func rejectDuplicateBaseDefinitions(cfg *PipelineConfig, metadata configkit.Metadata, sources []configkit.SourceInfo) error {
pathsByID := make(map[string]string, len(sources))
for _, source := range sources {
pathsByID[source.ID] = source.Path
}
var declarations []struct {
kind string
key string
path configkit.Path
}
if cfg.Scriptorium != nil {
for key := range cfg.Scriptorium.Artifacts {
declarations = append(declarations, struct {
kind, key string
path configkit.Path
}{"artifact", key, configkit.Keys("scriptorium", "artifacts", key)})
}
for key := range cfg.Scriptorium.ArtifactFamilies {
declarations = append(declarations, struct {
kind, key string
path configkit.Path
}{"artifact family", key, configkit.Keys("scriptorium", "artifact_families", key)})
}
}
sort.Slice(declarations, func(i, j int) bool { return declarations[i].path.String() < declarations[j].path.String() })
for _, declaration := range declarations {
seen := map[string]struct{}{}
for _, change := range metadata.History(declaration.path) {
if change.Origin.Layer != configkit.LayerBase {
continue
}
for _, location := range change.Origin.Locations {
seen[location.SourceID] = struct{}{}
}
}
if len(seen) > 1 {
claims := make([]string, 0, len(seen))
for sourceID := range seen {
claims = append(claims, pathsByID[sourceID])
}
sort.Strings(claims)
return fmt.Errorf("pipeline Scriptorium %s %q is declared by multiple base files: %s", declaration.kind, declaration.key, strings.Join(claims, ", "))
}
}
return nil
}
func intPtr(value int) *int { return &value }
func float64Ptr(value float64) *float64 { return &value }
func boolPtr(value bool) *bool { return &value }

View File

@@ -0,0 +1,102 @@
package config
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestPipelineConfigKitImportsProfileAndBaseSelection(t *testing.T) {
dir := t.TempDir()
writePipelineFixture(t, dir, "pipeline.yml", `configkit:
imports:
- path: config/workspace.yml
- path: config/artifact.yml
whisperx:
transcribe_url: https://transcription.example.test/transcribe
`)
writePipelineFixture(t, dir, "config/workspace.yml", "workspace:\n root: /srv/base\n")
writePipelineFixture(t, dir, "config/artifact.yml", `scriptorium:
artifacts:
recap:
enabled: false
output_path: artifacts/recap.md
`)
writePipelineFixture(t, dir, "profiles/testing.yml", "workspace:\n root: /srv/testing\n")
writePipelineFixture(t, dir, "profiles/unselected.yml", "unknown: [\n")
root := filepath.Join(dir, "pipeline.yml")
for _, profile := range []*string{nil, stringPointer(""), stringPointer("default")} {
cfg, err := LoadPipelineWithOptions(root, PipelineLoadOptions{Profile: profile})
if err != nil {
t.Fatalf("load base selection %#v: %v", profile, err)
}
if cfg.Workspace.Root != "/srv/base" {
t.Fatalf("base workspace = %q", cfg.Workspace.Root)
}
}
selected := "testing"
cfg, err := LoadPipelineWithOptions(root, PipelineLoadOptions{Profile: &selected})
if err != nil {
t.Fatal(err)
}
if cfg.Workspace.Root != "/srv/testing" || len(EffectivePipelineImports(cfg)) != 2 {
t.Fatalf("profiled pipeline = %#v, imports=%#v", cfg.Workspace, EffectivePipelineImports(cfg))
}
profile, ok := SelectedPipelineProfile(cfg)
if !ok || profile.Name != "testing" || profile.OverlayPath != filepath.Join(dir, "profiles", "testing.yml") {
t.Fatalf("selected profile = %#v, %v", profile, ok)
}
}
func TestPipelineConfigKitRejectsDuplicateBaseDeclarations(t *testing.T) {
for _, test := range []struct {
name, first, second string
}{
{name: "artifact", first: "scriptorium:\n artifacts:\n recap:\n prompt_id: recap\n", second: "scriptorium:\n artifacts:\n recap:\n output_path: artifacts/recap.md\n"},
{name: "artifact family", first: "scriptorium:\n artifact_families:\n recap:\n for_each: party.characters\n", second: "scriptorium:\n artifact_families:\n recap:\n output_path_pattern: artifacts/{character_id}.md\n"},
} {
t.Run(test.name, func(t *testing.T) {
dir := t.TempDir()
writePipelineFixture(t, dir, "pipeline.yml", `configkit:
imports:
- path: first.yml
- path: second.yml
whisperx:
transcribe_url: https://transcription.example.test/transcribe
`)
writePipelineFixture(t, dir, "first.yml", test.first)
writePipelineFixture(t, dir, "second.yml", test.second)
_, err := LoadPipeline(filepath.Join(dir, "pipeline.yml"))
if err == nil || !strings.Contains(err.Error(), "declared by multiple base files") ||
!strings.Contains(err.Error(), "first.yml") || !strings.Contains(err.Error(), "second.yml") {
t.Fatalf("duplicate declaration error = %v", err)
}
})
}
}
func TestPipelineConfigKitRejectsUnknownDirectiveAlias(t *testing.T) {
dir := t.TempDir()
path := writePipelineFixture(t, dir, "pipeline.yml", "composition:\n imports: []\n")
_, err := LoadPipeline(path)
if err == nil || !strings.Contains(err.Error(), "decode.unknown_field") {
t.Fatalf("composition alias error = %v", err)
}
}
func writePipelineFixture(t *testing.T, root, relative, contents string) string {
t.Helper()
path := filepath.Join(root, filepath.FromSlash(relative))
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte(contents), 0o644); err != nil {
t.Fatal(err)
}
return path
}
func stringPointer(value string) *string { return &value }

View File

@@ -1,479 +0,0 @@
package config
import (
"os"
"path/filepath"
"slices"
"strings"
"testing"
"github.com/google/go-cmp/cmp"
)
func TestLoadPipelineProfilesSelectDefaultAndExplicitOverlay(t *testing.T) {
dir := t.TempDir()
rootPath := writeProfilePipeline(t, dir, `composition:
imports: [conf.d/base.yml]
default_profile: production
profiles:
production:
overlay: profiles/production.yml
testing:
overlay: profiles/testing.yml
whisperx:
transcribe_url: https://transcription.example.com/transcribe
`)
importPath := writePipelineSource(t, dir, "conf.d/base.yml", `workspace:
root: /srv/base
audita:
modules: [base, shared]
`)
productionPath := writePipelineSource(t, dir, "profiles/production.yml", `workspace:
root: /srv/production
whisperx:
language: en
`)
testingPath := writePipelineSource(t, dir, "profiles/testing.yml", `workspace:
root: /srv/testing
whisperx:
language: fr
`)
production, err := LoadPipeline(rootPath)
if err != nil {
t.Fatal(err)
}
if production.Workspace.Root != "/srv/production" || production.WhisperX.Language != "en" {
t.Fatalf("default profile result = workspace=%q language=%q", production.Workspace.Root, production.WhisperX.Language)
}
assertSelectedPipelineProfile(t, production, "production", "default", productionPath)
if want := []string{absolutePath(t, rootPath), absolutePath(t, importPath), absolutePath(t, productionPath)}; !slices.Equal(production.resolution.sources, want) {
t.Fatalf("default sources = %#v, want %#v", production.resolution.sources, want)
}
assertPipelineFieldOwner(t, production, "workspace.root", absolutePath(t, productionPath))
assertPipelineFieldOwner(t, production, "audita.modules", absolutePath(t, importPath))
assertPipelineFieldOwner(t, production, "storage.backend", pipelineDefaultOwnershipSource)
profile := "testing"
testingCfg, err := LoadPipelineWithOptions(rootPath, PipelineLoadOptions{Profile: &profile})
if err != nil {
t.Fatal(err)
}
if testingCfg.Workspace.Root != "/srv/testing" || testingCfg.WhisperX.Language != "fr" {
t.Fatalf("explicit profile result = workspace=%q language=%q", testingCfg.Workspace.Root, testingCfg.WhisperX.Language)
}
assertSelectedPipelineProfile(t, testingCfg, "testing", "cli", testingPath)
}
func TestLoadPipelineProfileSelectionErrors(t *testing.T) {
for _, test := range []struct {
name string
root string
profile *string
want string
}{
{name: "profiles require selection", root: profileComposition("", "production"), want: "default_profile is omitted"},
{name: "unknown explicit", root: profileComposition("production", "production"), profile: profilePointer("unknown"), want: "not declared"},
{name: "stacked explicit", root: profileComposition("production", "production", "testing"), profile: profilePointer("production,testing"), want: "not declared"},
{name: "explicit empty", root: profileComposition("production", "production"), profile: profilePointer(""), want: "must be non-empty"},
{name: "explicit whitespace", root: profileComposition("production", "production"), profile: profilePointer(" production "), want: "surrounding whitespace"},
{name: "invalid default", root: profileComposition("unknown", "production"), want: "does not name a declared profile"},
{name: "empty default", root: profileComposition("EMPTY", "production"), want: "must be non-empty"},
{name: "empty profile name", root: `composition:
default_profile: production
profiles:
"":
overlay: production.yml
whisperx:
transcribe_url: https://transcription.example.com/transcribe
`, want: "profile name must be non-empty"},
} {
t.Run(test.name, func(t *testing.T) {
dir := t.TempDir()
root := test.root
if test.name == "empty default" {
root = strings.ReplaceAll(root, "default_profile: EMPTY", `default_profile: ""`)
}
rootPath := writeProfilePipeline(t, dir, root)
writePipelineSource(t, dir, "production.yml", "whisperx:\n language: en\n")
_, err := LoadPipelineWithOptions(rootPath, PipelineLoadOptions{Profile: test.profile})
if err == nil || !strings.Contains(strings.ToLower(err.Error()), strings.ToLower(test.want)) {
t.Fatalf("LoadPipelineWithOptions() error = %v, want containing %q", err, test.want)
}
})
}
t.Run("explicit against profile-free pipeline", func(t *testing.T) {
dir := t.TempDir()
rootPath := writeProfilePipeline(t, dir, testPipelineBaseYAML)
selected := "testing"
_, err := LoadPipelineWithOptions(rootPath, PipelineLoadOptions{Profile: &selected})
if err == nil || !strings.Contains(err.Error(), "declares no profiles") {
t.Fatalf("error = %v, want profile-free rejection", err)
}
})
}
func TestPipelineProfileNamesRejectControlCharacters(t *testing.T) {
if _, err := normalizePipelineProfileName("production\x00testing", "profile"); err == nil || !strings.Contains(err.Error(), "control characters") {
t.Fatalf("normalizePipelineProfileName() error = %v, want control-character rejection", err)
}
}
func TestLoadPipelineProfilesValidateEveryDeclaredOverlay(t *testing.T) {
for _, test := range []struct {
name string
path string
content string
setup func(*testing.T, string)
want string
}{
{name: "missing", path: "profiles/missing.yml", want: "open composition.profiles.testing.overlay"},
{name: "malformed", path: "profiles/testing.yml", content: "workspace: [\n", want: "decode yaml"},
{name: "duplicate keys", path: "profiles/testing.yml", content: "workspace:\n root: /one\n root: /two\n", want: "duplicate yaml key"},
{name: "trailing document", path: "profiles/testing.yml", content: "workspace:\n root: /one\n---\nworkspace:\n root: /two\n", want: "exactly one yaml document"},
{name: "nested composition", path: "profiles/testing.yml", content: "composition:\n imports: [nested.yml]\n", want: "only the root pipeline"},
{name: "inheritance", path: "profiles/testing.yml", content: "workspace:\n root: /testing\n", setup: func(t *testing.T, dir string) {
rootPath := filepath.Join(dir, "pipeline.yml")
data, err := os.ReadFile(rootPath)
if err != nil {
t.Fatal(err)
}
updated := strings.Replace(string(data), "testing:\n overlay:", "testing:\n extends: production\n overlay:", 1)
if err := os.WriteFile(rootPath, []byte(updated), 0o644); err != nil {
t.Fatal(err)
}
}, want: "only overlay is supported"},
{name: "traversal", path: "../testing.yml", want: "invalid"},
{name: "extension", path: "profiles/testing.json", content: "{}\n", want: ".yml or .yaml"},
{name: "directory", path: "profiles/testing.yml", setup: func(t *testing.T, dir string) {
if err := os.MkdirAll(filepath.Join(dir, "profiles/testing.yml"), 0o755); err != nil {
t.Fatal(err)
}
}, want: "is a directory"},
} {
t.Run(test.name, func(t *testing.T) {
dir := t.TempDir()
rootPath := writeProfilePipeline(t, dir, `composition:
default_profile: production
profiles:
production:
overlay: profiles/production.yml
testing:
overlay: `+test.path+`
whisperx:
transcribe_url: https://transcription.example.com/transcribe
`)
writePipelineSource(t, dir, "profiles/production.yml", "whisperx:\n language: en\n")
if test.content != "" {
writePipelineSource(t, dir, test.path, test.content)
}
if test.setup != nil {
test.setup(t, dir)
}
_, err := LoadPipeline(rootPath)
if err == nil || !strings.Contains(strings.ToLower(err.Error()), strings.ToLower(test.want)) {
t.Fatalf("LoadPipeline() error = %v, want containing %q", err, test.want)
}
})
}
}
func TestLoadPipelineProfileOverlaySemantics(t *testing.T) {
dir := t.TempDir()
rootPath := writeProfilePipeline(t, dir, `composition:
default_profile: testing
profiles:
testing:
overlay: testing.yml
workspace:
root: /base
cleanup_after_publish: true
whisperx:
transcribe_url: https://transcription.example.com/transcribe
retries: 3
audita:
modules: [base, shared]
scriptorium:
artifacts:
session_recap:
enabled: true
prompt_id: dnd.session_recap
output_path: artifacts/session_recap.md
`)
writePipelineSource(t, dir, "testing.yml", `workspace:
cleanup_after_publish: false
whisperx:
retries: 0
audita:
modules: []
scriptorium:
artifacts:
session_recap:
enabled: false
experimental:
enabled: false
`)
cfg, err := LoadPipeline(rootPath)
if err != nil {
t.Fatal(err)
}
if cfg.Workspace.Root != "/base" || cfg.Workspace.CleanupAfterPublish || cfg.WhisperX.Retries == nil || *cfg.WhisperX.Retries != 0 {
t.Fatalf("recursive/zero overlay = workspace=%#v retries=%#v", cfg.Workspace, cfg.WhisperX.Retries)
}
if cfg.Audita.Modules == nil || len(cfg.Audita.Modules) != 0 {
t.Fatalf("list replacement = %#v, want explicit empty list", cfg.Audita.Modules)
}
if cfg.Scriptorium == nil || cfg.Scriptorium.Artifacts["session_recap"].Enabled || len(cfg.Scriptorium.Artifacts) != 2 {
t.Fatalf("keyed artifact overlay = %#v", cfg.Scriptorium)
}
}
func TestLoadPipelineProfileRejectsNullAndKindChanges(t *testing.T) {
for _, test := range []struct {
name string
overlay string
want string
}{
{name: "null deletion", overlay: "workspace:\n root: null\n", want: "null cannot delete"},
{name: "mapping scalar", overlay: "workspace: /other\n", want: "kind change"},
{name: "list mapping", overlay: "audita:\n modules:\n testing: true\n", want: "kind change"},
} {
t.Run(test.name, func(t *testing.T) {
dir := t.TempDir()
rootPath := writeProfilePipeline(t, dir, `composition:
default_profile: testing
profiles:
testing:
overlay: testing.yml
workspace:
root: /base
audita:
modules: [base]
whisperx:
transcribe_url: https://transcription.example.com/transcribe
`)
writePipelineSource(t, dir, "testing.yml", test.overlay)
_, err := LoadPipeline(rootPath)
if err == nil || !strings.Contains(strings.ToLower(err.Error()), strings.ToLower(test.want)) {
t.Fatalf("LoadPipeline() error = %v, want containing %q", err, test.want)
}
})
}
}
func TestPipelineEffectiveDigestTracksNormalizedMeaning(t *testing.T) {
dir := t.TempDir()
monolithicPath := writePipelineSource(t, dir, "monolithic.yml", `workspace:
root: /srv/narratio
whisperx:
transcribe_url: https://transcription.example.com/transcribe
language: en
`)
composedPath := writePipelineSource(t, dir, "pipeline.yml", `composition:
imports: [workspace.yml]
whisperx:
transcribe_url: https://transcription.example.com/transcribe
language: en
`)
writePipelineSource(t, dir, "workspace.yml", "workspace:\n root: /srv/narratio\n")
monolithic, err := LoadPipeline(monolithicPath)
if err != nil {
t.Fatal(err)
}
composed, err := LoadPipeline(composedPath)
if err != nil {
t.Fatal(err)
}
if monolithic.resolution.effectiveDigest == "" || monolithic.resolution.effectiveDigest != composed.resolution.effectiveDigest {
t.Fatalf("equal effective results have digests %q and %q", monolithic.resolution.effectiveDigest, composed.resolution.effectiveDigest)
}
changedPath := writePipelineSource(t, dir, "changed.yml", `workspace:
root: /srv/narratio
whisperx:
transcribe_url: https://transcription.example.com/transcribe
language: fr
`)
changed, err := LoadPipeline(changedPath)
if err != nil {
t.Fatal(err)
}
if changed.resolution.effectiveDigest == monolithic.resolution.effectiveDigest {
t.Fatal("semantic value change retained effective digest")
}
t.Setenv("AWS_SECRET_ACCESS_KEY", "secret-one")
first, err := LoadPipeline(monolithicPath)
if err != nil {
t.Fatal(err)
}
t.Setenv("AWS_SECRET_ACCESS_KEY", "secret-two")
second, err := LoadPipeline(monolithicPath)
if err != nil {
t.Fatal(err)
}
if first.resolution.effectiveDigest != second.resolution.effectiveDigest || strings.Contains(first.resolution.effectiveDigest, "secret") {
t.Fatalf("environment secret affected or appeared in digest: %q / %q", first.resolution.effectiveDigest, second.resolution.effectiveDigest)
}
}
func TestEffectivePipelineValuesIgnoreEquivalentSourceLayout(t *testing.T) {
dir := t.TempDir()
monolithicPath := writePipelineSource(t, dir, "monolithic.yml", `workspace:
root: /srv/narratio
whisperx:
transcribe_url: https://transcription.example.com/transcribe
language: en
`)
composedPath := writePipelineSource(t, dir, "pipeline.yml", `composition:
imports: [workspace.yml]
whisperx:
language: en
transcribe_url: https://transcription.example.com/transcribe
`)
writePipelineSource(t, dir, "workspace.yml", "workspace:\n root: /srv/narratio\n")
monolithic, err := LoadPipeline(monolithicPath)
if err != nil {
t.Fatal(err)
}
composed, err := LoadPipeline(composedPath)
if err != nil {
t.Fatal(err)
}
monolithicValues, err := EffectivePipelineValues(monolithic)
if err != nil {
t.Fatal(err)
}
composedValues, err := EffectivePipelineValues(composed)
if err != nil {
t.Fatal(err)
}
if diff := cmp.Diff(monolithicValues, composedValues); diff != "" {
t.Fatalf("effective values differ (-monolithic +composed):\n%s", diff)
}
}
func TestPipelineEffectiveDigestExcludesProfileIdentity(t *testing.T) {
dir := t.TempDir()
rootPath := writeProfilePipeline(t, dir, profileComposition("production", "production", "testing"))
overlay := "workspace:\n root: /srv/narratio\n"
writePipelineSource(t, dir, "production.yml", overlay)
writePipelineSource(t, dir, "testing.yml", overlay)
production := "production"
productionCfg, err := LoadPipelineWithOptions(rootPath, PipelineLoadOptions{Profile: &production})
if err != nil {
t.Fatal(err)
}
testing := "testing"
testingCfg, err := LoadPipelineWithOptions(rootPath, PipelineLoadOptions{Profile: &testing})
if err != nil {
t.Fatal(err)
}
if productionCfg.resolution.effectiveDigest != testingCfg.resolution.effectiveDigest {
t.Fatalf("profile identity changed equal effective digests: %q / %q", productionCfg.resolution.effectiveDigest, testingCfg.resolution.effectiveDigest)
}
}
func TestLoadPipelineProfilePairResolvesIndependentEffectivePipelines(t *testing.T) {
dir := t.TempDir()
rootPath := writeProfilePipeline(t, dir, `composition:
imports: [base.yml]
default_profile: production
profiles:
production:
overlay: production.yml
testing:
overlay: testing.yml
whisperx:
transcribe_url: https://transcription.example.com/transcribe
`)
writePipelineSource(t, dir, "base.yml", "workspace:\n root: /srv/base\n")
writePipelineSource(t, dir, "production.yml", "workspace:\n root: /srv/production\nwhisperx:\n language: en\n")
writePipelineSource(t, dir, "testing.yml", "workspace:\n root: /srv/testing\nwhisperx:\n language: fr\n")
production, testing, err := LoadPipelineProfilePair(rootPath, "production", "testing")
if err != nil {
t.Fatal(err)
}
if production.Workspace.Root != "/srv/production" || production.WhisperX.Language != "en" {
t.Fatalf("production pair result = %#v", production)
}
if testing.Workspace.Root != "/srv/testing" || testing.WhisperX.Language != "fr" {
t.Fatalf("testing pair result = %#v", testing)
}
production.Workspace.Root = "/mutated-left"
if testing.Workspace.Root != "/srv/testing" {
t.Fatalf("right profile shared mutable state with left: %q", testing.Workspace.Root)
}
testingFirst, productionSecond, err := LoadPipelineProfilePair(rootPath, "testing", "production")
if err != nil {
t.Fatal(err)
}
if testingFirst.Workspace.Root != "/srv/testing" || productionSecond.Workspace.Root != "/srv/production" {
t.Fatalf("reversed profile pair = testing=%q production=%q", testingFirst.Workspace.Root, productionSecond.Workspace.Root)
}
if _, _, err := LoadPipelineProfilePair(rootPath, "production", "production"); err == nil || !strings.Contains(err.Error(), "must differ") {
t.Fatalf("equal profile pair error = %v, want selection rejection", err)
}
if _, _, err := LoadPipelineProfilePair(rootPath, "unknown", "testing"); err == nil || !strings.Contains(err.Error(), "not declared") {
t.Fatalf("unknown profile pair error = %v, want selection rejection", err)
}
}
func TestLoadWithSessionOptionsCarriesExplicitProfilePresence(t *testing.T) {
dir := t.TempDir()
rootPath := writeProfilePipeline(t, dir, profileComposition("production", "production", "testing"))
writePipelineSource(t, dir, "production.yml", "workspace:\n root: /production\n")
writePipelineSource(t, dir, "testing.yml", "workspace:\n root: /testing\n")
campaignPath := writePipelineSource(t, dir, "campaign.yml", "campaign_id: campaign\ninputs:\n speakers_file: speakers.yml\n autocorrect_file: autocorrect.yml\n glossary_file: glossary.yml\n party_file: party.yml\n")
writePipelineSource(t, dir, "party.yml", canonicalPartyFixture)
sessionPath := writePipelineSource(t, dir, "session.yml", "session_id: session\ncampaign: campaign\n")
selected := "testing"
cfg, err := LoadWithSessionOptions(rootPath, campaignPath, sessionPath, SessionLoadOptions{Profile: &selected})
if err != nil {
t.Fatal(err)
}
if cfg.Pipeline.Workspace.Root != "/testing" || cfg.Pipeline.resolution.selectedProfile.source != "cli" {
t.Fatalf("combined profile result = workspace=%q metadata=%#v", cfg.Pipeline.Workspace.Root, cfg.Pipeline.resolution.selectedProfile)
}
}
func writeProfilePipeline(t *testing.T, dir, content string) string {
t.Helper()
return writePipelineSource(t, dir, "pipeline.yml", content)
}
func profileComposition(defaultProfile string, profiles ...string) string {
var builder strings.Builder
builder.WriteString("composition:\n")
if defaultProfile != "" {
builder.WriteString(" default_profile: " + defaultProfile + "\n")
}
builder.WriteString(" profiles:\n")
for _, profile := range profiles {
builder.WriteString(" " + profile + ":\n overlay: " + profile + ".yml\n")
}
builder.WriteString("whisperx:\n transcribe_url: https://transcription.example.com/transcribe\n")
return builder.String()
}
func profilePointer(value string) *string {
return &value
}
func assertSelectedPipelineProfile(t *testing.T, cfg *PipelineConfig, name, source, overlayPath string) {
t.Helper()
if cfg == nil || cfg.resolution == nil || cfg.resolution.selectedProfile == nil {
t.Fatal("selected profile provenance is absent")
}
selection := cfg.resolution.selectedProfile
if selection.name != name || selection.source != source || selection.overlayPath != absolutePath(t, overlayPath) {
t.Fatalf("selected profile = %#v, want name=%q source=%q overlay=%q", selection, name, source, absolutePath(t, overlayPath))
}
}

View File

@@ -129,7 +129,7 @@ func TestScriptoriumLoadAndValidate(t *testing.T) {
source: narratio.transcript.polished
artifact: session_recap
`,
wantLoadErr: "field artifact not found",
wantLoadErr: "decode.unknown_field",
},
{
name: "canonical previous-session source is accepted",

View File

@@ -540,7 +540,7 @@ publish:
`
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
_, err := Load(pipelinePath, sessionPath)
if err == nil || !strings.Contains(err.Error(), "strict decode failed") || !strings.Contains(err.Error(), "field locks not found") {
if err == nil || !strings.Contains(err.Error(), "decode.unknown_field") {
t.Fatalf("Load() error = %v, want retired locks field rejected", err)
}
}

View File

@@ -57,12 +57,14 @@ func TestTrimLoadAndValidate(t *testing.T) {
},
},
{
name: "enabled omitted defaults enabled",
name: "explicit null remains absent",
trimYAML: `trim:
`,
assert: func(t *testing.T, cfg *Config) {
t.Helper()
assertDefaultTrimConfig(t, cfg)
if cfg.Pipeline.Trim != nil {
t.Fatalf("trim config = %#v, want nil", cfg.Pipeline.Trim)
}
},
},
{

View File

@@ -60,12 +60,7 @@ func validatePublish(cfg *PublishConfig, scriptorium *ScriptoriumConfig, notariu
}
dest := strings.TrimSpace(item.Dest)
if dest == "" {
derivedDest, err := artifactpolicy.ResolvePublishedDestinationWithExtractions(source, "", configuredOutputs, extractionOutputs)
if err != nil {
return fmt.Errorf("%s.dest is required when destination cannot be derived from %q: %w", prefix, source, err)
}
dest = derivedDest
cfg.Outputs[i].Dest = derivedDest
return fmt.Errorf("%s.dest is required when destination cannot be derived from %q", prefix, source)
}
normalizedDest, err := pathsafe.NormalizeRelativeDestination(dest)
if err != nil {