diff --git a/docs/cli.md b/docs/cli.md index 6d6850a..8418250 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -53,10 +53,10 @@ Rules: - if both positional `` and `--session-id` are provided, values must match. - `--previous-session-id` is a strict expectation: the selected session file must contain the same `previous_session_id`. -- `--profile` selects a declared pipeline profile. It may be supplied once; - an explicit empty or unknown value fails configuration resolution. When it is - omitted, a declared `default_profile` is used. The same selection applies to - all common-flag commands, including `regenerate-artifacts`. +- `--profile` selects `profiles/.yml` relative to the pipeline root. It + may be supplied once; omitted, empty, and the exact name `default` select the + base configuration, while an unknown named profile fails. The same selection + applies to all common-flag commands, including `regenerate-artifacts`. - `clean --all` cannot be combined with campaign/session selectors. - notification delivery is currently limited to the configured `noop` mode; see the [configuration reference](./config.md#notifications). @@ -101,7 +101,7 @@ Session, range, force, and artifact-execution flags are not accepted. `config validate` writes a concise root-path, selected-profile (or `none`), and effective-digest summary after successful complete validation. `config show` writes one deterministic, secret-free YAML document containing defaulted and -expanded concrete configuration. It omits composition declarations, artifact +expanded concrete configuration. It omits ConfigKit directives, artifact family declarations, and runtime provenance. `config sources` reports the same fully validated resolution without printing @@ -113,8 +113,8 @@ and generated family ownership. A generated party member has one family record and one party record at the same logical path. The output never reads or prints secret values. -`config diff` resolves both supplied profile names from one parsed root source -set and compares their fully resolved, secret-free effective mappings. It does +`config diff` resolves both supplied profile names independently and compares +their fully resolved, secret-free effective mappings. It does not accept `--profile`; the two positional names must be distinct, declared profiles. When party-driven families are present, both profiles must resolve to the same selected campaign and party. Use `--campaign-file` if profile-specific diff --git a/docs/config.md b/docs/config.md index 1975fb8..9078066 100644 --- a/docs/config.md +++ b/docs/config.md @@ -49,8 +49,8 @@ sources` use the same `--config`, `--campaign`, `--campaign-file`, and `--profile` selection rules as pipeline commands, but do not select, discover, or load a session. They do not read credential values or create runtime state. `narratio config diff ` uses the same pipeline and -campaign selectors, resolves each named profile independently from one parsed -root source set, and does not accept a separate `--profile` flag. +campaign selectors, resolves each named profile with an independent ConfigKit +load, and does not accept a separate `--profile` flag. Campaign selection is optional only when the resolved pipeline has no `scriptorium.artifact_families`. When families are declared, Narratio selects a @@ -59,7 +59,7 @@ then parses the campaign-owned party and expands concrete artifacts and any family publish rules before validation. `config validate` prints the resulting root, profile, and effective digest. `config show` emits the normalized effective pipeline YAML, with defaults and concrete expansion included but -composition and family declarations omitted. `config sources` prints a stable +ConfigKit directives and family declarations omitted. `config sources` prints a stable source projection instead of effective values: root/import/profile/default ownership plus campaign/party and generated-family records. Canonical derived players trace to the party. The [CLI reference](cli.md#config-validate-config-show-config-sources-and-config-diff) @@ -77,8 +77,9 @@ difference. Use the [production/testing bundle](../examples/production-testing/pipeline.yml) as the complete copyable migration reference. Split stable pipeline settings -into explicit additive imports, place production/testing differences in one -selected overlay, and retain a production `default_profile`. Convert campaign +into explicit additive imports and place production/testing differences in the +fixed `profiles/production.yml` and `profiles/testing.yml` overlays. Convert +campaign rosters to [canonical party input](integrations/party.md), then express character work as families. Inspect the result with `config validate`, `config show`, and `config sources`; use `config diff` @@ -96,16 +97,15 @@ remote state with an unsafe legacy identity must be migrated before use. ## Validation and Merge Rules -- YAML decode is strict (`KnownFields(true)`) and accepts exactly one document: - unknown fields or trailing documents fail load. +- ConfigKit performs strict typed YAML decoding and accepts exactly one document; + unknown fields, duplicate keys, and trailing documents fail load. - A pipeline file may explicitly import additive YAML fragments through the - root-only `composition.imports` list. Imported files contribute fields to one + root-only `configkit.imports` list. Imported files contribute fields to one logical pipeline document; they do not override fields supplied by the root or another import. -- A root pipeline may declare named profiles. Exactly one profile is selected - by an option-aware caller or by `composition.default_profile`; a caller's - explicit selection takes precedence. Declaring profiles without either form - of selection is an error. +- An omitted profile, an empty profile option, and the exact name `default` + select the base configuration. Any other name selects + `profiles/.yml` relative to the pipeline root. - Configured timeout and retry-delay durations must be positive. An omitted artifact timeout continues to inherit its configured Scriptorium timeout. - Session files must be concrete; unresolved `{{ ... }}` placeholders fail load. @@ -123,22 +123,16 @@ remote state with an unsafe legacy identity must be migrated before use. - local (`audio_dir` or `audio_files`), or - S3 (`audio_s3.prefix`). -### Pipeline composition +### Pipeline imports and profiles Large pipeline configurations may be split into explicitly named fragments and -may declare one overlay per selectable profile: +may use fixed-path profile overlays: ```yaml -composition: +configkit: imports: - - config/storage.yml - - config/integrations.yaml - default_profile: production - profiles: - production: - overlay: profiles/production.yml - testing: - overlay: profiles/testing.yml + - path: config/storage.yml + - path: config/integrations.yaml campaigns: root: /usr/local/share/narratio/campaigns @@ -152,36 +146,29 @@ Imports are resolved relative to the directory containing the root pipeline file and are loaded in declaration order. Narratio does not scan directories or infer fragments. Each import must be a confined regular `.yml` or `.yaml` file: absolute paths, traversal, symlinks, directories, duplicate files, and an -import of the root pipeline itself are rejected. Only the root pipeline may -contain `composition`; nested composition is rejected. +import of the root pipeline itself are rejected. Imported files cannot declare +another `configkit` control section. -Composition is additive. A map may be extended by multiple files when every +Base merging is additive. A map may be extended by multiple files when every leaf is distinct, but a scalar, list, or map/list/scalar kind cannot be claimed more than once, even when the repeated values are identical. Conflict errors -name the full field path and every source that claimed it. The assembled YAML is -then decoded against the normal strict pipeline schema and defaults are applied -once. +name the full field path and every source that claimed it. Narratio additionally +rejects an artifact or artifact-family key declared by more than one base file, +even when those files contribute disjoint children. -Profile names are case-sensitive, non-empty, trimmed, and cannot contain -control characters. If `profiles` is present, it must contain at least one -entry and every entry must contain only an `overlay` path. An explicit profile -selection overrides `default_profile`; unknown and explicitly empty selections -fail. Narratio never selects the first profile implicitly and does not read a -profile selection from the environment. +Profile names are case-sensitive and use ConfigKit's safe fixed-path naming +rules. Only the selected overlay is opened; a malformed unselected overlay does +not affect a base or different-profile load. Maps merge recursively, overlay +scalars replace base scalars, and overlay lists replace base lists completely. +Explicit `false`, zero, empty strings, empty containers, and null values remain +explicit inputs and are not refilled by defaults. Profiles cannot inherit, +stack, import files, or declare another ConfigKit control section. -Every declared overlay is resolved relative to the root pipeline directory and -must satisfy the same confined regular-YAML-file rules as an import. Narratio -parses every declared overlay even when it is not selected, then applies only -the selected one. Maps merge recursively, overlay scalars replace base scalars, -and overlay lists replace base lists completely. Explicit `false`, zero, empty -lists, and empty maps remain meaningful. YAML null cannot delete a value, and -kind changes are rejected. Profiles cannot inherit from or stack with other -profiles, and overlays cannot import files or declare profiles. - -After composition, Narratio strictly decodes the result, applies centralized -defaults once, resolves ordinary paths, and computes a deterministic effective +During ConfigKit loading, Narratio applies fresh typed defaults, recorded +entry-level defaults and normalization, read-only file-local validation, and +ordinary path resolution before computing a deterministic effective configuration digest. The digest represents the normalized, secret-free -runtime pipeline mapping; it excludes composition declarations, source +runtime pipeline mapping; it excludes ConfigKit directives, source provenance, profile identity, and raw environment secret values. Equivalent effective mappings therefore have the same digest regardless of how fields are split among the root and imports. @@ -274,9 +261,7 @@ Rules: | Field | Type | Required | Default / Rule | | --- | --- | --- | --- | -| `composition.imports[]` | list of strings | No | explicit additive pipeline fragments relative to the root pipeline directory; `.yml` or `.yaml` regular files only | -| `composition.default_profile` | string | Conditional | selected when profiles exist and no caller explicitly selects one; must name a declared profile | -| `composition.profiles..overlay` | string | Conditional | required for every declared profile; one confined `.yml` or `.yaml` overlay relative to the root pipeline directory | +| `configkit.imports[].path` | string | No | explicit additive pipeline fragment relative to the root pipeline directory; `.yml` or `.yaml` regular files only | | `pipeline.workspace.root` | string | No | `/var/lib/narratio` | | `pipeline.workspace.cleanup_after_publish` | bool | No | `false`; selects the published run's workspace target for final cleanup and requires effective `pipeline.publish.enabled=true` and `pipeline.publish.upload_run=true` | | `pipeline.campaigns.root` | string | No | `/usr/local/share/narratio/campaigns` | diff --git a/docs/internal/configuration.md b/docs/internal/configuration.md index 1ade825..6457c21 100644 --- a/docs/internal/configuration.md +++ b/docs/internal/configuration.md @@ -8,31 +8,21 @@ pipeline-loading boundary implemented by `internal/config`. `LoadPipeline` assembles and validates a pipeline in this order: -1. Parse the root YAML into a presence-aware composition tree. The tree retains - source names, full field paths, node kinds, declaration order, and explicit - zero, false, empty-map, and empty-list values. -2. Decode the root-only `composition` envelope, validate its declaration shapes - and cross-field profile semantics, then clone the root without that envelope - as the ordinary pipeline body. The parsed source document is not mutated. A - load option retains the difference between omitted and explicitly empty - profile selection. -3. Open each import relative to the root pipeline directory through the - confined regular-file boundary. Imports must use a `.yml` or `.yaml` - extension and cannot traverse, use symlinks, repeat a file, import the root, - or contain another composition envelope. -4. Resolve and structurally parse every declared profile overlay through the - same confined regular-file boundary. Missing or malformed unselected - overlays fail the load. Overlays cannot contain a composition envelope. -5. Additively merge the root body and imports. Distinct map leaves compose; - repeated scalar or list paths and node-kind disagreements are conflicts. -6. Select exactly one declared profile from an explicit option or the default, - then recursively merge its overlay. Overlay leaves replace base leaves, - lists are atomic replacements, and null or kind changes fail. -7. Emit deterministic canonical YAML and strictly decode it into - `PipelineConfig`. -8. Apply pipeline defaults once, explicitly normalize reusable domain values, - resolve ordinary relative pipeline paths from the root pipeline file, and - digest the normalized effective mapping. +1. Give ConfigKit a fresh typed `PipelineConfig` default value, the root file, + enabled imports, and the caller's profile selection. Omitted, empty, and + exact `default` selections use the base; named profiles use the fixed + `profiles/.yml` convention. +2. ConfigKit parses the root and declared imports, additively merges base + sources, applies only the selected overlay, and strictly decodes the result. +3. The definition resolver applies recorded entry-level and inherited defaults, + runs deterministic domain normalization, records derived publish destinations, + and resolves ordinary paths relative to the root pipeline file. +4. The definition validator performs read-only checks that require only the + pipeline file. Checks requiring canonical-party family expansion remain at + the existing explicit pipeline/campaign resolution boundary. +5. Narratio inspects public ConfigKit metadata to reject artifact and family keys + declared by multiple base files, adapts source ownership records, attaches + runtime-only state, and computes the normalized effective digest. This ordering preserves monolithic configuration behavior. Moving a field to an imported fragment changes its source ownership, not its path base, default, @@ -71,7 +61,7 @@ commands do not marshal runtime objects directly. `EffectivePipelineSources` and `EffectiveCampaignSources` provide the separate safe provenance projection for `config sources`. Pipeline ownership begins with -the complete logical field paths retained during composition and classifies +the complete logical field paths projected from the schema and classifies each contributor as root, import, profile, or centralized default. The projection replaces generated concrete member paths with paired family and canonical-party records, and does the same for generated publish rules. @@ -79,9 +69,8 @@ Campaign records identify campaign-owned fields and party inputs; canonical derived players point to the party source. The application command only joins these sorted records with selection metadata and never reparses configuration files. -`config diff` uses a paired profile loader that parses the root, imports, and -declared overlays once, then clones the additive base before independently -selecting, decoding, defaulting, and finalizing each profile. When campaign +`config diff` uses a thin paired loader that performs two independent ConfigKit +loads. When campaign resolution is needed, the command loads one selected campaign and party and expands both effective pipelines from that same party value. The configuration owner projects each normalized effective mapping directly from that same typed @@ -128,21 +117,22 @@ that candidate without changing it. Scriptorium validates artifact declarations and outputs, dependencies and inputs, variables, referenced outputs, and the final dependency graph in deterministic artifact order. -Diagnostics identify the relevant semantic field. Composition conflicts also +Diagnostics identify the relevant semantic field. Merge conflicts also retain every claiming source needed to correct an additive ownership conflict. Complete prose, punctuation, formatting, and private validation choreography are not compatibility contracts. ## Diagnostics And Runtime Metadata -Syntax, duplicate-key, composition, conflict, and schema failures include the +Syntax, duplicate-key, merge, profile, and schema failures include the relevant source name and full field path. Additive conflicts report every claiming source so operators can repair the split without repeatedly rediscovering additional conflicts. The loaded pipeline retains private runtime metadata for the absolute root path, ordered imports, selected profile name and selection source, selected -overlay, contributing sources, effective digest, and leaf ownership. Base +overlay, original ConfigKit metadata and source snapshots, effective digest, +and leaf ownership. Base leaves retain their root/import owners, replaced leaves belong to the selected overlay, and centrally supplied values use the synthetic `default` owner. This metadata does not participate in YAML decoding or alter the public @@ -152,7 +142,7 @@ The effective digest is SHA-256 over a versioned deterministic JSON encoding of the same typed safe projection used by show and diff. Runtime Notarius paths remain absolute for execution, but the digest substitutes their normalized logical values captured before root-relative resolution, so relocating an equivalent configuration -bundle does not change provenance. Because composition and resolution metadata +bundle does not change provenance. Because ConfigKit and resolution metadata are private, the digest excludes source layout, profile name, and ownership. Configuration stores environment variable names rather than resolving raw credentials, so raw secret values are neither loaded nor hashed. @@ -161,12 +151,11 @@ for later runtime expansion. ## Test Surfaces -`composition_test.go` protects the presence and merge algebra independently of -the public schema. `pipeline_composition_test.go` exercises explicit imports, -confinement, conflicts, strict decoding, metadata, and root-relative path -behavior through `LoadPipeline`. `pipeline_profiles_test.go` covers selection, -all-overlay validation, overlay behavior, provenance, option propagation, and -effective-digest stability. Application configuration-loader tests protect the +`pipeline_configkit_test.go` exercises real imports and fixed-path profiles, +base selection, unselected-overlay isolation, source attribution, and Narratio's +unique artifact/family base-declaration rule. Existing focused configuration +tests protect defaults, explicit-value semantics, strict decoding, validation, +metadata, and root-relative path behavior. Application configuration-loader tests protect the single-read boundary by changing the pipeline file after its initial load and confirming local session resolution retains the original pipeline. Other configuration tests continue to protect defaults and validation after assembly. diff --git a/docs/internal/overview.md b/docs/internal/overview.md index 524dae5..f30aa35 100644 --- a/docs/internal/overview.md +++ b/docs/internal/overview.md @@ -37,7 +37,7 @@ failed stage or required manifest save prevents later stages from starting. | --- | --- | --- | | Executable | `cmd/narratio` | Process entry, standard stream wiring, argument handoff, and exit status. | | Application orchestration | `internal/app` | Command dispatch, configuration selection, secret-file environment loading, production composition, session locking, planning, execution, restore, and user-facing reporting. | -| Configuration | [`internal/config`](configuration.md) | Presence-aware root/import/profile composition, canonical party and family expansion, strict YAML loading, defaults, normalization, session templating, and validation. | +| Configuration | [`internal/config`](configuration.md) | ConfigKit-backed pipeline imports and profiles, canonical party and family expansion, strict YAML loading, defaults, normalization, session templating, and validation. | | Pipeline stages | `internal/stage` | Canonical stage registry, shared stage contract, execution dependencies, and implemented stage behavior. | | External boundaries | `internal/adapters`, `internal/audio` | WhisperX HTTP, downstream subprocesses, notification, object storage, and S3 audio materialization behind Narratio contracts. | | Manifests | `internal/manifest` | Durable session progress, stage transitions, validation, and atomic persistence. | @@ -105,8 +105,8 @@ so any earlier replacement can stale a prior cleanup result. ## Focused Documentation -- [Configuration Internals](configuration.md): pipeline composition, import - confinement, field ownership, decoding, and root-relative path semantics. +- [Configuration Internals](configuration.md): pipeline imports and profiles, + field ownership, decoding, and root-relative path semantics. - [Adapter Internals](adapters.md): external adapter boundaries, composition, failure behavior, and test surfaces. - [Artifact Internals](artifacts.md): source identities, runtime catalog, diff --git a/docs/policy/architecture.md b/docs/policy/architecture.md index 7fc9bf6..1e2aa63 100644 --- a/docs/policy/architecture.md +++ b/docs/policy/architecture.md @@ -146,7 +146,7 @@ Configuration is strict, explicit, centralized, and operator-oriented. - YAML decoding rejects unknown fields. - Defaults are centralized and testable. - Empty configured values do not silently replace meaningful defaults. -- Validation rejects invalid composition before stage execution where +- Validation rejects invalid configuration before stage execution where practical. - Root-owned imports and one selected profile resolve deterministically through the configuration owner; commands do not implement their own merge rules. diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index fc6298a..88e176a 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -189,7 +189,10 @@ projection consumer needs the custom composition parser/renderer. ## Stage 4 — Cut pipeline loading over to ConfigKit -Status: Pending +Status: Complete — Replaced pipeline composition with ConfigKit v0.1.1 imports, +fixed-path profile selection, fresh typed defaults, recorded resolution, source +metadata adaptation, and duplicate base artifact/family enforcement; removed the +legacy parser and verified all tests plus the configuration race suite. Read: ConfigKit v0.1.1 `docs/config.md`, `docs/consumers/README.md`, and its Narratio-shaped compiled example; Narratio `internal/config/load.go`, diff --git a/examples/production-testing/pipeline.yml b/examples/production-testing/pipeline.yml index 64294fe..9e966f9 100644 --- a/examples/production-testing/pipeline.yml +++ b/examples/production-testing/pipeline.yml @@ -1,15 +1,9 @@ # Copyable production/testing pipeline entry point. Every fragment is named # explicitly; Narratio never scans conf.d automatically. -composition: +configkit: imports: - - conf.d/platform.yml - - conf.d/storage.yml - - conf.d/transcript.yml - - conf.d/artifacts.yml - - conf.d/publish.yml - default_profile: production - profiles: - production: - overlay: profiles/production.yml - testing: - overlay: profiles/testing.yml + - path: conf.d/platform.yml + - path: conf.d/storage.yml + - path: conf.d/transcript.yml + - path: conf.d/artifacts.yml + - path: conf.d/publish.yml diff --git a/go.mod b/go.mod index 7ee6b77..4c7871d 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module gitea.maximumdirect.net/eric/narratio go 1.26.0 require ( + gitea.maximumdirect.net/eric/ConfigKit v0.1.1 github.com/aws/aws-sdk-go-v2/config v1.32.17 github.com/aws/aws-sdk-go-v2/credentials v1.19.16 github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 @@ -28,4 +29,5 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect + go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect ) diff --git a/go.sum b/go.sum index 3342ad0..2ce74c7 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +gitea.maximumdirect.net/eric/ConfigKit v0.1.1 h1:/JXi95QDhgRtiQN7R/+6cGFcfc0+gfl+D44nKXPd52k= +gitea.maximumdirect.net/eric/ConfigKit v0.1.1/go.mod h1:tYWpsrbLyu05r8+1MapWp5fN6ZrX9I8hU2xEPedYmYk= github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= @@ -36,6 +38,8 @@ github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4= +go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= diff --git a/internal/app/assembled_workflow_test.go b/internal/app/assembled_workflow_test.go index a6ad489..7e97653 100644 --- a/internal/app/assembled_workflow_test.go +++ b/internal/app/assembled_workflow_test.go @@ -253,8 +253,8 @@ func TestAssembledSplitBundleCommandsUseOneEffectiveFamilyConfiguration(t *testi } var validated bytes.Buffer - if err := ConfigValidate(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath}, &validated); err != nil { - t.Fatalf("validate production default: %v", err) + if err := ConfigValidate(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath, "--profile", "production"}, &validated); err != nil { + t.Fatalf("validate production profile: %v", err) } if !strings.Contains(validated.String(), "profile=production") { t.Fatalf("validate output = %q, want production profile", validated.String()) @@ -330,7 +330,7 @@ func TestAssembledSplitBundleCommandsUseOneEffectiveFamilyConfiguration(t *testi } func TestAssembledSplitBundleProfileSwitchRecordsProvenanceAndLimitsReuse(t *testing.T) { - production := loadAssembledSplitBundleConfig(t, "") + production := loadAssembledSplitBundleConfig(t, "production") testingCfg := loadAssembledSplitBundleConfig(t, "testing") workspace := t.TempDir() production.Pipeline.Workspace.Root = workspace diff --git a/internal/app/config_commands_test.go b/internal/app/config_commands_test.go index da1dd80..a3ca888 100644 --- a/internal/app/config_commands_test.go +++ b/internal/app/config_commands_test.go @@ -304,17 +304,9 @@ func TestConfigCommandsInspectMaintainedSplitBundleWithoutRuntimeState(t *testin func writeInspectionProfiles(t *testing.T, pipelinePath, production, testing string) { t.Helper() - data, err := os.ReadFile(pipelinePath) - if err != nil { - t.Fatal(err) - } - composition := "composition:\n default_profile: production\n profiles:\n production:\n overlay: production.yml\n testing:\n overlay: testing.yml\n" - if err := os.WriteFile(pipelinePath, append([]byte(composition), data...), 0o644); err != nil { - t.Fatal(err) - } - dir := filepath.Dir(pipelinePath) - mustWriteTestFile(t, filepath.Join(dir, "production.yml"), production) - mustWriteTestFile(t, filepath.Join(dir, "testing.yml"), testing) + profilesDir := filepath.Join(filepath.Dir(pipelinePath), "profiles") + mustWriteTestFile(t, filepath.Join(profilesDir, "production.yml"), production) + mustWriteTestFile(t, filepath.Join(profilesDir, "testing.yml"), testing) } func TestConfigCommandsSelectProfilesAndCampaigns(t *testing.T) { @@ -326,7 +318,7 @@ func TestConfigCommandsSelectProfilesAndCampaigns(t *testing.T) { if err := ConfigValidate(context.Background(), []string{"--config", pipelinePath}, &defaultOut); err != nil { t.Fatalf("default ConfigValidate() error = %v", err) } - if !strings.Contains(defaultOut.String(), "profile=production") { + if !strings.Contains(defaultOut.String(), "profile=none") { t.Fatalf("default output = %q", defaultOut.String()) } @@ -398,8 +390,9 @@ func TestConfigValidateReportsCanonicalPartyAndImportErrors(t *testing.T) { dir := t.TempDir() rootPath := filepath.Join(dir, "pipeline.yml") - mustWriteTestFile(t, rootPath, `composition: - imports: [conf.yml] + mustWriteTestFile(t, rootPath, `configkit: + imports: + - path: conf.yml workspace: root: /one whisperx: @@ -407,7 +400,7 @@ whisperx: `) mustWriteTestFile(t, filepath.Join(dir, "conf.yml"), "workspace:\n root: /two\n") err := ConfigValidate(context.Background(), []string{"--config", rootPath}, io.Discard) - if err == nil || !strings.Contains(err.Error(), "workspace.root") || !strings.Contains(err.Error(), "pipeline.yml") || !strings.Contains(err.Error(), "conf.yml") { + if err == nil || !strings.Contains(err.Error(), "/workspace/root") || !strings.Contains(err.Error(), "pipeline.yml") || !strings.Contains(err.Error(), "conf.yml") { t.Fatalf("import diagnostic = %v", err) } } @@ -432,7 +425,7 @@ func TestConfigSourcesReportsStableOwnership(t *testing.T) { for _, want := range []string{ "root: ", "imports: none", - "profile: name=testing selection=cli overlay=", + "profile: name=testing selection=explicit overlay=", "campaign: id=sample-campaign source=", "party: source=", "digest: ", @@ -475,15 +468,7 @@ func TestConfigSourcesReportsCanonicalFamilyAndPublishOrigins(t *testing.T) { func TestConfigSourcesPreservesProfileOwnershipThroughFamilyExpansion(t *testing.T) { pipelinePath, campaignPath := writeInspectionFamilyConfig(t) - data, err := os.ReadFile(pipelinePath) - if err != nil { - t.Fatal(err) - } - profilePath := filepath.Join(filepath.Dir(pipelinePath), "testing.yml") - composition := "composition:\n default_profile: testing\n profiles:\n testing:\n overlay: testing.yml\n" - if err := os.WriteFile(pipelinePath, append([]byte(composition), data...), 0o644); err != nil { - t.Fatal(err) - } + profilePath := filepath.Join(filepath.Dir(pipelinePath), "profiles", "testing.yml") mustWriteTestFile(t, profilePath, `scriptorium: artifact_families: character_note: @@ -492,7 +477,7 @@ func TestConfigSourcesPreservesProfileOwnershipThroughFamilyExpansion(t *testing `) var out bytes.Buffer - if err := ConfigSources(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath}, &out); err != nil { + if err := ConfigSources(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath, "--profile", "testing"}, &out); err != nil { t.Fatalf("ConfigSources() error = %v", err) } want := "scriptorium.artifacts.character_note_arannis.enabled\tfamily\t" + profilePath @@ -505,13 +490,10 @@ func TestConfigSourcesReportsRootImportProfileAndDefaultOwnership(t *testing.T) dir := t.TempDir() rootPath := filepath.Join(dir, "pipeline.yml") importPath := filepath.Join(dir, "base.yml") - profilePath := filepath.Join(dir, "testing.yml") - mustWriteTestFile(t, rootPath, `composition: - imports: [base.yml] - default_profile: testing - profiles: - testing: - overlay: testing.yml + profilePath := filepath.Join(dir, "profiles", "testing.yml") + mustWriteTestFile(t, rootPath, `configkit: + imports: + - path: base.yml workspace: root: /srv/narratio whisperx: @@ -521,13 +503,13 @@ whisperx: mustWriteTestFile(t, profilePath, "whisperx:\n language: fr\n") var out bytes.Buffer - if err := ConfigSources(context.Background(), []string{"--config", rootPath}, &out); err != nil { + if err := ConfigSources(context.Background(), []string{"--config", rootPath, "--profile", "testing"}, &out); err != nil { t.Fatalf("ConfigSources() error = %v", err) } output := out.String() for _, want := range []string{ "import: " + importPath, - "profile: name=testing selection=default overlay=" + profilePath, + "profile: name=testing selection=explicit overlay=" + profilePath, "workspace.root\troot\t" + rootPath, "storage.backend\timport\t" + importPath, "whisperx.language\tprofile\t" + profilePath, diff --git a/internal/app/profile_commands_test.go b/internal/app/profile_commands_test.go index fb1dd5f..e662309 100644 --- a/internal/app/profile_commands_test.go +++ b/internal/app/profile_commands_test.go @@ -37,10 +37,10 @@ func TestRunProfileSelectionFlowsThroughSharedLoader(t *testing.T) { t.Fatal(err) } if language != "en" { - t.Fatalf("default profile language = %q, want en", language) + t.Fatalf("base configuration language = %q, want en", language) } - for _, profile := range []string{"unknown", ""} { + for _, profile := range []string{"unknown"} { err := runWithPipelineExecutor(context.Background(), []string{ "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--profile", profile, }, io.Discard, executor) @@ -48,19 +48,20 @@ func TestRunProfileSelectionFlowsThroughSharedLoader(t *testing.T) { t.Fatalf("profile %q error = %v, want selection rejection", profile, err) } } + + if err := runWithPipelineExecutor(context.Background(), []string{ + "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--profile", "", + }, io.Discard, executor); err != nil { + t.Fatalf("explicit empty profile should select base: %v", err) + } } func enableCommandTestProfiles(t *testing.T, pipelinePath string) { t.Helper() - data, err := os.ReadFile(pipelinePath) - if err != nil { + dir := filepath.Join(filepath.Dir(pipelinePath), "profiles") + if err := os.MkdirAll(dir, 0o755); err != nil { t.Fatal(err) } - composition := "composition:\n default_profile: production\n profiles:\n production:\n overlay: production.yml\n testing:\n overlay: testing.yml\n" - if err := os.WriteFile(pipelinePath, append([]byte(composition), data...), 0o644); err != nil { - t.Fatal(err) - } - dir := filepath.Dir(pipelinePath) if err := os.WriteFile(filepath.Join(dir, "production.yml"), []byte("whisperx:\n language: en\n"), 0o644); err != nil { t.Fatal(err) } diff --git a/internal/config/artifact_families_test.go b/internal/config/artifact_families_test.go index e69710c..9659f60 100644 --- a/internal/config/artifact_families_test.go +++ b/internal/config/artifact_families_test.go @@ -8,6 +8,15 @@ import ( "testing" ) +func absolutePath(t *testing.T, path string) string { + t.Helper() + absolute, err := filepath.Abs(path) + if err != nil { + t.Fatal(err) + } + return absolute +} + func TestArtifactFamiliesExpandCanonicalCharactersDeterministically(t *testing.T) { dir := t.TempDir() writePartyResolutionFile(t, dir+"/party.yml", `schema_version: narratio.party.v1 @@ -148,7 +157,7 @@ func TestArtifactFamilyStrictDecodeRejectsUnknownFields(t *testing.T) { dir := t.TempDir() pipelinePath, _, _ := writeArtifactFamilyConfig(t, dir, artifactFamilyPipelineYAML(validFamilyYAML+" unsupported: true\n")) _, err := LoadPipeline(pipelinePath) - if err == nil || !strings.Contains(err.Error(), "field unsupported not found") { + if err == nil || !strings.Contains(err.Error(), "decode.unknown_field") { t.Fatalf("LoadPipeline() error = %v", err) } } diff --git a/internal/config/composition.go b/internal/config/composition.go deleted file mode 100644 index 4222850..0000000 --- a/internal/config/composition.go +++ /dev/null @@ -1,783 +0,0 @@ -package config - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "sort" - "strconv" - "strings" - - "gopkg.in/yaml.v3" -) - -// compositionDocument is the presence-aware representation used while -// assembling pipeline configuration sources. It deliberately models YAML -// mechanics rather than duplicating PipelineConfig's field schema. -type compositionDocument struct { - root *compositionNode - sources []string -} - -type compositionNode struct { - kind yaml.Kind - tag string - value string - path string - sources []string - fields []compositionField - items []*compositionNode -} - -type compositionField struct { - key string - value *compositionNode -} - -// compositionValueRecord is a deterministic semantic leaf projection. Lists -// are atomic configuration values, while mappings are traversed recursively. -type compositionValueRecord struct { - Path string - Kind yaml.Kind - Value string - Sources []string -} - -// parseCompositionDocument parses exactly one YAML mapping while retaining -// source ownership, explicit zero values, and declaration order. -func parseCompositionDocument(source string, reader io.Reader) (*compositionDocument, error) { - if strings.TrimSpace(source) == "" { - return nil, fmt.Errorf("configuration source identity is required") - } - if reader == nil { - return nil, fmt.Errorf("configuration source %q: reader is nil", source) - } - - decoder := yaml.NewDecoder(reader) - var document yaml.Node - if err := decoder.Decode(&document); err != nil { - if err == io.EOF { - return nil, fmt.Errorf("configuration source %q: document is empty", source) - } - return nil, fmt.Errorf("configuration source %q: decode YAML: %w", source, err) - } - var trailing yaml.Node - if err := decoder.Decode(&trailing); err == nil { - return nil, fmt.Errorf("configuration source %q: must contain exactly one YAML document", source) - } else if err != io.EOF { - return nil, fmt.Errorf("configuration source %q: decode trailing YAML: %w", source, err) - } - if document.Kind != yaml.DocumentNode || len(document.Content) != 1 { - return nil, fmt.Errorf("configuration source %q: must contain exactly one YAML document", source) - } - if document.Content[0].Kind != yaml.MappingNode { - return nil, fmt.Errorf( - "configuration source %q: top-level document must be a mapping, got %s", - source, - yamlKindName(document.Content[0].Kind), - ) - } - - root, err := buildCompositionNode(document.Content[0], source, "") - if err != nil { - return nil, err - } - return &compositionDocument{root: root, sources: []string{source}}, nil -} - -func parseCompositionBytes(source string, data []byte) (*compositionDocument, error) { - return parseCompositionDocument(source, bytes.NewReader(data)) -} - -func buildCompositionNode(node *yaml.Node, source, path string) (*compositionNode, error) { - if node == nil { - return nil, fmt.Errorf("configuration source %q at %s: YAML node is nil", source, displayCompositionPath(path)) - } - if node.Kind == yaml.AliasNode { - return nil, compositionNodeError(source, path, node, "YAML aliases are not supported because source ownership would be ambiguous") - } - result := &compositionNode{ - kind: node.Kind, tag: node.Tag, value: node.Value, path: path, - sources: []string{source}, - } - - switch node.Kind { - case yaml.MappingNode: - if len(node.Content)%2 != 0 { - return nil, compositionNodeError(source, path, node, "mapping has an incomplete key/value pair") - } - seen := make(map[string]*yaml.Node, len(node.Content)/2) - for index := 0; index < len(node.Content); index += 2 { - keyNode := node.Content[index] - valueNode := node.Content[index+1] - if keyNode.Kind == yaml.AliasNode { - return nil, compositionNodeError(source, path, keyNode, "YAML aliases are not supported because source ownership would be ambiguous") - } - if valueNode.Kind == yaml.AliasNode { - return nil, compositionNodeError(source, appendCompositionPath(path, keyNode.Value), valueNode, "YAML aliases are not supported because source ownership would be ambiguous") - } - if keyNode.Kind != yaml.ScalarNode || keyNode.Tag != "!!str" { - return nil, compositionNodeError(source, path, keyNode, "mapping keys must be strings") - } - key := keyNode.Value - fieldPath := appendCompositionPath(path, key) - if prior, duplicate := seen[key]; duplicate { - return nil, fmt.Errorf( - "configuration source %q at %s: duplicate YAML key %q (first declared at line %d, column %d; repeated at line %d, column %d)", - source, displayCompositionPath(fieldPath), key, - prior.Line, prior.Column, keyNode.Line, keyNode.Column, - ) - } - seen[key] = keyNode - child, err := buildCompositionNode(valueNode, source, fieldPath) - if err != nil { - return nil, err - } - result.fields = append(result.fields, compositionField{key: key, value: child}) - } - case yaml.SequenceNode: - for index, childNode := range node.Content { - childPath := fmt.Sprintf("%s[%d]", path, index) - child, err := buildCompositionNode(childNode, source, childPath) - if err != nil { - return nil, err - } - result.items = append(result.items, child) - } - case yaml.ScalarNode: - // Scalar tag and lexical value retain distinctions such as explicit - // false, zero, an empty string, and null until final strict decoding. - default: - return nil, compositionNodeError(source, path, node, fmt.Sprintf("unsupported YAML node kind %s", yamlKindName(node.Kind))) - } - return result, nil -} - -// mergeAdditiveComposition recursively combines disjoint mappings. A scalar, -// list, or final keyed value may have only one base owner, regardless of -// whether duplicate values happen to be equal. -func mergeAdditiveComposition(base, incoming *compositionDocument) (*compositionDocument, error) { - return mergeAdditiveCompositions(base, incoming) -} - -// mergeAdditiveCompositions validates the complete base source set before -// merging so a conflict names every source that claims the same final path. -func mergeAdditiveCompositions(documents ...*compositionDocument) (*compositionDocument, error) { - if len(documents) == 0 { - return nil, fmt.Errorf("configuration additive base merge requires at least one document") - } - for index, document := range documents { - if err := validateCompositionDocument(document, fmt.Sprintf("base[%d]", index)); err != nil { - return nil, err - } - } - if err := validateAdditiveClaims(documents); err != nil { - return nil, err - } - - result := &compositionDocument{ - root: cloneCompositionNode(documents[0].root), - sources: append([]string(nil), documents[0].sources...), - } - for _, incoming := range documents[1:] { - merged, err := mergeAdditiveNodes(result.root, incoming.root) - if err != nil { - return nil, err - } - result.root = merged - result.sources = appendUniqueStrings(result.sources, incoming.sources...) - } - return result, nil -} - -func validateAdditiveClaims(documents []*compositionDocument) error { - claims := make(map[string][]*compositionNode) - for _, document := range documents { - appendCompositionClaims(document.root, claims) - } - paths := make([]string, 0, len(claims)) - for path := range claims { - paths = append(paths, path) - } - sort.Slice(paths, func(i, j int) bool { - leftDepth := compositionPathDepth(paths[i]) - rightDepth := compositionPathDepth(paths[j]) - if leftDepth != rightDepth { - return leftDepth < rightDepth - } - return paths[i] < paths[j] - }) - for _, path := range paths { - values := claims[path] - if len(values) < 2 { - continue - } - allPopulatedMappings := true - for _, value := range values { - if value.kind != yaml.MappingNode || len(value.fields) == 0 { - allPopulatedMappings = false - break - } - } - if !allPopulatedMappings { - return newCompositionConflict("additive base merge", path, values...) - } - } - return nil -} - -func appendCompositionClaims(node *compositionNode, claims map[string][]*compositionNode) { - if node == nil { - return - } - if node.path != "" { - claims[node.path] = append(claims[node.path], node) - } - for _, field := range node.fields { - appendCompositionClaims(field.value, claims) - } -} - -func compositionPathDepth(path string) int { - if path == "" { - return 0 - } - return strings.Count(path, ".") + strings.Count(path, "[") + 1 -} - -func mergeAdditiveNodes(base, incoming *compositionNode) (*compositionNode, error) { - if base.kind != yaml.MappingNode || incoming.kind != yaml.MappingNode { - return nil, newCompositionConflict("additive base merge", base.path, base, incoming) - } - base.sources = appendUniqueStrings(base.sources, incoming.sources...) - for _, incomingField := range incoming.fields { - index := compositionFieldIndex(base.fields, incomingField.key) - if index < 0 { - field := cloneCompositionField(incomingField) - base.fields = append(base.fields, field) - continue - } - baseValue := base.fields[index].value - incomingValue := incomingField.value - if baseValue.kind == yaml.MappingNode && incomingValue.kind == yaml.MappingNode { - if len(baseValue.fields) == 0 || len(incomingValue.fields) == 0 { - return nil, newCompositionConflict("additive base merge", incomingValue.path, baseValue, incomingValue) - } - merged, err := mergeAdditiveNodes(baseValue, incomingValue) - if err != nil { - return nil, err - } - base.fields[index].value = merged - continue - } - return nil, newCompositionConflict("additive base merge", incomingValue.path, baseValue, incomingValue) - } - return base, nil -} - -// mergeOverlayComposition applies the sole overwrite layer. Mappings merge -// recursively; same-kind scalars and lists replace; null and kind changes are -// rejected. -func mergeOverlayComposition(base, overlay *compositionDocument) (*compositionDocument, error) { - if err := validateCompositionDocument(base, "base"); err != nil { - return nil, err - } - if err := validateCompositionDocument(overlay, "overlay"); err != nil { - return nil, err - } - if null := firstNullCompositionNode(overlay.root); null != nil { - return nil, fmt.Errorf( - "configuration overlay at %s from %s: null cannot delete an effective value", - displayCompositionPath(null.path), formatCompositionSources(null.sources), - ) - } - merged, err := mergeOverlayNodes(cloneCompositionNode(base.root), overlay.root) - if err != nil { - return nil, err - } - return &compositionDocument{ - root: merged, - sources: appendUniqueStrings( - append([]string(nil), base.sources...), overlay.sources..., - ), - }, nil -} - -func mergeOverlayNodes(base, overlay *compositionNode) (*compositionNode, error) { - if base.kind != yaml.MappingNode || overlay.kind != yaml.MappingNode { - return nil, newCompositionConflict("profile overlay", base.path, base, overlay) - } - base.sources = appendUniqueStrings(base.sources, overlay.sources...) - for _, overlayField := range overlay.fields { - index := compositionFieldIndex(base.fields, overlayField.key) - if index < 0 { - field := cloneCompositionField(overlayField) - base.fields = append(base.fields, field) - continue - } - baseValue := base.fields[index].value - overlayValue := overlayField.value - if baseValue.kind != overlayValue.kind { - return nil, newCompositionConflict("profile overlay kind change", overlayValue.path, baseValue, overlayValue) - } - if baseValue.kind == yaml.MappingNode { - merged, err := mergeOverlayNodes(baseValue, overlayValue) - if err != nil { - return nil, err - } - base.fields[index].value = merged - continue - } - base.fields[index].value = cloneCompositionNode(overlayValue) - } - return base, nil -} - -// canonicalYAML renders the effective mapping with sorted keys and normalized -// presentation while retaining sequence order and scalar YAML types. -func (document *compositionDocument) canonicalYAML() ([]byte, error) { - if err := validateCompositionDocument(document, "document"); err != nil { - return nil, err - } - root, err := compositionYAMLNode(document.root) - if err != nil { - return nil, err - } - var buffer bytes.Buffer - encoder := yaml.NewEncoder(&buffer) - encoder.SetIndent(2) - if err := encoder.Encode(root); err != nil { - return nil, fmt.Errorf("render effective configuration YAML: %w", err) - } - if err := encoder.Close(); err != nil { - return nil, fmt.Errorf("finish effective configuration YAML: %w", err) - } - return buffer.Bytes(), nil -} - -// canonicalDigestInput provides a deterministic, formatting-independent byte -// representation for later secret-free effective configuration digesting. -func (document *compositionDocument) canonicalDigestInput() ([]byte, error) { - if err := validateCompositionDocument(document, "document"); err != nil { - return nil, err - } - value, err := canonicalCompositionValue(document.root) - if err != nil { - return nil, err - } - data, err := json.Marshal(value) - if err != nil { - return nil, fmt.Errorf("serialize canonical configuration digest input: %w", err) - } - return append(data, '\n'), nil -} - -// semanticRecords returns sorted atomic values for future effective diff and -// source-report projections. A caller receives copies of all source slices. -func (document *compositionDocument) semanticRecords() ([]compositionValueRecord, error) { - if err := validateCompositionDocument(document, "document"); err != nil { - return nil, err - } - var records []compositionValueRecord - if err := appendCompositionRecords(document.root, &records); err != nil { - return nil, err - } - sort.Slice(records, func(i, j int) bool { return records[i].Path < records[j].Path }) - return records, nil -} - -// compactSemanticRecords returns the same logical atomic paths as -// semanticRecords, but represents values as compact JSON-compatible YAML -// values rather than the typed structural form used for digesting. It is the -// stable human-facing projection for semantic comparisons. -func (document *compositionDocument) compactSemanticRecords() ([]compositionValueRecord, error) { - if err := validateCompositionDocument(document, "document"); err != nil { - return nil, err - } - var records []compositionValueRecord - if err := appendCompactCompositionRecords(document.root, &records); err != nil { - return nil, err - } - sort.Slice(records, func(i, j int) bool { return records[i].Path < records[j].Path }) - return records, nil -} - -func appendCompositionRecords(node *compositionNode, records *[]compositionValueRecord) error { - if node.kind == yaml.MappingNode && len(node.fields) > 0 { - for _, field := range node.fields { - if err := appendCompositionRecords(field.value, records); err != nil { - return err - } - } - return nil - } - value, err := canonicalCompositionValue(node) - if err != nil { - return err - } - encoded, err := json.Marshal(value) - if err != nil { - return fmt.Errorf("serialize configuration value at %s: %w", displayCompositionPath(node.path), err) - } - *records = append(*records, compositionValueRecord{ - Path: node.path, Kind: node.kind, Value: string(encoded), - Sources: append([]string(nil), node.sources...), - }) - return nil -} - -func appendCompactCompositionRecords(node *compositionNode, records *[]compositionValueRecord) error { - if node.kind == yaml.MappingNode && len(node.fields) > 0 { - for _, field := range node.fields { - if err := appendCompactCompositionRecords(field.value, records); err != nil { - return err - } - } - return nil - } - value, err := compactCompositionValue(node) - if err != nil { - return err - } - encoded, err := json.Marshal(value) - if err != nil { - return fmt.Errorf("serialize configuration value at %s: %w", displayCompositionPath(node.path), err) - } - *records = append(*records, compositionValueRecord{ - Path: node.path, Kind: node.kind, Value: string(encoded), - Sources: append([]string(nil), node.sources...), - }) - return nil -} - -func compactCompositionValue(node *compositionNode) (any, error) { - switch node.kind { - case yaml.MappingNode: - values := make(map[string]any, len(node.fields)) - for _, field := range node.fields { - value, err := compactCompositionValue(field.value) - if err != nil { - return nil, err - } - values[field.key] = value - } - return values, nil - case yaml.SequenceNode: - values := make([]any, 0, len(node.items)) - for _, item := range node.items { - value, err := compactCompositionValue(item) - if err != nil { - return nil, err - } - values = append(values, value) - } - return values, nil - case yaml.ScalarNode: - return canonicalScalarValue(node) - default: - return nil, fmt.Errorf("configuration at %s has unsupported YAML kind %s", displayCompositionPath(node.path), yamlKindName(node.kind)) - } -} - -type canonicalCompositionField struct { - Key string `json:"key"` - Value any `json:"value"` -} - -type canonicalCompositionNode struct { - Kind string `json:"kind"` - Tag string `json:"tag,omitempty"` - Value any `json:"value,omitempty"` - Fields []canonicalCompositionField `json:"fields,omitempty"` - Items []any `json:"items,omitempty"` -} - -func canonicalCompositionValue(node *compositionNode) (any, error) { - switch node.kind { - case yaml.MappingNode: - fields := append([]compositionField(nil), node.fields...) - sort.Slice(fields, func(i, j int) bool { return fields[i].key < fields[j].key }) - result := canonicalCompositionNode{Kind: "mapping"} - if len(fields) == 0 { - result.Fields = []canonicalCompositionField{} - } - for _, field := range fields { - value, err := canonicalCompositionValue(field.value) - if err != nil { - return nil, err - } - result.Fields = append(result.Fields, canonicalCompositionField{Key: field.key, Value: value}) - } - return result, nil - case yaml.SequenceNode: - result := canonicalCompositionNode{Kind: "sequence", Items: make([]any, 0, len(node.items))} - for _, item := range node.items { - value, err := canonicalCompositionValue(item) - if err != nil { - return nil, err - } - result.Items = append(result.Items, value) - } - return result, nil - case yaml.ScalarNode: - value, err := canonicalScalarValue(node) - if err != nil { - return nil, err - } - return canonicalCompositionNode{Kind: "scalar", Tag: node.tag, Value: value}, nil - default: - return nil, fmt.Errorf("configuration at %s has unsupported YAML kind %s", displayCompositionPath(node.path), yamlKindName(node.kind)) - } -} - -func canonicalScalarValue(node *compositionNode) (any, error) { - raw := &yaml.Node{Kind: yaml.ScalarNode, Tag: node.tag, Value: node.value} - var value any - if err := raw.Decode(&value); err != nil { - return nil, fmt.Errorf("decode scalar at %s: %w", displayCompositionPath(node.path), err) - } - switch typed := value.(type) { - case nil, bool, string, int, int64, uint64, float64: - return typed, nil - default: - // yaml.v3 may decode timestamps or uncommon scalar tags into types that - // encoding/json can serialize deterministically. Preserve the resolved - // tag alongside the value in the containing canonical node. - return typed, nil - } -} - -func compositionYAMLNode(node *compositionNode) (*yaml.Node, error) { - switch node.kind { - case yaml.MappingNode: - result := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"} - fields := append([]compositionField(nil), node.fields...) - sort.Slice(fields, func(i, j int) bool { return fields[i].key < fields[j].key }) - for _, field := range fields { - value, err := compositionYAMLNode(field.value) - if err != nil { - return nil, err - } - result.Content = append(result.Content, - &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: field.key}, - value, - ) - } - return result, nil - case yaml.SequenceNode: - result := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} - for _, item := range node.items { - value, err := compositionYAMLNode(item) - if err != nil { - return nil, err - } - result.Content = append(result.Content, value) - } - return result, nil - case yaml.ScalarNode: - return normalizedCompositionScalarNode(node) - default: - return nil, fmt.Errorf("configuration at %s has unsupported YAML kind %s", displayCompositionPath(node.path), yamlKindName(node.kind)) - } -} - -func normalizedCompositionScalarNode(node *compositionNode) (*yaml.Node, error) { - raw := &yaml.Node{Kind: yaml.ScalarNode, Tag: node.tag, Value: node.value} - var value any - if err := raw.Decode(&value); err != nil { - return nil, fmt.Errorf("normalize scalar at %s: %w", displayCompositionPath(node.path), err) - } - normalized := &yaml.Node{} - if err := normalized.Encode(value); err != nil { - return nil, fmt.Errorf("encode normalized scalar at %s: %w", displayCompositionPath(node.path), err) - } - if normalized.Kind != yaml.ScalarNode { - return nil, fmt.Errorf("normalize scalar at %s produced YAML kind %s", displayCompositionPath(node.path), yamlKindName(normalized.Kind)) - } - return normalized, nil -} - -func validateCompositionDocument(document *compositionDocument, role string) error { - if document == nil || document.root == nil { - return fmt.Errorf("configuration composition %s document is nil", role) - } - if document.root.kind != yaml.MappingNode { - return fmt.Errorf("configuration composition %s root must be a mapping", role) - } - return nil -} - -func firstNullCompositionNode(node *compositionNode) *compositionNode { - if node == nil { - return nil - } - if node.kind == yaml.ScalarNode && node.tag == "!!null" { - return node - } - for _, field := range node.fields { - if found := firstNullCompositionNode(field.value); found != nil { - return found - } - } - for _, item := range node.items { - if found := firstNullCompositionNode(item); found != nil { - return found - } - } - return nil -} - -func cloneCompositionNode(node *compositionNode) *compositionNode { - if node == nil { - return nil - } - clone := &compositionNode{ - kind: node.kind, tag: node.tag, value: node.value, path: node.path, - sources: append([]string(nil), node.sources...), - } - for _, field := range node.fields { - clone.fields = append(clone.fields, cloneCompositionField(field)) - } - for _, item := range node.items { - clone.items = append(clone.items, cloneCompositionNode(item)) - } - return clone -} - -func cloneCompositionField(field compositionField) compositionField { - return compositionField{key: field.key, value: cloneCompositionNode(field.value)} -} - -func compositionFieldIndex(fields []compositionField, key string) int { - for index := range fields { - if fields[index].key == key { - return index - } - } - return -1 -} - -func appendCompositionPath(parent, key string) string { - if isSimpleCompositionPathSegment(key) { - if parent == "" { - return key - } - return parent + "." + key - } - if parent == "" { - return "[" + strconv.Quote(key) + "]" - } - return parent + "[" + strconv.Quote(key) + "]" -} - -func isSimpleCompositionPathSegment(value string) bool { - if value == "" { - return false - } - for index, char := range value { - if (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || char == '_' || (index > 0 && char >= '0' && char <= '9') || (index > 0 && char == '-') { - continue - } - return false - } - return true -} - -func displayCompositionPath(path string) string { - if path == "" { - return "" - } - return path -} - -func compositionNodeError(source, path string, node *yaml.Node, message string) error { - line, column := 0, 0 - if node != nil { - line, column = node.Line, node.Column - } - return fmt.Errorf( - "configuration source %q at %s (line %d, column %d): %s", - source, displayCompositionPath(path), line, column, message, - ) -} - -func newCompositionConflict(operation, path string, values ...*compositionNode) error { - var sources []string - var kinds []string - for _, value := range values { - if value == nil { - continue - } - sources = appendUniqueStrings(sources, compositionClaimSources(value)...) - kind := yamlKindName(value.kind) - if !containsString(kinds, kind) { - kinds = append(kinds, kind) - } - } - return fmt.Errorf( - "configuration %s conflict at %s: claimed by %s (YAML kinds: %s)", - operation, displayCompositionPath(path), formatCompositionSources(sources), strings.Join(kinds, ", "), - ) -} - -func compositionClaimSources(node *compositionNode) []string { - if node == nil { - return nil - } - sources := append([]string(nil), node.sources...) - for _, field := range node.fields { - sources = appendUniqueStrings(sources, compositionClaimSources(field.value)...) - } - for _, item := range node.items { - sources = appendUniqueStrings(sources, compositionClaimSources(item)...) - } - return sources -} - -func appendUniqueStrings(values []string, additions ...string) []string { - seen := make(map[string]struct{}, len(values)+len(additions)) - result := make([]string, 0, len(values)+len(additions)) - for _, value := range append(append([]string(nil), values...), additions...) { - if _, exists := seen[value]; exists { - continue - } - seen[value] = struct{}{} - result = append(result, value) - } - return result -} - -func containsString(values []string, target string) bool { - for _, value := range values { - if value == target { - return true - } - } - return false -} - -func formatCompositionSources(sources []string) string { - quoted := make([]string, 0, len(sources)) - for _, source := range sources { - quoted = append(quoted, strconv.Quote(source)) - } - return strings.Join(quoted, ", ") -} - -func yamlKindName(kind yaml.Kind) string { - switch kind { - case yaml.DocumentNode: - return "document" - case yaml.MappingNode: - return "mapping" - case yaml.SequenceNode: - return "sequence" - case yaml.ScalarNode: - return "scalar" - case yaml.AliasNode: - return "alias" - default: - return fmt.Sprintf("kind(%d)", kind) - } -} diff --git a/internal/config/composition_test.go b/internal/config/composition_test.go deleted file mode 100644 index 9128e8c..0000000 --- a/internal/config/composition_test.go +++ /dev/null @@ -1,353 +0,0 @@ -package config - -import ( - "reflect" - "strings" - "testing" - - "gopkg.in/yaml.v3" -) - -func TestParseCompositionDocumentRetainsPresenceOwnershipAndDeclarationOrder(t *testing.T) { - document := mustParseComposition(t, "root.yml", `zeta: false -zero: 0 -empty_map: {} -empty_list: [] -nested: - value: "" -`) - - wantOrder := []string{"zeta", "zero", "empty_map", "empty_list", "nested"} - gotOrder := make([]string, 0, len(document.root.fields)) - for _, field := range document.root.fields { - gotOrder = append(gotOrder, field.key) - } - if !reflect.DeepEqual(gotOrder, wantOrder) { - t.Fatalf("declaration order = %#v, want %#v", gotOrder, wantOrder) - } - - tests := []struct { - path string - kind yaml.Kind - tag string - value string - }{ - {path: "zeta", kind: yaml.ScalarNode, tag: "!!bool", value: "false"}, - {path: "zero", kind: yaml.ScalarNode, tag: "!!int", value: "0"}, - {path: "empty_map", kind: yaml.MappingNode, tag: "!!map"}, - {path: "empty_list", kind: yaml.SequenceNode, tag: "!!seq"}, - {path: "nested.value", kind: yaml.ScalarNode, tag: "!!str", value: ""}, - } - for _, tt := range tests { - node := compositionNodeAtPath(t, document.root, tt.path) - if node.kind != tt.kind || node.tag != tt.tag || node.value != tt.value || !reflect.DeepEqual(node.sources, []string{"root.yml"}) { - t.Fatalf("node %s = kind=%v tag=%q value=%q sources=%#v", tt.path, node.kind, node.tag, node.value, node.sources) - } - } -} - -func TestParseCompositionDocumentRejectsAmbiguousOrMalformedYAML(t *testing.T) { - tests := []struct { - name string - yaml string - want string - }{ - {name: "duplicate top-level key", yaml: "value: 1\nvalue: 2\n", want: "duplicate YAML key"}, - {name: "duplicate nested key", yaml: "outer:\n value: 1\n value: 2\n", want: "outer.value"}, - {name: "alias", yaml: "base: &base\n value: 1\ncopy: *base\n", want: "aliases are not supported"}, - {name: "trailing document", yaml: "value: 1\n---\nvalue: 2\n", want: "exactly one YAML document"}, - {name: "top-level sequence", yaml: "- value\n", want: "top-level document must be a mapping"}, - {name: "non-string key", yaml: "1: value\n", want: "mapping keys must be strings"}, - {name: "malformed", yaml: "outer: [\n", want: "decode YAML"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - _, err := parseCompositionDocument("broken.yml", strings.NewReader(tt.yaml)) - if err == nil { - t.Fatal("parseCompositionDocument() error = nil") - } - if !strings.Contains(err.Error(), "broken.yml") || !strings.Contains(err.Error(), tt.want) { - t.Fatalf("error = %q, want source and %q", err, tt.want) - } - }) - } -} - -func TestMergeAdditiveCompositionJoinsOnlyDisjointMappings(t *testing.T) { - root := mustParseComposition(t, "pipeline.yml", `scriptorium: - artifacts: - recap: - enabled: true -zero: 0 -`) - imports := mustParseComposition(t, "conf.d/artifacts.yml", `scriptorium: - artifacts: - handout: - enabled: false -empty_list: [] -`) - - merged, err := mergeAdditiveComposition(root, imports) - if err != nil { - t.Fatalf("mergeAdditiveComposition() error = %v", err) - } - for _, path := range []string{ - "empty_list", "scriptorium.artifacts.handout.enabled", - "scriptorium.artifacts.recap.enabled", "zero", - } { - _ = compositionNodeAtPath(t, merged.root, path) - } - if got := compositionNodeAtPath(t, merged.root, "scriptorium.artifacts.handout.enabled").sources; !reflect.DeepEqual(got, []string{"conf.d/artifacts.yml"}) { - t.Fatalf("handout sources = %#v", got) - } - if got := compositionNodeAtPath(t, merged.root, "scriptorium.artifacts.recap.enabled").sources; !reflect.DeepEqual(got, []string{"pipeline.yml"}) { - t.Fatalf("recap sources = %#v", got) - } - - // Merge operations return a new document and retain the declared order in - // each input for source-aware diagnostics. - if len(root.root.fields) != 2 || len(imports.root.fields) != 2 { - t.Fatalf("merge mutated inputs: root=%d import=%d", len(root.root.fields), len(imports.root.fields)) - } -} - -func TestMergeAdditiveCompositionRejectsEveryDuplicateClass(t *testing.T) { - tests := []struct { - name string - baseYAML string - nextYAML string - path string - }{ - {name: "equal scalar", baseYAML: "value: true\n", nextYAML: "value: true\n", path: "value"}, - {name: "different scalar", baseYAML: "value: true\n", nextYAML: "value: false\n", path: "value"}, - {name: "atomic list", baseYAML: "values: [one]\n", nextYAML: "values: [two]\n", path: "values"}, - {name: "keyed entry", baseYAML: "items:\n shared:\n left: 1\n", nextYAML: "items:\n shared:\n left: 2\n", path: "items.shared.left"}, - {name: "kind conflict", baseYAML: "value:\n nested: true\n", nextYAML: "value: scalar\n", path: "value"}, - {name: "duplicate empty map", baseYAML: "value: {}\n", nextYAML: "value: {}\n", path: "value"}, - {name: "empty map then populated map", baseYAML: "value: {}\n", nextYAML: "value: {nested: true}\n", path: "value"}, - {name: "populated map then empty map", baseYAML: "value: {nested: true}\n", nextYAML: "value: {}\n", path: "value"}, - {name: "duplicate empty list", baseYAML: "value: []\n", nextYAML: "value: []\n", path: "value"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - base := mustParseComposition(t, "base.yml", tt.baseYAML) - next := mustParseComposition(t, "next.yml", tt.nextYAML) - _, err := mergeAdditiveComposition(base, next) - if err == nil { - t.Fatal("mergeAdditiveComposition() error = nil") - } - for _, want := range []string{tt.path, "base.yml", "next.yml"} { - if !strings.Contains(err.Error(), want) { - t.Fatalf("error = %q, want %q", err, want) - } - } - }) - } -} - -func TestMergeAdditiveCompositionReportsAllClaimingSources(t *testing.T) { - left := mustParseComposition(t, "left.yml", "group:\n left: 1\n") - right := mustParseComposition(t, "right.yml", "group:\n right: 2\n") - base, err := mergeAdditiveComposition(left, right) - if err != nil { - t.Fatal(err) - } - overlap := mustParseComposition(t, "overlap.yml", "group: scalar\n") - _, err = mergeAdditiveComposition(base, overlap) - if err == nil { - t.Fatal("mergeAdditiveComposition() error = nil") - } - for _, want := range []string{"group", "left.yml", "right.yml", "overlap.yml"} { - if !strings.Contains(err.Error(), want) { - t.Fatalf("error = %q, want %q", err, want) - } - } - - _, err = mergeAdditiveCompositions( - mustParseComposition(t, "first.yml", "value: 1\n"), - mustParseComposition(t, "second.yml", "value: 2\n"), - mustParseComposition(t, "third.yml", "value: 3\n"), - ) - if err == nil { - t.Fatal("mergeAdditiveCompositions() error = nil") - } - for _, want := range []string{"value", "first.yml", "second.yml", "third.yml"} { - if !strings.Contains(err.Error(), want) { - t.Fatalf("error = %q, want %q", err, want) - } - } -} - -func TestMergeOverlayCompositionRecursesMapsAndReplacesAtomicValues(t *testing.T) { - base := mustParseComposition(t, "base.yml", `feature: - enabled: true - retries: 3 - values: [one, two] - inherited: kept -artifacts: - recap: - enabled: true -`) - overlay := mustParseComposition(t, "testing.yml", `feature: - enabled: false - retries: 0 - values: [] - added: present -artifacts: - handout: - enabled: false -`) - - merged, err := mergeOverlayComposition(base, overlay) - if err != nil { - t.Fatalf("mergeOverlayComposition() error = %v", err) - } - assertCompositionScalar(t, merged.root, "feature.enabled", "!!bool", "false", "testing.yml") - assertCompositionScalar(t, merged.root, "feature.retries", "!!int", "0", "testing.yml") - assertCompositionScalar(t, merged.root, "feature.inherited", "!!str", "kept", "base.yml") - assertCompositionScalar(t, merged.root, "feature.added", "!!str", "present", "testing.yml") - if values := compositionNodeAtPath(t, merged.root, "feature.values"); values.kind != yaml.SequenceNode || len(values.items) != 0 || !reflect.DeepEqual(values.sources, []string{"testing.yml"}) { - t.Fatalf("replaced list = %#v", values) - } - _ = compositionNodeAtPath(t, merged.root, "artifacts.recap.enabled") - _ = compositionNodeAtPath(t, merged.root, "artifacts.handout.enabled") -} - -func TestMergeOverlayCompositionRejectsKindChangesAndNullDeletion(t *testing.T) { - kindTests := []struct { - name string - baseYAML string - overlay string - }{ - {name: "map to scalar", baseYAML: "value: {nested: true}\n", overlay: "value: replacement\n"}, - {name: "scalar to map", baseYAML: "value: original\n", overlay: "value: {nested: true}\n"}, - {name: "list to scalar", baseYAML: "value: [one]\n", overlay: "value: replacement\n"}, - {name: "scalar to list", baseYAML: "value: original\n", overlay: "value: [one]\n"}, - } - for _, tt := range kindTests { - t.Run(tt.name, func(t *testing.T) { - _, err := mergeOverlayComposition( - mustParseComposition(t, "base.yml", tt.baseYAML), - mustParseComposition(t, "overlay.yml", tt.overlay), - ) - if err == nil || !strings.Contains(err.Error(), "value") || !strings.Contains(err.Error(), "kind change") { - t.Fatalf("error = %v, want value kind change", err) - } - }) - } - - for _, overlayYAML := range []string{"value: null\n", "value: ~\n", "nested:\n value:\n"} { - _, err := mergeOverlayComposition( - mustParseComposition(t, "base.yml", "value: original\nnested:\n value: original\n"), - mustParseComposition(t, "overlay.yml", overlayYAML), - ) - if err == nil || !strings.Contains(err.Error(), "null cannot delete") || !strings.Contains(err.Error(), "overlay.yml") { - t.Fatalf("error = %v, want source-qualified null rejection", err) - } - } -} - -func TestCompositionCanonicalOutputsAreDeterministic(t *testing.T) { - first := mustParseComposition(t, "first.yml", `zeta: 01 -alpha: - list: [true, false] - empty: {} -`) - second := mustParseComposition(t, "second.yml", `alpha: - empty: {} - list: - - true - - false -zeta: 1 -`) - - firstYAML, err := first.canonicalYAML() - if err != nil { - t.Fatal(err) - } - secondYAML, err := second.canonicalYAML() - if err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(firstYAML, secondYAML) { - t.Fatalf("canonical YAML differs:\n%s\n---\n%s", firstYAML, secondYAML) - } - firstDigest, err := first.canonicalDigestInput() - if err != nil { - t.Fatal(err) - } - secondDigest, err := second.canonicalDigestInput() - if err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(firstDigest, secondDigest) { - t.Fatalf("digest input differs:\n%s\n---\n%s", firstDigest, secondDigest) - } - - left := mustParseComposition(t, "left.yml", "zeta: 1\n") - right := mustParseComposition(t, "right.yml", "alpha: 2\n") - leftRight, err := mergeAdditiveComposition(left, right) - if err != nil { - t.Fatal(err) - } - rightLeft, err := mergeAdditiveComposition(right, left) - if err != nil { - t.Fatal(err) - } - want, _ := leftRight.canonicalDigestInput() - got, _ := rightLeft.canonicalDigestInput() - if !reflect.DeepEqual(got, want) { - t.Fatalf("source traversal changed semantic digest input: got %s want %s", got, want) - } - wantYAML, _ := leftRight.canonicalYAML() - gotYAML, _ := rightLeft.canonicalYAML() - if !reflect.DeepEqual(gotYAML, wantYAML) { - t.Fatalf("source traversal changed canonical YAML: got %s want %s", gotYAML, wantYAML) - } - - records, err := first.semanticRecords() - if err != nil { - t.Fatal(err) - } - paths := make([]string, 0, len(records)) - for _, record := range records { - paths = append(paths, record.Path) - } - if wantPaths := []string{"alpha.empty", "alpha.list", "zeta"}; !reflect.DeepEqual(paths, wantPaths) { - t.Fatalf("semantic record paths = %#v, want %#v", paths, wantPaths) - } -} - -func mustParseComposition(t *testing.T, source, input string) *compositionDocument { - t.Helper() - document, err := parseCompositionBytes(source, []byte(input)) - if err != nil { - t.Fatalf("parseCompositionBytes(%q) error = %v", source, err) - } - return document -} - -func compositionNodeAtPath(t *testing.T, root *compositionNode, path string) *compositionNode { - t.Helper() - node := root - for _, segment := range strings.Split(path, ".") { - if node == nil || node.kind != yaml.MappingNode { - t.Fatalf("path %q reached non-mapping at %q", path, segment) - } - index := compositionFieldIndex(node.fields, segment) - if index < 0 { - t.Fatalf("path %q missing segment %q", path, segment) - } - node = node.fields[index].value - } - return node -} - -func assertCompositionScalar(t *testing.T, root *compositionNode, path, tag, value, source string) { - t.Helper() - node := compositionNodeAtPath(t, root, path) - if node.kind != yaml.ScalarNode || node.tag != tag || node.value != value || !reflect.DeepEqual(node.sources, []string{source}) { - t.Fatalf("%s = kind=%s tag=%q value=%q sources=%#v", path, yamlKindName(node.kind), node.tag, node.value, node.sources) - } -} diff --git a/internal/config/config.go b/internal/config/config.go index 7e795e0..75f17a6 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -191,8 +191,6 @@ type NormalizeConfig struct { OutputPath string `yaml:"output_path" json:"output_path"` OutputSchema string `yaml:"output_schema" json:"output_schema"` Report *bool `yaml:"report" json:"report"` - - outputPathSet bool `yaml:"-" json:"-"` } // TrimConfig configures trim-stage transcript boundary behavior. diff --git a/internal/config/effective_digest.go b/internal/config/effective_digest.go index ca6f7ed..6f6cc01 100644 --- a/internal/config/effective_digest.go +++ b/internal/config/effective_digest.go @@ -74,7 +74,7 @@ func finalizePipelineResolution(cfg *PipelineConfig) error { // recomputePipelineEffectiveDigest is the single package-owned hook for // refreshing provenance after later resolution expands concrete pipeline -// values. Composition declarations and runtime provenance are not serialized. +// values. Loader directives and runtime provenance are not serialized. func recomputePipelineEffectiveDigest(cfg *PipelineConfig) error { if cfg == nil || cfg.resolution == nil { return fmt.Errorf("pipeline resolution metadata is required") diff --git a/internal/config/effective_pipeline.go b/internal/config/effective_pipeline.go index 1f42f26..85cb196 100644 --- a/internal/config/effective_pipeline.go +++ b/internal/config/effective_pipeline.go @@ -55,7 +55,7 @@ type EffectivePipelineValueRecord struct { // EffectivePipelineValues projects a fully resolved pipeline into sorted // logical configuration values. Mappings are flattened, while sequences remain // atomic values. The projection uses the same normalized effective mapping as -// config show and therefore excludes composition and family declarations. +// config show and therefore excludes loader directives and family declarations. func EffectivePipelineValues(cfg *PipelineConfig) ([]EffectivePipelineValueRecord, error) { if cfg == nil { return nil, fmt.Errorf("pipeline config is required") @@ -301,6 +301,9 @@ func pipelineSourceRole(cfg *PipelineConfig, source string) string { if source == pipelineDefaultOwnershipSource { return "default" } + if source == "resolution" { + return "resolution" + } if cfg == nil || cfg.resolution == nil { return "pipeline" } @@ -319,7 +322,7 @@ func pipelineSourceRole(cfg *PipelineConfig, source string) string { } func normalizedProvenanceSource(source string) string { - if source == pipelineDefaultOwnershipSource { + if source == pipelineDefaultOwnershipSource || source == "resolution" { return source } return normalizedProvenancePath(source) diff --git a/internal/config/effective_projection.go b/internal/config/effective_projection.go index ea6f756..d1f69f3 100644 --- a/internal/config/effective_projection.go +++ b/internal/config/effective_projection.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "sort" + "strings" ) // effectivePipelineProjection is the single safe, schema-specific view used @@ -37,6 +38,11 @@ type effectiveScriptoriumConfig struct { Artifacts map[string]ScriptoriumArtifactConfig `yaml:"artifacts" json:"artifacts"` } +type projectedPipelineValue struct { + record EffectivePipelineValueRecord + segments []string +} + func projectEffectivePipeline(cfg *PipelineConfig, logicalNotarius bool) effectivePipelineProjection { projection := effectivePipelineProjection{ Workspace: cfg.Workspace, Campaigns: cfg.Campaigns, Storage: cfg.Storage, @@ -70,23 +76,35 @@ func projectionJSON(projection effectivePipelineProjection) ([]byte, error) { } func projectionValueRecords(projection effectivePipelineProjection) ([]EffectivePipelineValueRecord, error) { - data, err := projectionJSON(projection) + detailed, err := projectionDetailedRecords(projection) if err != nil { return nil, err } + records := make([]EffectivePipelineValueRecord, 0, len(detailed)) + for _, value := range detailed { + records = append(records, value.record) + } + return records, nil +} + +func projectionDetailedRecords(value any) ([]projectedPipelineValue, error) { + data, err := json.Marshal(value) + if err != nil { + return nil, fmt.Errorf("encode typed pipeline projection: %w", err) + } decoder := json.NewDecoder(bytes.NewReader(data)) decoder.UseNumber() var root map[string]any if err := decoder.Decode(&root); err != nil { return nil, fmt.Errorf("decode effective pipeline projection: %w", err) } - var records []EffectivePipelineValueRecord - appendProjectionRecords(&records, "", root) - sort.Slice(records, func(i, j int) bool { return records[i].Path < records[j].Path }) + var records []projectedPipelineValue + appendProjectionRecords(&records, nil, root) + sort.Slice(records, func(i, j int) bool { return records[i].record.Path < records[j].record.Path }) return records, nil } -func appendProjectionRecords(records *[]EffectivePipelineValueRecord, path string, value any) { +func appendProjectionRecords(records *[]projectedPipelineValue, segments []string, value any) { if mapping, ok := value.(map[string]any); ok && len(mapping) > 0 { keys := make([]string, 0, len(mapping)) for key := range mapping { @@ -94,14 +112,15 @@ func appendProjectionRecords(records *[]EffectivePipelineValueRecord, path strin } sort.Strings(keys) for _, key := range keys { - childPath := key - if path != "" { - childPath = path + "." + key - } - appendProjectionRecords(records, childPath, mapping[key]) + child := append(append([]string(nil), segments...), key) + appendProjectionRecords(records, child, mapping[key]) } return } encoded, _ := json.Marshal(value) - *records = append(*records, EffectivePipelineValueRecord{Path: path, Value: string(encoded)}) + path := strings.Join(segments, ".") + *records = append(*records, projectedPipelineValue{ + record: EffectivePipelineValueRecord{Path: path, Value: string(encoded)}, + segments: append([]string(nil), segments...), + }) } diff --git a/internal/config/load.go b/internal/config/load.go index 14ffc25..6be6eaf 100644 --- a/internal/config/load.go +++ b/internal/config/load.go @@ -17,9 +17,8 @@ func LoadPipeline(path string) (*PipelineConfig, error) { return LoadPipelineWithOptions(path, PipelineLoadOptions{}) } -// PipelineLoadOptions carries an optional explicit profile selection. A nil -// Profile means the caller omitted selection; a non-nil empty value is an -// explicit invalid selection. +// PipelineLoadOptions carries an optional explicit profile selection. Nil, +// empty, and the exact name "default" select the base configuration. type PipelineLoadOptions struct { Profile *string } @@ -27,77 +26,32 @@ type PipelineLoadOptions struct { // LoadPipelineWithOptions loads pipeline configuration with strict field // checking and optional named-profile selection. func LoadPipelineWithOptions(path string, opts PipelineLoadOptions) (*PipelineConfig, error) { - sources, err := loadPipelineCompositionSources(path) + cfg, err := loadPipelineWithConfigKit(path, opts) if err != nil { - return nil, fmt.Errorf("load pipeline config: %w", err) - } - if _, err := selectPipelineProfile(sources.envelope, opts); err != nil { - return nil, fmt.Errorf("load pipeline config: %w", err) - } - if err := sources.loadOverlays(); err != nil { - return nil, fmt.Errorf("load pipeline config: %w", err) - } - cfg, err := sources.resolve(opts) - if err != nil { - return nil, fmt.Errorf("load pipeline config: %w", err) + return nil, err } return finalizeLoadedPipeline(path, cfg) } -// LoadPipelineProfilePair resolves two explicit named profiles from one parsed -// pipeline root and its declared source set. Each result is independently -// decoded, defaulted, and finalized so later resolution can safely mutate one -// effective pipeline without affecting the other. +// LoadPipelineProfilePair resolves two profiles as independent ConfigKit loads. func LoadPipelineProfilePair(path, leftProfile, rightProfile string) (*PipelineConfig, *PipelineConfig, error) { - if _, err := normalizePipelineProfileName(leftProfile, "left profile selection"); err != nil { - return nil, nil, fmt.Errorf("load left pipeline profile: %w", err) - } - if _, err := normalizePipelineProfileName(rightProfile, "right profile selection"); err != nil { - return nil, nil, fmt.Errorf("load right pipeline profile: %w", err) - } if leftProfile == rightProfile { return nil, nil, fmt.Errorf("load pipeline profiles: left and right profile selections must differ") } - sources, err := loadPipelineCompositionSources(path) - if err != nil { - return nil, nil, fmt.Errorf("load pipeline config: %w", err) - } leftOptions := PipelineLoadOptions{Profile: &leftProfile} rightOptions := PipelineLoadOptions{Profile: &rightProfile} - if _, err := selectPipelineProfile(sources.envelope, leftOptions); err != nil { - return nil, nil, fmt.Errorf("load left pipeline profile: %w", err) - } - if _, err := selectPipelineProfile(sources.envelope, rightOptions); err != nil { - return nil, nil, fmt.Errorf("load right pipeline profile: %w", err) - } - if err := sources.loadOverlays(); err != nil { - return nil, nil, fmt.Errorf("load pipeline config: %w", err) - } - left, err := sources.resolve(leftOptions) + left, err := LoadPipelineWithOptions(path, leftOptions) if err != nil { return nil, nil, fmt.Errorf("load left pipeline profile: %w", err) } - right, err := sources.resolve(rightOptions) + right, err := LoadPipelineWithOptions(path, rightOptions) if err != nil { return nil, nil, fmt.Errorf("load right pipeline profile: %w", err) } - left, err = finalizeLoadedPipeline(path, left) - if err != nil { - return nil, nil, err - } - right, err = finalizeLoadedPipeline(path, right) - if err != nil { - return nil, nil, err - } return left, right, nil } func finalizeLoadedPipeline(path string, cfg *PipelineConfig) (*PipelineConfig, error) { - cfg.resolution.publishDeclared = cfg.Publish != nil - applyPipelineDefaults(cfg) - if err := NormalizePipelineConfig(cfg); err != nil { - return nil, fmt.Errorf("load pipeline config: %w", err) - } captureLogicalNotariusPaths(cfg) if err := resolveNotariusPaths(cfg, path); err != nil { return nil, fmt.Errorf("load pipeline config: %w", err) @@ -504,217 +458,6 @@ func shortName(path, fallback string) string { return base } -func applyPipelineDefaults(cfg *PipelineConfig) { - if cfg == nil { - return - } - applyWorkspaceDefaults(&cfg.Workspace) - applyCampaignsDefaults(&cfg.Campaigns) - applyStorageDefaults(&cfg.Storage) - applySpoolDefaults(&cfg.Spool) - applyCacheDefaults(&cfg.Cache) - applyPublishDefaults(&cfg.Publish) - applyWhisperXDefaults(&cfg.WhisperX) - applySeriatimDefaults(&cfg.Seriatim) - applyAuditaDefaults(&cfg.Audita) - if cfg.Normalize == nil { - cfg.Normalize = &NormalizeConfig{} - } - applyNormalizeDefaults(cfg.Normalize) - if cfg.Trim == nil { - cfg.Trim = &TrimConfig{} - } - applyTrimDefaults(cfg.Trim) - if trimEnabled(cfg.Trim) && cfg.Scriptorium == nil { - cfg.Scriptorium = &ScriptoriumConfig{} - } - applyRenderDefaults(&cfg.Render) - applyScriptoriumDefaults(cfg.Scriptorium) - applyNotariusDefaults(cfg.Notarius) - applyNotificationDefaults(&cfg.Notification) -} - -func applyNotificationDefaults(cfg *NotificationConfig) { - if cfg == nil { - return - } - if strings.TrimSpace(cfg.Mode) == "" { - cfg.Mode = DefaultNotificationMode - } -} - -func applyCampaignsDefaults(cfg *CampaignsConfig) { - if cfg == nil { - return - } - if cfg.Root == "" { - cfg.Root = DefaultCampaignsRoot - } -} - -func applyWorkspaceDefaults(cfg *WorkspaceConfig) { - if cfg == nil { - return - } - if cfg.Root == "" { - cfg.Root = DefaultWorkspaceRoot - } -} - -func applyStorageDefaults(cfg *StorageConfig) { - if cfg == nil { - return - } - backend := strings.ToLower(strings.TrimSpace(cfg.Backend)) - if backend == "" { - backend = StorageBackendLocal - } - cfg.Backend = backend - if cfg.S3 == nil { - return - } - if cfg.S3.RootPrefix == "" { - cfg.S3.RootPrefix = DefaultStorageS3RootPrefix - } - if cfg.S3.AccessKeyIDEnv == "" { - cfg.S3.AccessKeyIDEnv = DefaultS3AccessKeyIDEnv - } - if cfg.S3.SecretKeyEnv == "" { - cfg.S3.SecretKeyEnv = DefaultS3SecretAccessKeyEnv - } -} - -func applySpoolDefaults(cfg *SpoolConfig) { - if cfg == nil { - return - } - if cfg.Root == "" { - cfg.Root = DefaultSpoolRoot - } -} - -func applyCacheDefaults(cfg *CacheConfig) { - if cfg == nil { - return - } - if cfg.Root == "" { - cfg.Root = DefaultCacheRoot - } - if cfg.S3Audio == nil { - cfg.S3Audio = boolPtr(DefaultCacheS3Audio) - } -} - -func applyPublishDefaults(cfg **PublishConfig) { - if cfg == nil { - return - } - if *cfg == nil { - *cfg = &PublishConfig{} - } - - if (*cfg).Enabled == nil { - (*cfg).Enabled = boolPtr(DefaultArchiveEnabled) - } - if (*cfg).UploadRun == nil { - (*cfg).UploadRun = boolPtr(DefaultArchiveUploadRun) - } - if len((*cfg).Outputs) == 0 { - (*cfg).Outputs = append([]PublishOutputRule(nil), DefaultPublishOutputs...) - } - for i := range (*cfg).Outputs { - if (*cfg).Outputs[i].Required == nil { - (*cfg).Outputs[i].Required = boolPtr(true) - } - } -} - -func applyWhisperXDefaults(cfg *WhisperXConfig) { - if cfg == nil { - return - } - if cfg.Language == "" { - cfg.Language = DefaultWhisperXLanguage - } - if cfg.Timeout == "" { - cfg.Timeout = DefaultWhisperXTimeout - } - if cfg.RetryDelay == "" { - cfg.RetryDelay = DefaultWhisperXRetryDelay - } - if cfg.Concurrency == nil { - cfg.Concurrency = intPtr(DefaultWhisperXConcurrency) - } - if cfg.Retries == nil { - cfg.Retries = intPtr(DefaultWhisperXRetries) - } -} - -func intPtr(v int) *int { - p := v - return &p -} - -func applySeriatimDefaults(cfg *SeriatimConfig) { - if cfg == nil { - return - } - if cfg.Binary == "" { - cfg.Binary = DefaultSeriatimBinary - } - if cfg.Timeout == "" { - cfg.Timeout = DefaultSeriatimTimeout - } - if cfg.OutputSchema == "" { - cfg.OutputSchema = DefaultSeriatimOutputSchema - } - if cfg.CoalesceGap == nil { - cfg.CoalesceGap = float64Ptr(DefaultSeriatimCoalesceGap) - } - if cfg.Report == nil { - cfg.Report = boolPtr(DefaultSeriatimReport) - } -} - -func applyAuditaDefaults(cfg *AuditaConfig) { - if cfg == nil { - return - } - if cfg.Binary == "" { - cfg.Binary = DefaultAuditaBinary - } - if cfg.Timeout == "" { - cfg.Timeout = DefaultAuditaTimeout - } - if cfg.Report == nil { - cfg.Report = boolPtr(DefaultAuditaReport) - } -} - -func applyScriptoriumDefaults(cfg *ScriptoriumConfig) { - if cfg == nil { - return - } - if cfg.Binary == "" { - cfg.Binary = DefaultScriptoriumBinary - } - if cfg.Timeout == "" { - cfg.Timeout = DefaultScriptoriumTimeout - } -} - -func applyNotariusDefaults(cfg *NotariusConfig) { - if cfg == nil { - return - } - if cfg.Binary == "" { - cfg.Binary = DefaultNotariusBinary - } - if cfg.Timeout == "" { - cfg.Timeout = DefaultNotariusTimeout - } -} - func resolveNotariusPaths(cfg *PipelineConfig, pipelinePath string) error { if cfg == nil || cfg.Notarius == nil || !cfg.Notarius.Enabled { return nil @@ -753,83 +496,3 @@ func resolveNotariusPaths(cfg *PipelineConfig, pipelinePath string) error { cfg.Notarius.WorkingDirectory = workingDirectory return nil } - -func applyTrimDefaults(cfg *TrimConfig) { - if cfg == nil { - return - } - if cfg.Enabled == nil { - cfg.Enabled = boolPtr(DefaultTrimEnabled) - } - if strings.TrimSpace(cfg.OutputPath) == "" { - cfg.OutputPath = DefaultTrimOutputPath - } - if strings.TrimSpace(cfg.Bounds.PromptID) == "" { - cfg.Bounds.PromptID = DefaultTrimBoundsPromptID - } - if strings.TrimSpace(cfg.Bounds.TranscriptInputName) == "" { - cfg.Bounds.TranscriptInputName = DefaultTrimBoundsTranscriptInputName - } - if strings.TrimSpace(cfg.Bounds.OutputPath) == "" { - cfg.Bounds.OutputPath = DefaultTrimBoundsOutputPath - } - if cfg.Bounds.Timeout == "" { - cfg.Bounds.Timeout = DefaultTrimBoundsTimeout - } - if cfg.Seriatim.Report == nil { - cfg.Seriatim.Report = boolPtr(DefaultTrimSeriatimReport) - } -} - -func trimEnabled(cfg *TrimConfig) bool { - return cfg != nil && cfg.Enabled != nil && *cfg.Enabled -} - -func applyRenderDefaults(cfg **RenderConfig) { - if cfg == nil { - return - } - if *cfg == nil { - *cfg = &RenderConfig{} - } - if (*cfg).Enabled == nil { - (*cfg).Enabled = boolPtr(DefaultRenderEnabled) - } - if strings.TrimSpace((*cfg).Format) == "" { - (*cfg).Format = DefaultRenderFormat - } - if strings.TrimSpace((*cfg).Title) == "" { - (*cfg).Title = DefaultRenderTitle - } - if (*cfg).IncludeTimestamps == nil { - (*cfg).IncludeTimestamps = boolPtr(DefaultRenderTimestamps) - } - if (*cfg).IncludeSegmentIDs == nil { - (*cfg).IncludeSegmentIDs = boolPtr(DefaultRenderSegmentIDs) - } -} - -func applyNormalizeDefaults(cfg *NormalizeConfig) { - if cfg == nil { - return - } - if cfg.OutputSchema == "" { - cfg.OutputSchema = DefaultNormalizeOutputSchema - } - if cfg.OutputPath == "" && !cfg.outputPathWasSet() { - cfg.OutputPath = DefaultNormalizeOutputPath - } - if cfg.Report == nil { - cfg.Report = boolPtr(DefaultNormalizeReport) - } -} - -func float64Ptr(v float64) *float64 { - p := v - return &p -} - -func boolPtr(v bool) *bool { - p := v - return &p -} diff --git a/internal/config/loader_strict_test.go b/internal/config/loader_strict_test.go index 6754b8a..2f4d0b2 100644 --- a/internal/config/loader_strict_test.go +++ b/internal/config/loader_strict_test.go @@ -48,7 +48,7 @@ func TestLoadersRejectTrailingYAMLDocuments(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { err := tt.load() - if err == nil || !strings.Contains(err.Error(), "exactly one YAML document") { + if err == nil || (!strings.Contains(err.Error(), "exactly one YAML document") && !strings.Contains(err.Error(), "yaml.multiple_documents")) { t.Fatalf("load error = %v, want exactly-one-document error", err) } }) diff --git a/internal/config/normalize.go b/internal/config/normalize.go deleted file mode 100644 index 3b2dd7e..0000000 --- a/internal/config/normalize.go +++ /dev/null @@ -1,49 +0,0 @@ -package config - -import ( - "fmt" - - "gopkg.in/yaml.v3" -) - -// UnmarshalYAML tracks explicit normalize.output_path presence so validation can -// distinguish omitted vs explicitly empty values. -func (cfg *NormalizeConfig) UnmarshalYAML(node *yaml.Node) error { - if node.Kind != yaml.MappingNode { - return fmt.Errorf("normalize must be a mapping") - } - allowedKeys := map[string]struct{}{ - "output_path": {}, - "output_schema": {}, - "report": {}, - } - for i := 0; i+1 < len(node.Content); i += 2 { - key := node.Content[i].Value - if _, ok := allowedKeys[key]; !ok { - return fmt.Errorf("field %q not found in type config.NormalizeConfig", key) - } - } - - type rawNormalize NormalizeConfig - var raw rawNormalize - if err := node.Decode(&raw); err != nil { - return err - } - *cfg = NormalizeConfig(raw) - for i := 0; i+1 < len(node.Content); i += 2 { - if node.Content[i].Value == "output_path" { - cfg.outputPathSet = true - break - } - } - return nil -} - -// outputPathSet records whether normalize.output_path appeared in YAML. -// This is intentionally unexported and not serialized. -func (cfg *NormalizeConfig) outputPathWasSet() bool { - if cfg == nil { - return false - } - return cfg.outputPathSet -} diff --git a/internal/config/notarius_test.go b/internal/config/notarius_test.go index 486ff29..77eded3 100644 --- a/internal/config/notarius_test.go +++ b/internal/config/notarius_test.go @@ -107,6 +107,12 @@ notarius: enabled: true config_path: notarius/config.yml pipeline_id: dnd-session + outputs: + npc_registry: + lane_id: npc-registry + media_type: application/json + schema_id: notarius.dnd.npc_registry + schema_version: v1 ` paths := make([]string, 2) configs := make([]*PipelineConfig, 2) @@ -151,7 +157,7 @@ func TestNotariusStrictYAML(t *testing.T) { _, err := LoadPipeline(path) want := "strict decode failed" if tt.name == "duplicate reference selector" { - want = "duplicate YAML key" + want = "yaml.duplicate_key" } if err == nil || !strings.Contains(err.Error(), want) { t.Fatalf("LoadPipeline() error = %v, want containing %q", err, want) diff --git a/internal/config/notification_test.go b/internal/config/notification_test.go index c139b1d..8acda1c 100644 --- a/internal/config/notification_test.go +++ b/internal/config/notification_test.go @@ -23,12 +23,12 @@ func TestNotificationConfigSupportsOnlyNoopMode(t *testing.T) { { name: "backend is rejected", section: "notification:\n backend: email\n", - wantLoad: "field backend not found", + wantLoad: "decode.unknown_field", }, { name: "recipient is rejected", section: "notification:\n recipient: party@example.com\n", - wantLoad: "field recipient not found", + wantLoad: "decode.unknown_field", }, { name: "provider mode is rejected", diff --git a/internal/config/pipeline_composition.go b/internal/config/pipeline_composition.go deleted file mode 100644 index a750a2b..0000000 --- a/internal/config/pipeline_composition.go +++ /dev/null @@ -1,532 +0,0 @@ -package config - -import ( - "fmt" - "os" - "path/filepath" - "strings" - "unicode" - - "gitea.maximumdirect.net/eric/narratio/internal/pathsafe" - "gopkg.in/yaml.v3" -) - -type pipelineResolutionMetadata struct { - rootPath string - imports []string - sources []string - selectedProfile *pipelineProfileSelection - effectiveDigest string - logicalNotariusConfig string - logicalNotariusWorking string - logicalNotariusCaptured bool - ownership []pipelineFieldOwnership - artifactFamilies map[string]ScriptoriumArtifactFamilyConfig - artifactFamiliesExpanded bool - publishDeclared bool -} - -type pipelineProfileSelection struct { - name string - source string - overlayPath string -} - -type pipelineFieldOwnership struct { - path string - sources []string -} - -type pipelineCompositionEnvelope struct { - imports []string - defaultProfile *string - profiles []pipelineProfileDeclaration -} - -type pipelineProfileDeclaration struct { - name string - overlay string -} - -// pipelineCompositionSources retains one validated root source set. Each -// selected profile is resolved from a cloned base document so callers can -// safely compare or otherwise resolve multiple profiles without rereading or -// mutating the source set. -type pipelineCompositionSources struct { - rootPath string - envelope pipelineCompositionEnvelope - imports []loadedPipelineImport - overlays []loadedPipelineProfileOverlay - overlaysLoaded bool - base *compositionDocument -} - -func loadPipelineCompositionSources(path string) (*pipelineCompositionSources, error) { - rootPath, err := filepath.Abs(path) - if err != nil { - return nil, fmt.Errorf("resolve root pipeline path %q: %w", path, err) - } - rootFile, err := os.Open(rootPath) - if err != nil { - return nil, fmt.Errorf("pipeline file %q: open: %w", path, err) - } - rootDocument, parseErr := parseCompositionDocument(rootPath, rootFile) - closeErr := rootFile.Close() - if parseErr != nil { - return nil, parseErr - } - if closeErr != nil { - return nil, fmt.Errorf("pipeline file %q: close: %w", rootPath, closeErr) - } - - baseRoot, envelope, err := splitPipelineCompositionEnvelope(rootDocument) - if err != nil { - return nil, err - } - imports, err := loadPipelineImports(rootPath, envelope.imports) - if err != nil { - return nil, err - } - documents := make([]*compositionDocument, 0, len(imports)+1) - documents = append(documents, baseRoot) - for _, imported := range imports { - documents = append(documents, imported.document) - } - merged, err := mergeAdditiveCompositions(documents...) - if err != nil { - return nil, err - } - return &pipelineCompositionSources{ - rootPath: rootPath, - envelope: envelope, - imports: imports, - base: merged, - }, nil -} - -func (sources *pipelineCompositionSources) loadOverlays() error { - if sources == nil { - return fmt.Errorf("pipeline composition sources are required") - } - if sources.overlaysLoaded { - return nil - } - overlays, err := loadPipelineProfileOverlays(sources.rootPath, sources.envelope.profiles, sources.imports) - if err != nil { - return err - } - sources.overlays = overlays - sources.overlaysLoaded = true - return nil -} - -func (sources *pipelineCompositionSources) resolve(opts PipelineLoadOptions) (*PipelineConfig, error) { - if sources == nil || sources.base == nil { - return nil, fmt.Errorf("pipeline composition sources are required") - } - if !sources.overlaysLoaded { - return nil, fmt.Errorf("pipeline profile overlays have not been loaded") - } - selection, err := selectPipelineProfile(sources.envelope, opts) - if err != nil { - return nil, err - } - merged := &compositionDocument{ - root: cloneCompositionNode(sources.base.root), - sources: append([]string(nil), sources.base.sources...), - } - if selection != nil { - overlay, ok := loadedProfileOverlay(sources.overlays, selection.name) - if !ok { - return nil, fmt.Errorf("selected profile %q overlay was not loaded", selection.name) - } - merged, err = mergeOverlayComposition(merged, overlay.document) - if err != nil { - return nil, err - } - selection.overlayPath = overlay.path - } - - rendered, err := merged.canonicalYAML() - if err != nil { - return nil, err - } - var cfg PipelineConfig - if err := decodeStrictYAMLFromReader("pipeline", sources.rootPath, strings.NewReader(string(rendered)), &cfg); err != nil { - return nil, fmt.Errorf("assembled pipeline sources %s: %w", formatCompositionSources(merged.sources), err) - } - records, err := merged.semanticRecords() - if err != nil { - return nil, err - } - metadata := &pipelineResolutionMetadata{ - rootPath: sources.rootPath, - sources: append([]string(nil), merged.sources...), - selectedProfile: selection, - } - for _, imported := range sources.imports { - metadata.imports = append(metadata.imports, imported.path) - } - for _, record := range records { - metadata.ownership = append(metadata.ownership, pipelineFieldOwnership{ - path: record.Path, sources: append([]string(nil), record.Sources...), - }) - } - cfg.resolution = metadata - return &cfg, nil -} - -func splitPipelineCompositionEnvelope(document *compositionDocument) (*compositionDocument, pipelineCompositionEnvelope, error) { - if err := validateCompositionDocument(document, "root pipeline"); err != nil { - return nil, pipelineCompositionEnvelope{}, err - } - envelope, fieldIndex, err := decodePipelineCompositionEnvelope(document.root) - if err != nil { - return nil, pipelineCompositionEnvelope{}, err - } - if err := validatePipelineCompositionEnvelope(envelope); err != nil { - return nil, pipelineCompositionEnvelope{}, err - } - return pipelineBodyWithoutComposition(document, fieldIndex), envelope, nil -} - -func decodePipelineCompositionEnvelope(root *compositionNode) (pipelineCompositionEnvelope, int, error) { - index := compositionFieldIndex(root.fields, "composition") - if index < 0 { - return pipelineCompositionEnvelope{}, -1, nil - } - envelopeNode := root.fields[index].value - if envelopeNode.kind != yaml.MappingNode { - return pipelineCompositionEnvelope{}, -1, fmt.Errorf( - "configuration source %s at composition: expected a mapping, got %s", - formatCompositionSources(envelopeNode.sources), yamlKindName(envelopeNode.kind), - ) - } - - var envelope pipelineCompositionEnvelope - for _, field := range envelopeNode.fields { - switch field.key { - case "imports": - if field.value.kind != yaml.SequenceNode { - return pipelineCompositionEnvelope{}, -1, fmt.Errorf( - "configuration source %s at composition.imports: expected a list, got %s", - formatCompositionSources(field.value.sources), yamlKindName(field.value.kind), - ) - } - for itemIndex, item := range field.value.items { - if item.kind != yaml.ScalarNode || item.tag != "!!str" { - return pipelineCompositionEnvelope{}, -1, fmt.Errorf( - "configuration source %s at composition.imports[%d]: expected a string path", - formatCompositionSources(item.sources), itemIndex, - ) - } - envelope.imports = append(envelope.imports, item.value) - } - case "default_profile": - if field.value.kind != yaml.ScalarNode || field.value.tag != "!!str" { - return pipelineCompositionEnvelope{}, -1, fmt.Errorf( - "configuration source %s at composition.default_profile: expected a string profile name", - formatCompositionSources(field.value.sources), - ) - } - name, err := normalizePipelineProfileName(field.value.value, "composition.default_profile") - if err != nil { - return pipelineCompositionEnvelope{}, -1, err - } - envelope.defaultProfile = &name - case "profiles": - if field.value.kind != yaml.MappingNode { - return pipelineCompositionEnvelope{}, -1, fmt.Errorf( - "configuration source %s at composition.profiles: expected a mapping, got %s", - formatCompositionSources(field.value.sources), yamlKindName(field.value.kind), - ) - } - if len(field.value.fields) == 0 { - return pipelineCompositionEnvelope{}, -1, fmt.Errorf("composition.profiles must declare at least one named profile") - } - for _, profileField := range field.value.fields { - name, err := normalizePipelineProfileName(profileField.key, "composition.profiles profile name") - if err != nil { - return pipelineCompositionEnvelope{}, -1, err - } - profile, err := parsePipelineProfileDeclaration(name, profileField.value) - if err != nil { - return pipelineCompositionEnvelope{}, -1, err - } - envelope.profiles = append(envelope.profiles, profile) - } - default: - return pipelineCompositionEnvelope{}, -1, fmt.Errorf( - "configuration source %s at composition.%s: unknown composition field %q", - formatCompositionSources(field.value.sources), field.key, field.key, - ) - } - } - return envelope, index, nil -} - -func validatePipelineCompositionEnvelope(envelope pipelineCompositionEnvelope) error { - if envelope.defaultProfile != nil && !pipelineProfileDeclared(envelope.profiles, *envelope.defaultProfile) { - return fmt.Errorf( - "composition.default_profile %q does not name a declared profile", *envelope.defaultProfile, - ) - } - return nil -} - -func pipelineBodyWithoutComposition(document *compositionDocument, compositionIndex int) *compositionDocument { - root := cloneCompositionNode(document.root) - if compositionIndex >= 0 { - root.fields = append(root.fields[:compositionIndex], root.fields[compositionIndex+1:]...) - } - return &compositionDocument{root: root, sources: append([]string(nil), document.sources...)} -} - -func parsePipelineProfileDeclaration(name string, node *compositionNode) (pipelineProfileDeclaration, error) { - path := "composition.profiles." + name - if node.kind != yaml.MappingNode { - return pipelineProfileDeclaration{}, fmt.Errorf( - "configuration source %s at %s: expected a mapping, got %s", - formatCompositionSources(node.sources), path, yamlKindName(node.kind), - ) - } - profile := pipelineProfileDeclaration{name: name} - for _, field := range node.fields { - if field.key != "overlay" { - return pipelineProfileDeclaration{}, fmt.Errorf( - "configuration source %s at %s.%s: unknown profile field %q; only overlay is supported", - formatCompositionSources(field.value.sources), path, field.key, field.key, - ) - } - if field.value.kind != yaml.ScalarNode || field.value.tag != "!!str" { - return pipelineProfileDeclaration{}, fmt.Errorf( - "configuration source %s at %s.overlay: expected a string path", - formatCompositionSources(field.value.sources), path, - ) - } - profile.overlay = field.value.value - } - if profile.overlay == "" { - return pipelineProfileDeclaration{}, fmt.Errorf("%s.overlay is required", path) - } - return profile, nil -} - -func normalizePipelineProfileName(value, label string) (string, error) { - if value == "" || strings.TrimSpace(value) != value { - return "", fmt.Errorf("%s must be non-empty without surrounding whitespace", label) - } - for _, character := range value { - if unicode.IsControl(character) { - return "", fmt.Errorf("%s %q must not contain control characters", label, value) - } - } - return value, nil -} - -func pipelineProfileDeclared(profiles []pipelineProfileDeclaration, name string) bool { - for _, profile := range profiles { - if profile.name == name { - return true - } - } - return false -} - -func selectPipelineProfile(envelope pipelineCompositionEnvelope, opts PipelineLoadOptions) (*pipelineProfileSelection, error) { - if opts.Profile != nil { - name, err := normalizePipelineProfileName(*opts.Profile, "explicit profile selection") - if err != nil { - return nil, err - } - if len(envelope.profiles) == 0 { - return nil, fmt.Errorf("explicit profile %q was selected but the pipeline declares no profiles", name) - } - if !pipelineProfileDeclared(envelope.profiles, name) { - return nil, fmt.Errorf("explicit profile %q is not declared by the pipeline", name) - } - return &pipelineProfileSelection{name: name, source: "cli"}, nil - } - if len(envelope.profiles) == 0 { - return nil, nil - } - if envelope.defaultProfile == nil { - return nil, fmt.Errorf("pipeline declares profiles but composition.default_profile is omitted and no profile was explicitly selected") - } - return &pipelineProfileSelection{name: *envelope.defaultProfile, source: "default"}, nil -} - -type loadedPipelineImport struct { - path string - document *compositionDocument - info os.FileInfo -} - -type loadedPipelineProfileOverlay struct { - profile string - path string - document *compositionDocument - info os.FileInfo -} - -func loadPipelineProfileOverlays( - rootPath string, - declared []pipelineProfileDeclaration, - imports []loadedPipelineImport, -) ([]loadedPipelineProfileOverlay, error) { - if len(declared) == 0 { - return nil, nil - } - rootDir := filepath.Dir(rootPath) - rootInfo, err := os.Stat(rootPath) - if err != nil { - return nil, fmt.Errorf("inspect root pipeline file %q: %w", rootPath, err) - } - seenPaths := make(map[string]string, len(declared)) - loaded := make([]loadedPipelineProfileOverlay, 0, len(declared)) - for _, profile := range declared { - label := "composition.profiles." + profile.name + ".overlay" - raw := profile.overlay - if strings.TrimSpace(raw) != raw || raw == "" { - return nil, fmt.Errorf("%s must be a non-empty path without surrounding whitespace", label) - } - normalized, err := pathsafe.NormalizeRelativeDestination(raw) - if err != nil { - return nil, fmt.Errorf("%s path %q is invalid: %w", label, raw, err) - } - extension := filepath.Ext(filepath.FromSlash(normalized)) - if extension != ".yml" && extension != ".yaml" { - return nil, fmt.Errorf("%s path %q must use .yml or .yaml", label, raw) - } - if prior, duplicate := seenPaths[normalized]; duplicate { - return nil, fmt.Errorf("%s path %q duplicates profile %q overlay after normalization", label, raw, prior) - } - seenPaths[normalized] = profile.name - resolved := filepath.Join(rootDir, filepath.FromSlash(normalized)) - file, err := os.Open(resolved) - if err != nil { - return nil, fmt.Errorf("open %s path %q beneath root pipeline directory: %w", label, raw, err) - } - info, statErr := file.Stat() - if statErr != nil { - _ = file.Close() - return nil, fmt.Errorf("inspect %s path %q: %w", label, raw, statErr) - } - if os.SameFile(rootInfo, info) { - _ = file.Close() - return nil, fmt.Errorf("%s path %q references the root pipeline itself", label, raw) - } - for _, imported := range imports { - if os.SameFile(imported.info, info) { - _ = file.Close() - return nil, fmt.Errorf("%s path %q references the same file as imported source %q", label, raw, imported.path) - } - } - for _, prior := range loaded { - if os.SameFile(prior.info, info) { - _ = file.Close() - return nil, fmt.Errorf("%s path %q references the same file as profile %q overlay", label, raw, prior.profile) - } - } - document, parseErr := parseCompositionDocument(resolved, file) - closeErr := file.Close() - if parseErr != nil { - return nil, parseErr - } - if closeErr != nil { - return nil, fmt.Errorf("close %s path %q: %w", label, raw, closeErr) - } - if compositionFieldIndex(document.root.fields, "composition") >= 0 { - return nil, fmt.Errorf("profile overlay source %q declares composition; only the root pipeline may declare composition", resolved) - } - loaded = append(loaded, loadedPipelineProfileOverlay{ - profile: profile.name, path: resolved, document: document, info: info, - }) - } - return loaded, nil -} - -func loadedProfileOverlay(overlays []loadedPipelineProfileOverlay, name string) (loadedPipelineProfileOverlay, bool) { - for _, overlay := range overlays { - if overlay.profile == name { - return overlay, true - } - } - return loadedPipelineProfileOverlay{}, false -} - -func loadPipelineImports(rootPath string, declared []string) ([]loadedPipelineImport, error) { - if len(declared) == 0 { - return nil, nil - } - rootDir := filepath.Dir(rootPath) - rootInfo, err := os.Stat(rootPath) - if err != nil { - return nil, fmt.Errorf("inspect root pipeline file %q: %w", rootPath, err) - } - seenPaths := make(map[string]int, len(declared)) - loaded := make([]loadedPipelineImport, 0, len(declared)) - for index, raw := range declared { - if strings.TrimSpace(raw) != raw || raw == "" { - return nil, fmt.Errorf("composition.imports[%d] must be a non-empty path without surrounding whitespace", index) - } - normalized, err := pathsafe.NormalizeRelativeDestination(raw) - if err != nil { - return nil, fmt.Errorf("composition.imports[%d] path %q is invalid: %w", index, raw, err) - } - extension := filepath.Ext(filepath.FromSlash(normalized)) - if extension != ".yml" && extension != ".yaml" { - return nil, fmt.Errorf("composition.imports[%d] path %q must use .yml or .yaml", index, raw) - } - if prior, duplicate := seenPaths[normalized]; duplicate { - return nil, fmt.Errorf( - "composition.imports[%d] path %q duplicates composition.imports[%d] after normalization", - index, raw, prior, - ) - } - seenPaths[normalized] = index - resolved := filepath.Join(rootDir, filepath.FromSlash(normalized)) - if resolved == rootPath { - return nil, fmt.Errorf("composition.imports[%d] path %q imports the root pipeline itself", index, raw) - } - - file, err := os.Open(resolved) - if err != nil { - return nil, fmt.Errorf("open composition.imports[%d] path %q beneath root pipeline directory: %w", index, raw, err) - } - info, statErr := file.Stat() - if statErr != nil { - _ = file.Close() - return nil, fmt.Errorf("inspect composition.imports[%d] path %q: %w", index, raw, statErr) - } - if os.SameFile(rootInfo, info) { - _ = file.Close() - return nil, fmt.Errorf("composition.imports[%d] path %q imports the root pipeline itself", index, raw) - } - for priorIndex, prior := range loaded { - if os.SameFile(prior.info, info) { - _ = file.Close() - return nil, fmt.Errorf( - "composition.imports[%d] path %q references the same file as composition.imports[%d] %q", - index, raw, priorIndex, declared[priorIndex], - ) - } - } - document, parseErr := parseCompositionDocument(resolved, file) - closeErr := file.Close() - if parseErr != nil { - return nil, parseErr - } - if closeErr != nil { - return nil, fmt.Errorf("close composition.imports[%d] path %q: %w", index, raw, closeErr) - } - if compositionFieldIndex(document.root.fields, "composition") >= 0 { - return nil, fmt.Errorf( - "imported configuration source %q declares composition; only the root pipeline may declare composition", - resolved, - ) - } - loaded = append(loaded, loadedPipelineImport{path: resolved, document: document, info: info}) - } - return loaded, nil -} diff --git a/internal/config/pipeline_composition_test.go b/internal/config/pipeline_composition_test.go deleted file mode 100644 index 31257f4..0000000 --- a/internal/config/pipeline_composition_test.go +++ /dev/null @@ -1,431 +0,0 @@ -package config - -import ( - "os" - "path/filepath" - "reflect" - "slices" - "strings" - "testing" -) - -func TestSplitPipelineCompositionEnvelopePreservesInputAndDeclarationOrder(t *testing.T) { - document, err := parseCompositionBytes("pipeline.yml", []byte(`composition: - imports: [first.yml, second.yml] - default_profile: production - profiles: - production: - overlay: production.yml - testing: - overlay: testing.yml -workspace: - root: /srv/narratio -`)) - if err != nil { - t.Fatal(err) - } - originalRoot := cloneCompositionNode(document.root) - originalSources := append([]string(nil), document.sources...) - - body, envelope, err := splitPipelineCompositionEnvelope(document) - if err != nil { - t.Fatalf("splitPipelineCompositionEnvelope() error = %v", err) - } - if !reflect.DeepEqual(document.root, originalRoot) || !slices.Equal(document.sources, originalSources) { - t.Fatalf("splitPipelineCompositionEnvelope() mutated its input") - } - if compositionFieldIndex(body.root.fields, "composition") >= 0 { - t.Fatalf("pipeline body still contains composition envelope") - } - if compositionFieldIndex(body.root.fields, "workspace") < 0 { - t.Fatalf("pipeline body lost ordinary root fields") - } - if want := []string{"first.yml", "second.yml"}; !slices.Equal(envelope.imports, want) { - t.Fatalf("imports = %#v, want %#v", envelope.imports, want) - } - if len(envelope.profiles) != 2 || envelope.profiles[0].name != "production" || envelope.profiles[1].name != "testing" { - t.Fatalf("profiles = %#v, want declaration order", envelope.profiles) - } -} - -func TestSplitPipelineCompositionEnvelopeRejectsMalformedShapesWithoutMutation(t *testing.T) { - tests := []struct { - name string - yaml string - want string - }{ - {name: "envelope", yaml: "composition: []\n", want: "expected a mapping"}, - {name: "imports mapping", yaml: "composition:\n imports: {}\n", want: "expected a list"}, - {name: "import value", yaml: "composition:\n imports: [1]\n", want: "expected a string path"}, - {name: "default profile", yaml: "composition:\n default_profile: {}\n", want: "expected a string profile name"}, - {name: "profiles sequence", yaml: "composition:\n profiles: []\n", want: "expected a mapping"}, - {name: "empty profiles", yaml: "composition:\n profiles: {}\n", want: "at least one named profile"}, - {name: "profile declaration", yaml: "composition:\n profiles:\n production: production.yml\n", want: "expected a mapping"}, - {name: "invalid profile name", yaml: "composition:\n profiles:\n ' production ':\n overlay: production.yml\n", want: "surrounding whitespace"}, - {name: "unknown field", yaml: "composition:\n unexpected: true\n", want: "unknown composition field"}, - {name: "unknown default", yaml: "composition:\n default_profile: testing\n profiles:\n production:\n overlay: production.yml\n", want: "does not name a declared profile"}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - document, err := parseCompositionBytes("pipeline.yml", []byte(test.yaml)) - if err != nil { - t.Fatal(err) - } - originalRoot := cloneCompositionNode(document.root) - - _, _, err = splitPipelineCompositionEnvelope(document) - if err == nil || !strings.Contains(err.Error(), test.want) { - t.Fatalf("splitPipelineCompositionEnvelope() error = %v, want containing %q", err, test.want) - } - if !reflect.DeepEqual(document.root, originalRoot) { - t.Fatalf("splitPipelineCompositionEnvelope() mutated invalid input") - } - }) - } -} - -func TestLoadPipelineCompositionImportsDisjointFieldsAndTracksOwnership(t *testing.T) { - dir := t.TempDir() - rootPath := writePipelineSource(t, dir, "pipeline.yml", `composition: - imports: - - conf.d/platform.yml - - conf.d/artifacts.yml -whisperx: - transcribe_url: https://transcription.example.com/transcribe -`) - platformPath := writePipelineSource(t, dir, "conf.d/platform.yml", `workspace: - root: /srv/narratio/work -storage: - backend: local -`) - artifactsPath := writePipelineSource(t, dir, "conf.d/artifacts.yml", `scriptorium: - artifacts: - player_handout: - enabled: false - session_recap: - enabled: true - prompt_id: dnd.session_recap - output_path: artifacts/session_recap.md -`) - - cfg, err := LoadPipeline(rootPath) - if err != nil { - t.Fatalf("LoadPipeline() error = %v", err) - } - if cfg.Workspace.Root != "/srv/narratio/work" || cfg.Storage.Backend != StorageBackendLocal { - t.Fatalf("imported platform config = workspace=%q storage=%#v", cfg.Workspace.Root, cfg.Storage) - } - if cfg.Scriptorium == nil || len(cfg.Scriptorium.Artifacts) != 2 || cfg.Scriptorium.Artifacts["player_handout"].Enabled { - t.Fatalf("imported artifacts = %#v", cfg.Scriptorium) - } - if cfg.WhisperX.TranscribeURL != "https://transcription.example.com/transcribe" { - t.Fatalf("root field = %q", cfg.WhisperX.TranscribeURL) - } - if cfg.resolution == nil { - t.Fatal("pipeline resolution metadata = nil") - } - wantSources := []string{absolutePath(t, rootPath), absolutePath(t, platformPath), absolutePath(t, artifactsPath)} - if !slices.Equal(cfg.resolution.sources, wantSources) || !slices.Equal(cfg.resolution.imports, wantSources[1:]) { - t.Fatalf("resolution sources=%#v imports=%#v, want %#v / %#v", cfg.resolution.sources, cfg.resolution.imports, wantSources, wantSources[1:]) - } - assertPipelineFieldOwner(t, cfg, "whisperx.transcribe_url", absolutePath(t, rootPath)) - assertPipelineFieldOwner(t, cfg, "workspace.root", absolutePath(t, platformPath)) - assertPipelineFieldOwner(t, cfg, "scriptorium.artifacts.session_recap.prompt_id", absolutePath(t, artifactsPath)) -} - -func TestLoadPipelineCompositionMergesDisjointKeyedEntries(t *testing.T) { - dir := t.TempDir() - rootPath := writePipelineSource(t, dir, "pipeline.yml", `composition: - imports: [first.yml, second.yml] -whisperx: - transcribe_url: https://transcription.example.com/transcribe -scriptorium: - artifacts: - root_artifact: - enabled: false -`) - writePipelineSource(t, dir, "first.yml", `scriptorium: - artifacts: - first_artifact: - enabled: false -`) - writePipelineSource(t, dir, "second.yml", `scriptorium: - artifacts: - second_artifact: - enabled: false -`) - - cfg, err := LoadPipeline(rootPath) - if err != nil { - t.Fatal(err) - } - if got := len(cfg.Scriptorium.Artifacts); got != 3 { - t.Fatalf("artifact count = %d, want 3: %#v", got, cfg.Scriptorium.Artifacts) - } -} - -func TestLoadPipelineCompositionRejectsBaseConflictsWithAllSources(t *testing.T) { - tests := []struct { - name string - root string - imports map[string]string - path string - sources []string - }{ - { - name: "root and import identical scalar", - root: "whisperx:\n language: en\n", - imports: map[string]string{"one.yml": "whisperx:\n language: en\n"}, - path: "whisperx.language", sources: []string{"pipeline.yml", "one.yml"}, - }, - { - name: "all import claimants", - imports: map[string]string{ - "one.yml": "workspace:\n root: /one\n", - "two.yml": "workspace:\n root: /two\n", - "three.yml": "workspace:\n root: /three\n", - }, - path: "workspace.root", sources: []string{"one.yml", "two.yml", "three.yml"}, - }, - { - name: "atomic list", - root: "audita:\n modules: [one]\n", - imports: map[string]string{"one.yml": "audita:\n modules: [two]\n"}, - path: "audita.modules", sources: []string{"pipeline.yml", "one.yml"}, - }, - { - name: "kind conflict", - root: "workspace:\n root: /work\n", - imports: map[string]string{"one.yml": "workspace: invalid\n"}, - path: "workspace", sources: []string{"pipeline.yml", "one.yml"}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - dir := t.TempDir() - order := make([]string, 0, len(tt.imports)) - for _, name := range []string{"one.yml", "two.yml", "three.yml"} { - if _, ok := tt.imports[name]; ok { - order = append(order, name) - } - } - root := "composition:\n imports:\n" - for _, name := range order { - root += " - " + name + "\n" - } - root += tt.root - rootPath := writePipelineSource(t, dir, "pipeline.yml", root) - for name, content := range tt.imports { - writePipelineSource(t, dir, name, content) - } - - _, err := LoadPipeline(rootPath) - if err == nil { - t.Fatal("LoadPipeline() error = nil") - } - if !strings.Contains(err.Error(), tt.path) { - t.Fatalf("error = %q, want path %q", err, tt.path) - } - for _, source := range tt.sources { - if !strings.Contains(err.Error(), source) { - t.Fatalf("error = %q, want source %q", err, source) - } - } - }) - } -} - -func TestLoadPipelineCompositionRejectsUnsafeOrInvalidImports(t *testing.T) { - tests := []struct { - name string - imports []string - setup func(*testing.T, string) - want string - }{ - {name: "empty", imports: []string{""}, want: "non-empty path"}, - {name: "surrounding whitespace", imports: []string{" one.yml "}, want: "surrounding whitespace"}, - {name: "absolute", imports: []string{"/tmp/one.yml"}, want: "invalid"}, - {name: "traversal", imports: []string{"../one.yml"}, want: "invalid"}, - {name: "unsupported extension", imports: []string{"one.json"}, want: ".yml or .yaml"}, - {name: "missing", imports: []string{"missing.yml"}, want: "open composition.imports"}, - {name: "duplicate normalized", imports: []string{"one.yml", "./one.yml"}, setup: func(t *testing.T, dir string) { - writePipelineSource(t, dir, "one.yml", "workspace:\n root: /work\n") - }, want: "duplicates composition.imports"}, - {name: "root self import", imports: []string{"pipeline.yml"}, want: "root pipeline itself"}, - {name: "directory", imports: []string{"directory.yml"}, setup: func(t *testing.T, dir string) { - if err := os.Mkdir(filepath.Join(dir, "directory.yml"), 0o755); err != nil { - t.Fatal(err) - } - }, want: "is a directory"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - dir := t.TempDir() - if tt.setup != nil { - tt.setup(t, dir) - } - rootPath := writeImportRoot(t, dir, tt.imports) - _, err := LoadPipeline(rootPath) - if err == nil || !strings.Contains(strings.ToLower(err.Error()), strings.ToLower(tt.want)) { - t.Fatalf("LoadPipeline() error = %v, want containing %q", err, tt.want) - } - }) - } -} - -func TestLoadPipelineCompositionRejectsSameFileAliasesAndImportedComposition(t *testing.T) { - t.Run("same file through hard link", func(t *testing.T) { - dir := t.TempDir() - writePipelineSource(t, dir, "one.yml", "workspace:\n root: /work\n") - if err := os.Link(filepath.Join(dir, "one.yml"), filepath.Join(dir, "two.yml")); err != nil { - t.Skipf("hard links unavailable: %v", err) - } - rootPath := writeImportRoot(t, dir, []string{"one.yml", "two.yml"}) - _, err := LoadPipeline(rootPath) - if err == nil || !strings.Contains(err.Error(), "same file") { - t.Fatalf("LoadPipeline() error = %v, want same-file rejection", err) - } - }) - - t.Run("imported composition", func(t *testing.T) { - dir := t.TempDir() - rootPath := writeImportRoot(t, dir, []string{"nested.yml"}) - writePipelineSource(t, dir, "nested.yml", "composition:\n imports: []\n") - _, err := LoadPipeline(rootPath) - if err == nil || !strings.Contains(err.Error(), "only the root pipeline") || !strings.Contains(err.Error(), "nested.yml") { - t.Fatalf("LoadPipeline() error = %v, want imported composition rejection", err) - } - }) - - t.Run("empty profiles", func(t *testing.T) { - dir := t.TempDir() - rootPath := writePipelineSource(t, dir, "pipeline.yml", "composition:\n profiles: {}\n") - _, err := LoadPipeline(rootPath) - if err == nil || !strings.Contains(err.Error(), "must declare at least one named profile") { - t.Fatalf("LoadPipeline() error = %v, want empty profiles rejection", err) - } - }) -} - -func TestLoadPipelineCompositionReportsImportedParseAndSchemaSources(t *testing.T) { - t.Run("malformed imported YAML", func(t *testing.T) { - dir := t.TempDir() - rootPath := writeImportRoot(t, dir, []string{"broken.yml"}) - brokenPath := writePipelineSource(t, dir, "broken.yml", "workspace: [\n") - _, err := LoadPipeline(rootPath) - if err == nil || !strings.Contains(err.Error(), absolutePath(t, brokenPath)) || !strings.Contains(err.Error(), "decode YAML") { - t.Fatalf("LoadPipeline() error = %v, want imported parse source", err) - } - }) - - t.Run("unknown imported field", func(t *testing.T) { - dir := t.TempDir() - rootPath := writeImportRoot(t, dir, []string{"unknown.yml"}) - unknownPath := writePipelineSource(t, dir, "unknown.yml", "unknown_field: true\n") - _, err := LoadPipeline(rootPath) - if err == nil || !strings.Contains(err.Error(), absolutePath(t, unknownPath)) || !strings.Contains(err.Error(), "strict decode failed") { - t.Fatalf("LoadPipeline() error = %v, want assembled source-aware strict error", err) - } - }) -} - -func TestLoadPipelineCompositionKeepsRelativePathsRootBased(t *testing.T) { - rootDir := t.TempDir() - monolithicPath := writePipelineSource(t, rootDir, "monolithic.yml", testPipelineBaseYAML+` -notarius: - enabled: true - config_path: tool/notarius.yml - pipeline_id: dnd-session - outputs: - npc_registry: - lane_id: npc-registry - media_type: application/json - schema_id: notarius.dnd.npc_registry - schema_version: v1 -`) - composedPath := writePipelineSource(t, rootDir, "pipeline.yml", `composition: - imports: [conf.d/extraction.yml] -`+testPipelineBaseYAML) - writePipelineSource(t, rootDir, "conf.d/extraction.yml", `notarius: - enabled: true - config_path: tool/notarius.yml - pipeline_id: dnd-session - outputs: - npc_registry: - lane_id: npc-registry - media_type: application/json - schema_id: notarius.dnd.npc_registry - schema_version: v1 -`) - - monolithic, err := LoadPipeline(monolithicPath) - if err != nil { - t.Fatal(err) - } - composed, err := LoadPipeline(composedPath) - if err != nil { - t.Fatal(err) - } - want := filepath.Join(rootDir, "tool", "notarius.yml") - if monolithic.Notarius.ConfigPath != want || composed.Notarius.ConfigPath != want { - t.Fatalf("config paths = monolithic %q composed %q, want %q", monolithic.Notarius.ConfigPath, composed.Notarius.ConfigPath, want) - } - if monolithic.Notarius.WorkingDirectory != filepath.Dir(want) || composed.Notarius.WorkingDirectory != filepath.Dir(want) { - t.Fatalf("working directories = %q / %q", monolithic.Notarius.WorkingDirectory, composed.Notarius.WorkingDirectory) - } -} - -func writeImportRoot(t *testing.T, dir string, imports []string) string { - t.Helper() - var builder strings.Builder - builder.WriteString("composition:\n imports:\n") - for _, imported := range imports { - builder.WriteString(" - ") - if imported == "" { - builder.WriteString(`""`) - } else { - builder.WriteString(`"` + imported + `"`) - } - builder.WriteByte('\n') - } - builder.WriteString("whisperx:\n transcribe_url: https://transcription.example.com/transcribe\n") - return writePipelineSource(t, dir, "pipeline.yml", builder.String()) -} - -func writePipelineSource(t *testing.T, root, relative, content string) string { - t.Helper() - path := filepath.Join(root, filepath.FromSlash(relative)) - if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(path, []byte(content), 0o644); err != nil { - t.Fatal(err) - } - return path -} - -func absolutePath(t *testing.T, path string) string { - t.Helper() - absolute, err := filepath.Abs(path) - if err != nil { - t.Fatal(err) - } - return absolute -} - -func assertPipelineFieldOwner(t *testing.T, cfg *PipelineConfig, path, source string) { - t.Helper() - if cfg == nil || cfg.resolution == nil { - t.Fatal("pipeline resolution metadata is absent") - } - for _, ownership := range cfg.resolution.ownership { - if ownership.path == path { - if !slices.Equal(ownership.sources, []string{source}) { - t.Fatalf("owner of %s = %#v, want %q", path, ownership.sources, source) - } - return - } - } - t.Fatalf("ownership path %q not found: %#v", path, cfg.resolution.ownership) -} diff --git a/internal/config/pipeline_configkit.go b/internal/config/pipeline_configkit.go new file mode 100644 index 0000000..d688af0 --- /dev/null +++ b/internal/config/pipeline_configkit.go @@ -0,0 +1,391 @@ +package config + +import ( + "bytes" + "errors" + "fmt" + "path/filepath" + "reflect" + "sort" + "strings" + + configkit "gitea.maximumdirect.net/eric/ConfigKit" + "gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy" +) + +type pipelineResolutionMetadata struct { + rootPath string + imports []string + sources []string + selectedProfile *pipelineProfileSelection + effectiveDigest string + logicalNotariusConfig string + logicalNotariusWorking string + logicalNotariusCaptured bool + ownership []pipelineFieldOwnership + artifactFamilies map[string]ScriptoriumArtifactFamilyConfig + artifactFamiliesExpanded bool + publishDeclared bool + configkitMetadata configkit.Metadata + configkitSources []configkit.SourceInfo +} + +type pipelineProfileSelection struct { + name string + source string + overlayPath string +} + +type pipelineFieldOwnership struct { + path string + sources []string +} + +func loadPipelineWithConfigKit(path string, opts PipelineLoadOptions) (*PipelineConfig, error) { + profile := selectedPipelineProfileName(opts) + result, err := configkit.Load(configkit.File(path), configkit.Options[PipelineConfig]{ + Imports: true, + Profile: profile, + Definition: configkit.Definition[PipelineConfig]{ + Defaults: pipelineConfigDefaults, + Resolve: resolvePipelineConfig, + Validate: validatePipelineConfigKit, + }, + }) + if err != nil { + return nil, describePipelineLoadError(err) + } + cfg := result.Config + metadata, err := pipelineMetadataFromConfigKit(&cfg, result, opts) + if err != nil { + return nil, fmt.Errorf("load pipeline config: %w", err) + } + cfg.resolution = metadata + return &cfg, nil +} + +func describePipelineLoadError(err error) error { + var loadError *configkit.LoadError + if !errors.As(err, &loadError) { + return fmt.Errorf("load pipeline config: %w", err) + } + var diagnostics bytes.Buffer + resolved := loadError.Diagnostics() + sourcePaths := make(map[string]string, len(loadError.Sources())) + for _, source := range loadError.Sources() { + if source.Path != "" { + sourcePaths[source.ID] = source.Path + } else if source.Label != "" { + sourcePaths[source.ID] = source.Label + } + } + for index := range resolved { + if resolved[index].Primary != nil { + if name := sourcePaths[resolved[index].Primary.SourceID]; name != "" { + resolved[index].Primary.SourceID = name + } + } + for relatedIndex := range resolved[index].Related { + if name := sourcePaths[resolved[index].Related[relatedIndex].SourceID]; name != "" { + resolved[index].Related[relatedIndex].SourceID = name + } + } + } + if writeErr := configkit.WriteDiagnostics(&diagnostics, resolved); writeErr != nil { + return fmt.Errorf("load pipeline config: %w", err) + } + return fmt.Errorf("load pipeline config: strict decode failed or validation failed: %w: %s", err, strings.TrimSpace(diagnostics.String())) +} + +func selectedPipelineProfileName(opts PipelineLoadOptions) string { + if opts.Profile == nil || *opts.Profile == "" || *opts.Profile == "default" { + return "" + } + return *opts.Profile +} + +func pipelineConfigDefaults() PipelineConfig { + return PipelineConfig{ + Workspace: WorkspaceConfig{Root: DefaultWorkspaceRoot}, + Campaigns: CampaignsConfig{Root: DefaultCampaignsRoot}, + Storage: StorageConfig{Backend: StorageBackendLocal}, + Spool: SpoolConfig{Root: DefaultSpoolRoot}, + Cache: CacheConfig{Root: DefaultCacheRoot, S3Audio: boolPtr(DefaultCacheS3Audio)}, + Publish: &PublishConfig{ + Enabled: boolPtr(DefaultArchiveEnabled), UploadRun: boolPtr(DefaultArchiveUploadRun), + Outputs: defaultPublishOutputs(), + }, + WhisperX: WhisperXConfig{ + Language: DefaultWhisperXLanguage, Timeout: DefaultWhisperXTimeout, + RetryDelay: DefaultWhisperXRetryDelay, Concurrency: intPtr(DefaultWhisperXConcurrency), + Retries: intPtr(DefaultWhisperXRetries), + }, + Seriatim: SeriatimConfig{ + Binary: DefaultSeriatimBinary, Timeout: DefaultSeriatimTimeout, + OutputSchema: DefaultSeriatimOutputSchema, CoalesceGap: float64Ptr(DefaultSeriatimCoalesceGap), + Report: boolPtr(DefaultSeriatimReport), + }, + Audita: AuditaConfig{Binary: DefaultAuditaBinary, Timeout: DefaultAuditaTimeout, Report: boolPtr(DefaultAuditaReport)}, + Normalize: &NormalizeConfig{OutputPath: DefaultNormalizeOutputPath, OutputSchema: DefaultNormalizeOutputSchema, Report: boolPtr(DefaultNormalizeReport)}, + Trim: &TrimConfig{ + Enabled: boolPtr(DefaultTrimEnabled), OutputPath: DefaultTrimOutputPath, + Bounds: TrimBoundsConfig{ + PromptID: DefaultTrimBoundsPromptID, TranscriptInputName: DefaultTrimBoundsTranscriptInputName, + OutputPath: DefaultTrimBoundsOutputPath, Timeout: DefaultTrimBoundsTimeout, + }, + Seriatim: TrimSeriatimConfig{Report: boolPtr(DefaultTrimSeriatimReport)}, + }, + Render: &RenderConfig{ + Enabled: boolPtr(DefaultRenderEnabled), Format: DefaultRenderFormat, Title: DefaultRenderTitle, + IncludeTimestamps: boolPtr(DefaultRenderTimestamps), IncludeSegmentIDs: boolPtr(DefaultRenderSegmentIDs), + }, + Notification: NotificationConfig{Mode: DefaultNotificationMode}, + } +} + +func defaultPublishOutputs() []PublishOutputRule { + outputs := append([]PublishOutputRule(nil), DefaultPublishOutputs...) + for index := range outputs { + outputs[index].Required = boolPtr(true) + } + return outputs +} + +func resolvePipelineConfig(cfg *PipelineConfig, resolution *configkit.Resolution) error { + backend := strings.ToLower(strings.TrimSpace(cfg.Storage.Backend)) + if backend != cfg.Storage.Backend { + configkit.Normalize(resolution, configkit.Keys("storage", "backend"), &cfg.Storage.Backend, backend, "canonical storage backend") + } + if cfg.Storage.S3 != nil { + path := configkit.Keys("storage", "s3") + configkit.Default(resolution, path.Key("root_prefix"), &cfg.Storage.S3.RootPrefix, DefaultStorageS3RootPrefix, "default S3 root prefix") + configkit.Default(resolution, path.Key("access_key_id_env"), &cfg.Storage.S3.AccessKeyIDEnv, DefaultS3AccessKeyIDEnv, "default S3 access-key reference") + configkit.Default(resolution, path.Key("secret_access_key_env"), &cfg.Storage.S3.SecretKeyEnv, DefaultS3SecretAccessKeyEnv, "default S3 secret-key reference") + } + if cfg.Trim != nil && cfg.Trim.Enabled != nil && *cfg.Trim.Enabled && cfg.Scriptorium == nil { + value := &ScriptoriumConfig{Binary: DefaultScriptoriumBinary, Timeout: DefaultScriptoriumTimeout} + configkit.Default(resolution, configkit.Keys("scriptorium"), &cfg.Scriptorium, value, "trim requires Scriptorium") + } + if cfg.Scriptorium != nil { + path := configkit.Keys("scriptorium") + configkit.Default(resolution, path.Key("binary"), &cfg.Scriptorium.Binary, DefaultScriptoriumBinary, "default Scriptorium binary") + configkit.Default(resolution, path.Key("timeout"), &cfg.Scriptorium.Timeout, DefaultScriptoriumTimeout, "default Scriptorium timeout") + } + if cfg.Notarius != nil { + path := configkit.Keys("notarius") + configkit.Default(resolution, path.Key("binary"), &cfg.Notarius.Binary, DefaultNotariusBinary, "default Notarius binary") + configkit.Default(resolution, path.Key("timeout"), &cfg.Notarius.Timeout, DefaultNotariusTimeout, "default Notarius timeout") + if cfg.Notarius.Enabled && cfg.Notarius.ConfigPath != "" { + configkit.Default(resolution, path.Key("working_directory"), &cfg.Notarius.WorkingDirectory, + filepath.Dir(cfg.Notarius.ConfigPath), "derived from Notarius config path", path.Key("config_path")) + } + } + if cfg.Publish != nil { + defaultedOutput := false + for index := range cfg.Publish.Outputs { + output := cfg.Publish.Outputs[index] + path := configkit.Keys("publish", "outputs").Index(index).Key("required") + if !resolution.IsExplicit(path) { + defaultedOutput = true + } + configkit.Default(resolution, path, &output.Required, boolPtr(true), "publish outputs are required by default") + cfg.Publish.Outputs[index] = output + } + if defaultedOutput { + resolution.Record(configkit.Keys("publish", "outputs"), configkit.ChangeDerivedDefault, "publish output entry defaults") + } + } + before := cloneNotariusForResolution(cfg.Notarius) + if err := NormalizePipelineConfig(cfg); err != nil { + resolution.Error(configkit.Keys("notarius"), err.Error()) + return nil + } + if !reflect.DeepEqual(before, cfg.Notarius) { + resolution.Record(configkit.Keys("notarius"), configkit.ChangeNormalization, "canonical Notarius configuration") + } + if cfg.Publish != nil { + configuredOutputs := scriptoriumOutputPathMap(cfg.Scriptorium) + extractionOutputs := notariusOutputKeySet(cfg.Notarius) + for index := range cfg.Publish.Outputs { + output := cfg.Publish.Outputs[index] + if strings.TrimSpace(output.Dest) != "" { + continue + } + destination, err := artifactpolicy.ResolvePublishedDestinationWithExtractions(strings.TrimSpace(output.Source), "", configuredOutputs, extractionOutputs) + if err != nil { + continue + } + configkit.Normalize(resolution, configkit.Keys("publish", "outputs").Index(index).Key("dest"), &output.Dest, destination, "derived from publish source") + cfg.Publish.Outputs[index] = output + } + } + return nil +} + +func cloneNotariusForResolution(value *NotariusConfig) *NotariusConfig { + if value == nil { + return nil + } + clone := *value + clone.References = cloneStringMap(value.References) + if value.Outputs != nil { + clone.Outputs = make(map[string]NotariusOutputConfig, len(value.Outputs)) + for key, output := range value.Outputs { + clone.Outputs[key] = output + } + } + return &clone +} + +func cloneStringMap(value map[string]string) map[string]string { + if value == nil { + return nil + } + clone := make(map[string]string, len(value)) + for key, item := range value { + clone[key] = item + } + return clone +} + +func validatePipelineConfigKit(cfg *PipelineConfig, validation *configkit.Validation) { + // Checks that depend on family expansion or campaign/session context remain + // at Narratio's explicit resolution boundary. This early definition check is + // deliberately limited to an invariant that is wholly file-local. + if strings.TrimSpace(cfg.Workspace.Root) == "" { + validation.Error(configkit.Keys("workspace", "root"), "workspace root is required") + } +} + +func pipelineMetadataFromConfigKit(cfg *PipelineConfig, result *configkit.Result[PipelineConfig], opts PipelineLoadOptions) (*pipelineResolutionMetadata, error) { + sources := result.Sources() + metadata := result.Metadata() + if err := rejectDuplicateBaseDefinitions(cfg, metadata, sources); err != nil { + return nil, err + } + resolved := &pipelineResolutionMetadata{ + configkitMetadata: metadata, configkitSources: sources, + publishDeclared: metadata.IsExplicit(configkit.Keys("publish")), + } + pathsByID := make(map[string]string, len(sources)) + for index, source := range sources { + path := source.Path + if path == "" { + path = source.Label + } + pathsByID[source.ID] = path + resolved.sources = append(resolved.sources, path) + if index == 0 { + resolved.rootPath = path + } else if source.Layer == configkit.LayerBase { + resolved.imports = append(resolved.imports, path) + } + if source.Layer == configkit.LayerProfile { + resolved.selectedProfile = &pipelineProfileSelection{name: selectedPipelineProfileName(opts), source: "explicit", overlayPath: path} + } + } + detailed, err := pipelineMetadataPaths(cfg) + if err != nil { + return nil, err + } + for _, value := range detailed { + info, ok := metadata.Lookup(configkit.Keys(value.segments...)) + sourceNames := []string{pipelineDefaultOwnershipSource} + if ok { + sourceNames = ownershipSources(info.Origin, pathsByID) + } + resolved.ownership = append(resolved.ownership, pipelineFieldOwnership{path: value.record.Path, sources: sourceNames}) + } + return resolved, nil +} + +func pipelineMetadataPaths(cfg *PipelineConfig) ([]projectedPipelineValue, error) { + effective, err := projectionDetailedRecords(projectEffectivePipeline(cfg, false)) + if err != nil { + return nil, err + } + if cfg.Scriptorium == nil || len(cfg.Scriptorium.ArtifactFamilies) == 0 { + return effective, nil + } + families, err := projectionDetailedRecords(map[string]any{"scriptorium": map[string]any{"artifact_families": cfg.Scriptorium.ArtifactFamilies}}) + if err != nil { + return nil, err + } + return append(effective, families...), nil +} + +func ownershipSources(origin configkit.Origin, pathsByID map[string]string) []string { + if origin.Layer == configkit.LayerDefault || (origin.Layer == configkit.LayerResolution && origin.Operation == configkit.ChangeDerivedDefault) { + return []string{pipelineDefaultOwnershipSource} + } + if origin.Layer == configkit.LayerResolution { + return []string{"resolution"} + } + seen := map[string]struct{}{} + var sources []string + for _, location := range origin.Locations { + value := pathsByID[location.SourceID] + if value == "" { + value = location.SourceID + } + if _, ok := seen[value]; !ok { + seen[value] = struct{}{} + sources = append(sources, value) + } + } + if len(sources) == 0 { + return []string{pipelineDefaultOwnershipSource} + } + return sources +} + +func rejectDuplicateBaseDefinitions(cfg *PipelineConfig, metadata configkit.Metadata, sources []configkit.SourceInfo) error { + pathsByID := make(map[string]string, len(sources)) + for _, source := range sources { + pathsByID[source.ID] = source.Path + } + var declarations []struct { + kind string + key string + path configkit.Path + } + if cfg.Scriptorium != nil { + for key := range cfg.Scriptorium.Artifacts { + declarations = append(declarations, struct { + kind, key string + path configkit.Path + }{"artifact", key, configkit.Keys("scriptorium", "artifacts", key)}) + } + for key := range cfg.Scriptorium.ArtifactFamilies { + declarations = append(declarations, struct { + kind, key string + path configkit.Path + }{"artifact family", key, configkit.Keys("scriptorium", "artifact_families", key)}) + } + } + sort.Slice(declarations, func(i, j int) bool { return declarations[i].path.String() < declarations[j].path.String() }) + for _, declaration := range declarations { + seen := map[string]struct{}{} + for _, change := range metadata.History(declaration.path) { + if change.Origin.Layer != configkit.LayerBase { + continue + } + for _, location := range change.Origin.Locations { + seen[location.SourceID] = struct{}{} + } + } + if len(seen) > 1 { + claims := make([]string, 0, len(seen)) + for sourceID := range seen { + claims = append(claims, pathsByID[sourceID]) + } + sort.Strings(claims) + return fmt.Errorf("pipeline Scriptorium %s %q is declared by multiple base files: %s", declaration.kind, declaration.key, strings.Join(claims, ", ")) + } + } + return nil +} + +func intPtr(value int) *int { return &value } +func float64Ptr(value float64) *float64 { return &value } +func boolPtr(value bool) *bool { return &value } diff --git a/internal/config/pipeline_configkit_test.go b/internal/config/pipeline_configkit_test.go new file mode 100644 index 0000000..fb61169 --- /dev/null +++ b/internal/config/pipeline_configkit_test.go @@ -0,0 +1,102 @@ +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestPipelineConfigKitImportsProfileAndBaseSelection(t *testing.T) { + dir := t.TempDir() + writePipelineFixture(t, dir, "pipeline.yml", `configkit: + imports: + - path: config/workspace.yml + - path: config/artifact.yml +whisperx: + transcribe_url: https://transcription.example.test/transcribe +`) + writePipelineFixture(t, dir, "config/workspace.yml", "workspace:\n root: /srv/base\n") + writePipelineFixture(t, dir, "config/artifact.yml", `scriptorium: + artifacts: + recap: + enabled: false + output_path: artifacts/recap.md +`) + writePipelineFixture(t, dir, "profiles/testing.yml", "workspace:\n root: /srv/testing\n") + writePipelineFixture(t, dir, "profiles/unselected.yml", "unknown: [\n") + + root := filepath.Join(dir, "pipeline.yml") + for _, profile := range []*string{nil, stringPointer(""), stringPointer("default")} { + cfg, err := LoadPipelineWithOptions(root, PipelineLoadOptions{Profile: profile}) + if err != nil { + t.Fatalf("load base selection %#v: %v", profile, err) + } + if cfg.Workspace.Root != "/srv/base" { + t.Fatalf("base workspace = %q", cfg.Workspace.Root) + } + } + selected := "testing" + cfg, err := LoadPipelineWithOptions(root, PipelineLoadOptions{Profile: &selected}) + if err != nil { + t.Fatal(err) + } + if cfg.Workspace.Root != "/srv/testing" || len(EffectivePipelineImports(cfg)) != 2 { + t.Fatalf("profiled pipeline = %#v, imports=%#v", cfg.Workspace, EffectivePipelineImports(cfg)) + } + profile, ok := SelectedPipelineProfile(cfg) + if !ok || profile.Name != "testing" || profile.OverlayPath != filepath.Join(dir, "profiles", "testing.yml") { + t.Fatalf("selected profile = %#v, %v", profile, ok) + } +} + +func TestPipelineConfigKitRejectsDuplicateBaseDeclarations(t *testing.T) { + for _, test := range []struct { + name, first, second string + }{ + {name: "artifact", first: "scriptorium:\n artifacts:\n recap:\n prompt_id: recap\n", second: "scriptorium:\n artifacts:\n recap:\n output_path: artifacts/recap.md\n"}, + {name: "artifact family", first: "scriptorium:\n artifact_families:\n recap:\n for_each: party.characters\n", second: "scriptorium:\n artifact_families:\n recap:\n output_path_pattern: artifacts/{character_id}.md\n"}, + } { + t.Run(test.name, func(t *testing.T) { + dir := t.TempDir() + writePipelineFixture(t, dir, "pipeline.yml", `configkit: + imports: + - path: first.yml + - path: second.yml +whisperx: + transcribe_url: https://transcription.example.test/transcribe +`) + writePipelineFixture(t, dir, "first.yml", test.first) + writePipelineFixture(t, dir, "second.yml", test.second) + + _, err := LoadPipeline(filepath.Join(dir, "pipeline.yml")) + if err == nil || !strings.Contains(err.Error(), "declared by multiple base files") || + !strings.Contains(err.Error(), "first.yml") || !strings.Contains(err.Error(), "second.yml") { + t.Fatalf("duplicate declaration error = %v", err) + } + }) + } +} + +func TestPipelineConfigKitRejectsUnknownDirectiveAlias(t *testing.T) { + dir := t.TempDir() + path := writePipelineFixture(t, dir, "pipeline.yml", "composition:\n imports: []\n") + _, err := LoadPipeline(path) + if err == nil || !strings.Contains(err.Error(), "decode.unknown_field") { + t.Fatalf("composition alias error = %v", err) + } +} + +func writePipelineFixture(t *testing.T, root, relative, contents string) string { + t.Helper() + path := filepath.Join(root, filepath.FromSlash(relative)) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(contents), 0o644); err != nil { + t.Fatal(err) + } + return path +} + +func stringPointer(value string) *string { return &value } diff --git a/internal/config/pipeline_profiles_test.go b/internal/config/pipeline_profiles_test.go deleted file mode 100644 index b95e934..0000000 --- a/internal/config/pipeline_profiles_test.go +++ /dev/null @@ -1,479 +0,0 @@ -package config - -import ( - "os" - "path/filepath" - "slices" - "strings" - "testing" - - "github.com/google/go-cmp/cmp" -) - -func TestLoadPipelineProfilesSelectDefaultAndExplicitOverlay(t *testing.T) { - dir := t.TempDir() - rootPath := writeProfilePipeline(t, dir, `composition: - imports: [conf.d/base.yml] - default_profile: production - profiles: - production: - overlay: profiles/production.yml - testing: - overlay: profiles/testing.yml -whisperx: - transcribe_url: https://transcription.example.com/transcribe -`) - importPath := writePipelineSource(t, dir, "conf.d/base.yml", `workspace: - root: /srv/base -audita: - modules: [base, shared] -`) - productionPath := writePipelineSource(t, dir, "profiles/production.yml", `workspace: - root: /srv/production -whisperx: - language: en -`) - testingPath := writePipelineSource(t, dir, "profiles/testing.yml", `workspace: - root: /srv/testing -whisperx: - language: fr -`) - - production, err := LoadPipeline(rootPath) - if err != nil { - t.Fatal(err) - } - if production.Workspace.Root != "/srv/production" || production.WhisperX.Language != "en" { - t.Fatalf("default profile result = workspace=%q language=%q", production.Workspace.Root, production.WhisperX.Language) - } - assertSelectedPipelineProfile(t, production, "production", "default", productionPath) - if want := []string{absolutePath(t, rootPath), absolutePath(t, importPath), absolutePath(t, productionPath)}; !slices.Equal(production.resolution.sources, want) { - t.Fatalf("default sources = %#v, want %#v", production.resolution.sources, want) - } - assertPipelineFieldOwner(t, production, "workspace.root", absolutePath(t, productionPath)) - assertPipelineFieldOwner(t, production, "audita.modules", absolutePath(t, importPath)) - assertPipelineFieldOwner(t, production, "storage.backend", pipelineDefaultOwnershipSource) - - profile := "testing" - testingCfg, err := LoadPipelineWithOptions(rootPath, PipelineLoadOptions{Profile: &profile}) - if err != nil { - t.Fatal(err) - } - if testingCfg.Workspace.Root != "/srv/testing" || testingCfg.WhisperX.Language != "fr" { - t.Fatalf("explicit profile result = workspace=%q language=%q", testingCfg.Workspace.Root, testingCfg.WhisperX.Language) - } - assertSelectedPipelineProfile(t, testingCfg, "testing", "cli", testingPath) -} - -func TestLoadPipelineProfileSelectionErrors(t *testing.T) { - for _, test := range []struct { - name string - root string - profile *string - want string - }{ - {name: "profiles require selection", root: profileComposition("", "production"), want: "default_profile is omitted"}, - {name: "unknown explicit", root: profileComposition("production", "production"), profile: profilePointer("unknown"), want: "not declared"}, - {name: "stacked explicit", root: profileComposition("production", "production", "testing"), profile: profilePointer("production,testing"), want: "not declared"}, - {name: "explicit empty", root: profileComposition("production", "production"), profile: profilePointer(""), want: "must be non-empty"}, - {name: "explicit whitespace", root: profileComposition("production", "production"), profile: profilePointer(" production "), want: "surrounding whitespace"}, - {name: "invalid default", root: profileComposition("unknown", "production"), want: "does not name a declared profile"}, - {name: "empty default", root: profileComposition("EMPTY", "production"), want: "must be non-empty"}, - {name: "empty profile name", root: `composition: - default_profile: production - profiles: - "": - overlay: production.yml -whisperx: - transcribe_url: https://transcription.example.com/transcribe -`, want: "profile name must be non-empty"}, - } { - t.Run(test.name, func(t *testing.T) { - dir := t.TempDir() - root := test.root - if test.name == "empty default" { - root = strings.ReplaceAll(root, "default_profile: EMPTY", `default_profile: ""`) - } - rootPath := writeProfilePipeline(t, dir, root) - writePipelineSource(t, dir, "production.yml", "whisperx:\n language: en\n") - _, err := LoadPipelineWithOptions(rootPath, PipelineLoadOptions{Profile: test.profile}) - if err == nil || !strings.Contains(strings.ToLower(err.Error()), strings.ToLower(test.want)) { - t.Fatalf("LoadPipelineWithOptions() error = %v, want containing %q", err, test.want) - } - }) - } - - t.Run("explicit against profile-free pipeline", func(t *testing.T) { - dir := t.TempDir() - rootPath := writeProfilePipeline(t, dir, testPipelineBaseYAML) - selected := "testing" - _, err := LoadPipelineWithOptions(rootPath, PipelineLoadOptions{Profile: &selected}) - if err == nil || !strings.Contains(err.Error(), "declares no profiles") { - t.Fatalf("error = %v, want profile-free rejection", err) - } - }) -} - -func TestPipelineProfileNamesRejectControlCharacters(t *testing.T) { - if _, err := normalizePipelineProfileName("production\x00testing", "profile"); err == nil || !strings.Contains(err.Error(), "control characters") { - t.Fatalf("normalizePipelineProfileName() error = %v, want control-character rejection", err) - } -} - -func TestLoadPipelineProfilesValidateEveryDeclaredOverlay(t *testing.T) { - for _, test := range []struct { - name string - path string - content string - setup func(*testing.T, string) - want string - }{ - {name: "missing", path: "profiles/missing.yml", want: "open composition.profiles.testing.overlay"}, - {name: "malformed", path: "profiles/testing.yml", content: "workspace: [\n", want: "decode yaml"}, - {name: "duplicate keys", path: "profiles/testing.yml", content: "workspace:\n root: /one\n root: /two\n", want: "duplicate yaml key"}, - {name: "trailing document", path: "profiles/testing.yml", content: "workspace:\n root: /one\n---\nworkspace:\n root: /two\n", want: "exactly one yaml document"}, - {name: "nested composition", path: "profiles/testing.yml", content: "composition:\n imports: [nested.yml]\n", want: "only the root pipeline"}, - {name: "inheritance", path: "profiles/testing.yml", content: "workspace:\n root: /testing\n", setup: func(t *testing.T, dir string) { - rootPath := filepath.Join(dir, "pipeline.yml") - data, err := os.ReadFile(rootPath) - if err != nil { - t.Fatal(err) - } - updated := strings.Replace(string(data), "testing:\n overlay:", "testing:\n extends: production\n overlay:", 1) - if err := os.WriteFile(rootPath, []byte(updated), 0o644); err != nil { - t.Fatal(err) - } - }, want: "only overlay is supported"}, - {name: "traversal", path: "../testing.yml", want: "invalid"}, - {name: "extension", path: "profiles/testing.json", content: "{}\n", want: ".yml or .yaml"}, - {name: "directory", path: "profiles/testing.yml", setup: func(t *testing.T, dir string) { - if err := os.MkdirAll(filepath.Join(dir, "profiles/testing.yml"), 0o755); err != nil { - t.Fatal(err) - } - }, want: "is a directory"}, - } { - t.Run(test.name, func(t *testing.T) { - dir := t.TempDir() - rootPath := writeProfilePipeline(t, dir, `composition: - default_profile: production - profiles: - production: - overlay: profiles/production.yml - testing: - overlay: `+test.path+` -whisperx: - transcribe_url: https://transcription.example.com/transcribe -`) - writePipelineSource(t, dir, "profiles/production.yml", "whisperx:\n language: en\n") - if test.content != "" { - writePipelineSource(t, dir, test.path, test.content) - } - if test.setup != nil { - test.setup(t, dir) - } - _, err := LoadPipeline(rootPath) - if err == nil || !strings.Contains(strings.ToLower(err.Error()), strings.ToLower(test.want)) { - t.Fatalf("LoadPipeline() error = %v, want containing %q", err, test.want) - } - }) - } -} - -func TestLoadPipelineProfileOverlaySemantics(t *testing.T) { - dir := t.TempDir() - rootPath := writeProfilePipeline(t, dir, `composition: - default_profile: testing - profiles: - testing: - overlay: testing.yml -workspace: - root: /base - cleanup_after_publish: true -whisperx: - transcribe_url: https://transcription.example.com/transcribe - retries: 3 -audita: - modules: [base, shared] -scriptorium: - artifacts: - session_recap: - enabled: true - prompt_id: dnd.session_recap - output_path: artifacts/session_recap.md -`) - writePipelineSource(t, dir, "testing.yml", `workspace: - cleanup_after_publish: false -whisperx: - retries: 0 -audita: - modules: [] -scriptorium: - artifacts: - session_recap: - enabled: false - experimental: - enabled: false -`) - - cfg, err := LoadPipeline(rootPath) - if err != nil { - t.Fatal(err) - } - if cfg.Workspace.Root != "/base" || cfg.Workspace.CleanupAfterPublish || cfg.WhisperX.Retries == nil || *cfg.WhisperX.Retries != 0 { - t.Fatalf("recursive/zero overlay = workspace=%#v retries=%#v", cfg.Workspace, cfg.WhisperX.Retries) - } - if cfg.Audita.Modules == nil || len(cfg.Audita.Modules) != 0 { - t.Fatalf("list replacement = %#v, want explicit empty list", cfg.Audita.Modules) - } - if cfg.Scriptorium == nil || cfg.Scriptorium.Artifacts["session_recap"].Enabled || len(cfg.Scriptorium.Artifacts) != 2 { - t.Fatalf("keyed artifact overlay = %#v", cfg.Scriptorium) - } -} - -func TestLoadPipelineProfileRejectsNullAndKindChanges(t *testing.T) { - for _, test := range []struct { - name string - overlay string - want string - }{ - {name: "null deletion", overlay: "workspace:\n root: null\n", want: "null cannot delete"}, - {name: "mapping scalar", overlay: "workspace: /other\n", want: "kind change"}, - {name: "list mapping", overlay: "audita:\n modules:\n testing: true\n", want: "kind change"}, - } { - t.Run(test.name, func(t *testing.T) { - dir := t.TempDir() - rootPath := writeProfilePipeline(t, dir, `composition: - default_profile: testing - profiles: - testing: - overlay: testing.yml -workspace: - root: /base -audita: - modules: [base] -whisperx: - transcribe_url: https://transcription.example.com/transcribe -`) - writePipelineSource(t, dir, "testing.yml", test.overlay) - _, err := LoadPipeline(rootPath) - if err == nil || !strings.Contains(strings.ToLower(err.Error()), strings.ToLower(test.want)) { - t.Fatalf("LoadPipeline() error = %v, want containing %q", err, test.want) - } - }) - } -} - -func TestPipelineEffectiveDigestTracksNormalizedMeaning(t *testing.T) { - dir := t.TempDir() - monolithicPath := writePipelineSource(t, dir, "monolithic.yml", `workspace: - root: /srv/narratio -whisperx: - transcribe_url: https://transcription.example.com/transcribe - language: en -`) - composedPath := writePipelineSource(t, dir, "pipeline.yml", `composition: - imports: [workspace.yml] -whisperx: - transcribe_url: https://transcription.example.com/transcribe - language: en -`) - writePipelineSource(t, dir, "workspace.yml", "workspace:\n root: /srv/narratio\n") - - monolithic, err := LoadPipeline(monolithicPath) - if err != nil { - t.Fatal(err) - } - composed, err := LoadPipeline(composedPath) - if err != nil { - t.Fatal(err) - } - if monolithic.resolution.effectiveDigest == "" || monolithic.resolution.effectiveDigest != composed.resolution.effectiveDigest { - t.Fatalf("equal effective results have digests %q and %q", monolithic.resolution.effectiveDigest, composed.resolution.effectiveDigest) - } - - changedPath := writePipelineSource(t, dir, "changed.yml", `workspace: - root: /srv/narratio -whisperx: - transcribe_url: https://transcription.example.com/transcribe - language: fr -`) - changed, err := LoadPipeline(changedPath) - if err != nil { - t.Fatal(err) - } - if changed.resolution.effectiveDigest == monolithic.resolution.effectiveDigest { - t.Fatal("semantic value change retained effective digest") - } - - t.Setenv("AWS_SECRET_ACCESS_KEY", "secret-one") - first, err := LoadPipeline(monolithicPath) - if err != nil { - t.Fatal(err) - } - t.Setenv("AWS_SECRET_ACCESS_KEY", "secret-two") - second, err := LoadPipeline(monolithicPath) - if err != nil { - t.Fatal(err) - } - if first.resolution.effectiveDigest != second.resolution.effectiveDigest || strings.Contains(first.resolution.effectiveDigest, "secret") { - t.Fatalf("environment secret affected or appeared in digest: %q / %q", first.resolution.effectiveDigest, second.resolution.effectiveDigest) - } -} - -func TestEffectivePipelineValuesIgnoreEquivalentSourceLayout(t *testing.T) { - dir := t.TempDir() - monolithicPath := writePipelineSource(t, dir, "monolithic.yml", `workspace: - root: /srv/narratio -whisperx: - transcribe_url: https://transcription.example.com/transcribe - language: en -`) - composedPath := writePipelineSource(t, dir, "pipeline.yml", `composition: - imports: [workspace.yml] -whisperx: - language: en - transcribe_url: https://transcription.example.com/transcribe -`) - writePipelineSource(t, dir, "workspace.yml", "workspace:\n root: /srv/narratio\n") - - monolithic, err := LoadPipeline(monolithicPath) - if err != nil { - t.Fatal(err) - } - composed, err := LoadPipeline(composedPath) - if err != nil { - t.Fatal(err) - } - monolithicValues, err := EffectivePipelineValues(monolithic) - if err != nil { - t.Fatal(err) - } - composedValues, err := EffectivePipelineValues(composed) - if err != nil { - t.Fatal(err) - } - if diff := cmp.Diff(monolithicValues, composedValues); diff != "" { - t.Fatalf("effective values differ (-monolithic +composed):\n%s", diff) - } -} - -func TestPipelineEffectiveDigestExcludesProfileIdentity(t *testing.T) { - dir := t.TempDir() - rootPath := writeProfilePipeline(t, dir, profileComposition("production", "production", "testing")) - overlay := "workspace:\n root: /srv/narratio\n" - writePipelineSource(t, dir, "production.yml", overlay) - writePipelineSource(t, dir, "testing.yml", overlay) - - production := "production" - productionCfg, err := LoadPipelineWithOptions(rootPath, PipelineLoadOptions{Profile: &production}) - if err != nil { - t.Fatal(err) - } - testing := "testing" - testingCfg, err := LoadPipelineWithOptions(rootPath, PipelineLoadOptions{Profile: &testing}) - if err != nil { - t.Fatal(err) - } - if productionCfg.resolution.effectiveDigest != testingCfg.resolution.effectiveDigest { - t.Fatalf("profile identity changed equal effective digests: %q / %q", productionCfg.resolution.effectiveDigest, testingCfg.resolution.effectiveDigest) - } -} - -func TestLoadPipelineProfilePairResolvesIndependentEffectivePipelines(t *testing.T) { - dir := t.TempDir() - rootPath := writeProfilePipeline(t, dir, `composition: - imports: [base.yml] - default_profile: production - profiles: - production: - overlay: production.yml - testing: - overlay: testing.yml -whisperx: - transcribe_url: https://transcription.example.com/transcribe -`) - writePipelineSource(t, dir, "base.yml", "workspace:\n root: /srv/base\n") - writePipelineSource(t, dir, "production.yml", "workspace:\n root: /srv/production\nwhisperx:\n language: en\n") - writePipelineSource(t, dir, "testing.yml", "workspace:\n root: /srv/testing\nwhisperx:\n language: fr\n") - - production, testing, err := LoadPipelineProfilePair(rootPath, "production", "testing") - if err != nil { - t.Fatal(err) - } - if production.Workspace.Root != "/srv/production" || production.WhisperX.Language != "en" { - t.Fatalf("production pair result = %#v", production) - } - if testing.Workspace.Root != "/srv/testing" || testing.WhisperX.Language != "fr" { - t.Fatalf("testing pair result = %#v", testing) - } - production.Workspace.Root = "/mutated-left" - if testing.Workspace.Root != "/srv/testing" { - t.Fatalf("right profile shared mutable state with left: %q", testing.Workspace.Root) - } - testingFirst, productionSecond, err := LoadPipelineProfilePair(rootPath, "testing", "production") - if err != nil { - t.Fatal(err) - } - if testingFirst.Workspace.Root != "/srv/testing" || productionSecond.Workspace.Root != "/srv/production" { - t.Fatalf("reversed profile pair = testing=%q production=%q", testingFirst.Workspace.Root, productionSecond.Workspace.Root) - } - - if _, _, err := LoadPipelineProfilePair(rootPath, "production", "production"); err == nil || !strings.Contains(err.Error(), "must differ") { - t.Fatalf("equal profile pair error = %v, want selection rejection", err) - } - if _, _, err := LoadPipelineProfilePair(rootPath, "unknown", "testing"); err == nil || !strings.Contains(err.Error(), "not declared") { - t.Fatalf("unknown profile pair error = %v, want selection rejection", err) - } -} - -func TestLoadWithSessionOptionsCarriesExplicitProfilePresence(t *testing.T) { - dir := t.TempDir() - rootPath := writeProfilePipeline(t, dir, profileComposition("production", "production", "testing")) - writePipelineSource(t, dir, "production.yml", "workspace:\n root: /production\n") - writePipelineSource(t, dir, "testing.yml", "workspace:\n root: /testing\n") - campaignPath := writePipelineSource(t, dir, "campaign.yml", "campaign_id: campaign\ninputs:\n speakers_file: speakers.yml\n autocorrect_file: autocorrect.yml\n glossary_file: glossary.yml\n party_file: party.yml\n") - writePipelineSource(t, dir, "party.yml", canonicalPartyFixture) - sessionPath := writePipelineSource(t, dir, "session.yml", "session_id: session\ncampaign: campaign\n") - selected := "testing" - cfg, err := LoadWithSessionOptions(rootPath, campaignPath, sessionPath, SessionLoadOptions{Profile: &selected}) - if err != nil { - t.Fatal(err) - } - if cfg.Pipeline.Workspace.Root != "/testing" || cfg.Pipeline.resolution.selectedProfile.source != "cli" { - t.Fatalf("combined profile result = workspace=%q metadata=%#v", cfg.Pipeline.Workspace.Root, cfg.Pipeline.resolution.selectedProfile) - } -} - -func writeProfilePipeline(t *testing.T, dir, content string) string { - t.Helper() - return writePipelineSource(t, dir, "pipeline.yml", content) -} - -func profileComposition(defaultProfile string, profiles ...string) string { - var builder strings.Builder - builder.WriteString("composition:\n") - if defaultProfile != "" { - builder.WriteString(" default_profile: " + defaultProfile + "\n") - } - builder.WriteString(" profiles:\n") - for _, profile := range profiles { - builder.WriteString(" " + profile + ":\n overlay: " + profile + ".yml\n") - } - builder.WriteString("whisperx:\n transcribe_url: https://transcription.example.com/transcribe\n") - return builder.String() -} - -func profilePointer(value string) *string { - return &value -} - -func assertSelectedPipelineProfile(t *testing.T, cfg *PipelineConfig, name, source, overlayPath string) { - t.Helper() - if cfg == nil || cfg.resolution == nil || cfg.resolution.selectedProfile == nil { - t.Fatal("selected profile provenance is absent") - } - selection := cfg.resolution.selectedProfile - if selection.name != name || selection.source != source || selection.overlayPath != absolutePath(t, overlayPath) { - t.Fatalf("selected profile = %#v, want name=%q source=%q overlay=%q", selection, name, source, absolutePath(t, overlayPath)) - } -} diff --git a/internal/config/scriptorium_test.go b/internal/config/scriptorium_test.go index a9cb8ff..f033122 100644 --- a/internal/config/scriptorium_test.go +++ b/internal/config/scriptorium_test.go @@ -129,7 +129,7 @@ func TestScriptoriumLoadAndValidate(t *testing.T) { source: narratio.transcript.polished artifact: session_recap `, - wantLoadErr: "field artifact not found", + wantLoadErr: "decode.unknown_field", }, { name: "canonical previous-session source is accepted", diff --git a/internal/config/storage_publish_test.go b/internal/config/storage_publish_test.go index 68628b7..be1b63f 100644 --- a/internal/config/storage_publish_test.go +++ b/internal/config/storage_publish_test.go @@ -540,7 +540,7 @@ publish: ` pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML) _, err := Load(pipelinePath, sessionPath) - if err == nil || !strings.Contains(err.Error(), "strict decode failed") || !strings.Contains(err.Error(), "field locks not found") { + if err == nil || !strings.Contains(err.Error(), "decode.unknown_field") { t.Fatalf("Load() error = %v, want retired locks field rejected", err) } } diff --git a/internal/config/trim_test.go b/internal/config/trim_test.go index c303ea8..e8682af 100644 --- a/internal/config/trim_test.go +++ b/internal/config/trim_test.go @@ -57,12 +57,14 @@ func TestTrimLoadAndValidate(t *testing.T) { }, }, { - name: "enabled omitted defaults enabled", + name: "explicit null remains absent", trimYAML: `trim: `, assert: func(t *testing.T, cfg *Config) { t.Helper() - assertDefaultTrimConfig(t, cfg) + if cfg.Pipeline.Trim != nil { + t.Fatalf("trim config = %#v, want nil", cfg.Pipeline.Trim) + } }, }, { diff --git a/internal/config/validate_publish.go b/internal/config/validate_publish.go index 2fad95d..0bb08d5 100644 --- a/internal/config/validate_publish.go +++ b/internal/config/validate_publish.go @@ -60,12 +60,7 @@ func validatePublish(cfg *PublishConfig, scriptorium *ScriptoriumConfig, notariu } dest := strings.TrimSpace(item.Dest) if dest == "" { - derivedDest, err := artifactpolicy.ResolvePublishedDestinationWithExtractions(source, "", configuredOutputs, extractionOutputs) - if err != nil { - return fmt.Errorf("%s.dest is required when destination cannot be derived from %q: %w", prefix, source, err) - } - dest = derivedDest - cfg.Outputs[i].Dest = derivedDest + return fmt.Errorf("%s.dest is required when destination cannot be derived from %q", prefix, source) } normalizedDest, err := pathsafe.NormalizeRelativeDestination(dest) if err != nil {