Adopt ConfigKit for pipeline loading
This commit is contained in:
@@ -253,8 +253,8 @@ func TestAssembledSplitBundleCommandsUseOneEffectiveFamilyConfiguration(t *testi
|
||||
}
|
||||
|
||||
var validated bytes.Buffer
|
||||
if err := ConfigValidate(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath}, &validated); err != nil {
|
||||
t.Fatalf("validate production default: %v", err)
|
||||
if err := ConfigValidate(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath, "--profile", "production"}, &validated); err != nil {
|
||||
t.Fatalf("validate production profile: %v", err)
|
||||
}
|
||||
if !strings.Contains(validated.String(), "profile=production") {
|
||||
t.Fatalf("validate output = %q, want production profile", validated.String())
|
||||
@@ -330,7 +330,7 @@ func TestAssembledSplitBundleCommandsUseOneEffectiveFamilyConfiguration(t *testi
|
||||
}
|
||||
|
||||
func TestAssembledSplitBundleProfileSwitchRecordsProvenanceAndLimitsReuse(t *testing.T) {
|
||||
production := loadAssembledSplitBundleConfig(t, "")
|
||||
production := loadAssembledSplitBundleConfig(t, "production")
|
||||
testingCfg := loadAssembledSplitBundleConfig(t, "testing")
|
||||
workspace := t.TempDir()
|
||||
production.Pipeline.Workspace.Root = workspace
|
||||
|
||||
@@ -304,17 +304,9 @@ func TestConfigCommandsInspectMaintainedSplitBundleWithoutRuntimeState(t *testin
|
||||
|
||||
func writeInspectionProfiles(t *testing.T, pipelinePath, production, testing string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(pipelinePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
composition := "composition:\n default_profile: production\n profiles:\n production:\n overlay: production.yml\n testing:\n overlay: testing.yml\n"
|
||||
if err := os.WriteFile(pipelinePath, append([]byte(composition), data...), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dir := filepath.Dir(pipelinePath)
|
||||
mustWriteTestFile(t, filepath.Join(dir, "production.yml"), production)
|
||||
mustWriteTestFile(t, filepath.Join(dir, "testing.yml"), testing)
|
||||
profilesDir := filepath.Join(filepath.Dir(pipelinePath), "profiles")
|
||||
mustWriteTestFile(t, filepath.Join(profilesDir, "production.yml"), production)
|
||||
mustWriteTestFile(t, filepath.Join(profilesDir, "testing.yml"), testing)
|
||||
}
|
||||
|
||||
func TestConfigCommandsSelectProfilesAndCampaigns(t *testing.T) {
|
||||
@@ -326,7 +318,7 @@ func TestConfigCommandsSelectProfilesAndCampaigns(t *testing.T) {
|
||||
if err := ConfigValidate(context.Background(), []string{"--config", pipelinePath}, &defaultOut); err != nil {
|
||||
t.Fatalf("default ConfigValidate() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(defaultOut.String(), "profile=production") {
|
||||
if !strings.Contains(defaultOut.String(), "profile=none") {
|
||||
t.Fatalf("default output = %q", defaultOut.String())
|
||||
}
|
||||
|
||||
@@ -398,8 +390,9 @@ func TestConfigValidateReportsCanonicalPartyAndImportErrors(t *testing.T) {
|
||||
|
||||
dir := t.TempDir()
|
||||
rootPath := filepath.Join(dir, "pipeline.yml")
|
||||
mustWriteTestFile(t, rootPath, `composition:
|
||||
imports: [conf.yml]
|
||||
mustWriteTestFile(t, rootPath, `configkit:
|
||||
imports:
|
||||
- path: conf.yml
|
||||
workspace:
|
||||
root: /one
|
||||
whisperx:
|
||||
@@ -407,7 +400,7 @@ whisperx:
|
||||
`)
|
||||
mustWriteTestFile(t, filepath.Join(dir, "conf.yml"), "workspace:\n root: /two\n")
|
||||
err := ConfigValidate(context.Background(), []string{"--config", rootPath}, io.Discard)
|
||||
if err == nil || !strings.Contains(err.Error(), "workspace.root") || !strings.Contains(err.Error(), "pipeline.yml") || !strings.Contains(err.Error(), "conf.yml") {
|
||||
if err == nil || !strings.Contains(err.Error(), "/workspace/root") || !strings.Contains(err.Error(), "pipeline.yml") || !strings.Contains(err.Error(), "conf.yml") {
|
||||
t.Fatalf("import diagnostic = %v", err)
|
||||
}
|
||||
}
|
||||
@@ -432,7 +425,7 @@ func TestConfigSourcesReportsStableOwnership(t *testing.T) {
|
||||
for _, want := range []string{
|
||||
"root: ",
|
||||
"imports: none",
|
||||
"profile: name=testing selection=cli overlay=",
|
||||
"profile: name=testing selection=explicit overlay=",
|
||||
"campaign: id=sample-campaign source=",
|
||||
"party: source=",
|
||||
"digest: ",
|
||||
@@ -475,15 +468,7 @@ func TestConfigSourcesReportsCanonicalFamilyAndPublishOrigins(t *testing.T) {
|
||||
|
||||
func TestConfigSourcesPreservesProfileOwnershipThroughFamilyExpansion(t *testing.T) {
|
||||
pipelinePath, campaignPath := writeInspectionFamilyConfig(t)
|
||||
data, err := os.ReadFile(pipelinePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profilePath := filepath.Join(filepath.Dir(pipelinePath), "testing.yml")
|
||||
composition := "composition:\n default_profile: testing\n profiles:\n testing:\n overlay: testing.yml\n"
|
||||
if err := os.WriteFile(pipelinePath, append([]byte(composition), data...), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profilePath := filepath.Join(filepath.Dir(pipelinePath), "profiles", "testing.yml")
|
||||
mustWriteTestFile(t, profilePath, `scriptorium:
|
||||
artifact_families:
|
||||
character_note:
|
||||
@@ -492,7 +477,7 @@ func TestConfigSourcesPreservesProfileOwnershipThroughFamilyExpansion(t *testing
|
||||
`)
|
||||
|
||||
var out bytes.Buffer
|
||||
if err := ConfigSources(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath}, &out); err != nil {
|
||||
if err := ConfigSources(context.Background(), []string{"--config", pipelinePath, "--campaign-file", campaignPath, "--profile", "testing"}, &out); err != nil {
|
||||
t.Fatalf("ConfigSources() error = %v", err)
|
||||
}
|
||||
want := "scriptorium.artifacts.character_note_arannis.enabled\tfamily\t" + profilePath
|
||||
@@ -505,13 +490,10 @@ func TestConfigSourcesReportsRootImportProfileAndDefaultOwnership(t *testing.T)
|
||||
dir := t.TempDir()
|
||||
rootPath := filepath.Join(dir, "pipeline.yml")
|
||||
importPath := filepath.Join(dir, "base.yml")
|
||||
profilePath := filepath.Join(dir, "testing.yml")
|
||||
mustWriteTestFile(t, rootPath, `composition:
|
||||
imports: [base.yml]
|
||||
default_profile: testing
|
||||
profiles:
|
||||
testing:
|
||||
overlay: testing.yml
|
||||
profilePath := filepath.Join(dir, "profiles", "testing.yml")
|
||||
mustWriteTestFile(t, rootPath, `configkit:
|
||||
imports:
|
||||
- path: base.yml
|
||||
workspace:
|
||||
root: /srv/narratio
|
||||
whisperx:
|
||||
@@ -521,13 +503,13 @@ whisperx:
|
||||
mustWriteTestFile(t, profilePath, "whisperx:\n language: fr\n")
|
||||
|
||||
var out bytes.Buffer
|
||||
if err := ConfigSources(context.Background(), []string{"--config", rootPath}, &out); err != nil {
|
||||
if err := ConfigSources(context.Background(), []string{"--config", rootPath, "--profile", "testing"}, &out); err != nil {
|
||||
t.Fatalf("ConfigSources() error = %v", err)
|
||||
}
|
||||
output := out.String()
|
||||
for _, want := range []string{
|
||||
"import: " + importPath,
|
||||
"profile: name=testing selection=default overlay=" + profilePath,
|
||||
"profile: name=testing selection=explicit overlay=" + profilePath,
|
||||
"workspace.root\troot\t" + rootPath,
|
||||
"storage.backend\timport\t" + importPath,
|
||||
"whisperx.language\tprofile\t" + profilePath,
|
||||
|
||||
@@ -37,10 +37,10 @@ func TestRunProfileSelectionFlowsThroughSharedLoader(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if language != "en" {
|
||||
t.Fatalf("default profile language = %q, want en", language)
|
||||
t.Fatalf("base configuration language = %q, want en", language)
|
||||
}
|
||||
|
||||
for _, profile := range []string{"unknown", ""} {
|
||||
for _, profile := range []string{"unknown"} {
|
||||
err := runWithPipelineExecutor(context.Background(), []string{
|
||||
"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--profile", profile,
|
||||
}, io.Discard, executor)
|
||||
@@ -48,19 +48,20 @@ func TestRunProfileSelectionFlowsThroughSharedLoader(t *testing.T) {
|
||||
t.Fatalf("profile %q error = %v, want selection rejection", profile, err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := runWithPipelineExecutor(context.Background(), []string{
|
||||
"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--profile", "",
|
||||
}, io.Discard, executor); err != nil {
|
||||
t.Fatalf("explicit empty profile should select base: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func enableCommandTestProfiles(t *testing.T, pipelinePath string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(pipelinePath)
|
||||
if err != nil {
|
||||
dir := filepath.Join(filepath.Dir(pipelinePath), "profiles")
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
composition := "composition:\n default_profile: production\n profiles:\n production:\n overlay: production.yml\n testing:\n overlay: testing.yml\n"
|
||||
if err := os.WriteFile(pipelinePath, append([]byte(composition), data...), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dir := filepath.Dir(pipelinePath)
|
||||
if err := os.WriteFile(filepath.Join(dir, "production.yml"), []byte("whisperx:\n language: en\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -8,6 +8,15 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func absolutePath(t *testing.T, path string) string {
|
||||
t.Helper()
|
||||
absolute, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return absolute
|
||||
}
|
||||
|
||||
func TestArtifactFamiliesExpandCanonicalCharactersDeterministically(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writePartyResolutionFile(t, dir+"/party.yml", `schema_version: narratio.party.v1
|
||||
@@ -148,7 +157,7 @@ func TestArtifactFamilyStrictDecodeRejectsUnknownFields(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
pipelinePath, _, _ := writeArtifactFamilyConfig(t, dir, artifactFamilyPipelineYAML(validFamilyYAML+" unsupported: true\n"))
|
||||
_, err := LoadPipeline(pipelinePath)
|
||||
if err == nil || !strings.Contains(err.Error(), "field unsupported not found") {
|
||||
if err == nil || !strings.Contains(err.Error(), "decode.unknown_field") {
|
||||
t.Fatalf("LoadPipeline() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,783 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// compositionDocument is the presence-aware representation used while
|
||||
// assembling pipeline configuration sources. It deliberately models YAML
|
||||
// mechanics rather than duplicating PipelineConfig's field schema.
|
||||
type compositionDocument struct {
|
||||
root *compositionNode
|
||||
sources []string
|
||||
}
|
||||
|
||||
type compositionNode struct {
|
||||
kind yaml.Kind
|
||||
tag string
|
||||
value string
|
||||
path string
|
||||
sources []string
|
||||
fields []compositionField
|
||||
items []*compositionNode
|
||||
}
|
||||
|
||||
type compositionField struct {
|
||||
key string
|
||||
value *compositionNode
|
||||
}
|
||||
|
||||
// compositionValueRecord is a deterministic semantic leaf projection. Lists
|
||||
// are atomic configuration values, while mappings are traversed recursively.
|
||||
type compositionValueRecord struct {
|
||||
Path string
|
||||
Kind yaml.Kind
|
||||
Value string
|
||||
Sources []string
|
||||
}
|
||||
|
||||
// parseCompositionDocument parses exactly one YAML mapping while retaining
|
||||
// source ownership, explicit zero values, and declaration order.
|
||||
func parseCompositionDocument(source string, reader io.Reader) (*compositionDocument, error) {
|
||||
if strings.TrimSpace(source) == "" {
|
||||
return nil, fmt.Errorf("configuration source identity is required")
|
||||
}
|
||||
if reader == nil {
|
||||
return nil, fmt.Errorf("configuration source %q: reader is nil", source)
|
||||
}
|
||||
|
||||
decoder := yaml.NewDecoder(reader)
|
||||
var document yaml.Node
|
||||
if err := decoder.Decode(&document); err != nil {
|
||||
if err == io.EOF {
|
||||
return nil, fmt.Errorf("configuration source %q: document is empty", source)
|
||||
}
|
||||
return nil, fmt.Errorf("configuration source %q: decode YAML: %w", source, err)
|
||||
}
|
||||
var trailing yaml.Node
|
||||
if err := decoder.Decode(&trailing); err == nil {
|
||||
return nil, fmt.Errorf("configuration source %q: must contain exactly one YAML document", source)
|
||||
} else if err != io.EOF {
|
||||
return nil, fmt.Errorf("configuration source %q: decode trailing YAML: %w", source, err)
|
||||
}
|
||||
if document.Kind != yaml.DocumentNode || len(document.Content) != 1 {
|
||||
return nil, fmt.Errorf("configuration source %q: must contain exactly one YAML document", source)
|
||||
}
|
||||
if document.Content[0].Kind != yaml.MappingNode {
|
||||
return nil, fmt.Errorf(
|
||||
"configuration source %q: top-level document must be a mapping, got %s",
|
||||
source,
|
||||
yamlKindName(document.Content[0].Kind),
|
||||
)
|
||||
}
|
||||
|
||||
root, err := buildCompositionNode(document.Content[0], source, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &compositionDocument{root: root, sources: []string{source}}, nil
|
||||
}
|
||||
|
||||
func parseCompositionBytes(source string, data []byte) (*compositionDocument, error) {
|
||||
return parseCompositionDocument(source, bytes.NewReader(data))
|
||||
}
|
||||
|
||||
func buildCompositionNode(node *yaml.Node, source, path string) (*compositionNode, error) {
|
||||
if node == nil {
|
||||
return nil, fmt.Errorf("configuration source %q at %s: YAML node is nil", source, displayCompositionPath(path))
|
||||
}
|
||||
if node.Kind == yaml.AliasNode {
|
||||
return nil, compositionNodeError(source, path, node, "YAML aliases are not supported because source ownership would be ambiguous")
|
||||
}
|
||||
result := &compositionNode{
|
||||
kind: node.Kind, tag: node.Tag, value: node.Value, path: path,
|
||||
sources: []string{source},
|
||||
}
|
||||
|
||||
switch node.Kind {
|
||||
case yaml.MappingNode:
|
||||
if len(node.Content)%2 != 0 {
|
||||
return nil, compositionNodeError(source, path, node, "mapping has an incomplete key/value pair")
|
||||
}
|
||||
seen := make(map[string]*yaml.Node, len(node.Content)/2)
|
||||
for index := 0; index < len(node.Content); index += 2 {
|
||||
keyNode := node.Content[index]
|
||||
valueNode := node.Content[index+1]
|
||||
if keyNode.Kind == yaml.AliasNode {
|
||||
return nil, compositionNodeError(source, path, keyNode, "YAML aliases are not supported because source ownership would be ambiguous")
|
||||
}
|
||||
if valueNode.Kind == yaml.AliasNode {
|
||||
return nil, compositionNodeError(source, appendCompositionPath(path, keyNode.Value), valueNode, "YAML aliases are not supported because source ownership would be ambiguous")
|
||||
}
|
||||
if keyNode.Kind != yaml.ScalarNode || keyNode.Tag != "!!str" {
|
||||
return nil, compositionNodeError(source, path, keyNode, "mapping keys must be strings")
|
||||
}
|
||||
key := keyNode.Value
|
||||
fieldPath := appendCompositionPath(path, key)
|
||||
if prior, duplicate := seen[key]; duplicate {
|
||||
return nil, fmt.Errorf(
|
||||
"configuration source %q at %s: duplicate YAML key %q (first declared at line %d, column %d; repeated at line %d, column %d)",
|
||||
source, displayCompositionPath(fieldPath), key,
|
||||
prior.Line, prior.Column, keyNode.Line, keyNode.Column,
|
||||
)
|
||||
}
|
||||
seen[key] = keyNode
|
||||
child, err := buildCompositionNode(valueNode, source, fieldPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result.fields = append(result.fields, compositionField{key: key, value: child})
|
||||
}
|
||||
case yaml.SequenceNode:
|
||||
for index, childNode := range node.Content {
|
||||
childPath := fmt.Sprintf("%s[%d]", path, index)
|
||||
child, err := buildCompositionNode(childNode, source, childPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result.items = append(result.items, child)
|
||||
}
|
||||
case yaml.ScalarNode:
|
||||
// Scalar tag and lexical value retain distinctions such as explicit
|
||||
// false, zero, an empty string, and null until final strict decoding.
|
||||
default:
|
||||
return nil, compositionNodeError(source, path, node, fmt.Sprintf("unsupported YAML node kind %s", yamlKindName(node.Kind)))
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// mergeAdditiveComposition recursively combines disjoint mappings. A scalar,
|
||||
// list, or final keyed value may have only one base owner, regardless of
|
||||
// whether duplicate values happen to be equal.
|
||||
func mergeAdditiveComposition(base, incoming *compositionDocument) (*compositionDocument, error) {
|
||||
return mergeAdditiveCompositions(base, incoming)
|
||||
}
|
||||
|
||||
// mergeAdditiveCompositions validates the complete base source set before
|
||||
// merging so a conflict names every source that claims the same final path.
|
||||
func mergeAdditiveCompositions(documents ...*compositionDocument) (*compositionDocument, error) {
|
||||
if len(documents) == 0 {
|
||||
return nil, fmt.Errorf("configuration additive base merge requires at least one document")
|
||||
}
|
||||
for index, document := range documents {
|
||||
if err := validateCompositionDocument(document, fmt.Sprintf("base[%d]", index)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if err := validateAdditiveClaims(documents); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
result := &compositionDocument{
|
||||
root: cloneCompositionNode(documents[0].root),
|
||||
sources: append([]string(nil), documents[0].sources...),
|
||||
}
|
||||
for _, incoming := range documents[1:] {
|
||||
merged, err := mergeAdditiveNodes(result.root, incoming.root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result.root = merged
|
||||
result.sources = appendUniqueStrings(result.sources, incoming.sources...)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func validateAdditiveClaims(documents []*compositionDocument) error {
|
||||
claims := make(map[string][]*compositionNode)
|
||||
for _, document := range documents {
|
||||
appendCompositionClaims(document.root, claims)
|
||||
}
|
||||
paths := make([]string, 0, len(claims))
|
||||
for path := range claims {
|
||||
paths = append(paths, path)
|
||||
}
|
||||
sort.Slice(paths, func(i, j int) bool {
|
||||
leftDepth := compositionPathDepth(paths[i])
|
||||
rightDepth := compositionPathDepth(paths[j])
|
||||
if leftDepth != rightDepth {
|
||||
return leftDepth < rightDepth
|
||||
}
|
||||
return paths[i] < paths[j]
|
||||
})
|
||||
for _, path := range paths {
|
||||
values := claims[path]
|
||||
if len(values) < 2 {
|
||||
continue
|
||||
}
|
||||
allPopulatedMappings := true
|
||||
for _, value := range values {
|
||||
if value.kind != yaml.MappingNode || len(value.fields) == 0 {
|
||||
allPopulatedMappings = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if !allPopulatedMappings {
|
||||
return newCompositionConflict("additive base merge", path, values...)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func appendCompositionClaims(node *compositionNode, claims map[string][]*compositionNode) {
|
||||
if node == nil {
|
||||
return
|
||||
}
|
||||
if node.path != "" {
|
||||
claims[node.path] = append(claims[node.path], node)
|
||||
}
|
||||
for _, field := range node.fields {
|
||||
appendCompositionClaims(field.value, claims)
|
||||
}
|
||||
}
|
||||
|
||||
func compositionPathDepth(path string) int {
|
||||
if path == "" {
|
||||
return 0
|
||||
}
|
||||
return strings.Count(path, ".") + strings.Count(path, "[") + 1
|
||||
}
|
||||
|
||||
func mergeAdditiveNodes(base, incoming *compositionNode) (*compositionNode, error) {
|
||||
if base.kind != yaml.MappingNode || incoming.kind != yaml.MappingNode {
|
||||
return nil, newCompositionConflict("additive base merge", base.path, base, incoming)
|
||||
}
|
||||
base.sources = appendUniqueStrings(base.sources, incoming.sources...)
|
||||
for _, incomingField := range incoming.fields {
|
||||
index := compositionFieldIndex(base.fields, incomingField.key)
|
||||
if index < 0 {
|
||||
field := cloneCompositionField(incomingField)
|
||||
base.fields = append(base.fields, field)
|
||||
continue
|
||||
}
|
||||
baseValue := base.fields[index].value
|
||||
incomingValue := incomingField.value
|
||||
if baseValue.kind == yaml.MappingNode && incomingValue.kind == yaml.MappingNode {
|
||||
if len(baseValue.fields) == 0 || len(incomingValue.fields) == 0 {
|
||||
return nil, newCompositionConflict("additive base merge", incomingValue.path, baseValue, incomingValue)
|
||||
}
|
||||
merged, err := mergeAdditiveNodes(baseValue, incomingValue)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
base.fields[index].value = merged
|
||||
continue
|
||||
}
|
||||
return nil, newCompositionConflict("additive base merge", incomingValue.path, baseValue, incomingValue)
|
||||
}
|
||||
return base, nil
|
||||
}
|
||||
|
||||
// mergeOverlayComposition applies the sole overwrite layer. Mappings merge
|
||||
// recursively; same-kind scalars and lists replace; null and kind changes are
|
||||
// rejected.
|
||||
func mergeOverlayComposition(base, overlay *compositionDocument) (*compositionDocument, error) {
|
||||
if err := validateCompositionDocument(base, "base"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := validateCompositionDocument(overlay, "overlay"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if null := firstNullCompositionNode(overlay.root); null != nil {
|
||||
return nil, fmt.Errorf(
|
||||
"configuration overlay at %s from %s: null cannot delete an effective value",
|
||||
displayCompositionPath(null.path), formatCompositionSources(null.sources),
|
||||
)
|
||||
}
|
||||
merged, err := mergeOverlayNodes(cloneCompositionNode(base.root), overlay.root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &compositionDocument{
|
||||
root: merged,
|
||||
sources: appendUniqueStrings(
|
||||
append([]string(nil), base.sources...), overlay.sources...,
|
||||
),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func mergeOverlayNodes(base, overlay *compositionNode) (*compositionNode, error) {
|
||||
if base.kind != yaml.MappingNode || overlay.kind != yaml.MappingNode {
|
||||
return nil, newCompositionConflict("profile overlay", base.path, base, overlay)
|
||||
}
|
||||
base.sources = appendUniqueStrings(base.sources, overlay.sources...)
|
||||
for _, overlayField := range overlay.fields {
|
||||
index := compositionFieldIndex(base.fields, overlayField.key)
|
||||
if index < 0 {
|
||||
field := cloneCompositionField(overlayField)
|
||||
base.fields = append(base.fields, field)
|
||||
continue
|
||||
}
|
||||
baseValue := base.fields[index].value
|
||||
overlayValue := overlayField.value
|
||||
if baseValue.kind != overlayValue.kind {
|
||||
return nil, newCompositionConflict("profile overlay kind change", overlayValue.path, baseValue, overlayValue)
|
||||
}
|
||||
if baseValue.kind == yaml.MappingNode {
|
||||
merged, err := mergeOverlayNodes(baseValue, overlayValue)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
base.fields[index].value = merged
|
||||
continue
|
||||
}
|
||||
base.fields[index].value = cloneCompositionNode(overlayValue)
|
||||
}
|
||||
return base, nil
|
||||
}
|
||||
|
||||
// canonicalYAML renders the effective mapping with sorted keys and normalized
|
||||
// presentation while retaining sequence order and scalar YAML types.
|
||||
func (document *compositionDocument) canonicalYAML() ([]byte, error) {
|
||||
if err := validateCompositionDocument(document, "document"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
root, err := compositionYAMLNode(document.root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var buffer bytes.Buffer
|
||||
encoder := yaml.NewEncoder(&buffer)
|
||||
encoder.SetIndent(2)
|
||||
if err := encoder.Encode(root); err != nil {
|
||||
return nil, fmt.Errorf("render effective configuration YAML: %w", err)
|
||||
}
|
||||
if err := encoder.Close(); err != nil {
|
||||
return nil, fmt.Errorf("finish effective configuration YAML: %w", err)
|
||||
}
|
||||
return buffer.Bytes(), nil
|
||||
}
|
||||
|
||||
// canonicalDigestInput provides a deterministic, formatting-independent byte
|
||||
// representation for later secret-free effective configuration digesting.
|
||||
func (document *compositionDocument) canonicalDigestInput() ([]byte, error) {
|
||||
if err := validateCompositionDocument(document, "document"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
value, err := canonicalCompositionValue(document.root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("serialize canonical configuration digest input: %w", err)
|
||||
}
|
||||
return append(data, '\n'), nil
|
||||
}
|
||||
|
||||
// semanticRecords returns sorted atomic values for future effective diff and
|
||||
// source-report projections. A caller receives copies of all source slices.
|
||||
func (document *compositionDocument) semanticRecords() ([]compositionValueRecord, error) {
|
||||
if err := validateCompositionDocument(document, "document"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var records []compositionValueRecord
|
||||
if err := appendCompositionRecords(document.root, &records); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Slice(records, func(i, j int) bool { return records[i].Path < records[j].Path })
|
||||
return records, nil
|
||||
}
|
||||
|
||||
// compactSemanticRecords returns the same logical atomic paths as
|
||||
// semanticRecords, but represents values as compact JSON-compatible YAML
|
||||
// values rather than the typed structural form used for digesting. It is the
|
||||
// stable human-facing projection for semantic comparisons.
|
||||
func (document *compositionDocument) compactSemanticRecords() ([]compositionValueRecord, error) {
|
||||
if err := validateCompositionDocument(document, "document"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var records []compositionValueRecord
|
||||
if err := appendCompactCompositionRecords(document.root, &records); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Slice(records, func(i, j int) bool { return records[i].Path < records[j].Path })
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func appendCompositionRecords(node *compositionNode, records *[]compositionValueRecord) error {
|
||||
if node.kind == yaml.MappingNode && len(node.fields) > 0 {
|
||||
for _, field := range node.fields {
|
||||
if err := appendCompositionRecords(field.value, records); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
value, err := canonicalCompositionValue(node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
encoded, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return fmt.Errorf("serialize configuration value at %s: %w", displayCompositionPath(node.path), err)
|
||||
}
|
||||
*records = append(*records, compositionValueRecord{
|
||||
Path: node.path, Kind: node.kind, Value: string(encoded),
|
||||
Sources: append([]string(nil), node.sources...),
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
func appendCompactCompositionRecords(node *compositionNode, records *[]compositionValueRecord) error {
|
||||
if node.kind == yaml.MappingNode && len(node.fields) > 0 {
|
||||
for _, field := range node.fields {
|
||||
if err := appendCompactCompositionRecords(field.value, records); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
value, err := compactCompositionValue(node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
encoded, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return fmt.Errorf("serialize configuration value at %s: %w", displayCompositionPath(node.path), err)
|
||||
}
|
||||
*records = append(*records, compositionValueRecord{
|
||||
Path: node.path, Kind: node.kind, Value: string(encoded),
|
||||
Sources: append([]string(nil), node.sources...),
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
func compactCompositionValue(node *compositionNode) (any, error) {
|
||||
switch node.kind {
|
||||
case yaml.MappingNode:
|
||||
values := make(map[string]any, len(node.fields))
|
||||
for _, field := range node.fields {
|
||||
value, err := compactCompositionValue(field.value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
values[field.key] = value
|
||||
}
|
||||
return values, nil
|
||||
case yaml.SequenceNode:
|
||||
values := make([]any, 0, len(node.items))
|
||||
for _, item := range node.items {
|
||||
value, err := compactCompositionValue(item)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
values = append(values, value)
|
||||
}
|
||||
return values, nil
|
||||
case yaml.ScalarNode:
|
||||
return canonicalScalarValue(node)
|
||||
default:
|
||||
return nil, fmt.Errorf("configuration at %s has unsupported YAML kind %s", displayCompositionPath(node.path), yamlKindName(node.kind))
|
||||
}
|
||||
}
|
||||
|
||||
type canonicalCompositionField struct {
|
||||
Key string `json:"key"`
|
||||
Value any `json:"value"`
|
||||
}
|
||||
|
||||
type canonicalCompositionNode struct {
|
||||
Kind string `json:"kind"`
|
||||
Tag string `json:"tag,omitempty"`
|
||||
Value any `json:"value,omitempty"`
|
||||
Fields []canonicalCompositionField `json:"fields,omitempty"`
|
||||
Items []any `json:"items,omitempty"`
|
||||
}
|
||||
|
||||
func canonicalCompositionValue(node *compositionNode) (any, error) {
|
||||
switch node.kind {
|
||||
case yaml.MappingNode:
|
||||
fields := append([]compositionField(nil), node.fields...)
|
||||
sort.Slice(fields, func(i, j int) bool { return fields[i].key < fields[j].key })
|
||||
result := canonicalCompositionNode{Kind: "mapping"}
|
||||
if len(fields) == 0 {
|
||||
result.Fields = []canonicalCompositionField{}
|
||||
}
|
||||
for _, field := range fields {
|
||||
value, err := canonicalCompositionValue(field.value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result.Fields = append(result.Fields, canonicalCompositionField{Key: field.key, Value: value})
|
||||
}
|
||||
return result, nil
|
||||
case yaml.SequenceNode:
|
||||
result := canonicalCompositionNode{Kind: "sequence", Items: make([]any, 0, len(node.items))}
|
||||
for _, item := range node.items {
|
||||
value, err := canonicalCompositionValue(item)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result.Items = append(result.Items, value)
|
||||
}
|
||||
return result, nil
|
||||
case yaml.ScalarNode:
|
||||
value, err := canonicalScalarValue(node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return canonicalCompositionNode{Kind: "scalar", Tag: node.tag, Value: value}, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("configuration at %s has unsupported YAML kind %s", displayCompositionPath(node.path), yamlKindName(node.kind))
|
||||
}
|
||||
}
|
||||
|
||||
func canonicalScalarValue(node *compositionNode) (any, error) {
|
||||
raw := &yaml.Node{Kind: yaml.ScalarNode, Tag: node.tag, Value: node.value}
|
||||
var value any
|
||||
if err := raw.Decode(&value); err != nil {
|
||||
return nil, fmt.Errorf("decode scalar at %s: %w", displayCompositionPath(node.path), err)
|
||||
}
|
||||
switch typed := value.(type) {
|
||||
case nil, bool, string, int, int64, uint64, float64:
|
||||
return typed, nil
|
||||
default:
|
||||
// yaml.v3 may decode timestamps or uncommon scalar tags into types that
|
||||
// encoding/json can serialize deterministically. Preserve the resolved
|
||||
// tag alongside the value in the containing canonical node.
|
||||
return typed, nil
|
||||
}
|
||||
}
|
||||
|
||||
func compositionYAMLNode(node *compositionNode) (*yaml.Node, error) {
|
||||
switch node.kind {
|
||||
case yaml.MappingNode:
|
||||
result := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
|
||||
fields := append([]compositionField(nil), node.fields...)
|
||||
sort.Slice(fields, func(i, j int) bool { return fields[i].key < fields[j].key })
|
||||
for _, field := range fields {
|
||||
value, err := compositionYAMLNode(field.value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result.Content = append(result.Content,
|
||||
&yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: field.key},
|
||||
value,
|
||||
)
|
||||
}
|
||||
return result, nil
|
||||
case yaml.SequenceNode:
|
||||
result := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
|
||||
for _, item := range node.items {
|
||||
value, err := compositionYAMLNode(item)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result.Content = append(result.Content, value)
|
||||
}
|
||||
return result, nil
|
||||
case yaml.ScalarNode:
|
||||
return normalizedCompositionScalarNode(node)
|
||||
default:
|
||||
return nil, fmt.Errorf("configuration at %s has unsupported YAML kind %s", displayCompositionPath(node.path), yamlKindName(node.kind))
|
||||
}
|
||||
}
|
||||
|
||||
func normalizedCompositionScalarNode(node *compositionNode) (*yaml.Node, error) {
|
||||
raw := &yaml.Node{Kind: yaml.ScalarNode, Tag: node.tag, Value: node.value}
|
||||
var value any
|
||||
if err := raw.Decode(&value); err != nil {
|
||||
return nil, fmt.Errorf("normalize scalar at %s: %w", displayCompositionPath(node.path), err)
|
||||
}
|
||||
normalized := &yaml.Node{}
|
||||
if err := normalized.Encode(value); err != nil {
|
||||
return nil, fmt.Errorf("encode normalized scalar at %s: %w", displayCompositionPath(node.path), err)
|
||||
}
|
||||
if normalized.Kind != yaml.ScalarNode {
|
||||
return nil, fmt.Errorf("normalize scalar at %s produced YAML kind %s", displayCompositionPath(node.path), yamlKindName(normalized.Kind))
|
||||
}
|
||||
return normalized, nil
|
||||
}
|
||||
|
||||
func validateCompositionDocument(document *compositionDocument, role string) error {
|
||||
if document == nil || document.root == nil {
|
||||
return fmt.Errorf("configuration composition %s document is nil", role)
|
||||
}
|
||||
if document.root.kind != yaml.MappingNode {
|
||||
return fmt.Errorf("configuration composition %s root must be a mapping", role)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func firstNullCompositionNode(node *compositionNode) *compositionNode {
|
||||
if node == nil {
|
||||
return nil
|
||||
}
|
||||
if node.kind == yaml.ScalarNode && node.tag == "!!null" {
|
||||
return node
|
||||
}
|
||||
for _, field := range node.fields {
|
||||
if found := firstNullCompositionNode(field.value); found != nil {
|
||||
return found
|
||||
}
|
||||
}
|
||||
for _, item := range node.items {
|
||||
if found := firstNullCompositionNode(item); found != nil {
|
||||
return found
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func cloneCompositionNode(node *compositionNode) *compositionNode {
|
||||
if node == nil {
|
||||
return nil
|
||||
}
|
||||
clone := &compositionNode{
|
||||
kind: node.kind, tag: node.tag, value: node.value, path: node.path,
|
||||
sources: append([]string(nil), node.sources...),
|
||||
}
|
||||
for _, field := range node.fields {
|
||||
clone.fields = append(clone.fields, cloneCompositionField(field))
|
||||
}
|
||||
for _, item := range node.items {
|
||||
clone.items = append(clone.items, cloneCompositionNode(item))
|
||||
}
|
||||
return clone
|
||||
}
|
||||
|
||||
func cloneCompositionField(field compositionField) compositionField {
|
||||
return compositionField{key: field.key, value: cloneCompositionNode(field.value)}
|
||||
}
|
||||
|
||||
func compositionFieldIndex(fields []compositionField, key string) int {
|
||||
for index := range fields {
|
||||
if fields[index].key == key {
|
||||
return index
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func appendCompositionPath(parent, key string) string {
|
||||
if isSimpleCompositionPathSegment(key) {
|
||||
if parent == "" {
|
||||
return key
|
||||
}
|
||||
return parent + "." + key
|
||||
}
|
||||
if parent == "" {
|
||||
return "[" + strconv.Quote(key) + "]"
|
||||
}
|
||||
return parent + "[" + strconv.Quote(key) + "]"
|
||||
}
|
||||
|
||||
func isSimpleCompositionPathSegment(value string) bool {
|
||||
if value == "" {
|
||||
return false
|
||||
}
|
||||
for index, char := range value {
|
||||
if (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || char == '_' || (index > 0 && char >= '0' && char <= '9') || (index > 0 && char == '-') {
|
||||
continue
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func displayCompositionPath(path string) string {
|
||||
if path == "" {
|
||||
return "<root>"
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func compositionNodeError(source, path string, node *yaml.Node, message string) error {
|
||||
line, column := 0, 0
|
||||
if node != nil {
|
||||
line, column = node.Line, node.Column
|
||||
}
|
||||
return fmt.Errorf(
|
||||
"configuration source %q at %s (line %d, column %d): %s",
|
||||
source, displayCompositionPath(path), line, column, message,
|
||||
)
|
||||
}
|
||||
|
||||
func newCompositionConflict(operation, path string, values ...*compositionNode) error {
|
||||
var sources []string
|
||||
var kinds []string
|
||||
for _, value := range values {
|
||||
if value == nil {
|
||||
continue
|
||||
}
|
||||
sources = appendUniqueStrings(sources, compositionClaimSources(value)...)
|
||||
kind := yamlKindName(value.kind)
|
||||
if !containsString(kinds, kind) {
|
||||
kinds = append(kinds, kind)
|
||||
}
|
||||
}
|
||||
return fmt.Errorf(
|
||||
"configuration %s conflict at %s: claimed by %s (YAML kinds: %s)",
|
||||
operation, displayCompositionPath(path), formatCompositionSources(sources), strings.Join(kinds, ", "),
|
||||
)
|
||||
}
|
||||
|
||||
func compositionClaimSources(node *compositionNode) []string {
|
||||
if node == nil {
|
||||
return nil
|
||||
}
|
||||
sources := append([]string(nil), node.sources...)
|
||||
for _, field := range node.fields {
|
||||
sources = appendUniqueStrings(sources, compositionClaimSources(field.value)...)
|
||||
}
|
||||
for _, item := range node.items {
|
||||
sources = appendUniqueStrings(sources, compositionClaimSources(item)...)
|
||||
}
|
||||
return sources
|
||||
}
|
||||
|
||||
func appendUniqueStrings(values []string, additions ...string) []string {
|
||||
seen := make(map[string]struct{}, len(values)+len(additions))
|
||||
result := make([]string, 0, len(values)+len(additions))
|
||||
for _, value := range append(append([]string(nil), values...), additions...) {
|
||||
if _, exists := seen[value]; exists {
|
||||
continue
|
||||
}
|
||||
seen[value] = struct{}{}
|
||||
result = append(result, value)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func containsString(values []string, target string) bool {
|
||||
for _, value := range values {
|
||||
if value == target {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func formatCompositionSources(sources []string) string {
|
||||
quoted := make([]string, 0, len(sources))
|
||||
for _, source := range sources {
|
||||
quoted = append(quoted, strconv.Quote(source))
|
||||
}
|
||||
return strings.Join(quoted, ", ")
|
||||
}
|
||||
|
||||
func yamlKindName(kind yaml.Kind) string {
|
||||
switch kind {
|
||||
case yaml.DocumentNode:
|
||||
return "document"
|
||||
case yaml.MappingNode:
|
||||
return "mapping"
|
||||
case yaml.SequenceNode:
|
||||
return "sequence"
|
||||
case yaml.ScalarNode:
|
||||
return "scalar"
|
||||
case yaml.AliasNode:
|
||||
return "alias"
|
||||
default:
|
||||
return fmt.Sprintf("kind(%d)", kind)
|
||||
}
|
||||
}
|
||||
@@ -1,353 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func TestParseCompositionDocumentRetainsPresenceOwnershipAndDeclarationOrder(t *testing.T) {
|
||||
document := mustParseComposition(t, "root.yml", `zeta: false
|
||||
zero: 0
|
||||
empty_map: {}
|
||||
empty_list: []
|
||||
nested:
|
||||
value: ""
|
||||
`)
|
||||
|
||||
wantOrder := []string{"zeta", "zero", "empty_map", "empty_list", "nested"}
|
||||
gotOrder := make([]string, 0, len(document.root.fields))
|
||||
for _, field := range document.root.fields {
|
||||
gotOrder = append(gotOrder, field.key)
|
||||
}
|
||||
if !reflect.DeepEqual(gotOrder, wantOrder) {
|
||||
t.Fatalf("declaration order = %#v, want %#v", gotOrder, wantOrder)
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
path string
|
||||
kind yaml.Kind
|
||||
tag string
|
||||
value string
|
||||
}{
|
||||
{path: "zeta", kind: yaml.ScalarNode, tag: "!!bool", value: "false"},
|
||||
{path: "zero", kind: yaml.ScalarNode, tag: "!!int", value: "0"},
|
||||
{path: "empty_map", kind: yaml.MappingNode, tag: "!!map"},
|
||||
{path: "empty_list", kind: yaml.SequenceNode, tag: "!!seq"},
|
||||
{path: "nested.value", kind: yaml.ScalarNode, tag: "!!str", value: ""},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
node := compositionNodeAtPath(t, document.root, tt.path)
|
||||
if node.kind != tt.kind || node.tag != tt.tag || node.value != tt.value || !reflect.DeepEqual(node.sources, []string{"root.yml"}) {
|
||||
t.Fatalf("node %s = kind=%v tag=%q value=%q sources=%#v", tt.path, node.kind, node.tag, node.value, node.sources)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCompositionDocumentRejectsAmbiguousOrMalformedYAML(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
yaml string
|
||||
want string
|
||||
}{
|
||||
{name: "duplicate top-level key", yaml: "value: 1\nvalue: 2\n", want: "duplicate YAML key"},
|
||||
{name: "duplicate nested key", yaml: "outer:\n value: 1\n value: 2\n", want: "outer.value"},
|
||||
{name: "alias", yaml: "base: &base\n value: 1\ncopy: *base\n", want: "aliases are not supported"},
|
||||
{name: "trailing document", yaml: "value: 1\n---\nvalue: 2\n", want: "exactly one YAML document"},
|
||||
{name: "top-level sequence", yaml: "- value\n", want: "top-level document must be a mapping"},
|
||||
{name: "non-string key", yaml: "1: value\n", want: "mapping keys must be strings"},
|
||||
{name: "malformed", yaml: "outer: [\n", want: "decode YAML"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, err := parseCompositionDocument("broken.yml", strings.NewReader(tt.yaml))
|
||||
if err == nil {
|
||||
t.Fatal("parseCompositionDocument() error = nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "broken.yml") || !strings.Contains(err.Error(), tt.want) {
|
||||
t.Fatalf("error = %q, want source and %q", err, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeAdditiveCompositionJoinsOnlyDisjointMappings(t *testing.T) {
|
||||
root := mustParseComposition(t, "pipeline.yml", `scriptorium:
|
||||
artifacts:
|
||||
recap:
|
||||
enabled: true
|
||||
zero: 0
|
||||
`)
|
||||
imports := mustParseComposition(t, "conf.d/artifacts.yml", `scriptorium:
|
||||
artifacts:
|
||||
handout:
|
||||
enabled: false
|
||||
empty_list: []
|
||||
`)
|
||||
|
||||
merged, err := mergeAdditiveComposition(root, imports)
|
||||
if err != nil {
|
||||
t.Fatalf("mergeAdditiveComposition() error = %v", err)
|
||||
}
|
||||
for _, path := range []string{
|
||||
"empty_list", "scriptorium.artifacts.handout.enabled",
|
||||
"scriptorium.artifacts.recap.enabled", "zero",
|
||||
} {
|
||||
_ = compositionNodeAtPath(t, merged.root, path)
|
||||
}
|
||||
if got := compositionNodeAtPath(t, merged.root, "scriptorium.artifacts.handout.enabled").sources; !reflect.DeepEqual(got, []string{"conf.d/artifacts.yml"}) {
|
||||
t.Fatalf("handout sources = %#v", got)
|
||||
}
|
||||
if got := compositionNodeAtPath(t, merged.root, "scriptorium.artifacts.recap.enabled").sources; !reflect.DeepEqual(got, []string{"pipeline.yml"}) {
|
||||
t.Fatalf("recap sources = %#v", got)
|
||||
}
|
||||
|
||||
// Merge operations return a new document and retain the declared order in
|
||||
// each input for source-aware diagnostics.
|
||||
if len(root.root.fields) != 2 || len(imports.root.fields) != 2 {
|
||||
t.Fatalf("merge mutated inputs: root=%d import=%d", len(root.root.fields), len(imports.root.fields))
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeAdditiveCompositionRejectsEveryDuplicateClass(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
baseYAML string
|
||||
nextYAML string
|
||||
path string
|
||||
}{
|
||||
{name: "equal scalar", baseYAML: "value: true\n", nextYAML: "value: true\n", path: "value"},
|
||||
{name: "different scalar", baseYAML: "value: true\n", nextYAML: "value: false\n", path: "value"},
|
||||
{name: "atomic list", baseYAML: "values: [one]\n", nextYAML: "values: [two]\n", path: "values"},
|
||||
{name: "keyed entry", baseYAML: "items:\n shared:\n left: 1\n", nextYAML: "items:\n shared:\n left: 2\n", path: "items.shared.left"},
|
||||
{name: "kind conflict", baseYAML: "value:\n nested: true\n", nextYAML: "value: scalar\n", path: "value"},
|
||||
{name: "duplicate empty map", baseYAML: "value: {}\n", nextYAML: "value: {}\n", path: "value"},
|
||||
{name: "empty map then populated map", baseYAML: "value: {}\n", nextYAML: "value: {nested: true}\n", path: "value"},
|
||||
{name: "populated map then empty map", baseYAML: "value: {nested: true}\n", nextYAML: "value: {}\n", path: "value"},
|
||||
{name: "duplicate empty list", baseYAML: "value: []\n", nextYAML: "value: []\n", path: "value"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
base := mustParseComposition(t, "base.yml", tt.baseYAML)
|
||||
next := mustParseComposition(t, "next.yml", tt.nextYAML)
|
||||
_, err := mergeAdditiveComposition(base, next)
|
||||
if err == nil {
|
||||
t.Fatal("mergeAdditiveComposition() error = nil")
|
||||
}
|
||||
for _, want := range []string{tt.path, "base.yml", "next.yml"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Fatalf("error = %q, want %q", err, want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeAdditiveCompositionReportsAllClaimingSources(t *testing.T) {
|
||||
left := mustParseComposition(t, "left.yml", "group:\n left: 1\n")
|
||||
right := mustParseComposition(t, "right.yml", "group:\n right: 2\n")
|
||||
base, err := mergeAdditiveComposition(left, right)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
overlap := mustParseComposition(t, "overlap.yml", "group: scalar\n")
|
||||
_, err = mergeAdditiveComposition(base, overlap)
|
||||
if err == nil {
|
||||
t.Fatal("mergeAdditiveComposition() error = nil")
|
||||
}
|
||||
for _, want := range []string{"group", "left.yml", "right.yml", "overlap.yml"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Fatalf("error = %q, want %q", err, want)
|
||||
}
|
||||
}
|
||||
|
||||
_, err = mergeAdditiveCompositions(
|
||||
mustParseComposition(t, "first.yml", "value: 1\n"),
|
||||
mustParseComposition(t, "second.yml", "value: 2\n"),
|
||||
mustParseComposition(t, "third.yml", "value: 3\n"),
|
||||
)
|
||||
if err == nil {
|
||||
t.Fatal("mergeAdditiveCompositions() error = nil")
|
||||
}
|
||||
for _, want := range []string{"value", "first.yml", "second.yml", "third.yml"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Fatalf("error = %q, want %q", err, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeOverlayCompositionRecursesMapsAndReplacesAtomicValues(t *testing.T) {
|
||||
base := mustParseComposition(t, "base.yml", `feature:
|
||||
enabled: true
|
||||
retries: 3
|
||||
values: [one, two]
|
||||
inherited: kept
|
||||
artifacts:
|
||||
recap:
|
||||
enabled: true
|
||||
`)
|
||||
overlay := mustParseComposition(t, "testing.yml", `feature:
|
||||
enabled: false
|
||||
retries: 0
|
||||
values: []
|
||||
added: present
|
||||
artifacts:
|
||||
handout:
|
||||
enabled: false
|
||||
`)
|
||||
|
||||
merged, err := mergeOverlayComposition(base, overlay)
|
||||
if err != nil {
|
||||
t.Fatalf("mergeOverlayComposition() error = %v", err)
|
||||
}
|
||||
assertCompositionScalar(t, merged.root, "feature.enabled", "!!bool", "false", "testing.yml")
|
||||
assertCompositionScalar(t, merged.root, "feature.retries", "!!int", "0", "testing.yml")
|
||||
assertCompositionScalar(t, merged.root, "feature.inherited", "!!str", "kept", "base.yml")
|
||||
assertCompositionScalar(t, merged.root, "feature.added", "!!str", "present", "testing.yml")
|
||||
if values := compositionNodeAtPath(t, merged.root, "feature.values"); values.kind != yaml.SequenceNode || len(values.items) != 0 || !reflect.DeepEqual(values.sources, []string{"testing.yml"}) {
|
||||
t.Fatalf("replaced list = %#v", values)
|
||||
}
|
||||
_ = compositionNodeAtPath(t, merged.root, "artifacts.recap.enabled")
|
||||
_ = compositionNodeAtPath(t, merged.root, "artifacts.handout.enabled")
|
||||
}
|
||||
|
||||
func TestMergeOverlayCompositionRejectsKindChangesAndNullDeletion(t *testing.T) {
|
||||
kindTests := []struct {
|
||||
name string
|
||||
baseYAML string
|
||||
overlay string
|
||||
}{
|
||||
{name: "map to scalar", baseYAML: "value: {nested: true}\n", overlay: "value: replacement\n"},
|
||||
{name: "scalar to map", baseYAML: "value: original\n", overlay: "value: {nested: true}\n"},
|
||||
{name: "list to scalar", baseYAML: "value: [one]\n", overlay: "value: replacement\n"},
|
||||
{name: "scalar to list", baseYAML: "value: original\n", overlay: "value: [one]\n"},
|
||||
}
|
||||
for _, tt := range kindTests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, err := mergeOverlayComposition(
|
||||
mustParseComposition(t, "base.yml", tt.baseYAML),
|
||||
mustParseComposition(t, "overlay.yml", tt.overlay),
|
||||
)
|
||||
if err == nil || !strings.Contains(err.Error(), "value") || !strings.Contains(err.Error(), "kind change") {
|
||||
t.Fatalf("error = %v, want value kind change", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
for _, overlayYAML := range []string{"value: null\n", "value: ~\n", "nested:\n value:\n"} {
|
||||
_, err := mergeOverlayComposition(
|
||||
mustParseComposition(t, "base.yml", "value: original\nnested:\n value: original\n"),
|
||||
mustParseComposition(t, "overlay.yml", overlayYAML),
|
||||
)
|
||||
if err == nil || !strings.Contains(err.Error(), "null cannot delete") || !strings.Contains(err.Error(), "overlay.yml") {
|
||||
t.Fatalf("error = %v, want source-qualified null rejection", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompositionCanonicalOutputsAreDeterministic(t *testing.T) {
|
||||
first := mustParseComposition(t, "first.yml", `zeta: 01
|
||||
alpha:
|
||||
list: [true, false]
|
||||
empty: {}
|
||||
`)
|
||||
second := mustParseComposition(t, "second.yml", `alpha:
|
||||
empty: {}
|
||||
list:
|
||||
- true
|
||||
- false
|
||||
zeta: 1
|
||||
`)
|
||||
|
||||
firstYAML, err := first.canonicalYAML()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secondYAML, err := second.canonicalYAML()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(firstYAML, secondYAML) {
|
||||
t.Fatalf("canonical YAML differs:\n%s\n---\n%s", firstYAML, secondYAML)
|
||||
}
|
||||
firstDigest, err := first.canonicalDigestInput()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secondDigest, err := second.canonicalDigestInput()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(firstDigest, secondDigest) {
|
||||
t.Fatalf("digest input differs:\n%s\n---\n%s", firstDigest, secondDigest)
|
||||
}
|
||||
|
||||
left := mustParseComposition(t, "left.yml", "zeta: 1\n")
|
||||
right := mustParseComposition(t, "right.yml", "alpha: 2\n")
|
||||
leftRight, err := mergeAdditiveComposition(left, right)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rightLeft, err := mergeAdditiveComposition(right, left)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want, _ := leftRight.canonicalDigestInput()
|
||||
got, _ := rightLeft.canonicalDigestInput()
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("source traversal changed semantic digest input: got %s want %s", got, want)
|
||||
}
|
||||
wantYAML, _ := leftRight.canonicalYAML()
|
||||
gotYAML, _ := rightLeft.canonicalYAML()
|
||||
if !reflect.DeepEqual(gotYAML, wantYAML) {
|
||||
t.Fatalf("source traversal changed canonical YAML: got %s want %s", gotYAML, wantYAML)
|
||||
}
|
||||
|
||||
records, err := first.semanticRecords()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
paths := make([]string, 0, len(records))
|
||||
for _, record := range records {
|
||||
paths = append(paths, record.Path)
|
||||
}
|
||||
if wantPaths := []string{"alpha.empty", "alpha.list", "zeta"}; !reflect.DeepEqual(paths, wantPaths) {
|
||||
t.Fatalf("semantic record paths = %#v, want %#v", paths, wantPaths)
|
||||
}
|
||||
}
|
||||
|
||||
func mustParseComposition(t *testing.T, source, input string) *compositionDocument {
|
||||
t.Helper()
|
||||
document, err := parseCompositionBytes(source, []byte(input))
|
||||
if err != nil {
|
||||
t.Fatalf("parseCompositionBytes(%q) error = %v", source, err)
|
||||
}
|
||||
return document
|
||||
}
|
||||
|
||||
func compositionNodeAtPath(t *testing.T, root *compositionNode, path string) *compositionNode {
|
||||
t.Helper()
|
||||
node := root
|
||||
for _, segment := range strings.Split(path, ".") {
|
||||
if node == nil || node.kind != yaml.MappingNode {
|
||||
t.Fatalf("path %q reached non-mapping at %q", path, segment)
|
||||
}
|
||||
index := compositionFieldIndex(node.fields, segment)
|
||||
if index < 0 {
|
||||
t.Fatalf("path %q missing segment %q", path, segment)
|
||||
}
|
||||
node = node.fields[index].value
|
||||
}
|
||||
return node
|
||||
}
|
||||
|
||||
func assertCompositionScalar(t *testing.T, root *compositionNode, path, tag, value, source string) {
|
||||
t.Helper()
|
||||
node := compositionNodeAtPath(t, root, path)
|
||||
if node.kind != yaml.ScalarNode || node.tag != tag || node.value != value || !reflect.DeepEqual(node.sources, []string{source}) {
|
||||
t.Fatalf("%s = kind=%s tag=%q value=%q sources=%#v", path, yamlKindName(node.kind), node.tag, node.value, node.sources)
|
||||
}
|
||||
}
|
||||
@@ -191,8 +191,6 @@ type NormalizeConfig struct {
|
||||
OutputPath string `yaml:"output_path" json:"output_path"`
|
||||
OutputSchema string `yaml:"output_schema" json:"output_schema"`
|
||||
Report *bool `yaml:"report" json:"report"`
|
||||
|
||||
outputPathSet bool `yaml:"-" json:"-"`
|
||||
}
|
||||
|
||||
// TrimConfig configures trim-stage transcript boundary behavior.
|
||||
|
||||
@@ -74,7 +74,7 @@ func finalizePipelineResolution(cfg *PipelineConfig) error {
|
||||
|
||||
// recomputePipelineEffectiveDigest is the single package-owned hook for
|
||||
// refreshing provenance after later resolution expands concrete pipeline
|
||||
// values. Composition declarations and runtime provenance are not serialized.
|
||||
// values. Loader directives and runtime provenance are not serialized.
|
||||
func recomputePipelineEffectiveDigest(cfg *PipelineConfig) error {
|
||||
if cfg == nil || cfg.resolution == nil {
|
||||
return fmt.Errorf("pipeline resolution metadata is required")
|
||||
|
||||
@@ -55,7 +55,7 @@ type EffectivePipelineValueRecord struct {
|
||||
// EffectivePipelineValues projects a fully resolved pipeline into sorted
|
||||
// logical configuration values. Mappings are flattened, while sequences remain
|
||||
// atomic values. The projection uses the same normalized effective mapping as
|
||||
// config show and therefore excludes composition and family declarations.
|
||||
// config show and therefore excludes loader directives and family declarations.
|
||||
func EffectivePipelineValues(cfg *PipelineConfig) ([]EffectivePipelineValueRecord, error) {
|
||||
if cfg == nil {
|
||||
return nil, fmt.Errorf("pipeline config is required")
|
||||
@@ -301,6 +301,9 @@ func pipelineSourceRole(cfg *PipelineConfig, source string) string {
|
||||
if source == pipelineDefaultOwnershipSource {
|
||||
return "default"
|
||||
}
|
||||
if source == "resolution" {
|
||||
return "resolution"
|
||||
}
|
||||
if cfg == nil || cfg.resolution == nil {
|
||||
return "pipeline"
|
||||
}
|
||||
@@ -319,7 +322,7 @@ func pipelineSourceRole(cfg *PipelineConfig, source string) string {
|
||||
}
|
||||
|
||||
func normalizedProvenanceSource(source string) string {
|
||||
if source == pipelineDefaultOwnershipSource {
|
||||
if source == pipelineDefaultOwnershipSource || source == "resolution" {
|
||||
return source
|
||||
}
|
||||
return normalizedProvenancePath(source)
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// effectivePipelineProjection is the single safe, schema-specific view used
|
||||
@@ -37,6 +38,11 @@ type effectiveScriptoriumConfig struct {
|
||||
Artifacts map[string]ScriptoriumArtifactConfig `yaml:"artifacts" json:"artifacts"`
|
||||
}
|
||||
|
||||
type projectedPipelineValue struct {
|
||||
record EffectivePipelineValueRecord
|
||||
segments []string
|
||||
}
|
||||
|
||||
func projectEffectivePipeline(cfg *PipelineConfig, logicalNotarius bool) effectivePipelineProjection {
|
||||
projection := effectivePipelineProjection{
|
||||
Workspace: cfg.Workspace, Campaigns: cfg.Campaigns, Storage: cfg.Storage,
|
||||
@@ -70,23 +76,35 @@ func projectionJSON(projection effectivePipelineProjection) ([]byte, error) {
|
||||
}
|
||||
|
||||
func projectionValueRecords(projection effectivePipelineProjection) ([]EffectivePipelineValueRecord, error) {
|
||||
data, err := projectionJSON(projection)
|
||||
detailed, err := projectionDetailedRecords(projection)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
records := make([]EffectivePipelineValueRecord, 0, len(detailed))
|
||||
for _, value := range detailed {
|
||||
records = append(records, value.record)
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func projectionDetailedRecords(value any) ([]projectedPipelineValue, error) {
|
||||
data, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("encode typed pipeline projection: %w", err)
|
||||
}
|
||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||
decoder.UseNumber()
|
||||
var root map[string]any
|
||||
if err := decoder.Decode(&root); err != nil {
|
||||
return nil, fmt.Errorf("decode effective pipeline projection: %w", err)
|
||||
}
|
||||
var records []EffectivePipelineValueRecord
|
||||
appendProjectionRecords(&records, "", root)
|
||||
sort.Slice(records, func(i, j int) bool { return records[i].Path < records[j].Path })
|
||||
var records []projectedPipelineValue
|
||||
appendProjectionRecords(&records, nil, root)
|
||||
sort.Slice(records, func(i, j int) bool { return records[i].record.Path < records[j].record.Path })
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func appendProjectionRecords(records *[]EffectivePipelineValueRecord, path string, value any) {
|
||||
func appendProjectionRecords(records *[]projectedPipelineValue, segments []string, value any) {
|
||||
if mapping, ok := value.(map[string]any); ok && len(mapping) > 0 {
|
||||
keys := make([]string, 0, len(mapping))
|
||||
for key := range mapping {
|
||||
@@ -94,14 +112,15 @@ func appendProjectionRecords(records *[]EffectivePipelineValueRecord, path strin
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
childPath := key
|
||||
if path != "" {
|
||||
childPath = path + "." + key
|
||||
}
|
||||
appendProjectionRecords(records, childPath, mapping[key])
|
||||
child := append(append([]string(nil), segments...), key)
|
||||
appendProjectionRecords(records, child, mapping[key])
|
||||
}
|
||||
return
|
||||
}
|
||||
encoded, _ := json.Marshal(value)
|
||||
*records = append(*records, EffectivePipelineValueRecord{Path: path, Value: string(encoded)})
|
||||
path := strings.Join(segments, ".")
|
||||
*records = append(*records, projectedPipelineValue{
|
||||
record: EffectivePipelineValueRecord{Path: path, Value: string(encoded)},
|
||||
segments: append([]string(nil), segments...),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -17,9 +17,8 @@ func LoadPipeline(path string) (*PipelineConfig, error) {
|
||||
return LoadPipelineWithOptions(path, PipelineLoadOptions{})
|
||||
}
|
||||
|
||||
// PipelineLoadOptions carries an optional explicit profile selection. A nil
|
||||
// Profile means the caller omitted selection; a non-nil empty value is an
|
||||
// explicit invalid selection.
|
||||
// PipelineLoadOptions carries an optional explicit profile selection. Nil,
|
||||
// empty, and the exact name "default" select the base configuration.
|
||||
type PipelineLoadOptions struct {
|
||||
Profile *string
|
||||
}
|
||||
@@ -27,77 +26,32 @@ type PipelineLoadOptions struct {
|
||||
// LoadPipelineWithOptions loads pipeline configuration with strict field
|
||||
// checking and optional named-profile selection.
|
||||
func LoadPipelineWithOptions(path string, opts PipelineLoadOptions) (*PipelineConfig, error) {
|
||||
sources, err := loadPipelineCompositionSources(path)
|
||||
cfg, err := loadPipelineWithConfigKit(path, opts)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load pipeline config: %w", err)
|
||||
}
|
||||
if _, err := selectPipelineProfile(sources.envelope, opts); err != nil {
|
||||
return nil, fmt.Errorf("load pipeline config: %w", err)
|
||||
}
|
||||
if err := sources.loadOverlays(); err != nil {
|
||||
return nil, fmt.Errorf("load pipeline config: %w", err)
|
||||
}
|
||||
cfg, err := sources.resolve(opts)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load pipeline config: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
return finalizeLoadedPipeline(path, cfg)
|
||||
}
|
||||
|
||||
// LoadPipelineProfilePair resolves two explicit named profiles from one parsed
|
||||
// pipeline root and its declared source set. Each result is independently
|
||||
// decoded, defaulted, and finalized so later resolution can safely mutate one
|
||||
// effective pipeline without affecting the other.
|
||||
// LoadPipelineProfilePair resolves two profiles as independent ConfigKit loads.
|
||||
func LoadPipelineProfilePair(path, leftProfile, rightProfile string) (*PipelineConfig, *PipelineConfig, error) {
|
||||
if _, err := normalizePipelineProfileName(leftProfile, "left profile selection"); err != nil {
|
||||
return nil, nil, fmt.Errorf("load left pipeline profile: %w", err)
|
||||
}
|
||||
if _, err := normalizePipelineProfileName(rightProfile, "right profile selection"); err != nil {
|
||||
return nil, nil, fmt.Errorf("load right pipeline profile: %w", err)
|
||||
}
|
||||
if leftProfile == rightProfile {
|
||||
return nil, nil, fmt.Errorf("load pipeline profiles: left and right profile selections must differ")
|
||||
}
|
||||
sources, err := loadPipelineCompositionSources(path)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("load pipeline config: %w", err)
|
||||
}
|
||||
leftOptions := PipelineLoadOptions{Profile: &leftProfile}
|
||||
rightOptions := PipelineLoadOptions{Profile: &rightProfile}
|
||||
if _, err := selectPipelineProfile(sources.envelope, leftOptions); err != nil {
|
||||
return nil, nil, fmt.Errorf("load left pipeline profile: %w", err)
|
||||
}
|
||||
if _, err := selectPipelineProfile(sources.envelope, rightOptions); err != nil {
|
||||
return nil, nil, fmt.Errorf("load right pipeline profile: %w", err)
|
||||
}
|
||||
if err := sources.loadOverlays(); err != nil {
|
||||
return nil, nil, fmt.Errorf("load pipeline config: %w", err)
|
||||
}
|
||||
left, err := sources.resolve(leftOptions)
|
||||
left, err := LoadPipelineWithOptions(path, leftOptions)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("load left pipeline profile: %w", err)
|
||||
}
|
||||
right, err := sources.resolve(rightOptions)
|
||||
right, err := LoadPipelineWithOptions(path, rightOptions)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("load right pipeline profile: %w", err)
|
||||
}
|
||||
left, err = finalizeLoadedPipeline(path, left)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
right, err = finalizeLoadedPipeline(path, right)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return left, right, nil
|
||||
}
|
||||
|
||||
func finalizeLoadedPipeline(path string, cfg *PipelineConfig) (*PipelineConfig, error) {
|
||||
cfg.resolution.publishDeclared = cfg.Publish != nil
|
||||
applyPipelineDefaults(cfg)
|
||||
if err := NormalizePipelineConfig(cfg); err != nil {
|
||||
return nil, fmt.Errorf("load pipeline config: %w", err)
|
||||
}
|
||||
captureLogicalNotariusPaths(cfg)
|
||||
if err := resolveNotariusPaths(cfg, path); err != nil {
|
||||
return nil, fmt.Errorf("load pipeline config: %w", err)
|
||||
@@ -504,217 +458,6 @@ func shortName(path, fallback string) string {
|
||||
return base
|
||||
}
|
||||
|
||||
func applyPipelineDefaults(cfg *PipelineConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
applyWorkspaceDefaults(&cfg.Workspace)
|
||||
applyCampaignsDefaults(&cfg.Campaigns)
|
||||
applyStorageDefaults(&cfg.Storage)
|
||||
applySpoolDefaults(&cfg.Spool)
|
||||
applyCacheDefaults(&cfg.Cache)
|
||||
applyPublishDefaults(&cfg.Publish)
|
||||
applyWhisperXDefaults(&cfg.WhisperX)
|
||||
applySeriatimDefaults(&cfg.Seriatim)
|
||||
applyAuditaDefaults(&cfg.Audita)
|
||||
if cfg.Normalize == nil {
|
||||
cfg.Normalize = &NormalizeConfig{}
|
||||
}
|
||||
applyNormalizeDefaults(cfg.Normalize)
|
||||
if cfg.Trim == nil {
|
||||
cfg.Trim = &TrimConfig{}
|
||||
}
|
||||
applyTrimDefaults(cfg.Trim)
|
||||
if trimEnabled(cfg.Trim) && cfg.Scriptorium == nil {
|
||||
cfg.Scriptorium = &ScriptoriumConfig{}
|
||||
}
|
||||
applyRenderDefaults(&cfg.Render)
|
||||
applyScriptoriumDefaults(cfg.Scriptorium)
|
||||
applyNotariusDefaults(cfg.Notarius)
|
||||
applyNotificationDefaults(&cfg.Notification)
|
||||
}
|
||||
|
||||
func applyNotificationDefaults(cfg *NotificationConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
if strings.TrimSpace(cfg.Mode) == "" {
|
||||
cfg.Mode = DefaultNotificationMode
|
||||
}
|
||||
}
|
||||
|
||||
func applyCampaignsDefaults(cfg *CampaignsConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
if cfg.Root == "" {
|
||||
cfg.Root = DefaultCampaignsRoot
|
||||
}
|
||||
}
|
||||
|
||||
func applyWorkspaceDefaults(cfg *WorkspaceConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
if cfg.Root == "" {
|
||||
cfg.Root = DefaultWorkspaceRoot
|
||||
}
|
||||
}
|
||||
|
||||
func applyStorageDefaults(cfg *StorageConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
backend := strings.ToLower(strings.TrimSpace(cfg.Backend))
|
||||
if backend == "" {
|
||||
backend = StorageBackendLocal
|
||||
}
|
||||
cfg.Backend = backend
|
||||
if cfg.S3 == nil {
|
||||
return
|
||||
}
|
||||
if cfg.S3.RootPrefix == "" {
|
||||
cfg.S3.RootPrefix = DefaultStorageS3RootPrefix
|
||||
}
|
||||
if cfg.S3.AccessKeyIDEnv == "" {
|
||||
cfg.S3.AccessKeyIDEnv = DefaultS3AccessKeyIDEnv
|
||||
}
|
||||
if cfg.S3.SecretKeyEnv == "" {
|
||||
cfg.S3.SecretKeyEnv = DefaultS3SecretAccessKeyEnv
|
||||
}
|
||||
}
|
||||
|
||||
func applySpoolDefaults(cfg *SpoolConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
if cfg.Root == "" {
|
||||
cfg.Root = DefaultSpoolRoot
|
||||
}
|
||||
}
|
||||
|
||||
func applyCacheDefaults(cfg *CacheConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
if cfg.Root == "" {
|
||||
cfg.Root = DefaultCacheRoot
|
||||
}
|
||||
if cfg.S3Audio == nil {
|
||||
cfg.S3Audio = boolPtr(DefaultCacheS3Audio)
|
||||
}
|
||||
}
|
||||
|
||||
func applyPublishDefaults(cfg **PublishConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
if *cfg == nil {
|
||||
*cfg = &PublishConfig{}
|
||||
}
|
||||
|
||||
if (*cfg).Enabled == nil {
|
||||
(*cfg).Enabled = boolPtr(DefaultArchiveEnabled)
|
||||
}
|
||||
if (*cfg).UploadRun == nil {
|
||||
(*cfg).UploadRun = boolPtr(DefaultArchiveUploadRun)
|
||||
}
|
||||
if len((*cfg).Outputs) == 0 {
|
||||
(*cfg).Outputs = append([]PublishOutputRule(nil), DefaultPublishOutputs...)
|
||||
}
|
||||
for i := range (*cfg).Outputs {
|
||||
if (*cfg).Outputs[i].Required == nil {
|
||||
(*cfg).Outputs[i].Required = boolPtr(true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func applyWhisperXDefaults(cfg *WhisperXConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
if cfg.Language == "" {
|
||||
cfg.Language = DefaultWhisperXLanguage
|
||||
}
|
||||
if cfg.Timeout == "" {
|
||||
cfg.Timeout = DefaultWhisperXTimeout
|
||||
}
|
||||
if cfg.RetryDelay == "" {
|
||||
cfg.RetryDelay = DefaultWhisperXRetryDelay
|
||||
}
|
||||
if cfg.Concurrency == nil {
|
||||
cfg.Concurrency = intPtr(DefaultWhisperXConcurrency)
|
||||
}
|
||||
if cfg.Retries == nil {
|
||||
cfg.Retries = intPtr(DefaultWhisperXRetries)
|
||||
}
|
||||
}
|
||||
|
||||
func intPtr(v int) *int {
|
||||
p := v
|
||||
return &p
|
||||
}
|
||||
|
||||
func applySeriatimDefaults(cfg *SeriatimConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
if cfg.Binary == "" {
|
||||
cfg.Binary = DefaultSeriatimBinary
|
||||
}
|
||||
if cfg.Timeout == "" {
|
||||
cfg.Timeout = DefaultSeriatimTimeout
|
||||
}
|
||||
if cfg.OutputSchema == "" {
|
||||
cfg.OutputSchema = DefaultSeriatimOutputSchema
|
||||
}
|
||||
if cfg.CoalesceGap == nil {
|
||||
cfg.CoalesceGap = float64Ptr(DefaultSeriatimCoalesceGap)
|
||||
}
|
||||
if cfg.Report == nil {
|
||||
cfg.Report = boolPtr(DefaultSeriatimReport)
|
||||
}
|
||||
}
|
||||
|
||||
func applyAuditaDefaults(cfg *AuditaConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
if cfg.Binary == "" {
|
||||
cfg.Binary = DefaultAuditaBinary
|
||||
}
|
||||
if cfg.Timeout == "" {
|
||||
cfg.Timeout = DefaultAuditaTimeout
|
||||
}
|
||||
if cfg.Report == nil {
|
||||
cfg.Report = boolPtr(DefaultAuditaReport)
|
||||
}
|
||||
}
|
||||
|
||||
func applyScriptoriumDefaults(cfg *ScriptoriumConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
if cfg.Binary == "" {
|
||||
cfg.Binary = DefaultScriptoriumBinary
|
||||
}
|
||||
if cfg.Timeout == "" {
|
||||
cfg.Timeout = DefaultScriptoriumTimeout
|
||||
}
|
||||
}
|
||||
|
||||
func applyNotariusDefaults(cfg *NotariusConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
if cfg.Binary == "" {
|
||||
cfg.Binary = DefaultNotariusBinary
|
||||
}
|
||||
if cfg.Timeout == "" {
|
||||
cfg.Timeout = DefaultNotariusTimeout
|
||||
}
|
||||
}
|
||||
|
||||
func resolveNotariusPaths(cfg *PipelineConfig, pipelinePath string) error {
|
||||
if cfg == nil || cfg.Notarius == nil || !cfg.Notarius.Enabled {
|
||||
return nil
|
||||
@@ -753,83 +496,3 @@ func resolveNotariusPaths(cfg *PipelineConfig, pipelinePath string) error {
|
||||
cfg.Notarius.WorkingDirectory = workingDirectory
|
||||
return nil
|
||||
}
|
||||
|
||||
func applyTrimDefaults(cfg *TrimConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
if cfg.Enabled == nil {
|
||||
cfg.Enabled = boolPtr(DefaultTrimEnabled)
|
||||
}
|
||||
if strings.TrimSpace(cfg.OutputPath) == "" {
|
||||
cfg.OutputPath = DefaultTrimOutputPath
|
||||
}
|
||||
if strings.TrimSpace(cfg.Bounds.PromptID) == "" {
|
||||
cfg.Bounds.PromptID = DefaultTrimBoundsPromptID
|
||||
}
|
||||
if strings.TrimSpace(cfg.Bounds.TranscriptInputName) == "" {
|
||||
cfg.Bounds.TranscriptInputName = DefaultTrimBoundsTranscriptInputName
|
||||
}
|
||||
if strings.TrimSpace(cfg.Bounds.OutputPath) == "" {
|
||||
cfg.Bounds.OutputPath = DefaultTrimBoundsOutputPath
|
||||
}
|
||||
if cfg.Bounds.Timeout == "" {
|
||||
cfg.Bounds.Timeout = DefaultTrimBoundsTimeout
|
||||
}
|
||||
if cfg.Seriatim.Report == nil {
|
||||
cfg.Seriatim.Report = boolPtr(DefaultTrimSeriatimReport)
|
||||
}
|
||||
}
|
||||
|
||||
func trimEnabled(cfg *TrimConfig) bool {
|
||||
return cfg != nil && cfg.Enabled != nil && *cfg.Enabled
|
||||
}
|
||||
|
||||
func applyRenderDefaults(cfg **RenderConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
if *cfg == nil {
|
||||
*cfg = &RenderConfig{}
|
||||
}
|
||||
if (*cfg).Enabled == nil {
|
||||
(*cfg).Enabled = boolPtr(DefaultRenderEnabled)
|
||||
}
|
||||
if strings.TrimSpace((*cfg).Format) == "" {
|
||||
(*cfg).Format = DefaultRenderFormat
|
||||
}
|
||||
if strings.TrimSpace((*cfg).Title) == "" {
|
||||
(*cfg).Title = DefaultRenderTitle
|
||||
}
|
||||
if (*cfg).IncludeTimestamps == nil {
|
||||
(*cfg).IncludeTimestamps = boolPtr(DefaultRenderTimestamps)
|
||||
}
|
||||
if (*cfg).IncludeSegmentIDs == nil {
|
||||
(*cfg).IncludeSegmentIDs = boolPtr(DefaultRenderSegmentIDs)
|
||||
}
|
||||
}
|
||||
|
||||
func applyNormalizeDefaults(cfg *NormalizeConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
if cfg.OutputSchema == "" {
|
||||
cfg.OutputSchema = DefaultNormalizeOutputSchema
|
||||
}
|
||||
if cfg.OutputPath == "" && !cfg.outputPathWasSet() {
|
||||
cfg.OutputPath = DefaultNormalizeOutputPath
|
||||
}
|
||||
if cfg.Report == nil {
|
||||
cfg.Report = boolPtr(DefaultNormalizeReport)
|
||||
}
|
||||
}
|
||||
|
||||
func float64Ptr(v float64) *float64 {
|
||||
p := v
|
||||
return &p
|
||||
}
|
||||
|
||||
func boolPtr(v bool) *bool {
|
||||
p := v
|
||||
return &p
|
||||
}
|
||||
|
||||
@@ -48,7 +48,7 @@ func TestLoadersRejectTrailingYAMLDocuments(t *testing.T) {
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := tt.load()
|
||||
if err == nil || !strings.Contains(err.Error(), "exactly one YAML document") {
|
||||
if err == nil || (!strings.Contains(err.Error(), "exactly one YAML document") && !strings.Contains(err.Error(), "yaml.multiple_documents")) {
|
||||
t.Fatalf("load error = %v, want exactly-one-document error", err)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -1,49 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// UnmarshalYAML tracks explicit normalize.output_path presence so validation can
|
||||
// distinguish omitted vs explicitly empty values.
|
||||
func (cfg *NormalizeConfig) UnmarshalYAML(node *yaml.Node) error {
|
||||
if node.Kind != yaml.MappingNode {
|
||||
return fmt.Errorf("normalize must be a mapping")
|
||||
}
|
||||
allowedKeys := map[string]struct{}{
|
||||
"output_path": {},
|
||||
"output_schema": {},
|
||||
"report": {},
|
||||
}
|
||||
for i := 0; i+1 < len(node.Content); i += 2 {
|
||||
key := node.Content[i].Value
|
||||
if _, ok := allowedKeys[key]; !ok {
|
||||
return fmt.Errorf("field %q not found in type config.NormalizeConfig", key)
|
||||
}
|
||||
}
|
||||
|
||||
type rawNormalize NormalizeConfig
|
||||
var raw rawNormalize
|
||||
if err := node.Decode(&raw); err != nil {
|
||||
return err
|
||||
}
|
||||
*cfg = NormalizeConfig(raw)
|
||||
for i := 0; i+1 < len(node.Content); i += 2 {
|
||||
if node.Content[i].Value == "output_path" {
|
||||
cfg.outputPathSet = true
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// outputPathSet records whether normalize.output_path appeared in YAML.
|
||||
// This is intentionally unexported and not serialized.
|
||||
func (cfg *NormalizeConfig) outputPathWasSet() bool {
|
||||
if cfg == nil {
|
||||
return false
|
||||
}
|
||||
return cfg.outputPathSet
|
||||
}
|
||||
@@ -107,6 +107,12 @@ notarius:
|
||||
enabled: true
|
||||
config_path: notarius/config.yml
|
||||
pipeline_id: dnd-session
|
||||
outputs:
|
||||
npc_registry:
|
||||
lane_id: npc-registry
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.npc_registry
|
||||
schema_version: v1
|
||||
`
|
||||
paths := make([]string, 2)
|
||||
configs := make([]*PipelineConfig, 2)
|
||||
@@ -151,7 +157,7 @@ func TestNotariusStrictYAML(t *testing.T) {
|
||||
_, err := LoadPipeline(path)
|
||||
want := "strict decode failed"
|
||||
if tt.name == "duplicate reference selector" {
|
||||
want = "duplicate YAML key"
|
||||
want = "yaml.duplicate_key"
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), want) {
|
||||
t.Fatalf("LoadPipeline() error = %v, want containing %q", err, want)
|
||||
|
||||
@@ -23,12 +23,12 @@ func TestNotificationConfigSupportsOnlyNoopMode(t *testing.T) {
|
||||
{
|
||||
name: "backend is rejected",
|
||||
section: "notification:\n backend: email\n",
|
||||
wantLoad: "field backend not found",
|
||||
wantLoad: "decode.unknown_field",
|
||||
},
|
||||
{
|
||||
name: "recipient is rejected",
|
||||
section: "notification:\n recipient: party@example.com\n",
|
||||
wantLoad: "field recipient not found",
|
||||
wantLoad: "decode.unknown_field",
|
||||
},
|
||||
{
|
||||
name: "provider mode is rejected",
|
||||
|
||||
@@ -1,532 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type pipelineResolutionMetadata struct {
|
||||
rootPath string
|
||||
imports []string
|
||||
sources []string
|
||||
selectedProfile *pipelineProfileSelection
|
||||
effectiveDigest string
|
||||
logicalNotariusConfig string
|
||||
logicalNotariusWorking string
|
||||
logicalNotariusCaptured bool
|
||||
ownership []pipelineFieldOwnership
|
||||
artifactFamilies map[string]ScriptoriumArtifactFamilyConfig
|
||||
artifactFamiliesExpanded bool
|
||||
publishDeclared bool
|
||||
}
|
||||
|
||||
type pipelineProfileSelection struct {
|
||||
name string
|
||||
source string
|
||||
overlayPath string
|
||||
}
|
||||
|
||||
type pipelineFieldOwnership struct {
|
||||
path string
|
||||
sources []string
|
||||
}
|
||||
|
||||
type pipelineCompositionEnvelope struct {
|
||||
imports []string
|
||||
defaultProfile *string
|
||||
profiles []pipelineProfileDeclaration
|
||||
}
|
||||
|
||||
type pipelineProfileDeclaration struct {
|
||||
name string
|
||||
overlay string
|
||||
}
|
||||
|
||||
// pipelineCompositionSources retains one validated root source set. Each
|
||||
// selected profile is resolved from a cloned base document so callers can
|
||||
// safely compare or otherwise resolve multiple profiles without rereading or
|
||||
// mutating the source set.
|
||||
type pipelineCompositionSources struct {
|
||||
rootPath string
|
||||
envelope pipelineCompositionEnvelope
|
||||
imports []loadedPipelineImport
|
||||
overlays []loadedPipelineProfileOverlay
|
||||
overlaysLoaded bool
|
||||
base *compositionDocument
|
||||
}
|
||||
|
||||
func loadPipelineCompositionSources(path string) (*pipelineCompositionSources, error) {
|
||||
rootPath, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve root pipeline path %q: %w", path, err)
|
||||
}
|
||||
rootFile, err := os.Open(rootPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("pipeline file %q: open: %w", path, err)
|
||||
}
|
||||
rootDocument, parseErr := parseCompositionDocument(rootPath, rootFile)
|
||||
closeErr := rootFile.Close()
|
||||
if parseErr != nil {
|
||||
return nil, parseErr
|
||||
}
|
||||
if closeErr != nil {
|
||||
return nil, fmt.Errorf("pipeline file %q: close: %w", rootPath, closeErr)
|
||||
}
|
||||
|
||||
baseRoot, envelope, err := splitPipelineCompositionEnvelope(rootDocument)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
imports, err := loadPipelineImports(rootPath, envelope.imports)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
documents := make([]*compositionDocument, 0, len(imports)+1)
|
||||
documents = append(documents, baseRoot)
|
||||
for _, imported := range imports {
|
||||
documents = append(documents, imported.document)
|
||||
}
|
||||
merged, err := mergeAdditiveCompositions(documents...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &pipelineCompositionSources{
|
||||
rootPath: rootPath,
|
||||
envelope: envelope,
|
||||
imports: imports,
|
||||
base: merged,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (sources *pipelineCompositionSources) loadOverlays() error {
|
||||
if sources == nil {
|
||||
return fmt.Errorf("pipeline composition sources are required")
|
||||
}
|
||||
if sources.overlaysLoaded {
|
||||
return nil
|
||||
}
|
||||
overlays, err := loadPipelineProfileOverlays(sources.rootPath, sources.envelope.profiles, sources.imports)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sources.overlays = overlays
|
||||
sources.overlaysLoaded = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sources *pipelineCompositionSources) resolve(opts PipelineLoadOptions) (*PipelineConfig, error) {
|
||||
if sources == nil || sources.base == nil {
|
||||
return nil, fmt.Errorf("pipeline composition sources are required")
|
||||
}
|
||||
if !sources.overlaysLoaded {
|
||||
return nil, fmt.Errorf("pipeline profile overlays have not been loaded")
|
||||
}
|
||||
selection, err := selectPipelineProfile(sources.envelope, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
merged := &compositionDocument{
|
||||
root: cloneCompositionNode(sources.base.root),
|
||||
sources: append([]string(nil), sources.base.sources...),
|
||||
}
|
||||
if selection != nil {
|
||||
overlay, ok := loadedProfileOverlay(sources.overlays, selection.name)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("selected profile %q overlay was not loaded", selection.name)
|
||||
}
|
||||
merged, err = mergeOverlayComposition(merged, overlay.document)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
selection.overlayPath = overlay.path
|
||||
}
|
||||
|
||||
rendered, err := merged.canonicalYAML()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var cfg PipelineConfig
|
||||
if err := decodeStrictYAMLFromReader("pipeline", sources.rootPath, strings.NewReader(string(rendered)), &cfg); err != nil {
|
||||
return nil, fmt.Errorf("assembled pipeline sources %s: %w", formatCompositionSources(merged.sources), err)
|
||||
}
|
||||
records, err := merged.semanticRecords()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
metadata := &pipelineResolutionMetadata{
|
||||
rootPath: sources.rootPath,
|
||||
sources: append([]string(nil), merged.sources...),
|
||||
selectedProfile: selection,
|
||||
}
|
||||
for _, imported := range sources.imports {
|
||||
metadata.imports = append(metadata.imports, imported.path)
|
||||
}
|
||||
for _, record := range records {
|
||||
metadata.ownership = append(metadata.ownership, pipelineFieldOwnership{
|
||||
path: record.Path, sources: append([]string(nil), record.Sources...),
|
||||
})
|
||||
}
|
||||
cfg.resolution = metadata
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
func splitPipelineCompositionEnvelope(document *compositionDocument) (*compositionDocument, pipelineCompositionEnvelope, error) {
|
||||
if err := validateCompositionDocument(document, "root pipeline"); err != nil {
|
||||
return nil, pipelineCompositionEnvelope{}, err
|
||||
}
|
||||
envelope, fieldIndex, err := decodePipelineCompositionEnvelope(document.root)
|
||||
if err != nil {
|
||||
return nil, pipelineCompositionEnvelope{}, err
|
||||
}
|
||||
if err := validatePipelineCompositionEnvelope(envelope); err != nil {
|
||||
return nil, pipelineCompositionEnvelope{}, err
|
||||
}
|
||||
return pipelineBodyWithoutComposition(document, fieldIndex), envelope, nil
|
||||
}
|
||||
|
||||
func decodePipelineCompositionEnvelope(root *compositionNode) (pipelineCompositionEnvelope, int, error) {
|
||||
index := compositionFieldIndex(root.fields, "composition")
|
||||
if index < 0 {
|
||||
return pipelineCompositionEnvelope{}, -1, nil
|
||||
}
|
||||
envelopeNode := root.fields[index].value
|
||||
if envelopeNode.kind != yaml.MappingNode {
|
||||
return pipelineCompositionEnvelope{}, -1, fmt.Errorf(
|
||||
"configuration source %s at composition: expected a mapping, got %s",
|
||||
formatCompositionSources(envelopeNode.sources), yamlKindName(envelopeNode.kind),
|
||||
)
|
||||
}
|
||||
|
||||
var envelope pipelineCompositionEnvelope
|
||||
for _, field := range envelopeNode.fields {
|
||||
switch field.key {
|
||||
case "imports":
|
||||
if field.value.kind != yaml.SequenceNode {
|
||||
return pipelineCompositionEnvelope{}, -1, fmt.Errorf(
|
||||
"configuration source %s at composition.imports: expected a list, got %s",
|
||||
formatCompositionSources(field.value.sources), yamlKindName(field.value.kind),
|
||||
)
|
||||
}
|
||||
for itemIndex, item := range field.value.items {
|
||||
if item.kind != yaml.ScalarNode || item.tag != "!!str" {
|
||||
return pipelineCompositionEnvelope{}, -1, fmt.Errorf(
|
||||
"configuration source %s at composition.imports[%d]: expected a string path",
|
||||
formatCompositionSources(item.sources), itemIndex,
|
||||
)
|
||||
}
|
||||
envelope.imports = append(envelope.imports, item.value)
|
||||
}
|
||||
case "default_profile":
|
||||
if field.value.kind != yaml.ScalarNode || field.value.tag != "!!str" {
|
||||
return pipelineCompositionEnvelope{}, -1, fmt.Errorf(
|
||||
"configuration source %s at composition.default_profile: expected a string profile name",
|
||||
formatCompositionSources(field.value.sources),
|
||||
)
|
||||
}
|
||||
name, err := normalizePipelineProfileName(field.value.value, "composition.default_profile")
|
||||
if err != nil {
|
||||
return pipelineCompositionEnvelope{}, -1, err
|
||||
}
|
||||
envelope.defaultProfile = &name
|
||||
case "profiles":
|
||||
if field.value.kind != yaml.MappingNode {
|
||||
return pipelineCompositionEnvelope{}, -1, fmt.Errorf(
|
||||
"configuration source %s at composition.profiles: expected a mapping, got %s",
|
||||
formatCompositionSources(field.value.sources), yamlKindName(field.value.kind),
|
||||
)
|
||||
}
|
||||
if len(field.value.fields) == 0 {
|
||||
return pipelineCompositionEnvelope{}, -1, fmt.Errorf("composition.profiles must declare at least one named profile")
|
||||
}
|
||||
for _, profileField := range field.value.fields {
|
||||
name, err := normalizePipelineProfileName(profileField.key, "composition.profiles profile name")
|
||||
if err != nil {
|
||||
return pipelineCompositionEnvelope{}, -1, err
|
||||
}
|
||||
profile, err := parsePipelineProfileDeclaration(name, profileField.value)
|
||||
if err != nil {
|
||||
return pipelineCompositionEnvelope{}, -1, err
|
||||
}
|
||||
envelope.profiles = append(envelope.profiles, profile)
|
||||
}
|
||||
default:
|
||||
return pipelineCompositionEnvelope{}, -1, fmt.Errorf(
|
||||
"configuration source %s at composition.%s: unknown composition field %q",
|
||||
formatCompositionSources(field.value.sources), field.key, field.key,
|
||||
)
|
||||
}
|
||||
}
|
||||
return envelope, index, nil
|
||||
}
|
||||
|
||||
func validatePipelineCompositionEnvelope(envelope pipelineCompositionEnvelope) error {
|
||||
if envelope.defaultProfile != nil && !pipelineProfileDeclared(envelope.profiles, *envelope.defaultProfile) {
|
||||
return fmt.Errorf(
|
||||
"composition.default_profile %q does not name a declared profile", *envelope.defaultProfile,
|
||||
)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func pipelineBodyWithoutComposition(document *compositionDocument, compositionIndex int) *compositionDocument {
|
||||
root := cloneCompositionNode(document.root)
|
||||
if compositionIndex >= 0 {
|
||||
root.fields = append(root.fields[:compositionIndex], root.fields[compositionIndex+1:]...)
|
||||
}
|
||||
return &compositionDocument{root: root, sources: append([]string(nil), document.sources...)}
|
||||
}
|
||||
|
||||
func parsePipelineProfileDeclaration(name string, node *compositionNode) (pipelineProfileDeclaration, error) {
|
||||
path := "composition.profiles." + name
|
||||
if node.kind != yaml.MappingNode {
|
||||
return pipelineProfileDeclaration{}, fmt.Errorf(
|
||||
"configuration source %s at %s: expected a mapping, got %s",
|
||||
formatCompositionSources(node.sources), path, yamlKindName(node.kind),
|
||||
)
|
||||
}
|
||||
profile := pipelineProfileDeclaration{name: name}
|
||||
for _, field := range node.fields {
|
||||
if field.key != "overlay" {
|
||||
return pipelineProfileDeclaration{}, fmt.Errorf(
|
||||
"configuration source %s at %s.%s: unknown profile field %q; only overlay is supported",
|
||||
formatCompositionSources(field.value.sources), path, field.key, field.key,
|
||||
)
|
||||
}
|
||||
if field.value.kind != yaml.ScalarNode || field.value.tag != "!!str" {
|
||||
return pipelineProfileDeclaration{}, fmt.Errorf(
|
||||
"configuration source %s at %s.overlay: expected a string path",
|
||||
formatCompositionSources(field.value.sources), path,
|
||||
)
|
||||
}
|
||||
profile.overlay = field.value.value
|
||||
}
|
||||
if profile.overlay == "" {
|
||||
return pipelineProfileDeclaration{}, fmt.Errorf("%s.overlay is required", path)
|
||||
}
|
||||
return profile, nil
|
||||
}
|
||||
|
||||
func normalizePipelineProfileName(value, label string) (string, error) {
|
||||
if value == "" || strings.TrimSpace(value) != value {
|
||||
return "", fmt.Errorf("%s must be non-empty without surrounding whitespace", label)
|
||||
}
|
||||
for _, character := range value {
|
||||
if unicode.IsControl(character) {
|
||||
return "", fmt.Errorf("%s %q must not contain control characters", label, value)
|
||||
}
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func pipelineProfileDeclared(profiles []pipelineProfileDeclaration, name string) bool {
|
||||
for _, profile := range profiles {
|
||||
if profile.name == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func selectPipelineProfile(envelope pipelineCompositionEnvelope, opts PipelineLoadOptions) (*pipelineProfileSelection, error) {
|
||||
if opts.Profile != nil {
|
||||
name, err := normalizePipelineProfileName(*opts.Profile, "explicit profile selection")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(envelope.profiles) == 0 {
|
||||
return nil, fmt.Errorf("explicit profile %q was selected but the pipeline declares no profiles", name)
|
||||
}
|
||||
if !pipelineProfileDeclared(envelope.profiles, name) {
|
||||
return nil, fmt.Errorf("explicit profile %q is not declared by the pipeline", name)
|
||||
}
|
||||
return &pipelineProfileSelection{name: name, source: "cli"}, nil
|
||||
}
|
||||
if len(envelope.profiles) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
if envelope.defaultProfile == nil {
|
||||
return nil, fmt.Errorf("pipeline declares profiles but composition.default_profile is omitted and no profile was explicitly selected")
|
||||
}
|
||||
return &pipelineProfileSelection{name: *envelope.defaultProfile, source: "default"}, nil
|
||||
}
|
||||
|
||||
type loadedPipelineImport struct {
|
||||
path string
|
||||
document *compositionDocument
|
||||
info os.FileInfo
|
||||
}
|
||||
|
||||
type loadedPipelineProfileOverlay struct {
|
||||
profile string
|
||||
path string
|
||||
document *compositionDocument
|
||||
info os.FileInfo
|
||||
}
|
||||
|
||||
func loadPipelineProfileOverlays(
|
||||
rootPath string,
|
||||
declared []pipelineProfileDeclaration,
|
||||
imports []loadedPipelineImport,
|
||||
) ([]loadedPipelineProfileOverlay, error) {
|
||||
if len(declared) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
rootDir := filepath.Dir(rootPath)
|
||||
rootInfo, err := os.Stat(rootPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("inspect root pipeline file %q: %w", rootPath, err)
|
||||
}
|
||||
seenPaths := make(map[string]string, len(declared))
|
||||
loaded := make([]loadedPipelineProfileOverlay, 0, len(declared))
|
||||
for _, profile := range declared {
|
||||
label := "composition.profiles." + profile.name + ".overlay"
|
||||
raw := profile.overlay
|
||||
if strings.TrimSpace(raw) != raw || raw == "" {
|
||||
return nil, fmt.Errorf("%s must be a non-empty path without surrounding whitespace", label)
|
||||
}
|
||||
normalized, err := pathsafe.NormalizeRelativeDestination(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s path %q is invalid: %w", label, raw, err)
|
||||
}
|
||||
extension := filepath.Ext(filepath.FromSlash(normalized))
|
||||
if extension != ".yml" && extension != ".yaml" {
|
||||
return nil, fmt.Errorf("%s path %q must use .yml or .yaml", label, raw)
|
||||
}
|
||||
if prior, duplicate := seenPaths[normalized]; duplicate {
|
||||
return nil, fmt.Errorf("%s path %q duplicates profile %q overlay after normalization", label, raw, prior)
|
||||
}
|
||||
seenPaths[normalized] = profile.name
|
||||
resolved := filepath.Join(rootDir, filepath.FromSlash(normalized))
|
||||
file, err := os.Open(resolved)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open %s path %q beneath root pipeline directory: %w", label, raw, err)
|
||||
}
|
||||
info, statErr := file.Stat()
|
||||
if statErr != nil {
|
||||
_ = file.Close()
|
||||
return nil, fmt.Errorf("inspect %s path %q: %w", label, raw, statErr)
|
||||
}
|
||||
if os.SameFile(rootInfo, info) {
|
||||
_ = file.Close()
|
||||
return nil, fmt.Errorf("%s path %q references the root pipeline itself", label, raw)
|
||||
}
|
||||
for _, imported := range imports {
|
||||
if os.SameFile(imported.info, info) {
|
||||
_ = file.Close()
|
||||
return nil, fmt.Errorf("%s path %q references the same file as imported source %q", label, raw, imported.path)
|
||||
}
|
||||
}
|
||||
for _, prior := range loaded {
|
||||
if os.SameFile(prior.info, info) {
|
||||
_ = file.Close()
|
||||
return nil, fmt.Errorf("%s path %q references the same file as profile %q overlay", label, raw, prior.profile)
|
||||
}
|
||||
}
|
||||
document, parseErr := parseCompositionDocument(resolved, file)
|
||||
closeErr := file.Close()
|
||||
if parseErr != nil {
|
||||
return nil, parseErr
|
||||
}
|
||||
if closeErr != nil {
|
||||
return nil, fmt.Errorf("close %s path %q: %w", label, raw, closeErr)
|
||||
}
|
||||
if compositionFieldIndex(document.root.fields, "composition") >= 0 {
|
||||
return nil, fmt.Errorf("profile overlay source %q declares composition; only the root pipeline may declare composition", resolved)
|
||||
}
|
||||
loaded = append(loaded, loadedPipelineProfileOverlay{
|
||||
profile: profile.name, path: resolved, document: document, info: info,
|
||||
})
|
||||
}
|
||||
return loaded, nil
|
||||
}
|
||||
|
||||
func loadedProfileOverlay(overlays []loadedPipelineProfileOverlay, name string) (loadedPipelineProfileOverlay, bool) {
|
||||
for _, overlay := range overlays {
|
||||
if overlay.profile == name {
|
||||
return overlay, true
|
||||
}
|
||||
}
|
||||
return loadedPipelineProfileOverlay{}, false
|
||||
}
|
||||
|
||||
func loadPipelineImports(rootPath string, declared []string) ([]loadedPipelineImport, error) {
|
||||
if len(declared) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
rootDir := filepath.Dir(rootPath)
|
||||
rootInfo, err := os.Stat(rootPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("inspect root pipeline file %q: %w", rootPath, err)
|
||||
}
|
||||
seenPaths := make(map[string]int, len(declared))
|
||||
loaded := make([]loadedPipelineImport, 0, len(declared))
|
||||
for index, raw := range declared {
|
||||
if strings.TrimSpace(raw) != raw || raw == "" {
|
||||
return nil, fmt.Errorf("composition.imports[%d] must be a non-empty path without surrounding whitespace", index)
|
||||
}
|
||||
normalized, err := pathsafe.NormalizeRelativeDestination(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("composition.imports[%d] path %q is invalid: %w", index, raw, err)
|
||||
}
|
||||
extension := filepath.Ext(filepath.FromSlash(normalized))
|
||||
if extension != ".yml" && extension != ".yaml" {
|
||||
return nil, fmt.Errorf("composition.imports[%d] path %q must use .yml or .yaml", index, raw)
|
||||
}
|
||||
if prior, duplicate := seenPaths[normalized]; duplicate {
|
||||
return nil, fmt.Errorf(
|
||||
"composition.imports[%d] path %q duplicates composition.imports[%d] after normalization",
|
||||
index, raw, prior,
|
||||
)
|
||||
}
|
||||
seenPaths[normalized] = index
|
||||
resolved := filepath.Join(rootDir, filepath.FromSlash(normalized))
|
||||
if resolved == rootPath {
|
||||
return nil, fmt.Errorf("composition.imports[%d] path %q imports the root pipeline itself", index, raw)
|
||||
}
|
||||
|
||||
file, err := os.Open(resolved)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open composition.imports[%d] path %q beneath root pipeline directory: %w", index, raw, err)
|
||||
}
|
||||
info, statErr := file.Stat()
|
||||
if statErr != nil {
|
||||
_ = file.Close()
|
||||
return nil, fmt.Errorf("inspect composition.imports[%d] path %q: %w", index, raw, statErr)
|
||||
}
|
||||
if os.SameFile(rootInfo, info) {
|
||||
_ = file.Close()
|
||||
return nil, fmt.Errorf("composition.imports[%d] path %q imports the root pipeline itself", index, raw)
|
||||
}
|
||||
for priorIndex, prior := range loaded {
|
||||
if os.SameFile(prior.info, info) {
|
||||
_ = file.Close()
|
||||
return nil, fmt.Errorf(
|
||||
"composition.imports[%d] path %q references the same file as composition.imports[%d] %q",
|
||||
index, raw, priorIndex, declared[priorIndex],
|
||||
)
|
||||
}
|
||||
}
|
||||
document, parseErr := parseCompositionDocument(resolved, file)
|
||||
closeErr := file.Close()
|
||||
if parseErr != nil {
|
||||
return nil, parseErr
|
||||
}
|
||||
if closeErr != nil {
|
||||
return nil, fmt.Errorf("close composition.imports[%d] path %q: %w", index, raw, closeErr)
|
||||
}
|
||||
if compositionFieldIndex(document.root.fields, "composition") >= 0 {
|
||||
return nil, fmt.Errorf(
|
||||
"imported configuration source %q declares composition; only the root pipeline may declare composition",
|
||||
resolved,
|
||||
)
|
||||
}
|
||||
loaded = append(loaded, loadedPipelineImport{path: resolved, document: document, info: info})
|
||||
}
|
||||
return loaded, nil
|
||||
}
|
||||
@@ -1,431 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSplitPipelineCompositionEnvelopePreservesInputAndDeclarationOrder(t *testing.T) {
|
||||
document, err := parseCompositionBytes("pipeline.yml", []byte(`composition:
|
||||
imports: [first.yml, second.yml]
|
||||
default_profile: production
|
||||
profiles:
|
||||
production:
|
||||
overlay: production.yml
|
||||
testing:
|
||||
overlay: testing.yml
|
||||
workspace:
|
||||
root: /srv/narratio
|
||||
`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
originalRoot := cloneCompositionNode(document.root)
|
||||
originalSources := append([]string(nil), document.sources...)
|
||||
|
||||
body, envelope, err := splitPipelineCompositionEnvelope(document)
|
||||
if err != nil {
|
||||
t.Fatalf("splitPipelineCompositionEnvelope() error = %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(document.root, originalRoot) || !slices.Equal(document.sources, originalSources) {
|
||||
t.Fatalf("splitPipelineCompositionEnvelope() mutated its input")
|
||||
}
|
||||
if compositionFieldIndex(body.root.fields, "composition") >= 0 {
|
||||
t.Fatalf("pipeline body still contains composition envelope")
|
||||
}
|
||||
if compositionFieldIndex(body.root.fields, "workspace") < 0 {
|
||||
t.Fatalf("pipeline body lost ordinary root fields")
|
||||
}
|
||||
if want := []string{"first.yml", "second.yml"}; !slices.Equal(envelope.imports, want) {
|
||||
t.Fatalf("imports = %#v, want %#v", envelope.imports, want)
|
||||
}
|
||||
if len(envelope.profiles) != 2 || envelope.profiles[0].name != "production" || envelope.profiles[1].name != "testing" {
|
||||
t.Fatalf("profiles = %#v, want declaration order", envelope.profiles)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitPipelineCompositionEnvelopeRejectsMalformedShapesWithoutMutation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
yaml string
|
||||
want string
|
||||
}{
|
||||
{name: "envelope", yaml: "composition: []\n", want: "expected a mapping"},
|
||||
{name: "imports mapping", yaml: "composition:\n imports: {}\n", want: "expected a list"},
|
||||
{name: "import value", yaml: "composition:\n imports: [1]\n", want: "expected a string path"},
|
||||
{name: "default profile", yaml: "composition:\n default_profile: {}\n", want: "expected a string profile name"},
|
||||
{name: "profiles sequence", yaml: "composition:\n profiles: []\n", want: "expected a mapping"},
|
||||
{name: "empty profiles", yaml: "composition:\n profiles: {}\n", want: "at least one named profile"},
|
||||
{name: "profile declaration", yaml: "composition:\n profiles:\n production: production.yml\n", want: "expected a mapping"},
|
||||
{name: "invalid profile name", yaml: "composition:\n profiles:\n ' production ':\n overlay: production.yml\n", want: "surrounding whitespace"},
|
||||
{name: "unknown field", yaml: "composition:\n unexpected: true\n", want: "unknown composition field"},
|
||||
{name: "unknown default", yaml: "composition:\n default_profile: testing\n profiles:\n production:\n overlay: production.yml\n", want: "does not name a declared profile"},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
document, err := parseCompositionBytes("pipeline.yml", []byte(test.yaml))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
originalRoot := cloneCompositionNode(document.root)
|
||||
|
||||
_, _, err = splitPipelineCompositionEnvelope(document)
|
||||
if err == nil || !strings.Contains(err.Error(), test.want) {
|
||||
t.Fatalf("splitPipelineCompositionEnvelope() error = %v, want containing %q", err, test.want)
|
||||
}
|
||||
if !reflect.DeepEqual(document.root, originalRoot) {
|
||||
t.Fatalf("splitPipelineCompositionEnvelope() mutated invalid input")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadPipelineCompositionImportsDisjointFieldsAndTracksOwnership(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
rootPath := writePipelineSource(t, dir, "pipeline.yml", `composition:
|
||||
imports:
|
||||
- conf.d/platform.yml
|
||||
- conf.d/artifacts.yml
|
||||
whisperx:
|
||||
transcribe_url: https://transcription.example.com/transcribe
|
||||
`)
|
||||
platformPath := writePipelineSource(t, dir, "conf.d/platform.yml", `workspace:
|
||||
root: /srv/narratio/work
|
||||
storage:
|
||||
backend: local
|
||||
`)
|
||||
artifactsPath := writePipelineSource(t, dir, "conf.d/artifacts.yml", `scriptorium:
|
||||
artifacts:
|
||||
player_handout:
|
||||
enabled: false
|
||||
session_recap:
|
||||
enabled: true
|
||||
prompt_id: dnd.session_recap
|
||||
output_path: artifacts/session_recap.md
|
||||
`)
|
||||
|
||||
cfg, err := LoadPipeline(rootPath)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadPipeline() error = %v", err)
|
||||
}
|
||||
if cfg.Workspace.Root != "/srv/narratio/work" || cfg.Storage.Backend != StorageBackendLocal {
|
||||
t.Fatalf("imported platform config = workspace=%q storage=%#v", cfg.Workspace.Root, cfg.Storage)
|
||||
}
|
||||
if cfg.Scriptorium == nil || len(cfg.Scriptorium.Artifacts) != 2 || cfg.Scriptorium.Artifacts["player_handout"].Enabled {
|
||||
t.Fatalf("imported artifacts = %#v", cfg.Scriptorium)
|
||||
}
|
||||
if cfg.WhisperX.TranscribeURL != "https://transcription.example.com/transcribe" {
|
||||
t.Fatalf("root field = %q", cfg.WhisperX.TranscribeURL)
|
||||
}
|
||||
if cfg.resolution == nil {
|
||||
t.Fatal("pipeline resolution metadata = nil")
|
||||
}
|
||||
wantSources := []string{absolutePath(t, rootPath), absolutePath(t, platformPath), absolutePath(t, artifactsPath)}
|
||||
if !slices.Equal(cfg.resolution.sources, wantSources) || !slices.Equal(cfg.resolution.imports, wantSources[1:]) {
|
||||
t.Fatalf("resolution sources=%#v imports=%#v, want %#v / %#v", cfg.resolution.sources, cfg.resolution.imports, wantSources, wantSources[1:])
|
||||
}
|
||||
assertPipelineFieldOwner(t, cfg, "whisperx.transcribe_url", absolutePath(t, rootPath))
|
||||
assertPipelineFieldOwner(t, cfg, "workspace.root", absolutePath(t, platformPath))
|
||||
assertPipelineFieldOwner(t, cfg, "scriptorium.artifacts.session_recap.prompt_id", absolutePath(t, artifactsPath))
|
||||
}
|
||||
|
||||
func TestLoadPipelineCompositionMergesDisjointKeyedEntries(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
rootPath := writePipelineSource(t, dir, "pipeline.yml", `composition:
|
||||
imports: [first.yml, second.yml]
|
||||
whisperx:
|
||||
transcribe_url: https://transcription.example.com/transcribe
|
||||
scriptorium:
|
||||
artifacts:
|
||||
root_artifact:
|
||||
enabled: false
|
||||
`)
|
||||
writePipelineSource(t, dir, "first.yml", `scriptorium:
|
||||
artifacts:
|
||||
first_artifact:
|
||||
enabled: false
|
||||
`)
|
||||
writePipelineSource(t, dir, "second.yml", `scriptorium:
|
||||
artifacts:
|
||||
second_artifact:
|
||||
enabled: false
|
||||
`)
|
||||
|
||||
cfg, err := LoadPipeline(rootPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := len(cfg.Scriptorium.Artifacts); got != 3 {
|
||||
t.Fatalf("artifact count = %d, want 3: %#v", got, cfg.Scriptorium.Artifacts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadPipelineCompositionRejectsBaseConflictsWithAllSources(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
root string
|
||||
imports map[string]string
|
||||
path string
|
||||
sources []string
|
||||
}{
|
||||
{
|
||||
name: "root and import identical scalar",
|
||||
root: "whisperx:\n language: en\n",
|
||||
imports: map[string]string{"one.yml": "whisperx:\n language: en\n"},
|
||||
path: "whisperx.language", sources: []string{"pipeline.yml", "one.yml"},
|
||||
},
|
||||
{
|
||||
name: "all import claimants",
|
||||
imports: map[string]string{
|
||||
"one.yml": "workspace:\n root: /one\n",
|
||||
"two.yml": "workspace:\n root: /two\n",
|
||||
"three.yml": "workspace:\n root: /three\n",
|
||||
},
|
||||
path: "workspace.root", sources: []string{"one.yml", "two.yml", "three.yml"},
|
||||
},
|
||||
{
|
||||
name: "atomic list",
|
||||
root: "audita:\n modules: [one]\n",
|
||||
imports: map[string]string{"one.yml": "audita:\n modules: [two]\n"},
|
||||
path: "audita.modules", sources: []string{"pipeline.yml", "one.yml"},
|
||||
},
|
||||
{
|
||||
name: "kind conflict",
|
||||
root: "workspace:\n root: /work\n",
|
||||
imports: map[string]string{"one.yml": "workspace: invalid\n"},
|
||||
path: "workspace", sources: []string{"pipeline.yml", "one.yml"},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
order := make([]string, 0, len(tt.imports))
|
||||
for _, name := range []string{"one.yml", "two.yml", "three.yml"} {
|
||||
if _, ok := tt.imports[name]; ok {
|
||||
order = append(order, name)
|
||||
}
|
||||
}
|
||||
root := "composition:\n imports:\n"
|
||||
for _, name := range order {
|
||||
root += " - " + name + "\n"
|
||||
}
|
||||
root += tt.root
|
||||
rootPath := writePipelineSource(t, dir, "pipeline.yml", root)
|
||||
for name, content := range tt.imports {
|
||||
writePipelineSource(t, dir, name, content)
|
||||
}
|
||||
|
||||
_, err := LoadPipeline(rootPath)
|
||||
if err == nil {
|
||||
t.Fatal("LoadPipeline() error = nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), tt.path) {
|
||||
t.Fatalf("error = %q, want path %q", err, tt.path)
|
||||
}
|
||||
for _, source := range tt.sources {
|
||||
if !strings.Contains(err.Error(), source) {
|
||||
t.Fatalf("error = %q, want source %q", err, source)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadPipelineCompositionRejectsUnsafeOrInvalidImports(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
imports []string
|
||||
setup func(*testing.T, string)
|
||||
want string
|
||||
}{
|
||||
{name: "empty", imports: []string{""}, want: "non-empty path"},
|
||||
{name: "surrounding whitespace", imports: []string{" one.yml "}, want: "surrounding whitespace"},
|
||||
{name: "absolute", imports: []string{"/tmp/one.yml"}, want: "invalid"},
|
||||
{name: "traversal", imports: []string{"../one.yml"}, want: "invalid"},
|
||||
{name: "unsupported extension", imports: []string{"one.json"}, want: ".yml or .yaml"},
|
||||
{name: "missing", imports: []string{"missing.yml"}, want: "open composition.imports"},
|
||||
{name: "duplicate normalized", imports: []string{"one.yml", "./one.yml"}, setup: func(t *testing.T, dir string) {
|
||||
writePipelineSource(t, dir, "one.yml", "workspace:\n root: /work\n")
|
||||
}, want: "duplicates composition.imports"},
|
||||
{name: "root self import", imports: []string{"pipeline.yml"}, want: "root pipeline itself"},
|
||||
{name: "directory", imports: []string{"directory.yml"}, setup: func(t *testing.T, dir string) {
|
||||
if err := os.Mkdir(filepath.Join(dir, "directory.yml"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}, want: "is a directory"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if tt.setup != nil {
|
||||
tt.setup(t, dir)
|
||||
}
|
||||
rootPath := writeImportRoot(t, dir, tt.imports)
|
||||
_, err := LoadPipeline(rootPath)
|
||||
if err == nil || !strings.Contains(strings.ToLower(err.Error()), strings.ToLower(tt.want)) {
|
||||
t.Fatalf("LoadPipeline() error = %v, want containing %q", err, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadPipelineCompositionRejectsSameFileAliasesAndImportedComposition(t *testing.T) {
|
||||
t.Run("same file through hard link", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writePipelineSource(t, dir, "one.yml", "workspace:\n root: /work\n")
|
||||
if err := os.Link(filepath.Join(dir, "one.yml"), filepath.Join(dir, "two.yml")); err != nil {
|
||||
t.Skipf("hard links unavailable: %v", err)
|
||||
}
|
||||
rootPath := writeImportRoot(t, dir, []string{"one.yml", "two.yml"})
|
||||
_, err := LoadPipeline(rootPath)
|
||||
if err == nil || !strings.Contains(err.Error(), "same file") {
|
||||
t.Fatalf("LoadPipeline() error = %v, want same-file rejection", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("imported composition", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
rootPath := writeImportRoot(t, dir, []string{"nested.yml"})
|
||||
writePipelineSource(t, dir, "nested.yml", "composition:\n imports: []\n")
|
||||
_, err := LoadPipeline(rootPath)
|
||||
if err == nil || !strings.Contains(err.Error(), "only the root pipeline") || !strings.Contains(err.Error(), "nested.yml") {
|
||||
t.Fatalf("LoadPipeline() error = %v, want imported composition rejection", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("empty profiles", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
rootPath := writePipelineSource(t, dir, "pipeline.yml", "composition:\n profiles: {}\n")
|
||||
_, err := LoadPipeline(rootPath)
|
||||
if err == nil || !strings.Contains(err.Error(), "must declare at least one named profile") {
|
||||
t.Fatalf("LoadPipeline() error = %v, want empty profiles rejection", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestLoadPipelineCompositionReportsImportedParseAndSchemaSources(t *testing.T) {
|
||||
t.Run("malformed imported YAML", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
rootPath := writeImportRoot(t, dir, []string{"broken.yml"})
|
||||
brokenPath := writePipelineSource(t, dir, "broken.yml", "workspace: [\n")
|
||||
_, err := LoadPipeline(rootPath)
|
||||
if err == nil || !strings.Contains(err.Error(), absolutePath(t, brokenPath)) || !strings.Contains(err.Error(), "decode YAML") {
|
||||
t.Fatalf("LoadPipeline() error = %v, want imported parse source", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("unknown imported field", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
rootPath := writeImportRoot(t, dir, []string{"unknown.yml"})
|
||||
unknownPath := writePipelineSource(t, dir, "unknown.yml", "unknown_field: true\n")
|
||||
_, err := LoadPipeline(rootPath)
|
||||
if err == nil || !strings.Contains(err.Error(), absolutePath(t, unknownPath)) || !strings.Contains(err.Error(), "strict decode failed") {
|
||||
t.Fatalf("LoadPipeline() error = %v, want assembled source-aware strict error", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestLoadPipelineCompositionKeepsRelativePathsRootBased(t *testing.T) {
|
||||
rootDir := t.TempDir()
|
||||
monolithicPath := writePipelineSource(t, rootDir, "monolithic.yml", testPipelineBaseYAML+`
|
||||
notarius:
|
||||
enabled: true
|
||||
config_path: tool/notarius.yml
|
||||
pipeline_id: dnd-session
|
||||
outputs:
|
||||
npc_registry:
|
||||
lane_id: npc-registry
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.npc_registry
|
||||
schema_version: v1
|
||||
`)
|
||||
composedPath := writePipelineSource(t, rootDir, "pipeline.yml", `composition:
|
||||
imports: [conf.d/extraction.yml]
|
||||
`+testPipelineBaseYAML)
|
||||
writePipelineSource(t, rootDir, "conf.d/extraction.yml", `notarius:
|
||||
enabled: true
|
||||
config_path: tool/notarius.yml
|
||||
pipeline_id: dnd-session
|
||||
outputs:
|
||||
npc_registry:
|
||||
lane_id: npc-registry
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.npc_registry
|
||||
schema_version: v1
|
||||
`)
|
||||
|
||||
monolithic, err := LoadPipeline(monolithicPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
composed, err := LoadPipeline(composedPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := filepath.Join(rootDir, "tool", "notarius.yml")
|
||||
if monolithic.Notarius.ConfigPath != want || composed.Notarius.ConfigPath != want {
|
||||
t.Fatalf("config paths = monolithic %q composed %q, want %q", monolithic.Notarius.ConfigPath, composed.Notarius.ConfigPath, want)
|
||||
}
|
||||
if monolithic.Notarius.WorkingDirectory != filepath.Dir(want) || composed.Notarius.WorkingDirectory != filepath.Dir(want) {
|
||||
t.Fatalf("working directories = %q / %q", monolithic.Notarius.WorkingDirectory, composed.Notarius.WorkingDirectory)
|
||||
}
|
||||
}
|
||||
|
||||
func writeImportRoot(t *testing.T, dir string, imports []string) string {
|
||||
t.Helper()
|
||||
var builder strings.Builder
|
||||
builder.WriteString("composition:\n imports:\n")
|
||||
for _, imported := range imports {
|
||||
builder.WriteString(" - ")
|
||||
if imported == "" {
|
||||
builder.WriteString(`""`)
|
||||
} else {
|
||||
builder.WriteString(`"` + imported + `"`)
|
||||
}
|
||||
builder.WriteByte('\n')
|
||||
}
|
||||
builder.WriteString("whisperx:\n transcribe_url: https://transcription.example.com/transcribe\n")
|
||||
return writePipelineSource(t, dir, "pipeline.yml", builder.String())
|
||||
}
|
||||
|
||||
func writePipelineSource(t *testing.T, root, relative, content string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(root, filepath.FromSlash(relative))
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func absolutePath(t *testing.T, path string) string {
|
||||
t.Helper()
|
||||
absolute, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return absolute
|
||||
}
|
||||
|
||||
func assertPipelineFieldOwner(t *testing.T, cfg *PipelineConfig, path, source string) {
|
||||
t.Helper()
|
||||
if cfg == nil || cfg.resolution == nil {
|
||||
t.Fatal("pipeline resolution metadata is absent")
|
||||
}
|
||||
for _, ownership := range cfg.resolution.ownership {
|
||||
if ownership.path == path {
|
||||
if !slices.Equal(ownership.sources, []string{source}) {
|
||||
t.Fatalf("owner of %s = %#v, want %q", path, ownership.sources, source)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatalf("ownership path %q not found: %#v", path, cfg.resolution.ownership)
|
||||
}
|
||||
391
internal/config/pipeline_configkit.go
Normal file
391
internal/config/pipeline_configkit.go
Normal file
@@ -0,0 +1,391 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
|
||||
)
|
||||
|
||||
type pipelineResolutionMetadata struct {
|
||||
rootPath string
|
||||
imports []string
|
||||
sources []string
|
||||
selectedProfile *pipelineProfileSelection
|
||||
effectiveDigest string
|
||||
logicalNotariusConfig string
|
||||
logicalNotariusWorking string
|
||||
logicalNotariusCaptured bool
|
||||
ownership []pipelineFieldOwnership
|
||||
artifactFamilies map[string]ScriptoriumArtifactFamilyConfig
|
||||
artifactFamiliesExpanded bool
|
||||
publishDeclared bool
|
||||
configkitMetadata configkit.Metadata
|
||||
configkitSources []configkit.SourceInfo
|
||||
}
|
||||
|
||||
type pipelineProfileSelection struct {
|
||||
name string
|
||||
source string
|
||||
overlayPath string
|
||||
}
|
||||
|
||||
type pipelineFieldOwnership struct {
|
||||
path string
|
||||
sources []string
|
||||
}
|
||||
|
||||
func loadPipelineWithConfigKit(path string, opts PipelineLoadOptions) (*PipelineConfig, error) {
|
||||
profile := selectedPipelineProfileName(opts)
|
||||
result, err := configkit.Load(configkit.File(path), configkit.Options[PipelineConfig]{
|
||||
Imports: true,
|
||||
Profile: profile,
|
||||
Definition: configkit.Definition[PipelineConfig]{
|
||||
Defaults: pipelineConfigDefaults,
|
||||
Resolve: resolvePipelineConfig,
|
||||
Validate: validatePipelineConfigKit,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, describePipelineLoadError(err)
|
||||
}
|
||||
cfg := result.Config
|
||||
metadata, err := pipelineMetadataFromConfigKit(&cfg, result, opts)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load pipeline config: %w", err)
|
||||
}
|
||||
cfg.resolution = metadata
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
func describePipelineLoadError(err error) error {
|
||||
var loadError *configkit.LoadError
|
||||
if !errors.As(err, &loadError) {
|
||||
return fmt.Errorf("load pipeline config: %w", err)
|
||||
}
|
||||
var diagnostics bytes.Buffer
|
||||
resolved := loadError.Diagnostics()
|
||||
sourcePaths := make(map[string]string, len(loadError.Sources()))
|
||||
for _, source := range loadError.Sources() {
|
||||
if source.Path != "" {
|
||||
sourcePaths[source.ID] = source.Path
|
||||
} else if source.Label != "" {
|
||||
sourcePaths[source.ID] = source.Label
|
||||
}
|
||||
}
|
||||
for index := range resolved {
|
||||
if resolved[index].Primary != nil {
|
||||
if name := sourcePaths[resolved[index].Primary.SourceID]; name != "" {
|
||||
resolved[index].Primary.SourceID = name
|
||||
}
|
||||
}
|
||||
for relatedIndex := range resolved[index].Related {
|
||||
if name := sourcePaths[resolved[index].Related[relatedIndex].SourceID]; name != "" {
|
||||
resolved[index].Related[relatedIndex].SourceID = name
|
||||
}
|
||||
}
|
||||
}
|
||||
if writeErr := configkit.WriteDiagnostics(&diagnostics, resolved); writeErr != nil {
|
||||
return fmt.Errorf("load pipeline config: %w", err)
|
||||
}
|
||||
return fmt.Errorf("load pipeline config: strict decode failed or validation failed: %w: %s", err, strings.TrimSpace(diagnostics.String()))
|
||||
}
|
||||
|
||||
func selectedPipelineProfileName(opts PipelineLoadOptions) string {
|
||||
if opts.Profile == nil || *opts.Profile == "" || *opts.Profile == "default" {
|
||||
return ""
|
||||
}
|
||||
return *opts.Profile
|
||||
}
|
||||
|
||||
func pipelineConfigDefaults() PipelineConfig {
|
||||
return PipelineConfig{
|
||||
Workspace: WorkspaceConfig{Root: DefaultWorkspaceRoot},
|
||||
Campaigns: CampaignsConfig{Root: DefaultCampaignsRoot},
|
||||
Storage: StorageConfig{Backend: StorageBackendLocal},
|
||||
Spool: SpoolConfig{Root: DefaultSpoolRoot},
|
||||
Cache: CacheConfig{Root: DefaultCacheRoot, S3Audio: boolPtr(DefaultCacheS3Audio)},
|
||||
Publish: &PublishConfig{
|
||||
Enabled: boolPtr(DefaultArchiveEnabled), UploadRun: boolPtr(DefaultArchiveUploadRun),
|
||||
Outputs: defaultPublishOutputs(),
|
||||
},
|
||||
WhisperX: WhisperXConfig{
|
||||
Language: DefaultWhisperXLanguage, Timeout: DefaultWhisperXTimeout,
|
||||
RetryDelay: DefaultWhisperXRetryDelay, Concurrency: intPtr(DefaultWhisperXConcurrency),
|
||||
Retries: intPtr(DefaultWhisperXRetries),
|
||||
},
|
||||
Seriatim: SeriatimConfig{
|
||||
Binary: DefaultSeriatimBinary, Timeout: DefaultSeriatimTimeout,
|
||||
OutputSchema: DefaultSeriatimOutputSchema, CoalesceGap: float64Ptr(DefaultSeriatimCoalesceGap),
|
||||
Report: boolPtr(DefaultSeriatimReport),
|
||||
},
|
||||
Audita: AuditaConfig{Binary: DefaultAuditaBinary, Timeout: DefaultAuditaTimeout, Report: boolPtr(DefaultAuditaReport)},
|
||||
Normalize: &NormalizeConfig{OutputPath: DefaultNormalizeOutputPath, OutputSchema: DefaultNormalizeOutputSchema, Report: boolPtr(DefaultNormalizeReport)},
|
||||
Trim: &TrimConfig{
|
||||
Enabled: boolPtr(DefaultTrimEnabled), OutputPath: DefaultTrimOutputPath,
|
||||
Bounds: TrimBoundsConfig{
|
||||
PromptID: DefaultTrimBoundsPromptID, TranscriptInputName: DefaultTrimBoundsTranscriptInputName,
|
||||
OutputPath: DefaultTrimBoundsOutputPath, Timeout: DefaultTrimBoundsTimeout,
|
||||
},
|
||||
Seriatim: TrimSeriatimConfig{Report: boolPtr(DefaultTrimSeriatimReport)},
|
||||
},
|
||||
Render: &RenderConfig{
|
||||
Enabled: boolPtr(DefaultRenderEnabled), Format: DefaultRenderFormat, Title: DefaultRenderTitle,
|
||||
IncludeTimestamps: boolPtr(DefaultRenderTimestamps), IncludeSegmentIDs: boolPtr(DefaultRenderSegmentIDs),
|
||||
},
|
||||
Notification: NotificationConfig{Mode: DefaultNotificationMode},
|
||||
}
|
||||
}
|
||||
|
||||
func defaultPublishOutputs() []PublishOutputRule {
|
||||
outputs := append([]PublishOutputRule(nil), DefaultPublishOutputs...)
|
||||
for index := range outputs {
|
||||
outputs[index].Required = boolPtr(true)
|
||||
}
|
||||
return outputs
|
||||
}
|
||||
|
||||
func resolvePipelineConfig(cfg *PipelineConfig, resolution *configkit.Resolution) error {
|
||||
backend := strings.ToLower(strings.TrimSpace(cfg.Storage.Backend))
|
||||
if backend != cfg.Storage.Backend {
|
||||
configkit.Normalize(resolution, configkit.Keys("storage", "backend"), &cfg.Storage.Backend, backend, "canonical storage backend")
|
||||
}
|
||||
if cfg.Storage.S3 != nil {
|
||||
path := configkit.Keys("storage", "s3")
|
||||
configkit.Default(resolution, path.Key("root_prefix"), &cfg.Storage.S3.RootPrefix, DefaultStorageS3RootPrefix, "default S3 root prefix")
|
||||
configkit.Default(resolution, path.Key("access_key_id_env"), &cfg.Storage.S3.AccessKeyIDEnv, DefaultS3AccessKeyIDEnv, "default S3 access-key reference")
|
||||
configkit.Default(resolution, path.Key("secret_access_key_env"), &cfg.Storage.S3.SecretKeyEnv, DefaultS3SecretAccessKeyEnv, "default S3 secret-key reference")
|
||||
}
|
||||
if cfg.Trim != nil && cfg.Trim.Enabled != nil && *cfg.Trim.Enabled && cfg.Scriptorium == nil {
|
||||
value := &ScriptoriumConfig{Binary: DefaultScriptoriumBinary, Timeout: DefaultScriptoriumTimeout}
|
||||
configkit.Default(resolution, configkit.Keys("scriptorium"), &cfg.Scriptorium, value, "trim requires Scriptorium")
|
||||
}
|
||||
if cfg.Scriptorium != nil {
|
||||
path := configkit.Keys("scriptorium")
|
||||
configkit.Default(resolution, path.Key("binary"), &cfg.Scriptorium.Binary, DefaultScriptoriumBinary, "default Scriptorium binary")
|
||||
configkit.Default(resolution, path.Key("timeout"), &cfg.Scriptorium.Timeout, DefaultScriptoriumTimeout, "default Scriptorium timeout")
|
||||
}
|
||||
if cfg.Notarius != nil {
|
||||
path := configkit.Keys("notarius")
|
||||
configkit.Default(resolution, path.Key("binary"), &cfg.Notarius.Binary, DefaultNotariusBinary, "default Notarius binary")
|
||||
configkit.Default(resolution, path.Key("timeout"), &cfg.Notarius.Timeout, DefaultNotariusTimeout, "default Notarius timeout")
|
||||
if cfg.Notarius.Enabled && cfg.Notarius.ConfigPath != "" {
|
||||
configkit.Default(resolution, path.Key("working_directory"), &cfg.Notarius.WorkingDirectory,
|
||||
filepath.Dir(cfg.Notarius.ConfigPath), "derived from Notarius config path", path.Key("config_path"))
|
||||
}
|
||||
}
|
||||
if cfg.Publish != nil {
|
||||
defaultedOutput := false
|
||||
for index := range cfg.Publish.Outputs {
|
||||
output := cfg.Publish.Outputs[index]
|
||||
path := configkit.Keys("publish", "outputs").Index(index).Key("required")
|
||||
if !resolution.IsExplicit(path) {
|
||||
defaultedOutput = true
|
||||
}
|
||||
configkit.Default(resolution, path, &output.Required, boolPtr(true), "publish outputs are required by default")
|
||||
cfg.Publish.Outputs[index] = output
|
||||
}
|
||||
if defaultedOutput {
|
||||
resolution.Record(configkit.Keys("publish", "outputs"), configkit.ChangeDerivedDefault, "publish output entry defaults")
|
||||
}
|
||||
}
|
||||
before := cloneNotariusForResolution(cfg.Notarius)
|
||||
if err := NormalizePipelineConfig(cfg); err != nil {
|
||||
resolution.Error(configkit.Keys("notarius"), err.Error())
|
||||
return nil
|
||||
}
|
||||
if !reflect.DeepEqual(before, cfg.Notarius) {
|
||||
resolution.Record(configkit.Keys("notarius"), configkit.ChangeNormalization, "canonical Notarius configuration")
|
||||
}
|
||||
if cfg.Publish != nil {
|
||||
configuredOutputs := scriptoriumOutputPathMap(cfg.Scriptorium)
|
||||
extractionOutputs := notariusOutputKeySet(cfg.Notarius)
|
||||
for index := range cfg.Publish.Outputs {
|
||||
output := cfg.Publish.Outputs[index]
|
||||
if strings.TrimSpace(output.Dest) != "" {
|
||||
continue
|
||||
}
|
||||
destination, err := artifactpolicy.ResolvePublishedDestinationWithExtractions(strings.TrimSpace(output.Source), "", configuredOutputs, extractionOutputs)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
configkit.Normalize(resolution, configkit.Keys("publish", "outputs").Index(index).Key("dest"), &output.Dest, destination, "derived from publish source")
|
||||
cfg.Publish.Outputs[index] = output
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func cloneNotariusForResolution(value *NotariusConfig) *NotariusConfig {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
clone := *value
|
||||
clone.References = cloneStringMap(value.References)
|
||||
if value.Outputs != nil {
|
||||
clone.Outputs = make(map[string]NotariusOutputConfig, len(value.Outputs))
|
||||
for key, output := range value.Outputs {
|
||||
clone.Outputs[key] = output
|
||||
}
|
||||
}
|
||||
return &clone
|
||||
}
|
||||
|
||||
func cloneStringMap(value map[string]string) map[string]string {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
clone := make(map[string]string, len(value))
|
||||
for key, item := range value {
|
||||
clone[key] = item
|
||||
}
|
||||
return clone
|
||||
}
|
||||
|
||||
func validatePipelineConfigKit(cfg *PipelineConfig, validation *configkit.Validation) {
|
||||
// Checks that depend on family expansion or campaign/session context remain
|
||||
// at Narratio's explicit resolution boundary. This early definition check is
|
||||
// deliberately limited to an invariant that is wholly file-local.
|
||||
if strings.TrimSpace(cfg.Workspace.Root) == "" {
|
||||
validation.Error(configkit.Keys("workspace", "root"), "workspace root is required")
|
||||
}
|
||||
}
|
||||
|
||||
func pipelineMetadataFromConfigKit(cfg *PipelineConfig, result *configkit.Result[PipelineConfig], opts PipelineLoadOptions) (*pipelineResolutionMetadata, error) {
|
||||
sources := result.Sources()
|
||||
metadata := result.Metadata()
|
||||
if err := rejectDuplicateBaseDefinitions(cfg, metadata, sources); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resolved := &pipelineResolutionMetadata{
|
||||
configkitMetadata: metadata, configkitSources: sources,
|
||||
publishDeclared: metadata.IsExplicit(configkit.Keys("publish")),
|
||||
}
|
||||
pathsByID := make(map[string]string, len(sources))
|
||||
for index, source := range sources {
|
||||
path := source.Path
|
||||
if path == "" {
|
||||
path = source.Label
|
||||
}
|
||||
pathsByID[source.ID] = path
|
||||
resolved.sources = append(resolved.sources, path)
|
||||
if index == 0 {
|
||||
resolved.rootPath = path
|
||||
} else if source.Layer == configkit.LayerBase {
|
||||
resolved.imports = append(resolved.imports, path)
|
||||
}
|
||||
if source.Layer == configkit.LayerProfile {
|
||||
resolved.selectedProfile = &pipelineProfileSelection{name: selectedPipelineProfileName(opts), source: "explicit", overlayPath: path}
|
||||
}
|
||||
}
|
||||
detailed, err := pipelineMetadataPaths(cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, value := range detailed {
|
||||
info, ok := metadata.Lookup(configkit.Keys(value.segments...))
|
||||
sourceNames := []string{pipelineDefaultOwnershipSource}
|
||||
if ok {
|
||||
sourceNames = ownershipSources(info.Origin, pathsByID)
|
||||
}
|
||||
resolved.ownership = append(resolved.ownership, pipelineFieldOwnership{path: value.record.Path, sources: sourceNames})
|
||||
}
|
||||
return resolved, nil
|
||||
}
|
||||
|
||||
func pipelineMetadataPaths(cfg *PipelineConfig) ([]projectedPipelineValue, error) {
|
||||
effective, err := projectionDetailedRecords(projectEffectivePipeline(cfg, false))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if cfg.Scriptorium == nil || len(cfg.Scriptorium.ArtifactFamilies) == 0 {
|
||||
return effective, nil
|
||||
}
|
||||
families, err := projectionDetailedRecords(map[string]any{"scriptorium": map[string]any{"artifact_families": cfg.Scriptorium.ArtifactFamilies}})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append(effective, families...), nil
|
||||
}
|
||||
|
||||
func ownershipSources(origin configkit.Origin, pathsByID map[string]string) []string {
|
||||
if origin.Layer == configkit.LayerDefault || (origin.Layer == configkit.LayerResolution && origin.Operation == configkit.ChangeDerivedDefault) {
|
||||
return []string{pipelineDefaultOwnershipSource}
|
||||
}
|
||||
if origin.Layer == configkit.LayerResolution {
|
||||
return []string{"resolution"}
|
||||
}
|
||||
seen := map[string]struct{}{}
|
||||
var sources []string
|
||||
for _, location := range origin.Locations {
|
||||
value := pathsByID[location.SourceID]
|
||||
if value == "" {
|
||||
value = location.SourceID
|
||||
}
|
||||
if _, ok := seen[value]; !ok {
|
||||
seen[value] = struct{}{}
|
||||
sources = append(sources, value)
|
||||
}
|
||||
}
|
||||
if len(sources) == 0 {
|
||||
return []string{pipelineDefaultOwnershipSource}
|
||||
}
|
||||
return sources
|
||||
}
|
||||
|
||||
func rejectDuplicateBaseDefinitions(cfg *PipelineConfig, metadata configkit.Metadata, sources []configkit.SourceInfo) error {
|
||||
pathsByID := make(map[string]string, len(sources))
|
||||
for _, source := range sources {
|
||||
pathsByID[source.ID] = source.Path
|
||||
}
|
||||
var declarations []struct {
|
||||
kind string
|
||||
key string
|
||||
path configkit.Path
|
||||
}
|
||||
if cfg.Scriptorium != nil {
|
||||
for key := range cfg.Scriptorium.Artifacts {
|
||||
declarations = append(declarations, struct {
|
||||
kind, key string
|
||||
path configkit.Path
|
||||
}{"artifact", key, configkit.Keys("scriptorium", "artifacts", key)})
|
||||
}
|
||||
for key := range cfg.Scriptorium.ArtifactFamilies {
|
||||
declarations = append(declarations, struct {
|
||||
kind, key string
|
||||
path configkit.Path
|
||||
}{"artifact family", key, configkit.Keys("scriptorium", "artifact_families", key)})
|
||||
}
|
||||
}
|
||||
sort.Slice(declarations, func(i, j int) bool { return declarations[i].path.String() < declarations[j].path.String() })
|
||||
for _, declaration := range declarations {
|
||||
seen := map[string]struct{}{}
|
||||
for _, change := range metadata.History(declaration.path) {
|
||||
if change.Origin.Layer != configkit.LayerBase {
|
||||
continue
|
||||
}
|
||||
for _, location := range change.Origin.Locations {
|
||||
seen[location.SourceID] = struct{}{}
|
||||
}
|
||||
}
|
||||
if len(seen) > 1 {
|
||||
claims := make([]string, 0, len(seen))
|
||||
for sourceID := range seen {
|
||||
claims = append(claims, pathsByID[sourceID])
|
||||
}
|
||||
sort.Strings(claims)
|
||||
return fmt.Errorf("pipeline Scriptorium %s %q is declared by multiple base files: %s", declaration.kind, declaration.key, strings.Join(claims, ", "))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func intPtr(value int) *int { return &value }
|
||||
func float64Ptr(value float64) *float64 { return &value }
|
||||
func boolPtr(value bool) *bool { return &value }
|
||||
102
internal/config/pipeline_configkit_test.go
Normal file
102
internal/config/pipeline_configkit_test.go
Normal file
@@ -0,0 +1,102 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPipelineConfigKitImportsProfileAndBaseSelection(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writePipelineFixture(t, dir, "pipeline.yml", `configkit:
|
||||
imports:
|
||||
- path: config/workspace.yml
|
||||
- path: config/artifact.yml
|
||||
whisperx:
|
||||
transcribe_url: https://transcription.example.test/transcribe
|
||||
`)
|
||||
writePipelineFixture(t, dir, "config/workspace.yml", "workspace:\n root: /srv/base\n")
|
||||
writePipelineFixture(t, dir, "config/artifact.yml", `scriptorium:
|
||||
artifacts:
|
||||
recap:
|
||||
enabled: false
|
||||
output_path: artifacts/recap.md
|
||||
`)
|
||||
writePipelineFixture(t, dir, "profiles/testing.yml", "workspace:\n root: /srv/testing\n")
|
||||
writePipelineFixture(t, dir, "profiles/unselected.yml", "unknown: [\n")
|
||||
|
||||
root := filepath.Join(dir, "pipeline.yml")
|
||||
for _, profile := range []*string{nil, stringPointer(""), stringPointer("default")} {
|
||||
cfg, err := LoadPipelineWithOptions(root, PipelineLoadOptions{Profile: profile})
|
||||
if err != nil {
|
||||
t.Fatalf("load base selection %#v: %v", profile, err)
|
||||
}
|
||||
if cfg.Workspace.Root != "/srv/base" {
|
||||
t.Fatalf("base workspace = %q", cfg.Workspace.Root)
|
||||
}
|
||||
}
|
||||
selected := "testing"
|
||||
cfg, err := LoadPipelineWithOptions(root, PipelineLoadOptions{Profile: &selected})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.Workspace.Root != "/srv/testing" || len(EffectivePipelineImports(cfg)) != 2 {
|
||||
t.Fatalf("profiled pipeline = %#v, imports=%#v", cfg.Workspace, EffectivePipelineImports(cfg))
|
||||
}
|
||||
profile, ok := SelectedPipelineProfile(cfg)
|
||||
if !ok || profile.Name != "testing" || profile.OverlayPath != filepath.Join(dir, "profiles", "testing.yml") {
|
||||
t.Fatalf("selected profile = %#v, %v", profile, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPipelineConfigKitRejectsDuplicateBaseDeclarations(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name, first, second string
|
||||
}{
|
||||
{name: "artifact", first: "scriptorium:\n artifacts:\n recap:\n prompt_id: recap\n", second: "scriptorium:\n artifacts:\n recap:\n output_path: artifacts/recap.md\n"},
|
||||
{name: "artifact family", first: "scriptorium:\n artifact_families:\n recap:\n for_each: party.characters\n", second: "scriptorium:\n artifact_families:\n recap:\n output_path_pattern: artifacts/{character_id}.md\n"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writePipelineFixture(t, dir, "pipeline.yml", `configkit:
|
||||
imports:
|
||||
- path: first.yml
|
||||
- path: second.yml
|
||||
whisperx:
|
||||
transcribe_url: https://transcription.example.test/transcribe
|
||||
`)
|
||||
writePipelineFixture(t, dir, "first.yml", test.first)
|
||||
writePipelineFixture(t, dir, "second.yml", test.second)
|
||||
|
||||
_, err := LoadPipeline(filepath.Join(dir, "pipeline.yml"))
|
||||
if err == nil || !strings.Contains(err.Error(), "declared by multiple base files") ||
|
||||
!strings.Contains(err.Error(), "first.yml") || !strings.Contains(err.Error(), "second.yml") {
|
||||
t.Fatalf("duplicate declaration error = %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPipelineConfigKitRejectsUnknownDirectiveAlias(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := writePipelineFixture(t, dir, "pipeline.yml", "composition:\n imports: []\n")
|
||||
_, err := LoadPipeline(path)
|
||||
if err == nil || !strings.Contains(err.Error(), "decode.unknown_field") {
|
||||
t.Fatalf("composition alias error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func writePipelineFixture(t *testing.T, root, relative, contents string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(root, filepath.FromSlash(relative))
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(contents), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func stringPointer(value string) *string { return &value }
|
||||
@@ -1,479 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
)
|
||||
|
||||
func TestLoadPipelineProfilesSelectDefaultAndExplicitOverlay(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
rootPath := writeProfilePipeline(t, dir, `composition:
|
||||
imports: [conf.d/base.yml]
|
||||
default_profile: production
|
||||
profiles:
|
||||
production:
|
||||
overlay: profiles/production.yml
|
||||
testing:
|
||||
overlay: profiles/testing.yml
|
||||
whisperx:
|
||||
transcribe_url: https://transcription.example.com/transcribe
|
||||
`)
|
||||
importPath := writePipelineSource(t, dir, "conf.d/base.yml", `workspace:
|
||||
root: /srv/base
|
||||
audita:
|
||||
modules: [base, shared]
|
||||
`)
|
||||
productionPath := writePipelineSource(t, dir, "profiles/production.yml", `workspace:
|
||||
root: /srv/production
|
||||
whisperx:
|
||||
language: en
|
||||
`)
|
||||
testingPath := writePipelineSource(t, dir, "profiles/testing.yml", `workspace:
|
||||
root: /srv/testing
|
||||
whisperx:
|
||||
language: fr
|
||||
`)
|
||||
|
||||
production, err := LoadPipeline(rootPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if production.Workspace.Root != "/srv/production" || production.WhisperX.Language != "en" {
|
||||
t.Fatalf("default profile result = workspace=%q language=%q", production.Workspace.Root, production.WhisperX.Language)
|
||||
}
|
||||
assertSelectedPipelineProfile(t, production, "production", "default", productionPath)
|
||||
if want := []string{absolutePath(t, rootPath), absolutePath(t, importPath), absolutePath(t, productionPath)}; !slices.Equal(production.resolution.sources, want) {
|
||||
t.Fatalf("default sources = %#v, want %#v", production.resolution.sources, want)
|
||||
}
|
||||
assertPipelineFieldOwner(t, production, "workspace.root", absolutePath(t, productionPath))
|
||||
assertPipelineFieldOwner(t, production, "audita.modules", absolutePath(t, importPath))
|
||||
assertPipelineFieldOwner(t, production, "storage.backend", pipelineDefaultOwnershipSource)
|
||||
|
||||
profile := "testing"
|
||||
testingCfg, err := LoadPipelineWithOptions(rootPath, PipelineLoadOptions{Profile: &profile})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if testingCfg.Workspace.Root != "/srv/testing" || testingCfg.WhisperX.Language != "fr" {
|
||||
t.Fatalf("explicit profile result = workspace=%q language=%q", testingCfg.Workspace.Root, testingCfg.WhisperX.Language)
|
||||
}
|
||||
assertSelectedPipelineProfile(t, testingCfg, "testing", "cli", testingPath)
|
||||
}
|
||||
|
||||
func TestLoadPipelineProfileSelectionErrors(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
root string
|
||||
profile *string
|
||||
want string
|
||||
}{
|
||||
{name: "profiles require selection", root: profileComposition("", "production"), want: "default_profile is omitted"},
|
||||
{name: "unknown explicit", root: profileComposition("production", "production"), profile: profilePointer("unknown"), want: "not declared"},
|
||||
{name: "stacked explicit", root: profileComposition("production", "production", "testing"), profile: profilePointer("production,testing"), want: "not declared"},
|
||||
{name: "explicit empty", root: profileComposition("production", "production"), profile: profilePointer(""), want: "must be non-empty"},
|
||||
{name: "explicit whitespace", root: profileComposition("production", "production"), profile: profilePointer(" production "), want: "surrounding whitespace"},
|
||||
{name: "invalid default", root: profileComposition("unknown", "production"), want: "does not name a declared profile"},
|
||||
{name: "empty default", root: profileComposition("EMPTY", "production"), want: "must be non-empty"},
|
||||
{name: "empty profile name", root: `composition:
|
||||
default_profile: production
|
||||
profiles:
|
||||
"":
|
||||
overlay: production.yml
|
||||
whisperx:
|
||||
transcribe_url: https://transcription.example.com/transcribe
|
||||
`, want: "profile name must be non-empty"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
root := test.root
|
||||
if test.name == "empty default" {
|
||||
root = strings.ReplaceAll(root, "default_profile: EMPTY", `default_profile: ""`)
|
||||
}
|
||||
rootPath := writeProfilePipeline(t, dir, root)
|
||||
writePipelineSource(t, dir, "production.yml", "whisperx:\n language: en\n")
|
||||
_, err := LoadPipelineWithOptions(rootPath, PipelineLoadOptions{Profile: test.profile})
|
||||
if err == nil || !strings.Contains(strings.ToLower(err.Error()), strings.ToLower(test.want)) {
|
||||
t.Fatalf("LoadPipelineWithOptions() error = %v, want containing %q", err, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("explicit against profile-free pipeline", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
rootPath := writeProfilePipeline(t, dir, testPipelineBaseYAML)
|
||||
selected := "testing"
|
||||
_, err := LoadPipelineWithOptions(rootPath, PipelineLoadOptions{Profile: &selected})
|
||||
if err == nil || !strings.Contains(err.Error(), "declares no profiles") {
|
||||
t.Fatalf("error = %v, want profile-free rejection", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestPipelineProfileNamesRejectControlCharacters(t *testing.T) {
|
||||
if _, err := normalizePipelineProfileName("production\x00testing", "profile"); err == nil || !strings.Contains(err.Error(), "control characters") {
|
||||
t.Fatalf("normalizePipelineProfileName() error = %v, want control-character rejection", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadPipelineProfilesValidateEveryDeclaredOverlay(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
path string
|
||||
content string
|
||||
setup func(*testing.T, string)
|
||||
want string
|
||||
}{
|
||||
{name: "missing", path: "profiles/missing.yml", want: "open composition.profiles.testing.overlay"},
|
||||
{name: "malformed", path: "profiles/testing.yml", content: "workspace: [\n", want: "decode yaml"},
|
||||
{name: "duplicate keys", path: "profiles/testing.yml", content: "workspace:\n root: /one\n root: /two\n", want: "duplicate yaml key"},
|
||||
{name: "trailing document", path: "profiles/testing.yml", content: "workspace:\n root: /one\n---\nworkspace:\n root: /two\n", want: "exactly one yaml document"},
|
||||
{name: "nested composition", path: "profiles/testing.yml", content: "composition:\n imports: [nested.yml]\n", want: "only the root pipeline"},
|
||||
{name: "inheritance", path: "profiles/testing.yml", content: "workspace:\n root: /testing\n", setup: func(t *testing.T, dir string) {
|
||||
rootPath := filepath.Join(dir, "pipeline.yml")
|
||||
data, err := os.ReadFile(rootPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
updated := strings.Replace(string(data), "testing:\n overlay:", "testing:\n extends: production\n overlay:", 1)
|
||||
if err := os.WriteFile(rootPath, []byte(updated), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}, want: "only overlay is supported"},
|
||||
{name: "traversal", path: "../testing.yml", want: "invalid"},
|
||||
{name: "extension", path: "profiles/testing.json", content: "{}\n", want: ".yml or .yaml"},
|
||||
{name: "directory", path: "profiles/testing.yml", setup: func(t *testing.T, dir string) {
|
||||
if err := os.MkdirAll(filepath.Join(dir, "profiles/testing.yml"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}, want: "is a directory"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
rootPath := writeProfilePipeline(t, dir, `composition:
|
||||
default_profile: production
|
||||
profiles:
|
||||
production:
|
||||
overlay: profiles/production.yml
|
||||
testing:
|
||||
overlay: `+test.path+`
|
||||
whisperx:
|
||||
transcribe_url: https://transcription.example.com/transcribe
|
||||
`)
|
||||
writePipelineSource(t, dir, "profiles/production.yml", "whisperx:\n language: en\n")
|
||||
if test.content != "" {
|
||||
writePipelineSource(t, dir, test.path, test.content)
|
||||
}
|
||||
if test.setup != nil {
|
||||
test.setup(t, dir)
|
||||
}
|
||||
_, err := LoadPipeline(rootPath)
|
||||
if err == nil || !strings.Contains(strings.ToLower(err.Error()), strings.ToLower(test.want)) {
|
||||
t.Fatalf("LoadPipeline() error = %v, want containing %q", err, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadPipelineProfileOverlaySemantics(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
rootPath := writeProfilePipeline(t, dir, `composition:
|
||||
default_profile: testing
|
||||
profiles:
|
||||
testing:
|
||||
overlay: testing.yml
|
||||
workspace:
|
||||
root: /base
|
||||
cleanup_after_publish: true
|
||||
whisperx:
|
||||
transcribe_url: https://transcription.example.com/transcribe
|
||||
retries: 3
|
||||
audita:
|
||||
modules: [base, shared]
|
||||
scriptorium:
|
||||
artifacts:
|
||||
session_recap:
|
||||
enabled: true
|
||||
prompt_id: dnd.session_recap
|
||||
output_path: artifacts/session_recap.md
|
||||
`)
|
||||
writePipelineSource(t, dir, "testing.yml", `workspace:
|
||||
cleanup_after_publish: false
|
||||
whisperx:
|
||||
retries: 0
|
||||
audita:
|
||||
modules: []
|
||||
scriptorium:
|
||||
artifacts:
|
||||
session_recap:
|
||||
enabled: false
|
||||
experimental:
|
||||
enabled: false
|
||||
`)
|
||||
|
||||
cfg, err := LoadPipeline(rootPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.Workspace.Root != "/base" || cfg.Workspace.CleanupAfterPublish || cfg.WhisperX.Retries == nil || *cfg.WhisperX.Retries != 0 {
|
||||
t.Fatalf("recursive/zero overlay = workspace=%#v retries=%#v", cfg.Workspace, cfg.WhisperX.Retries)
|
||||
}
|
||||
if cfg.Audita.Modules == nil || len(cfg.Audita.Modules) != 0 {
|
||||
t.Fatalf("list replacement = %#v, want explicit empty list", cfg.Audita.Modules)
|
||||
}
|
||||
if cfg.Scriptorium == nil || cfg.Scriptorium.Artifacts["session_recap"].Enabled || len(cfg.Scriptorium.Artifacts) != 2 {
|
||||
t.Fatalf("keyed artifact overlay = %#v", cfg.Scriptorium)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadPipelineProfileRejectsNullAndKindChanges(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
overlay string
|
||||
want string
|
||||
}{
|
||||
{name: "null deletion", overlay: "workspace:\n root: null\n", want: "null cannot delete"},
|
||||
{name: "mapping scalar", overlay: "workspace: /other\n", want: "kind change"},
|
||||
{name: "list mapping", overlay: "audita:\n modules:\n testing: true\n", want: "kind change"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
rootPath := writeProfilePipeline(t, dir, `composition:
|
||||
default_profile: testing
|
||||
profiles:
|
||||
testing:
|
||||
overlay: testing.yml
|
||||
workspace:
|
||||
root: /base
|
||||
audita:
|
||||
modules: [base]
|
||||
whisperx:
|
||||
transcribe_url: https://transcription.example.com/transcribe
|
||||
`)
|
||||
writePipelineSource(t, dir, "testing.yml", test.overlay)
|
||||
_, err := LoadPipeline(rootPath)
|
||||
if err == nil || !strings.Contains(strings.ToLower(err.Error()), strings.ToLower(test.want)) {
|
||||
t.Fatalf("LoadPipeline() error = %v, want containing %q", err, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPipelineEffectiveDigestTracksNormalizedMeaning(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
monolithicPath := writePipelineSource(t, dir, "monolithic.yml", `workspace:
|
||||
root: /srv/narratio
|
||||
whisperx:
|
||||
transcribe_url: https://transcription.example.com/transcribe
|
||||
language: en
|
||||
`)
|
||||
composedPath := writePipelineSource(t, dir, "pipeline.yml", `composition:
|
||||
imports: [workspace.yml]
|
||||
whisperx:
|
||||
transcribe_url: https://transcription.example.com/transcribe
|
||||
language: en
|
||||
`)
|
||||
writePipelineSource(t, dir, "workspace.yml", "workspace:\n root: /srv/narratio\n")
|
||||
|
||||
monolithic, err := LoadPipeline(monolithicPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
composed, err := LoadPipeline(composedPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if monolithic.resolution.effectiveDigest == "" || monolithic.resolution.effectiveDigest != composed.resolution.effectiveDigest {
|
||||
t.Fatalf("equal effective results have digests %q and %q", monolithic.resolution.effectiveDigest, composed.resolution.effectiveDigest)
|
||||
}
|
||||
|
||||
changedPath := writePipelineSource(t, dir, "changed.yml", `workspace:
|
||||
root: /srv/narratio
|
||||
whisperx:
|
||||
transcribe_url: https://transcription.example.com/transcribe
|
||||
language: fr
|
||||
`)
|
||||
changed, err := LoadPipeline(changedPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if changed.resolution.effectiveDigest == monolithic.resolution.effectiveDigest {
|
||||
t.Fatal("semantic value change retained effective digest")
|
||||
}
|
||||
|
||||
t.Setenv("AWS_SECRET_ACCESS_KEY", "secret-one")
|
||||
first, err := LoadPipeline(monolithicPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("AWS_SECRET_ACCESS_KEY", "secret-two")
|
||||
second, err := LoadPipeline(monolithicPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if first.resolution.effectiveDigest != second.resolution.effectiveDigest || strings.Contains(first.resolution.effectiveDigest, "secret") {
|
||||
t.Fatalf("environment secret affected or appeared in digest: %q / %q", first.resolution.effectiveDigest, second.resolution.effectiveDigest)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectivePipelineValuesIgnoreEquivalentSourceLayout(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
monolithicPath := writePipelineSource(t, dir, "monolithic.yml", `workspace:
|
||||
root: /srv/narratio
|
||||
whisperx:
|
||||
transcribe_url: https://transcription.example.com/transcribe
|
||||
language: en
|
||||
`)
|
||||
composedPath := writePipelineSource(t, dir, "pipeline.yml", `composition:
|
||||
imports: [workspace.yml]
|
||||
whisperx:
|
||||
language: en
|
||||
transcribe_url: https://transcription.example.com/transcribe
|
||||
`)
|
||||
writePipelineSource(t, dir, "workspace.yml", "workspace:\n root: /srv/narratio\n")
|
||||
|
||||
monolithic, err := LoadPipeline(monolithicPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
composed, err := LoadPipeline(composedPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
monolithicValues, err := EffectivePipelineValues(monolithic)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
composedValues, err := EffectivePipelineValues(composed)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if diff := cmp.Diff(monolithicValues, composedValues); diff != "" {
|
||||
t.Fatalf("effective values differ (-monolithic +composed):\n%s", diff)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPipelineEffectiveDigestExcludesProfileIdentity(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
rootPath := writeProfilePipeline(t, dir, profileComposition("production", "production", "testing"))
|
||||
overlay := "workspace:\n root: /srv/narratio\n"
|
||||
writePipelineSource(t, dir, "production.yml", overlay)
|
||||
writePipelineSource(t, dir, "testing.yml", overlay)
|
||||
|
||||
production := "production"
|
||||
productionCfg, err := LoadPipelineWithOptions(rootPath, PipelineLoadOptions{Profile: &production})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
testing := "testing"
|
||||
testingCfg, err := LoadPipelineWithOptions(rootPath, PipelineLoadOptions{Profile: &testing})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if productionCfg.resolution.effectiveDigest != testingCfg.resolution.effectiveDigest {
|
||||
t.Fatalf("profile identity changed equal effective digests: %q / %q", productionCfg.resolution.effectiveDigest, testingCfg.resolution.effectiveDigest)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadPipelineProfilePairResolvesIndependentEffectivePipelines(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
rootPath := writeProfilePipeline(t, dir, `composition:
|
||||
imports: [base.yml]
|
||||
default_profile: production
|
||||
profiles:
|
||||
production:
|
||||
overlay: production.yml
|
||||
testing:
|
||||
overlay: testing.yml
|
||||
whisperx:
|
||||
transcribe_url: https://transcription.example.com/transcribe
|
||||
`)
|
||||
writePipelineSource(t, dir, "base.yml", "workspace:\n root: /srv/base\n")
|
||||
writePipelineSource(t, dir, "production.yml", "workspace:\n root: /srv/production\nwhisperx:\n language: en\n")
|
||||
writePipelineSource(t, dir, "testing.yml", "workspace:\n root: /srv/testing\nwhisperx:\n language: fr\n")
|
||||
|
||||
production, testing, err := LoadPipelineProfilePair(rootPath, "production", "testing")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if production.Workspace.Root != "/srv/production" || production.WhisperX.Language != "en" {
|
||||
t.Fatalf("production pair result = %#v", production)
|
||||
}
|
||||
if testing.Workspace.Root != "/srv/testing" || testing.WhisperX.Language != "fr" {
|
||||
t.Fatalf("testing pair result = %#v", testing)
|
||||
}
|
||||
production.Workspace.Root = "/mutated-left"
|
||||
if testing.Workspace.Root != "/srv/testing" {
|
||||
t.Fatalf("right profile shared mutable state with left: %q", testing.Workspace.Root)
|
||||
}
|
||||
testingFirst, productionSecond, err := LoadPipelineProfilePair(rootPath, "testing", "production")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if testingFirst.Workspace.Root != "/srv/testing" || productionSecond.Workspace.Root != "/srv/production" {
|
||||
t.Fatalf("reversed profile pair = testing=%q production=%q", testingFirst.Workspace.Root, productionSecond.Workspace.Root)
|
||||
}
|
||||
|
||||
if _, _, err := LoadPipelineProfilePair(rootPath, "production", "production"); err == nil || !strings.Contains(err.Error(), "must differ") {
|
||||
t.Fatalf("equal profile pair error = %v, want selection rejection", err)
|
||||
}
|
||||
if _, _, err := LoadPipelineProfilePair(rootPath, "unknown", "testing"); err == nil || !strings.Contains(err.Error(), "not declared") {
|
||||
t.Fatalf("unknown profile pair error = %v, want selection rejection", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadWithSessionOptionsCarriesExplicitProfilePresence(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
rootPath := writeProfilePipeline(t, dir, profileComposition("production", "production", "testing"))
|
||||
writePipelineSource(t, dir, "production.yml", "workspace:\n root: /production\n")
|
||||
writePipelineSource(t, dir, "testing.yml", "workspace:\n root: /testing\n")
|
||||
campaignPath := writePipelineSource(t, dir, "campaign.yml", "campaign_id: campaign\ninputs:\n speakers_file: speakers.yml\n autocorrect_file: autocorrect.yml\n glossary_file: glossary.yml\n party_file: party.yml\n")
|
||||
writePipelineSource(t, dir, "party.yml", canonicalPartyFixture)
|
||||
sessionPath := writePipelineSource(t, dir, "session.yml", "session_id: session\ncampaign: campaign\n")
|
||||
selected := "testing"
|
||||
cfg, err := LoadWithSessionOptions(rootPath, campaignPath, sessionPath, SessionLoadOptions{Profile: &selected})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.Pipeline.Workspace.Root != "/testing" || cfg.Pipeline.resolution.selectedProfile.source != "cli" {
|
||||
t.Fatalf("combined profile result = workspace=%q metadata=%#v", cfg.Pipeline.Workspace.Root, cfg.Pipeline.resolution.selectedProfile)
|
||||
}
|
||||
}
|
||||
|
||||
func writeProfilePipeline(t *testing.T, dir, content string) string {
|
||||
t.Helper()
|
||||
return writePipelineSource(t, dir, "pipeline.yml", content)
|
||||
}
|
||||
|
||||
func profileComposition(defaultProfile string, profiles ...string) string {
|
||||
var builder strings.Builder
|
||||
builder.WriteString("composition:\n")
|
||||
if defaultProfile != "" {
|
||||
builder.WriteString(" default_profile: " + defaultProfile + "\n")
|
||||
}
|
||||
builder.WriteString(" profiles:\n")
|
||||
for _, profile := range profiles {
|
||||
builder.WriteString(" " + profile + ":\n overlay: " + profile + ".yml\n")
|
||||
}
|
||||
builder.WriteString("whisperx:\n transcribe_url: https://transcription.example.com/transcribe\n")
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
func profilePointer(value string) *string {
|
||||
return &value
|
||||
}
|
||||
|
||||
func assertSelectedPipelineProfile(t *testing.T, cfg *PipelineConfig, name, source, overlayPath string) {
|
||||
t.Helper()
|
||||
if cfg == nil || cfg.resolution == nil || cfg.resolution.selectedProfile == nil {
|
||||
t.Fatal("selected profile provenance is absent")
|
||||
}
|
||||
selection := cfg.resolution.selectedProfile
|
||||
if selection.name != name || selection.source != source || selection.overlayPath != absolutePath(t, overlayPath) {
|
||||
t.Fatalf("selected profile = %#v, want name=%q source=%q overlay=%q", selection, name, source, absolutePath(t, overlayPath))
|
||||
}
|
||||
}
|
||||
@@ -129,7 +129,7 @@ func TestScriptoriumLoadAndValidate(t *testing.T) {
|
||||
source: narratio.transcript.polished
|
||||
artifact: session_recap
|
||||
`,
|
||||
wantLoadErr: "field artifact not found",
|
||||
wantLoadErr: "decode.unknown_field",
|
||||
},
|
||||
{
|
||||
name: "canonical previous-session source is accepted",
|
||||
|
||||
@@ -540,7 +540,7 @@ publish:
|
||||
`
|
||||
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
|
||||
_, err := Load(pipelinePath, sessionPath)
|
||||
if err == nil || !strings.Contains(err.Error(), "strict decode failed") || !strings.Contains(err.Error(), "field locks not found") {
|
||||
if err == nil || !strings.Contains(err.Error(), "decode.unknown_field") {
|
||||
t.Fatalf("Load() error = %v, want retired locks field rejected", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -57,12 +57,14 @@ func TestTrimLoadAndValidate(t *testing.T) {
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "enabled omitted defaults enabled",
|
||||
name: "explicit null remains absent",
|
||||
trimYAML: `trim:
|
||||
`,
|
||||
assert: func(t *testing.T, cfg *Config) {
|
||||
t.Helper()
|
||||
assertDefaultTrimConfig(t, cfg)
|
||||
if cfg.Pipeline.Trim != nil {
|
||||
t.Fatalf("trim config = %#v, want nil", cfg.Pipeline.Trim)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
|
||||
@@ -60,12 +60,7 @@ func validatePublish(cfg *PublishConfig, scriptorium *ScriptoriumConfig, notariu
|
||||
}
|
||||
dest := strings.TrimSpace(item.Dest)
|
||||
if dest == "" {
|
||||
derivedDest, err := artifactpolicy.ResolvePublishedDestinationWithExtractions(source, "", configuredOutputs, extractionOutputs)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s.dest is required when destination cannot be derived from %q: %w", prefix, source, err)
|
||||
}
|
||||
dest = derivedDest
|
||||
cfg.Outputs[i].Dest = derivedDest
|
||||
return fmt.Errorf("%s.dest is required when destination cannot be derived from %q", prefix, source)
|
||||
}
|
||||
normalizedDest, err := pathsafe.NormalizeRelativeDestination(dest)
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user