eric/narratio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 27 Wiki Activity

Unify effective config projection

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-07 03:35:46 +00:00
parent 10a2e89965
commit 0e6ba414b3
7 changed files with 336 additions and 268 deletions
Download Patch File Download Diff File Expand all files Collapse all files

328
internal/config/config.go
View File

@@ -17,25 +17,25 @@ type Config struct {
// PipelineConfig contains durable pipeline-level settings.
type PipelineConfig struct {
Workspace WorkspaceConfig `yaml:"workspace"`
Campaigns CampaignsConfig `yaml:"campaigns"`
Storage StorageConfig `yaml:"storage"`
Spool SpoolConfig `yaml:"spool"`
Cache CacheConfig `yaml:"cache"`
Publish *PublishConfig `yaml:"publish"`
Secrets *SecretsConfig `yaml:"secrets"`
WhisperX WhisperXConfig `yaml:"whisperx"`
Seriatim SeriatimConfig `yaml:"seriatim"`
Audita AuditaConfig `yaml:"audita"`
Normalize *NormalizeConfig `yaml:"normalize"`
Trim *TrimConfig `yaml:"trim"`
Render *RenderConfig `yaml:"render"`
Scriptorium *ScriptoriumConfig `yaml:"scriptorium"`
Notarius *NotariusConfig `yaml:"notarius"`
Notification NotificationConfig `yaml:"notification"`
Workspace WorkspaceConfig `yaml:"workspace" json:"workspace"`
Campaigns CampaignsConfig `yaml:"campaigns" json:"campaigns"`
Storage StorageConfig `yaml:"storage" json:"storage"`
Spool SpoolConfig `yaml:"spool" json:"spool"`
Cache CacheConfig `yaml:"cache" json:"cache"`
Publish *PublishConfig `yaml:"publish" json:"publish"`
Secrets *SecretsConfig `yaml:"secrets" json:"secrets"`
WhisperX WhisperXConfig `yaml:"whisperx" json:"whisperx"`
Seriatim SeriatimConfig `yaml:"seriatim" json:"seriatim"`
Audita AuditaConfig `yaml:"audita" json:"audita"`
Normalize *NormalizeConfig `yaml:"normalize" json:"normalize"`
Trim *TrimConfig `yaml:"trim" json:"trim"`
Render *RenderConfig `yaml:"render" json:"render"`
Scriptorium *ScriptoriumConfig `yaml:"scriptorium" json:"scriptorium"`
Notarius *NotariusConfig `yaml:"notarius" json:"notarius"`
Notification NotificationConfig `yaml:"notification" json:"notification"`
resolution *pipelineResolutionMetadata `yaml:"-"`
familyCatalog ArtifactFamilyCatalog `yaml:"-"`
resolution *pipelineResolutionMetadata `yaml:"-" json:"-"`
familyCatalog ArtifactFamilyCatalog `yaml:"-" json:"-"`
}
// ArtifactFamilies returns a copy of runtime-only expansion provenance.
@@ -48,51 +48,51 @@ func ArtifactFamilies(cfg *PipelineConfig) ArtifactFamilyCatalog {
// CampaignsConfig configures the local campaign registry.
type CampaignsConfig struct {
Root string `yaml:"root"`
DefaultCampaignID string `yaml:"default_campaign_id"`
Root string `yaml:"root" json:"root"`
DefaultCampaignID string `yaml:"default_campaign_id" json:"default_campaign_id"`
}
// CampaignConfig contains stable campaign-level identity and input defaults.
type CampaignConfig struct {
CampaignID string `yaml:"campaign_id"`
SessionTemplateFile string `yaml:"session_template_file"`
Inputs CampaignInputsConfig `yaml:"inputs"`
CampaignID string `yaml:"campaign_id" json:"campaign_id"`
SessionTemplateFile string `yaml:"session_template_file" json:"session_template_file"`
Inputs CampaignInputsConfig `yaml:"inputs" json:"inputs"`
}
// CampaignInputsConfig contains stable campaign-level input file references.
type CampaignInputsConfig struct {
SpeakersFile string `yaml:"speakers_file"`
AutocorrectFile string `yaml:"autocorrect_file"`
GlossaryFile string `yaml:"glossary_file"`
PartyFile string `yaml:"party_file"`
SpellCatalogFile string `yaml:"spell_catalog_file"`
SpeakersFile string `yaml:"speakers_file" json:"speakers_file"`
AutocorrectFile string `yaml:"autocorrect_file" json:"autocorrect_file"`
GlossaryFile string `yaml:"glossary_file" json:"glossary_file"`
PartyFile string `yaml:"party_file" json:"party_file"`
SpellCatalogFile string `yaml:"spell_catalog_file" json:"spell_catalog_file"`
}
// SessionConfig contains per-session inputs and metadata.
type SessionConfig struct {
SessionID string `yaml:"session_id"`
PreviousSessionID string `yaml:"previous_session_id"`
Campaign string `yaml:"campaign"`
Date string `yaml:"date"`
Title string `yaml:"title"`
Inputs SessionInputsConfig `yaml:"inputs"`
SessionID string `yaml:"session_id" json:"session_id"`
PreviousSessionID string `yaml:"previous_session_id" json:"previous_session_id"`
Campaign string `yaml:"campaign" json:"campaign"`
Date string `yaml:"date" json:"date"`
Title string `yaml:"title" json:"title"`
Inputs SessionInputsConfig `yaml:"inputs" json:"inputs"`
}
// WorkspaceConfig configures local workspace behavior.
type WorkspaceConfig struct {
Root string `yaml:"root"`
CleanupAfterPublish bool `yaml:"cleanup_after_publish"`
Root string `yaml:"root" json:"root"`
CleanupAfterPublish bool `yaml:"cleanup_after_publish" json:"cleanup_after_publish"`
}
// SecretsConfig configures optional local filesystem secret loading.
type SecretsConfig struct {
EnvDir string `yaml:"env_dir"`
EnvDir string `yaml:"env_dir" json:"env_dir"`
}
// StorageConfig configures storage backends and related parameters.
type StorageConfig struct {
Backend string `yaml:"backend"`
S3 *StorageS3Config `yaml:"s3"`
Backend string `yaml:"backend" json:"backend"`
S3 *StorageS3Config `yaml:"s3" json:"s3"`
}
const (
@@ -104,178 +104,178 @@ const (
// StorageS3Config configures S3 storage coordinates.
type StorageS3Config struct {
Bucket string `yaml:"bucket"`
RootPrefix string `yaml:"root_prefix"`
Region string `yaml:"region"`
Endpoint string `yaml:"endpoint"`
ForcePathStyle bool `yaml:"force_path_style"`
AccessKeyIDEnv string `yaml:"access_key_id_env"`
SecretKeyEnv string `yaml:"secret_access_key_env"`
Bucket string `yaml:"bucket" json:"bucket"`
RootPrefix string `yaml:"root_prefix" json:"root_prefix"`
Region string `yaml:"region" json:"region"`
Endpoint string `yaml:"endpoint" json:"endpoint"`
ForcePathStyle bool `yaml:"force_path_style" json:"force_path_style"`
AccessKeyIDEnv string `yaml:"access_key_id_env" json:"access_key_id_env"`
SecretKeyEnv string `yaml:"secret_access_key_env" json:"secret_access_key_env"`
}
// SpoolConfig configures local spool storage for staged data.
type SpoolConfig struct {
Root string `yaml:"root"`
DeleteAudioAfterPublish bool `yaml:"delete_audio_after_publish"`
Root string `yaml:"root" json:"root"`
DeleteAudioAfterPublish bool `yaml:"delete_audio_after_publish" json:"delete_audio_after_publish"`
}
// CacheConfig configures durable local caches for reusable remote inputs.
type CacheConfig struct {
Root string `yaml:"root"`
S3Audio *bool `yaml:"s3_audio"`
Root string `yaml:"root" json:"root"`
S3Audio *bool `yaml:"s3_audio" json:"s3_audio"`
}
// PublishConfig configures publish behavior and source-based output uploads.
type PublishConfig struct {
Enabled *bool `yaml:"enabled"`
UploadRun *bool `yaml:"upload_run"`
Outputs []PublishOutputRule `yaml:"outputs"`
Enabled *bool `yaml:"enabled" json:"enabled"`
UploadRun *bool `yaml:"upload_run" json:"upload_run"`
Outputs []PublishOutputRule `yaml:"outputs" json:"outputs"`
}
// PublishOutputRule configures one source-to-destination publish mapping.
type PublishOutputRule struct {
Source string `yaml:"source"`
Dest string `yaml:"dest"`
Required *bool `yaml:"required"`
Source string `yaml:"source" json:"source"`
Dest string `yaml:"dest" json:"dest"`
Required *bool `yaml:"required" json:"required"`
}
// WhisperXConfig configures WhisperX adapter settings.
type WhisperXConfig struct {
TranscribeURL string `yaml:"transcribe_url"`
Language string `yaml:"language"`
Timeout string `yaml:"timeout"`
Retries *int `yaml:"retries"`
RetryDelay string `yaml:"retry_delay"`
Concurrency *int `yaml:"concurrency"`
TranscribeURL string `yaml:"transcribe_url" json:"transcribe_url"`
Language string `yaml:"language" json:"language"`
Timeout string `yaml:"timeout" json:"timeout"`
Retries *int `yaml:"retries" json:"retries"`
RetryDelay string `yaml:"retry_delay" json:"retry_delay"`
Concurrency *int `yaml:"concurrency" json:"concurrency"`
}
// SeriatimConfig configures seriatim adapter settings.
type SeriatimConfig struct {
Binary string `yaml:"binary"`
Timeout string `yaml:"timeout"`
OutputSchema string `yaml:"output_schema"`
CoalesceGap *float64 `yaml:"coalesce_gap"`
Report *bool `yaml:"report"`
Env SeriatimEnvConfig `yaml:"env"`
Binary string `yaml:"binary" json:"binary"`
Timeout string `yaml:"timeout" json:"timeout"`
OutputSchema string `yaml:"output_schema" json:"output_schema"`
CoalesceGap *float64 `yaml:"coalesce_gap" json:"coalesce_gap"`
Report *bool `yaml:"report" json:"report"`
Env SeriatimEnvConfig `yaml:"env" json:"env"`
}
// SeriatimEnvConfig configures optional advanced seriatim environment tuning.
type SeriatimEnvConfig struct {
OverlapWordRunGap *float64 `yaml:"overlap_word_run_gap"`
OverlapWordRunReorderWindow *float64 `yaml:"overlap_word_run_reorder_window"`
BackchannelMaxDuration *float64 `yaml:"backchannel_max_duration"`
FillerMaxDuration *float64 `yaml:"filler_max_duration"`
OverlapWordRunGap *float64 `yaml:"overlap_word_run_gap" json:"overlap_word_run_gap"`
OverlapWordRunReorderWindow *float64 `yaml:"overlap_word_run_reorder_window" json:"overlap_word_run_reorder_window"`
BackchannelMaxDuration *float64 `yaml:"backchannel_max_duration" json:"backchannel_max_duration"`
FillerMaxDuration *float64 `yaml:"filler_max_duration" json:"filler_max_duration"`
}
// AuditaConfig configures audita adapter settings.
type AuditaConfig struct {
Binary string `yaml:"binary"`
Timeout string `yaml:"timeout"`
LLMAPIKeyEnv string `yaml:"llm_api_key_env"`
Modules []string `yaml:"modules"`
BaseURL string `yaml:"base_url"`
Model string `yaml:"model"`
TotalLLMConcurrency *int `yaml:"total_llm_concurrency"`
ProposalLLMConcurrency *int `yaml:"proposal_llm_concurrency"`
ValidationModel string `yaml:"validation_model"`
ValidationLLMConcurrency *int `yaml:"validation_llm_concurrency"`
TranscriptDescription string `yaml:"transcript_description"`
ConfigPath string `yaml:"config_path"`
OutputSchema string `yaml:"output_schema"`
WorkDirRetention string `yaml:"work_dir_retention"`
Report *bool `yaml:"report"`
Binary string `yaml:"binary" json:"binary"`
Timeout string `yaml:"timeout" json:"timeout"`
LLMAPIKeyEnv string `yaml:"llm_api_key_env" json:"llm_api_key_env"`
Modules []string `yaml:"modules" json:"modules"`
BaseURL string `yaml:"base_url" json:"base_url"`
Model string `yaml:"model" json:"model"`
TotalLLMConcurrency *int `yaml:"total_llm_concurrency" json:"total_llm_concurrency"`
ProposalLLMConcurrency *int `yaml:"proposal_llm_concurrency" json:"proposal_llm_concurrency"`
ValidationModel string `yaml:"validation_model" json:"validation_model"`
ValidationLLMConcurrency *int `yaml:"validation_llm_concurrency" json:"validation_llm_concurrency"`
TranscriptDescription string `yaml:"transcript_description" json:"transcript_description"`
ConfigPath string `yaml:"config_path" json:"config_path"`
OutputSchema string `yaml:"output_schema" json:"output_schema"`
WorkDirRetention string `yaml:"work_dir_retention" json:"work_dir_retention"`
Report *bool `yaml:"report" json:"report"`
}
// NormalizeConfig configures normalize-stage transcript schema/output behavior.
type NormalizeConfig struct {
OutputPath string `yaml:"output_path"`
OutputSchema string `yaml:"output_schema"`
Report *bool `yaml:"report"`
OutputPath string `yaml:"output_path" json:"output_path"`
OutputSchema string `yaml:"output_schema" json:"output_schema"`
Report *bool `yaml:"report" json:"report"`
outputPathSet bool `yaml:"-"`
outputPathSet bool `yaml:"-" json:"-"`
}
// TrimConfig configures trim-stage transcript boundary behavior.
type TrimConfig struct {
Enabled *bool `yaml:"enabled"`
OutputPath string `yaml:"output_path"`
Bounds TrimBoundsConfig `yaml:"bounds"`
Seriatim TrimSeriatimConfig `yaml:"seriatim"`
Enabled *bool `yaml:"enabled" json:"enabled"`
OutputPath string `yaml:"output_path" json:"output_path"`
Bounds TrimBoundsConfig `yaml:"bounds" json:"bounds"`
Seriatim TrimSeriatimConfig `yaml:"seriatim" json:"seriatim"`
}
// TrimBoundsConfig configures Scriptorium bounds prompt execution for trim.
type TrimBoundsConfig struct {
PromptID string `yaml:"prompt_id"`
ProfileID string `yaml:"profile_id"`
TranscriptInputName string `yaml:"transcript_input_name"`
OutputPath string `yaml:"output_path"`
Timeout string `yaml:"timeout"`
RenderDebug bool `yaml:"render_debug"`
RenderOutputPath string `yaml:"render_output_path"`
PromptID string `yaml:"prompt_id" json:"prompt_id"`
ProfileID string `yaml:"profile_id" json:"profile_id"`
TranscriptInputName string `yaml:"transcript_input_name" json:"transcript_input_name"`
OutputPath string `yaml:"output_path" json:"output_path"`
Timeout string `yaml:"timeout" json:"timeout"`
RenderDebug bool `yaml:"render_debug" json:"render_debug"`
RenderOutputPath string `yaml:"render_output_path" json:"render_output_path"`
}
// TrimSeriatimConfig configures Seriatim options for trim output construction.
type TrimSeriatimConfig struct {
Report *bool `yaml:"report"`
Report *bool `yaml:"report" json:"report"`
}
// RenderConfig configures render-stage output formatting behavior.
type RenderConfig struct {
Enabled *bool `yaml:"enabled"`
Format string `yaml:"format"`
Title string `yaml:"title"`
IncludeTimestamps *bool `yaml:"include_timestamps"`
IncludeSegmentIDs *bool `yaml:"include_segment_ids"`
IncludeMetadata bool `yaml:"include_metadata"`
Enabled *bool `yaml:"enabled" json:"enabled"`
Format string `yaml:"format" json:"format"`
Title string `yaml:"title" json:"title"`
IncludeTimestamps *bool `yaml:"include_timestamps" json:"include_timestamps"`
IncludeSegmentIDs *bool `yaml:"include_segment_ids" json:"include_segment_ids"`
IncludeMetadata bool `yaml:"include_metadata" json:"include_metadata"`
}
// ScriptoriumConfig configures Scriptorium-backed artifact generation.
type ScriptoriumConfig struct {
Binary string `yaml:"binary"`
ConfigPath string `yaml:"config_path"`
Timeout string `yaml:"timeout"`
RenderDebug bool `yaml:"render_debug"`
Artifacts map[string]ScriptoriumArtifactConfig `yaml:"artifacts"`
ArtifactFamilies map[string]ScriptoriumArtifactFamilyConfig `yaml:"artifact_families"`
Binary string `yaml:"binary" json:"binary"`
ConfigPath string `yaml:"config_path" json:"config_path"`
Timeout string `yaml:"timeout" json:"timeout"`
RenderDebug bool `yaml:"render_debug" json:"render_debug"`
Artifacts map[string]ScriptoriumArtifactConfig `yaml:"artifacts" json:"artifacts"`
ArtifactFamilies map[string]ScriptoriumArtifactFamilyConfig `yaml:"artifact_families" json:"artifact_families"`
}
// ScriptoriumArtifactConfig configures one named output artifact workflow.
type ScriptoriumArtifactConfig struct {
Enabled bool `yaml:"enabled"`
DependsOn []string `yaml:"depends_on"`
RenderDebug *bool `yaml:"render_debug"`
PromptID string `yaml:"prompt_id"`
ProfileID string `yaml:"profile_id"`
OutputPath string `yaml:"output_path"`
Timeout string `yaml:"timeout"`
Inputs map[string]ScriptoriumInputConfig `yaml:"inputs"`
Vars map[string]any `yaml:"vars"`
Enabled bool `yaml:"enabled" json:"enabled"`
DependsOn []string `yaml:"depends_on" json:"depends_on"`
RenderDebug *bool `yaml:"render_debug" json:"render_debug"`
PromptID string `yaml:"prompt_id" json:"prompt_id"`
ProfileID string `yaml:"profile_id" json:"profile_id"`
OutputPath string `yaml:"output_path" json:"output_path"`
Timeout string `yaml:"timeout" json:"timeout"`
Inputs map[string]ScriptoriumInputConfig `yaml:"inputs" json:"inputs"`
Vars map[string]any `yaml:"vars" json:"vars"`
}
// ScriptoriumArtifactFamilyConfig declares the shared configuration expanded
// into one ordinary artifact for each canonical party character.
type ScriptoriumArtifactFamilyConfig struct {
Enabled bool `yaml:"enabled"`
ForEach string `yaml:"for_each"`
DependsOn []string `yaml:"depends_on"`
RenderDebug *bool `yaml:"render_debug"`
PromptID string `yaml:"prompt_id"`
ProfileID string `yaml:"profile_id"`
OutputPathPattern string `yaml:"output_path_pattern"`
Timeout string `yaml:"timeout"`
Inputs map[string]ScriptoriumInputConfig `yaml:"inputs"`
Vars map[string]any `yaml:"vars"`
MemberVars map[string]string `yaml:"member_vars"`
MemberDependencies []string `yaml:"member_dependencies"`
Publish *ScriptoriumArtifactFamilyPublishConfig `yaml:"publish"`
Enabled bool `yaml:"enabled" json:"enabled"`
ForEach string `yaml:"for_each" json:"for_each"`
DependsOn []string `yaml:"depends_on" json:"depends_on"`
RenderDebug *bool `yaml:"render_debug" json:"render_debug"`
PromptID string `yaml:"prompt_id" json:"prompt_id"`
ProfileID string `yaml:"profile_id" json:"profile_id"`
OutputPathPattern string `yaml:"output_path_pattern" json:"output_path_pattern"`
Timeout string `yaml:"timeout" json:"timeout"`
Inputs map[string]ScriptoriumInputConfig `yaml:"inputs" json:"inputs"`
Vars map[string]any `yaml:"vars" json:"vars"`
MemberVars map[string]string `yaml:"member_vars" json:"member_vars"`
MemberDependencies []string `yaml:"member_dependencies" json:"member_dependencies"`
Publish *ScriptoriumArtifactFamilyPublishConfig `yaml:"publish" json:"publish"`
}
// ScriptoriumArtifactFamilyPublishConfig retains the typed declaration for
// family publishing. It is resolved by the publish owner.
type ScriptoriumArtifactFamilyPublishConfig struct {
Enabled bool `yaml:"enabled"`
Required bool `yaml:"required"`
DestPattern string `yaml:"dest_pattern"`
Enabled bool `yaml:"enabled" json:"enabled"`
Required bool `yaml:"required" json:"required"`
DestPattern string `yaml:"dest_pattern" json:"dest_pattern"`
}
// ArtifactFamilyCatalog records the runtime-only origin of expanded artifacts.
@@ -305,50 +305,50 @@ type ArtifactFamilyMemberOrigin struct {
// ScriptoriumInputConfig configures one named prompt input source.
type ScriptoriumInputConfig struct {
Source string `yaml:"source"`
Required bool `yaml:"required"`
Source string `yaml:"source" json:"source"`
Required bool `yaml:"required" json:"required"`
}
// NotariusConfig configures structured artifact extraction by Notarius.
type NotariusConfig struct {
Enabled bool `yaml:"enabled"`
Binary string `yaml:"binary"`
ConfigPath string `yaml:"config_path"`
PipelineID string `yaml:"pipeline_id"`
Timeout string `yaml:"timeout"`
WorkingDirectory string `yaml:"working_directory"`
References map[string]string `yaml:"references"`
Outputs map[string]NotariusOutputConfig `yaml:"outputs"`
Enabled bool `yaml:"enabled" json:"enabled"`
Binary string `yaml:"binary" json:"binary"`
ConfigPath string `yaml:"config_path" json:"config_path"`
PipelineID string `yaml:"pipeline_id" json:"pipeline_id"`
Timeout string `yaml:"timeout" json:"timeout"`
WorkingDirectory string `yaml:"working_directory" json:"working_directory"`
References map[string]string `yaml:"references" json:"references"`
Outputs map[string]NotariusOutputConfig `yaml:"outputs" json:"outputs"`
}
// NotariusOutputConfig declares one required extraction lane contract.
type NotariusOutputConfig struct {
LaneID string `yaml:"lane_id"`
MediaType string `yaml:"media_type"`
SchemaID string `yaml:"schema_id"`
SchemaVersion string `yaml:"schema_version"`
ModuleKey string `yaml:"module_key"`
LaneID string `yaml:"lane_id" json:"lane_id"`
MediaType string `yaml:"media_type" json:"media_type"`
SchemaID string `yaml:"schema_id" json:"schema_id"`
SchemaVersion string `yaml:"schema_version" json:"schema_version"`
ModuleKey string `yaml:"module_key" json:"module_key"`
}
// NotificationConfig configures the supported notification behavior.
type NotificationConfig struct {
Mode string `yaml:"mode"`
Mode string `yaml:"mode" json:"mode"`
}
// SessionInputsConfig contains per-session input references.
type SessionInputsConfig struct {
AudioDir string `yaml:"audio_dir"`
AudioFiles []string `yaml:"audio_files"`
AudioS3 *SessionAudioS3Input `yaml:"audio_s3"`
SpeakersFile string `yaml:"speakers_file"`
AutocorrectFile string `yaml:"autocorrect_file"`
GlossaryFile string `yaml:"glossary_file"`
SpellCatalogFile string `yaml:"spell_catalog_file"`
AudioDir string `yaml:"audio_dir" json:"audio_dir"`
AudioFiles []string `yaml:"audio_files" json:"audio_files"`
AudioS3 *SessionAudioS3Input `yaml:"audio_s3" json:"audio_s3"`
SpeakersFile string `yaml:"speakers_file" json:"speakers_file"`
AutocorrectFile string `yaml:"autocorrect_file" json:"autocorrect_file"`
GlossaryFile string `yaml:"glossary_file" json:"glossary_file"`
SpellCatalogFile string `yaml:"spell_catalog_file" json:"spell_catalog_file"`
}
// SessionAudioS3Input configures S3 session-audio input discovery.
type SessionAudioS3Input struct {
Prefix string `yaml:"prefix"`
Prefix string `yaml:"prefix" json:"prefix"`
}
// ResolvedStableInputs records where stable input file paths came from after

38
internal/config/effective_digest.go
View File

@@ -5,14 +5,13 @@ import (
"encoding/hex"
"fmt"
"path/filepath"
"sort"
"strings"
"gopkg.in/yaml.v3"
)
const (
pipelineDefaultOwnershipSource = "default"
effectivePipelineDigestVersion = "narratio-effective-pipeline-v2\n"
effectivePipelineDigestVersion = "narratio-effective-pipeline-v3\n"
)
// PipelineProfileProvenance identifies the profile selected while resolving a
@@ -49,15 +48,7 @@ func finalizePipelineResolution(cfg *PipelineConfig) error {
for _, ownership := range cfg.resolution.ownership {
declared[ownership.path] = append([]string(nil), ownership.sources...)
}
data, err := yaml.Marshal(cfg)
if err != nil {
return fmt.Errorf("serialize normalized effective pipeline: %w", err)
}
document, err := parseCompositionBytes("normalized effective pipeline", data)
if err != nil {
return err
}
records, err := document.semanticRecords()
records, err := projectionValueRecords(projectEffectivePipeline(cfg, true))
if err != nil {
return err
}
@@ -71,6 +62,12 @@ func finalizePipelineResolution(cfg *PipelineConfig) error {
path: record.Path, sources: append([]string(nil), sources...),
})
}
for path, sources := range declared {
if strings.HasPrefix(path, "scriptorium.artifact_families.") {
ownership = append(ownership, pipelineFieldOwnership{path: path, sources: append([]string(nil), sources...)})
}
}
sort.Slice(ownership, func(i, j int) bool { return ownership[i].path < ownership[j].path })
cfg.resolution.ownership = ownership
return recomputePipelineEffectiveDigest(cfg)
}
@@ -82,22 +79,7 @@ func recomputePipelineEffectiveDigest(cfg *PipelineConfig) error {
if cfg == nil || cfg.resolution == nil {
return fmt.Errorf("pipeline resolution metadata is required")
}
digestConfig := *cfg
if cfg.Notarius != nil && cfg.resolution.logicalNotariusCaptured {
notarius := *cfg.Notarius
notarius.ConfigPath = cfg.resolution.logicalNotariusConfig
notarius.WorkingDirectory = cfg.resolution.logicalNotariusWorking
digestConfig.Notarius = &notarius
}
data, err := yaml.Marshal(&digestConfig)
if err != nil {
return fmt.Errorf("serialize normalized effective pipeline: %w", err)
}
document, err := parseCompositionBytes("normalized effective pipeline", data)
if err != nil {
return err
}
canonical, err := document.canonicalDigestInput()
canonical, err := projectionJSON(projectEffectivePipeline(cfg, true))
if err != nil {
return err
}

74
internal/config/effective_pipeline.go
View File

@@ -60,23 +60,7 @@ func EffectivePipelineValues(cfg *PipelineConfig) ([]EffectivePipelineValueRecor
if cfg == nil {
return nil, fmt.Errorf("pipeline config is required")
}
data, err := MarshalEffectivePipeline(cfg)
if err != nil {
return nil, err
}
document, err := parseCompositionBytes("effective pipeline", data)
if err != nil {
return nil, err
}
records, err := document.compactSemanticRecords()
if err != nil {
return nil, err
}
values := make([]EffectivePipelineValueRecord, 0, len(records))
for _, record := range records {
values = append(values, EffectivePipelineValueRecord{Path: record.Path, Value: record.Value})
}
return values, nil
return projectionValueRecords(projectEffectivePipeline(cfg, true))
}
// EffectivePipelineSources projects pipeline ownership after defaults and
@@ -86,15 +70,14 @@ func EffectivePipelineSources(cfg *PipelineConfig, party ResolvedParty) ([]Effec
if cfg == nil {
return nil, fmt.Errorf("pipeline config is required")
}
data, err := MarshalEffectivePipeline(cfg)
values, err := EffectivePipelineValues(cfg)
if err != nil {
return nil, err
}
document, err := parseCompositionBytes("effective pipeline", data)
if err != nil {
return nil, err
paths := make([]string, 0, len(values))
for _, value := range values {
paths = append(paths, value.Path)
}
paths := effectivePipelineFieldPaths(document.root)
owners := effectivePipelineOwners(cfg)
families := ArtifactFamilies(cfg)
publishCount := 0
@@ -168,54 +151,11 @@ func MarshalEffectivePipeline(cfg *PipelineConfig) ([]byte, error) {
return nil, fmt.Errorf("pipeline config is required")
}
data, err := yaml.Marshal(cfg)
data, err := yaml.Marshal(projectEffectivePipeline(cfg, true))
if err != nil {
return nil, fmt.Errorf("serialize effective pipeline: %w", err)
}
document, err := parseCompositionBytes("effective pipeline", data)
if err != nil {
return nil, err
}
removeResolutionOnlyPipelineFields(document)
return document.canonicalYAML()
}
func removeResolutionOnlyPipelineFields(document *compositionDocument) {
if document == nil || document.root == nil {
return
}
scriptoriumIndex := compositionFieldIndex(document.root.fields, "scriptorium")
if scriptoriumIndex < 0 {
return
}
scriptorium := document.root.fields[scriptoriumIndex].value
if scriptorium == nil || scriptorium.kind != yaml.MappingNode {
return
}
familyIndex := compositionFieldIndex(scriptorium.fields, "artifact_families")
if familyIndex < 0 {
return
}
scriptorium.fields = append(scriptorium.fields[:familyIndex], scriptorium.fields[familyIndex+1:]...)
}
func effectivePipelineFieldPaths(node *compositionNode) []string {
var paths []string
var visit func(*compositionNode)
visit = func(current *compositionNode) {
if current == nil {
return
}
if current.kind == yaml.MappingNode && len(current.fields) > 0 {
for _, field := range current.fields {
visit(field.value)
}
return
}
paths = append(paths, current.path)
}
visit(node)
return paths
return data, nil
}
func effectivePipelineOwners(cfg *PipelineConfig) map[string][]string {

107
internal/config/effective_projection.go Normal file
View File

@@ -0,0 +1,107 @@
package config
import (
"bytes"
"encoding/json"
"fmt"
"sort"
)
// effectivePipelineProjection is the single safe, schema-specific view used
// for inspection, comparison, and identity. Runtime metadata and unresolved
// family declarations have no slot in this type.
type effectivePipelineProjection struct {
Workspace WorkspaceConfig `yaml:"workspace" json:"workspace"`
Campaigns CampaignsConfig `yaml:"campaigns" json:"campaigns"`
Storage StorageConfig `yaml:"storage" json:"storage"`
Spool SpoolConfig `yaml:"spool" json:"spool"`
Cache CacheConfig `yaml:"cache" json:"cache"`
Publish *PublishConfig `yaml:"publish" json:"publish"`
Secrets *SecretsConfig `yaml:"secrets" json:"secrets"`
WhisperX WhisperXConfig `yaml:"whisperx" json:"whisperx"`
Seriatim SeriatimConfig `yaml:"seriatim" json:"seriatim"`
Audita AuditaConfig `yaml:"audita" json:"audita"`
Normalize *NormalizeConfig `yaml:"normalize" json:"normalize"`
Trim *TrimConfig `yaml:"trim" json:"trim"`
Render *RenderConfig `yaml:"render" json:"render"`
Scriptorium *effectiveScriptoriumConfig `yaml:"scriptorium" json:"scriptorium"`
Notarius *NotariusConfig `yaml:"notarius" json:"notarius"`
Notification NotificationConfig `yaml:"notification" json:"notification"`
}
type effectiveScriptoriumConfig struct {
Binary string `yaml:"binary" json:"binary"`
ConfigPath string `yaml:"config_path" json:"config_path"`
Timeout string `yaml:"timeout" json:"timeout"`
RenderDebug bool `yaml:"render_debug" json:"render_debug"`
Artifacts map[string]ScriptoriumArtifactConfig `yaml:"artifacts" json:"artifacts"`
}
func projectEffectivePipeline(cfg *PipelineConfig, logicalNotarius bool) effectivePipelineProjection {
projection := effectivePipelineProjection{
Workspace: cfg.Workspace, Campaigns: cfg.Campaigns, Storage: cfg.Storage,
Spool: cfg.Spool, Cache: cfg.Cache, Publish: cfg.Publish, Secrets: cfg.Secrets,
WhisperX: cfg.WhisperX, Seriatim: cfg.Seriatim, Audita: cfg.Audita,
Normalize: cfg.Normalize, Trim: cfg.Trim, Render: cfg.Render,
Notarius: cfg.Notarius, Notification: cfg.Notification,
}
if cfg.Scriptorium != nil {
projection.Scriptorium = &effectiveScriptoriumConfig{
Binary: cfg.Scriptorium.Binary, ConfigPath: cfg.Scriptorium.ConfigPath,
Timeout: cfg.Scriptorium.Timeout, RenderDebug: cfg.Scriptorium.RenderDebug,
Artifacts: cfg.Scriptorium.Artifacts,
}
}
if logicalNotarius && cfg.Notarius != nil && cfg.resolution != nil && cfg.resolution.logicalNotariusCaptured {
notarius := *cfg.Notarius
notarius.ConfigPath = cfg.resolution.logicalNotariusConfig
notarius.WorkingDirectory = cfg.resolution.logicalNotariusWorking
projection.Notarius = &notarius
}
return projection
}
func projectionJSON(projection effectivePipelineProjection) ([]byte, error) {
data, err := json.Marshal(projection)
if err != nil {
return nil, fmt.Errorf("encode effective pipeline projection: %w", err)
}
return data, nil
}
func projectionValueRecords(projection effectivePipelineProjection) ([]EffectivePipelineValueRecord, error) {
data, err := projectionJSON(projection)
if err != nil {
return nil, err
}
decoder := json.NewDecoder(bytes.NewReader(data))
decoder.UseNumber()
var root map[string]any
if err := decoder.Decode(&root); err != nil {
return nil, fmt.Errorf("decode effective pipeline projection: %w", err)
}
var records []EffectivePipelineValueRecord
appendProjectionRecords(&records, "", root)
sort.Slice(records, func(i, j int) bool { return records[i].Path < records[j].Path })
return records, nil
}
func appendProjectionRecords(records *[]EffectivePipelineValueRecord, path string, value any) {
if mapping, ok := value.(map[string]any); ok && len(mapping) > 0 {
keys := make([]string, 0, len(mapping))
for key := range mapping {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
childPath := key
if path != "" {
childPath = path + "." + key
}
appendProjectionRecords(records, childPath, mapping[key])
}
return
}
encoded, _ := json.Marshal(value)
*records = append(*records, EffectivePipelineValueRecord{Path: path, Value: string(encoded)})
}

36
internal/config/load_validate_test.go
View File

@@ -83,6 +83,42 @@ whisperx:
}
}
func TestEffectivePipelineValuesPreserveNilEmptyAndExactIntegers(t *testing.T) {
base := &PipelineConfig{Audita: AuditaConfig{Modules: nil}}
nilValues, err := EffectivePipelineValues(base)
if err != nil {
t.Fatal(err)
}
base.Audita.Modules = []string{}
base.Scriptorium = &ScriptoriumConfig{Artifacts: map[string]ScriptoriumArtifactConfig{
"recap": {Vars: map[string]any{"exact": int64(9007199254740993)}},
}}
emptyValues, err := EffectivePipelineValues(base)
if err != nil {
t.Fatal(err)
}
if got := effectiveValueForTest(t, nilValues, "audita.modules"); got != "null" {
t.Fatalf("nil sequence value = %q, want null", got)
}
if got := effectiveValueForTest(t, emptyValues, "audita.modules"); got != "[]" {
t.Fatalf("empty sequence value = %q, want []", got)
}
if got := effectiveValueForTest(t, emptyValues, "scriptorium.artifacts.recap.vars.exact"); got != "9007199254740993" {
t.Fatalf("exact integer value = %q", got)
}
}
func effectiveValueForTest(t *testing.T, values []EffectivePipelineValueRecord, path string) string {
t.Helper()
for _, value := range values {
if value.Path == path {
return value.Value
}
}
t.Fatalf("effective value %q not found", path)
return ""
}
func TestValidateMissingAudioSource(t *testing.T) {
cfg := loadedValidConfig(t)
cfg.Session.Inputs.AudioDir = ""