diff --git a/docs/internal/configuration.md b/docs/internal/configuration.md index dbc88db..1ade825 100644 --- a/docs/internal/configuration.md +++ b/docs/internal/configuration.md @@ -64,11 +64,10 @@ validation, which expands its canonical-party members and generated publish rules. A pipeline without those declarations may be validated by itself. `MarshalEffectivePipeline` is the configuration-owned projection for `config -show`. It serializes the typed, defaulted effective mapping through the -deterministic composition renderer, then removes resolution-only artifact -family declarations. The result contains no composition envelope or private -provenance fields and has one trailing newline; commands do not marshal runtime -objects directly. +show`. It serializes one explicit typed safe projection with the existing YAML +encoder. The projection has no slots for resolution-only artifact family +declarations or private provenance. The result has one trailing newline; +commands do not marshal runtime objects directly. `EffectivePipelineSources` and `EffectiveCampaignSources` provide the separate safe provenance projection for `config sources`. Pipeline ownership begins with @@ -85,7 +84,8 @@ declared overlays once, then clones the additive base before independently selecting, decoding, defaulting, and finalizing each profile. When campaign resolution is needed, the command loads one selected campaign and party and expands both effective pipelines from that same party value. The configuration -owner projects each normalized effective mapping into sorted logical paths; +owner projects each normalized effective mapping directly from that same typed +projection into sorted logical paths using JSON number preservation; mapping leaves are compared individually while sequence values remain atomic. Values are compact deterministic JSON representations for command output, not raw YAML fragments, ownership records, or secret material. A differing digest @@ -148,8 +148,8 @@ overlay, and centrally supplied values use the synthetic `default` owner. This metadata does not participate in YAML decoding or alter the public configuration model. -The effective digest is SHA-256 over deterministic canonical YAML produced from -the defaulted `PipelineConfig`. Runtime Notarius paths remain absolute for +The effective digest is SHA-256 over a versioned deterministic JSON encoding of +the same typed safe projection used by show and diff. Runtime Notarius paths remain absolute for execution, but the digest substitutes their normalized logical values captured before root-relative resolution, so relocating an equivalent configuration bundle does not change provenance. Because composition and resolution metadata diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 0001666..fc6298a 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -150,7 +150,10 @@ Do not add a separate mutation test for every trivial validator. ## Stage 3 — Replace composition-dependent effective projection -Status: Pending +Status: Complete — Introduced one typed safe projection for effective YAML, +semantic values, source paths, and the versioned v3 digest; removed composition +parsing from those consumers and verified ordering, expansion, relocation, +nil/empty, exact-integer, CLI diff, and semantic-resume behavior. Read: `internal/config/effective_pipeline.go`, `effective_digest.go`, `pipeline_composition.go`, `internal/app/config_commands.go`, the config-diff diff --git a/internal/config/config.go b/internal/config/config.go index d24cb3d..7e795e0 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -17,25 +17,25 @@ type Config struct { // PipelineConfig contains durable pipeline-level settings. type PipelineConfig struct { - Workspace WorkspaceConfig `yaml:"workspace"` - Campaigns CampaignsConfig `yaml:"campaigns"` - Storage StorageConfig `yaml:"storage"` - Spool SpoolConfig `yaml:"spool"` - Cache CacheConfig `yaml:"cache"` - Publish *PublishConfig `yaml:"publish"` - Secrets *SecretsConfig `yaml:"secrets"` - WhisperX WhisperXConfig `yaml:"whisperx"` - Seriatim SeriatimConfig `yaml:"seriatim"` - Audita AuditaConfig `yaml:"audita"` - Normalize *NormalizeConfig `yaml:"normalize"` - Trim *TrimConfig `yaml:"trim"` - Render *RenderConfig `yaml:"render"` - Scriptorium *ScriptoriumConfig `yaml:"scriptorium"` - Notarius *NotariusConfig `yaml:"notarius"` - Notification NotificationConfig `yaml:"notification"` + Workspace WorkspaceConfig `yaml:"workspace" json:"workspace"` + Campaigns CampaignsConfig `yaml:"campaigns" json:"campaigns"` + Storage StorageConfig `yaml:"storage" json:"storage"` + Spool SpoolConfig `yaml:"spool" json:"spool"` + Cache CacheConfig `yaml:"cache" json:"cache"` + Publish *PublishConfig `yaml:"publish" json:"publish"` + Secrets *SecretsConfig `yaml:"secrets" json:"secrets"` + WhisperX WhisperXConfig `yaml:"whisperx" json:"whisperx"` + Seriatim SeriatimConfig `yaml:"seriatim" json:"seriatim"` + Audita AuditaConfig `yaml:"audita" json:"audita"` + Normalize *NormalizeConfig `yaml:"normalize" json:"normalize"` + Trim *TrimConfig `yaml:"trim" json:"trim"` + Render *RenderConfig `yaml:"render" json:"render"` + Scriptorium *ScriptoriumConfig `yaml:"scriptorium" json:"scriptorium"` + Notarius *NotariusConfig `yaml:"notarius" json:"notarius"` + Notification NotificationConfig `yaml:"notification" json:"notification"` - resolution *pipelineResolutionMetadata `yaml:"-"` - familyCatalog ArtifactFamilyCatalog `yaml:"-"` + resolution *pipelineResolutionMetadata `yaml:"-" json:"-"` + familyCatalog ArtifactFamilyCatalog `yaml:"-" json:"-"` } // ArtifactFamilies returns a copy of runtime-only expansion provenance. @@ -48,51 +48,51 @@ func ArtifactFamilies(cfg *PipelineConfig) ArtifactFamilyCatalog { // CampaignsConfig configures the local campaign registry. type CampaignsConfig struct { - Root string `yaml:"root"` - DefaultCampaignID string `yaml:"default_campaign_id"` + Root string `yaml:"root" json:"root"` + DefaultCampaignID string `yaml:"default_campaign_id" json:"default_campaign_id"` } // CampaignConfig contains stable campaign-level identity and input defaults. type CampaignConfig struct { - CampaignID string `yaml:"campaign_id"` - SessionTemplateFile string `yaml:"session_template_file"` - Inputs CampaignInputsConfig `yaml:"inputs"` + CampaignID string `yaml:"campaign_id" json:"campaign_id"` + SessionTemplateFile string `yaml:"session_template_file" json:"session_template_file"` + Inputs CampaignInputsConfig `yaml:"inputs" json:"inputs"` } // CampaignInputsConfig contains stable campaign-level input file references. type CampaignInputsConfig struct { - SpeakersFile string `yaml:"speakers_file"` - AutocorrectFile string `yaml:"autocorrect_file"` - GlossaryFile string `yaml:"glossary_file"` - PartyFile string `yaml:"party_file"` - SpellCatalogFile string `yaml:"spell_catalog_file"` + SpeakersFile string `yaml:"speakers_file" json:"speakers_file"` + AutocorrectFile string `yaml:"autocorrect_file" json:"autocorrect_file"` + GlossaryFile string `yaml:"glossary_file" json:"glossary_file"` + PartyFile string `yaml:"party_file" json:"party_file"` + SpellCatalogFile string `yaml:"spell_catalog_file" json:"spell_catalog_file"` } // SessionConfig contains per-session inputs and metadata. type SessionConfig struct { - SessionID string `yaml:"session_id"` - PreviousSessionID string `yaml:"previous_session_id"` - Campaign string `yaml:"campaign"` - Date string `yaml:"date"` - Title string `yaml:"title"` - Inputs SessionInputsConfig `yaml:"inputs"` + SessionID string `yaml:"session_id" json:"session_id"` + PreviousSessionID string `yaml:"previous_session_id" json:"previous_session_id"` + Campaign string `yaml:"campaign" json:"campaign"` + Date string `yaml:"date" json:"date"` + Title string `yaml:"title" json:"title"` + Inputs SessionInputsConfig `yaml:"inputs" json:"inputs"` } // WorkspaceConfig configures local workspace behavior. type WorkspaceConfig struct { - Root string `yaml:"root"` - CleanupAfterPublish bool `yaml:"cleanup_after_publish"` + Root string `yaml:"root" json:"root"` + CleanupAfterPublish bool `yaml:"cleanup_after_publish" json:"cleanup_after_publish"` } // SecretsConfig configures optional local filesystem secret loading. type SecretsConfig struct { - EnvDir string `yaml:"env_dir"` + EnvDir string `yaml:"env_dir" json:"env_dir"` } // StorageConfig configures storage backends and related parameters. type StorageConfig struct { - Backend string `yaml:"backend"` - S3 *StorageS3Config `yaml:"s3"` + Backend string `yaml:"backend" json:"backend"` + S3 *StorageS3Config `yaml:"s3" json:"s3"` } const ( @@ -104,178 +104,178 @@ const ( // StorageS3Config configures S3 storage coordinates. type StorageS3Config struct { - Bucket string `yaml:"bucket"` - RootPrefix string `yaml:"root_prefix"` - Region string `yaml:"region"` - Endpoint string `yaml:"endpoint"` - ForcePathStyle bool `yaml:"force_path_style"` - AccessKeyIDEnv string `yaml:"access_key_id_env"` - SecretKeyEnv string `yaml:"secret_access_key_env"` + Bucket string `yaml:"bucket" json:"bucket"` + RootPrefix string `yaml:"root_prefix" json:"root_prefix"` + Region string `yaml:"region" json:"region"` + Endpoint string `yaml:"endpoint" json:"endpoint"` + ForcePathStyle bool `yaml:"force_path_style" json:"force_path_style"` + AccessKeyIDEnv string `yaml:"access_key_id_env" json:"access_key_id_env"` + SecretKeyEnv string `yaml:"secret_access_key_env" json:"secret_access_key_env"` } // SpoolConfig configures local spool storage for staged data. type SpoolConfig struct { - Root string `yaml:"root"` - DeleteAudioAfterPublish bool `yaml:"delete_audio_after_publish"` + Root string `yaml:"root" json:"root"` + DeleteAudioAfterPublish bool `yaml:"delete_audio_after_publish" json:"delete_audio_after_publish"` } // CacheConfig configures durable local caches for reusable remote inputs. type CacheConfig struct { - Root string `yaml:"root"` - S3Audio *bool `yaml:"s3_audio"` + Root string `yaml:"root" json:"root"` + S3Audio *bool `yaml:"s3_audio" json:"s3_audio"` } // PublishConfig configures publish behavior and source-based output uploads. type PublishConfig struct { - Enabled *bool `yaml:"enabled"` - UploadRun *bool `yaml:"upload_run"` - Outputs []PublishOutputRule `yaml:"outputs"` + Enabled *bool `yaml:"enabled" json:"enabled"` + UploadRun *bool `yaml:"upload_run" json:"upload_run"` + Outputs []PublishOutputRule `yaml:"outputs" json:"outputs"` } // PublishOutputRule configures one source-to-destination publish mapping. type PublishOutputRule struct { - Source string `yaml:"source"` - Dest string `yaml:"dest"` - Required *bool `yaml:"required"` + Source string `yaml:"source" json:"source"` + Dest string `yaml:"dest" json:"dest"` + Required *bool `yaml:"required" json:"required"` } // WhisperXConfig configures WhisperX adapter settings. type WhisperXConfig struct { - TranscribeURL string `yaml:"transcribe_url"` - Language string `yaml:"language"` - Timeout string `yaml:"timeout"` - Retries *int `yaml:"retries"` - RetryDelay string `yaml:"retry_delay"` - Concurrency *int `yaml:"concurrency"` + TranscribeURL string `yaml:"transcribe_url" json:"transcribe_url"` + Language string `yaml:"language" json:"language"` + Timeout string `yaml:"timeout" json:"timeout"` + Retries *int `yaml:"retries" json:"retries"` + RetryDelay string `yaml:"retry_delay" json:"retry_delay"` + Concurrency *int `yaml:"concurrency" json:"concurrency"` } // SeriatimConfig configures seriatim adapter settings. type SeriatimConfig struct { - Binary string `yaml:"binary"` - Timeout string `yaml:"timeout"` - OutputSchema string `yaml:"output_schema"` - CoalesceGap *float64 `yaml:"coalesce_gap"` - Report *bool `yaml:"report"` - Env SeriatimEnvConfig `yaml:"env"` + Binary string `yaml:"binary" json:"binary"` + Timeout string `yaml:"timeout" json:"timeout"` + OutputSchema string `yaml:"output_schema" json:"output_schema"` + CoalesceGap *float64 `yaml:"coalesce_gap" json:"coalesce_gap"` + Report *bool `yaml:"report" json:"report"` + Env SeriatimEnvConfig `yaml:"env" json:"env"` } // SeriatimEnvConfig configures optional advanced seriatim environment tuning. type SeriatimEnvConfig struct { - OverlapWordRunGap *float64 `yaml:"overlap_word_run_gap"` - OverlapWordRunReorderWindow *float64 `yaml:"overlap_word_run_reorder_window"` - BackchannelMaxDuration *float64 `yaml:"backchannel_max_duration"` - FillerMaxDuration *float64 `yaml:"filler_max_duration"` + OverlapWordRunGap *float64 `yaml:"overlap_word_run_gap" json:"overlap_word_run_gap"` + OverlapWordRunReorderWindow *float64 `yaml:"overlap_word_run_reorder_window" json:"overlap_word_run_reorder_window"` + BackchannelMaxDuration *float64 `yaml:"backchannel_max_duration" json:"backchannel_max_duration"` + FillerMaxDuration *float64 `yaml:"filler_max_duration" json:"filler_max_duration"` } // AuditaConfig configures audita adapter settings. type AuditaConfig struct { - Binary string `yaml:"binary"` - Timeout string `yaml:"timeout"` - LLMAPIKeyEnv string `yaml:"llm_api_key_env"` - Modules []string `yaml:"modules"` - BaseURL string `yaml:"base_url"` - Model string `yaml:"model"` - TotalLLMConcurrency *int `yaml:"total_llm_concurrency"` - ProposalLLMConcurrency *int `yaml:"proposal_llm_concurrency"` - ValidationModel string `yaml:"validation_model"` - ValidationLLMConcurrency *int `yaml:"validation_llm_concurrency"` - TranscriptDescription string `yaml:"transcript_description"` - ConfigPath string `yaml:"config_path"` - OutputSchema string `yaml:"output_schema"` - WorkDirRetention string `yaml:"work_dir_retention"` - Report *bool `yaml:"report"` + Binary string `yaml:"binary" json:"binary"` + Timeout string `yaml:"timeout" json:"timeout"` + LLMAPIKeyEnv string `yaml:"llm_api_key_env" json:"llm_api_key_env"` + Modules []string `yaml:"modules" json:"modules"` + BaseURL string `yaml:"base_url" json:"base_url"` + Model string `yaml:"model" json:"model"` + TotalLLMConcurrency *int `yaml:"total_llm_concurrency" json:"total_llm_concurrency"` + ProposalLLMConcurrency *int `yaml:"proposal_llm_concurrency" json:"proposal_llm_concurrency"` + ValidationModel string `yaml:"validation_model" json:"validation_model"` + ValidationLLMConcurrency *int `yaml:"validation_llm_concurrency" json:"validation_llm_concurrency"` + TranscriptDescription string `yaml:"transcript_description" json:"transcript_description"` + ConfigPath string `yaml:"config_path" json:"config_path"` + OutputSchema string `yaml:"output_schema" json:"output_schema"` + WorkDirRetention string `yaml:"work_dir_retention" json:"work_dir_retention"` + Report *bool `yaml:"report" json:"report"` } // NormalizeConfig configures normalize-stage transcript schema/output behavior. type NormalizeConfig struct { - OutputPath string `yaml:"output_path"` - OutputSchema string `yaml:"output_schema"` - Report *bool `yaml:"report"` + OutputPath string `yaml:"output_path" json:"output_path"` + OutputSchema string `yaml:"output_schema" json:"output_schema"` + Report *bool `yaml:"report" json:"report"` - outputPathSet bool `yaml:"-"` + outputPathSet bool `yaml:"-" json:"-"` } // TrimConfig configures trim-stage transcript boundary behavior. type TrimConfig struct { - Enabled *bool `yaml:"enabled"` - OutputPath string `yaml:"output_path"` - Bounds TrimBoundsConfig `yaml:"bounds"` - Seriatim TrimSeriatimConfig `yaml:"seriatim"` + Enabled *bool `yaml:"enabled" json:"enabled"` + OutputPath string `yaml:"output_path" json:"output_path"` + Bounds TrimBoundsConfig `yaml:"bounds" json:"bounds"` + Seriatim TrimSeriatimConfig `yaml:"seriatim" json:"seriatim"` } // TrimBoundsConfig configures Scriptorium bounds prompt execution for trim. type TrimBoundsConfig struct { - PromptID string `yaml:"prompt_id"` - ProfileID string `yaml:"profile_id"` - TranscriptInputName string `yaml:"transcript_input_name"` - OutputPath string `yaml:"output_path"` - Timeout string `yaml:"timeout"` - RenderDebug bool `yaml:"render_debug"` - RenderOutputPath string `yaml:"render_output_path"` + PromptID string `yaml:"prompt_id" json:"prompt_id"` + ProfileID string `yaml:"profile_id" json:"profile_id"` + TranscriptInputName string `yaml:"transcript_input_name" json:"transcript_input_name"` + OutputPath string `yaml:"output_path" json:"output_path"` + Timeout string `yaml:"timeout" json:"timeout"` + RenderDebug bool `yaml:"render_debug" json:"render_debug"` + RenderOutputPath string `yaml:"render_output_path" json:"render_output_path"` } // TrimSeriatimConfig configures Seriatim options for trim output construction. type TrimSeriatimConfig struct { - Report *bool `yaml:"report"` + Report *bool `yaml:"report" json:"report"` } // RenderConfig configures render-stage output formatting behavior. type RenderConfig struct { - Enabled *bool `yaml:"enabled"` - Format string `yaml:"format"` - Title string `yaml:"title"` - IncludeTimestamps *bool `yaml:"include_timestamps"` - IncludeSegmentIDs *bool `yaml:"include_segment_ids"` - IncludeMetadata bool `yaml:"include_metadata"` + Enabled *bool `yaml:"enabled" json:"enabled"` + Format string `yaml:"format" json:"format"` + Title string `yaml:"title" json:"title"` + IncludeTimestamps *bool `yaml:"include_timestamps" json:"include_timestamps"` + IncludeSegmentIDs *bool `yaml:"include_segment_ids" json:"include_segment_ids"` + IncludeMetadata bool `yaml:"include_metadata" json:"include_metadata"` } // ScriptoriumConfig configures Scriptorium-backed artifact generation. type ScriptoriumConfig struct { - Binary string `yaml:"binary"` - ConfigPath string `yaml:"config_path"` - Timeout string `yaml:"timeout"` - RenderDebug bool `yaml:"render_debug"` - Artifacts map[string]ScriptoriumArtifactConfig `yaml:"artifacts"` - ArtifactFamilies map[string]ScriptoriumArtifactFamilyConfig `yaml:"artifact_families"` + Binary string `yaml:"binary" json:"binary"` + ConfigPath string `yaml:"config_path" json:"config_path"` + Timeout string `yaml:"timeout" json:"timeout"` + RenderDebug bool `yaml:"render_debug" json:"render_debug"` + Artifacts map[string]ScriptoriumArtifactConfig `yaml:"artifacts" json:"artifacts"` + ArtifactFamilies map[string]ScriptoriumArtifactFamilyConfig `yaml:"artifact_families" json:"artifact_families"` } // ScriptoriumArtifactConfig configures one named output artifact workflow. type ScriptoriumArtifactConfig struct { - Enabled bool `yaml:"enabled"` - DependsOn []string `yaml:"depends_on"` - RenderDebug *bool `yaml:"render_debug"` - PromptID string `yaml:"prompt_id"` - ProfileID string `yaml:"profile_id"` - OutputPath string `yaml:"output_path"` - Timeout string `yaml:"timeout"` - Inputs map[string]ScriptoriumInputConfig `yaml:"inputs"` - Vars map[string]any `yaml:"vars"` + Enabled bool `yaml:"enabled" json:"enabled"` + DependsOn []string `yaml:"depends_on" json:"depends_on"` + RenderDebug *bool `yaml:"render_debug" json:"render_debug"` + PromptID string `yaml:"prompt_id" json:"prompt_id"` + ProfileID string `yaml:"profile_id" json:"profile_id"` + OutputPath string `yaml:"output_path" json:"output_path"` + Timeout string `yaml:"timeout" json:"timeout"` + Inputs map[string]ScriptoriumInputConfig `yaml:"inputs" json:"inputs"` + Vars map[string]any `yaml:"vars" json:"vars"` } // ScriptoriumArtifactFamilyConfig declares the shared configuration expanded // into one ordinary artifact for each canonical party character. type ScriptoriumArtifactFamilyConfig struct { - Enabled bool `yaml:"enabled"` - ForEach string `yaml:"for_each"` - DependsOn []string `yaml:"depends_on"` - RenderDebug *bool `yaml:"render_debug"` - PromptID string `yaml:"prompt_id"` - ProfileID string `yaml:"profile_id"` - OutputPathPattern string `yaml:"output_path_pattern"` - Timeout string `yaml:"timeout"` - Inputs map[string]ScriptoriumInputConfig `yaml:"inputs"` - Vars map[string]any `yaml:"vars"` - MemberVars map[string]string `yaml:"member_vars"` - MemberDependencies []string `yaml:"member_dependencies"` - Publish *ScriptoriumArtifactFamilyPublishConfig `yaml:"publish"` + Enabled bool `yaml:"enabled" json:"enabled"` + ForEach string `yaml:"for_each" json:"for_each"` + DependsOn []string `yaml:"depends_on" json:"depends_on"` + RenderDebug *bool `yaml:"render_debug" json:"render_debug"` + PromptID string `yaml:"prompt_id" json:"prompt_id"` + ProfileID string `yaml:"profile_id" json:"profile_id"` + OutputPathPattern string `yaml:"output_path_pattern" json:"output_path_pattern"` + Timeout string `yaml:"timeout" json:"timeout"` + Inputs map[string]ScriptoriumInputConfig `yaml:"inputs" json:"inputs"` + Vars map[string]any `yaml:"vars" json:"vars"` + MemberVars map[string]string `yaml:"member_vars" json:"member_vars"` + MemberDependencies []string `yaml:"member_dependencies" json:"member_dependencies"` + Publish *ScriptoriumArtifactFamilyPublishConfig `yaml:"publish" json:"publish"` } // ScriptoriumArtifactFamilyPublishConfig retains the typed declaration for // family publishing. It is resolved by the publish owner. type ScriptoriumArtifactFamilyPublishConfig struct { - Enabled bool `yaml:"enabled"` - Required bool `yaml:"required"` - DestPattern string `yaml:"dest_pattern"` + Enabled bool `yaml:"enabled" json:"enabled"` + Required bool `yaml:"required" json:"required"` + DestPattern string `yaml:"dest_pattern" json:"dest_pattern"` } // ArtifactFamilyCatalog records the runtime-only origin of expanded artifacts. @@ -305,50 +305,50 @@ type ArtifactFamilyMemberOrigin struct { // ScriptoriumInputConfig configures one named prompt input source. type ScriptoriumInputConfig struct { - Source string `yaml:"source"` - Required bool `yaml:"required"` + Source string `yaml:"source" json:"source"` + Required bool `yaml:"required" json:"required"` } // NotariusConfig configures structured artifact extraction by Notarius. type NotariusConfig struct { - Enabled bool `yaml:"enabled"` - Binary string `yaml:"binary"` - ConfigPath string `yaml:"config_path"` - PipelineID string `yaml:"pipeline_id"` - Timeout string `yaml:"timeout"` - WorkingDirectory string `yaml:"working_directory"` - References map[string]string `yaml:"references"` - Outputs map[string]NotariusOutputConfig `yaml:"outputs"` + Enabled bool `yaml:"enabled" json:"enabled"` + Binary string `yaml:"binary" json:"binary"` + ConfigPath string `yaml:"config_path" json:"config_path"` + PipelineID string `yaml:"pipeline_id" json:"pipeline_id"` + Timeout string `yaml:"timeout" json:"timeout"` + WorkingDirectory string `yaml:"working_directory" json:"working_directory"` + References map[string]string `yaml:"references" json:"references"` + Outputs map[string]NotariusOutputConfig `yaml:"outputs" json:"outputs"` } // NotariusOutputConfig declares one required extraction lane contract. type NotariusOutputConfig struct { - LaneID string `yaml:"lane_id"` - MediaType string `yaml:"media_type"` - SchemaID string `yaml:"schema_id"` - SchemaVersion string `yaml:"schema_version"` - ModuleKey string `yaml:"module_key"` + LaneID string `yaml:"lane_id" json:"lane_id"` + MediaType string `yaml:"media_type" json:"media_type"` + SchemaID string `yaml:"schema_id" json:"schema_id"` + SchemaVersion string `yaml:"schema_version" json:"schema_version"` + ModuleKey string `yaml:"module_key" json:"module_key"` } // NotificationConfig configures the supported notification behavior. type NotificationConfig struct { - Mode string `yaml:"mode"` + Mode string `yaml:"mode" json:"mode"` } // SessionInputsConfig contains per-session input references. type SessionInputsConfig struct { - AudioDir string `yaml:"audio_dir"` - AudioFiles []string `yaml:"audio_files"` - AudioS3 *SessionAudioS3Input `yaml:"audio_s3"` - SpeakersFile string `yaml:"speakers_file"` - AutocorrectFile string `yaml:"autocorrect_file"` - GlossaryFile string `yaml:"glossary_file"` - SpellCatalogFile string `yaml:"spell_catalog_file"` + AudioDir string `yaml:"audio_dir" json:"audio_dir"` + AudioFiles []string `yaml:"audio_files" json:"audio_files"` + AudioS3 *SessionAudioS3Input `yaml:"audio_s3" json:"audio_s3"` + SpeakersFile string `yaml:"speakers_file" json:"speakers_file"` + AutocorrectFile string `yaml:"autocorrect_file" json:"autocorrect_file"` + GlossaryFile string `yaml:"glossary_file" json:"glossary_file"` + SpellCatalogFile string `yaml:"spell_catalog_file" json:"spell_catalog_file"` } // SessionAudioS3Input configures S3 session-audio input discovery. type SessionAudioS3Input struct { - Prefix string `yaml:"prefix"` + Prefix string `yaml:"prefix" json:"prefix"` } // ResolvedStableInputs records where stable input file paths came from after diff --git a/internal/config/effective_digest.go b/internal/config/effective_digest.go index 9e9fde2..ca6f7ed 100644 --- a/internal/config/effective_digest.go +++ b/internal/config/effective_digest.go @@ -5,14 +5,13 @@ import ( "encoding/hex" "fmt" "path/filepath" + "sort" "strings" - - "gopkg.in/yaml.v3" ) const ( pipelineDefaultOwnershipSource = "default" - effectivePipelineDigestVersion = "narratio-effective-pipeline-v2\n" + effectivePipelineDigestVersion = "narratio-effective-pipeline-v3\n" ) // PipelineProfileProvenance identifies the profile selected while resolving a @@ -49,15 +48,7 @@ func finalizePipelineResolution(cfg *PipelineConfig) error { for _, ownership := range cfg.resolution.ownership { declared[ownership.path] = append([]string(nil), ownership.sources...) } - data, err := yaml.Marshal(cfg) - if err != nil { - return fmt.Errorf("serialize normalized effective pipeline: %w", err) - } - document, err := parseCompositionBytes("normalized effective pipeline", data) - if err != nil { - return err - } - records, err := document.semanticRecords() + records, err := projectionValueRecords(projectEffectivePipeline(cfg, true)) if err != nil { return err } @@ -71,6 +62,12 @@ func finalizePipelineResolution(cfg *PipelineConfig) error { path: record.Path, sources: append([]string(nil), sources...), }) } + for path, sources := range declared { + if strings.HasPrefix(path, "scriptorium.artifact_families.") { + ownership = append(ownership, pipelineFieldOwnership{path: path, sources: append([]string(nil), sources...)}) + } + } + sort.Slice(ownership, func(i, j int) bool { return ownership[i].path < ownership[j].path }) cfg.resolution.ownership = ownership return recomputePipelineEffectiveDigest(cfg) } @@ -82,22 +79,7 @@ func recomputePipelineEffectiveDigest(cfg *PipelineConfig) error { if cfg == nil || cfg.resolution == nil { return fmt.Errorf("pipeline resolution metadata is required") } - digestConfig := *cfg - if cfg.Notarius != nil && cfg.resolution.logicalNotariusCaptured { - notarius := *cfg.Notarius - notarius.ConfigPath = cfg.resolution.logicalNotariusConfig - notarius.WorkingDirectory = cfg.resolution.logicalNotariusWorking - digestConfig.Notarius = ¬arius - } - data, err := yaml.Marshal(&digestConfig) - if err != nil { - return fmt.Errorf("serialize normalized effective pipeline: %w", err) - } - document, err := parseCompositionBytes("normalized effective pipeline", data) - if err != nil { - return err - } - canonical, err := document.canonicalDigestInput() + canonical, err := projectionJSON(projectEffectivePipeline(cfg, true)) if err != nil { return err } diff --git a/internal/config/effective_pipeline.go b/internal/config/effective_pipeline.go index a14c4c0..1f42f26 100644 --- a/internal/config/effective_pipeline.go +++ b/internal/config/effective_pipeline.go @@ -60,23 +60,7 @@ func EffectivePipelineValues(cfg *PipelineConfig) ([]EffectivePipelineValueRecor if cfg == nil { return nil, fmt.Errorf("pipeline config is required") } - data, err := MarshalEffectivePipeline(cfg) - if err != nil { - return nil, err - } - document, err := parseCompositionBytes("effective pipeline", data) - if err != nil { - return nil, err - } - records, err := document.compactSemanticRecords() - if err != nil { - return nil, err - } - values := make([]EffectivePipelineValueRecord, 0, len(records)) - for _, record := range records { - values = append(values, EffectivePipelineValueRecord{Path: record.Path, Value: record.Value}) - } - return values, nil + return projectionValueRecords(projectEffectivePipeline(cfg, true)) } // EffectivePipelineSources projects pipeline ownership after defaults and @@ -86,15 +70,14 @@ func EffectivePipelineSources(cfg *PipelineConfig, party ResolvedParty) ([]Effec if cfg == nil { return nil, fmt.Errorf("pipeline config is required") } - data, err := MarshalEffectivePipeline(cfg) + values, err := EffectivePipelineValues(cfg) if err != nil { return nil, err } - document, err := parseCompositionBytes("effective pipeline", data) - if err != nil { - return nil, err + paths := make([]string, 0, len(values)) + for _, value := range values { + paths = append(paths, value.Path) } - paths := effectivePipelineFieldPaths(document.root) owners := effectivePipelineOwners(cfg) families := ArtifactFamilies(cfg) publishCount := 0 @@ -168,54 +151,11 @@ func MarshalEffectivePipeline(cfg *PipelineConfig) ([]byte, error) { return nil, fmt.Errorf("pipeline config is required") } - data, err := yaml.Marshal(cfg) + data, err := yaml.Marshal(projectEffectivePipeline(cfg, true)) if err != nil { return nil, fmt.Errorf("serialize effective pipeline: %w", err) } - document, err := parseCompositionBytes("effective pipeline", data) - if err != nil { - return nil, err - } - removeResolutionOnlyPipelineFields(document) - return document.canonicalYAML() -} - -func removeResolutionOnlyPipelineFields(document *compositionDocument) { - if document == nil || document.root == nil { - return - } - scriptoriumIndex := compositionFieldIndex(document.root.fields, "scriptorium") - if scriptoriumIndex < 0 { - return - } - scriptorium := document.root.fields[scriptoriumIndex].value - if scriptorium == nil || scriptorium.kind != yaml.MappingNode { - return - } - familyIndex := compositionFieldIndex(scriptorium.fields, "artifact_families") - if familyIndex < 0 { - return - } - scriptorium.fields = append(scriptorium.fields[:familyIndex], scriptorium.fields[familyIndex+1:]...) -} - -func effectivePipelineFieldPaths(node *compositionNode) []string { - var paths []string - var visit func(*compositionNode) - visit = func(current *compositionNode) { - if current == nil { - return - } - if current.kind == yaml.MappingNode && len(current.fields) > 0 { - for _, field := range current.fields { - visit(field.value) - } - return - } - paths = append(paths, current.path) - } - visit(node) - return paths + return data, nil } func effectivePipelineOwners(cfg *PipelineConfig) map[string][]string { diff --git a/internal/config/effective_projection.go b/internal/config/effective_projection.go new file mode 100644 index 0000000..ea6f756 --- /dev/null +++ b/internal/config/effective_projection.go @@ -0,0 +1,107 @@ +package config + +import ( + "bytes" + "encoding/json" + "fmt" + "sort" +) + +// effectivePipelineProjection is the single safe, schema-specific view used +// for inspection, comparison, and identity. Runtime metadata and unresolved +// family declarations have no slot in this type. +type effectivePipelineProjection struct { + Workspace WorkspaceConfig `yaml:"workspace" json:"workspace"` + Campaigns CampaignsConfig `yaml:"campaigns" json:"campaigns"` + Storage StorageConfig `yaml:"storage" json:"storage"` + Spool SpoolConfig `yaml:"spool" json:"spool"` + Cache CacheConfig `yaml:"cache" json:"cache"` + Publish *PublishConfig `yaml:"publish" json:"publish"` + Secrets *SecretsConfig `yaml:"secrets" json:"secrets"` + WhisperX WhisperXConfig `yaml:"whisperx" json:"whisperx"` + Seriatim SeriatimConfig `yaml:"seriatim" json:"seriatim"` + Audita AuditaConfig `yaml:"audita" json:"audita"` + Normalize *NormalizeConfig `yaml:"normalize" json:"normalize"` + Trim *TrimConfig `yaml:"trim" json:"trim"` + Render *RenderConfig `yaml:"render" json:"render"` + Scriptorium *effectiveScriptoriumConfig `yaml:"scriptorium" json:"scriptorium"` + Notarius *NotariusConfig `yaml:"notarius" json:"notarius"` + Notification NotificationConfig `yaml:"notification" json:"notification"` +} + +type effectiveScriptoriumConfig struct { + Binary string `yaml:"binary" json:"binary"` + ConfigPath string `yaml:"config_path" json:"config_path"` + Timeout string `yaml:"timeout" json:"timeout"` + RenderDebug bool `yaml:"render_debug" json:"render_debug"` + Artifacts map[string]ScriptoriumArtifactConfig `yaml:"artifacts" json:"artifacts"` +} + +func projectEffectivePipeline(cfg *PipelineConfig, logicalNotarius bool) effectivePipelineProjection { + projection := effectivePipelineProjection{ + Workspace: cfg.Workspace, Campaigns: cfg.Campaigns, Storage: cfg.Storage, + Spool: cfg.Spool, Cache: cfg.Cache, Publish: cfg.Publish, Secrets: cfg.Secrets, + WhisperX: cfg.WhisperX, Seriatim: cfg.Seriatim, Audita: cfg.Audita, + Normalize: cfg.Normalize, Trim: cfg.Trim, Render: cfg.Render, + Notarius: cfg.Notarius, Notification: cfg.Notification, + } + if cfg.Scriptorium != nil { + projection.Scriptorium = &effectiveScriptoriumConfig{ + Binary: cfg.Scriptorium.Binary, ConfigPath: cfg.Scriptorium.ConfigPath, + Timeout: cfg.Scriptorium.Timeout, RenderDebug: cfg.Scriptorium.RenderDebug, + Artifacts: cfg.Scriptorium.Artifacts, + } + } + if logicalNotarius && cfg.Notarius != nil && cfg.resolution != nil && cfg.resolution.logicalNotariusCaptured { + notarius := *cfg.Notarius + notarius.ConfigPath = cfg.resolution.logicalNotariusConfig + notarius.WorkingDirectory = cfg.resolution.logicalNotariusWorking + projection.Notarius = ¬arius + } + return projection +} + +func projectionJSON(projection effectivePipelineProjection) ([]byte, error) { + data, err := json.Marshal(projection) + if err != nil { + return nil, fmt.Errorf("encode effective pipeline projection: %w", err) + } + return data, nil +} + +func projectionValueRecords(projection effectivePipelineProjection) ([]EffectivePipelineValueRecord, error) { + data, err := projectionJSON(projection) + if err != nil { + return nil, err + } + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.UseNumber() + var root map[string]any + if err := decoder.Decode(&root); err != nil { + return nil, fmt.Errorf("decode effective pipeline projection: %w", err) + } + var records []EffectivePipelineValueRecord + appendProjectionRecords(&records, "", root) + sort.Slice(records, func(i, j int) bool { return records[i].Path < records[j].Path }) + return records, nil +} + +func appendProjectionRecords(records *[]EffectivePipelineValueRecord, path string, value any) { + if mapping, ok := value.(map[string]any); ok && len(mapping) > 0 { + keys := make([]string, 0, len(mapping)) + for key := range mapping { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + childPath := key + if path != "" { + childPath = path + "." + key + } + appendProjectionRecords(records, childPath, mapping[key]) + } + return + } + encoded, _ := json.Marshal(value) + *records = append(*records, EffectivePipelineValueRecord{Path: path, Value: string(encoded)}) +} diff --git a/internal/config/load_validate_test.go b/internal/config/load_validate_test.go index 5edc274..35b319c 100644 --- a/internal/config/load_validate_test.go +++ b/internal/config/load_validate_test.go @@ -83,6 +83,42 @@ whisperx: } } +func TestEffectivePipelineValuesPreserveNilEmptyAndExactIntegers(t *testing.T) { + base := &PipelineConfig{Audita: AuditaConfig{Modules: nil}} + nilValues, err := EffectivePipelineValues(base) + if err != nil { + t.Fatal(err) + } + base.Audita.Modules = []string{} + base.Scriptorium = &ScriptoriumConfig{Artifacts: map[string]ScriptoriumArtifactConfig{ + "recap": {Vars: map[string]any{"exact": int64(9007199254740993)}}, + }} + emptyValues, err := EffectivePipelineValues(base) + if err != nil { + t.Fatal(err) + } + if got := effectiveValueForTest(t, nilValues, "audita.modules"); got != "null" { + t.Fatalf("nil sequence value = %q, want null", got) + } + if got := effectiveValueForTest(t, emptyValues, "audita.modules"); got != "[]" { + t.Fatalf("empty sequence value = %q, want []", got) + } + if got := effectiveValueForTest(t, emptyValues, "scriptorium.artifacts.recap.vars.exact"); got != "9007199254740993" { + t.Fatalf("exact integer value = %q", got) + } +} + +func effectiveValueForTest(t *testing.T, values []EffectivePipelineValueRecord, path string) string { + t.Helper() + for _, value := range values { + if value.Path == path { + return value.Value + } + } + t.Fatalf("effective value %q not found", path) + return "" +} + func TestValidateMissingAudioSource(t *testing.T) { cfg := loadedValidConfig(t) cfg.Session.Inputs.AudioDir = ""