diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index ec5c02e7..f2a10786 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -259,6 +259,8 @@ not combine it with source-origin or CLI work. ## Stage 4 — Convert Documents Into Runtime Configuration +**Status:** Completed 2026-09-13. + **Goal:** Produce the existing typed runtime composition from the new document. **Work:** diff --git a/internal/core/config/loaded_config.go b/internal/core/config/loaded_config.go new file mode 100644 index 00000000..7fd245c7 --- /dev/null +++ b/internal/core/config/loaded_config.go @@ -0,0 +1,321 @@ +package config + +import ( + "fmt" + "io" + + configkit "gitea.maximumdirect.net/eric/ConfigKit" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" +) + +// LoadedConfig keeps the finalized runtime configuration together with the +// private ConfigKit inspection snapshot that produced it. +type LoadedConfig struct { + Config Config + + result *configkit.Result[Document] +} + +// RuntimeConfig returns an independently owned runtime configuration. +func (loaded *LoadedConfig) RuntimeConfig() Config { + if loaded == nil { + return Config{} + } + return cloneConfig(loaded.Config) +} + +// WriteEffective renders ConfigKit's frozen, redacted effective document. +func (loaded *LoadedConfig) WriteEffective(output io.Writer) error { + if loaded == nil || loaded.result == nil { + return fmt.Errorf("loaded configuration is unavailable") + } + return loaded.result.WriteEffective(output) +} + +// WriteExplanation renders ConfigKit's safe explanation for a JSON-pointer +// path without exposing ConfigKit types at this package boundary. +func (loaded *LoadedConfig) WriteExplanation(output io.Writer, pointer string) error { + if loaded == nil || loaded.result == nil { + return fmt.Errorf("loaded configuration is unavailable") + } + path, err := configkit.ParsePath(pointer) + if err != nil { + return fmt.Errorf("parse configuration explanation path: %w", err) + } + return loaded.result.WriteExplanation(output, path) +} + +func loadConfig(path string, options LoadOptions) (*LoadedConfig, error) { + result, err := loadDocument(path, options) + if err != nil { + return nil, err + } + return newLoadedConfig(result) +} + +func loadConfigSource(source configkit.Source, options LoadOptions) (*LoadedConfig, error) { + result, err := loadDocumentSource(source, options) + if err != nil { + return nil, err + } + return newLoadedConfig(result) +} + +func newLoadedConfig(result *configkit.Result[Document]) (*LoadedConfig, error) { + if result == nil { + return nil, fmt.Errorf("configuration result is unavailable") + } + runtime, err := documentToConfig(result.Config) + if err != nil { + return nil, err + } + return &LoadedConfig{Config: runtime, result: result}, nil +} + +func documentToConfig(document Document) (Config, error) { + config := Default() + config.Pipelines = documentPipelinesToRuntime(document.Pipelines) + if document.Concurrency != nil { + config.Concurrency.StageWorkers = documentIntMap(document.Concurrency.StageWorkers) + if document.Concurrency.TotalLLM != nil { + config.Concurrency.TotalLLM = *document.Concurrency.TotalLLM + } + } + if workers, ok := config.Concurrency.StageWorkers["extract"]; ok { + config.Concurrency.extractWorkersConfigured = true + config.Concurrency.defaultedExtractWorkers = workers + } + if document.PromptKit != nil { + if document.PromptKit.ProfileDir != nil { + config.PromptKit.ProfileDir = *document.PromptKit.ProfileDir + } + if document.PromptKit.ProfileFile != nil { + config.PromptKit.ProfileFile = *document.PromptKit.ProfileFile + } + if document.PromptKit.LocalBackend != nil { + backend := PromptKitLocalBackendConfig{} + if document.PromptKit.LocalBackend.Endpoint != nil { + backend.Endpoint = *document.PromptKit.LocalBackend.Endpoint + } + if document.PromptKit.LocalBackend.ConcurrencyLimit != nil { + backend.ConcurrencyLimit = *document.PromptKit.LocalBackend.ConcurrencyLimit + } + config.PromptKit.LocalBackend = &backend + } + } + if document.Output != nil && document.Output.Directory != nil { + config.Output.Directory = *document.Output.Directory + } + if document.Cache != nil { + if document.Cache.ChunkPlans != nil { + if document.Cache.ChunkPlans.Directory != nil { + config.Cache.ChunkPlans.Directory = *document.Cache.ChunkPlans.Directory + } + if document.Cache.ChunkPlans.Mode != nil { + mode, err := pipeline.ParseChunkCacheMode(*document.Cache.ChunkPlans.Mode) + if err != nil { + return Config{}, fmt.Errorf("convert chunk-plan cache mode: %w", err) + } + config.Cache.ChunkPlans.Mode = mode + } + } + if document.Cache.Checkpoints != nil { + if document.Cache.Checkpoints.Enabled != nil { + config.Cache.Checkpoints.Enabled = *document.Cache.Checkpoints.Enabled + } + if document.Cache.Checkpoints.Directory != nil { + config.Cache.Checkpoints.Directory = *document.Cache.Checkpoints.Directory + } + } + } + if document.Debug != nil && document.Debug.Directory != nil { + config.Debug.Directory = *document.Debug.Directory + } + return config, nil +} + +func documentPipelinesToRuntime(documents map[string]PipelineDocument) map[string]pipeline.PipelineProfile { + if documents == nil { + return nil + } + profiles := make(map[string]pipeline.PipelineProfile, len(documents)) + for id, document := range documents { + profiles[id] = documentPipelineToRuntime(id, document) + } + return profiles +} + +func documentPipelineToRuntime(id string, document PipelineDocument) pipeline.PipelineProfile { + profile := pipeline.PipelineProfile{ + ID: id, + StructuredOutputRepairAttempts: cloneDocumentInt(document.StructuredOutputRepairAttempts), + ValidationPolicy: documentPolicyToRuntime(document.ValidationPolicy), + Input: documentBindingToRuntime(document.Input), + Artifacts: documentLanesToRuntime(document.Artifacts), + References: documentReferencesToRuntime(document.References), + } + if document.LLMProfile != nil { + profile.LLMProfile = *document.LLMProfile + } + if document.Chunk != nil { + profile.Chunk = documentBindingToRuntime(*document.Chunk) + } + if document.Output != nil { + profile.Output = documentBindingToRuntime(*document.Output) + } + if document.Steps != nil { + profile.Steps = make([]pipeline.PipelineStepProfile, len(document.Steps)) + for index, step := range document.Steps { + profile.Steps[index] = pipeline.PipelineStepProfile{ + ID: step.ID, + Artifacts: documentLanesToRuntime(step.Artifacts), + References: documentReferencesToRuntime(step.References), + } + } + } + return profile +} + +func documentLanesToRuntime(documents map[string]ArtifactLaneDocument) map[string]pipeline.ArtifactLaneProfile { + if documents == nil { + return nil + } + lanes := make(map[string]pipeline.ArtifactLaneProfile, len(documents)) + for id, document := range documents { + lane := pipeline.ArtifactLaneProfile{ + Extract: documentBindingToRuntime(document.Extract), + References: documentReferencesToRuntime(document.References), + } + if document.Merge != nil { + lane.Merge = documentBindingToRuntime(*document.Merge) + } + if document.Normalize != nil { + lane.Normalize = documentBindingToRuntime(*document.Normalize) + } + if document.Validators != nil { + lane.Validators = make([]pipeline.ModuleBinding, len(document.Validators)) + for index, validator := range document.Validators { + lane.Validators[index] = documentValidatorToRuntime(validator) + } + } + lanes[id] = lane + } + return lanes +} + +func documentBindingToRuntime(document BindingDocument) pipeline.ModuleBinding { + binding := pipeline.ModuleBinding{ + Module: document.Module, + StructuredOutputRepairAttempts: cloneDocumentInt(document.StructuredOutputRepairAttempts), + ValidationPolicy: documentPolicyToRuntime(document.ValidationPolicy), + Retries: document.Retries, + Options: documentOptions(document.Options), + References: documentReferencesToRuntime(document.References), + } + if document.LLMProfile != nil { + binding.LLMProfile = *document.LLMProfile + } + if document.Validators != nil { + binding.Validators.Set = true + binding.Validators.Validators = make([]pipeline.ModuleBinding, len(document.Validators)) + for index, validator := range document.Validators { + binding.Validators.Validators[index] = documentValidatorToRuntime(validator) + } + } + return binding +} + +func documentValidatorToRuntime(document ValidatorDocument) pipeline.ModuleBinding { + binding := pipeline.ModuleBinding{ + Module: document.Module, + StructuredOutputRepairAttempts: cloneDocumentInt(document.StructuredOutputRepairAttempts), + Retries: document.Retries, + Options: documentOptions(document.Options), + } + if document.LLMProfile != nil { + binding.LLMProfile = *document.LLMProfile + } + return binding +} + +func documentReferencesToRuntime(documents map[string]ReferenceDocument) map[string]pipeline.ReferenceSource { + if documents == nil { + return nil + } + references := make(map[string]pipeline.ReferenceSource, len(documents)) + for name, document := range documents { + if document.Artifact != nil { + references[name] = pipeline.GeneratedReference(document.Artifact.Step, document.Artifact.Lane) + continue + } + if document.Path != nil { + references[name] = pipeline.ExternalReference(*document.Path) + } + } + return references +} + +func documentPolicyToRuntime(document *ValidationPolicyDocument) *pipeline.ValidationPolicyOverride { + if document == nil { + return nil + } + policy := pipeline.ValidationPolicyOverride{} + if document.ProducerStructuralFailure != nil { + value := *document.ProducerStructuralFailure + policy.ProducerStructuralFailure = &value + } + if document.SemanticRejection != nil { + value := *document.SemanticRejection + policy.SemanticRejection = &value + } + if document.ValidatorFailure != nil { + value := *document.ValidatorFailure + policy.ValidatorFailure = &value + } + return &policy +} + +func documentIntMap(values map[string]int) map[string]int { + if values == nil { + return nil + } + result := make(map[string]int, len(values)) + for key, value := range values { + result[key] = value + } + return result +} + +func cloneDocumentInt(value *int) *int { + if value == nil { + return nil + } + result := *value + return &result +} + +func documentOptions(values map[string]any) map[string]any { + if values == nil { + return nil + } + result := make(map[string]any, len(values)) + for key, value := range values { + result[key] = cloneDocumentOption(value) + } + return result +} + +func cloneDocumentOption(value any) any { + switch typed := value.(type) { + case map[string]any: + return documentOptions(typed) + case []any: + result := make([]any, len(typed)) + for index, item := range typed { + result[index] = cloneDocumentOption(item) + } + return result + default: + return typed + } +} diff --git a/internal/core/config/loaded_config_test.go b/internal/core/config/loaded_config_test.go new file mode 100644 index 00000000..67b74d1c --- /dev/null +++ b/internal/core/config/loaded_config_test.go @@ -0,0 +1,102 @@ +package config + +import ( + "bytes" + "strings" + "testing" + + configkit "gitea.maximumdirect.net/eric/ConfigKit" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" +) + +func TestLoadedConfigConvertsDocumentsAndResolvesRuntimePipeline(t *testing.T) { + t.Parallel() + + loaded, err := loadConfigSource(configkit.Bytes("document.yml", []byte(`version: 5 +pipelines: + session: + llm_profile: local + structured_output_repair_attempts: 0 + input: + module: input + chunk: + module: generic + validators: [] + artifacts: + score: + extract: + module: extract + validators: + - module: configured-a + options: + nested: + level: 1 +`)), LoadOptions{}) + if err != nil { + t.Fatalf("loadConfigSource() error = %v", err) + } + profile := loaded.Config.Pipelines["session"] + if profile.LLMProfile != "local" || profile.StructuredOutputRepairAttempts == nil || *profile.StructuredOutputRepairAttempts != 0 { + t.Fatalf("pipeline profile = %#v", profile) + } + if !profile.Chunk.Validators.Set || profile.Chunk.Validators.Validators == nil || len(profile.Chunk.Validators.Validators) != 0 { + t.Fatalf("empty validator chain was not preserved: %#v", profile.Chunk.Validators) + } + validator := profile.Artifacts["score"].Extract.Validators.Validators[0] + if validator.Module != "configured-a" || validator.Options["nested"].(map[string]any)["level"] == nil { + t.Fatalf("validator conversion = %#v", validator) + } + + effective, err := loaded.Config.Resolve(ResolveInput{PipelineID: "session", Catalog: effectiveCatalog(t)}) + if err != nil { + t.Fatalf("Resolve() error = %v", err) + } + if effective.ResolvedPipeline.ID != "session" || effective.ResolvedPipeline.Digest == "" { + t.Fatalf("resolved pipeline = %#v", effective.ResolvedPipeline) + } +} + +func TestLoadedConfigOwnsRuntimeValuesAndKeepsFrozenInspection(t *testing.T) { + t.Parallel() + + loaded, err := loadConfigSource(configkit.Bytes("document.yml", []byte(`version: 5 +pipelines: + session: + input: {module: input} + artifacts: + score: + extract: + module: extract + options: + nested: {value: original} +`)), LoadOptions{}) + if err != nil { + t.Fatalf("loadConfigSource() error = %v", err) + } + + first := loaded.RuntimeConfig() + first.Pipelines["session"] = pipelineProfileWithChangedOption(first.Pipelines["session"]) + second := loaded.RuntimeConfig() + if got := second.Pipelines["session"].Artifacts["score"].Extract.Options["nested"].(map[string]any)["value"]; got != "original" { + t.Fatalf("runtime configuration aliases caller mutation: %#v", got) + } + + loaded.Config.Pipelines["session"] = pipelineProfileWithChangedOption(loaded.Config.Pipelines["session"]) + var effective bytes.Buffer + if err := loaded.WriteEffective(&effective); err != nil { + t.Fatalf("WriteEffective() error = %v", err) + } + if !strings.Contains(effective.String(), "original") || strings.Contains(effective.String(), "changed") { + t.Fatalf("effective document changed with runtime mutation: %q", effective.String()) + } +} + +func pipelineProfileWithChangedOption(profile pipeline.PipelineProfile) pipeline.PipelineProfile { + profile = clonePipelineProfile(profile) + nested := profile.Artifacts["score"].Extract.Options["nested"].(map[string]any) + nested["value"] = "changed" + lane := profile.Artifacts["score"] + lane.Extract.Options["nested"] = nested + profile.Artifacts["score"] = lane + return profile +}