Convert ConfigKit documents to runtime configuration

docs/roadmap/implementation.md

@@ -259,6 +259,8 @@ not combine it with source-origin or CLI work.
 
 ## Stage 4 — Convert Documents Into Runtime Configuration
 
+**Status:** Completed 2026-09-13.
+
 **Goal:** Produce the existing typed runtime composition from the new document.
 
 **Work:**

internal/core/config/loaded_config.go

@@ -0,0 +1,321 @@
+package config
+
+import (
+	"fmt"
+	"io"
+
+	configkit "gitea.maximumdirect.net/eric/ConfigKit"
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
+)
+
+// LoadedConfig keeps the finalized runtime configuration together with the
+// private ConfigKit inspection snapshot that produced it.
+type LoadedConfig struct {
+	Config Config
+
+	result *configkit.Result[Document]
+}
+
+// RuntimeConfig returns an independently owned runtime configuration.
+func (loaded *LoadedConfig) RuntimeConfig() Config {
+	if loaded == nil {
+		return Config{}
+	}
+	return cloneConfig(loaded.Config)
+}
+
+// WriteEffective renders ConfigKit's frozen, redacted effective document.
+func (loaded *LoadedConfig) WriteEffective(output io.Writer) error {
+	if loaded == nil || loaded.result == nil {
+		return fmt.Errorf("loaded configuration is unavailable")
+	}
+	return loaded.result.WriteEffective(output)
+}
+
+// WriteExplanation renders ConfigKit's safe explanation for a JSON-pointer
+// path without exposing ConfigKit types at this package boundary.
+func (loaded *LoadedConfig) WriteExplanation(output io.Writer, pointer string) error {
+	if loaded == nil || loaded.result == nil {
+		return fmt.Errorf("loaded configuration is unavailable")
+	}
+	path, err := configkit.ParsePath(pointer)
+	if err != nil {
+		return fmt.Errorf("parse configuration explanation path: %w", err)
+	}
+	return loaded.result.WriteExplanation(output, path)
+}
+
+func loadConfig(path string, options LoadOptions) (*LoadedConfig, error) {
+	result, err := loadDocument(path, options)
+	if err != nil {
+		return nil, err
+	}
+	return newLoadedConfig(result)
+}
+
+func loadConfigSource(source configkit.Source, options LoadOptions) (*LoadedConfig, error) {
+	result, err := loadDocumentSource(source, options)
+	if err != nil {
+		return nil, err
+	}
+	return newLoadedConfig(result)
+}
+
+func newLoadedConfig(result *configkit.Result[Document]) (*LoadedConfig, error) {
+	if result == nil {
+		return nil, fmt.Errorf("configuration result is unavailable")
+	}
+	runtime, err := documentToConfig(result.Config)
+	if err != nil {
+		return nil, err
+	}
+	return &LoadedConfig{Config: runtime, result: result}, nil
+}
+
+func documentToConfig(document Document) (Config, error) {
+	config := Default()
+	config.Pipelines = documentPipelinesToRuntime(document.Pipelines)
+	if document.Concurrency != nil {
+		config.Concurrency.StageWorkers = documentIntMap(document.Concurrency.StageWorkers)
+		if document.Concurrency.TotalLLM != nil {
+			config.Concurrency.TotalLLM = *document.Concurrency.TotalLLM
+		}
+	}
+	if workers, ok := config.Concurrency.StageWorkers["extract"]; ok {
+		config.Concurrency.extractWorkersConfigured = true
+		config.Concurrency.defaultedExtractWorkers = workers
+	}
+	if document.PromptKit != nil {
+		if document.PromptKit.ProfileDir != nil {
+			config.PromptKit.ProfileDir = *document.PromptKit.ProfileDir
+		}
+		if document.PromptKit.ProfileFile != nil {
+			config.PromptKit.ProfileFile = *document.PromptKit.ProfileFile
+		}
+		if document.PromptKit.LocalBackend != nil {
+			backend := PromptKitLocalBackendConfig{}
+			if document.PromptKit.LocalBackend.Endpoint != nil {
+				backend.Endpoint = *document.PromptKit.LocalBackend.Endpoint
+			}
+			if document.PromptKit.LocalBackend.ConcurrencyLimit != nil {
+				backend.ConcurrencyLimit = *document.PromptKit.LocalBackend.ConcurrencyLimit
+			}
+			config.PromptKit.LocalBackend = &backend
+		}
+	}
+	if document.Output != nil && document.Output.Directory != nil {
+		config.Output.Directory = *document.Output.Directory
+	}
+	if document.Cache != nil {
+		if document.Cache.ChunkPlans != nil {
+			if document.Cache.ChunkPlans.Directory != nil {
+				config.Cache.ChunkPlans.Directory = *document.Cache.ChunkPlans.Directory
+			}
+			if document.Cache.ChunkPlans.Mode != nil {
+				mode, err := pipeline.ParseChunkCacheMode(*document.Cache.ChunkPlans.Mode)
+				if err != nil {
+					return Config{}, fmt.Errorf("convert chunk-plan cache mode: %w", err)
+				}
+				config.Cache.ChunkPlans.Mode = mode
+			}
+		}
+		if document.Cache.Checkpoints != nil {
+			if document.Cache.Checkpoints.Enabled != nil {
+				config.Cache.Checkpoints.Enabled = *document.Cache.Checkpoints.Enabled
+			}
+			if document.Cache.Checkpoints.Directory != nil {
+				config.Cache.Checkpoints.Directory = *document.Cache.Checkpoints.Directory
+			}
+		}
+	}
+	if document.Debug != nil && document.Debug.Directory != nil {
+		config.Debug.Directory = *document.Debug.Directory
+	}
+	return config, nil
+}
+
+func documentPipelinesToRuntime(documents map[string]PipelineDocument) map[string]pipeline.PipelineProfile {
+	if documents == nil {
+		return nil
+	}
+	profiles := make(map[string]pipeline.PipelineProfile, len(documents))
+	for id, document := range documents {
+		profiles[id] = documentPipelineToRuntime(id, document)
+	}
+	return profiles
+}
+
+func documentPipelineToRuntime(id string, document PipelineDocument) pipeline.PipelineProfile {
+	profile := pipeline.PipelineProfile{
+		ID:                             id,
+		StructuredOutputRepairAttempts: cloneDocumentInt(document.StructuredOutputRepairAttempts),
+		ValidationPolicy:               documentPolicyToRuntime(document.ValidationPolicy),
+		Input:                          documentBindingToRuntime(document.Input),
+		Artifacts:                      documentLanesToRuntime(document.Artifacts),
+		References:                     documentReferencesToRuntime(document.References),
+	}
+	if document.LLMProfile != nil {
+		profile.LLMProfile = *document.LLMProfile
+	}
+	if document.Chunk != nil {
+		profile.Chunk = documentBindingToRuntime(*document.Chunk)
+	}
+	if document.Output != nil {
+		profile.Output = documentBindingToRuntime(*document.Output)
+	}
+	if document.Steps != nil {
+		profile.Steps = make([]pipeline.PipelineStepProfile, len(document.Steps))
+		for index, step := range document.Steps {
+			profile.Steps[index] = pipeline.PipelineStepProfile{
+				ID:         step.ID,
+				Artifacts:  documentLanesToRuntime(step.Artifacts),
+				References: documentReferencesToRuntime(step.References),
+			}
+		}
+	}
+	return profile
+}
+
+func documentLanesToRuntime(documents map[string]ArtifactLaneDocument) map[string]pipeline.ArtifactLaneProfile {
+	if documents == nil {
+		return nil
+	}
+	lanes := make(map[string]pipeline.ArtifactLaneProfile, len(documents))
+	for id, document := range documents {
+		lane := pipeline.ArtifactLaneProfile{
+			Extract:    documentBindingToRuntime(document.Extract),
+			References: documentReferencesToRuntime(document.References),
+		}
+		if document.Merge != nil {
+			lane.Merge = documentBindingToRuntime(*document.Merge)
+		}
+		if document.Normalize != nil {
+			lane.Normalize = documentBindingToRuntime(*document.Normalize)
+		}
+		if document.Validators != nil {
+			lane.Validators = make([]pipeline.ModuleBinding, len(document.Validators))
+			for index, validator := range document.Validators {
+				lane.Validators[index] = documentValidatorToRuntime(validator)
+			}
+		}
+		lanes[id] = lane
+	}
+	return lanes
+}
+
+func documentBindingToRuntime(document BindingDocument) pipeline.ModuleBinding {
+	binding := pipeline.ModuleBinding{
+		Module:                         document.Module,
+		StructuredOutputRepairAttempts: cloneDocumentInt(document.StructuredOutputRepairAttempts),
+		ValidationPolicy:               documentPolicyToRuntime(document.ValidationPolicy),
+		Retries:                        document.Retries,
+		Options:                        documentOptions(document.Options),
+		References:                     documentReferencesToRuntime(document.References),
+	}
+	if document.LLMProfile != nil {
+		binding.LLMProfile = *document.LLMProfile
+	}
+	if document.Validators != nil {
+		binding.Validators.Set = true
+		binding.Validators.Validators = make([]pipeline.ModuleBinding, len(document.Validators))
+		for index, validator := range document.Validators {
+			binding.Validators.Validators[index] = documentValidatorToRuntime(validator)
+		}
+	}
+	return binding
+}
+
+func documentValidatorToRuntime(document ValidatorDocument) pipeline.ModuleBinding {
+	binding := pipeline.ModuleBinding{
+		Module:                         document.Module,
+		StructuredOutputRepairAttempts: cloneDocumentInt(document.StructuredOutputRepairAttempts),
+		Retries:                        document.Retries,
+		Options:                        documentOptions(document.Options),
+	}
+	if document.LLMProfile != nil {
+		binding.LLMProfile = *document.LLMProfile
+	}
+	return binding
+}
+
+func documentReferencesToRuntime(documents map[string]ReferenceDocument) map[string]pipeline.ReferenceSource {
+	if documents == nil {
+		return nil
+	}
+	references := make(map[string]pipeline.ReferenceSource, len(documents))
+	for name, document := range documents {
+		if document.Artifact != nil {
+			references[name] = pipeline.GeneratedReference(document.Artifact.Step, document.Artifact.Lane)
+			continue
+		}
+		if document.Path != nil {
+			references[name] = pipeline.ExternalReference(*document.Path)
+		}
+	}
+	return references
+}
+
+func documentPolicyToRuntime(document *ValidationPolicyDocument) *pipeline.ValidationPolicyOverride {
+	if document == nil {
+		return nil
+	}
+	policy := pipeline.ValidationPolicyOverride{}
+	if document.ProducerStructuralFailure != nil {
+		value := *document.ProducerStructuralFailure
+		policy.ProducerStructuralFailure = &value
+	}
+	if document.SemanticRejection != nil {
+		value := *document.SemanticRejection
+		policy.SemanticRejection = &value
+	}
+	if document.ValidatorFailure != nil {
+		value := *document.ValidatorFailure
+		policy.ValidatorFailure = &value
+	}
+	return &policy
+}
+
+func documentIntMap(values map[string]int) map[string]int {
+	if values == nil {
+		return nil
+	}
+	result := make(map[string]int, len(values))
+	for key, value := range values {
+		result[key] = value
+	}
+	return result
+}
+
+func cloneDocumentInt(value *int) *int {
+	if value == nil {
+		return nil
+	}
+	result := *value
+	return &result
+}
+
+func documentOptions(values map[string]any) map[string]any {
+	if values == nil {
+		return nil
+	}
+	result := make(map[string]any, len(values))
+	for key, value := range values {
+		result[key] = cloneDocumentOption(value)
+	}
+	return result
+}
+
+func cloneDocumentOption(value any) any {
+	switch typed := value.(type) {
+	case map[string]any:
+		return documentOptions(typed)
+	case []any:
+		result := make([]any, len(typed))
+		for index, item := range typed {
+			result[index] = cloneDocumentOption(item)
+		}
+		return result
+	default:
+		return typed
+	}
+}

internal/core/config/loaded_config_test.go

@@ -0,0 +1,102 @@
+package config
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+
+	configkit "gitea.maximumdirect.net/eric/ConfigKit"
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
+)
+
+func TestLoadedConfigConvertsDocumentsAndResolvesRuntimePipeline(t *testing.T) {
+	t.Parallel()
+
+	loaded, err := loadConfigSource(configkit.Bytes("document.yml", []byte(`version: 5
+pipelines:
+  session:
+    llm_profile: local
+    structured_output_repair_attempts: 0
+    input:
+      module: input
+    chunk:
+      module: generic
+      validators: []
+    artifacts:
+      score:
+        extract:
+          module: extract
+          validators:
+            - module: configured-a
+              options:
+                nested:
+                  level: 1
+`)), LoadOptions{})
+	if err != nil {
+		t.Fatalf("loadConfigSource() error = %v", err)
+	}
+	profile := loaded.Config.Pipelines["session"]
+	if profile.LLMProfile != "local" || profile.StructuredOutputRepairAttempts == nil || *profile.StructuredOutputRepairAttempts != 0 {
+		t.Fatalf("pipeline profile = %#v", profile)
+	}
+	if !profile.Chunk.Validators.Set || profile.Chunk.Validators.Validators == nil || len(profile.Chunk.Validators.Validators) != 0 {
+		t.Fatalf("empty validator chain was not preserved: %#v", profile.Chunk.Validators)
+	}
+	validator := profile.Artifacts["score"].Extract.Validators.Validators[0]
+	if validator.Module != "configured-a" || validator.Options["nested"].(map[string]any)["level"] == nil {
+		t.Fatalf("validator conversion = %#v", validator)
+	}
+
+	effective, err := loaded.Config.Resolve(ResolveInput{PipelineID: "session", Catalog: effectiveCatalog(t)})
+	if err != nil {
+		t.Fatalf("Resolve() error = %v", err)
+	}
+	if effective.ResolvedPipeline.ID != "session" || effective.ResolvedPipeline.Digest == "" {
+		t.Fatalf("resolved pipeline = %#v", effective.ResolvedPipeline)
+	}
+}
+
+func TestLoadedConfigOwnsRuntimeValuesAndKeepsFrozenInspection(t *testing.T) {
+	t.Parallel()
+
+	loaded, err := loadConfigSource(configkit.Bytes("document.yml", []byte(`version: 5
+pipelines:
+  session:
+    input: {module: input}
+    artifacts:
+      score:
+        extract:
+          module: extract
+          options:
+            nested: {value: original}
+`)), LoadOptions{})
+	if err != nil {
+		t.Fatalf("loadConfigSource() error = %v", err)
+	}
+
+	first := loaded.RuntimeConfig()
+	first.Pipelines["session"] = pipelineProfileWithChangedOption(first.Pipelines["session"])
+	second := loaded.RuntimeConfig()
+	if got := second.Pipelines["session"].Artifacts["score"].Extract.Options["nested"].(map[string]any)["value"]; got != "original" {
+		t.Fatalf("runtime configuration aliases caller mutation: %#v", got)
+	}
+
+	loaded.Config.Pipelines["session"] = pipelineProfileWithChangedOption(loaded.Config.Pipelines["session"])
+	var effective bytes.Buffer
+	if err := loaded.WriteEffective(&effective); err != nil {
+		t.Fatalf("WriteEffective() error = %v", err)
+	}
+	if !strings.Contains(effective.String(), "original") || strings.Contains(effective.String(), "changed") {
+		t.Fatalf("effective document changed with runtime mutation: %q", effective.String())
+	}
+}
+
+func pipelineProfileWithChangedOption(profile pipeline.PipelineProfile) pipeline.PipelineProfile {
+	profile = clonePipelineProfile(profile)
+	nested := profile.Artifacts["score"].Extract.Options["nested"].(map[string]any)
+	nested["value"] = "changed"
+	lane := profile.Artifacts["score"]
+	lane.Extract.Options["nested"] = nested
+	profile.Artifacts["score"] = lane
+	return profile
+}