Convert ConfigKit documents to runtime configuration
This commit is contained in:
@@ -259,6 +259,8 @@ not combine it with source-origin or CLI work.
|
||||
|
||||
## Stage 4 — Convert Documents Into Runtime Configuration
|
||||
|
||||
**Status:** Completed 2026-09-13.
|
||||
|
||||
**Goal:** Produce the existing typed runtime composition from the new document.
|
||||
|
||||
**Work:**
|
||||
|
||||
321
internal/core/config/loaded_config.go
Normal file
321
internal/core/config/loaded_config.go
Normal file
@@ -0,0 +1,321 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
// LoadedConfig keeps the finalized runtime configuration together with the
|
||||
// private ConfigKit inspection snapshot that produced it.
|
||||
type LoadedConfig struct {
|
||||
Config Config
|
||||
|
||||
result *configkit.Result[Document]
|
||||
}
|
||||
|
||||
// RuntimeConfig returns an independently owned runtime configuration.
|
||||
func (loaded *LoadedConfig) RuntimeConfig() Config {
|
||||
if loaded == nil {
|
||||
return Config{}
|
||||
}
|
||||
return cloneConfig(loaded.Config)
|
||||
}
|
||||
|
||||
// WriteEffective renders ConfigKit's frozen, redacted effective document.
|
||||
func (loaded *LoadedConfig) WriteEffective(output io.Writer) error {
|
||||
if loaded == nil || loaded.result == nil {
|
||||
return fmt.Errorf("loaded configuration is unavailable")
|
||||
}
|
||||
return loaded.result.WriteEffective(output)
|
||||
}
|
||||
|
||||
// WriteExplanation renders ConfigKit's safe explanation for a JSON-pointer
|
||||
// path without exposing ConfigKit types at this package boundary.
|
||||
func (loaded *LoadedConfig) WriteExplanation(output io.Writer, pointer string) error {
|
||||
if loaded == nil || loaded.result == nil {
|
||||
return fmt.Errorf("loaded configuration is unavailable")
|
||||
}
|
||||
path, err := configkit.ParsePath(pointer)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse configuration explanation path: %w", err)
|
||||
}
|
||||
return loaded.result.WriteExplanation(output, path)
|
||||
}
|
||||
|
||||
func loadConfig(path string, options LoadOptions) (*LoadedConfig, error) {
|
||||
result, err := loadDocument(path, options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return newLoadedConfig(result)
|
||||
}
|
||||
|
||||
func loadConfigSource(source configkit.Source, options LoadOptions) (*LoadedConfig, error) {
|
||||
result, err := loadDocumentSource(source, options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return newLoadedConfig(result)
|
||||
}
|
||||
|
||||
func newLoadedConfig(result *configkit.Result[Document]) (*LoadedConfig, error) {
|
||||
if result == nil {
|
||||
return nil, fmt.Errorf("configuration result is unavailable")
|
||||
}
|
||||
runtime, err := documentToConfig(result.Config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &LoadedConfig{Config: runtime, result: result}, nil
|
||||
}
|
||||
|
||||
func documentToConfig(document Document) (Config, error) {
|
||||
config := Default()
|
||||
config.Pipelines = documentPipelinesToRuntime(document.Pipelines)
|
||||
if document.Concurrency != nil {
|
||||
config.Concurrency.StageWorkers = documentIntMap(document.Concurrency.StageWorkers)
|
||||
if document.Concurrency.TotalLLM != nil {
|
||||
config.Concurrency.TotalLLM = *document.Concurrency.TotalLLM
|
||||
}
|
||||
}
|
||||
if workers, ok := config.Concurrency.StageWorkers["extract"]; ok {
|
||||
config.Concurrency.extractWorkersConfigured = true
|
||||
config.Concurrency.defaultedExtractWorkers = workers
|
||||
}
|
||||
if document.PromptKit != nil {
|
||||
if document.PromptKit.ProfileDir != nil {
|
||||
config.PromptKit.ProfileDir = *document.PromptKit.ProfileDir
|
||||
}
|
||||
if document.PromptKit.ProfileFile != nil {
|
||||
config.PromptKit.ProfileFile = *document.PromptKit.ProfileFile
|
||||
}
|
||||
if document.PromptKit.LocalBackend != nil {
|
||||
backend := PromptKitLocalBackendConfig{}
|
||||
if document.PromptKit.LocalBackend.Endpoint != nil {
|
||||
backend.Endpoint = *document.PromptKit.LocalBackend.Endpoint
|
||||
}
|
||||
if document.PromptKit.LocalBackend.ConcurrencyLimit != nil {
|
||||
backend.ConcurrencyLimit = *document.PromptKit.LocalBackend.ConcurrencyLimit
|
||||
}
|
||||
config.PromptKit.LocalBackend = &backend
|
||||
}
|
||||
}
|
||||
if document.Output != nil && document.Output.Directory != nil {
|
||||
config.Output.Directory = *document.Output.Directory
|
||||
}
|
||||
if document.Cache != nil {
|
||||
if document.Cache.ChunkPlans != nil {
|
||||
if document.Cache.ChunkPlans.Directory != nil {
|
||||
config.Cache.ChunkPlans.Directory = *document.Cache.ChunkPlans.Directory
|
||||
}
|
||||
if document.Cache.ChunkPlans.Mode != nil {
|
||||
mode, err := pipeline.ParseChunkCacheMode(*document.Cache.ChunkPlans.Mode)
|
||||
if err != nil {
|
||||
return Config{}, fmt.Errorf("convert chunk-plan cache mode: %w", err)
|
||||
}
|
||||
config.Cache.ChunkPlans.Mode = mode
|
||||
}
|
||||
}
|
||||
if document.Cache.Checkpoints != nil {
|
||||
if document.Cache.Checkpoints.Enabled != nil {
|
||||
config.Cache.Checkpoints.Enabled = *document.Cache.Checkpoints.Enabled
|
||||
}
|
||||
if document.Cache.Checkpoints.Directory != nil {
|
||||
config.Cache.Checkpoints.Directory = *document.Cache.Checkpoints.Directory
|
||||
}
|
||||
}
|
||||
}
|
||||
if document.Debug != nil && document.Debug.Directory != nil {
|
||||
config.Debug.Directory = *document.Debug.Directory
|
||||
}
|
||||
return config, nil
|
||||
}
|
||||
|
||||
func documentPipelinesToRuntime(documents map[string]PipelineDocument) map[string]pipeline.PipelineProfile {
|
||||
if documents == nil {
|
||||
return nil
|
||||
}
|
||||
profiles := make(map[string]pipeline.PipelineProfile, len(documents))
|
||||
for id, document := range documents {
|
||||
profiles[id] = documentPipelineToRuntime(id, document)
|
||||
}
|
||||
return profiles
|
||||
}
|
||||
|
||||
func documentPipelineToRuntime(id string, document PipelineDocument) pipeline.PipelineProfile {
|
||||
profile := pipeline.PipelineProfile{
|
||||
ID: id,
|
||||
StructuredOutputRepairAttempts: cloneDocumentInt(document.StructuredOutputRepairAttempts),
|
||||
ValidationPolicy: documentPolicyToRuntime(document.ValidationPolicy),
|
||||
Input: documentBindingToRuntime(document.Input),
|
||||
Artifacts: documentLanesToRuntime(document.Artifacts),
|
||||
References: documentReferencesToRuntime(document.References),
|
||||
}
|
||||
if document.LLMProfile != nil {
|
||||
profile.LLMProfile = *document.LLMProfile
|
||||
}
|
||||
if document.Chunk != nil {
|
||||
profile.Chunk = documentBindingToRuntime(*document.Chunk)
|
||||
}
|
||||
if document.Output != nil {
|
||||
profile.Output = documentBindingToRuntime(*document.Output)
|
||||
}
|
||||
if document.Steps != nil {
|
||||
profile.Steps = make([]pipeline.PipelineStepProfile, len(document.Steps))
|
||||
for index, step := range document.Steps {
|
||||
profile.Steps[index] = pipeline.PipelineStepProfile{
|
||||
ID: step.ID,
|
||||
Artifacts: documentLanesToRuntime(step.Artifacts),
|
||||
References: documentReferencesToRuntime(step.References),
|
||||
}
|
||||
}
|
||||
}
|
||||
return profile
|
||||
}
|
||||
|
||||
func documentLanesToRuntime(documents map[string]ArtifactLaneDocument) map[string]pipeline.ArtifactLaneProfile {
|
||||
if documents == nil {
|
||||
return nil
|
||||
}
|
||||
lanes := make(map[string]pipeline.ArtifactLaneProfile, len(documents))
|
||||
for id, document := range documents {
|
||||
lane := pipeline.ArtifactLaneProfile{
|
||||
Extract: documentBindingToRuntime(document.Extract),
|
||||
References: documentReferencesToRuntime(document.References),
|
||||
}
|
||||
if document.Merge != nil {
|
||||
lane.Merge = documentBindingToRuntime(*document.Merge)
|
||||
}
|
||||
if document.Normalize != nil {
|
||||
lane.Normalize = documentBindingToRuntime(*document.Normalize)
|
||||
}
|
||||
if document.Validators != nil {
|
||||
lane.Validators = make([]pipeline.ModuleBinding, len(document.Validators))
|
||||
for index, validator := range document.Validators {
|
||||
lane.Validators[index] = documentValidatorToRuntime(validator)
|
||||
}
|
||||
}
|
||||
lanes[id] = lane
|
||||
}
|
||||
return lanes
|
||||
}
|
||||
|
||||
func documentBindingToRuntime(document BindingDocument) pipeline.ModuleBinding {
|
||||
binding := pipeline.ModuleBinding{
|
||||
Module: document.Module,
|
||||
StructuredOutputRepairAttempts: cloneDocumentInt(document.StructuredOutputRepairAttempts),
|
||||
ValidationPolicy: documentPolicyToRuntime(document.ValidationPolicy),
|
||||
Retries: document.Retries,
|
||||
Options: documentOptions(document.Options),
|
||||
References: documentReferencesToRuntime(document.References),
|
||||
}
|
||||
if document.LLMProfile != nil {
|
||||
binding.LLMProfile = *document.LLMProfile
|
||||
}
|
||||
if document.Validators != nil {
|
||||
binding.Validators.Set = true
|
||||
binding.Validators.Validators = make([]pipeline.ModuleBinding, len(document.Validators))
|
||||
for index, validator := range document.Validators {
|
||||
binding.Validators.Validators[index] = documentValidatorToRuntime(validator)
|
||||
}
|
||||
}
|
||||
return binding
|
||||
}
|
||||
|
||||
func documentValidatorToRuntime(document ValidatorDocument) pipeline.ModuleBinding {
|
||||
binding := pipeline.ModuleBinding{
|
||||
Module: document.Module,
|
||||
StructuredOutputRepairAttempts: cloneDocumentInt(document.StructuredOutputRepairAttempts),
|
||||
Retries: document.Retries,
|
||||
Options: documentOptions(document.Options),
|
||||
}
|
||||
if document.LLMProfile != nil {
|
||||
binding.LLMProfile = *document.LLMProfile
|
||||
}
|
||||
return binding
|
||||
}
|
||||
|
||||
func documentReferencesToRuntime(documents map[string]ReferenceDocument) map[string]pipeline.ReferenceSource {
|
||||
if documents == nil {
|
||||
return nil
|
||||
}
|
||||
references := make(map[string]pipeline.ReferenceSource, len(documents))
|
||||
for name, document := range documents {
|
||||
if document.Artifact != nil {
|
||||
references[name] = pipeline.GeneratedReference(document.Artifact.Step, document.Artifact.Lane)
|
||||
continue
|
||||
}
|
||||
if document.Path != nil {
|
||||
references[name] = pipeline.ExternalReference(*document.Path)
|
||||
}
|
||||
}
|
||||
return references
|
||||
}
|
||||
|
||||
func documentPolicyToRuntime(document *ValidationPolicyDocument) *pipeline.ValidationPolicyOverride {
|
||||
if document == nil {
|
||||
return nil
|
||||
}
|
||||
policy := pipeline.ValidationPolicyOverride{}
|
||||
if document.ProducerStructuralFailure != nil {
|
||||
value := *document.ProducerStructuralFailure
|
||||
policy.ProducerStructuralFailure = &value
|
||||
}
|
||||
if document.SemanticRejection != nil {
|
||||
value := *document.SemanticRejection
|
||||
policy.SemanticRejection = &value
|
||||
}
|
||||
if document.ValidatorFailure != nil {
|
||||
value := *document.ValidatorFailure
|
||||
policy.ValidatorFailure = &value
|
||||
}
|
||||
return &policy
|
||||
}
|
||||
|
||||
func documentIntMap(values map[string]int) map[string]int {
|
||||
if values == nil {
|
||||
return nil
|
||||
}
|
||||
result := make(map[string]int, len(values))
|
||||
for key, value := range values {
|
||||
result[key] = value
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func cloneDocumentInt(value *int) *int {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
result := *value
|
||||
return &result
|
||||
}
|
||||
|
||||
func documentOptions(values map[string]any) map[string]any {
|
||||
if values == nil {
|
||||
return nil
|
||||
}
|
||||
result := make(map[string]any, len(values))
|
||||
for key, value := range values {
|
||||
result[key] = cloneDocumentOption(value)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func cloneDocumentOption(value any) any {
|
||||
switch typed := value.(type) {
|
||||
case map[string]any:
|
||||
return documentOptions(typed)
|
||||
case []any:
|
||||
result := make([]any, len(typed))
|
||||
for index, item := range typed {
|
||||
result[index] = cloneDocumentOption(item)
|
||||
}
|
||||
return result
|
||||
default:
|
||||
return typed
|
||||
}
|
||||
}
|
||||
102
internal/core/config/loaded_config_test.go
Normal file
102
internal/core/config/loaded_config_test.go
Normal file
@@ -0,0 +1,102 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestLoadedConfigConvertsDocumentsAndResolvesRuntimePipeline(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
loaded, err := loadConfigSource(configkit.Bytes("document.yml", []byte(`version: 5
|
||||
pipelines:
|
||||
session:
|
||||
llm_profile: local
|
||||
structured_output_repair_attempts: 0
|
||||
input:
|
||||
module: input
|
||||
chunk:
|
||||
module: generic
|
||||
validators: []
|
||||
artifacts:
|
||||
score:
|
||||
extract:
|
||||
module: extract
|
||||
validators:
|
||||
- module: configured-a
|
||||
options:
|
||||
nested:
|
||||
level: 1
|
||||
`)), LoadOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("loadConfigSource() error = %v", err)
|
||||
}
|
||||
profile := loaded.Config.Pipelines["session"]
|
||||
if profile.LLMProfile != "local" || profile.StructuredOutputRepairAttempts == nil || *profile.StructuredOutputRepairAttempts != 0 {
|
||||
t.Fatalf("pipeline profile = %#v", profile)
|
||||
}
|
||||
if !profile.Chunk.Validators.Set || profile.Chunk.Validators.Validators == nil || len(profile.Chunk.Validators.Validators) != 0 {
|
||||
t.Fatalf("empty validator chain was not preserved: %#v", profile.Chunk.Validators)
|
||||
}
|
||||
validator := profile.Artifacts["score"].Extract.Validators.Validators[0]
|
||||
if validator.Module != "configured-a" || validator.Options["nested"].(map[string]any)["level"] == nil {
|
||||
t.Fatalf("validator conversion = %#v", validator)
|
||||
}
|
||||
|
||||
effective, err := loaded.Config.Resolve(ResolveInput{PipelineID: "session", Catalog: effectiveCatalog(t)})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
if effective.ResolvedPipeline.ID != "session" || effective.ResolvedPipeline.Digest == "" {
|
||||
t.Fatalf("resolved pipeline = %#v", effective.ResolvedPipeline)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadedConfigOwnsRuntimeValuesAndKeepsFrozenInspection(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
loaded, err := loadConfigSource(configkit.Bytes("document.yml", []byte(`version: 5
|
||||
pipelines:
|
||||
session:
|
||||
input: {module: input}
|
||||
artifacts:
|
||||
score:
|
||||
extract:
|
||||
module: extract
|
||||
options:
|
||||
nested: {value: original}
|
||||
`)), LoadOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("loadConfigSource() error = %v", err)
|
||||
}
|
||||
|
||||
first := loaded.RuntimeConfig()
|
||||
first.Pipelines["session"] = pipelineProfileWithChangedOption(first.Pipelines["session"])
|
||||
second := loaded.RuntimeConfig()
|
||||
if got := second.Pipelines["session"].Artifacts["score"].Extract.Options["nested"].(map[string]any)["value"]; got != "original" {
|
||||
t.Fatalf("runtime configuration aliases caller mutation: %#v", got)
|
||||
}
|
||||
|
||||
loaded.Config.Pipelines["session"] = pipelineProfileWithChangedOption(loaded.Config.Pipelines["session"])
|
||||
var effective bytes.Buffer
|
||||
if err := loaded.WriteEffective(&effective); err != nil {
|
||||
t.Fatalf("WriteEffective() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(effective.String(), "original") || strings.Contains(effective.String(), "changed") {
|
||||
t.Fatalf("effective document changed with runtime mutation: %q", effective.String())
|
||||
}
|
||||
}
|
||||
|
||||
func pipelineProfileWithChangedOption(profile pipeline.PipelineProfile) pipeline.PipelineProfile {
|
||||
profile = clonePipelineProfile(profile)
|
||||
nested := profile.Artifacts["score"].Extract.Options["nested"].(map[string]any)
|
||||
nested["value"] = "changed"
|
||||
lane := profile.Artifacts["score"]
|
||||
lane.Extract.Options["nested"] = nested
|
||||
profile.Artifacts["score"] = lane
|
||||
return profile
|
||||
}
|
||||
Reference in New Issue
Block a user