Add ConfigKit version 5 document boundary
This commit is contained in:
122
internal/core/config/document.go
Normal file
122
internal/core/config/document.go
Normal file
@@ -0,0 +1,122 @@
|
||||
package config
|
||||
|
||||
import "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
|
||||
// Document is the version 5 configuration shape decoded by ConfigKit. It is
|
||||
// deliberately separate from Config so YAML presence and composition concerns
|
||||
// do not reach runtime pipeline types.
|
||||
type Document struct {
|
||||
Version string `yaml:"version"`
|
||||
PromptKit *PromptKitDocument `yaml:"promptkit,omitempty"`
|
||||
Pipelines map[string]PipelineDocument `yaml:"pipelines,omitempty"`
|
||||
Concurrency *ConcurrencyDocument `yaml:"concurrency,omitempty"`
|
||||
Output *OutputDocument `yaml:"output,omitempty"`
|
||||
Cache *CacheDocument `yaml:"cache,omitempty"`
|
||||
Debug *DebugDocument `yaml:"debug,omitempty"`
|
||||
}
|
||||
|
||||
type PromptKitDocument struct {
|
||||
ProfileDir *string `yaml:"profile_dir,omitempty"`
|
||||
ProfileFile *string `yaml:"profile_file,omitempty"`
|
||||
LocalBackend *PromptKitLocalBackendDocument `yaml:"local_backend,omitempty"`
|
||||
}
|
||||
|
||||
type PromptKitLocalBackendDocument struct {
|
||||
Endpoint *string `yaml:"endpoint,omitempty"`
|
||||
ConcurrencyLimit *int `yaml:"concurrency_limit,omitempty"`
|
||||
}
|
||||
|
||||
type PipelineDocument struct {
|
||||
LLMProfile *string `yaml:"llm_profile,omitempty"`
|
||||
StructuredOutputRepairAttempts *int `yaml:"structured_output_repair_attempts,omitempty"`
|
||||
ValidationPolicy *ValidationPolicyDocument `yaml:"validation_policy,omitempty"`
|
||||
Input BindingDocument `yaml:"input"`
|
||||
Chunk *BindingDocument `yaml:"chunk,omitempty"`
|
||||
Artifacts map[string]ArtifactLaneDocument `yaml:"artifacts,omitempty"`
|
||||
Steps []PipelineStepDocument `yaml:"steps,omitempty"`
|
||||
Output *BindingDocument `yaml:"output,omitempty"`
|
||||
References map[string]ReferenceDocument `yaml:"references,omitempty"`
|
||||
}
|
||||
|
||||
type PipelineStepDocument struct {
|
||||
ID string `yaml:"id"`
|
||||
Artifacts map[string]ArtifactLaneDocument `yaml:"artifacts"`
|
||||
References map[string]ReferenceDocument `yaml:"references,omitempty"`
|
||||
}
|
||||
|
||||
type ArtifactLaneDocument struct {
|
||||
Extract BindingDocument `yaml:"extract"`
|
||||
Merge *BindingDocument `yaml:"merge,omitempty"`
|
||||
Normalize *BindingDocument `yaml:"normalize,omitempty"`
|
||||
Validators []ValidatorDocument `yaml:"validators,omitempty"`
|
||||
References map[string]ReferenceDocument `yaml:"references,omitempty"`
|
||||
}
|
||||
|
||||
// BindingDocument represents a producer binding. ValidatorDocument is a
|
||||
// deliberately finite subset so nested validator-only fields are rejected by
|
||||
// ConfigKit's strict decoder.
|
||||
type BindingDocument struct {
|
||||
Module string `yaml:"module"`
|
||||
LLMProfile *string `yaml:"llm_profile,omitempty"`
|
||||
StructuredOutputRepairAttempts *int `yaml:"structured_output_repair_attempts,omitempty"`
|
||||
ValidationPolicy *ValidationPolicyDocument `yaml:"validation_policy,omitempty"`
|
||||
Retries int `yaml:"retries,omitempty"`
|
||||
Options map[string]any `yaml:"options,omitempty"`
|
||||
References map[string]ReferenceDocument `yaml:"references,omitempty"`
|
||||
Validators []ValidatorDocument `yaml:"validators,omitempty"`
|
||||
}
|
||||
|
||||
type ValidatorDocument struct {
|
||||
Module string `yaml:"module"`
|
||||
LLMProfile *string `yaml:"llm_profile,omitempty"`
|
||||
StructuredOutputRepairAttempts *int `yaml:"structured_output_repair_attempts,omitempty"`
|
||||
Retries int `yaml:"retries,omitempty"`
|
||||
Options map[string]any `yaml:"options,omitempty"`
|
||||
}
|
||||
|
||||
// ReferenceDocument is the explicit version 5 union. Later document
|
||||
// validation establishes which locations permit generated references and
|
||||
// requires exactly one populated branch.
|
||||
type ReferenceDocument struct {
|
||||
Path *string `yaml:"path,omitempty"`
|
||||
Artifact *ArtifactReferenceDocument `yaml:"artifact,omitempty"`
|
||||
}
|
||||
|
||||
type ArtifactReferenceDocument struct {
|
||||
Step string `yaml:"step"`
|
||||
Lane string `yaml:"lane"`
|
||||
}
|
||||
|
||||
type ValidationPolicyDocument struct {
|
||||
ProducerStructuralFailure *pipeline.ProducerStructuralFailureAction `yaml:"producer_structural_failure,omitempty"`
|
||||
SemanticRejection *pipeline.SemanticRejectionAction `yaml:"semantic_rejection,omitempty"`
|
||||
ValidatorFailure *pipeline.ValidatorFailureAction `yaml:"validator_failure,omitempty"`
|
||||
}
|
||||
|
||||
type ConcurrencyDocument struct {
|
||||
TotalLLM *int `yaml:"total_llm,omitempty"`
|
||||
StageWorkers map[string]int `yaml:"stage_workers,omitempty"`
|
||||
}
|
||||
|
||||
type OutputDocument struct {
|
||||
Directory *string `yaml:"directory,omitempty"`
|
||||
}
|
||||
|
||||
type CacheDocument struct {
|
||||
ChunkPlans *ChunkPlanCacheDocument `yaml:"chunk_plans,omitempty"`
|
||||
Checkpoints *CheckpointCacheDocument `yaml:"checkpoints,omitempty"`
|
||||
}
|
||||
|
||||
type ChunkPlanCacheDocument struct {
|
||||
Directory *string `yaml:"directory,omitempty"`
|
||||
Mode *string `yaml:"mode,omitempty"`
|
||||
}
|
||||
|
||||
type CheckpointCacheDocument struct {
|
||||
Enabled *bool `yaml:"enabled,omitempty"`
|
||||
Directory *string `yaml:"directory,omitempty"`
|
||||
}
|
||||
|
||||
type DebugDocument struct {
|
||||
Directory *string `yaml:"directory,omitempty"`
|
||||
}
|
||||
154
internal/core/config/document_test.go
Normal file
154
internal/core/config/document_test.go
Normal file
@@ -0,0 +1,154 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
)
|
||||
|
||||
func TestDocumentLoadAcceptsVersion5AndCanonicalShapes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, source := range []string{
|
||||
`version: 5
|
||||
pipelines:
|
||||
session:
|
||||
input:
|
||||
module: seriatim
|
||||
references:
|
||||
party:
|
||||
path: ./party.txt
|
||||
steps:
|
||||
- id: describe
|
||||
artifacts:
|
||||
spells:
|
||||
extract:
|
||||
module: dnd/spells
|
||||
validators:
|
||||
- module: generic/valid_json
|
||||
options:
|
||||
strict: true
|
||||
- id: use-artifact
|
||||
references:
|
||||
spells:
|
||||
artifact:
|
||||
step: describe
|
||||
lane: spells
|
||||
artifacts:
|
||||
locations:
|
||||
extract:
|
||||
module: dnd/locations
|
||||
`,
|
||||
`version: "5"
|
||||
pipelines:
|
||||
session:
|
||||
input:
|
||||
module: seriatim
|
||||
`,
|
||||
} {
|
||||
loaded, err := loadDocumentSource(configkit.Bytes("document.yml", []byte(source)), LoadOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("loadDocumentSource() error = %v", err)
|
||||
}
|
||||
profile, ok := loaded.Config.Pipelines["session"]
|
||||
if !ok || profile.Input.Module != "seriatim" {
|
||||
t.Fatalf("pipeline = %#v", profile)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDocumentLoadRejectsUnsupportedVersionsBeforeDocumentDecoding(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
source string
|
||||
code configkit.Code
|
||||
}{
|
||||
{name: "missing", source: "unknown: value\n", code: configkit.CodeVersionError},
|
||||
{name: "null", source: "version: null\nunknown: value\n", code: configkit.CodeVersionError},
|
||||
{name: "version four", source: "version: 4\nunknown: value\n", code: configkit.CodeVersionError},
|
||||
{name: "unsupported", source: "version: 6\nunknown: value\n", code: configkit.CodeVersionError},
|
||||
{name: "noninteger", source: "version: 5.0\n", code: configkit.CodeVersionType},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
_, err := loadDocumentSource(configkit.Bytes("document.yml", []byte(test.source)), LoadOptions{})
|
||||
var loadError *configkit.LoadError
|
||||
if !errors.As(err, &loadError) {
|
||||
t.Fatalf("error = %v, want ConfigKit load error", err)
|
||||
}
|
||||
diagnostics := loadError.Diagnostics()
|
||||
if len(diagnostics) != 1 || diagnostics[0].Code != test.code || diagnostics[0].Path.String() != "/version" {
|
||||
t.Fatalf("diagnostics = %#v", diagnostics)
|
||||
}
|
||||
if strings.Contains(diagnostics[0].Message, "unknown") {
|
||||
t.Fatalf("document decoding ran before version validation: %#v", diagnostics)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDocumentLoadRejectsShorthandAndUnknownFields(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
source string
|
||||
code configkit.Code
|
||||
path string
|
||||
}{
|
||||
{
|
||||
name: "binding shorthand",
|
||||
source: "version: 5\npipelines:\n session:\n input: seriatim\n",
|
||||
code: configkit.CodeDecodeType,
|
||||
path: "/pipelines/session/input",
|
||||
},
|
||||
{
|
||||
name: "validator nested producer field",
|
||||
source: "version: 5\npipelines:\n session:\n input:\n module: seriatim\n validators:\n - module: generic/valid_json\n validators: []\n",
|
||||
code: configkit.CodeDecodeUnknownField,
|
||||
path: "/pipelines/session/input/validators/0/validators",
|
||||
},
|
||||
{
|
||||
name: "unknown field",
|
||||
source: "version: 5\npipelines:\n session:\n input:\n module: seriatim\n unknown: true\n",
|
||||
code: configkit.CodeDecodeUnknownField,
|
||||
path: "/pipelines/session/input/unknown",
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
_, err := loadDocumentSource(configkit.Bytes("document.yml", []byte(test.source)), LoadOptions{})
|
||||
var loadError *configkit.LoadError
|
||||
if !errors.As(err, &loadError) {
|
||||
t.Fatalf("error = %v, want ConfigKit load error", err)
|
||||
}
|
||||
diagnostics := loadError.Diagnostics()
|
||||
if len(diagnostics) != 1 || diagnostics[0].Code != test.code || diagnostics[0].Path.String() != test.path {
|
||||
t.Fatalf("diagnostics = %#v", diagnostics)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDocumentDefinitionReturnsIndependentDefaults(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
first, err := loadDocumentSource(configkit.Bytes("first.yml", []byte("version: 5\n")), LoadOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first.Config.Pipelines["changed"] = PipelineDocument{}
|
||||
first.Config.Concurrency.StageWorkers["extract"] = 3
|
||||
|
||||
second, err := loadDocumentSource(configkit.Bytes("second.yml", []byte("version: 5\n")), LoadOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(second.Config.Pipelines) != 0 || len(second.Config.Concurrency.StageWorkers) != 0 {
|
||||
t.Fatalf("defaults aliased across loads: %#v", second.Config)
|
||||
}
|
||||
}
|
||||
60
internal/core/config/load.go
Normal file
60
internal/core/config/load.go
Normal file
@@ -0,0 +1,60 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
)
|
||||
|
||||
// LoadOptions contains configuration selections supplied by an application
|
||||
// boundary. It is intentionally narrow: configuration discovery and CLI flag
|
||||
// parsing remain outside this package.
|
||||
type LoadOptions struct {
|
||||
Profile string
|
||||
Overrides []configkit.Override
|
||||
}
|
||||
|
||||
// loadDocument is the private ConfigKit load boundary under construction.
|
||||
// Production commands continue to use the existing version 4 loader until the
|
||||
// runtime conversion and command wiring are complete.
|
||||
func loadDocument(path string, options LoadOptions) (*configkit.Result[Document], error) {
|
||||
return loadDocumentSource(configkit.File(path), options)
|
||||
}
|
||||
|
||||
func loadDocumentSource(source configkit.Source, options LoadOptions) (*configkit.Result[Document], error) {
|
||||
return configkit.Load(source, configkit.Options[Document]{
|
||||
Definition: documentDefinition(),
|
||||
Profile: options.Profile,
|
||||
Overrides: options.Overrides,
|
||||
})
|
||||
}
|
||||
|
||||
func documentDefinition() configkit.Definition[Document] {
|
||||
return configkit.Definition[Document]{
|
||||
Defaults: func() Document {
|
||||
return Document{
|
||||
Pipelines: map[string]PipelineDocument{},
|
||||
Concurrency: &ConcurrencyDocument{
|
||||
StageWorkers: map[string]int{},
|
||||
},
|
||||
}
|
||||
},
|
||||
Version: &configkit.VersionRule{
|
||||
Path: configkit.Keys("version"),
|
||||
Check: validateDocumentVersion,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func validateDocumentVersion(version configkit.Version, validation *configkit.Validation) error {
|
||||
if !version.Present || version.Null {
|
||||
validation.Error(configkit.Keys("version"), "configuration version is required")
|
||||
return nil
|
||||
}
|
||||
if version.Text == "4" {
|
||||
validation.Error(configkit.Keys("version"), "configuration version 4 is no longer supported; migrate to version 5")
|
||||
return nil
|
||||
}
|
||||
if version.Text != "5" {
|
||||
validation.Error(configkit.Keys("version"), "unsupported configuration version")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user