Add version 3 configuration contract tests
This commit is contained in:
364
internal/core/config/file_config_contract_test.go
Normal file
364
internal/core/config/file_config_contract_test.go
Normal file
@@ -0,0 +1,364 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestDefaultReturnsDocumentedValuesAndIndependentMaps(t *testing.T) {
|
||||
first := Default()
|
||||
if first.Concurrency.TotalLLM != 1 || first.Concurrency.StageWorkers["extract"] != 1 {
|
||||
t.Fatalf("concurrency defaults = %#v", first.Concurrency)
|
||||
}
|
||||
if first.Output.Directory != "./notarius-output" || first.Debug.Directory != "./notarius-debug" {
|
||||
t.Fatalf("output/debug defaults = %#v, %#v", first.Output, first.Debug)
|
||||
}
|
||||
if first.Cache.ChunkPlans.Mode != pipeline.ChunkCacheAuto || first.Cache.ChunkPlans.Directory != "" || first.Cache.Checkpoints.Directory != "" {
|
||||
t.Fatalf("cache defaults = %#v", first.Cache)
|
||||
}
|
||||
if len(first.Pipelines) != 0 {
|
||||
t.Fatalf("pipeline defaults = %#v", first.Pipelines)
|
||||
}
|
||||
|
||||
first.Concurrency.StageWorkers["extract"] = 99
|
||||
first.Concurrency.StageWorkers["other"] = 100
|
||||
first.Pipelines["changed"] = pipeline.PipelineProfile{}
|
||||
second := Default()
|
||||
if second.Concurrency.StageWorkers["extract"] != 1 || len(second.Concurrency.StageWorkers) != 1 || len(second.Pipelines) != 0 {
|
||||
t.Fatalf("Default() returned state shared with an earlier result: %#v", second)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileConfigMinimalVersion3AppliesOverDefaults(t *testing.T) {
|
||||
file := parseFileConfig(t, "version: 3\n")
|
||||
cfg := Default()
|
||||
if err := cfg.ApplyFileConfig(file); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.Output.Directory != "./notarius-output" || cfg.Debug.Directory != "./notarius-debug" || cfg.Cache.ChunkPlans.Mode != pipeline.ChunkCacheAuto {
|
||||
t.Fatalf("minimal file changed unrelated defaults: %#v", cfg)
|
||||
}
|
||||
if cfg.Concurrency.TotalLLM != 1 || cfg.Concurrency.StageWorkers["extract"] != 1 || len(cfg.Pipelines) != 0 {
|
||||
t.Fatalf("minimal file did not retain defaults: %#v", cfg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileConfigMissingVersionIsReportedBeforeFieldDecoding(t *testing.T) {
|
||||
_, err := ParseFileConfigYAML([]byte("workspace:\n directory: /tmp/old\n"))
|
||||
if err == nil || !strings.Contains(err.Error(), "config version is required") {
|
||||
t.Fatalf("missing version error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileConfigRejectsUnknownCurrentAndRemovedFields(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
yaml string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "removed diagnostics",
|
||||
yaml: "version: 3\ndiagnostics: {}\n",
|
||||
want: "field diagnostics not found",
|
||||
},
|
||||
{
|
||||
name: "removed llm profiles",
|
||||
yaml: "version: 3\nllm_profiles: {}\n",
|
||||
want: "field llm_profiles not found",
|
||||
},
|
||||
{
|
||||
name: "pipeline field",
|
||||
yaml: "version: 3\npipelines:\n main:\n unknown: true\n",
|
||||
want: "field unknown not found",
|
||||
},
|
||||
{
|
||||
name: "lane field",
|
||||
yaml: "version: 3\npipelines:\n main:\n artifacts:\n spells:\n unknown: true\n",
|
||||
want: "field unknown not found",
|
||||
},
|
||||
{
|
||||
name: "module binding field",
|
||||
yaml: "version: 3\npipelines:\n main:\n input:\n module: seriatim\n unknown: true\n",
|
||||
want: "field unknown not found in module binding",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, err := ParseFileConfigYAML([]byte(tt.yaml))
|
||||
if err == nil || !strings.Contains(err.Error(), tt.want) {
|
||||
t.Fatalf("error = %v, want context %q", err, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileConfigModuleBindingsPreserveFormsAndValidatorPresence(t *testing.T) {
|
||||
cfg := applyFileConfig(t, `version: 3
|
||||
pipelines:
|
||||
main:
|
||||
input: seriatim
|
||||
chunk:
|
||||
module: generic
|
||||
llm_profile: chunk-profile
|
||||
retries: 2
|
||||
options:
|
||||
max_units: 25
|
||||
references:
|
||||
glossary: ./glossary.md
|
||||
validators: []
|
||||
artifacts:
|
||||
spells:
|
||||
extract:
|
||||
module: dnd/spells
|
||||
options:
|
||||
nested:
|
||||
enabled: true
|
||||
merge: appendorder
|
||||
normalize: noop
|
||||
`)
|
||||
profile := cfg.Pipelines["main"]
|
||||
if profile.Input.Module != "seriatim" || profile.Input.Validators.Set {
|
||||
t.Fatalf("shorthand binding = %#v", profile.Input)
|
||||
}
|
||||
if profile.Chunk.Module != "generic" || profile.Chunk.LLMProfile != "chunk-profile" || profile.Chunk.Retries != 2 ||
|
||||
!reflect.DeepEqual(profile.Chunk.Options, map[string]any{"max_units": 25}) ||
|
||||
!reflect.DeepEqual(profile.Chunk.References, map[string]string{"glossary": "./glossary.md"}) {
|
||||
t.Fatalf("object binding = %#v", profile.Chunk)
|
||||
}
|
||||
if !profile.Chunk.Validators.Set || len(profile.Chunk.Validators.Validators) != 0 {
|
||||
t.Fatalf("explicit empty validators = %#v", profile.Chunk.Validators)
|
||||
}
|
||||
if profile.Artifacts["spells"].Extract.Module != "dnd/spells" ||
|
||||
!reflect.DeepEqual(profile.Artifacts["spells"].Extract.Options, map[string]any{
|
||||
"nested": map[string]any{"enabled": true},
|
||||
}) {
|
||||
t.Fatalf("extract binding = %#v", profile.Artifacts["spells"].Extract)
|
||||
}
|
||||
if profile.Artifacts["spells"].Merge.Module != "appendorder" || profile.Artifacts["spells"].Normalize.Module != "noop" {
|
||||
t.Fatalf("stage shorthand bindings = %#v", profile.Artifacts["spells"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileConfigReferencePrecedenceIsRetained(t *testing.T) {
|
||||
cfg := applyFileConfig(t, `version: 3
|
||||
pipelines:
|
||||
main:
|
||||
input: seriatim
|
||||
references:
|
||||
pipeline-only: ./pipeline.txt
|
||||
shared: ./pipeline-shared.txt
|
||||
chunk:
|
||||
module: generic
|
||||
references:
|
||||
chunk-only: ./chunk.txt
|
||||
artifacts:
|
||||
spells:
|
||||
references:
|
||||
lane-only: ./lane.txt
|
||||
shared: ./lane-shared.txt
|
||||
overridden: ./lane.txt
|
||||
extract:
|
||||
module: dnd/spells
|
||||
references:
|
||||
extract-only: ./extract.txt
|
||||
overridden: ./extract-overridden.txt
|
||||
merge:
|
||||
module: appendorder
|
||||
references:
|
||||
merge-only: ./merge.txt
|
||||
normalize:
|
||||
module: noop
|
||||
references:
|
||||
normalize-only: ./normalize.txt
|
||||
`)
|
||||
profile := cfg.Pipelines["main"]
|
||||
if !reflect.DeepEqual(profile.References, map[string]string{
|
||||
"pipeline-only": "./pipeline.txt",
|
||||
"shared": "./pipeline-shared.txt",
|
||||
}) {
|
||||
t.Fatalf("pipeline references = %#v", profile.References)
|
||||
}
|
||||
if !reflect.DeepEqual(profile.Chunk.References, map[string]string{"chunk-only": "./chunk.txt"}) {
|
||||
t.Fatalf("chunk references = %#v", profile.Chunk.References)
|
||||
}
|
||||
lane := profile.Artifacts["spells"]
|
||||
if !reflect.DeepEqual(lane.References, map[string]string{
|
||||
"lane-only": "./lane.txt",
|
||||
"shared": "./lane-shared.txt",
|
||||
"overridden": "./lane.txt",
|
||||
}) {
|
||||
t.Fatalf("lane compatibility references = %#v", lane.References)
|
||||
}
|
||||
if !reflect.DeepEqual(lane.Extract.References, map[string]string{
|
||||
"lane-only": "./lane.txt",
|
||||
"shared": "./lane-shared.txt",
|
||||
"overridden": "./extract-overridden.txt",
|
||||
"extract-only": "./extract.txt",
|
||||
}) {
|
||||
t.Fatalf("extract references = %#v", lane.Extract.References)
|
||||
}
|
||||
if !reflect.DeepEqual(lane.Merge.References, map[string]string{"merge-only": "./merge.txt"}) ||
|
||||
!reflect.DeepEqual(lane.Normalize.References, map[string]string{"normalize-only": "./normalize.txt"}) {
|
||||
t.Fatalf("merge/normalize references = %#v, %#v", lane.Merge.References, lane.Normalize.References)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileConfigStageLocalValidatorsPreserveOrderAndFields(t *testing.T) {
|
||||
cfg := applyFileConfig(t, `version: 3
|
||||
pipelines:
|
||||
main:
|
||||
input: seriatim
|
||||
chunk:
|
||||
module: generic
|
||||
validators:
|
||||
- generic/always_accept
|
||||
- module: generic/valid_json
|
||||
llm_profile: validator-profile
|
||||
options:
|
||||
schema: compact
|
||||
artifacts:
|
||||
spells:
|
||||
extract:
|
||||
module: dnd/spells
|
||||
validators:
|
||||
- module: extract/dnd/spells/shape
|
||||
options:
|
||||
strict: true
|
||||
merge:
|
||||
module: appendorder
|
||||
validators:
|
||||
- generic/always_accept
|
||||
normalize:
|
||||
module: noop
|
||||
validators:
|
||||
- module: generic/valid_json
|
||||
options:
|
||||
mode: normalized
|
||||
`)
|
||||
profile := cfg.Pipelines["main"]
|
||||
chunkValidators := profile.Chunk.Validators.Validators
|
||||
if !profile.Chunk.Validators.Set || len(chunkValidators) != 2 || chunkValidators[0].Module != "generic/always_accept" ||
|
||||
chunkValidators[1].Module != "generic/valid_json" || chunkValidators[1].LLMProfile != "validator-profile" ||
|
||||
!reflect.DeepEqual(chunkValidators[1].Options, map[string]any{"schema": "compact"}) {
|
||||
t.Fatalf("chunk validators = %#v", profile.Chunk.Validators)
|
||||
}
|
||||
lane := profile.Artifacts["spells"]
|
||||
if len(lane.Extract.Validators.Validators) != 1 || lane.Extract.Validators.Validators[0].Module != "extract/dnd/spells/shape" ||
|
||||
!reflect.DeepEqual(lane.Extract.Validators.Validators[0].Options, map[string]any{"strict": true}) {
|
||||
t.Fatalf("extract validators = %#v", lane.Extract.Validators)
|
||||
}
|
||||
if len(lane.Merge.Validators.Validators) != 1 || lane.Merge.Validators.Validators[0].Module != "generic/always_accept" {
|
||||
t.Fatalf("merge validators = %#v", lane.Merge.Validators)
|
||||
}
|
||||
if len(lane.Normalize.Validators.Validators) != 1 || lane.Normalize.Validators.Validators[0].Module != "generic/valid_json" ||
|
||||
!reflect.DeepEqual(lane.Normalize.Validators.Validators[0].Options, map[string]any{"mode": "normalized"}) {
|
||||
t.Fatalf("normalize validators = %#v", lane.Normalize.Validators)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileConfigStateSectionsApplyIndependently(t *testing.T) {
|
||||
cfg := applyFileConfig(t, `version: 3
|
||||
scriptorium:
|
||||
profile_dir: ./profiles
|
||||
concurrency:
|
||||
total_llm: 7
|
||||
output:
|
||||
directory: ./output
|
||||
cache:
|
||||
chunk_plans:
|
||||
directory: ./plans
|
||||
mode: bypass
|
||||
checkpoints:
|
||||
directory: ./checkpoints
|
||||
debug:
|
||||
directory: ./debug
|
||||
`)
|
||||
if cfg.Scriptorium.ProfileDir != "./profiles" || cfg.Scriptorium.ProfileFile != "" {
|
||||
t.Fatalf("scriptorium = %#v", cfg.Scriptorium)
|
||||
}
|
||||
if cfg.Concurrency.TotalLLM != 7 || cfg.Concurrency.StageWorkers["extract"] != 7 {
|
||||
t.Fatalf("concurrency = %#v", cfg.Concurrency)
|
||||
}
|
||||
if cfg.Output.Directory != "./output" || cfg.Cache.ChunkPlans.Directory != "plans" || cfg.Cache.ChunkPlans.Mode != pipeline.ChunkCacheBypass ||
|
||||
cfg.Cache.Checkpoints.Directory != "checkpoints" || cfg.Debug.Directory != "./debug" {
|
||||
t.Fatalf("state sections = %#v, %#v, %#v, %#v", cfg.Output, cfg.Cache, cfg.Debug, cfg.Scriptorium)
|
||||
}
|
||||
if cfg.Output.Directory == cfg.Cache.ChunkPlans.Directory || cfg.Cache.ChunkPlans.Directory == cfg.Cache.Checkpoints.Directory || cfg.Cache.Checkpoints.Directory == cfg.Debug.Directory {
|
||||
t.Fatal("state roots were coupled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileConfigRejectsTrimmedKeyCollisions(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
yaml string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "pipeline ids",
|
||||
yaml: "version: 3\npipelines:\n main: {}\n ' main ': {}\n",
|
||||
want: "pipeline id \"main\" is duplicated after trimming",
|
||||
},
|
||||
{
|
||||
name: "lane ids",
|
||||
yaml: "version: 3\npipelines:\n main:\n artifacts:\n spells: {}\n ' spells ': {}\n",
|
||||
want: "artifact lane id \"spells\" is duplicated after trimming",
|
||||
},
|
||||
{
|
||||
name: "reference slots",
|
||||
yaml: "version: 3\npipelines:\n main:\n references:\n slot: ./one.txt\n ' slot ': ./two.txt\n",
|
||||
want: "reference slot \"slot\" is duplicated after trimming",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
file := parseFileConfig(t, tt.yaml)
|
||||
cfg := Default()
|
||||
err := cfg.ApplyFileConfig(file)
|
||||
if err == nil || !strings.Contains(err.Error(), tt.want) {
|
||||
t.Fatalf("ApplyFileConfig() error = %v, want context %q", err, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileConfigReportsPathAndOperationContext(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
missing := filepath.Join(dir, "missing.yml")
|
||||
_, err := LoadFileConfig(missing)
|
||||
if err == nil || !strings.Contains(err.Error(), "read config file") || !strings.Contains(err.Error(), missing) {
|
||||
t.Fatalf("missing-file error = %v", err)
|
||||
}
|
||||
|
||||
malformed := filepath.Join(dir, "malformed.yml")
|
||||
if err := os.WriteFile(malformed, []byte("version: [\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = LoadFileConfig(malformed)
|
||||
if err == nil || !strings.Contains(err.Error(), "parse config file") || !strings.Contains(err.Error(), malformed) {
|
||||
t.Fatalf("malformed-file error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func parseFileConfig(t *testing.T, source string) FileConfig {
|
||||
t.Helper()
|
||||
file, err := ParseFileConfigYAML([]byte(source))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML() error = %v", err)
|
||||
}
|
||||
return file
|
||||
}
|
||||
|
||||
func applyFileConfig(t *testing.T, source string) Config {
|
||||
t.Helper()
|
||||
cfg := Default()
|
||||
if err := cfg.ApplyFileConfig(parseFileConfig(t, source)); err != nil {
|
||||
t.Fatalf("ApplyFileConfig() error = %v", err)
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
Reference in New Issue
Block a user