diff --git a/decode.go b/decode.go new file mode 100644 index 0000000..86f00cd --- /dev/null +++ b/decode.go @@ -0,0 +1,455 @@ +package configkit + +import ( + "encoding" + "math" + "reflect" + "regexp" + "strconv" + "time" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" +) + +var integerPattern = regexp.MustCompile(`^-?(0|[1-9][0-9]*)$`) +var unsignedIntegerPattern = regexp.MustCompile(`^(0|[1-9][0-9]*)$`) + +type decodeIssue struct { + code Code + path Path + message string + location document.Location +} + +type inputPresence struct { + null bool + location document.Location +} + +type decodedConfiguration struct { + value reflect.Value + presence map[string]inputPresence + origins map[string]Origin + baseline *document.Node + final *document.Node +} + +func decodeConfiguration(schema *schemaDescriptor, defaults reflect.Value, input *document.Node) (decodedConfiguration, []decodeIssue) { + if schema == nil || schema.root == nil || !defaults.IsValid() || defaults.Type() != schema.root.typeOf { + return decodedConfiguration{}, []decodeIssue{{code: CodeSchemaUnsupportedType, message: "configuration defaults do not match the schema"}} + } + baseline, projectionIssue := projectConfiguration(schema, defaults, defaultProjectionLimits()) + if projectionIssue != nil { + return decodedConfiguration{}, []decodeIssue{decodeIssueFromProjection(projectionIssue)} + } + working, cloneIssue := cloneConfiguration(schema, defaults, defaultProjectionLimits()) + if cloneIssue != nil { + return decodedConfiguration{}, []decodeIssue{decodeIssueFromProjection(cloneIssue)} + } + + presence := make(map[string]inputPresence) + if input != nil { + collectPresence(input, Path{}, presence) + decoded, issues := decodeValue(input, working, schema.root, Path{}, schema) + if len(issues) != 0 { + return decodedConfiguration{}, issues + } + working = decoded + } + final, projectionIssue := projectConfiguration(schema, working, defaultProjectionLimits()) + if projectionIssue != nil { + return decodedConfiguration{}, []decodeIssue{decodeIssueFromProjection(projectionIssue)} + } + origins := make(map[string]Origin) + collectOrigins(final, Path{}, presence, origins) + return decodedConfiguration{ + value: working, + presence: presence, + origins: origins, + baseline: baseline, + final: final, + }, nil +} + +func decodeValue(node *document.Node, base reflect.Value, descriptor *typeDescriptor, path Path, schema *schemaDescriptor) (reflect.Value, []decodeIssue) { + if node.Kind == document.Null { + switch descriptor.kind { + case descriptorPointer, descriptorMap, descriptorSlice, descriptorDynamic: + return reflect.Zero(descriptor.typeOf), nil + default: + return reflect.Value{}, []decodeIssue{newDecodeIssue(CodeDecodeNull, path, node, "null is not allowed for this configuration type")} + } + } + if descriptor.kind == descriptorPointer { + var elementBase reflect.Value + if base.IsNil() { + elementBase = reflect.Zero(descriptor.typeOf.Elem()) + } else { + elementBase = base.Elem() + } + decoded, issues := decodeValue(node, elementBase, descriptor.element, path, schema) + if len(issues) != 0 { + return reflect.Value{}, issues + } + result := reflect.New(descriptor.typeOf.Elem()) + result.Elem().Set(decoded) + return result, nil + } + + switch descriptor.kind { + case descriptorStruct: + return decodeStruct(node, base, descriptor, path, schema) + case descriptorMap: + return decodeMap(node, base, descriptor, path, schema) + case descriptorSlice: + return decodeSlice(node, descriptor, path, schema) + case descriptorArray: + return decodeArray(node, descriptor, path, schema) + case descriptorDynamic: + return decodeDynamic(node, base, path, schema) + case descriptorString: + if node.Kind != document.String { + return typeMismatch(path, node, "string") + } + result := reflect.New(descriptor.typeOf).Elem() + result.SetString(node.Text) + return result, nil + case descriptorBoolean: + if node.Kind != document.Boolean { + return typeMismatch(path, node, "boolean") + } + result := reflect.New(descriptor.typeOf).Elem() + result.SetBool(node.Text == "true") + return result, nil + case descriptorSigned: + if node.Kind != document.Number || !integerPattern.MatchString(node.Text) { + return typeMismatch(path, node, "integer") + } + value, err := strconv.ParseInt(node.Text, 10, descriptor.typeOf.Bits()) + if err != nil { + return rangeFailure(path, node, "integer") + } + result := reflect.New(descriptor.typeOf).Elem() + result.SetInt(value) + return result, nil + case descriptorUnsigned: + if node.Kind != document.Number || !unsignedIntegerPattern.MatchString(node.Text) { + return typeMismatch(path, node, "unsigned integer") + } + value, err := strconv.ParseUint(node.Text, 10, descriptor.typeOf.Bits()) + if err != nil { + return rangeFailure(path, node, "unsigned integer") + } + result := reflect.New(descriptor.typeOf).Elem() + result.SetUint(value) + return result, nil + case descriptorFloat: + if node.Kind != document.Number { + return typeMismatch(path, node, "number") + } + value, err := strconv.ParseFloat(node.Text, descriptor.typeOf.Bits()) + if err != nil || math.IsNaN(value) || math.IsInf(value, 0) { + return rangeFailure(path, node, "number") + } + result := reflect.New(descriptor.typeOf).Elem() + result.SetFloat(value) + return result, nil + case descriptorDuration: + if node.Kind != document.String { + return typeMismatch(path, node, "duration string") + } + value, err := time.ParseDuration(node.Text) + if err != nil { + return reflect.Value{}, []decodeIssue{newDecodeIssue(CodeDecodeDuration, path, node, "duration string is invalid")} + } + result := reflect.New(descriptor.typeOf).Elem() + result.SetInt(int64(value)) + return result, nil + case descriptorText: + if node.Kind != document.String { + return typeMismatch(path, node, "text string") + } + result := reflect.New(descriptor.typeOf).Elem() + unmarshaler := result.Addr().Interface().(encoding.TextUnmarshaler) + if err := unmarshaler.UnmarshalText([]byte(node.Text)); err != nil { + return reflect.Value{}, []decodeIssue{newDecodeIssue(CodeDecodeText, path, node, "text value is invalid for its configuration type")} + } + return result, nil + default: + return reflect.Value{}, []decodeIssue{newDecodeIssue(CodeSchemaUnsupportedType, path, node, "configuration type is not supported")} + } +} + +func decodeStruct(node *document.Node, base reflect.Value, descriptor *typeDescriptor, path Path, schema *schemaDescriptor) (reflect.Value, []decodeIssue) { + if node.Kind != document.Mapping { + return typeMismatch(path, node, "mapping") + } + result := reflect.New(descriptor.typeOf).Elem() + result.Set(base) + var issues []decodeIssue + for _, entry := range node.Mapping { + childPath := path.Key(entry.Key) + if position, ok := descriptor.fieldByName[entry.Key]; ok { + field := descriptor.fields[position] + target, ok := fieldForDecode(result, field.index, true) + if !ok { + issues = append(issues, decodeIssue{code: CodeSchemaUnsupportedType, path: childPath, message: "configuration field could not be addressed", location: entry.ValueLocation}) + continue + } + decoded, fieldIssues := decodeValue(entry.Value, target, field.descriptor, childPath, schema) + if len(fieldIssues) != 0 { + issues = append(issues, fieldIssues...) + continue + } + target.Set(decoded) + continue + } + if descriptor.inlineMap != nil { + if issue := decodeInlineMapEntry(result, *descriptor.inlineMap, entry, childPath, schema); issue != nil { + issues = append(issues, issue...) + } + continue + } + issues = append(issues, decodeIssue{code: CodeDecodeUnknownField, path: childPath, message: "configuration field is not declared", location: entry.KeyLocation}) + } + return result, issues +} + +func decodeInlineMapEntry(target reflect.Value, field fieldDescriptor, entry document.Entry, path Path, schema *schemaDescriptor) []decodeIssue { + mapValue, ok := fieldForDecode(target, field.index, true) + if !ok || mapValue.Kind() != reflect.Map { + return []decodeIssue{{code: CodeSchemaInline, path: path, message: "inline map could not be addressed", location: entry.KeyLocation}} + } + if mapValue.IsNil() { + mapValue.Set(reflect.MakeMap(mapValue.Type())) + } + key := reflect.New(mapValue.Type().Key()).Elem() + key.SetString(entry.Key) + base := mapValue.MapIndex(key) + if !base.IsValid() { + base = reflect.Zero(mapValue.Type().Elem()) + } + decoded, issues := decodeValue(entry.Value, base, field.descriptor.element, path, schema) + if len(issues) == 0 { + mapValue.SetMapIndex(key, decoded) + } + return issues +} + +func decodeMap(node *document.Node, base reflect.Value, descriptor *typeDescriptor, path Path, schema *schemaDescriptor) (reflect.Value, []decodeIssue) { + if node.Kind != document.Mapping { + return typeMismatch(path, node, "mapping") + } + result := base + if result.IsNil() { + result = reflect.MakeMap(descriptor.typeOf) + } + var issues []decodeIssue + for _, entry := range node.Mapping { + key := reflect.New(descriptor.typeOf.Key()).Elem() + key.SetString(entry.Key) + entryBase := result.MapIndex(key) + if !entryBase.IsValid() { + entryBase = reflect.Zero(descriptor.typeOf.Elem()) + } + decoded, entryIssues := decodeValue(entry.Value, entryBase, descriptor.element, path.Key(entry.Key), schema) + if len(entryIssues) != 0 { + issues = append(issues, entryIssues...) + continue + } + result.SetMapIndex(key, decoded) + } + return result, issues +} + +func decodeSlice(node *document.Node, descriptor *typeDescriptor, path Path, schema *schemaDescriptor) (reflect.Value, []decodeIssue) { + if node.Kind != document.Sequence { + return typeMismatch(path, node, "sequence") + } + result := reflect.MakeSlice(descriptor.typeOf, len(node.Sequence), len(node.Sequence)) + var issues []decodeIssue + for i, child := range node.Sequence { + decoded, childIssues := decodeValue(child, reflect.Zero(descriptor.typeOf.Elem()), descriptor.element, path.Index(i), schema) + if len(childIssues) != 0 { + issues = append(issues, childIssues...) + continue + } + result.Index(i).Set(decoded) + } + return result, issues +} + +func decodeArray(node *document.Node, descriptor *typeDescriptor, path Path, schema *schemaDescriptor) (reflect.Value, []decodeIssue) { + if node.Kind != document.Sequence { + return typeMismatch(path, node, "sequence") + } + if len(node.Sequence) != descriptor.length { + return reflect.Value{}, []decodeIssue{newDecodeIssue(CodeDecodeArrayLength, path, node, "configuration array has the wrong length")} + } + result := reflect.New(descriptor.typeOf).Elem() + var issues []decodeIssue + for i, child := range node.Sequence { + decoded, childIssues := decodeValue(child, reflect.Zero(descriptor.typeOf.Elem()), descriptor.element, path.Index(i), schema) + if len(childIssues) != 0 { + issues = append(issues, childIssues...) + continue + } + result.Index(i).Set(decoded) + } + return result, issues +} + +func decodeDynamic(node *document.Node, base reflect.Value, path Path, schema *schemaDescriptor) (reflect.Value, []decodeIssue) { + if node.Kind == document.Mapping && base.IsValid() && !base.IsNil() { + concrete := base.Elem() + operation := valueOperation{schema: schema} + descriptor, issue := operation.dynamicDescriptor(concrete.Type(), path) + if issue == nil && acceptsMapping(descriptor) { + decoded, issues := decodeValue(node, concrete, descriptor, path, schema) + if len(issues) != 0 { + return reflect.Value{}, issues + } + result := reflect.New(base.Type()).Elem() + result.Set(decoded) + return result, nil + } + } + var value any + switch node.Kind { + case document.Null: + return reflect.Zero(base.Type()), nil + case document.String: + value = node.Text + case document.Boolean: + value = node.Text == "true" + case document.Number: + if integerPattern.MatchString(node.Text) { + parsed, err := strconv.ParseInt(node.Text, 10, 64) + if err != nil { + return rangeFailure(path, node, "dynamic integer") + } + value = parsed + } else { + parsed, err := strconv.ParseFloat(node.Text, 64) + if err != nil || math.IsNaN(parsed) || math.IsInf(parsed, 0) { + return rangeFailure(path, node, "dynamic number") + } + value = parsed + } + case document.Sequence: + sequence := make([]any, len(node.Sequence)) + var issues []decodeIssue + for i, child := range node.Sequence { + decoded, childIssues := decodeDynamic(child, reflect.Zero(reflect.TypeFor[any]()), path.Index(i), schema) + if len(childIssues) != 0 { + issues = append(issues, childIssues...) + continue + } + sequence[i] = decoded.Interface() + } + if len(issues) != 0 { + return reflect.Value{}, issues + } + value = sequence + case document.Mapping: + mapping := make(map[string]any, len(node.Mapping)) + var issues []decodeIssue + for _, entry := range node.Mapping { + decoded, childIssues := decodeDynamic(entry.Value, reflect.Zero(reflect.TypeFor[any]()), path.Key(entry.Key), schema) + if len(childIssues) != 0 { + issues = append(issues, childIssues...) + continue + } + mapping[entry.Key] = decoded.Interface() + } + if len(issues) != 0 { + return reflect.Value{}, issues + } + value = mapping + default: + return reflect.Value{}, []decodeIssue{newDecodeIssue(CodeDecodeType, path, node, "dynamic configuration value has an unsupported kind")} + } + result := reflect.New(base.Type()).Elem() + if value != nil { + result.Set(reflect.ValueOf(value)) + } + return result, nil +} + +func acceptsMapping(descriptor *typeDescriptor) bool { + for descriptor != nil && descriptor.kind == descriptorPointer { + descriptor = descriptor.element + } + return descriptor != nil && (descriptor.kind == descriptorStruct || descriptor.kind == descriptorMap || descriptor.kind == descriptorDynamic) +} + +func fieldForDecode(root reflect.Value, index []int, allocate bool) (reflect.Value, bool) { + current := root + for position, fieldIndex := range index { + if current.Kind() != reflect.Struct { + return reflect.Value{}, false + } + current = current.Field(fieldIndex) + if position == len(index)-1 { + return current, current.CanSet() + } + for current.Kind() == reflect.Pointer { + if current.IsNil() { + if !allocate || !current.CanSet() { + return reflect.Value{}, false + } + current.Set(reflect.New(current.Type().Elem())) + } + current = current.Elem() + } + } + return reflect.Value{}, false +} + +func collectPresence(node *document.Node, path Path, presence map[string]inputPresence) { + presence[path.String()] = inputPresence{null: node.Kind == document.Null, location: node.Location} + switch node.Kind { + case document.Mapping: + for _, entry := range node.Mapping { + collectPresence(entry.Value, path.Key(entry.Key), presence) + } + case document.Sequence: + for i, child := range node.Sequence { + collectPresence(child, path.Index(i), presence) + } + } +} + +func collectOrigins(node *document.Node, path Path, presence map[string]inputPresence, origins map[string]Origin) { + if input, ok := presence[path.String()]; ok { + origins[path.String()] = Origin{Layer: LayerBase, Locations: []Location{{SourceID: input.location.SourceID, Line: input.location.Line, Column: input.location.Column}}} + } else { + origins[path.String()] = Origin{Layer: LayerDefault} + } + switch node.Kind { + case document.Mapping: + for _, entry := range node.Mapping { + collectOrigins(entry.Value, path.Key(entry.Key), presence, origins) + } + case document.Sequence: + for i, child := range node.Sequence { + collectOrigins(child, path.Index(i), presence, origins) + } + } +} + +func decodeIssueFromProjection(issue *projectionIssue) decodeIssue { + return decodeIssue{code: issue.code, path: issue.path.clone(), message: issue.message} +} + +func newDecodeIssue(code Code, path Path, node *document.Node, message string) decodeIssue { + return decodeIssue{code: code, path: path.clone(), message: message, location: node.Location} +} + +func typeMismatch(path Path, node *document.Node, expected string) (reflect.Value, []decodeIssue) { + return reflect.Value{}, []decodeIssue{newDecodeIssue(CodeDecodeType, path, node, "configuration value must be a "+expected)} +} + +func rangeFailure(path Path, node *document.Node, expected string) (reflect.Value, []decodeIssue) { + return reflect.Value{}, []decodeIssue{newDecodeIssue(CodeDecodeRange, path, node, expected+" is outside the supported range")} +} diff --git a/decode_test.go b/decode_test.go new file mode 100644 index 0000000..351b86b --- /dev/null +++ b/decode_test.go @@ -0,0 +1,312 @@ +package configkit + +import ( + "errors" + "math" + "reflect" + "strings" + "testing" + "time" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" +) + +type namedSmall int8 + +type validatedText string + +func (value validatedText) MarshalText() ([]byte, error) { + return []byte(value), nil +} + +func (value *validatedText) UnmarshalText(text []byte) error { + if string(text) == "bad-secret-canary" { + return errors.New("CANARY_TEXT_ERROR: bad-secret-canary") + } + *value = validatedText(text) + return nil +} + +func TestDecodeAppliesInputOverDetachedDefaults(t *testing.T) { + t.Parallel() + + type nested struct { + A string `yaml:"a"` + B string `yaml:"b"` + } + type config struct { + Enabled bool `yaml:"enabled"` + Workers int `yaml:"workers"` + Label string `yaml:"label"` + Nested nested `yaml:"nested"` + EmptyMapping nested `yaml:"empty_mapping"` + Objects map[string]nested `yaml:"objects"` + Sequence []string `yaml:"sequence"` + Replacement []string `yaml:"replacement"` + Pointer *nested `yaml:"pointer"` + NilPointer *nested `yaml:"nil_pointer"` + Nullable map[string]string `yaml:"nullable"` + Dynamic any `yaml:"dynamic"` + DynamicShape any `yaml:"dynamic_shape"` + DynamicNull any `yaml:"dynamic_null"` + Fixed [2]int `yaml:"fixed"` + Small namedSmall `yaml:"small"` + Delay time.Duration `yaml:"delay"` + Custom validatedText `yaml:"custom"` + } + defaults := config{ + Enabled: true, + Workers: 4, + Label: "default", + Nested: nested{A: "keep", B: "before"}, + EmptyMapping: nested{A: "keep-empty", B: "keep-empty"}, + Objects: map[string]nested{ + "existing": {A: "keep-map", B: "before"}, + }, + Sequence: []string{"default"}, + Replacement: []string{"default"}, + Pointer: &nested{A: "keep-pointer", B: "before"}, + Nullable: map[string]string{"default": "value"}, + Dynamic: map[string]any{"keep": "default", "count": int64(4)}, + DynamicShape: nested{A: "keep-dynamic", B: "before"}, + DynamicNull: "default", + Fixed: [2]int{1, 2}, + Small: 7, + Delay: time.Second, + Custom: "default", + } + input := ` +enabled: false +workers: 0 +label: "" +nested: {b: changed} +empty_mapping: {} +objects: + existing: {b: changed} + new: {b: added} +sequence: [] +replacement: [one, two] +pointer: {b: changed} +nil_pointer: {b: added} +nullable: null +dynamic: {count: 0, fraction: 1.5} +dynamic_shape: {b: changed} +dynamic_null: null +fixed: [3, 4] +small: -8 +delay: 1.5s +custom: accepted +` + schema := mustSchemaFor[config](t) + documentRoot := parseDocumentForDecode(t, input) + decoded, issues := decodeConfiguration(schema, reflect.ValueOf(defaults), documentRoot) + if len(issues) != 0 { + t.Fatalf("decode issues = %#v", issues) + } + got := decoded.value.Interface().(config) + if got.Enabled || got.Workers != 0 || got.Label != "" { + t.Fatalf("explicit zero values lost: %#v", got) + } + if got.Nested != (nested{A: "keep", B: "changed"}) || got.EmptyMapping != defaults.EmptyMapping { + t.Fatalf("mapping defaults = %#v / %#v", got.Nested, got.EmptyMapping) + } + if got.Objects["existing"] != (nested{A: "keep-map", B: "changed"}) || got.Objects["new"] != (nested{B: "added"}) { + t.Fatalf("map defaults = %#v", got.Objects) + } + if got.Sequence == nil || len(got.Sequence) != 0 || !reflect.DeepEqual(got.Replacement, []string{"one", "two"}) { + t.Fatalf("sequence replacement = %#v / %#v", got.Sequence, got.Replacement) + } + if got.Pointer == defaults.Pointer || *got.Pointer != (nested{A: "keep-pointer", B: "changed"}) || got.NilPointer == nil || *got.NilPointer != (nested{B: "added"}) { + t.Fatalf("pointer defaults = %#v / %#v", got.Pointer, got.NilPointer) + } + if got.Nullable != nil || got.DynamicNull != nil { + t.Fatalf("null values = %#v / %#v", got.Nullable, got.DynamicNull) + } + dynamic := got.Dynamic.(map[string]any) + if dynamic["keep"] != "default" || dynamic["count"] != int64(0) || dynamic["fraction"] != float64(1.5) { + t.Fatalf("dynamic mapping = %#v", dynamic) + } + if got.DynamicShape != (nested{A: "keep-dynamic", B: "changed"}) { + t.Fatalf("dynamic concrete mapping = %#v", got.DynamicShape) + } + if got.Fixed != [2]int{3, 4} || got.Small != -8 || got.Delay != 1500*time.Millisecond || got.Custom != "accepted" { + t.Fatalf("scalar decoding = %#v", got) + } + + for _, path := range []Path{Keys(), Keys("enabled"), Keys("nested"), Keys("nested", "b"), Keys("empty_mapping")} { + if _, ok := decoded.presence[path.String()]; !ok { + t.Errorf("explicit path %q missing", path.String()) + } + } + if _, ok := decoded.presence[Keys("nested", "a").String()]; ok { + t.Fatal("parent mapping made omitted child explicit") + } + if decoded.origins[Keys("nested", "a").String()].Layer != LayerDefault || decoded.origins[Keys("nested", "b").String()].Layer != LayerBase { + t.Fatalf("origins = %#v", decoded.origins) + } + if decoded.origins[Keys("nested", "b").String()].Locations[0].Line == 0 { + t.Fatal("input origin lost source coordinates") + } + + defaults.Objects["existing"] = nested{A: "mutated"} + defaults.Pointer.B = "mutated" + defaults.Sequence[0] = "mutated" + if got.Objects["existing"].A != "keep-map" || got.Pointer.B != "changed" || len(got.Sequence) != 0 { + t.Fatal("decoded value aliases defaults") + } + got.Pointer.B = "caller mutation" + if mappingValue(t, mappingValue(t, decoded.final, "pointer"), "b").Text != "changed" { + t.Fatal("final snapshot aliases returned typed value") + } + if mappingValue(t, mappingValue(t, decoded.baseline, "nested"), "b").Text != "before" { + t.Fatal("baseline snapshot was mutated by decode") + } +} + +func TestDecodeDefaultsOnlyAndRepeatedResultsAreIndependent(t *testing.T) { + t.Parallel() + + type config struct { + Values map[string][]string `yaml:"values"` + } + defaults := config{Values: map[string][]string{"one": {"a"}}} + schema := mustSchemaFor[config](t) + first, issues := decodeConfiguration(schema, reflect.ValueOf(defaults), nil) + if len(issues) != 0 { + t.Fatal(issues) + } + second, issues := decodeConfiguration(schema, reflect.ValueOf(defaults), nil) + if len(issues) != 0 { + t.Fatal(issues) + } + firstValue := first.value.Interface().(config) + secondValue := second.value.Interface().(config) + firstValue.Values["one"][0] = "changed" + if secondValue.Values["one"][0] != "a" || defaults.Values["one"][0] != "a" { + t.Fatal("repeated defaults-only results share aliases") + } + if len(first.presence) != 0 || first.origins[Keys("values", "one").String()].Layer != LayerDefault { + t.Fatalf("defaults-only metadata = %#v / %#v", first.presence, first.origins) + } +} + +func TestDecodeCollectsIndependentTypeAndShapeFailures(t *testing.T) { + t.Parallel() + + type nested struct { + Known int `yaml:"known"` + } + type config struct { + Small int8 `yaml:"small"` + Unsigned uint8 `yaml:"unsigned"` + Number float32 `yaml:"number"` + Text string `yaml:"text"` + Flag bool `yaml:"flag"` + Delay time.Duration `yaml:"delay"` + Fixed [2]int `yaml:"fixed"` + Nested nested `yaml:"nested"` + Custom validatedText `yaml:"custom"` + Dynamic any `yaml:"dynamic"` + } + input := ` +small: 128 +unsigned: -1 +number: 1e999 +text: 3 +flag: "true" +delay: 10 +fixed: [1] +nested: {first_unknown: 1, second_unknown: 2} +custom: bad-secret-canary +dynamic: 9223372036854775808 +` + schema := mustSchemaFor[config](t) + _, issues := decodeConfiguration(schema, reflect.ValueOf(config{Custom: "ok"}), parseDocumentForDecode(t, input)) + if len(issues) != 11 { + t.Fatalf("issues = %d: %#v", len(issues), issues) + } + want := map[string]Code{ + "/small": CodeDecodeRange, + "/unsigned": CodeDecodeType, + "/number": CodeDecodeRange, + "/text": CodeDecodeType, + "/flag": CodeDecodeType, + "/delay": CodeDecodeType, + "/fixed": CodeDecodeArrayLength, + "/nested/first_unknown": CodeDecodeUnknownField, + "/nested/second_unknown": CodeDecodeUnknownField, + "/custom": CodeDecodeText, + "/dynamic": CodeDecodeRange, + } + for _, issue := range issues { + if wantCode, ok := want[issue.path.String()]; !ok || wantCode != issue.code { + t.Errorf("unexpected issue %#v", issue) + } + if issue.location.Line == 0 { + t.Errorf("issue lost input location: %#v", issue) + } + if strings.Contains(issue.message, "CANARY") || strings.Contains(issue.message, "bad-secret") { + t.Errorf("issue exposed value: %#v", issue) + } + } +} + +func TestDecodeNullabilityAndNestedMapStrictness(t *testing.T) { + t.Parallel() + + type nested struct { + Value int `yaml:"value"` + } + type config struct { + Scalar int `yaml:"scalar"` + Object nested `yaml:"object"` + Array [1]int `yaml:"array"` + Map map[string]nested `yaml:"map"` + } + input := "scalar: null\nobject: null\narray: null\nmap:\n item: {unknown: 1}\n" + _, issues := decodeConfiguration(mustSchemaFor[config](t), reflect.ValueOf(config{}), parseDocumentForDecode(t, input)) + want := map[string]Code{ + "/scalar": CodeDecodeNull, + "/object": CodeDecodeNull, + "/array": CodeDecodeNull, + "/map/item/unknown": CodeDecodeUnknownField, + } + if len(issues) != len(want) { + t.Fatalf("issues = %#v", issues) + } + for _, issue := range issues { + if want[issue.path.String()] != issue.code { + t.Errorf("issue = %#v", issue) + } + } +} + +func TestDecodeRejectsInvalidDefaultsBeforeApplyingInput(t *testing.T) { + t.Parallel() + + type config struct { + Name string `yaml:"name"` + Extras map[string]any `yaml:",inline"` + Float float64 `yaml:"float"` + } + schema := mustSchemaFor[config](t) + for name, defaults := range map[string]config{ + "inline collision": {Name: "declared", Extras: map[string]any{"name": "duplicate"}}, + "non-finite": {Float: math.NaN()}, + } { + t.Run(name, func(t *testing.T) { + if _, issues := decodeConfiguration(schema, reflect.ValueOf(defaults), parseDocumentForDecode(t, "{}\n")); len(issues) != 1 || !strings.HasPrefix(string(issues[0].code), "schema.") { + t.Fatalf("issues = %#v", issues) + } + }) + } +} + +func parseDocumentForDecode(t *testing.T, input string) *document.Node { + t.Helper() + parsed, diagnostics := parseSource(Bytes("decode.yml", []byte(input)), "source-1", defaultParseLimits()) + if len(diagnostics) != 0 { + t.Fatalf("parse diagnostics = %#v", diagnostics) + } + return parsed.document +} diff --git a/docs/config.md b/docs/config.md index 27514c4..9479a40 100644 --- a/docs/config.md +++ b/docs/config.md @@ -1,8 +1,9 @@ # Configuration Input Contract -This document owns ConfigKit's currently implemented source-input and YAML syntax -contracts. The parser and owned document representation are private; a public typed -loading lifecycle has not been implemented yet. +This document owns ConfigKit's currently implemented source-input, YAML syntax, +schema, typed-decoding, and ownership contracts. The parser, owned document +representation, and decoder are private; a public typed loading lifecycle has not +been implemented yet. ## Sources @@ -101,8 +102,50 @@ tokens become indices only in a sequence. Sequence indices cannot have leading zeroes; fixed-array indices are bounds checked. Dynamic `any` and inline-map entries remain schema-valid for application-defined descendants. -Descriptors do not yet decode documents. Document decoding will reuse this graph as -its implementation lands. +Descriptors are reused by typed decoding, projection, and cloning so those operations +cannot acquire different field bindings. + +## Defaults and typed decoding + +The private decoder starts from an independently owned clone of the application's +typed defaults and applies only explicitly present input nodes. Defaults therefore +never become explicit, while equal-valued input remains explicit. Successful decode +produces detached typed state, baseline and final canonical trees, an input-presence +index, and default/input origins for the later public loading lifecycle. + +| Configured input | Effect on the typed baseline | +| --- | --- | +| Absent field | Retain the default | +| Explicit false, zero, or empty string | Replace the default | +| Partial mapping or string-keyed map | Recurse and retain omitted default children or entries | +| Empty mapping | Retain existing mapping children | +| Sequence, including an empty sequence | Replace the entire sequence | +| Pointer object | Recurse into a non-nil default; otherwise allocate an object whose omitted children have ordinary Go zero values | +| Allowed null | Replace the value with nil and suppress its default | + +Null is allowed only for pointers, maps, slices, and dynamic `any` values. It is an +error for scalars, structs, and arrays. New map entries and sequence elements begin +from their Go zero values; ConfigKit does not invent entry-level defaults. + +Decoding is strict at every typed depth. Unknown struct fields, type mismatches, +invalid nulls, numeric overflow, and arrays with the wrong length are errors at the +original semantic path and source coordinates. String-keyed maps accept arbitrary +keys but preserve the strict schema of their values; inline maps accept unmatched +keys according to their element descriptor. Multiple independent field failures are +collected without publishing the partially decoded object. + +Booleans and strings are not coerced. Integer destinations require integer syntax +and a value in the destination type's range. Float destinations accept finite JSON +numbers representable by their destination precision. Dynamic numbers become +`int64` for integer syntax and `float64` for fractional or exponent syntax, with +range and finiteness checks. Named scalar types retain their identity. Durations +decode only from strings accepted by `time.ParseDuration`; integers are rejected. +Paired text scalars decode only from strings through `encoding.TextUnmarshaler`, and +codec failures are classified without exposing application error text or input +values. + +This decoder is currently an internal lifecycle foundation, not a public conversion +API. Applications cannot call typed loading until `Load` is introduced. ## Typed projection and ownership diff --git a/docs/internal/extraction.md b/docs/internal/extraction.md index 38e1a4d..3e3c282 100644 --- a/docs/internal/extraction.md +++ b/docs/internal/extraction.md @@ -170,6 +170,19 @@ Assigned across schema, decoding, resolution, validation, and consumer examples: application-owned; every domain mutation is recorded. Campaign/session fallback, search paths, fetching, storage, and concrete default values remain in Narratio. +Implementation record (2026-09-06): ConfigKit generalized Narratio's fresh-default, +strict nested-type, and explicit-presence cases into `decode.go` and +`decode_test.go`. The decoder applies the owned document tree over a detached typed +baseline, treats mappings recursively and sequences atomically, preserves explicit +false/zero/empty/null, rejects unknown nested fields, and returns private presence +and default/input origin artifacts. Unlike Narratio's custom YAML hooks and +pointer/zero inference, presence comes directly from the parsed tree and equal-valued +input remains explicit. ConfigKit also adds destination-width numeric checks, +dynamic `int64`/`float64`, exact arrays, duration strings, paired text codecs, safe +multi-error collection, and repeated-result isolation. Resolution, validation, +consumer defaults, and all campaign/session algorithms remain assigned to later +work. + ### Secrets bootstrap Assigned to the explicit bootstrap operation: diff --git a/docs/internal/overview.md b/docs/internal/overview.md index a5710d9..2d8332c 100644 --- a/docs/internal/overview.md +++ b/docs/internal/overview.md @@ -7,18 +7,21 @@ module and package documentation boundary. Its implemented public primitives are - detached provenance and diagnostic record types with empty `Metadata` queries; - canonical, control-safe diagnostic rendering and structured `LoadError` access; - opaque file, byte, reader, and no-input source construction and bounded reads; -- a parser-independent ordered tree in `internal/document`; and +- a parser-independent ordered tree in `internal/document`; - the sole upstream YAML integration in `internal/yaml`, which translates validated - input into that owned tree and exposes no parser types outside its boundary; and + input into that owned tree and exposes no parser types outside its boundary; - a per-operation private reflection descriptor graph in `schema.go` for supported types, YAML field binding, inlining, secret tags, recursion, and semantic paths; - and - descriptor-driven canonical projection, detached typed cloning, cycle/limit - enforcement, and structural comparison in `projection.go`. + enforcement, and structural comparison in `projection.go`; and +- strict schema-driven decoding over detached defaults in `decode.go`, including + mapping/sequence semantics, null and numeric policy, safe codec errors, presence, + baseline/final trees, and initial default/input origins. -The [configuration contract](../config.md) owns implemented input, schema, and -ownership behavior. ConfigKit intentionally exposes no typed loading or decoding API yet. -Private metadata population will be integrated when that lifecycle exists. +The [configuration contract](../config.md) owns implemented input, schema, decoding, +and ownership behavior. ConfigKit intentionally exposes no typed loading API yet; +the decoder and its initial metadata artifacts remain private until that lifecycle +exists. The source-extraction inventory records the application foundations assigned to future implementation work: diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index cfbfdc3..0abbcb9 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -2,7 +2,7 @@ ## Status and ownership -Status: implementation in progress; Stages 1–5 are complete and later stages are +Status: implementation in progress; Stages 1–6 are complete and later stages are pending. Audience: a `gpt-5.6-terra` coding agent implementing one stage per prompt, in @@ -564,7 +564,31 @@ Acceptance: ## Stage 6 — Decode scalar and container values over defaults -Status: pending. +Status: complete. + +Completion evidence (2026-09-06): + +- Added schema-driven typed decoding in [`decode.go`](../../decode.go), starting + from the Stage 5 detached clone and returning only private typed state, input + presence, projected baseline/final trees, and per-path default/base origins. +- Implemented recursive mapping retention, atomic sequence replacement, exact array + lengths, pointer-object zero allocation versus non-nil retention, nullable-shape + clearing, strict nested map values, inline-map decoding, and unknown-field errors. +- Added checked destination-width integers and floats, dynamic `int64`/`float64`, + named scalars, duration strings, paired text codecs, safe value-free failures, and + collection of independent decode errors at original semantic paths/coordinates. + The [current configuration contract](../config.md#defaults-and-typed-decoding) and + component inventory now own this behavior. +- Adapted Narratio's fresh-default, presence, strict nested-type, and repeated-load + cases, replacing custom YAML hooks and pointer/zero guesses with tree-derived + presence; explicit empty and null are retained and equal-valued input is explicit. + Focused coverage includes the defaults operation table, new/existing map entries, + empty mappings/sequences, nil/non-nil pointer objects, default isolation, numeric + range and coercion failures, integer durations, inline collisions, nested unknown + fields, coordinates, safe codec failures, and independent repeated results. + Verified `go test ./...`, `go vet ./...`, `go build ./...`, `go mod tidy -diff`, + and `git diff --check` with Go 1.26.7 on linux/amd64. No deviations were required; + public loading and schema-version checks remain later work. Read: feature [defaults/null](initial.md#static-defaults-and-null) and typed decoding. diff --git a/docs/roadmap/initial.md b/docs/roadmap/initial.md index ec4809d..ee9520e 100644 --- a/docs/roadmap/initial.md +++ b/docs/roadmap/initial.md @@ -552,69 +552,21 @@ Applications own flags and commands that select this behavior. ## Static defaults and null -Applications supply a defaults factory that creates a fresh configuration per -load. Mutable maps, slices, and pointers must not be shared between factory -results. ConfigKit must not retain mutable aliases between returned loads or -between the application result and its internal metadata snapshots. - -Defaults obey the same recursive mapping/atomic replacement rules as a lower -layer, rather than incidental decoder behavior: - -| Configured input | Effect on defaults | -| --- | --- | -| Absent field | Keep the default | -| Explicit false, zero, or empty string | Replace the default | -| Partial mapping, including a string-keyed Go map | Retain omitted default children/entries | -| `{}` over an existing mapping | Retain its default children | -| Sequence, including `[]` | Replace the entire default sequence | -| Partial object represented by a pointer | Retain children of a non-nil default object; otherwise allocate and use ordinary zero values for omitted children | -| Allowed null | Replace with nil; suppress lower defaults | - -The factory supplies actual defaults; ConfigKit does not invent per-entry -defaults for newly introduced map entries or sequence elements. Application -resolution may supply those. Go zero values are part of the initial typed -baseline; do not imply that the application explicitly declared every zero as -a domain default. Avoid serialization through `omitempty` that would erase -baseline values or presence information. - -Null is valid for pointers, maps, slices, and dynamic `any` values. It is invalid -for ordinary scalar fields, structs, and arrays. Thus `workers: null` cannot -silently leave an integer default intact. Applications can further reject null -through validation. Null never means reset to the application default. +The defaults merge and nullability model is implemented privately. The +[current defaults and typed-decoding contract](../config.md#defaults-and-typed-decoding) +owns recursive mapping, atomic sequence, pointer allocation, explicit-presence, +null, and detached-baseline behavior. The public defaults factory and result +lifecycle remain target work in the sections below. ## Typed decoding -The schema-binding portion of this target is implemented. The [current application -schema contract](../config.md#application-schema-binding) owns root/type support, -field and tag binding, inlining, text-codec pairing, and schema-aware path rules. -The remaining document-to-value behavior below is target decoding work. - -Strictness must be enforced by ConfigKit across the effective tree: - -- Unknown struct fields, type mismatches, overflow, and invalid null are errors. -- Without an inline map, unmatched keys remain errors. Default inline-map entries - colliding with declared fields are checked when runtime defaults are introduced. -- String-keyed maps accept application-defined keys. Struct values inside them - remain strict. `map[string]any` permits arbitrary keys and recursively accepts - the supported scalar/container model; application validation owns its schema. -- Do not coerce numbers or booleans to strings, or quoted numbers to numeric - fields. Integer targets require integer syntax and range checks. Float targets - accept finite decimal numbers subject to representable range checks. -- Dynamic integer values use checked `int64`, and fractional/exponent values use - checked finite `float64`; values outside those ranges require a typed schema - or a quoted string rather than silent rounding of integer nodes. -- Support `time.Duration` from a duration string. Custom scalar types must - implement both `encoding.TextUnmarshaler` and `encoding.TextMarshaler` so - decoding and inspection have an agreed representation. Duration integers are - rejected. See the [type and projection contract](#supported-types-and-projection). -- Parser-specific unmarshal hooks are not a public extension mechanism. Do not - let them bypass key, null, or type policy. Custom structural decoders are - deferred; unsupported type shapes fail with a clear schema diagnostic. - -Keep parser nodes and errors internal under the -[upstream isolation contract](#yaml-dependency-and-public-api-isolation). -The field-binding model must also support defaults, provenance, secret -classification, and rendering consistently. +Schema binding and strict document-to-value decoding are implemented privately. +The [current application schema contract](../config.md#application-schema-binding) +owns root/type support, field and tag binding, inlining, codec pairing, and +schema-aware path rules. The [current defaults and typed-decoding contract](../config.md#defaults-and-typed-decoding) +owns strict conversions, container application, presence, safe failures, and initial +origins. Public lifecycle integration, layered provenance, secret handling, and +rendering remain target work below. ## Recorded resolution and validation diff --git a/records.go b/records.go index ada004e..8c7bac0 100644 --- a/records.go +++ b/records.go @@ -64,6 +64,14 @@ const ( CodeSchemaCodec Code = "schema.codec" CodeSchemaNonFinite Code = "schema.non_finite" CodeSchemaDynamicType Code = "schema.dynamic_type" + + CodeDecodeType Code = "decode.type" + CodeDecodeNull Code = "decode.null" + CodeDecodeRange Code = "decode.range" + CodeDecodeDuration Code = "decode.duration" + CodeDecodeText Code = "decode.text" + CodeDecodeUnknownField Code = "decode.unknown_field" + CodeDecodeArrayLength Code = "decode.array_length" ) // Location identifies a position in a configuration source. Zero line and column