package config import ( "sort" "strings" configkit "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" ) // documentLoadState keeps the spelling supplied by the user while resolver // operations move map entries to their canonical keys. It is intentionally // load-local: source-location handling consumes it during a later resolver // pass and it never becomes runtime configuration state. type documentLoadState struct { originalPaths map[string]configkit.Path mainDirectory string invocationDirectory string } func newDocumentLoadState(invocationDirectory string) *documentLoadState { return &documentLoadState{originalPaths: make(map[string]configkit.Path), invocationDirectory: cleanInvocationDirectory(invocationDirectory)} } func (state *documentLoadState) rememberPath(normalized, original configkit.Path) { if state != nil { state.originalPaths[normalized.String()] = original } } // originalPath follows recorded map-key moves back to the spelling present in // the source. Resolver code that needs a source coordinate must use this // rather than reconstructing a path from a normalized identifier. func (state *documentLoadState) originalPath(path configkit.Path) configkit.Path { if state == nil || len(state.originalPaths) == 0 { return path } value := path.String() for { changed := false segments := strings.Split(strings.TrimPrefix(value, "/"), "/") for length := 1; length <= len(segments); length++ { prefix := "/" + strings.Join(segments[:length], "/") original, ok := state.originalPaths[prefix] if !ok { continue } value = original.String() + strings.TrimPrefix(value, prefix) changed = true break } if !changed { break } } if original, err := configkit.ParsePath(value); err == nil { return original } return path } func normalizeDocumentPipelines(document *Document, resolution *configkit.Resolution, state *documentLoadState) { if document == nil { return } validateDocumentPipelinePresence(document, resolution) normalizer := documentNormalizer{Resolution: resolution} pipelinesPath := configkit.Keys("pipelines") for _, id := range sortedPipelineDocumentKeys(document.Pipelines) { profile := document.Pipelines[id] normalizePipelineDocument(&profile, pipelinesPath.Key(id), normalizer, state) document.Pipelines[id] = profile } document.Pipelines = normalizeDocumentMap(document.Pipelines, pipelinesPath, "pipeline identifier", normalizer, state) } func validateDocumentPipelinePresence(document *Document, resolution *configkit.Resolution) { pipelinesPath := configkit.Keys("pipelines") for _, id := range sortedPipelineDocumentKeys(document.Pipelines) { profile := document.Pipelines[id] path := pipelinesPath.Key(id) validatePipelineDocumentPresence(profile, path, resolution) } } func validatePipelineDocumentPresence(profile PipelineDocument, path configkit.Path, resolution *configkit.Resolution) { rejectDocumentNull(resolution, path.Key("llm_profile"), profile.LLMProfile == nil) rejectDocumentNull(resolution, path.Key("structured_output_repair_attempts"), profile.StructuredOutputRepairAttempts == nil) rejectDocumentNull(resolution, path.Key("validation_policy"), profile.ValidationPolicy == nil) validateDocumentPolicyPresence(profile.ValidationPolicy, path.Key("validation_policy"), resolution) stepsPath := path.Key("steps") if resolution.IsExplicit(stepsPath) && profile.Steps == nil { resolution.Error(stepsPath, "must not be null") } if resolution.IsExplicit(stepsPath) && len(profile.Steps) == 0 { resolution.Error(stepsPath, "must contain at least one ordered step") } if resolution.IsExplicit(stepsPath) && resolution.IsExplicit(path.Key("artifacts")) { resolution.Error(stepsPath, "must not be declared with artifacts", path.Key("artifacts")) } validateBindingDocumentPresence(profile.Input, path.Key("input"), resolution) validateOptionalBindingDocumentPresence(profile.Chunk, path.Key("chunk"), resolution) validateOptionalBindingDocumentPresence(profile.Output, path.Key("output"), resolution) validateReferenceDocumentMapPresence(profile.References, path.Key("references"), resolution) for _, laneID := range sortedLaneDocumentKeys(profile.Artifacts) { validateArtifactLaneDocumentPresence(profile.Artifacts[laneID], path.Key("artifacts").Key(laneID), resolution) } for index, step := range profile.Steps { stepPath := stepsPath.Index(index) validateReferenceDocumentMapPresence(step.References, stepPath.Key("references"), resolution) for _, laneID := range sortedLaneDocumentKeys(step.Artifacts) { validateArtifactLaneDocumentPresence(step.Artifacts[laneID], stepPath.Key("artifacts").Key(laneID), resolution) } } } func validateArtifactLaneDocumentPresence(lane ArtifactLaneDocument, path configkit.Path, resolution *configkit.Resolution) { if resolution.IsExplicit(path.Key("references")) { resolution.Error(path.Key("references"), "artifact-lane references are not supported; use binding references") } validateBindingDocumentPresence(lane.Extract, path.Key("extract"), resolution) validateOptionalBindingDocumentPresence(lane.Merge, path.Key("merge"), resolution) validateOptionalBindingDocumentPresence(lane.Normalize, path.Key("normalize"), resolution) } func validateOptionalBindingDocumentPresence(binding *BindingDocument, path configkit.Path, resolution *configkit.Resolution) { if binding == nil { if resolution.IsExplicit(path) { resolution.Error(path, "must not be null when configured") } return } validateBindingDocumentPresence(*binding, path, resolution) } func validateBindingDocumentPresence(binding BindingDocument, path configkit.Path, resolution *configkit.Resolution) { rejectDocumentNull(resolution, path.Key("llm_profile"), binding.LLMProfile == nil) rejectDocumentNull(resolution, path.Key("structured_output_repair_attempts"), binding.StructuredOutputRepairAttempts == nil) rejectDocumentNull(resolution, path.Key("validation_policy"), binding.ValidationPolicy == nil) validateDocumentPolicyPresence(binding.ValidationPolicy, path.Key("validation_policy"), resolution) if binding.Validators == nil && resolution.IsExplicit(path.Key("validators")) { resolution.Error(path.Key("validators"), "must not be null") } validateReferenceDocumentMapPresence(binding.References, path.Key("references"), resolution) for index, validator := range binding.Validators { validatorPath := path.Key("validators").Index(index) rejectDocumentNull(resolution, validatorPath.Key("llm_profile"), validator.LLMProfile == nil) rejectDocumentNull(resolution, validatorPath.Key("structured_output_repair_attempts"), validator.StructuredOutputRepairAttempts == nil) } } func validateReferenceDocumentMapPresence(references map[string]ReferenceDocument, path configkit.Path, resolution *configkit.Resolution) { for _, name := range sortedReferenceDocumentKeys(references) { source := references[name] sourcePath := path.Key(name) rejectDocumentNull(resolution, sourcePath.Key("path"), source.Path == nil) rejectDocumentNull(resolution, sourcePath.Key("artifact"), source.Artifact == nil) } } func rejectDocumentNull(resolution *configkit.Resolution, path configkit.Path, isNil bool) { if isNil && resolution.IsExplicit(path) { resolution.Error(path, "must not be null") } } func validateDocumentPolicyPresence(policy *ValidationPolicyDocument, path configkit.Path, resolution *configkit.Resolution) { if policy == nil { return } rejectDocumentNull(resolution, path.Key("producer_structural_failure"), policy.ProducerStructuralFailure == nil) rejectDocumentNull(resolution, path.Key("semantic_rejection"), policy.SemanticRejection == nil) rejectDocumentNull(resolution, path.Key("validator_failure"), policy.ValidatorFailure == nil) } func normalizePipelineDocument(profile *PipelineDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) { normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &profile.LLMProfile, "trimmed pipeline LLM profile") normalizeDocumentBinding(&profile.Input, path.Key("input"), resolution, state) if profile.Chunk != nil { normalizeDocumentBinding(profile.Chunk, path.Key("chunk"), resolution, state) } if profile.Output != nil { normalizeDocumentBinding(profile.Output, path.Key("output"), resolution, state) } profile.References = normalizeDocumentReferences(profile.References, path.Key("references"), resolution, state) for _, laneID := range sortedLaneDocumentKeys(profile.Artifacts) { lane := profile.Artifacts[laneID] normalizeArtifactLaneDocument(&lane, path.Key("artifacts").Key(laneID), resolution, state) profile.Artifacts[laneID] = lane } profile.Artifacts = normalizeDocumentMap(profile.Artifacts, path.Key("artifacts"), "artifact lane identifier", resolution, state) resolution = resolution.inSequence(path.Key("steps")) for index := range profile.Steps { step := &profile.Steps[index] stepPath := path.Key("steps").Index(index) normalizeDocumentValue(resolution, stepPath.Key("id"), &step.ID, strings.TrimSpace(step.ID), "trimmed step identifier") step.References = normalizeDocumentReferences(step.References, stepPath.Key("references"), resolution, state) for _, laneID := range sortedLaneDocumentKeys(step.Artifacts) { lane := step.Artifacts[laneID] normalizeArtifactLaneDocument(&lane, stepPath.Key("artifacts").Key(laneID), resolution, state) step.Artifacts[laneID] = lane } step.Artifacts = normalizeDocumentMap(step.Artifacts, stepPath.Key("artifacts"), "artifact lane identifier", resolution, state) } } func normalizeArtifactLaneDocument(lane *ArtifactLaneDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) { normalizeDocumentBinding(&lane.Extract, path.Key("extract"), resolution, state) if lane.Merge != nil { normalizeDocumentBinding(lane.Merge, path.Key("merge"), resolution, state) } if lane.Normalize != nil { normalizeDocumentBinding(lane.Normalize, path.Key("normalize"), resolution, state) } resolution = resolution.inSequence(path.Key("validators")) for index := range lane.Validators { normalizeValidatorDocument(&lane.Validators[index], path.Key("validators").Index(index), resolution) } } func normalizeDocumentBinding(binding *BindingDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) { normalizeDocumentValue(resolution, path.Key("module"), &binding.Module, strings.TrimSpace(binding.Module), "trimmed module identifier") normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &binding.LLMProfile, "trimmed binding LLM profile") binding.References = normalizeDocumentReferences(binding.References, path.Key("references"), resolution, state) resolution = resolution.inSequence(path.Key("validators")) for index := range binding.Validators { normalizeValidatorDocument(&binding.Validators[index], path.Key("validators").Index(index), resolution) } } func normalizeValidatorDocument(validator *ValidatorDocument, path configkit.Path, resolution documentNormalizer) { normalizeDocumentValue(resolution, path.Key("module"), &validator.Module, strings.TrimSpace(validator.Module), "trimmed validator module identifier") normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &validator.LLMProfile, "trimmed validator LLM profile") } func normalizeDocumentReferences(references map[string]ReferenceDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) map[string]ReferenceDocument { for _, name := range sortedReferenceDocumentKeys(references) { source := references[name] sourcePath := path.Key(name) normalizePipelineDocumentString(resolution, sourcePath.Key("path"), &source.Path, "trimmed reference path") if source.Artifact != nil { normalizeDocumentValue(resolution, sourcePath.Key("artifact").Key("step"), &source.Artifact.Step, strings.TrimSpace(source.Artifact.Step), "trimmed artifact step selector") normalizeDocumentValue(resolution, sourcePath.Key("artifact").Key("lane"), &source.Artifact.Lane, strings.TrimSpace(source.Artifact.Lane), "trimmed artifact lane selector") } references[name] = source } return normalizeDocumentMap(references, path, "reference name", resolution, state) } func normalizeDocumentValue[T comparable](resolution documentNormalizer, path configkit.Path, target *T, value T, reason string) { if target == nil || !resolution.IsExplicit(path) { return } if *target != value { resolution.Record(path, configkit.ChangeNormalization, reason) *target = value } } func normalizePipelineDocumentString(resolution documentNormalizer, path configkit.Path, target **string, reason string) { if target == nil || *target == nil || !resolution.IsExplicit(path) { return } value := strings.TrimSpace(**target) if **target != value { resolution.Record(path, configkit.ChangeNormalization, reason) **target = value } } func normalizeDocumentMap[T any](values map[string]T, path configkit.Path, kind string, resolution documentNormalizer, state *documentLoadState) map[string]T { if len(values) == 0 { return values } keys := make([]string, 0, len(values)) for key := range values { keys = append(keys, key) } sort.Strings(keys) canonical := make(map[string]string, len(values)) valid := true for _, key := range keys { trimmed := strings.TrimSpace(key) if trimmed == "" { resolution.Error(path.Key(key), kind+" must not be empty") valid = false continue } if first, exists := canonical[trimmed]; exists { resolution.Error(path.Key(key), kind+" collides after whitespace normalization", path.Key(first)) valid = false continue } canonical[trimmed] = key } if !valid { return values } result := make(map[string]T, len(values)) changed := false dependencies := make([]configkit.Path, 0, len(keys)) for _, key := range keys { trimmed := strings.TrimSpace(key) result[trimmed] = values[key] dependencies = append(dependencies, path.Key(key)) if trimmed != key { changed = true state.rememberPath(path.Key(trimmed), path.Key(key)) } } if changed { resolution.Record(path, configkit.ChangeNormalization, "normalized "+kind+" keys", dependencies...) values = result } return values } func validateDocumentPipelines(document *Document, validation *configkit.Validation) { if document == nil { return } pipelinesPath := configkit.Keys("pipelines") for _, id := range sortedPipelineDocumentKeys(document.Pipelines) { profile := document.Pipelines[id] path := pipelinesPath.Key(id) validatePipelineDocument(profile, path, validation) } } func validatePipelineDocument(profile PipelineDocument, path configkit.Path, validation *configkit.Validation) { validateSelectedDocumentProfile(profile.LLMProfile, path.Key("llm_profile"), validation) validateDocumentRepairAttempts(profile.StructuredOutputRepairAttempts, path.Key("structured_output_repair_attempts"), validation) validateDocumentPolicy(profile.ValidationPolicy, path.Key("validation_policy"), validation) validateDocumentBinding(profile.Input, path.Key("input"), "input", false, false, validation) if profile.Chunk != nil { validateDocumentBinding(*profile.Chunk, path.Key("chunk"), "chunk", true, true, validation) } if profile.Output != nil { validateDocumentBinding(*profile.Output, path.Key("output"), "output", false, false, validation) } validateDocumentReferences(profile.References, path.Key("references"), false, validation) seenLanes := make(map[string]configkit.Path) if profile.Steps != nil { seenSteps := make(map[string]configkit.Path) for index, step := range profile.Steps { stepPath := path.Key("steps").Index(index) if strings.TrimSpace(step.ID) == "" { validation.Error(stepPath.Key("id"), "must not be empty") } else if first, exists := seenSteps[step.ID]; exists { validation.Error(stepPath.Key("id"), "must be unique", first) } else { seenSteps[step.ID] = stepPath.Key("id") } validateDocumentReferences(step.References, stepPath.Key("references"), true, validation) validateDocumentLanes(step.Artifacts, stepPath.Key("artifacts"), seenLanes, validation) } return } validateDocumentLanes(profile.Artifacts, path.Key("artifacts"), seenLanes, validation) } func validateDocumentLanes(lanes map[string]ArtifactLaneDocument, path configkit.Path, seen map[string]configkit.Path, validation *configkit.Validation) { for _, id := range sortedLaneDocumentKeys(lanes) { lanePath := path.Key(id) if first, exists := seen[id]; exists { validation.Error(lanePath, "artifact lane identifier must be unique across steps", first) } else { seen[id] = lanePath } lane := lanes[id] validateDocumentBinding(lane.Extract, lanePath.Key("extract"), "extract", true, true, validation) if lane.Merge != nil { validateDocumentBinding(*lane.Merge, lanePath.Key("merge"), "merge", true, true, validation) } if lane.Normalize != nil { validateDocumentBinding(*lane.Normalize, lanePath.Key("normalize"), "normalize", true, true, validation) } if len(lane.Validators) > 0 { validation.Error(lanePath.Key("validators"), "artifact-lane validators are not supported; attach validators to a binding") } } } func validateDocumentBinding(binding BindingDocument, path configkit.Path, slot string, referencesAllowed, validatorsAllowed bool, validation *configkit.Validation) { if strings.TrimSpace(binding.Module) == "" { validation.Error(path.Key("module"), "must not be empty") } validateSelectedDocumentProfile(binding.LLMProfile, path.Key("llm_profile"), validation) validateDocumentRepairAttempts(binding.StructuredOutputRepairAttempts, path.Key("structured_output_repair_attempts"), validation) if binding.Retries < 0 { validation.Error(path.Key("retries"), "must be greater than or equal to zero") } if binding.ValidationPolicy != nil { if slot != "chunk" && slot != "extract" && slot != "merge" && slot != "normalize" { validation.Error(path.Key("validation_policy"), "is not supported for this binding") } validateDocumentPolicy(binding.ValidationPolicy, path.Key("validation_policy"), validation) } if len(binding.References) > 0 && !referencesAllowed { validation.Error(path.Key("references"), "are not supported for this binding") } if referencesAllowed { validateDocumentReferences(binding.References, path.Key("references"), true, validation) } if binding.Validators != nil && !validatorsAllowed { validation.Error(path.Key("validators"), "are not supported for this binding") } for index, validator := range binding.Validators { validateValidatorDocument(validator, path.Key("validators").Index(index), validation) } } func validateValidatorDocument(validator ValidatorDocument, path configkit.Path, validation *configkit.Validation) { if strings.TrimSpace(validator.Module) == "" { validation.Error(path.Key("module"), "must not be empty") } validateSelectedDocumentProfile(validator.LLMProfile, path.Key("llm_profile"), validation) validateDocumentRepairAttempts(validator.StructuredOutputRepairAttempts, path.Key("structured_output_repair_attempts"), validation) if validator.Retries < 0 { validation.Error(path.Key("retries"), "must be greater than or equal to zero") } } func validateDocumentReferences(references map[string]ReferenceDocument, path configkit.Path, generatedAllowed bool, validation *configkit.Validation) { for _, name := range sortedReferenceDocumentKeys(references) { source := references[name] sourcePath := path.Key(name) hasPath := source.Path != nil hasArtifact := source.Artifact != nil if hasPath == hasArtifact { validation.Error(sourcePath, "must select exactly one reference source") continue } if hasPath { if strings.TrimSpace(*source.Path) == "" { validation.Error(sourcePath.Key("path"), "must not be empty") } continue } if !generatedAllowed { validation.Error(sourcePath.Key("artifact"), "is not supported here; use an external path") continue } if strings.TrimSpace(source.Artifact.Step) == "" { validation.Error(sourcePath.Key("artifact").Key("step"), "must not be empty") } if strings.TrimSpace(source.Artifact.Lane) == "" { validation.Error(sourcePath.Key("artifact").Key("lane"), "must not be empty") } } } func validateSelectedDocumentProfile(value *string, path configkit.Path, validation *configkit.Validation) { if value != nil && strings.TrimSpace(*value) == "" { validation.Error(path, "must not be empty when set") } } func validateDocumentRepairAttempts(value *int, path configkit.Path, validation *configkit.Validation) { if value != nil && (*value < 0 || *value > 3) { validation.Error(path, "must be between zero and three") } } func validateDocumentPolicy(policy *ValidationPolicyDocument, path configkit.Path, validation *configkit.Validation) { if policy == nil { return } if policy.ProducerStructuralFailure != nil { switch *policy.ProducerStructuralFailure { case pipeline.ProducerStructuralFailureFailRun, pipeline.ProducerStructuralFailureRejectOutput: default: validation.Error(path.Key("producer_structural_failure"), "must be fail_run or reject_output") } } if policy.SemanticRejection != nil { switch *policy.SemanticRejection { case pipeline.SemanticRejectionFailRun, pipeline.SemanticRejectionRejectOutput: default: validation.Error(path.Key("semantic_rejection"), "must be fail_run or reject_output") } } if policy.ValidatorFailure != nil { switch *policy.ValidatorFailure { case pipeline.ValidatorFailureWarnContinue, pipeline.ValidatorFailureFailRun: default: validation.Error(path.Key("validator_failure"), "must be warn_continue or fail_run") } } } func sortedPipelineDocumentKeys(values map[string]PipelineDocument) []string { return sortedDocumentKeys(values) } func sortedLaneDocumentKeys(values map[string]ArtifactLaneDocument) []string { return sortedDocumentKeys(values) } func sortedReferenceDocumentKeys(values map[string]ReferenceDocument) []string { return sortedDocumentKeys(values) } func sortedDocumentKeys[T any](values map[string]T) []string { keys := make([]string, 0, len(values)) for key := range values { keys = append(keys, key) } sort.Strings(keys) return keys }