Files
notarius/internal/core/config/pipeline_document.go

514 lines
22 KiB
Go

package config
import (
"sort"
"strings"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
// documentLoadState keeps the spelling supplied by the user while resolver
// operations move map entries to their canonical keys. It is intentionally
// load-local: source-location handling consumes it during a later resolver
// pass and it never becomes runtime configuration state.
type documentLoadState struct {
originalPaths map[string]configkit.Path
mainDirectory string
invocationDirectory string
}
func newDocumentLoadState(invocationDirectory string) *documentLoadState {
return &documentLoadState{originalPaths: make(map[string]configkit.Path), invocationDirectory: cleanInvocationDirectory(invocationDirectory)}
}
func (state *documentLoadState) rememberPath(normalized, original configkit.Path) {
if state != nil {
state.originalPaths[normalized.String()] = original
}
}
// originalPath follows recorded map-key moves back to the spelling present in
// the source. Resolver code that needs a source coordinate must use this
// rather than reconstructing a path from a normalized identifier.
func (state *documentLoadState) originalPath(path configkit.Path) configkit.Path {
if state == nil || len(state.originalPaths) == 0 {
return path
}
value := path.String()
for {
changed := false
segments := strings.Split(strings.TrimPrefix(value, "/"), "/")
for length := 1; length <= len(segments); length++ {
prefix := "/" + strings.Join(segments[:length], "/")
original, ok := state.originalPaths[prefix]
if !ok {
continue
}
value = original.String() + strings.TrimPrefix(value, prefix)
changed = true
break
}
if !changed {
break
}
}
if original, err := configkit.ParsePath(value); err == nil {
return original
}
return path
}
func normalizeDocumentPipelines(document *Document, resolution *configkit.Resolution, state *documentLoadState) {
if document == nil {
return
}
validateDocumentPipelinePresence(document, resolution)
normalizer := documentNormalizer{Resolution: resolution}
pipelinesPath := configkit.Keys("pipelines")
for _, id := range sortedPipelineDocumentKeys(document.Pipelines) {
profile := document.Pipelines[id]
normalizePipelineDocument(&profile, pipelinesPath.Key(id), normalizer, state)
document.Pipelines[id] = profile
}
document.Pipelines = normalizeDocumentMap(document.Pipelines, pipelinesPath, "pipeline identifier", normalizer, state)
}
func validateDocumentPipelinePresence(document *Document, resolution *configkit.Resolution) {
pipelinesPath := configkit.Keys("pipelines")
for _, id := range sortedPipelineDocumentKeys(document.Pipelines) {
profile := document.Pipelines[id]
path := pipelinesPath.Key(id)
validatePipelineDocumentPresence(profile, path, resolution)
}
}
func validatePipelineDocumentPresence(profile PipelineDocument, path configkit.Path, resolution *configkit.Resolution) {
rejectDocumentNull(resolution, path.Key("llm_profile"), profile.LLMProfile == nil)
rejectDocumentNull(resolution, path.Key("structured_output_repair_attempts"), profile.StructuredOutputRepairAttempts == nil)
rejectDocumentNull(resolution, path.Key("validation_policy"), profile.ValidationPolicy == nil)
validateDocumentPolicyPresence(profile.ValidationPolicy, path.Key("validation_policy"), resolution)
stepsPath := path.Key("steps")
if resolution.IsExplicit(stepsPath) && profile.Steps == nil {
resolution.Error(stepsPath, "must not be null")
}
if resolution.IsExplicit(stepsPath) && len(profile.Steps) == 0 {
resolution.Error(stepsPath, "must contain at least one ordered step")
}
if resolution.IsExplicit(stepsPath) && resolution.IsExplicit(path.Key("artifacts")) {
resolution.Error(stepsPath, "must not be declared with artifacts", path.Key("artifacts"))
}
validateBindingDocumentPresence(profile.Input, path.Key("input"), resolution)
validateOptionalBindingDocumentPresence(profile.Chunk, path.Key("chunk"), resolution)
validateOptionalBindingDocumentPresence(profile.Output, path.Key("output"), resolution)
validateReferenceDocumentMapPresence(profile.References, path.Key("references"), resolution)
for _, laneID := range sortedLaneDocumentKeys(profile.Artifacts) {
validateArtifactLaneDocumentPresence(profile.Artifacts[laneID], path.Key("artifacts").Key(laneID), resolution)
}
for index, step := range profile.Steps {
stepPath := stepsPath.Index(index)
validateReferenceDocumentMapPresence(step.References, stepPath.Key("references"), resolution)
for _, laneID := range sortedLaneDocumentKeys(step.Artifacts) {
validateArtifactLaneDocumentPresence(step.Artifacts[laneID], stepPath.Key("artifacts").Key(laneID), resolution)
}
}
}
func validateArtifactLaneDocumentPresence(lane ArtifactLaneDocument, path configkit.Path, resolution *configkit.Resolution) {
if resolution.IsExplicit(path.Key("references")) {
resolution.Error(path.Key("references"), "artifact-lane references are not supported; use binding references")
}
validateBindingDocumentPresence(lane.Extract, path.Key("extract"), resolution)
validateOptionalBindingDocumentPresence(lane.Merge, path.Key("merge"), resolution)
validateOptionalBindingDocumentPresence(lane.Normalize, path.Key("normalize"), resolution)
}
func validateOptionalBindingDocumentPresence(binding *BindingDocument, path configkit.Path, resolution *configkit.Resolution) {
if binding == nil {
if resolution.IsExplicit(path) {
resolution.Error(path, "must not be null when configured")
}
return
}
validateBindingDocumentPresence(*binding, path, resolution)
}
func validateBindingDocumentPresence(binding BindingDocument, path configkit.Path, resolution *configkit.Resolution) {
rejectDocumentNull(resolution, path.Key("llm_profile"), binding.LLMProfile == nil)
rejectDocumentNull(resolution, path.Key("structured_output_repair_attempts"), binding.StructuredOutputRepairAttempts == nil)
rejectDocumentNull(resolution, path.Key("validation_policy"), binding.ValidationPolicy == nil)
validateDocumentPolicyPresence(binding.ValidationPolicy, path.Key("validation_policy"), resolution)
if binding.Validators == nil && resolution.IsExplicit(path.Key("validators")) {
resolution.Error(path.Key("validators"), "must not be null")
}
validateReferenceDocumentMapPresence(binding.References, path.Key("references"), resolution)
for index, validator := range binding.Validators {
validatorPath := path.Key("validators").Index(index)
rejectDocumentNull(resolution, validatorPath.Key("llm_profile"), validator.LLMProfile == nil)
rejectDocumentNull(resolution, validatorPath.Key("structured_output_repair_attempts"), validator.StructuredOutputRepairAttempts == nil)
}
}
func validateReferenceDocumentMapPresence(references map[string]ReferenceDocument, path configkit.Path, resolution *configkit.Resolution) {
for _, name := range sortedReferenceDocumentKeys(references) {
source := references[name]
sourcePath := path.Key(name)
rejectDocumentNull(resolution, sourcePath.Key("path"), source.Path == nil)
rejectDocumentNull(resolution, sourcePath.Key("artifact"), source.Artifact == nil)
}
}
func rejectDocumentNull(resolution *configkit.Resolution, path configkit.Path, isNil bool) {
if isNil && resolution.IsExplicit(path) {
resolution.Error(path, "must not be null")
}
}
func validateDocumentPolicyPresence(policy *ValidationPolicyDocument, path configkit.Path, resolution *configkit.Resolution) {
if policy == nil {
return
}
rejectDocumentNull(resolution, path.Key("producer_structural_failure"), policy.ProducerStructuralFailure == nil)
rejectDocumentNull(resolution, path.Key("semantic_rejection"), policy.SemanticRejection == nil)
rejectDocumentNull(resolution, path.Key("validator_failure"), policy.ValidatorFailure == nil)
}
func normalizePipelineDocument(profile *PipelineDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) {
normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &profile.LLMProfile, "trimmed pipeline LLM profile")
normalizeDocumentBinding(&profile.Input, path.Key("input"), resolution, state)
if profile.Chunk != nil {
normalizeDocumentBinding(profile.Chunk, path.Key("chunk"), resolution, state)
}
if profile.Output != nil {
normalizeDocumentBinding(profile.Output, path.Key("output"), resolution, state)
}
profile.References = normalizeDocumentReferences(profile.References, path.Key("references"), resolution, state)
for _, laneID := range sortedLaneDocumentKeys(profile.Artifacts) {
lane := profile.Artifacts[laneID]
normalizeArtifactLaneDocument(&lane, path.Key("artifacts").Key(laneID), resolution, state)
profile.Artifacts[laneID] = lane
}
profile.Artifacts = normalizeDocumentMap(profile.Artifacts, path.Key("artifacts"), "artifact lane identifier", resolution, state)
resolution = resolution.inSequence(path.Key("steps"))
for index := range profile.Steps {
step := &profile.Steps[index]
stepPath := path.Key("steps").Index(index)
normalizeDocumentValue(resolution, stepPath.Key("id"), &step.ID, strings.TrimSpace(step.ID), "trimmed step identifier")
step.References = normalizeDocumentReferences(step.References, stepPath.Key("references"), resolution, state)
for _, laneID := range sortedLaneDocumentKeys(step.Artifacts) {
lane := step.Artifacts[laneID]
normalizeArtifactLaneDocument(&lane, stepPath.Key("artifacts").Key(laneID), resolution, state)
step.Artifacts[laneID] = lane
}
step.Artifacts = normalizeDocumentMap(step.Artifacts, stepPath.Key("artifacts"), "artifact lane identifier", resolution, state)
}
}
func normalizeArtifactLaneDocument(lane *ArtifactLaneDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) {
normalizeDocumentBinding(&lane.Extract, path.Key("extract"), resolution, state)
if lane.Merge != nil {
normalizeDocumentBinding(lane.Merge, path.Key("merge"), resolution, state)
}
if lane.Normalize != nil {
normalizeDocumentBinding(lane.Normalize, path.Key("normalize"), resolution, state)
}
resolution = resolution.inSequence(path.Key("validators"))
for index := range lane.Validators {
normalizeValidatorDocument(&lane.Validators[index], path.Key("validators").Index(index), resolution)
}
}
func normalizeDocumentBinding(binding *BindingDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) {
normalizeDocumentValue(resolution, path.Key("module"), &binding.Module, strings.TrimSpace(binding.Module), "trimmed module identifier")
normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &binding.LLMProfile, "trimmed binding LLM profile")
binding.References = normalizeDocumentReferences(binding.References, path.Key("references"), resolution, state)
resolution = resolution.inSequence(path.Key("validators"))
for index := range binding.Validators {
normalizeValidatorDocument(&binding.Validators[index], path.Key("validators").Index(index), resolution)
}
}
func normalizeValidatorDocument(validator *ValidatorDocument, path configkit.Path, resolution documentNormalizer) {
normalizeDocumentValue(resolution, path.Key("module"), &validator.Module, strings.TrimSpace(validator.Module), "trimmed validator module identifier")
normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &validator.LLMProfile, "trimmed validator LLM profile")
}
func normalizeDocumentReferences(references map[string]ReferenceDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) map[string]ReferenceDocument {
for _, name := range sortedReferenceDocumentKeys(references) {
source := references[name]
sourcePath := path.Key(name)
normalizePipelineDocumentString(resolution, sourcePath.Key("path"), &source.Path, "trimmed reference path")
if source.Artifact != nil {
normalizeDocumentValue(resolution, sourcePath.Key("artifact").Key("step"), &source.Artifact.Step, strings.TrimSpace(source.Artifact.Step), "trimmed artifact step selector")
normalizeDocumentValue(resolution, sourcePath.Key("artifact").Key("lane"), &source.Artifact.Lane, strings.TrimSpace(source.Artifact.Lane), "trimmed artifact lane selector")
}
references[name] = source
}
return normalizeDocumentMap(references, path, "reference name", resolution, state)
}
func normalizeDocumentValue[T comparable](resolution documentNormalizer, path configkit.Path, target *T, value T, reason string) {
if target == nil || !resolution.IsExplicit(path) {
return
}
if *target != value {
resolution.Record(path, configkit.ChangeNormalization, reason)
*target = value
}
}
func normalizePipelineDocumentString(resolution documentNormalizer, path configkit.Path, target **string, reason string) {
if target == nil || *target == nil || !resolution.IsExplicit(path) {
return
}
value := strings.TrimSpace(**target)
if **target != value {
resolution.Record(path, configkit.ChangeNormalization, reason)
**target = value
}
}
func normalizeDocumentMap[T any](values map[string]T, path configkit.Path, kind string, resolution documentNormalizer, state *documentLoadState) map[string]T {
if len(values) == 0 {
return values
}
keys := make([]string, 0, len(values))
for key := range values {
keys = append(keys, key)
}
sort.Strings(keys)
canonical := make(map[string]string, len(values))
valid := true
for _, key := range keys {
trimmed := strings.TrimSpace(key)
if trimmed == "" {
resolution.Error(path.Key(key), kind+" must not be empty")
valid = false
continue
}
if first, exists := canonical[trimmed]; exists {
resolution.Error(path.Key(key), kind+" collides after whitespace normalization", path.Key(first))
valid = false
continue
}
canonical[trimmed] = key
}
if !valid {
return values
}
result := make(map[string]T, len(values))
changed := false
dependencies := make([]configkit.Path, 0, len(keys))
for _, key := range keys {
trimmed := strings.TrimSpace(key)
result[trimmed] = values[key]
dependencies = append(dependencies, path.Key(key))
if trimmed != key {
changed = true
state.rememberPath(path.Key(trimmed), path.Key(key))
}
}
if changed {
resolution.Record(path, configkit.ChangeNormalization, "normalized "+kind+" keys", dependencies...)
values = result
}
return values
}
func validateDocumentPipelines(document *Document, validation *configkit.Validation) {
if document == nil {
return
}
pipelinesPath := configkit.Keys("pipelines")
for _, id := range sortedPipelineDocumentKeys(document.Pipelines) {
profile := document.Pipelines[id]
path := pipelinesPath.Key(id)
validatePipelineDocument(profile, path, validation)
}
}
func validatePipelineDocument(profile PipelineDocument, path configkit.Path, validation *configkit.Validation) {
validateSelectedDocumentProfile(profile.LLMProfile, path.Key("llm_profile"), validation)
validateDocumentRepairAttempts(profile.StructuredOutputRepairAttempts, path.Key("structured_output_repair_attempts"), validation)
validateDocumentPolicy(profile.ValidationPolicy, path.Key("validation_policy"), validation)
validateDocumentBinding(profile.Input, path.Key("input"), "input", false, false, validation)
if profile.Chunk != nil {
validateDocumentBinding(*profile.Chunk, path.Key("chunk"), "chunk", true, true, validation)
}
if profile.Output != nil {
validateDocumentBinding(*profile.Output, path.Key("output"), "output", false, false, validation)
}
validateDocumentReferences(profile.References, path.Key("references"), false, validation)
seenLanes := make(map[string]configkit.Path)
if profile.Steps != nil {
seenSteps := make(map[string]configkit.Path)
for index, step := range profile.Steps {
stepPath := path.Key("steps").Index(index)
if strings.TrimSpace(step.ID) == "" {
validation.Error(stepPath.Key("id"), "must not be empty")
} else if first, exists := seenSteps[step.ID]; exists {
validation.Error(stepPath.Key("id"), "must be unique", first)
} else {
seenSteps[step.ID] = stepPath.Key("id")
}
validateDocumentReferences(step.References, stepPath.Key("references"), true, validation)
validateDocumentLanes(step.Artifacts, stepPath.Key("artifacts"), seenLanes, validation)
}
return
}
validateDocumentLanes(profile.Artifacts, path.Key("artifacts"), seenLanes, validation)
}
func validateDocumentLanes(lanes map[string]ArtifactLaneDocument, path configkit.Path, seen map[string]configkit.Path, validation *configkit.Validation) {
for _, id := range sortedLaneDocumentKeys(lanes) {
lanePath := path.Key(id)
if first, exists := seen[id]; exists {
validation.Error(lanePath, "artifact lane identifier must be unique across steps", first)
} else {
seen[id] = lanePath
}
lane := lanes[id]
validateDocumentBinding(lane.Extract, lanePath.Key("extract"), "extract", true, true, validation)
if lane.Merge != nil {
validateDocumentBinding(*lane.Merge, lanePath.Key("merge"), "merge", true, true, validation)
}
if lane.Normalize != nil {
validateDocumentBinding(*lane.Normalize, lanePath.Key("normalize"), "normalize", true, true, validation)
}
if len(lane.Validators) > 0 {
validation.Error(lanePath.Key("validators"), "artifact-lane validators are not supported; attach validators to a binding")
}
}
}
func validateDocumentBinding(binding BindingDocument, path configkit.Path, slot string, referencesAllowed, validatorsAllowed bool, validation *configkit.Validation) {
if strings.TrimSpace(binding.Module) == "" {
validation.Error(path.Key("module"), "must not be empty")
}
validateSelectedDocumentProfile(binding.LLMProfile, path.Key("llm_profile"), validation)
validateDocumentRepairAttempts(binding.StructuredOutputRepairAttempts, path.Key("structured_output_repair_attempts"), validation)
if binding.Retries < 0 {
validation.Error(path.Key("retries"), "must be greater than or equal to zero")
}
if binding.ValidationPolicy != nil {
if slot != "chunk" && slot != "extract" && slot != "merge" && slot != "normalize" {
validation.Error(path.Key("validation_policy"), "is not supported for this binding")
}
validateDocumentPolicy(binding.ValidationPolicy, path.Key("validation_policy"), validation)
}
if len(binding.References) > 0 && !referencesAllowed {
validation.Error(path.Key("references"), "are not supported for this binding")
}
if referencesAllowed {
validateDocumentReferences(binding.References, path.Key("references"), true, validation)
}
if binding.Validators != nil && !validatorsAllowed {
validation.Error(path.Key("validators"), "are not supported for this binding")
}
for index, validator := range binding.Validators {
validateValidatorDocument(validator, path.Key("validators").Index(index), validation)
}
}
func validateValidatorDocument(validator ValidatorDocument, path configkit.Path, validation *configkit.Validation) {
if strings.TrimSpace(validator.Module) == "" {
validation.Error(path.Key("module"), "must not be empty")
}
validateSelectedDocumentProfile(validator.LLMProfile, path.Key("llm_profile"), validation)
validateDocumentRepairAttempts(validator.StructuredOutputRepairAttempts, path.Key("structured_output_repair_attempts"), validation)
if validator.Retries < 0 {
validation.Error(path.Key("retries"), "must be greater than or equal to zero")
}
}
func validateDocumentReferences(references map[string]ReferenceDocument, path configkit.Path, generatedAllowed bool, validation *configkit.Validation) {
for _, name := range sortedReferenceDocumentKeys(references) {
source := references[name]
sourcePath := path.Key(name)
hasPath := source.Path != nil
hasArtifact := source.Artifact != nil
if hasPath == hasArtifact {
validation.Error(sourcePath, "must select exactly one reference source")
continue
}
if hasPath {
if strings.TrimSpace(*source.Path) == "" {
validation.Error(sourcePath.Key("path"), "must not be empty")
}
continue
}
if !generatedAllowed {
validation.Error(sourcePath.Key("artifact"), "is not supported here; use an external path")
continue
}
if strings.TrimSpace(source.Artifact.Step) == "" {
validation.Error(sourcePath.Key("artifact").Key("step"), "must not be empty")
}
if strings.TrimSpace(source.Artifact.Lane) == "" {
validation.Error(sourcePath.Key("artifact").Key("lane"), "must not be empty")
}
}
}
func validateSelectedDocumentProfile(value *string, path configkit.Path, validation *configkit.Validation) {
if value != nil && strings.TrimSpace(*value) == "" {
validation.Error(path, "must not be empty when set")
}
}
func validateDocumentRepairAttempts(value *int, path configkit.Path, validation *configkit.Validation) {
if value != nil && (*value < 0 || *value > 3) {
validation.Error(path, "must be between zero and three")
}
}
func validateDocumentPolicy(policy *ValidationPolicyDocument, path configkit.Path, validation *configkit.Validation) {
if policy == nil {
return
}
if policy.ProducerStructuralFailure != nil {
switch *policy.ProducerStructuralFailure {
case pipeline.ProducerStructuralFailureFailRun, pipeline.ProducerStructuralFailureRejectOutput:
default:
validation.Error(path.Key("producer_structural_failure"), "must be fail_run or reject_output")
}
}
if policy.SemanticRejection != nil {
switch *policy.SemanticRejection {
case pipeline.SemanticRejectionFailRun, pipeline.SemanticRejectionRejectOutput:
default:
validation.Error(path.Key("semantic_rejection"), "must be fail_run or reject_output")
}
}
if policy.ValidatorFailure != nil {
switch *policy.ValidatorFailure {
case pipeline.ValidatorFailureWarnContinue, pipeline.ValidatorFailureFailRun:
default:
validation.Error(path.Key("validator_failure"), "must be warn_continue or fail_run")
}
}
}
func sortedPipelineDocumentKeys(values map[string]PipelineDocument) []string {
return sortedDocumentKeys(values)
}
func sortedLaneDocumentKeys(values map[string]ArtifactLaneDocument) []string {
return sortedDocumentKeys(values)
}
func sortedReferenceDocumentKeys(values map[string]ReferenceDocument) []string {
return sortedDocumentKeys(values)
}
func sortedDocumentKeys[T any](values map[string]T) []string {
keys := make([]string, 0, len(values))
for key := range values {
keys = append(keys, key)
}
sort.Strings(keys)
return keys
}