eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Execute complete validator chains

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-27 00:18:16 +00:00
parent 04ba87e174
commit 8dd7a4324d
8 changed files with 524 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
internal/framework/pipeline/runner.go
View File

@@ -476,20 +476,19 @@ func (r *Runner) validateChunks(ctx context.Context, doc *source.SourceDocument,
return nil, nil, fmt.Errorf("encode canonical chunks for validation: %w", err)
}
schema := contracts.ArtifactSchema{ID: "notarius.source.chunks", Name: "notarius_source_chunks", Version: "v1", JSONSchema: []byte(`{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"array"}`)}
var warnings []contracts.Warning
for index, item := range prepared.validators {
report, err := executeValidationChain(ctx, prepared, func(validatorCtx context.Context, item preparedValidator, validatorAttempt int) (validationInvocation, error) {
binding := item.resolved.Binding
started := time.Now().UTC()
attemptPath := path.Join("validate", fileio.EncodePathComponent(string(StageChunk)), "", fileio.EncodePathComponent(moduleKey), fmt.Sprintf("%02d-%s-attempt-%02d", index+1, fileio.EncodePathComponent(binding.Module), attempt))
validatorCtx, llmScope := withIsolatedDebugLLMScope(ctx, attemptPath)
attemptPath := validatorAttemptPath(path.Join("validate", fileio.EncodePathComponent(string(StageChunk)), "", fileio.EncodePathComponent(moduleKey), fmt.Sprintf("%02d-%s-attempt-%02d", item.position, fileio.EncodePathComponent(binding.Module), attempt)), validatorAttempt)
validatorCtx, llmScope := withIsolatedDebugLLMScope(validatorCtx, attemptPath)
var result contracts.ValidationResult
requestMetadata, cloneErr := cloneMetadata(metadata)
if cloneErr != nil {
return nil, nil, fmt.Errorf("clone chunk validation metadata: %w", cloneErr)
return validationInvocation{}, fatalValidationError(fmt.Errorf("clone chunk validation metadata: %w", cloneErr))
}
requestChunks, cloneErr := cloneSourceChunks(chunks)
if cloneErr != nil {
return nil, nil, fmt.Errorf("clone chunks for validation: %w", cloneErr)
return validationInvocation{}, fatalValidationError(fmt.Errorf("clone chunks for validation: %w", cloneErr))
}
switch item.resolved.Target {
case ValidatorTargetChunk:
@@ -497,7 +496,7 @@ func (r *Runner) validateChunks(ctx context.Context, doc *source.SourceDocument,
case ValidatorTargetSerialized:
result, err = item.serialized.Validate(validatorCtx, contracts.SerializedValidationRequest{Stage: string(StageChunk), ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts), Metadata: requestMetadata, Chunks: requestChunks, Schema: contracts.CloneArtifactSchema(schema), MediaType: "application/json", Content: append([]byte(nil), content...)})
default:
return nil, nil, fmt.Errorf("validator %q is incompatible with chunk validation", binding.Module)
return validationInvocation{}, fatalValidationError(fmt.Errorf("validator %q is incompatible with chunk validation", binding.Module))
}
if err == nil {
err = contracts.ValidateValidationResult(result)
@@ -511,27 +510,25 @@ func (r *Runner) validateChunks(ctx context.Context, doc *source.SourceDocument,
if err != nil {
validationErr := fmt.Errorf("validate chunks with validator %q: %w", binding.Module, err)
if debugErr := writeDebugAttempt(debug, attemptPath, debugTimedEnvelope{Stage: string(StageChunk), ModuleKey: moduleKey, Attempt: attempt, StartedAt: started, Payload: debugCall, Error: debugCall.Error}, llmScope); debugErr != nil {
return warnings, nil, errors.Join(validationErr, fmt.Errorf("write chunk validator attempt debug artifact: %w", debugErr))
return validationInvocation{}, fatalValidationError(errors.Join(validationErr, fmt.Errorf("write chunk validator attempt debug artifact: %w", debugErr)))
}
return warnings, nil, validationErr
return validationInvocation{}, validationErr
}
if debugErr := writeDebugAttempt(debug, attemptPath, debugTimedEnvelope{Stage: string(StageChunk), ModuleKey: moduleKey, Attempt: attempt, StartedAt: started, Payload: debugCall}, llmScope); debugErr != nil {
return warnings, nil, fmt.Errorf("write chunk validator attempt debug artifact: %w", debugErr)
return validationInvocation{}, fatalValidationError(fmt.Errorf("write chunk validator attempt debug artifact: %w", debugErr))
}
if !result.Approved {
reason := result.ReasonCode
if reason == "" {
reason = "output_rejected"
}
message := result.Message
if message == "" {
message = "output rejected"
}
return warnings, &contracts.RejectedOutput{Stage: string(StageChunk), ModuleKey: moduleKey, ValidatorName: binding.Module, ReasonCode: reason, Message: message, AttemptCount: attempt, DiagnosticArtifactPath: result.DiagnosticArtifactPath}, nil
}
warnings = append(warnings, result.Warnings...)
return validationInvocation{result: result}, nil
})
if err != nil {
return report.Warnings(), nil, err
}
return warnings, nil, nil
if failure := report.FirstFailure(); failure != nil {
return report.Warnings(), nil, validatorFailureError(*failure)
}
if rejection := report.FirstRejection(); rejection != nil {
return report.Warnings(), &contracts.RejectedOutput{Stage: string(StageChunk), ModuleKey: moduleKey, ValidatorName: rejection.validatorName, ReasonCode: rejection.reasonCode, Message: rejection.message, AttemptCount: attempt, DiagnosticArtifactPath: rejection.diagnosticPath}, nil
}
return report.Warnings(), nil, nil
}
func resolvedValidatorChain(stage ModuleStage, laneID string, moduleKey string, chains []ResolvedValidatorChain) ResolvedValidatorChain {