Add generic raw output validators

This commit is contained in:
2026-07-07 21:32:56 +00:00
parent 5ef027b6f0
commit 3e67be6ac3
16 changed files with 550 additions and 12 deletions

View File

@@ -188,6 +188,9 @@ lane, module, source, chunk, schema, session, reference, LLM client/profile,
binding option, and run metadata context. Merge validators also receive the
ordered extract outputs used by the merge, and normalize validators receive the
accepted merge output. Empty raw validation chains approve output by default.
Response-schema provenance may include in-memory JSON schema bytes for
validators. Those bytes are omitted from manifests, diagnostics, and encoded
output files.
Resolved validator chains come from central default mappings unless a
stage-local config override is set on `chunk`, lane `extract`, lane `merge`, or

6
go.mod
View File

@@ -4,10 +4,8 @@ go 1.25.5
require (
gitea.maximumdirect.net/eric/scriptorium v0.11.0
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
gopkg.in/yaml.v3 v3.0.1
)
require (
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
golang.org/x/text v0.14.0 // indirect
)
require golang.org/x/text v0.14.0 // indirect

View File

@@ -248,9 +248,10 @@ type Validator interface {
}
type ResponseSchema struct {
ID string `json:"id,omitempty"`
Name string `json:"name,omitempty"`
Version string `json:"version,omitempty"`
ID string `json:"id,omitempty"`
Name string `json:"name,omitempty"`
Version string `json:"version,omitempty"`
JSONSchema []byte `json:"-"`
}
type ExtractOutput struct {

View File

@@ -322,6 +322,32 @@ func TestLLMInputSetCloneCopiesContent(t *testing.T) {
}
}
func TestResponseSchemaJSONOmitRawSchemaContent(t *testing.T) {
schema := ResponseSchema{
ID: "schema-id",
Name: "schema-name",
Version: "v1",
JSONSchema: []byte(`{"type":"object"}`),
}
encoded, err := json.Marshal(schema)
if err != nil {
t.Fatalf("json.Marshal() error = %v, want nil", err)
}
var got map[string]any
if err := json.Unmarshal(encoded, &got); err != nil {
t.Fatalf("json.Unmarshal() error = %v, want nil", err)
}
if got["id"] != "schema-id" || got["name"] != "schema-name" || got["version"] != "v1" {
t.Fatalf("encoded schema = %#v, want schema provenance", got)
}
if _, ok := got["json_schema"]; ok {
t.Fatalf("encoded schema leaked raw schema content: %s", encoded)
}
if _, ok := got["JSONSchema"]; ok {
t.Fatalf("encoded schema leaked raw schema content: %s", encoded)
}
}
func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
extractOutput := ExtractOutput{
LaneID: "generic-lane",

View File

@@ -551,7 +551,7 @@ func (target rawValidationTarget) validationRequest(binding ModuleBinding) contr
LLMProfile: binding.LLMProfile,
Options: cloneOptions(binding.Options),
Metadata: cloneMetadata(target.metadata),
Schema: target.schema,
Schema: cloneResponseSchema(target.schema),
Payload: cloneRawPayload(target.payload),
ChunkID: target.chunkID,
ChunkIndex: target.chunkIndex,
@@ -1012,6 +1012,11 @@ func cloneRawPayload(payload contracts.RawPayload) contracts.RawPayload {
}
}
func cloneResponseSchema(schema contracts.ResponseSchema) contracts.ResponseSchema {
schema.JSONSchema = append([]byte(nil), schema.JSONSchema...)
return schema
}
func cloneSourceChunkPtr(chunk *contracts.SourceChunk) *contracts.SourceChunk {
if chunk == nil {
return nil
@@ -1050,6 +1055,7 @@ func cloneSourceUnits(units []source.SourceUnit) []source.SourceUnit {
}
func cloneExtractOutput(output contracts.ExtractOutput) contracts.ExtractOutput {
output.Schema = cloneResponseSchema(output.Schema)
output.Payload = cloneRawPayload(output.Payload)
return output
}
@@ -1066,11 +1072,13 @@ func cloneExtractOutputs(outputs []contracts.ExtractOutput) []contracts.ExtractO
}
func cloneMergeOutput(output contracts.MergeOutput) contracts.MergeOutput {
output.Schema = cloneResponseSchema(output.Schema)
output.Payload = cloneRawPayload(output.Payload)
return output
}
func cloneNormalizeOutput(output contracts.NormalizeOutput) contracts.NormalizeOutput {
output.Schema = cloneResponseSchema(output.Schema)
output.Payload = cloneRawPayload(output.Payload)
return output
}

View File

@@ -115,12 +115,17 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.ExtractionRequest
return contracts.ExtractionResult{}, extractorErrorf("marshal raw output: %w", err)
}
}
schema, err := loadResponseSchema()
if err != nil {
return contracts.ExtractionResult{}, extractorErrorf("load response schema: %w", err)
}
return contracts.ExtractionResult{
Output: contracts.ExtractOutput{
Schema: contracts.ResponseSchema{
ID: ResponseSchemaID,
Name: ResponseSchemaName,
Version: SchemaVersion,
ID: ResponseSchemaID,
Name: ResponseSchemaName,
Version: SchemaVersion,
JSONSchema: append([]byte(nil), schema.JSONSchema...),
},
Payload: contracts.RawPayload{
Content: content,

View File

@@ -60,6 +60,9 @@ func TestExtractReturnsRawOutputFromStructuredResponse(t *testing.T) {
if result.Output.Schema.ID != ResponseSchemaID || result.Output.Schema.Name != ResponseSchemaName || result.Output.Schema.Version != SchemaVersion {
t.Fatalf("schema = %#v, want response schema provenance", result.Output.Schema)
}
if !json.Valid(result.Output.Schema.JSONSchema) {
t.Fatalf("schema JSON is invalid or missing: %s", result.Output.Schema.JSONSchema)
}
if got := string(result.Output.Payload.Content); got != string(client.content) {
t.Fatalf("content = %q, want exact raw completion content", got)
}

View File

@@ -189,18 +189,28 @@ func commonSchema(outputs []contracts.ExtractOutput) contracts.ResponseSchema {
}
schema := outputs[0].Schema
for _, output := range outputs[1:] {
if output.Schema != schema {
if !sameResponseSchema(output.Schema, schema) {
return contracts.ResponseSchema{}
}
}
return schema
}
func sameResponseSchema(left contracts.ResponseSchema, right contracts.ResponseSchema) bool {
return left.ID == right.ID && left.Name == right.Name && left.Version == right.Version && string(left.JSONSchema) == string(right.JSONSchema)
}
func cloneExtractOutput(output contracts.ExtractOutput) contracts.ExtractOutput {
output.Schema = cloneResponseSchema(output.Schema)
output.Payload = cloneRawPayload(output.Payload)
return output
}
func cloneResponseSchema(schema contracts.ResponseSchema) contracts.ResponseSchema {
schema.JSONSchema = append([]byte(nil), schema.JSONSchema...)
return schema
}
func cloneRawPayload(payload contracts.RawPayload) contracts.RawPayload {
return contracts.RawPayload{
Content: append([]byte(nil), payload.Content...),

View File

@@ -0,0 +1,43 @@
package alwaysaccept
import (
"context"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
const Key = "generic/always_accept"
var _ contracts.Validator = (*Validator)(nil)
type Validator struct{}
func New() *Validator {
return &Validator{}
}
func (v *Validator) Name() string {
return Key
}
func (v *Validator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v *Validator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: true}, nil
}
func Spec() pipeline.ValidatorSpec {
return pipeline.ValidatorSpec{
Key: Key,
ExecutionClass: contracts.ExecutionClassDeterministic,
}
}
func Register(registry *pipeline.ValidatorRegistry) error {
return registry.RegisterWithSpec(Spec(), func() (contracts.Validator, error) {
return New(), nil
})
}

View File

@@ -0,0 +1,40 @@
package alwaysaccept
import (
"context"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func TestValidatorApproves(t *testing.T) {
result, err := New().Validate(context.Background(), contracts.ValidationRequest{})
if err != nil {
t.Fatalf("Validate() error = %v, want nil", err)
}
if !result.Approved {
t.Fatalf("Approved = false, want true")
}
if result.ReasonCode != "" || result.Message != "" {
t.Fatalf("result = %#v, want approval without rejection details", result)
}
}
func TestSpecAndRegister(t *testing.T) {
if Spec().Key != Key || Spec().ExecutionClass != contracts.ExecutionClassDeterministic {
t.Fatalf("Spec() = %#v, want key and deterministic execution", Spec())
}
registry := pipeline.NewValidatorRegistry()
if err := Register(registry); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
validator, err := registry.Build(Key)
if err != nil {
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
}
if validator.Name() != Key {
t.Fatalf("Name() = %q, want %q", validator.Name(), Key)
}
}

View File

@@ -0,0 +1,48 @@
package alwaysreject
import (
"context"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
const Key = "generic/always_reject"
const ReasonCode = "always_reject"
var _ contracts.Validator = (*Validator)(nil)
type Validator struct{}
func New() *Validator {
return &Validator{}
}
func (v *Validator) Name() string {
return Key
}
func (v *Validator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v *Validator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
return contracts.ValidationResult{
Approved: false,
ReasonCode: ReasonCode,
Message: "output rejected by always-reject validator",
}, nil
}
func Spec() pipeline.ValidatorSpec {
return pipeline.ValidatorSpec{
Key: Key,
ExecutionClass: contracts.ExecutionClassDeterministic,
}
}
func Register(registry *pipeline.ValidatorRegistry) error {
return registry.RegisterWithSpec(Spec(), func() (contracts.Validator, error) {
return New(), nil
})
}

View File

@@ -0,0 +1,43 @@
package alwaysreject
import (
"context"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func TestValidatorRejects(t *testing.T) {
result, err := New().Validate(context.Background(), contracts.ValidationRequest{})
if err != nil {
t.Fatalf("Validate() error = %v, want nil", err)
}
if result.Approved {
t.Fatalf("Approved = true, want false")
}
if result.ReasonCode != ReasonCode {
t.Fatalf("ReasonCode = %q, want %q", result.ReasonCode, ReasonCode)
}
if result.Message == "" {
t.Fatal("Message = empty, want rejection message")
}
}
func TestSpecAndRegister(t *testing.T) {
if Spec().Key != Key || Spec().ExecutionClass != contracts.ExecutionClassDeterministic {
t.Fatalf("Spec() = %#v, want key and deterministic execution", Spec())
}
registry := pipeline.NewValidatorRegistry()
if err := Register(registry); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
validator, err := registry.Build(Key)
if err != nil {
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
}
if validator.Name() != Key {
t.Fatalf("Name() = %q, want %q", validator.Name(), Key)
}
}

View File

@@ -0,0 +1,52 @@
package validjson
import (
"context"
"encoding/json"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
const Key = "generic/valid_json"
const ReasonCodeInvalidJSON = "invalid_json"
var _ contracts.Validator = (*Validator)(nil)
type Validator struct{}
func New() *Validator {
return &Validator{}
}
func (v *Validator) Name() string {
return Key
}
func (v *Validator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v *Validator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
if !json.Valid(req.Payload.Content) {
return contracts.ValidationResult{
Approved: false,
ReasonCode: ReasonCodeInvalidJSON,
Message: "payload is not valid JSON",
}, nil
}
return contracts.ValidationResult{Approved: true}, nil
}
func Spec() pipeline.ValidatorSpec {
return pipeline.ValidatorSpec{
Key: Key,
ExecutionClass: contracts.ExecutionClassDeterministic,
}
}
func Register(registry *pipeline.ValidatorRegistry) error {
return registry.RegisterWithSpec(Spec(), func() (contracts.Validator, error) {
return New(), nil
})
}

View File

@@ -0,0 +1,66 @@
package validjson
import (
"context"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func TestValidatorAcceptsValidJSON(t *testing.T) {
tests := []string{
`{"value":true}`,
`[1,2,3]`,
`"value"`,
}
for _, payload := range tests {
result, err := New().Validate(context.Background(), requestWithPayload(payload))
if err != nil {
t.Fatalf("Validate(%s) error = %v, want nil", payload, err)
}
if !result.Approved {
t.Fatalf("Validate(%s) = %#v, want approved", payload, result)
}
}
}
func TestValidatorRejectsInvalidJSON(t *testing.T) {
result, err := New().Validate(context.Background(), requestWithPayload(`{"value":`))
if err != nil {
t.Fatalf("Validate() error = %v, want nil", err)
}
if result.Approved {
t.Fatalf("Approved = true, want false")
}
if result.ReasonCode != ReasonCodeInvalidJSON {
t.Fatalf("ReasonCode = %q, want %q", result.ReasonCode, ReasonCodeInvalidJSON)
}
}
func TestSpecAndRegister(t *testing.T) {
if Spec().Key != Key || Spec().ExecutionClass != contracts.ExecutionClassDeterministic {
t.Fatalf("Spec() = %#v, want key and deterministic execution", Spec())
}
registry := pipeline.NewValidatorRegistry()
if err := Register(registry); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
validator, err := registry.Build(Key)
if err != nil {
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
}
if validator.Name() != Key {
t.Fatalf("Name() = %q, want %q", validator.Name(), Key)
}
}
func requestWithPayload(payload string) contracts.ValidationRequest {
return contracts.ValidationRequest{
Payload: contracts.RawPayload{
Content: []byte(payload),
MediaType: "application/json",
},
}
}

View File

@@ -0,0 +1,81 @@
package validjsonschema
import (
"bytes"
"context"
"fmt"
"github.com/santhosh-tekuri/jsonschema/v6"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
const Key = "generic/valid_json_schema"
const ReasonCodeInvalidJSON = "invalid_json"
const ReasonCodeSchemaInvalid = "json_schema_invalid"
var _ contracts.Validator = (*Validator)(nil)
type Validator struct{}
func New() *Validator {
return &Validator{}
}
func (v *Validator) Name() string {
return Key
}
func (v *Validator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v *Validator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
if len(req.Schema.JSONSchema) == 0 {
return contracts.ValidationResult{}, fmt.Errorf("response schema content is not available")
}
instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(req.Payload.Content))
if err != nil {
return contracts.ValidationResult{
Approved: false,
ReasonCode: ReasonCodeInvalidJSON,
Message: "payload is not valid JSON",
}, nil
}
schemaDocument, err := jsonschema.UnmarshalJSON(bytes.NewReader(req.Schema.JSONSchema))
if err != nil {
return contracts.ValidationResult{}, fmt.Errorf("parse response schema: %w", err)
}
compiler := jsonschema.NewCompiler()
if err := compiler.AddResource("schema.json", schemaDocument); err != nil {
return contracts.ValidationResult{}, fmt.Errorf("load response schema: %w", err)
}
schema, err := compiler.Compile("schema.json")
if err != nil {
return contracts.ValidationResult{}, fmt.Errorf("compile response schema: %w", err)
}
if err := schema.Validate(instance); err != nil {
return contracts.ValidationResult{
Approved: false,
ReasonCode: ReasonCodeSchemaInvalid,
Message: "payload does not conform to response schema",
}, nil
}
return contracts.ValidationResult{Approved: true}, nil
}
func Spec() pipeline.ValidatorSpec {
return pipeline.ValidatorSpec{
Key: Key,
ExecutionClass: contracts.ExecutionClassDeterministic,
}
}
func Register(registry *pipeline.ValidatorRegistry) error {
return registry.RegisterWithSpec(Spec(), func() (contracts.Validator, error) {
return New(), nil
})
}

View File

@@ -0,0 +1,111 @@
package validjsonschema
import (
"context"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func TestValidatorAcceptsSchemaConformantJSON(t *testing.T) {
result, err := New().Validate(context.Background(), requestWithSchema(`{"name":"Aria"}`, objectSchema()))
if err != nil {
t.Fatalf("Validate() error = %v, want nil", err)
}
if !result.Approved {
t.Fatalf("Validate() = %#v, want approved", result)
}
}
func TestValidatorRejectsInvalidPayloadJSON(t *testing.T) {
result, err := New().Validate(context.Background(), requestWithSchema(`{"name":`, objectSchema()))
if err != nil {
t.Fatalf("Validate() error = %v, want nil", err)
}
if result.Approved {
t.Fatalf("Approved = true, want false")
}
if result.ReasonCode != ReasonCodeInvalidJSON {
t.Fatalf("ReasonCode = %q, want %q", result.ReasonCode, ReasonCodeInvalidJSON)
}
}
func TestValidatorRejectsSchemaNonConformance(t *testing.T) {
result, err := New().Validate(context.Background(), requestWithSchema(`{"name":3}`, objectSchema()))
if err != nil {
t.Fatalf("Validate() error = %v, want nil", err)
}
if result.Approved {
t.Fatalf("Approved = true, want false")
}
if result.ReasonCode != ReasonCodeSchemaInvalid {
t.Fatalf("ReasonCode = %q, want %q", result.ReasonCode, ReasonCodeSchemaInvalid)
}
}
func TestValidatorErrorsWhenSchemaContentMissing(t *testing.T) {
_, err := New().Validate(context.Background(), requestWithSchema(`{"name":"Aria"}`, nil))
if err == nil {
t.Fatal("Validate() error = nil, want missing schema content error")
}
if !strings.Contains(err.Error(), "schema content") {
t.Fatalf("Validate() error = %q, want schema content context", err.Error())
}
}
func TestValidatorErrorsWhenSchemaContentIsMalformed(t *testing.T) {
_, err := New().Validate(context.Background(), requestWithSchema(`{"name":"Aria"}`, []byte(`{"type":`)))
if err == nil {
t.Fatal("Validate() error = nil, want malformed schema error")
}
if !strings.Contains(err.Error(), "parse response schema") {
t.Fatalf("Validate() error = %q, want parse schema context", err.Error())
}
}
func TestSpecAndRegister(t *testing.T) {
if Spec().Key != Key || Spec().ExecutionClass != contracts.ExecutionClassDeterministic {
t.Fatalf("Spec() = %#v, want key and deterministic execution", Spec())
}
registry := pipeline.NewValidatorRegistry()
if err := Register(registry); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
validator, err := registry.Build(Key)
if err != nil {
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
}
if validator.Name() != Key {
t.Fatalf("Name() = %q, want %q", validator.Name(), Key)
}
}
func requestWithSchema(payload string, schema []byte) contracts.ValidationRequest {
return contracts.ValidationRequest{
Schema: contracts.ResponseSchema{
ID: "test.schema",
Name: "test_schema",
Version: "v1",
JSONSchema: append([]byte(nil), schema...),
},
Payload: contracts.RawPayload{
Content: []byte(payload),
MediaType: "application/json",
},
}
}
func objectSchema() []byte {
return []byte(`{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"required": ["name"],
"properties": {
"name": {"type": "string"}
},
"additionalProperties": false
}`)
}