Add generic raw output validators

internal/framework/contracts/contracts.go

@@ -248,9 +248,10 @@ type Validator interface {
 }
 
 type ResponseSchema struct {
-	ID      string `json:"id,omitempty"`
-	Name    string `json:"name,omitempty"`
-	Version string `json:"version,omitempty"`
+	ID         string `json:"id,omitempty"`
+	Name       string `json:"name,omitempty"`
+	Version    string `json:"version,omitempty"`
+	JSONSchema []byte `json:"-"`
 }
 
 type ExtractOutput struct {

internal/framework/contracts/contracts_test.go

@@ -322,6 +322,32 @@ func TestLLMInputSetCloneCopiesContent(t *testing.T) {
 	}
 }
 
+func TestResponseSchemaJSONOmitRawSchemaContent(t *testing.T) {
+	schema := ResponseSchema{
+		ID:         "schema-id",
+		Name:       "schema-name",
+		Version:    "v1",
+		JSONSchema: []byte(`{"type":"object"}`),
+	}
+	encoded, err := json.Marshal(schema)
+	if err != nil {
+		t.Fatalf("json.Marshal() error = %v, want nil", err)
+	}
+	var got map[string]any
+	if err := json.Unmarshal(encoded, &got); err != nil {
+		t.Fatalf("json.Unmarshal() error = %v, want nil", err)
+	}
+	if got["id"] != "schema-id" || got["name"] != "schema-name" || got["version"] != "v1" {
+		t.Fatalf("encoded schema = %#v, want schema provenance", got)
+	}
+	if _, ok := got["json_schema"]; ok {
+		t.Fatalf("encoded schema leaked raw schema content: %s", encoded)
+	}
+	if _, ok := got["JSONSchema"]; ok {
+		t.Fatalf("encoded schema leaked raw schema content: %s", encoded)
+	}
+}
+
 func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
 	extractOutput := ExtractOutput{
 		LaneID:       "generic-lane",

internal/framework/pipeline/runner.go

@@ -551,7 +551,7 @@ func (target rawValidationTarget) validationRequest(binding ModuleBinding) contr
 		LLMProfile:     binding.LLMProfile,
 		Options:        cloneOptions(binding.Options),
 		Metadata:       cloneMetadata(target.metadata),
-		Schema:         target.schema,
+		Schema:         cloneResponseSchema(target.schema),
 		Payload:        cloneRawPayload(target.payload),
 		ChunkID:        target.chunkID,
 		ChunkIndex:     target.chunkIndex,
@@ -1012,6 +1012,11 @@ func cloneRawPayload(payload contracts.RawPayload) contracts.RawPayload {
 	}
 }
 
+func cloneResponseSchema(schema contracts.ResponseSchema) contracts.ResponseSchema {
+	schema.JSONSchema = append([]byte(nil), schema.JSONSchema...)
+	return schema
+}
+
 func cloneSourceChunkPtr(chunk *contracts.SourceChunk) *contracts.SourceChunk {
 	if chunk == nil {
 		return nil
@@ -1050,6 +1055,7 @@ func cloneSourceUnits(units []source.SourceUnit) []source.SourceUnit {
 }
 
 func cloneExtractOutput(output contracts.ExtractOutput) contracts.ExtractOutput {
+	output.Schema = cloneResponseSchema(output.Schema)
 	output.Payload = cloneRawPayload(output.Payload)
 	return output
 }
@@ -1066,11 +1072,13 @@ func cloneExtractOutputs(outputs []contracts.ExtractOutput) []contracts.ExtractO
 }
 
 func cloneMergeOutput(output contracts.MergeOutput) contracts.MergeOutput {
+	output.Schema = cloneResponseSchema(output.Schema)
 	output.Payload = cloneRawPayload(output.Payload)
 	return output
 }
 
 func cloneNormalizeOutput(output contracts.NormalizeOutput) contracts.NormalizeOutput {
+	output.Schema = cloneResponseSchema(output.Schema)
 	output.Payload = cloneRawPayload(output.Payload)
 	return output
 }

internal/modules/extract/dnd/spells/extractor.go

@@ -115,12 +115,17 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.ExtractionRequest
 			return contracts.ExtractionResult{}, extractorErrorf("marshal raw output: %w", err)
 		}
 	}
+	schema, err := loadResponseSchema()
+	if err != nil {
+		return contracts.ExtractionResult{}, extractorErrorf("load response schema: %w", err)
+	}
 	return contracts.ExtractionResult{
 		Output: contracts.ExtractOutput{
 			Schema: contracts.ResponseSchema{
-				ID:      ResponseSchemaID,
-				Name:    ResponseSchemaName,
-				Version: SchemaVersion,
+				ID:         ResponseSchemaID,
+				Name:       ResponseSchemaName,
+				Version:    SchemaVersion,
+				JSONSchema: append([]byte(nil), schema.JSONSchema...),
 			},
 			Payload: contracts.RawPayload{
 				Content:   content,

internal/modules/extract/dnd/spells/extractor_test.go

@@ -60,6 +60,9 @@ func TestExtractReturnsRawOutputFromStructuredResponse(t *testing.T) {
 	if result.Output.Schema.ID != ResponseSchemaID || result.Output.Schema.Name != ResponseSchemaName || result.Output.Schema.Version != SchemaVersion {
 		t.Fatalf("schema = %#v, want response schema provenance", result.Output.Schema)
 	}
+	if !json.Valid(result.Output.Schema.JSONSchema) {
+		t.Fatalf("schema JSON is invalid or missing: %s", result.Output.Schema.JSONSchema)
+	}
 	if got := string(result.Output.Payload.Content); got != string(client.content) {
 		t.Fatalf("content = %q, want exact raw completion content", got)
 	}

internal/modules/merge/appendorder/merger.go

@@ -189,18 +189,28 @@ func commonSchema(outputs []contracts.ExtractOutput) contracts.ResponseSchema {
 	}
 	schema := outputs[0].Schema
 	for _, output := range outputs[1:] {
-		if output.Schema != schema {
+		if !sameResponseSchema(output.Schema, schema) {
 			return contracts.ResponseSchema{}
 		}
 	}
 	return schema
 }
 
+func sameResponseSchema(left contracts.ResponseSchema, right contracts.ResponseSchema) bool {
+	return left.ID == right.ID && left.Name == right.Name && left.Version == right.Version && string(left.JSONSchema) == string(right.JSONSchema)
+}
+
 func cloneExtractOutput(output contracts.ExtractOutput) contracts.ExtractOutput {
+	output.Schema = cloneResponseSchema(output.Schema)
 	output.Payload = cloneRawPayload(output.Payload)
 	return output
 }
 
+func cloneResponseSchema(schema contracts.ResponseSchema) contracts.ResponseSchema {
+	schema.JSONSchema = append([]byte(nil), schema.JSONSchema...)
+	return schema
+}
+
 func cloneRawPayload(payload contracts.RawPayload) contracts.RawPayload {
 	return contracts.RawPayload{
 		Content:   append([]byte(nil), payload.Content...),

internal/validators/generic/always_accept/validator.go

@@ -0,0 +1,43 @@
+package alwaysaccept
+
+import (
+	"context"
+
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
+)
+
+const Key = "generic/always_accept"
+
+var _ contracts.Validator = (*Validator)(nil)
+
+type Validator struct{}
+
+func New() *Validator {
+	return &Validator{}
+}
+
+func (v *Validator) Name() string {
+	return Key
+}
+
+func (v *Validator) ExecutionClass() contracts.ExecutionClass {
+	return contracts.ExecutionClassDeterministic
+}
+
+func (v *Validator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
+	return contracts.ValidationResult{Approved: true}, nil
+}
+
+func Spec() pipeline.ValidatorSpec {
+	return pipeline.ValidatorSpec{
+		Key:            Key,
+		ExecutionClass: contracts.ExecutionClassDeterministic,
+	}
+}
+
+func Register(registry *pipeline.ValidatorRegistry) error {
+	return registry.RegisterWithSpec(Spec(), func() (contracts.Validator, error) {
+		return New(), nil
+	})
+}

internal/validators/generic/always_accept/validator_test.go

@@ -0,0 +1,40 @@
+package alwaysaccept
+
+import (
+	"context"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
+)
+
+func TestValidatorApproves(t *testing.T) {
+	result, err := New().Validate(context.Background(), contracts.ValidationRequest{})
+	if err != nil {
+		t.Fatalf("Validate() error = %v, want nil", err)
+	}
+	if !result.Approved {
+		t.Fatalf("Approved = false, want true")
+	}
+	if result.ReasonCode != "" || result.Message != "" {
+		t.Fatalf("result = %#v, want approval without rejection details", result)
+	}
+}
+
+func TestSpecAndRegister(t *testing.T) {
+	if Spec().Key != Key || Spec().ExecutionClass != contracts.ExecutionClassDeterministic {
+		t.Fatalf("Spec() = %#v, want key and deterministic execution", Spec())
+	}
+
+	registry := pipeline.NewValidatorRegistry()
+	if err := Register(registry); err != nil {
+		t.Fatalf("Register() error = %v, want nil", err)
+	}
+	validator, err := registry.Build(Key)
+	if err != nil {
+		t.Fatalf("Build(%q) error = %v, want nil", Key, err)
+	}
+	if validator.Name() != Key {
+		t.Fatalf("Name() = %q, want %q", validator.Name(), Key)
+	}
+}

internal/validators/generic/always_reject/validator.go

@@ -0,0 +1,48 @@
+package alwaysreject
+
+import (
+	"context"
+
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
+)
+
+const Key = "generic/always_reject"
+const ReasonCode = "always_reject"
+
+var _ contracts.Validator = (*Validator)(nil)
+
+type Validator struct{}
+
+func New() *Validator {
+	return &Validator{}
+}
+
+func (v *Validator) Name() string {
+	return Key
+}
+
+func (v *Validator) ExecutionClass() contracts.ExecutionClass {
+	return contracts.ExecutionClassDeterministic
+}
+
+func (v *Validator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
+	return contracts.ValidationResult{
+		Approved:   false,
+		ReasonCode: ReasonCode,
+		Message:    "output rejected by always-reject validator",
+	}, nil
+}
+
+func Spec() pipeline.ValidatorSpec {
+	return pipeline.ValidatorSpec{
+		Key:            Key,
+		ExecutionClass: contracts.ExecutionClassDeterministic,
+	}
+}
+
+func Register(registry *pipeline.ValidatorRegistry) error {
+	return registry.RegisterWithSpec(Spec(), func() (contracts.Validator, error) {
+		return New(), nil
+	})
+}

internal/validators/generic/always_reject/validator_test.go

@@ -0,0 +1,43 @@
+package alwaysreject
+
+import (
+	"context"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
+)
+
+func TestValidatorRejects(t *testing.T) {
+	result, err := New().Validate(context.Background(), contracts.ValidationRequest{})
+	if err != nil {
+		t.Fatalf("Validate() error = %v, want nil", err)
+	}
+	if result.Approved {
+		t.Fatalf("Approved = true, want false")
+	}
+	if result.ReasonCode != ReasonCode {
+		t.Fatalf("ReasonCode = %q, want %q", result.ReasonCode, ReasonCode)
+	}
+	if result.Message == "" {
+		t.Fatal("Message = empty, want rejection message")
+	}
+}
+
+func TestSpecAndRegister(t *testing.T) {
+	if Spec().Key != Key || Spec().ExecutionClass != contracts.ExecutionClassDeterministic {
+		t.Fatalf("Spec() = %#v, want key and deterministic execution", Spec())
+	}
+
+	registry := pipeline.NewValidatorRegistry()
+	if err := Register(registry); err != nil {
+		t.Fatalf("Register() error = %v, want nil", err)
+	}
+	validator, err := registry.Build(Key)
+	if err != nil {
+		t.Fatalf("Build(%q) error = %v, want nil", Key, err)
+	}
+	if validator.Name() != Key {
+		t.Fatalf("Name() = %q, want %q", validator.Name(), Key)
+	}
+}

internal/validators/generic/valid_json/validator.go

@@ -0,0 +1,52 @@
+package validjson
+
+import (
+	"context"
+	"encoding/json"
+
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
+)
+
+const Key = "generic/valid_json"
+const ReasonCodeInvalidJSON = "invalid_json"
+
+var _ contracts.Validator = (*Validator)(nil)
+
+type Validator struct{}
+
+func New() *Validator {
+	return &Validator{}
+}
+
+func (v *Validator) Name() string {
+	return Key
+}
+
+func (v *Validator) ExecutionClass() contracts.ExecutionClass {
+	return contracts.ExecutionClassDeterministic
+}
+
+func (v *Validator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
+	if !json.Valid(req.Payload.Content) {
+		return contracts.ValidationResult{
+			Approved:   false,
+			ReasonCode: ReasonCodeInvalidJSON,
+			Message:    "payload is not valid JSON",
+		}, nil
+	}
+	return contracts.ValidationResult{Approved: true}, nil
+}
+
+func Spec() pipeline.ValidatorSpec {
+	return pipeline.ValidatorSpec{
+		Key:            Key,
+		ExecutionClass: contracts.ExecutionClassDeterministic,
+	}
+}
+
+func Register(registry *pipeline.ValidatorRegistry) error {
+	return registry.RegisterWithSpec(Spec(), func() (contracts.Validator, error) {
+		return New(), nil
+	})
+}

internal/validators/generic/valid_json/validator_test.go

@@ -0,0 +1,66 @@
+package validjson
+
+import (
+	"context"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
+)
+
+func TestValidatorAcceptsValidJSON(t *testing.T) {
+	tests := []string{
+		`{"value":true}`,
+		`[1,2,3]`,
+		`"value"`,
+	}
+	for _, payload := range tests {
+		result, err := New().Validate(context.Background(), requestWithPayload(payload))
+		if err != nil {
+			t.Fatalf("Validate(%s) error = %v, want nil", payload, err)
+		}
+		if !result.Approved {
+			t.Fatalf("Validate(%s) = %#v, want approved", payload, result)
+		}
+	}
+}
+
+func TestValidatorRejectsInvalidJSON(t *testing.T) {
+	result, err := New().Validate(context.Background(), requestWithPayload(`{"value":`))
+	if err != nil {
+		t.Fatalf("Validate() error = %v, want nil", err)
+	}
+	if result.Approved {
+		t.Fatalf("Approved = true, want false")
+	}
+	if result.ReasonCode != ReasonCodeInvalidJSON {
+		t.Fatalf("ReasonCode = %q, want %q", result.ReasonCode, ReasonCodeInvalidJSON)
+	}
+}
+
+func TestSpecAndRegister(t *testing.T) {
+	if Spec().Key != Key || Spec().ExecutionClass != contracts.ExecutionClassDeterministic {
+		t.Fatalf("Spec() = %#v, want key and deterministic execution", Spec())
+	}
+
+	registry := pipeline.NewValidatorRegistry()
+	if err := Register(registry); err != nil {
+		t.Fatalf("Register() error = %v, want nil", err)
+	}
+	validator, err := registry.Build(Key)
+	if err != nil {
+		t.Fatalf("Build(%q) error = %v, want nil", Key, err)
+	}
+	if validator.Name() != Key {
+		t.Fatalf("Name() = %q, want %q", validator.Name(), Key)
+	}
+}
+
+func requestWithPayload(payload string) contracts.ValidationRequest {
+	return contracts.ValidationRequest{
+		Payload: contracts.RawPayload{
+			Content:   []byte(payload),
+			MediaType: "application/json",
+		},
+	}
+}

internal/validators/generic/valid_json_schema/validator.go

@@ -0,0 +1,81 @@
+package validjsonschema
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+
+	"github.com/santhosh-tekuri/jsonschema/v6"
+
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
+)
+
+const Key = "generic/valid_json_schema"
+const ReasonCodeInvalidJSON = "invalid_json"
+const ReasonCodeSchemaInvalid = "json_schema_invalid"
+
+var _ contracts.Validator = (*Validator)(nil)
+
+type Validator struct{}
+
+func New() *Validator {
+	return &Validator{}
+}
+
+func (v *Validator) Name() string {
+	return Key
+}
+
+func (v *Validator) ExecutionClass() contracts.ExecutionClass {
+	return contracts.ExecutionClassDeterministic
+}
+
+func (v *Validator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
+	if len(req.Schema.JSONSchema) == 0 {
+		return contracts.ValidationResult{}, fmt.Errorf("response schema content is not available")
+	}
+
+	instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(req.Payload.Content))
+	if err != nil {
+		return contracts.ValidationResult{
+			Approved:   false,
+			ReasonCode: ReasonCodeInvalidJSON,
+			Message:    "payload is not valid JSON",
+		}, nil
+	}
+
+	schemaDocument, err := jsonschema.UnmarshalJSON(bytes.NewReader(req.Schema.JSONSchema))
+	if err != nil {
+		return contracts.ValidationResult{}, fmt.Errorf("parse response schema: %w", err)
+	}
+	compiler := jsonschema.NewCompiler()
+	if err := compiler.AddResource("schema.json", schemaDocument); err != nil {
+		return contracts.ValidationResult{}, fmt.Errorf("load response schema: %w", err)
+	}
+	schema, err := compiler.Compile("schema.json")
+	if err != nil {
+		return contracts.ValidationResult{}, fmt.Errorf("compile response schema: %w", err)
+	}
+	if err := schema.Validate(instance); err != nil {
+		return contracts.ValidationResult{
+			Approved:   false,
+			ReasonCode: ReasonCodeSchemaInvalid,
+			Message:    "payload does not conform to response schema",
+		}, nil
+	}
+	return contracts.ValidationResult{Approved: true}, nil
+}
+
+func Spec() pipeline.ValidatorSpec {
+	return pipeline.ValidatorSpec{
+		Key:            Key,
+		ExecutionClass: contracts.ExecutionClassDeterministic,
+	}
+}
+
+func Register(registry *pipeline.ValidatorRegistry) error {
+	return registry.RegisterWithSpec(Spec(), func() (contracts.Validator, error) {
+		return New(), nil
+	})
+}

internal/validators/generic/valid_json_schema/validator_test.go

@@ -0,0 +1,111 @@
+package validjsonschema
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
+)
+
+func TestValidatorAcceptsSchemaConformantJSON(t *testing.T) {
+	result, err := New().Validate(context.Background(), requestWithSchema(`{"name":"Aria"}`, objectSchema()))
+	if err != nil {
+		t.Fatalf("Validate() error = %v, want nil", err)
+	}
+	if !result.Approved {
+		t.Fatalf("Validate() = %#v, want approved", result)
+	}
+}
+
+func TestValidatorRejectsInvalidPayloadJSON(t *testing.T) {
+	result, err := New().Validate(context.Background(), requestWithSchema(`{"name":`, objectSchema()))
+	if err != nil {
+		t.Fatalf("Validate() error = %v, want nil", err)
+	}
+	if result.Approved {
+		t.Fatalf("Approved = true, want false")
+	}
+	if result.ReasonCode != ReasonCodeInvalidJSON {
+		t.Fatalf("ReasonCode = %q, want %q", result.ReasonCode, ReasonCodeInvalidJSON)
+	}
+}
+
+func TestValidatorRejectsSchemaNonConformance(t *testing.T) {
+	result, err := New().Validate(context.Background(), requestWithSchema(`{"name":3}`, objectSchema()))
+	if err != nil {
+		t.Fatalf("Validate() error = %v, want nil", err)
+	}
+	if result.Approved {
+		t.Fatalf("Approved = true, want false")
+	}
+	if result.ReasonCode != ReasonCodeSchemaInvalid {
+		t.Fatalf("ReasonCode = %q, want %q", result.ReasonCode, ReasonCodeSchemaInvalid)
+	}
+}
+
+func TestValidatorErrorsWhenSchemaContentMissing(t *testing.T) {
+	_, err := New().Validate(context.Background(), requestWithSchema(`{"name":"Aria"}`, nil))
+	if err == nil {
+		t.Fatal("Validate() error = nil, want missing schema content error")
+	}
+	if !strings.Contains(err.Error(), "schema content") {
+		t.Fatalf("Validate() error = %q, want schema content context", err.Error())
+	}
+}
+
+func TestValidatorErrorsWhenSchemaContentIsMalformed(t *testing.T) {
+	_, err := New().Validate(context.Background(), requestWithSchema(`{"name":"Aria"}`, []byte(`{"type":`)))
+	if err == nil {
+		t.Fatal("Validate() error = nil, want malformed schema error")
+	}
+	if !strings.Contains(err.Error(), "parse response schema") {
+		t.Fatalf("Validate() error = %q, want parse schema context", err.Error())
+	}
+}
+
+func TestSpecAndRegister(t *testing.T) {
+	if Spec().Key != Key || Spec().ExecutionClass != contracts.ExecutionClassDeterministic {
+		t.Fatalf("Spec() = %#v, want key and deterministic execution", Spec())
+	}
+
+	registry := pipeline.NewValidatorRegistry()
+	if err := Register(registry); err != nil {
+		t.Fatalf("Register() error = %v, want nil", err)
+	}
+	validator, err := registry.Build(Key)
+	if err != nil {
+		t.Fatalf("Build(%q) error = %v, want nil", Key, err)
+	}
+	if validator.Name() != Key {
+		t.Fatalf("Name() = %q, want %q", validator.Name(), Key)
+	}
+}
+
+func requestWithSchema(payload string, schema []byte) contracts.ValidationRequest {
+	return contracts.ValidationRequest{
+		Schema: contracts.ResponseSchema{
+			ID:         "test.schema",
+			Name:       "test_schema",
+			Version:    "v1",
+			JSONSchema: append([]byte(nil), schema...),
+		},
+		Payload: contracts.RawPayload{
+			Content:   []byte(payload),
+			MediaType: "application/json",
+		},
+	}
+}
+
+func objectSchema() []byte {
+	return []byte(`{
+		"$schema": "https://json-schema.org/draft/2020-12/schema",
+		"type": "object",
+		"required": ["name"],
+		"properties": {
+			"name": {"type": "string"}
+		},
+		"additionalProperties": false
+	}`)
+}