Add shared semantic reconciliation engine
This commit is contained in:
@@ -1,3 +1,3 @@
|
||||
// Package semanticreconcile prepares and validates bounded materials for
|
||||
// domain-neutral semantic reconciliation.
|
||||
// Package semanticreconcile prepares, executes, and validates bounded
|
||||
// domain-neutral semantic reconciliation requests.
|
||||
package semanticreconcile
|
||||
|
||||
210
internal/framework/semanticreconcile/engine.go
Normal file
210
internal/framework/semanticreconcile/engine.go
Normal file
@@ -0,0 +1,210 @@
|
||||
package semanticreconcile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
)
|
||||
|
||||
// PromptSpec identifies the exact prompt selected by a reconciliation owner.
|
||||
type PromptSpec struct {
|
||||
ID string
|
||||
Version string
|
||||
SHA256 string
|
||||
}
|
||||
|
||||
// Validate rejects incomplete prompt identity and non-canonical digests.
|
||||
func (spec PromptSpec) Validate() error {
|
||||
if strings.TrimSpace(spec.ID) == "" {
|
||||
return fmt.Errorf("semantic reconciliation prompt ID must not be empty")
|
||||
}
|
||||
if strings.TrimSpace(spec.Version) == "" {
|
||||
return fmt.Errorf("semantic reconciliation prompt version must not be empty")
|
||||
}
|
||||
if err := validateSHA256(spec.SHA256); err != nil {
|
||||
return fmt.Errorf("semantic reconciliation prompt digest: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DefaultPromptSpec returns the identity of the core-owned generic prompt.
|
||||
func DefaultPromptSpec() (PromptSpec, error) {
|
||||
digest, err := PromptHash()
|
||||
if err != nil {
|
||||
return PromptSpec{}, err
|
||||
}
|
||||
return PromptSpec{ID: PromptID, Version: PromptVersion, SHA256: digest}, nil
|
||||
}
|
||||
|
||||
// Request contains one typed owner's source-backed reconciliation input.
|
||||
type Request struct {
|
||||
StageName string
|
||||
Source *source.SourceDocument
|
||||
Candidates []Candidate
|
||||
ProfileID string
|
||||
SessionID string
|
||||
}
|
||||
|
||||
// ResultDisposition classifies a provider-neutral reconciliation outcome.
|
||||
type ResultDisposition uint8
|
||||
|
||||
const (
|
||||
Complete ResultDisposition = iota + 1
|
||||
RetryableInvalidStructuredOutput
|
||||
RetryableDiscardedProposalGroups
|
||||
SkippedInsufficientCandidates
|
||||
SkippedLimitExceeded
|
||||
)
|
||||
|
||||
// Result owns the safe plan and neutral diagnostics from one call.
|
||||
type Result struct {
|
||||
disposition ResultDisposition
|
||||
plan Plan
|
||||
issues []Issue
|
||||
discardedGroupCount int
|
||||
candidateMappings []CandidateMapping
|
||||
}
|
||||
|
||||
// Disposition returns the classified outcome.
|
||||
func (result Result) Disposition() ResultDisposition { return result.disposition }
|
||||
|
||||
// Plan returns an independently owned safe plan.
|
||||
func (result Result) Plan() Plan { return result.planCopy() }
|
||||
|
||||
// Issues returns an owned copy of stable proposal issues.
|
||||
func (result Result) Issues() []Issue { return append([]Issue(nil), result.issues...) }
|
||||
|
||||
// DiscardedGroupCount returns the number of excluded proposal groups.
|
||||
func (result Result) DiscardedGroupCount() int { return result.discardedGroupCount }
|
||||
|
||||
// CandidateMappings returns the request-local handle mapping used for this call.
|
||||
func (result Result) CandidateMappings() []CandidateMapping {
|
||||
return append([]CandidateMapping(nil), result.candidateMappings...)
|
||||
}
|
||||
|
||||
func (result Result) planCopy() Plan {
|
||||
return Plan{groups: result.plan.Groups()}
|
||||
}
|
||||
|
||||
// Engine prepares bounded material, performs one structured completion, and
|
||||
// classifies the deterministic assessment without applying it to typed values.
|
||||
type Engine struct {
|
||||
client contracts.StructuredLLMClient
|
||||
prompt PromptSpec
|
||||
schema llm.ResponseSchema
|
||||
limits Limits
|
||||
}
|
||||
|
||||
// NewEngine constructs a reconciliation engine using the core response schema.
|
||||
func NewEngine(client contracts.StructuredLLMClient, prompt PromptSpec, limits Limits) (*Engine, error) {
|
||||
if client == nil {
|
||||
return nil, fmt.Errorf("construct semantic reconciliation engine: LLM client must not be nil")
|
||||
}
|
||||
if err := prompt.Validate(); err != nil {
|
||||
return nil, fmt.Errorf("construct semantic reconciliation engine: %w", err)
|
||||
}
|
||||
if err := limits.Validate(); err != nil {
|
||||
return nil, fmt.Errorf("construct semantic reconciliation engine: %w", err)
|
||||
}
|
||||
schema, err := LoadResponseSchema()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("construct semantic reconciliation engine: load response schema: %w", err)
|
||||
}
|
||||
return newEngine(client, prompt, schema, limits), nil
|
||||
}
|
||||
|
||||
func newEngine(client contracts.StructuredLLMClient, prompt PromptSpec, schema llm.ResponseSchema, limits Limits) *Engine {
|
||||
return &Engine{client: client, prompt: prompt, schema: schema, limits: limits}
|
||||
}
|
||||
|
||||
// Reconcile prepares and assesses one request. Retryable semantic outcomes are
|
||||
// returned as results; provider and transport failures remain errors.
|
||||
func (engine *Engine) Reconcile(ctx context.Context, request Request) (Result, error) {
|
||||
if err := engine.validate(); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if ctx == nil {
|
||||
return Result{}, fmt.Errorf("semantic reconciliation %q: context must not be nil", request.StageName)
|
||||
}
|
||||
if strings.TrimSpace(request.StageName) == "" {
|
||||
return Result{}, fmt.Errorf("semantic reconciliation stage name must not be empty")
|
||||
}
|
||||
if request.Source == nil {
|
||||
return Result{}, fmt.Errorf("semantic reconciliation %q: source document must not be nil", request.StageName)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return Result{}, fmt.Errorf("semantic reconciliation %q: context error before preparation: %w", request.StageName, err)
|
||||
}
|
||||
|
||||
preparation, err := Prepare(request.Source, request.Candidates, engine.limits)
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("semantic reconciliation %q: prepare materials: %w", request.StageName, err)
|
||||
}
|
||||
result := Result{candidateMappings: preparation.CandidateMappings()}
|
||||
switch preparation.Disposition() {
|
||||
case InsufficientCandidates:
|
||||
result.disposition = SkippedInsufficientCandidates
|
||||
return result, nil
|
||||
case LimitExceeded:
|
||||
result.disposition = SkippedLimitExceeded
|
||||
return result, nil
|
||||
case Ready:
|
||||
default:
|
||||
return Result{}, fmt.Errorf("semantic reconciliation %q: unknown preparation disposition %d", request.StageName, preparation.Disposition())
|
||||
}
|
||||
|
||||
if err := ctx.Err(); err != nil {
|
||||
return Result{}, fmt.Errorf("semantic reconciliation %q: context error before completion: %w", request.StageName, err)
|
||||
}
|
||||
var response ProposalResponse
|
||||
_, err = engine.client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
StageName: request.StageName,
|
||||
PromptID: engine.prompt.ID,
|
||||
PromptVersion: engine.prompt.Version,
|
||||
ProfileID: request.ProfileID,
|
||||
SessionID: request.SessionID,
|
||||
Inputs: preparation.Materials(),
|
||||
}, &response)
|
||||
if err != nil {
|
||||
if errors.Is(err, contracts.ErrInvalidStructuredOutput) {
|
||||
result.disposition = RetryableInvalidStructuredOutput
|
||||
return result, nil
|
||||
}
|
||||
return Result{}, fmt.Errorf("semantic reconciliation %q: complete structured output: %w", request.StageName, err)
|
||||
}
|
||||
|
||||
assessment := preparation.Assess(response)
|
||||
result.plan = assessment.Plan()
|
||||
result.issues = assessment.Issues()
|
||||
result.discardedGroupCount = assessment.DiscardedGroupCount()
|
||||
if result.discardedGroupCount > 0 {
|
||||
result.disposition = RetryableDiscardedProposalGroups
|
||||
} else {
|
||||
result.disposition = Complete
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (engine *Engine) validate() error {
|
||||
if engine == nil {
|
||||
return fmt.Errorf("semantic reconciliation engine must not be nil")
|
||||
}
|
||||
if engine.client == nil {
|
||||
return fmt.Errorf("semantic reconciliation engine: LLM client must not be nil")
|
||||
}
|
||||
if err := engine.prompt.Validate(); err != nil {
|
||||
return fmt.Errorf("semantic reconciliation engine: invalid construction state: %w", err)
|
||||
}
|
||||
if err := engine.limits.Validate(); err != nil {
|
||||
return fmt.Errorf("semantic reconciliation engine: invalid construction state: %w", err)
|
||||
}
|
||||
if err := validateResponseSchemaIdentity(engine.schema); err != nil {
|
||||
return fmt.Errorf("semantic reconciliation engine: invalid construction state: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
304
internal/framework/semanticreconcile/engine_test.go
Normal file
304
internal/framework/semanticreconcile/engine_test.go
Normal file
@@ -0,0 +1,304 @@
|
||||
package semanticreconcile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
)
|
||||
|
||||
func TestNewEngineValidatesConstruction(t *testing.T) {
|
||||
prompt := testPromptSpec("a")
|
||||
tests := []struct {
|
||||
name string
|
||||
client contracts.StructuredLLMClient
|
||||
prompt PromptSpec
|
||||
limits Limits
|
||||
want string
|
||||
}{
|
||||
{name: "nil client", prompt: prompt, limits: DefaultLimits(), want: "client"},
|
||||
{name: "empty prompt ID", client: &recordingReconciliationClient{}, prompt: PromptSpec{Version: "v1", SHA256: testDigest("a")}, limits: DefaultLimits(), want: "prompt ID"},
|
||||
{name: "empty prompt version", client: &recordingReconciliationClient{}, prompt: PromptSpec{ID: "prompt", SHA256: testDigest("a")}, limits: DefaultLimits(), want: "prompt version"},
|
||||
{name: "missing digest", client: &recordingReconciliationClient{}, prompt: PromptSpec{ID: "prompt", Version: "v1"}, limits: DefaultLimits(), want: "digest"},
|
||||
{name: "wrong digest algorithm", client: &recordingReconciliationClient{}, prompt: PromptSpec{ID: "prompt", Version: "v1", SHA256: "md5:" + strings.Repeat("a", 32)}, limits: DefaultLimits(), want: "sha256:"},
|
||||
{name: "non canonical digest", client: &recordingReconciliationClient{}, prompt: PromptSpec{ID: "prompt", Version: "v1", SHA256: "sha256:" + strings.Repeat("A", 64)}, limits: DefaultLimits(), want: "lowercase"},
|
||||
{name: "invalid limits", client: &recordingReconciliationClient{}, prompt: prompt, limits: Limits{}, want: "limits"},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if _, err := NewEngine(test.client, test.prompt, test.limits); err == nil || !strings.Contains(err.Error(), test.want) {
|
||||
t.Fatalf("NewEngine() error = %v, want %q", err, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
defaultPrompt, err := DefaultPromptSpec()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if defaultPrompt.ID != PromptID || defaultPrompt.Version != PromptVersion || defaultPrompt.SHA256 == "" {
|
||||
t.Fatalf("DefaultPromptSpec() = %#v, want complete generic prompt identity", defaultPrompt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnginePropagatesRequestAndAssessesResponse(t *testing.T) {
|
||||
client := &recordingReconciliationClient{responses: []ProposalResponse{{DuplicateGroups: []DuplicateGroup{{
|
||||
CandidateIDs: []int{1, 2}, CanonicalCandidateID: 2,
|
||||
}}}}}
|
||||
engine := newTestEngine(t, client, DefaultLimits())
|
||||
request := readyEngineRequest()
|
||||
request.ProfileID = " profile-as-resolved "
|
||||
request.SessionID = " session-as-supplied "
|
||||
|
||||
result, err := engine.Reconcile(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatalf("Reconcile() error = %v, want nil", err)
|
||||
}
|
||||
if result.Disposition() != Complete || result.DiscardedGroupCount() != 0 || len(result.Issues()) != 0 {
|
||||
t.Fatalf("result = disposition %v discarded %d issues %#v", result.Disposition(), result.DiscardedGroupCount(), result.Issues())
|
||||
}
|
||||
groups := result.Plan().Groups()
|
||||
if len(groups) != 1 || !reflect.DeepEqual(groups[0].MemberPositions(), []int{0, 1}) || groups[0].CanonicalPosition() != 1 {
|
||||
t.Fatalf("safe plan = %#v", groups)
|
||||
}
|
||||
if len(client.requests) != 1 {
|
||||
t.Fatalf("completion calls = %d, want exactly one", len(client.requests))
|
||||
}
|
||||
got := client.requests[0]
|
||||
if got.StageName != request.StageName || got.PromptID != engine.prompt.ID || got.PromptVersion != engine.prompt.Version || got.ProfileID != request.ProfileID || got.SessionID != request.SessionID {
|
||||
t.Fatalf("structured request = %#v, want exact routing values", got)
|
||||
}
|
||||
if len(got.Inputs) != 2 || got.Inputs["candidates"].Name != "candidates" || got.Inputs["transcript"].Name != "transcript" || len(got.Vars) != 0 {
|
||||
t.Fatalf("structured request inputs = %#v vars = %#v, want only candidate and transcript materials", got.Inputs, got.Vars)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEngineClassifiesSemanticAndTransportOutcomes(t *testing.T) {
|
||||
transportErr := errors.New("provider unavailable")
|
||||
tests := []struct {
|
||||
name string
|
||||
response ProposalResponse
|
||||
completion error
|
||||
want ResultDisposition
|
||||
wantDiscard int
|
||||
wantIssues bool
|
||||
wantError error
|
||||
}{
|
||||
{name: "empty groups complete", response: ProposalResponse{DuplicateGroups: []DuplicateGroup{}}, want: Complete},
|
||||
{name: "discarded proposal retryable", response: ProposalResponse{DuplicateGroups: []DuplicateGroup{{CandidateIDs: []int{1, 99}, CanonicalCandidateID: 1}}}, want: RetryableDiscardedProposalGroups, wantDiscard: 1, wantIssues: true},
|
||||
{name: "invalid structured output retryable", completion: fmt.Errorf("decode response: %w", contracts.ErrInvalidStructuredOutput), want: RetryableInvalidStructuredOutput},
|
||||
{name: "transport failure", completion: transportErr, wantError: transportErr},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
client := &recordingReconciliationClient{responses: []ProposalResponse{test.response}, errors: []error{test.completion}}
|
||||
result, err := newTestEngine(t, client, DefaultLimits()).Reconcile(context.Background(), readyEngineRequest())
|
||||
if test.wantError != nil {
|
||||
if !errors.Is(err, test.wantError) || !strings.Contains(err.Error(), readyEngineRequest().StageName) {
|
||||
t.Fatalf("Reconcile() error = %v, want contextual %v", err, test.wantError)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("Reconcile() error = %v, want nil", err)
|
||||
}
|
||||
if result.Disposition() != test.want || result.DiscardedGroupCount() != test.wantDiscard || (len(result.Issues()) > 0) != test.wantIssues {
|
||||
t.Fatalf("result = disposition %v discarded %d issues %#v", result.Disposition(), result.DiscardedGroupCount(), result.Issues())
|
||||
}
|
||||
if len(client.requests) != 1 {
|
||||
t.Fatalf("completion calls = %d, want one", len(client.requests))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEngineSkipsDeterministicOutcomesWithoutCompletion(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
request Request
|
||||
limits Limits
|
||||
want ResultDisposition
|
||||
mappingLen int
|
||||
}{
|
||||
{name: "insufficient candidates", request: engineRequestWithCandidateCount(1), limits: DefaultLimits(), want: SkippedInsufficientCandidates, mappingLen: 1},
|
||||
{name: "candidate limit", request: engineRequestWithCandidateCount(2), limits: Limits{ContextRadius: 0, MaximumCandidates: 1, MaximumMaterialBytes: 10000}, want: SkippedLimitExceeded, mappingLen: 2},
|
||||
{name: "material limit", request: engineRequestWithCandidateCount(2), limits: Limits{ContextRadius: 0, MaximumCandidates: 2, MaximumMaterialBytes: 1}, want: SkippedLimitExceeded, mappingLen: 2},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
client := &recordingReconciliationClient{}
|
||||
result, err := newTestEngine(t, client, test.limits).Reconcile(context.Background(), test.request)
|
||||
if err != nil {
|
||||
t.Fatalf("Reconcile() error = %v, want nil", err)
|
||||
}
|
||||
if result.Disposition() != test.want || len(result.CandidateMappings()) != test.mappingLen {
|
||||
t.Fatalf("result disposition = %v mappings = %#v", result.Disposition(), result.CandidateMappings())
|
||||
}
|
||||
if len(client.requests) != 0 {
|
||||
t.Fatalf("completion calls = %d, want zero", len(client.requests))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEngineRejectsInvalidInvocationAndHonorsCancellation(t *testing.T) {
|
||||
request := readyEngineRequest()
|
||||
client := &recordingReconciliationClient{}
|
||||
engine := newTestEngine(t, client, DefaultLimits())
|
||||
|
||||
var nilEngine *Engine
|
||||
if _, err := nilEngine.Reconcile(context.Background(), request); err == nil || !strings.Contains(err.Error(), "engine must not be nil") {
|
||||
t.Fatalf("nil engine error = %v", err)
|
||||
}
|
||||
if _, err := (&Engine{}).Reconcile(context.Background(), request); err == nil || !strings.Contains(err.Error(), "client") {
|
||||
t.Fatalf("zero engine error = %v", err)
|
||||
}
|
||||
invalid := newEngine(&recordingReconciliationClient{}, testPromptSpec("a"), llm.ResponseSchema{}, DefaultLimits())
|
||||
if _, err := invalid.Reconcile(context.Background(), request); err == nil || !strings.Contains(err.Error(), "invalid construction state") {
|
||||
t.Fatalf("invalid construction error = %v", err)
|
||||
}
|
||||
if _, err := engine.Reconcile(nil, request); err == nil || !strings.Contains(err.Error(), "context") {
|
||||
t.Fatalf("nil context error = %v", err)
|
||||
}
|
||||
withoutStage := request
|
||||
withoutStage.StageName = " "
|
||||
if _, err := engine.Reconcile(context.Background(), withoutStage); err == nil || !strings.Contains(err.Error(), "stage name") {
|
||||
t.Fatalf("empty stage error = %v", err)
|
||||
}
|
||||
withoutSource := request
|
||||
withoutSource.Source = nil
|
||||
if _, err := engine.Reconcile(context.Background(), withoutSource); err == nil || !strings.Contains(err.Error(), "source document") {
|
||||
t.Fatalf("nil source error = %v", err)
|
||||
}
|
||||
|
||||
canceled, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := engine.Reconcile(canceled, request); !errors.Is(err, context.Canceled) || !strings.Contains(err.Error(), "before preparation") {
|
||||
t.Fatalf("preparation cancellation error = %v", err)
|
||||
}
|
||||
if _, err := engine.Reconcile(&cancelBeforeCompletionContext{}, request); !errors.Is(err, context.Canceled) || !strings.Contains(err.Error(), "before completion") {
|
||||
t.Fatalf("completion cancellation error = %v", err)
|
||||
}
|
||||
if len(client.requests) != 0 {
|
||||
t.Fatalf("completion calls = %d, want zero for invalid and canceled invocations", len(client.requests))
|
||||
}
|
||||
}
|
||||
|
||||
func TestEngineCallsAreIndependentAndResultsAreOwned(t *testing.T) {
|
||||
client := &recordingReconciliationClient{responses: []ProposalResponse{
|
||||
{DuplicateGroups: []DuplicateGroup{
|
||||
{CandidateIDs: []int{1, 2}, CanonicalCandidateID: 1},
|
||||
{CandidateIDs: []int{2, 99}, CanonicalCandidateID: 2},
|
||||
}},
|
||||
{DuplicateGroups: []DuplicateGroup{}},
|
||||
}}
|
||||
engine := newTestEngine(t, client, DefaultLimits())
|
||||
first, err := engine.Reconcile(context.Background(), readyEngineRequest())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first.Plan().Groups()[0].memberPositions[0] = 99
|
||||
firstIssues := first.Issues()
|
||||
firstIssues[0].Category = "changed"
|
||||
firstMappings := first.CandidateMappings()
|
||||
firstMappings[0].CandidatePosition = 99
|
||||
if first.Plan().Groups()[0].MemberPositions()[0] != 0 || first.Issues()[0].Category == "changed" || first.CandidateMappings()[0].CandidatePosition != 0 {
|
||||
t.Fatal("result accessors exposed retained state")
|
||||
}
|
||||
|
||||
second, err := engine.Reconcile(context.Background(), readyEngineRequest())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if second.Disposition() != Complete || len(second.Plan().Groups()) != 0 || len(second.Issues()) != 0 || second.DiscardedGroupCount() != 0 || len(second.CandidateMappings()) != 2 {
|
||||
t.Fatalf("second result retained prior call state: disposition %v plan %#v issues %#v discarded %d mappings %#v", second.Disposition(), second.Plan().Groups(), second.Issues(), second.DiscardedGroupCount(), second.CandidateMappings())
|
||||
}
|
||||
}
|
||||
|
||||
type recordingReconciliationClient struct {
|
||||
requests []contracts.StructuredCompletionRequest
|
||||
responses []ProposalResponse
|
||||
errors []error
|
||||
}
|
||||
|
||||
func (client *recordingReconciliationClient) CompleteStructured(_ context.Context, request contracts.StructuredCompletionRequest, output any) (contracts.StructuredCompletionResponse, error) {
|
||||
request.Inputs = request.Inputs.Clone()
|
||||
client.requests = append(client.requests, request)
|
||||
index := len(client.requests) - 1
|
||||
if index < len(client.errors) && client.errors[index] != nil {
|
||||
return contracts.StructuredCompletionResponse{}, client.errors[index]
|
||||
}
|
||||
response := ProposalResponse{DuplicateGroups: []DuplicateGroup{}}
|
||||
if index < len(client.responses) {
|
||||
response = cloneProposalResponse(client.responses[index])
|
||||
}
|
||||
target, ok := output.(*ProposalResponse)
|
||||
if !ok {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("output type = %T", output)
|
||||
}
|
||||
*target = response
|
||||
return contracts.StructuredCompletionResponse{}, nil
|
||||
}
|
||||
|
||||
func cloneProposalResponse(response ProposalResponse) ProposalResponse {
|
||||
cloned := ProposalResponse{DuplicateGroups: make([]DuplicateGroup, len(response.DuplicateGroups))}
|
||||
for index, group := range response.DuplicateGroups {
|
||||
cloned.DuplicateGroups[index] = DuplicateGroup{
|
||||
CandidateIDs: append([]int(nil), group.CandidateIDs...), CanonicalCandidateID: group.CanonicalCandidateID,
|
||||
}
|
||||
}
|
||||
return cloned
|
||||
}
|
||||
|
||||
func newTestEngine(t *testing.T, client contracts.StructuredLLMClient, limits Limits) *Engine {
|
||||
t.Helper()
|
||||
engine, err := NewEngine(client, testPromptSpec("a"), limits)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine() error = %v", err)
|
||||
}
|
||||
return engine
|
||||
}
|
||||
|
||||
func testPromptSpec(digestCharacter string) PromptSpec {
|
||||
return PromptSpec{ID: "test.semantic_reconciliation", Version: "v1", SHA256: testDigest(digestCharacter)}
|
||||
}
|
||||
|
||||
func testDigest(character string) string { return "sha256:" + strings.Repeat(character, 64) }
|
||||
|
||||
func readyEngineRequest() Request { return engineRequestWithCandidateCount(2) }
|
||||
|
||||
func engineRequestWithCandidateCount(count int) Request {
|
||||
document := &source.SourceDocument{ID: "source", Units: []source.SourceUnit{
|
||||
{ID: 1, Kind: "speech", Text: "Mira arrived."},
|
||||
{ID: 2, Kind: "speech", Text: "The captain spoke."},
|
||||
}}
|
||||
candidates := make([]Candidate, count)
|
||||
for index := range candidates {
|
||||
unitID := index%len(document.Units) + 1
|
||||
candidates[index] = Candidate{
|
||||
Label: fmt.Sprintf("candidate-%d", index+1),
|
||||
SourceRefs: []source.SourceRef{{SourceID: document.ID, StartUnitID: unitID, EndUnitID: unitID}},
|
||||
}
|
||||
}
|
||||
return Request{StageName: "test/normalize", Source: document, Candidates: candidates, ProfileID: "profile", SessionID: "session"}
|
||||
}
|
||||
|
||||
type cancelBeforeCompletionContext struct{ calls int }
|
||||
|
||||
func (ctx *cancelBeforeCompletionContext) Deadline() (time.Time, bool) { return time.Time{}, false }
|
||||
func (ctx *cancelBeforeCompletionContext) Done() <-chan struct{} { return nil }
|
||||
func (ctx *cancelBeforeCompletionContext) Value(any) any { return nil }
|
||||
func (ctx *cancelBeforeCompletionContext) Err() error {
|
||||
ctx.calls++
|
||||
if ctx.calls > 1 {
|
||||
return context.Canceled
|
||||
}
|
||||
return nil
|
||||
}
|
||||
101
internal/framework/semanticreconcile/identity.go
Normal file
101
internal/framework/semanticreconcile/identity.go
Normal file
@@ -0,0 +1,101 @@
|
||||
package semanticreconcile
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
// Policy identifies the framework-owned reconciliation and assessment rules.
|
||||
const Policy = "semantic_reconciliation.v1"
|
||||
|
||||
var _ contracts.ManifestMetadataProvider = (*Engine)(nil)
|
||||
var _ pipeline.CheckpointFingerprintProvider = (*Engine)(nil)
|
||||
|
||||
// ManifestMetadata returns fresh, content-free identity for the complete core
|
||||
// reconciliation mechanism.
|
||||
func (engine *Engine) ManifestMetadata() map[string]any {
|
||||
if engine == nil || engine.validate() != nil {
|
||||
return nil
|
||||
}
|
||||
return map[string]any{
|
||||
"prompt_id": engine.prompt.ID,
|
||||
"prompt_version": engine.prompt.Version,
|
||||
"prompt_sha256": engine.prompt.SHA256,
|
||||
"response_schema_key": string(engine.schema.Key),
|
||||
"response_schema_id": engine.schema.ID,
|
||||
"response_schema_name": engine.schema.Name,
|
||||
"response_schema_version": engine.schema.Version,
|
||||
"response_schema_sha256": engine.schema.SHA256,
|
||||
"semantic_reconciliation_policy": Policy,
|
||||
"semantic_reconciliation_limits": map[string]any{
|
||||
"context_radius": engine.limits.ContextRadius,
|
||||
"maximum_candidates": engine.limits.MaximumCandidates,
|
||||
"maximum_material_bytes": engine.limits.MaximumMaterialBytes,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// CheckpointFingerprints returns fresh canonical identities for prompt,
|
||||
// schema, reconciliation policy, and the complete limit policy.
|
||||
func (engine *Engine) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
|
||||
if engine == nil || engine.validate() != nil {
|
||||
return nil
|
||||
}
|
||||
return []pipeline.CheckpointFingerprint{
|
||||
{Name: "prompt", Value: identityDigest(engine.prompt.ID, engine.prompt.Version, engine.prompt.SHA256)},
|
||||
{Name: "response_schema", Value: identityDigest(string(engine.schema.Key), engine.schema.ID, engine.schema.Version, engine.schema.Name, engine.schema.SHA256)},
|
||||
{Name: "semantic_reconciliation_policy", Value: Policy},
|
||||
{Name: "semantic_reconciliation_limits", Value: limitPolicyDigest(engine.limits)},
|
||||
}
|
||||
}
|
||||
|
||||
func limitPolicyDigest(limits Limits) string {
|
||||
return identityDigest(
|
||||
strconv.Itoa(limits.ContextRadius),
|
||||
strconv.Itoa(limits.MaximumCandidates),
|
||||
strconv.Itoa(limits.MaximumMaterialBytes),
|
||||
)
|
||||
}
|
||||
|
||||
func identityDigest(parts ...string) string {
|
||||
hash := sha256.New()
|
||||
for _, part := range parts {
|
||||
_, _ = hash.Write([]byte(strconv.Itoa(len(part))))
|
||||
_, _ = hash.Write([]byte{':'})
|
||||
_, _ = hash.Write([]byte(part))
|
||||
}
|
||||
return "sha256:" + hex.EncodeToString(hash.Sum(nil))
|
||||
}
|
||||
|
||||
func validateResponseSchemaIdentity(schema llm.ResponseSchema) error {
|
||||
if strings.TrimSpace(string(schema.Key)) == "" || strings.TrimSpace(schema.ID) == "" || strings.TrimSpace(schema.Version) == "" || strings.TrimSpace(schema.Name) == "" {
|
||||
return fmt.Errorf("response schema identity must be complete")
|
||||
}
|
||||
if err := validateSHA256(schema.SHA256); err != nil {
|
||||
return fmt.Errorf("response schema digest: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateSHA256(value string) error {
|
||||
const prefix = "sha256:"
|
||||
if !strings.HasPrefix(value, prefix) {
|
||||
return fmt.Errorf("must use sha256: prefix")
|
||||
}
|
||||
hexValue := strings.TrimPrefix(value, prefix)
|
||||
if len(hexValue) != sha256.Size*2 || hexValue != strings.ToLower(hexValue) {
|
||||
return fmt.Errorf("must contain 64 lowercase hexadecimal characters")
|
||||
}
|
||||
decoded, err := hex.DecodeString(hexValue)
|
||||
if err != nil || len(decoded) != sha256.Size {
|
||||
return fmt.Errorf("must contain 64 lowercase hexadecimal characters")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
93
internal/framework/semanticreconcile/identity_test.go
Normal file
93
internal/framework/semanticreconcile/identity_test.go
Normal file
@@ -0,0 +1,93 @@
|
||||
package semanticreconcile
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestEngineMetadataAndFingerprintsCoverCoreIdentity(t *testing.T) {
|
||||
base := newTestEngine(t, &recordingReconciliationClient{}, DefaultLimits())
|
||||
metadata := base.ManifestMetadata()
|
||||
for _, key := range []string{
|
||||
"prompt_id", "prompt_version", "prompt_sha256",
|
||||
"response_schema_key", "response_schema_id", "response_schema_name", "response_schema_version", "response_schema_sha256",
|
||||
"semantic_reconciliation_policy", "semantic_reconciliation_limits",
|
||||
} {
|
||||
if metadata[key] == nil || metadata[key] == "" {
|
||||
t.Fatalf("metadata[%q] = %#v, want populated core identity", key, metadata[key])
|
||||
}
|
||||
}
|
||||
limits, ok := metadata["semantic_reconciliation_limits"].(map[string]any)
|
||||
if !ok || len(limits) != 3 || limits["context_radius"] == nil || limits["maximum_candidates"] == nil || limits["maximum_material_bytes"] == nil {
|
||||
t.Fatalf("limit metadata = %#v, want complete limits", metadata["semantic_reconciliation_limits"])
|
||||
}
|
||||
fingerprints := base.CheckpointFingerprints()
|
||||
wantNames := []string{"prompt", "response_schema", "semantic_reconciliation_policy", "semantic_reconciliation_limits"}
|
||||
if len(fingerprints) != len(wantNames) {
|
||||
t.Fatalf("fingerprints = %#v, want required categories", fingerprints)
|
||||
}
|
||||
for index, want := range wantNames {
|
||||
if fingerprints[index].Name != want || fingerprints[index].Value == "" {
|
||||
t.Fatalf("fingerprint %d = %#v, want %q with value", index, fingerprints[index], want)
|
||||
}
|
||||
}
|
||||
|
||||
metadata["prompt_id"] = "changed"
|
||||
limits["context_radius"] = -1
|
||||
fingerprints[0].Name = "changed"
|
||||
if got := base.ManifestMetadata(); got["prompt_id"] == "changed" || got["semantic_reconciliation_limits"].(map[string]any)["context_radius"] == -1 {
|
||||
t.Fatalf("ManifestMetadata() exposed retained state: %#v", got)
|
||||
}
|
||||
if got := base.CheckpointFingerprints(); got[0].Name == "changed" {
|
||||
t.Fatalf("CheckpointFingerprints() exposed retained state: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoreFingerprintsChangeWithBehavioralIdentity(t *testing.T) {
|
||||
base := newTestEngine(t, &recordingReconciliationClient{}, DefaultLimits())
|
||||
baseFingerprints := base.CheckpointFingerprints()
|
||||
|
||||
promptChanged, err := NewEngine(&recordingReconciliationClient{}, testPromptSpec("b"), DefaultLimits())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertOnlyFingerprintChanged(t, baseFingerprints, promptChanged.CheckpointFingerprints(), "prompt")
|
||||
|
||||
schema, err := LoadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
schema.SHA256 = testDigest("b")
|
||||
schemaChanged := newEngine(&recordingReconciliationClient{}, base.prompt, schema, DefaultLimits())
|
||||
assertOnlyFingerprintChanged(t, baseFingerprints, schemaChanged.CheckpointFingerprints(), "response_schema")
|
||||
|
||||
limits := DefaultLimits()
|
||||
limits.ContextRadius++
|
||||
limitsChanged, err := NewEngine(&recordingReconciliationClient{}, base.prompt, limits)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertOnlyFingerprintChanged(t, baseFingerprints, limitsChanged.CheckpointFingerprints(), "semantic_reconciliation_limits")
|
||||
}
|
||||
|
||||
func TestInvalidEngineIdentityHasNoMetadataOrFingerprints(t *testing.T) {
|
||||
invalid := newEngine(&recordingReconciliationClient{}, testPromptSpec("a"), llm.ResponseSchema{}, DefaultLimits())
|
||||
if invalid.ManifestMetadata() != nil || invalid.CheckpointFingerprints() != nil {
|
||||
t.Fatalf("invalid engine exposed identity: metadata %#v fingerprints %#v", invalid.ManifestMetadata(), invalid.CheckpointFingerprints())
|
||||
}
|
||||
}
|
||||
|
||||
func assertOnlyFingerprintChanged(t *testing.T, before, after []pipeline.CheckpointFingerprint, changedName string) {
|
||||
t.Helper()
|
||||
if len(before) != len(after) {
|
||||
t.Fatalf("fingerprint counts differ: %#v %#v", before, after)
|
||||
}
|
||||
for index := range before {
|
||||
changed := before[index] != after[index]
|
||||
if changed != (before[index].Name == changedName) {
|
||||
t.Fatalf("fingerprint %q change = %t, want only %q changed\nbefore: %#v\nafter: %#v", before[index].Name, changed, changedName, before, after)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user