eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Expose chunk plan provenance and safe diagnostics

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-18 00:45:01 +00:00
parent 8ba5228c01
commit 6fc6ce0adb
15 changed files with 363 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
internal/framework/pipeline/runner_chunk_plan_test.go
View File

@@ -54,17 +54,30 @@ type manifestChunker struct {
}
func (c manifestChunker) ManifestMetadata() map[string]any { return c.metadata }
func (manifestChunker) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassLLMBacked
}
type llmCountingChunker struct {
terminalChunker
llmCalls *int
}
func (llmCountingChunker) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassLLMBacked
}
func (c llmCountingChunker) Plan(ctx context.Context, request contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
(*c.llmCalls)++
return c.terminalChunker.Plan(ctx, request)
}
type deterministicChunker struct{ terminalChunker }
func (deterministicChunker) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
type retryingChunker struct {
key string
plan source.ChunkPlan
@@ -93,19 +106,20 @@ func (l *dependencyLoader) Extract(_ string, _ string, dependencies []Checkpoint
func TestRunnerChunkPlanModeMatrix(t *testing.T) {
tests := []struct {
name string
mode ChunkCacheMode
decision ChunkPlanDecision
wantLoads int
wantSaves int
wantCalls int
name string
mode ChunkCacheMode
decision ChunkPlanDecision
wantLoads int
wantSaves int
wantCalls int
wantAction string
}{
{name: "auto hit", mode: ChunkCacheAuto, decision: ChunkPlanDecision{Status: ChunkPlanHit}, wantLoads: 1},
{name: "auto missing", mode: ChunkCacheAuto, decision: ChunkPlanDecision{Status: ChunkPlanMissing}, wantLoads: 1, wantSaves: 1, wantCalls: 1},
{name: "auto invalid", mode: ChunkCacheAuto, decision: ChunkPlanDecision{Status: ChunkPlanInvalid}, wantLoads: 1, wantSaves: 1, wantCalls: 1},
{name: "bypass", mode: ChunkCacheBypass, wantCalls: 1},
{name: "empty bypass", wantCalls: 1},
{name: "refresh", mode: ChunkCacheRefresh, wantSaves: 1, wantCalls: 1},
{name: "auto hit", mode: ChunkCacheAuto, decision: ChunkPlanDecision{Status: ChunkPlanHit}, wantLoads: 1, wantAction: "reused"},
{name: "auto missing", mode: ChunkCacheAuto, decision: ChunkPlanDecision{Status: ChunkPlanMissing}, wantLoads: 1, wantSaves: 1, wantCalls: 1, wantAction: "generated"},
{name: "auto invalid", mode: ChunkCacheAuto, decision: ChunkPlanDecision{Status: ChunkPlanInvalid}, wantLoads: 1, wantSaves: 1, wantCalls: 1, wantAction: "generated"},
{name: "bypass", mode: ChunkCacheBypass, wantCalls: 1, wantAction: "bypassed"},
{name: "empty bypass", wantCalls: 1, wantAction: "bypassed"},
{name: "refresh", mode: ChunkCacheRefresh, wantSaves: 1, wantCalls: 1, wantAction: "refreshed"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
@@ -114,16 +128,58 @@ func TestRunnerChunkPlanModeMatrix(t *testing.T) {
llmCalls := 0
prepared.chunker = llmCountingChunker{terminalChunker: terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls}, llmCalls: &llmCalls}
store := &recordingChunkPlanStore{record: chunkPlanRecord(t, prepared, plan), decision: tc.decision}
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: tc.mode, ChunkPlans: store}); err != nil {
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: tc.mode, ChunkPlans: store})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if store.loads != tc.wantLoads || store.saves != tc.wantSaves || calls != tc.wantCalls {
t.Fatalf("calls = load %d save %d module %d, want %d %d %d", store.loads, store.saves, calls, tc.wantLoads, tc.wantSaves, tc.wantCalls)
}
if output.Manifest.ChunkPlan == nil || output.Manifest.ChunkPlan.Action != tc.wantAction || output.ChunkPlan == nil || output.ChunkPlan.Action != tc.wantAction {
t.Fatalf("chunk plan manifest = %#v summary = %#v, want action %q", output.Manifest.ChunkPlan, output.ChunkPlan, tc.wantAction)
}
})
}
}
func TestRunnerChunkPlanHitUsesStoredProducerProvenance(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
prepared.resolved.Chunk.Module = "chunk/requested"
prepared.resolved.Chunk.LLMProfile = "current-profile"
record := chunkPlanRecord(t, prepared, plan)
record.Producer.InputModule = "input/original"
record.Producer.ChunkModule = "chunk/original"
record.Producer.LLMProfile = "original-profile"
record.Producer.Metadata = map[string]any{"prompt": map[string]any{"id": "original"}}
store := &recordingChunkPlanStore{record: record, decision: ChunkPlanDecision{Status: ChunkPlanHit}}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store})
if err != nil {
t.Fatal(err)
}
manifest := output.Manifest.ChunkPlan
if manifest.RequestedModule != "chunk/requested" || manifest.ProducerModule != "chunk/original" || manifest.ProducerLLMProfile != "original-profile" {
t.Fatalf("chunk plan manifest = %#v", manifest)
}
storedNested := record.Producer.Metadata["prompt"].(map[string]any)
storedNested["id"] = "mutated"
if got := manifest.ProducerMetadata["prompt"].(map[string]any)["id"]; got != "original" {
t.Fatalf("producer metadata changed through stored alias: %v", got)
}
}
func TestRunnerChunkPlanManifestRetainsCandidateOnRejection(t *testing.T) {
prepared, _ := preparedTerminalDebugPipeline(t)
validator := &countingChunkValidator{result: contracts.ValidationResult{Approved: false, ReasonCode: "policy", Message: "no"}}
prepared.chunkValidators.validators = []preparedValidator{{resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk}, chunk: validator}}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheRefresh, ChunkPlans: &recordingChunkPlanStore{}})
if err != nil {
t.Fatal(err)
}
if output.Manifest.ChunkPlan.Action != "refreshed" || output.Manifest.ChunkPlan.PlanDigest == "" || output.ChunkPlan.ValidationStatus != "rejected" || output.ChunkPlan.PublicationStatus != "not_published" {
t.Fatalf("manifest = %#v summary = %#v", output.Manifest.ChunkPlan, output.ChunkPlan)
}
}
func TestRunnerValidatesChunkPlanPolicyInputs(t *testing.T) {
for _, tc := range []struct {
name string
@@ -328,6 +384,20 @@ func TestRunnerStoresProducerProvenanceAndProducerWarnings(t *testing.T) {
}
}
func TestRunnerOmitsProducerProfileForDeterministicChunker(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
prepared.resolved.Chunk.LLMProfile = "configured-but-unused"
prepared.chunker = deterministicChunker{terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan}}
store := &recordingChunkPlanStore{}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheRefresh, ChunkPlans: store})
if err != nil {
t.Fatal(err)
}
if store.saved.Producer.LLMProfile != "" || output.Manifest.ChunkPlan.ProducerLLMProfile != "" {
t.Fatalf("deterministic producer profile = stored %q manifest %q", store.saved.Producer.LLMProfile, output.Manifest.ChunkPlan.ProducerLLMProfile)
}
}
func TestRunnerRefreshChangesDownstreamChunkFingerprint(t *testing.T) {
doc := typedTestDocumentWithUnits(2)
prepared := preparedConcurrentPipeline(t, 1)