From 071a78ae22b595684576d7c813d884518bf663f8 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 9 Aug 2026 19:42:33 +0000 Subject: [PATCH] Replace evidence context with source unit excerpts --- docs/roadmap/evidence-context.md | 174 ++++++++++ docs/roadmap/implementation.md | 315 ++++++++++++++++++ .../schemas/source_evidence_context.v1.json | 39 +-- internal/framework/evidencecontext/build.go | 182 +++------- internal/framework/evidencecontext/codec.go | 187 +++-------- .../evidencecontext/evidencecontext_test.go | 263 ++++----------- internal/framework/evidencecontext/model.go | 37 +- .../testdata/source_evidence_context.v1.json | 2 +- .../framework/pipeline/evidence_output.go | 24 +- .../pipeline/evidence_output_test.go | 11 +- 10 files changed, 650 insertions(+), 584 deletions(-) create mode 100644 docs/roadmap/evidence-context.md create mode 100644 docs/roadmap/implementation.md diff --git a/docs/roadmap/evidence-context.md b/docs/roadmap/evidence-context.md new file mode 100644 index 0000000..6b60330 --- /dev/null +++ b/docs/roadmap/evidence-context.md @@ -0,0 +1,174 @@ +# Minimal Evidence Context + +## Purpose + +Redefine the optional `evidence-context.json` output as a compact reading +excerpt rather than a second provenance model. The artifact should contain the +smallest useful source projection: the ordered union of source units cited by +selected accepted lane artifacts, expanded by the configured surrounding +window. + +The normalized lane artifacts remain authoritative for which lane cited which +source ranges. The original source remains authoritative for the complete +input. Evidence context is an optional convenience artifact for consumers that +need the relevant source material alongside the extracted results. + +## Motivation + +The current artifact repeats source units inside context objects and adds +context ranges, lane attribution, direct evidence references, source identity, +source digest, selected lanes, and window configuration. With several lanes +and a nonzero window, this representation can be substantially larger than the +complete input by both bytes and tokens. That defeats its intended value as a +convenient evidence excerpt. + +The application already publishes authoritative source references in each +normalized artifact. Repeating their provenance in `evidence-context.json` +does not justify the additional size or contract complexity. + +## Target Contract + +`evidence-context.json` remains an optional pipeline-wide JSON artifact with +the existing logical filename, artifact kind, media type, schema identity, and +schema version. Because Notarius is pre-release, the v1 schema may be replaced +in place and no reader compatibility or migration path is required. + +The payload is a top-level JSON array of generic source units. It has no +evidence-context-specific wrapper or metadata. Every element uses the existing +`source.SourceUnit` representation and therefore preserves its required +`id`, `kind`, `text`, and self-reference together with any metadata already +owned by the source unit. “No additional metadata” means that evidence-context +construction does not annotate, reshape, or enrich a source unit; it does not +mean stripping metadata supplied by the input adapter. + +An enabled output with no accepted cited evidence emits `[]`, not `null` and +not an absent artifact. The artifact remains absent when evidence-context +publication is not enabled. + +## Selection Semantics + +Construction must: + +1. obtain source references only through the typed evidence projections of + accepted normalized outputs from the configured lane allowlist; +2. validate every projected reference against the current source document; +3. expand each valid referenced range by `window_units` positions on both + sides, clamping at the source boundaries; +4. take the union of all expanded ranges; +5. emit every selected source unit exactly once and in source-document + position order; and +6. return owned copies so later mutation cannot alias the source document or + output artifact. + +Overlapping and adjacent ranges may be coalesced internally, but range groups +are not represented in the payload. Repeated citations, citations from +multiple lanes, and overlapping windows never duplicate a source unit. +Rejected, failed, absent, inactive, and unselected lanes contribute nothing. + +The selected set is therefore bounded by the source document: it can contain +at most every source unit once. With broad evidence coverage or a sufficiently +large window, it may legitimately equal the complete generic source document. +No byte- or token-size guarantee is made because generic source-unit +serialization can differ from the external input format and output formatting +has its own overhead. The application must not truncate a complete excerpt to +meet an arbitrary size limit. + +## Configuration And Publication + +Retain the current `output.options.evidence_context` configuration: + +- `enabled` continues to control publication; +- `lanes` remains the non-empty allowlist of configured artifact lanes whose + accepted normalized evidence contributes to the excerpt; and +- `window_units` remains a non-negative optional value with the existing + default of three. + +Retain `evidence-context.json` and the pipeline-wide `index.json` +`evidence_context` descriptor. The descriptor remains the canonical place for +artifact identity and discovery. It must not be copied into the payload. + +## Internal Design + +Keep evidence preparation at the current framework and output boundary. The +generic framework should continue to consume typed evidence projections rather +than inspect domain JSON or import domain artifact types. + +Simplify `internal/framework/evidencecontext` around the new contract: + +- represent the durable document as a collection of source units rather than + contexts and evidence contributions; +- reduce the build request to the source document, window size, and projected + source references needed to select units; +- remove durable and internal types used only for lane attribution, context + ranges, and evidence-reference publication; +- retain strict schema validation, deterministic serialization, source and + reference validation, ownership, and stable artifact identity; and +- preserve the existing pipeline preparation logic that validates configured + lanes and typed evidence projections. + +The pipeline may continue to record a content-free debug summary containing +artifact identity, selected lane configuration, window size, source digest, +and emitted unit count. Remove the obsolete context count. Debug information +is operational state outside `evidence-context.json` and must not contain the +selected source text or source-unit metadata. + +## Documentation Impact + +When the implementation lands, rewrite the durable contract in +`docs/integrations/evidence-context.md`. Update the configuration, JSON-output, +subprocess-consumer, operations, and internal-module documentation wherever it +describes the old context or provenance structure. Preserve canonical +ownership: the integration contract defines the payload; other documents give +only the information appropriate to their audience and link to that contract. + +The maintained complete D&D configuration should retain its existing evidence +publication settings. No new example is needed because the configuration +surface is unchanged. + +## Testing Expectations + +Tests should protect the revised behavior rather than the former internal +shape. At the narrowest stable boundaries, verify: + +- window expansion and source-boundary clamping; +- union, deduplication, and source-position ordering, including non-monotonic + unit IDs; +- unchanged source-unit values and owned metadata; +- `[]` for enabled publication with no accepted cited evidence; +- rejection of invalid sources, references, and durable payloads; +- omission when publication is disabled; +- contribution only from selected accepted normalized outputs, including + checkpoint reconstruction; and +- correct JSON bundle publication and index discovery. + +Do not add an exact byte-length, token-count, or ratio test. Such a test would +be format-sensitive and would not protect the durable contract. The meaningful +boundedness invariant is structural: every emitted element corresponds to one +distinct unit from the source document, and no source unit is emitted more +than once. + +## Documentation Decision + +No new ADR is required. This work simplifies a pre-release output contract but +does not change the architectural ownership of source evidence, typed +projections, pipeline stages, or output publication. The durable contract and +current implementation documentation should be updated when the behavior is +implemented; this roadmap owns the proposed behavior until then. + +## Acceptance Criteria + +- `evidence-context.json` is a top-level array containing only selected generic + source units. +- The payload contains no context groups, lane IDs, evidence references, + source-level wrapper fields, or evidence-context-specific annotations. +- Units are copied unchanged from the source document, appear once, and retain + source-document order. +- Existing lane selection and `window_units` behavior remain configurable and + retain their current validation and defaults. +- The artifact remains optional, keeps its filename and identity, and remains + discoverable through `index.json`. +- The implementation retains strict validation, deterministic output, + ownership, failure propagation, checkpoint behavior, and content-free debug + reporting. +- Current documentation and maintained examples accurately describe the new + contract, and the repository-wide tests, vet, and build pass. diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md new file mode 100644 index 0000000..758bea2 --- /dev/null +++ b/docs/roadmap/implementation.md @@ -0,0 +1,315 @@ +# Minimal Evidence Context Implementation Plan + +## Objective + +Implement the target state defined in +[Minimal Evidence Context](evidence-context.md): retain the optional +`evidence-context.json` output and its existing configuration while replacing +its provenance-heavy payload with the ordered, deduplicated union of selected +generic source units. + +This plan intentionally changes the existing v1 contract in place. Notarius is +pre-release, so do not add legacy decoding, schema-version branching, +migration, dual publication, or other backward-compatibility machinery. + +## Global Implementation Rules + +Apply these rules in every stage: + +- Follow `docs/policy/architecture.md`, `docs/policy/documentation.md`, and + `docs/policy/testing.md`. +- Treat the normalized lane artifacts as authoritative for citations and the + source document as authoritative for source-unit content. +- Preserve the typed evidence-projection boundary. Generic code must not + inspect domain JSON or depend on D&D artifact types. +- Preserve the logical filename `evidence-context.json`, artifact kind + `source/evidence-context`, media type `application/json`, schema ID + `notarius.source.evidence_context`, schema name + `notarius_source_evidence_context_v1`, and schema version `v1`. +- Preserve the current configuration keys, validation, lane allowlist + semantics, and default `window_units` value. +- Do not introduce a payload wrapper. The durable JSON value is a top-level + array of generic source units; the empty value is `[]`. +- Preserve source-unit fields and metadata exactly as represented in the + validated generic source document. Add no evidence-context-specific fields. +- Do not add byte-count, token-count, or compression-ratio tests. Test the + structural boundedness and behavioral contract instead. +- Use focused tests while iterating and leave the repository compiling and its + relevant test suites passing at the end of every stage. + +## Stage 1: Replace The Core Evidence-Context Contract + +### Goal + +Replace the context/provenance model with a strict top-level source-unit array +and simplify construction to select the union of expanded source ranges. + +### Required Changes + +1. Update `internal/framework/evidencecontext/model.go`: + - represent `Document` as the durable collection of `source.SourceUnit` + values that serializes directly as a JSON array; + - retain the existing artifact identity constants; + - reduce `BuildRequest` to `Source`, `WindowUnits`, and a flat collection of + projected `SourceRefs`; + - remove `Context`, `EvidenceRef`, `LaneEvidence`, and any other types whose + only purpose was durable lane attribution or context grouping. +2. Update `internal/framework/evidencecontext/build.go`: + - validate the window and complete source document, including its digest; + - validate every projected source reference against the source index; + - convert referenced endpoints to source positions, expand and clamp them, + and merge their union; + - append owned copies of the selected source units once in document-position + order; + - return an initialized empty document when there are no references; + - remove sorting keys and helpers that exist only to order lane attribution + or evidence records. +3. Update `internal/framework/evidencecontext/codec.go` and + `internal/framework/evidencecontext/assets/schemas/source_evidence_context.v1.json`: + - define the root schema as an array of strict source-unit objects; + - preserve the existing source-unit contract: required `id`, `kind`, `text`, + and `ref`, with optional open-ended source-owned `metadata`; + - ensure decode rejects `null`, wrapper objects, malformed units, unknown + fixed unit/reference fields, invalid self-references, and noncanonical + documents; + - ensure encode and decode own all slice and nested metadata values and + serialize an empty document as `[]`; + - remove canonicalization logic used only by the deleted wrapper and context + structures. +4. Replace the fixture in + `internal/framework/evidencecontext/testdata/source_evidence_context.v1.json` + with a compact top-level unit array. +5. Rewrite `internal/framework/evidencecontext/evidencecontext_test.go` around + durable behavior: + - cover expanded overlapping and adjacent ranges, boundary clamping, + repeated references, and non-monotonic unit IDs; + - prove that the output is source-ordered, contains each selected source + unit once, and never invents a unit; + - prove preservation and ownership of source-unit metadata; + - cover empty-array encoding and strict invalid-input rejection; + - retain artifact identity and deterministic round-trip coverage without + testing deleted internal structures. +6. Adapt `internal/framework/pipeline/evidence_output.go` so it flattens the + typed references from prepared selected lanes into the new build request. + Preserve all existing checks for duplicate accepted lane outputs, source and + artifact-kind compatibility, codec decoding, and evidence projection + failures. +7. Adapt the pipeline debug summary and its use in `runner.go` as necessary: + - calculate selected lanes, window size, and source digest from the prepared + plan and source document rather than the payload; + - report the emitted unit count directly; + - remove `context_count`; + - keep source text and unit metadata out of debug summaries. +8. Rewrite the focused tests in + `internal/framework/pipeline/evidence_output_test.go` to assert the new + payload while preserving coverage of selected accepted outputs, absent and + rejected lanes, incompatible artifacts, failure propagation, ownership, + checkpoint reconstruction, opt-in behavior, and redacted debug reporting. + +### Acceptance Criteria + +- The evidence-context package has no durable or internal model for context + groups, lane attribution, or published evidence references. +- Its codec accepts and emits only the new top-level array contract. +- Its builder returns the exact ordered union of source units selected by the + expanded validated ranges. +- Pipeline construction still obtains references exclusively through prepared + typed evidence projections. +- The payload remains absent when publication is not configured and is `[]` + when enabled with no contributing evidence. +- Focused framework tests pass. + +### Validation + +```sh +gofmt -w internal/framework/evidencecontext internal/framework/pipeline/evidence_output.go internal/framework/pipeline/evidence_output_test.go +go test ./internal/framework/evidencecontext ./internal/framework/pipeline +``` + +### Prompt Scope + +This is one coherent but substantial gpt-5.6-terra prompt. Keep the work +limited to the core contract, its pipeline construction boundary, and their +focused tests; do not update generic JSON output tests, production integration +tests, or documentation in this stage. + +## Stage 2: Update Output And Production Integration Coverage + +### Goal + +Carry the revised artifact through the generic JSON output and representative +assembled D&D workflows without changing publication or discovery semantics. + +### Required Changes + +1. Review `internal/modules/generic/output/json/encoder.go` and retain the + existing option decoding, default window, lane validation, logical filename, + artifact identity checks, pretty JSON publication, and `index.json` + descriptor. Change production code only where it assumes the old payload. +2. Rewrite evidence-context fixtures and assertions in + `internal/modules/generic/output/json/encoder_test.go`: + - validate and publish a top-level unit array; + - retain descriptor, omission, strict artifact validation, and + content-nondisclosure failure tests; + - remove expectations for wrapper fields or contexts. +3. Update evidence assertions in the representative integration and CLI + contract tests, including: + - `internal/modules/integration/dnd_npc_grounded_test.go`; + - `internal/modules/integration/dnd_location_registry_runner_test.go`; + - `internal/cli/dnd_enemy_events_contract_test.go`; and + - any additional test located by searching for `evidence-context`, + `EvidenceContext`, `Contexts`, `context_count`, or old fixture fields. +4. Ensure the integration assertions protect the behavior that matters: + - only selected accepted lanes contribute references; + - multiple lanes and overlapping windows produce one ordered unit union; + - non-monotonic unit IDs do not change source-position order; + - scene-description selection remains explicit; + - the output descriptor remains pipeline-wide rather than a lane artifact. +5. Remove obsolete helpers and fixtures that only inspect old context groups or + per-lane evidence records. Do not preserve compatibility helpers. + +### Acceptance Criteria + +- The generic JSON encoder publishes the new payload under the existing + filename and descriptor. +- Maintained integration and CLI workflows observe a deduplicated array of + unchanged source units. +- No production code or test outside the archived roadmap refers to deleted + payload fields or types. +- Focused output, integration, and CLI tests pass. + +### Validation + +```sh +gofmt -w internal/modules/generic/output/json internal/modules/integration internal/cli +go test ./internal/modules/generic/output/json ./internal/modules/integration ./internal/cli +rg -n 'context_ref|evidence_refs|selected_lanes|source_digest|context_count|\.Contexts\b' internal --glob '*.go' --glob '*.json' +``` + +Review every remaining search result rather than requiring the search to be +empty: `source_digest` and similar terms are valid in unrelated contracts and +content-free debug state. + +### Prompt Scope + +This stage is appropriately sized for one gpt-5.6-terra prompt. It owns output +and assembled-workflow adaptation only; do not revise public documentation in +this stage. + +## Stage 3: Rewrite The Canonical Documentation + +### Goal + +Make all current documentation describe the implemented minimal evidence +excerpt, with the durable contract defined in one canonical location. + +### Required Changes + +1. Rewrite `docs/integrations/evidence-context.md` as the canonical durable + contract: + - retain identity, discovery, filename, and optionality; + - define the top-level source-unit array and empty-array behavior; + - define typed-evidence selection, window expansion, union, deduplication, + source-position ordering, and unchanged-unit semantics; + - state that the excerpt can include at most every generic source unit once + but has no byte/token-size guarantee; + - state that lane artifacts remain authoritative for citations and the + excerpt contains no lane attribution; + - preserve sensitivity and consumer validation guidance. +2. Update `docs/config.md` only where it describes the result of + `evidence_context`. Preserve the existing keys, validation rules, and default + window. Link to the integration contract for payload details. +3. Update `docs/integrations/json-output.md` to describe the logical file as the + optional selected source-unit excerpt without duplicating its schema. +4. Update `docs/consumers/subprocess.md` so consumers decode the array and use + lane artifacts—not the excerpt—for authoritative citations and lane + provenance. +5. Update `docs/operations.md`, `docs/internal/modules.md`, and any other current + documentation that describes context groups, provenance records, or the old + wrapper. Keep operational sensitivity guidance and internal ownership in + their canonical documents. +6. Review `examples/dnd-complete.config.yml`. Retain the current + `evidence_context` configuration and lane list unless repository inspection + reveals an independently stale module key. Do not add a new example solely + for this unchanged configuration surface. +7. Do not modify ADRs or add a new ADR. Do not rewrite archived audit history + to match current behavior. + +### Acceptance Criteria + +- `docs/integrations/evidence-context.md` is the sole canonical definition of + the payload. +- Other current documents accurately summarize their audience-specific aspect + and link to the contract instead of duplicating it. +- No current documentation claims that `evidence-context.json` contains + context groups, evidence-reference records, lane attribution, source-level + wrapper metadata, or a guaranteed byte/token reduction. +- The maintained complete example remains valid and reflects implemented + configuration. + +### Validation + +```sh +rg -n 'context_ref|evidence_refs|selected_lanes|context_count' README.md docs examples --glob '!docs/roadmap/**' --glob '!docs/adr/**' +go test ./internal/config ./internal/modules/generic/output/json ./internal/cli +``` + +Inspect any search matches for legitimate historical or unrelated use. If the +repository provides no automated Markdown link checker, manually verify every +link changed in this stage. + +### Prompt Scope + +This documentation migration is small enough for one gpt-5.6-terra prompt. + +## Stage 4: Final Verification And Cleanup + +### Goal + +Verify the complete feature across package, integration, documentation, and +repository boundaries, then remove only obsolete implementation debris found +by that verification. + +### Required Changes + +1. Review the diff against `docs/roadmap/evidence-context.md` and confirm every + acceptance criterion is implemented. +2. Search the non-archived repository for deleted Go types, old JSON fields, + obsolete fixture shapes, and statements that describe the former contract. +3. Confirm the schema, codec, builder, pipeline request, debug summary, output + encoder, index descriptor, example configuration, and integration contract + agree on the final shape and identities. +4. Run formatting and the complete offline validation suite. +5. Make only narrowly scoped corrections required by failed checks or direct + roadmap divergence. Do not introduce unrelated refactors or new features. +6. Leave the active roadmap documents in place for post-implementation review; + retirement occurs only when separately requested after the implementation + is accepted. + +### Acceptance Criteria + +- All roadmap acceptance criteria are satisfied with no legacy compatibility + path. +- No current code, schema, fixture, example, or documentation depends on the + former wrapper/context/provenance payload. +- The structural bound is protected: the artifact contains only distinct, + unchanged units from the source document in source order. +- Repository formatting, tests, vet, and build all pass. +- The worktree contains only intentional feature changes. + +### Validation + +```sh +gofmt -w internal/framework/evidencecontext internal/framework/pipeline internal/modules/generic/output/json internal/modules/integration internal/cli +go test ./... +go vet ./... +go build ./cmd/notarius +git diff --check +git status --short +``` + +### Prompt Scope + +This verification pass is small enough for one gpt-5.6-terra prompt. If it +uncovers a material unrelated defect, report it separately instead of widening +this feature. diff --git a/internal/framework/evidencecontext/assets/schemas/source_evidence_context.v1.json b/internal/framework/evidencecontext/assets/schemas/source_evidence_context.v1.json index 169c9c0..3d22efa 100644 --- a/internal/framework/evidencecontext/assets/schemas/source_evidence_context.v1.json +++ b/internal/framework/evidencecontext/assets/schemas/source_evidence_context.v1.json @@ -2,24 +2,8 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "notarius.source.evidence_context", "title": "notarius_source_evidence_context_v1", - "type": "object", - "additionalProperties": false, - "required": ["source_id", "source_digest", "window_units", "selected_lanes", "contexts"], - "properties": { - "source_id": {"type": "string", "minLength": 1}, - "source_digest": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"}, - "window_units": {"type": "integer", "minimum": 0}, - "selected_lanes": { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": {"type": "string", "minLength": 1} - }, - "contexts": { - "type": "array", - "items": {"$ref": "#/$defs/context"} - } - }, + "type": "array", + "items": {"$ref": "#/$defs/unit"}, "$defs": { "source_ref": { "type": "object", @@ -42,25 +26,6 @@ "ref": {"$ref": "#/$defs/source_ref"}, "metadata": {"type": "object", "additionalProperties": true} } - }, - "evidence_ref": { - "type": "object", - "additionalProperties": false, - "required": ["lane_id", "source_ref"], - "properties": { - "lane_id": {"type": "string", "minLength": 1}, - "source_ref": {"$ref": "#/$defs/source_ref"} - } - }, - "context": { - "type": "object", - "additionalProperties": false, - "required": ["context_ref", "evidence_refs", "units"], - "properties": { - "context_ref": {"$ref": "#/$defs/source_ref"}, - "evidence_refs": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/evidence_ref"}}, - "units": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/unit"}} - } } } } diff --git a/internal/framework/evidencecontext/build.go b/internal/framework/evidencecontext/build.go index d862a2d..26b908a 100644 --- a/internal/framework/evidencecontext/build.go +++ b/internal/framework/evidencecontext/build.go @@ -3,118 +3,58 @@ package evidencecontext import ( "fmt" "sort" - "strings" "gitea.maximumdirect.net/eric/notarius/internal/core/source" ) -type contribution struct { - laneID string - ref source.SourceRef +type expandedRange struct { startPos int endPos int } -type expandedRange struct { - startPos int - endPos int - contributions []contribution -} - -// Build validates accepted direct references, expands them by source-document -// position, and returns their deterministic context union. +// Build validates projected source references, expands them by source-document +// position, and returns their ordered union as an owned source-unit excerpt. func Build(request BuildRequest) (Document, error) { if request.WindowUnits < 0 { - return Document{}, fmt.Errorf("window_units must not be negative") - } - lanes, err := normalizeSelectedLanes(request.SelectedLanes) - if err != nil { - return Document{}, err + return nil, fmt.Errorf("window_units must not be negative") } if err := source.ValidateDocument(request.Source); err != nil { - return Document{}, fmt.Errorf("validate source document: %w", err) + return nil, fmt.Errorf("validate source document: %w", err) } digest, err := source.DigestDocument(request.Source) if err != nil { - return Document{}, fmt.Errorf("digest source document: %w", err) + return nil, fmt.Errorf("digest source document: %w", err) } if digest != request.Source.Digest { - return Document{}, fmt.Errorf("source digest does not match source document digest") + return nil, fmt.Errorf("source digest does not match source document digest") } - selected := make(map[string]struct{}, len(lanes)) - for _, laneID := range lanes { - selected[laneID] = struct{}{} - } index := source.NewDocumentIndex(request.Source) - seen := make(map[evidenceKey]struct{}) - contributions := make([]contribution, 0) - for laneIndex, laneEvidence := range request.LaneEvidence { - laneID := strings.TrimSpace(laneEvidence.LaneID) - if _, ok := selected[laneID]; !ok { - return Document{}, fmt.Errorf("lane evidence[%d] lane %q is not selected", laneIndex, laneID) + ranges := make([]expandedRange, 0, len(request.SourceRefs)) + for refIndex, ref := range request.SourceRefs { + if err := index.ValidateRef(ref); err != nil { + return nil, fmt.Errorf("source reference[%d]: %w", refIndex, err) } - for refIndex, ref := range laneEvidence.SourceRefs { - if err := index.ValidateRef(ref); err != nil { - return Document{}, fmt.Errorf("lane %q source reference[%d]: %w", laneID, refIndex, err) + startPos, _ := index.Position(ref.StartUnitID) + endPos, _ := index.Position(ref.EndUnitID) + ranges = append(ranges, expandedRange{ + startPos: expandStart(startPos, request.WindowUnits), + endPos: expandEnd(endPos, len(request.Source.Units), request.WindowUnits), + }) + } + + merged := mergeRanges(ranges) + document := make(Document, 0) + for _, value := range merged { + for position := value.startPos; position <= value.endPos; position++ { + unit, err := cloneSourceUnit(request.Source.Units[position]) + if err != nil { + return nil, fmt.Errorf("clone source unit at position %d: %w", position, err) } - key := evidenceKey{laneID: laneID, ref: ref} - if _, exists := seen[key]; exists { - continue - } - seen[key] = struct{}{} - startPos, _ := index.Position(ref.StartUnitID) - endPos, _ := index.Position(ref.EndUnitID) - contributions = append(contributions, contribution{laneID: laneID, ref: ref, startPos: expandStart(startPos, request.WindowUnits), endPos: expandEnd(endPos, len(request.Source.Units), request.WindowUnits)}) + document = append(document, unit) } } - - sort.Slice(contributions, func(i, j int) bool { return lessContribution(contributions[i], contributions[j]) }) - document := Document{ - SourceID: request.Source.ID, - SourceDigest: digest, - WindowUnits: request.WindowUnits, - SelectedLanes: lanes, - Contexts: make([]Context, 0), - } - for _, rangeValue := range mergeRanges(contributions) { - context, err := buildContext(request.Source, rangeValue) - if err != nil { - return Document{}, err - } - document.Contexts = append(document.Contexts, context) - } - canonical, err := canonicalizeOwned(document) - if err != nil { - return Document{}, fmt.Errorf("validate evidence context: %w", err) - } - return canonical, nil -} - -type evidenceKey struct { - laneID string - ref source.SourceRef -} - -func normalizeSelectedLanes(values []string) ([]string, error) { - if len(values) == 0 { - return nil, fmt.Errorf("selected_lanes must not be empty") - } - seen := make(map[string]struct{}, len(values)) - lanes := make([]string, 0, len(values)) - for index, raw := range values { - laneID := strings.TrimSpace(raw) - if laneID == "" { - return nil, fmt.Errorf("selected_lanes[%d] must not be empty", index) - } - if _, exists := seen[laneID]; exists { - return nil, fmt.Errorf("selected_lanes lane %q is duplicated", laneID) - } - seen[laneID] = struct{}{} - lanes = append(lanes, laneID) - } - sort.Strings(lanes) - return lanes, nil + return document, nil } func expandStart(position, window int) int { @@ -132,65 +72,25 @@ func expandEnd(position, length, window int) int { return position + window } -func lessContribution(left, right contribution) bool { - if left.startPos != right.startPos { - return left.startPos < right.startPos - } - if left.endPos != right.endPos { - return left.endPos < right.endPos - } - return lessEvidenceRef(EvidenceRef{LaneID: left.laneID, SourceRef: left.ref}, EvidenceRef{LaneID: right.laneID, SourceRef: right.ref}) -} - -func mergeRanges(values []contribution) []expandedRange { +func mergeRanges(values []expandedRange) []expandedRange { if len(values) == 0 { return nil } - ranges := make([]expandedRange, 0, len(values)) + sort.Slice(values, func(i, j int) bool { + if values[i].startPos != values[j].startPos { + return values[i].startPos < values[j].startPos + } + return values[i].endPos < values[j].endPos + }) + merged := make([]expandedRange, 0, len(values)) for _, value := range values { - if len(ranges) == 0 || value.startPos > ranges[len(ranges)-1].endPos+1 { - ranges = append(ranges, expandedRange{startPos: value.startPos, endPos: value.endPos, contributions: []contribution{value}}) + if len(merged) == 0 || value.startPos > merged[len(merged)-1].endPos+1 { + merged = append(merged, value) continue } - current := &ranges[len(ranges)-1] - if value.endPos > current.endPos { - current.endPos = value.endPos + if value.endPos > merged[len(merged)-1].endPos { + merged[len(merged)-1].endPos = value.endPos } - current.contributions = append(current.contributions, value) } - return ranges -} - -func buildContext(document *source.SourceDocument, value expandedRange) (Context, error) { - evidenceRefs := make([]EvidenceRef, 0, len(value.contributions)) - for _, contribution := range value.contributions { - evidenceRefs = append(evidenceRefs, EvidenceRef{LaneID: contribution.laneID, SourceRef: contribution.ref}) - } - sort.Slice(evidenceRefs, func(i, j int) bool { return lessEvidenceRef(evidenceRefs[i], evidenceRefs[j]) }) - units := make([]source.SourceUnit, 0, value.endPos-value.startPos+1) - for position := value.startPos; position <= value.endPos; position++ { - unit, err := cloneSourceUnit(document.Units[position]) - if err != nil { - return Context{}, fmt.Errorf("clone source unit at position %d: %w", position, err) - } - units = append(units, unit) - } - return Context{ - ContextRef: source.SourceRef{SourceID: document.ID, StartUnitID: units[0].ID, EndUnitID: units[len(units)-1].ID}, - EvidenceRefs: evidenceRefs, - Units: units, - }, nil -} - -func lessEvidenceRef(left, right EvidenceRef) bool { - if left.LaneID != right.LaneID { - return left.LaneID < right.LaneID - } - if left.SourceRef.SourceID != right.SourceRef.SourceID { - return left.SourceRef.SourceID < right.SourceRef.SourceID - } - if left.SourceRef.StartUnitID != right.SourceRef.StartUnitID { - return left.SourceRef.StartUnitID < right.SourceRef.StartUnitID - } - return left.SourceRef.EndUnitID < right.SourceRef.EndUnitID + return merged } diff --git a/internal/framework/evidencecontext/codec.go b/internal/framework/evidencecontext/codec.go index a4091e1..1f4bbff 100644 --- a/internal/framework/evidencecontext/codec.go +++ b/internal/framework/evidencecontext/codec.go @@ -6,7 +6,6 @@ import ( "encoding/json" "fmt" "io" - "regexp" "strings" "sync" @@ -18,8 +17,6 @@ import ( //go:embed assets/schemas/source_evidence_context.v1.json var schemaAssets embed.FS -var digestPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`) - var ( loadSchemaOnce sync.Once loadedSchema []byte @@ -78,24 +75,24 @@ func (c *Codec) Encode(value Document) ([]byte, error) { func (c *Codec) Decode(content []byte) (Document, error) { if _, err := c.schemaBytes(); err != nil { - return Document{}, err + return nil, err } if err := validateSchemaInstance(content); err != nil { - return Document{}, fmt.Errorf("decode evidence context: %w", err) + return nil, fmt.Errorf("decode evidence context: %w", err) } decoder := json.NewDecoder(bytes.NewReader(content)) decoder.DisallowUnknownFields() var value Document if err := decoder.Decode(&value); err != nil { - return Document{}, fmt.Errorf("decode evidence context: %w", err) + return nil, fmt.Errorf("decode evidence context: %w", err) } var trailing any if err := decoder.Decode(&trailing); err != io.EOF { - return Document{}, fmt.Errorf("decode evidence context: multiple JSON values") + return nil, fmt.Errorf("decode evidence context: multiple JSON values") } canonical, err := canonicalizeOwned(value) if err != nil { - return Document{}, fmt.Errorf("decode evidence context: %w", err) + return nil, fmt.Errorf("decode evidence context: %w", err) } return canonical, nil } @@ -115,17 +112,16 @@ func loadAndCompileSchema() { return } var identity struct { - ID string `json:"$id"` - Title string `json:"title"` - Type string `json:"type"` - Required []string `json:"required"` + ID string `json:"$id"` + Title string `json:"title"` + Type string `json:"type"` } if err := json.Unmarshal(raw, &identity); err != nil { loadSchemaErr = fmt.Errorf("decode source evidence context schema: %w", err) return } - if identity.ID != SchemaID || identity.Title != SchemaName || identity.Type != "object" || !hasRequiredFields(identity.Required) { - loadSchemaErr = fmt.Errorf("source evidence context schema identity or required fields are invalid") + if identity.ID != SchemaID || identity.Title != SchemaName || identity.Type != "array" { + loadSchemaErr = fmt.Errorf("source evidence context schema identity is invalid") return } schemaDocument, err := jsonschema.UnmarshalJSON(bytes.NewReader(raw)) @@ -158,164 +154,59 @@ func validateSchemaInstance(content []byte) error { return nil } -func hasRequiredFields(required []string) bool { - want := map[string]bool{"source_id": true, "source_digest": true, "window_units": true, "selected_lanes": true, "contexts": true} - for _, field := range required { - delete(want, field) - } - return len(want) == 0 -} - func canonicalize(value Document) (Document, error) { owned, err := clone(value) if err != nil { - return Document{}, err + return nil, err } return canonicalizeOwned(owned) } func canonicalizeOwned(value Document) (Document, error) { - if err := requireIdentity("source_id", value.SourceID); err != nil { - return Document{}, err + if value == nil { + return nil, fmt.Errorf("document must be a JSON array") } - if !digestPattern.MatchString(value.SourceDigest) { - return Document{}, fmt.Errorf("source_digest must be a sha256 digest") - } - if value.WindowUnits < 0 { - return Document{}, fmt.Errorf("window_units must not be negative") - } - if err := validateSelectedLanes(value.SelectedLanes); err != nil { - return Document{}, err - } - if value.Contexts == nil { - value.Contexts = make([]Context, 0) - } - selected := make(map[string]struct{}, len(value.SelectedLanes)) - for _, laneID := range value.SelectedLanes { - selected[laneID] = struct{}{} - } - seenUnits := make(map[int]struct{}) - for contextIndex := range value.Contexts { - context, err := canonicalizeContext(value.SourceID, selected, seenUnits, value.Contexts[contextIndex], contextIndex) - if err != nil { - return Document{}, err - } - value.Contexts[contextIndex] = context - } - return value, nil -} - -func validateSelectedLanes(lanes []string) error { - if len(lanes) == 0 { - return fmt.Errorf("selected_lanes must not be empty") - } - for index, laneID := range lanes { - if err := requireIdentity(fmt.Sprintf("selected_lanes[%d]", index), laneID); err != nil { - return err - } - if index > 0 && lanes[index-1] >= laneID { - return fmt.Errorf("selected_lanes must be unique and in lexical order") - } - } - return nil -} - -func canonicalizeContext(sourceID string, selected map[string]struct{}, seenUnits map[int]struct{}, value Context, contextIndex int) (Context, error) { - prefix := fmt.Sprintf("contexts[%d]", contextIndex) - if len(value.EvidenceRefs) == 0 { - return Context{}, fmt.Errorf("%s.evidence_refs must not be empty", prefix) - } - if len(value.Units) == 0 { - return Context{}, fmt.Errorf("%s.units must not be empty", prefix) - } - if err := validateRefIdentity(sourceID, value.ContextRef, prefix+".context_ref"); err != nil { - return Context{}, err - } - positions := make(map[int]int, len(value.Units)) - for unitIndex := range value.Units { - unit := value.Units[unitIndex] + seenUnitIDs := make(map[int]struct{}, len(value)) + for unitIndex := range value { + unit := value[unitIndex] if unit.ID <= 0 || strings.TrimSpace(unit.Kind) == "" || strings.TrimSpace(unit.Text) == "" { - return Context{}, fmt.Errorf("%s.units[%d] has invalid required fields", prefix, unitIndex) + return nil, fmt.Errorf("units[%d] has invalid required fields", unitIndex) } - if err := validateRefIdentity(sourceID, unit.Ref, fmt.Sprintf("%s.units[%d].ref", prefix, unitIndex)); err != nil { - return Context{}, err + if err := validateUnitRef(unit, unitIndex); err != nil { + return nil, err } - if unit.Ref.StartUnitID != unit.ID || unit.Ref.EndUnitID != unit.ID { - return Context{}, fmt.Errorf("%s.units[%d].ref must identify unit id %d", prefix, unitIndex, unit.ID) - } - if _, exists := positions[unit.ID]; exists { - return Context{}, fmt.Errorf("%s.units contains duplicate unit id %d", prefix, unit.ID) - } - if _, exists := seenUnits[unit.ID]; exists { - return Context{}, fmt.Errorf("contexts contain duplicate unit id %d", unit.ID) - } - positions[unit.ID] = unitIndex - seenUnits[unit.ID] = struct{}{} - } - if value.ContextRef.StartUnitID != value.Units[0].ID || value.ContextRef.EndUnitID != value.Units[len(value.Units)-1].ID { - return Context{}, fmt.Errorf("%s.context_ref must identify the first and last units", prefix) - } - for evidenceIndex := range value.EvidenceRefs { - evidence := value.EvidenceRefs[evidenceIndex] - if _, ok := selected[evidence.LaneID]; !ok { - return Context{}, fmt.Errorf("%s.evidence_refs[%d].lane_id is not selected", prefix, evidenceIndex) - } - if err := requireIdentity(fmt.Sprintf("%s.evidence_refs[%d].lane_id", prefix, evidenceIndex), evidence.LaneID); err != nil { - return Context{}, err - } - if err := validateRefIdentity(sourceID, evidence.SourceRef, fmt.Sprintf("%s.evidence_refs[%d].source_ref", prefix, evidenceIndex)); err != nil { - return Context{}, err - } - start, startOK := positions[evidence.SourceRef.StartUnitID] - end, endOK := positions[evidence.SourceRef.EndUnitID] - if !startOK || !endOK || start > end { - return Context{}, fmt.Errorf("%s.evidence_refs[%d].source_ref is outside context units", prefix, evidenceIndex) - } - if evidenceIndex > 0 && !lessEvidenceRef(value.EvidenceRefs[evidenceIndex-1], evidence) { - return Context{}, fmt.Errorf("%s.evidence_refs must be unique and in deterministic order", prefix) + if _, exists := seenUnitIDs[unit.ID]; exists { + return nil, fmt.Errorf("units contains duplicate unit id %d", unit.ID) } + seenUnitIDs[unit.ID] = struct{}{} } return value, nil } -func validateRefIdentity(sourceID string, ref source.SourceRef, field string) error { - if ref.SourceID != sourceID { - return fmt.Errorf("%s.source_id does not match source_id", field) +func validateUnitRef(unit source.SourceUnit, unitIndex int) error { + prefix := fmt.Sprintf("units[%d].ref", unitIndex) + if strings.TrimSpace(unit.Ref.SourceID) == "" || strings.TrimSpace(unit.Ref.SourceID) != unit.Ref.SourceID { + return fmt.Errorf("%s.source_id must be a non-empty trimmed string", prefix) } - if ref.StartUnitID <= 0 || ref.EndUnitID <= 0 { - return fmt.Errorf("%s endpoints must be positive", field) - } - return nil -} - -func requireIdentity(field, value string) error { - if strings.TrimSpace(value) == "" || strings.TrimSpace(value) != value { - return fmt.Errorf("%s must be a non-empty trimmed string", field) + if unit.Ref.StartUnitID != unit.ID || unit.Ref.EndUnitID != unit.ID { + return fmt.Errorf("%s must identify unit id %d", prefix, unit.ID) } return nil } func clone(value Document) (Document, error) { - value.SelectedLanes = append([]string(nil), value.SelectedLanes...) - if value.Contexts == nil { - value.Contexts = make([]Context, 0) - } else { - contexts := make([]Context, len(value.Contexts)) - for contextIndex, context := range value.Contexts { - contexts[contextIndex].ContextRef = context.ContextRef - contexts[contextIndex].EvidenceRefs = append([]EvidenceRef(nil), context.EvidenceRefs...) - contexts[contextIndex].Units = make([]source.SourceUnit, len(context.Units)) - for unitIndex, unit := range context.Units { - cloned, err := cloneSourceUnit(unit) - if err != nil { - return Document{}, fmt.Errorf("clone contexts[%d].units[%d]: %w", contextIndex, unitIndex, err) - } - contexts[contextIndex].Units[unitIndex] = cloned - } - } - value.Contexts = contexts + if value == nil { + return nil, nil } - return value, nil + cloned := make(Document, len(value)) + for unitIndex, unit := range value { + owned, err := cloneSourceUnit(unit) + if err != nil { + return nil, fmt.Errorf("clone units[%d]: %w", unitIndex, err) + } + cloned[unitIndex] = owned + } + return cloned, nil } func cloneSourceUnit(unit source.SourceUnit) (source.SourceUnit, error) { diff --git a/internal/framework/evidencecontext/evidencecontext_test.go b/internal/framework/evidencecontext/evidencecontext_test.go index 5f6854a..66318b4 100644 --- a/internal/framework/evidencecontext/evidencecontext_test.go +++ b/internal/framework/evidencecontext/evidencecontext_test.go @@ -2,7 +2,6 @@ package evidencecontext import ( "bytes" - "encoding/json" "math" "os" "reflect" @@ -12,126 +11,56 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/core/source" ) -func TestBuildExpandsAndMergesEvidenceByDocumentPosition(t *testing.T) { +func TestBuildSelectsExpandedSourceUnitUnion(t *testing.T) { for _, test := range []struct { - name string - window int - evidence []LaneEvidence - wantUnits [][]int - wantRefs [][]EvidenceRef + name string + window int + refs []source.SourceRef + wantIDs []int }{ - { - name: "zero window", - evidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3)}}}, - wantUnits: [][]int{{3}}, - wantRefs: [][]EvidenceRef{{{LaneID: "npcs", SourceRef: ref(3, 3)}}}, - }, - { - name: "non monotonic ids use positions and clip boundaries", - window: 1, - evidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3)}}}, - wantUnits: [][]int{{10, 3, 30}}, - wantRefs: [][]EvidenceRef{{{LaneID: "npcs", SourceRef: ref(3, 3)}}}, - }, - { - name: "separate gaps stay separate", - evidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(10, 10), ref(50, 50)}}}, - wantUnits: [][]int{{10}, {50}}, - wantRefs: [][]EvidenceRef{{{LaneID: "npcs", SourceRef: ref(10, 10)}}, {{LaneID: "npcs", SourceRef: ref(50, 50)}}}, - }, - { - name: "overlapping windows merge", - window: 1, - evidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3), ref(30, 30)}}}, - wantUnits: [][]int{{10, 3, 30, 7}}, - wantRefs: [][]EvidenceRef{{{LaneID: "npcs", SourceRef: ref(3, 3)}, {LaneID: "npcs", SourceRef: ref(30, 30)}}}, - }, - { - name: "contiguous windows merge", - window: 1, - evidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(10, 10), ref(7, 7)}}}, - wantUnits: [][]int{{10, 3, 30, 7, 50}}, - wantRefs: [][]EvidenceRef{{{LaneID: "npcs", SourceRef: ref(7, 7)}, {LaneID: "npcs", SourceRef: ref(10, 10)}}}, - }, - { - name: "duplicate contributions retain unique lane attribution", - evidence: []LaneEvidence{ - {LaneID: "spells", SourceRefs: []source.SourceRef{ref(30, 30), ref(30, 30)}}, - {LaneID: "npcs", SourceRefs: []source.SourceRef{ref(30, 30)}}, - }, - wantUnits: [][]int{{30}}, - wantRefs: [][]EvidenceRef{{{LaneID: "npcs", SourceRef: ref(30, 30)}, {LaneID: "spells", SourceRef: ref(30, 30)}}}, - }, - { - name: "empty contributions retain explicit empty contexts", - evidence: []LaneEvidence{{LaneID: "npcs"}}, - wantUnits: [][]int{}, - wantRefs: [][]EvidenceRef{}, - }, - { - name: "largest window clips without overflow", - window: math.MaxInt, - evidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(30, 30)}}}, - wantUnits: [][]int{{10, 3, 30, 7, 50}}, - wantRefs: [][]EvidenceRef{{{LaneID: "npcs", SourceRef: ref(30, 30)}}}, - }, + {name: "zero window", refs: []source.SourceRef{ref(3, 3)}, wantIDs: []int{3}}, + {name: "non monotonic IDs use document positions", window: 1, refs: []source.SourceRef{ref(3, 3)}, wantIDs: []int{10, 3, 30}}, + {name: "boundary clamping", window: 1, refs: []source.SourceRef{ref(10, 10), ref(50, 50)}, wantIDs: []int{10, 3, 7, 50}}, + {name: "overlapping and adjacent windows merge", window: 1, refs: []source.SourceRef{ref(3, 3), ref(30, 30), ref(30, 30)}, wantIDs: []int{10, 3, 30, 7}}, + {name: "adjacent expanded ranges merge", window: 1, refs: []source.SourceRef{ref(10, 10), ref(7, 7)}, wantIDs: []int{10, 3, 30, 7, 50}}, + {name: "largest window clips without overflow", window: math.MaxInt, refs: []source.SourceRef{ref(30, 30)}, wantIDs: []int{10, 3, 30, 7, 50}}, + {name: "no references returns an initialized empty document", wantIDs: []int{}}, } { t.Run(test.name, func(t *testing.T) { - document := testDocument(t) - got, err := Build(BuildRequest{Source: document, WindowUnits: test.window, SelectedLanes: []string{"spells", "npcs"}, LaneEvidence: test.evidence}) + got, err := Build(BuildRequest{Source: testDocument(t), WindowUnits: test.window, SourceRefs: test.refs}) if err != nil { t.Fatalf("Build() error = %v", err) } - if want := []string{"npcs", "spells"}; !reflect.DeepEqual(got.SelectedLanes, want) { - t.Fatalf("SelectedLanes = %#v, want %#v", got.SelectedLanes, want) + if got == nil { + t.Fatal("Build() returned a nil document") } - if got.WindowUnits != test.window || got.SourceID != document.ID || got.SourceDigest != document.Digest { - t.Fatalf("Build() identity = %#v, want source and window identity", got) - } - if actual := contextUnitIDs(got.Contexts); !reflect.DeepEqual(actual, test.wantUnits) { - t.Fatalf("context unit ids = %#v, want %#v", actual, test.wantUnits) - } - if actual := contextEvidenceRefs(got.Contexts); !reflect.DeepEqual(actual, test.wantRefs) { - t.Fatalf("context evidence refs = %#v, want %#v", actual, test.wantRefs) + if actual := unitIDs(got); !reflect.DeepEqual(actual, test.wantIDs) { + t.Fatalf("unit IDs = %#v, want %#v", actual, test.wantIDs) } }) } } -func TestBuildIsStableAndOwnsSourceAndInputs(t *testing.T) { +func TestBuildCopiesSelectedUnitsAndMetadata(t *testing.T) { document := testDocument(t) - refs := []source.SourceRef{ref(30, 30), ref(3, 3)} - request := BuildRequest{ - Source: document, - WindowUnits: 1, - SelectedLanes: []string{"spells", "npcs"}, - LaneEvidence: []LaneEvidence{{LaneID: "spells", SourceRefs: refs}, {LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3)}}}, - } - first, err := Build(request) + first, err := Build(BuildRequest{Source: document, WindowUnits: 1, SourceRefs: []source.SourceRef{ref(3, 3)}}) if err != nil { t.Fatal(err) } - secondRequest := request - secondRequest.LaneEvidence = []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3)}}, {LaneID: "spells", SourceRefs: []source.SourceRef{ref(3, 3), ref(30, 30)}}} - second, err := Build(secondRequest) + second, err := Build(BuildRequest{Source: document, WindowUnits: 1, SourceRefs: []source.SourceRef{ref(3, 3)}}) if err != nil { t.Fatal(err) } - if !reflect.DeepEqual(first, second) { - t.Fatalf("Build() order differs:\nfirst: %#v\nsecond: %#v", first, second) + if !reflect.DeepEqual(first[0], document.Units[0]) { + t.Fatalf("first unit = %#v, want unchanged source unit %#v", first[0], document.Units[0]) } - first.SelectedLanes[0] = "changed" - first.Contexts[0].Units[0].Metadata["nested"].(map[string]any)["value"] = "changed" + first[0].Metadata["nested"].(map[string]any)["value"] = "changed" if document.Units[0].Metadata["nested"].(map[string]any)["value"] != "original" { - t.Fatal("Build() returned metadata aliases to source document") + t.Fatal("Build() returned metadata aliases to the source document") } document.Units[0].Metadata["nested"].(map[string]any)["value"] = "later" - if second.Contexts[0].Units[0].Metadata["nested"].(map[string]any)["value"] != "original" { - t.Fatal("Build() retained metadata aliases to source document") - } - refs[0].StartUnitID = 999 - if !containsEvidenceRef(second.Contexts[0].EvidenceRefs, ref(30, 30)) { - t.Fatal("Build() retained source-reference input aliases") + if second[0].Metadata["nested"].(map[string]any)["value"] != "original" { + t.Fatal("Build() retained metadata aliases to the source document") } } @@ -142,18 +71,11 @@ func TestBuildRejectsInvalidInputs(t *testing.T) { want string }{ {name: "negative window", mutate: func(request *BuildRequest) { request.WindowUnits = -1 }, want: "window_units"}, - {name: "blank selected lane", mutate: func(request *BuildRequest) { request.SelectedLanes = []string{" "} }, want: "selected_lanes"}, - {name: "duplicate selected lane", mutate: func(request *BuildRequest) { request.SelectedLanes = []string{"npcs", " npcs "} }, want: "duplicated"}, - {name: "unselected contribution", mutate: func(request *BuildRequest) { - request.LaneEvidence = []LaneEvidence{{LaneID: "other", SourceRefs: []source.SourceRef{ref(3, 3)}}} - }, want: "not selected"}, {name: "source digest mismatch", mutate: func(request *BuildRequest) { request.Source.Digest = "sha256:" + strings.Repeat("0", 64) }, want: "does not match"}, - {name: "invalid reference", mutate: func(request *BuildRequest) { - request.LaneEvidence = []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(99, 99)}}} - }, want: "source reference[0]"}, + {name: "invalid reference", mutate: func(request *BuildRequest) { request.SourceRefs = []source.SourceRef{ref(99, 99)} }, want: "source reference[0]"}, } { t.Run(test.name, func(t *testing.T) { - request := BuildRequest{Source: testDocument(t), SelectedLanes: []string{"npcs"}, LaneEvidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3)}}}} + request := BuildRequest{Source: testDocument(t), SourceRefs: []source.SourceRef{ref(3, 3)}} test.mutate(&request) if _, err := Build(request); err == nil || !strings.Contains(err.Error(), test.want) { t.Fatalf("Build() error = %v, want %q", err, test.want) @@ -162,7 +84,7 @@ func TestBuildRejectsInvalidInputs(t *testing.T) { } } -func TestCodecRoundTripsCompactFixtureAndOwnsDecodedValues(t *testing.T) { +func TestCodecRoundTripsFixtureAndOwnsValues(t *testing.T) { fixture, err := os.ReadFile("testdata/source_evidence_context.v1.json") if err != nil { t.Fatal(err) @@ -179,15 +101,16 @@ func TestCodecRoundTripsCompactFixtureAndOwnsDecodedValues(t *testing.T) { if !bytes.Equal(encoded, bytes.TrimSpace(fixture)) { t.Fatalf("fixture does not use canonical encoding\nwant: %s\n got: %s", fixture, encoded) } - value.Contexts[0].Units[0].Text = "changed" + value[0].Text = "changed" decoded, err := codec.Decode(encoded) if err != nil { t.Fatal(err) } - if decoded.Contexts[0].Units[0].Text != "The party meets Rowan." { + if decoded[0].Text != "The party meets Rowan." { t.Fatal("Encode() retained mutable document storage") } - built, err := Build(BuildRequest{Source: testDocument(t), WindowUnits: 1, SelectedLanes: []string{"npcs"}, LaneEvidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3)}}}}) + + built, err := Build(BuildRequest{Source: testDocument(t), WindowUnits: 1, SourceRefs: []source.SourceRef{ref(3, 3)}}) if err != nil { t.Fatal(err) } @@ -203,82 +126,51 @@ func TestCodecRoundTripsCompactFixtureAndOwnsDecodedValues(t *testing.T) { if err != nil { t.Fatal(err) } - first.Contexts[0].Units[0].Metadata["nested"].(map[string]any)["value"] = "changed" - if second.Contexts[0].Units[0].Metadata["nested"].(map[string]any)["value"] != "original" { + first[0].Metadata["nested"].(map[string]any)["value"] = "changed" + if second[0].Metadata["nested"].(map[string]any)["value"] != "original" { t.Fatal("Decode() returned metadata aliases") } } -func TestCodecRejectsInvalidDurableBoundaries(t *testing.T) { - value, err := Build(BuildRequest{Source: testDocument(t), SelectedLanes: []string{"npcs"}, LaneEvidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3)}}}}) - if err != nil { - t.Fatal(err) - } +func TestCodecRejectsInvalidDurablePayloads(t *testing.T) { for _, test := range []struct { - name string - mutate func(*Document) + name string + content string }{ - {name: "unsorted lanes", mutate: func(value *Document) { value.SelectedLanes = []string{"z", "a"} }}, - {name: "context range mismatch", mutate: func(value *Document) { value.Contexts[0].ContextRef.EndUnitID = 999 }}, - {name: "mismatched evidence source", mutate: func(value *Document) { value.Contexts[0].EvidenceRefs[0].SourceRef.SourceID = "other" }}, - {name: "invalid evidence range", mutate: func(value *Document) { - value.Contexts[0].EvidenceRefs[0].SourceRef.StartUnitID = 10 - }}, - {name: "duplicate context unit", mutate: func(value *Document) { value.Contexts = append(value.Contexts, value.Contexts[0]) }}, + {name: "null", content: "null"}, + {name: "wrapper object", content: `{"units":[]}`}, + {name: "missing required unit field", content: `[{"id":1,"kind":"segment","ref":{"source_id":"session","start_unit_id":1,"end_unit_id":1}}]`}, + {name: "unknown unit field", content: `[{"id":1,"kind":"segment","text":"text","ref":{"source_id":"session","start_unit_id":1,"end_unit_id":1},"unknown":true}]`}, + {name: "unknown reference field", content: `[{"id":1,"kind":"segment","text":"text","ref":{"source_id":"session","start_unit_id":1,"end_unit_id":1,"unknown":true}}]`}, + {name: "invalid self reference", content: `[{"id":1,"kind":"segment","text":"text","ref":{"source_id":"session","start_unit_id":1,"end_unit_id":2}}]`}, + {name: "duplicate units", content: `[{"id":1,"kind":"segment","text":"one","ref":{"source_id":"session","start_unit_id":1,"end_unit_id":1}},{"id":1,"kind":"segment","text":"two","ref":{"source_id":"session","start_unit_id":1,"end_unit_id":1}}]`}, + {name: "multiple JSON values", content: `[] []`}, } { t.Run(test.name, func(t *testing.T) { - candidate, err := clone(value) - if err != nil { - t.Fatal(err) - } - test.mutate(&candidate) - if _, err := New().Encode(candidate); err == nil { - t.Fatal("Encode() error = nil, want durable model rejection") - } - }) - } - content, err := New().Encode(value) - if err != nil { - t.Fatal(err) - } - for _, test := range []struct { - name string - mutate func(map[string]any) - }{ - {name: "missing contexts", mutate: func(value map[string]any) { delete(value, "contexts") }}, - {name: "null contexts", mutate: func(value map[string]any) { value["contexts"] = nil }}, - {name: "unknown fixed field", mutate: func(value map[string]any) { value["unknown"] = true }}, - {name: "missing units", mutate: func(value map[string]any) { delete(contextObject(value, 0), "units") }}, - {name: "null evidence refs", mutate: func(value map[string]any) { contextObject(value, 0)["evidence_refs"] = nil }}, - } { - t.Run(test.name, func(t *testing.T) { - raw := decodeJSON(t, content) - test.mutate(raw) - mutated, err := json.Marshal(raw) - if err != nil { - t.Fatal(err) - } - if _, err := New().Decode(mutated); err == nil { + if _, err := New().Decode([]byte(test.content)); err == nil { t.Fatal("Decode() error = nil, want strict payload rejection") } }) } - if _, err := New().Decode(append(content, []byte(" {}")...)); err == nil { - t.Fatal("Decode() error = nil, want trailing JSON rejection") + if _, err := New().Encode(nil); err == nil { + t.Fatal("Encode(nil) error = nil, want array rejection") } } -func TestSerializeUsesFixedArtifactIdentity(t *testing.T) { - artifact, err := Serialize(BuildRequest{Source: testDocument(t), SelectedLanes: []string{"npcs"}}) +func TestSerializeUsesFixedArtifactIdentityAndEmptyArray(t *testing.T) { + artifact, err := Serialize(BuildRequest{Source: testDocument(t)}) if err != nil { t.Fatal(err) } if artifact.Kind != ArtifactKind || artifact.MediaType != MediaType || artifact.Schema.ID != SchemaID || artifact.Schema.Name != SchemaName || artifact.Schema.Version != SchemaVersion { t.Fatalf("Serialize() = %#v, want fixed artifact identity", artifact) } + if string(artifact.Content) != "[]" { + t.Fatalf("Serialize() content = %s, want []", artifact.Content) + } decoded, err := New().Decode(artifact.Content) - if err != nil || len(decoded.Contexts) != 0 || decoded.Contexts == nil { - t.Fatalf("Decode(Serialize()) = %#v, %v; want explicit empty contexts", decoded, err) + if err != nil || decoded == nil || len(decoded) != 0 { + t.Fatalf("Decode(Serialize()) = %#v, %v; want explicit empty array", decoded, err) } } @@ -306,45 +198,10 @@ func ref(start, end int) source.SourceRef { return source.SourceRef{SourceID: "session", StartUnitID: start, EndUnitID: end} } -func contextUnitIDs(contexts []Context) [][]int { - values := make([][]int, len(contexts)) - for index, context := range contexts { - values[index] = make([]int, len(context.Units)) - for unitIndex, unit := range context.Units { - values[index][unitIndex] = unit.ID - } +func unitIDs(units Document) []int { + values := make([]int, len(units)) + for index, unit := range units { + values[index] = unit.ID } return values } - -func contextEvidenceRefs(contexts []Context) [][]EvidenceRef { - values := make([][]EvidenceRef, len(contexts)) - for index, context := range contexts { - values[index] = append([]EvidenceRef(nil), context.EvidenceRefs...) - } - return values -} - -func containsEvidenceRef(values []EvidenceRef, want source.SourceRef) bool { - for _, value := range values { - if value.SourceRef == want { - return true - } - } - return false -} - -func decodeJSON(t *testing.T, content []byte) map[string]any { - t.Helper() - decoder := json.NewDecoder(bytes.NewReader(content)) - decoder.UseNumber() - var value map[string]any - if err := decoder.Decode(&value); err != nil { - t.Fatal(err) - } - return value -} - -func contextObject(value map[string]any, index int) map[string]any { - return value["contexts"].([]any)[index].(map[string]any) -} diff --git a/internal/framework/evidencecontext/model.go b/internal/framework/evidencecontext/model.go index c053e58..cf93dc0 100644 --- a/internal/framework/evidencecontext/model.go +++ b/internal/framework/evidencecontext/model.go @@ -14,37 +14,12 @@ const ( MediaType = "application/json" ) -// Document is the durable union of direct evidence and surrounding source -// context selected for one accepted source document. -type Document struct { - SourceID string `json:"source_id"` - SourceDigest string `json:"source_digest"` - WindowUnits int `json:"window_units"` - SelectedLanes []string `json:"selected_lanes"` - Contexts []Context `json:"contexts"` -} +// Document is the durable selected source-unit excerpt. +type Document []source.SourceUnit -type Context struct { - ContextRef source.SourceRef `json:"context_ref"` - EvidenceRefs []EvidenceRef `json:"evidence_refs"` - Units []source.SourceUnit `json:"units"` -} - -type EvidenceRef struct { - LaneID string `json:"lane_id"` - SourceRef source.SourceRef `json:"source_ref"` -} - -// LaneEvidence attributes direct source references to one selected lane. -type LaneEvidence struct { - LaneID string `json:"lane_id"` - SourceRefs []source.SourceRef `json:"source_refs"` -} - -// BuildRequest supplies accepted source material and direct lane evidence. +// BuildRequest supplies accepted source material and projected source references. type BuildRequest struct { - Source *source.SourceDocument - WindowUnits int - SelectedLanes []string - LaneEvidence []LaneEvidence + Source *source.SourceDocument + WindowUnits int + SourceRefs []source.SourceRef } diff --git a/internal/framework/evidencecontext/testdata/source_evidence_context.v1.json b/internal/framework/evidencecontext/testdata/source_evidence_context.v1.json index 38175b0..ea838fb 100644 --- a/internal/framework/evidencecontext/testdata/source_evidence_context.v1.json +++ b/internal/framework/evidencecontext/testdata/source_evidence_context.v1.json @@ -1 +1 @@ -{"source_id":"session-alpha","source_digest":"sha256:0000000000000000000000000000000000000000000000000000000000000000","window_units":0,"selected_lanes":["npcs"],"contexts":[{"context_ref":{"source_id":"session-alpha","start_unit_id":13,"end_unit_id":13},"evidence_refs":[{"lane_id":"npcs","source_ref":{"source_id":"session-alpha","start_unit_id":13,"end_unit_id":13}}],"units":[{"id":13,"kind":"transcript_segment","text":"The party meets Rowan.","ref":{"source_id":"session-alpha","start_unit_id":13,"end_unit_id":13}}]}]} +[{"id":13,"kind":"transcript_segment","text":"The party meets Rowan.","ref":{"source_id":"session-alpha","start_unit_id":13,"end_unit_id":13}}] diff --git a/internal/framework/pipeline/evidence_output.go b/internal/framework/pipeline/evidence_output.go index b1bcaf5..3713f8e 100644 --- a/internal/framework/pipeline/evidence_output.go +++ b/internal/framework/pipeline/evidence_output.go @@ -20,7 +20,6 @@ type debugEvidenceContextSummary struct { SchemaVersion string `json:"schema_version"` SelectedLanes []string `json:"selected_lanes"` WindowUnits int `json:"window_units"` - ContextCount int `json:"context_count"` UnitCount int `json:"unit_count"` SourceDigest string `json:"source_digest"` } @@ -49,10 +48,9 @@ func buildOutputEvidenceContext(prepared *PreparedPipeline, doc *source.SourceDo } request := evidencecontext.BuildRequest{ - Source: doc, - WindowUnits: prepared.evidencePlan.policy.WindowUnits, - SelectedLanes: append([]string(nil), prepared.evidencePlan.policy.LaneIDs...), - LaneEvidence: make([]evidencecontext.LaneEvidence, 0, len(prepared.evidencePlan.lanes)), + Source: doc, + WindowUnits: prepared.evidencePlan.policy.WindowUnits, + SourceRefs: make([]source.SourceRef, 0), } for _, lane := range prepared.evidencePlan.lanes { output, ok := byLane[lane.laneID] @@ -73,10 +71,7 @@ func buildOutputEvidenceContext(prepared *PreparedPipeline, doc *source.SourceDo if err != nil { return nil, nil, fmt.Errorf("evidence context output lane %q: accepted normalized artifact cannot be projected", lane.laneID) } - request.LaneEvidence = append(request.LaneEvidence, evidencecontext.LaneEvidence{ - LaneID: lane.laneID, - SourceRefs: append([]source.SourceRef(nil), references...), - }) + request.SourceRefs = append(request.SourceRefs, references...) } document, err := evidencecontext.Build(request) @@ -99,13 +94,10 @@ func buildOutputEvidenceContext(prepared *PreparedPipeline, doc *source.SourceDo SchemaID: artifact.Schema.ID, SchemaName: artifact.Schema.Name, SchemaVersion: artifact.Schema.Version, - SelectedLanes: append([]string(nil), document.SelectedLanes...), - WindowUnits: document.WindowUnits, - ContextCount: len(document.Contexts), - SourceDigest: document.SourceDigest, - } - for _, context := range document.Contexts { - summary.UnitCount += len(context.Units) + SelectedLanes: append([]string(nil), prepared.evidencePlan.policy.LaneIDs...), + WindowUnits: prepared.evidencePlan.policy.WindowUnits, + SourceDigest: doc.Digest, + UnitCount: len(document), } return contracts.CloneSerializedArtifactPointer(artifact), &summary, nil } diff --git a/internal/framework/pipeline/evidence_output_test.go b/internal/framework/pipeline/evidence_output_test.go index 6989ed1..22b3f06 100644 --- a/internal/framework/pipeline/evidence_output_test.go +++ b/internal/framework/pipeline/evidence_output_test.go @@ -97,14 +97,11 @@ func TestRunnerBuildsEvidenceContextFromSelectedNormalizedOutputs(t *testing.T) } value := decodeCapturedEvidence(t, encoder) - if !reflect.DeepEqual(value.SelectedLanes, []string{"alpha", "beta", "inactive"}) || len(value.Contexts) != 1 || len(value.Contexts[0].Units) != 3 { + if actual := []int{value[0].ID, value[1].ID, value[2].ID}; !reflect.DeepEqual(actual, []int{1, 2, 3}) { t.Fatalf("evidence context = %#v, want selected union", value) } - if got := value.Contexts[0].EvidenceRefs; len(got) != 2 || got[0].LaneID != "alpha" || got[1].LaneID != "beta" { - t.Fatalf("evidence refs = %#v, want both selected lanes", got) - } debugJSON := string(debug.json["output/evidence-context.json"]) - if strings.Contains(debugJSON, "text-1") || strings.Contains(debugJSON, "metadata") || !strings.Contains(debugJSON, `"artifact_kind":"source/evidence-context"`) || !strings.Contains(debugJSON, `"schema_id":"notarius.source.evidence_context"`) || !strings.Contains(debugJSON, `"context_count":1`) || !strings.Contains(debugJSON, `"unit_count":3`) { + if strings.Contains(debugJSON, "text-1") || strings.Contains(debugJSON, "metadata") || !strings.Contains(debugJSON, `"artifact_kind":"source/evidence-context"`) || !strings.Contains(debugJSON, `"schema_id":"notarius.source.evidence_context"`) || strings.Contains(debugJSON, "context_count") || !strings.Contains(debugJSON, `"unit_count":3`) { t.Fatalf("evidence debug envelope = %s, want only allowlisted summary", debugJSON) } } @@ -134,7 +131,7 @@ func TestRunnerEvidenceContextOmitsAbsentAndRejectedLanes(t *testing.T) { t.Fatalf("rejections = %#v, want rejected lane unchanged", result.Rejected) } value := decodeCapturedEvidence(t, encoder) - if len(value.Contexts) != 1 || len(value.Contexts[0].EvidenceRefs) != 1 || value.Contexts[0].EvidenceRefs[0].LaneID != "present" { + if len(value) != 1 || value[0].ID != 1 { t.Fatalf("evidence context = %#v, want present lane only", value) } } @@ -232,7 +229,7 @@ func TestRunnerEvidenceContextRebuildsFromAcceptedCheckpoint(t *testing.T) { t.Fatalf("Run() error = %v", err) } value := decodeCapturedEvidence(t, encoder) - if len(value.Contexts) != 1 || value.Contexts[0].EvidenceRefs[0].LaneID != "notes" { + if len(value) != 1 || value[0].ID != 1 { t.Fatalf("evidence context = %#v, want checkpointed normalized output", value) } }