diff --git a/docs/roadmap/accepted-chunk-map-export.md b/docs/roadmap/accepted-chunk-map-export.md deleted file mode 100644 index 6d8365e..0000000 --- a/docs/roadmap/accepted-chunk-map-export.md +++ /dev/null @@ -1,261 +0,0 @@ -# Accepted Chunk Map Export - -Status: Implemented - -## Purpose - -Notarius already creates and validates one materialized chunk map before any -artifact lane executes. That map contains stable chunk identities, ordered -current-source ranges, and accepted namespaced annotations. It is useful -downstream data, but today it is visible only in internal execution and -explicit debug surfaces. - -Add an opt-in durable chunk-map artifact to the output bundle. Export the exact -accepted chunks used by the run rather than reconstructing them through an -extractor or exposing a chunker's unvalidated model response. - -This is a framework and output concern. Chunking remains pipeline-wide and -precedes all artifact lanes. - -## Desired End State - -The production JSON output encoder can be configured to add a canonical -`chunk-map.json` file to the logical output bundle. The file describes the -accepted materialized chunks used for lane execution without embedding source -units or transcript text. - -Use these durable identities: - -| Concern | Identity | -| --- | --- | -| Artifact kind | `source/chunk-map` | -| Logical file | `chunk-map.json` | -| Schema ID | `notarius.source.chunk_map` | -| Schema name | `notarius_source_chunk_map_v1` | -| Schema version | `v1` | -| Media type | `application/json` | - -The framework owns the artifact model, schema, validation, and canonical -encoding. Output encoders receive the serialized artifact through the generic -output request. The JSON encoder owns only the opt-in decision, logical file -placement, and output-index entry. - -## Configuration - -Add one strict option to the production `json` output module: - -```yaml -pipelines: - dnd-session: - input: seriatim - output: - module: json - options: - include_chunk_map: true -``` - -`include_chunk_map` is a boolean and defaults to `false`. Unknown options and -non-boolean values remain configuration errors. Do not add a separate CLI flag -or top-level filesystem option: this choice changes the logical files produced -by an output module, not their physical destination. - -Other output encoders may ignore the available chunk-map artifact unless and -until they define their own explicit export behavior. - -## Durable Artifact Contract - -The payload is one strict JSON object with this shape: - -```json -{ - "source_id": "session-7", - "source_digest": "sha256:0123456789abcdef...", - "plan_digest": "sha256:abcdef0123456789...", - "requested_chunker": "dnd/scenes", - "producer": { - "input_module": "seriatim", - "chunk_module": "dnd/scenes", - "llm_profile": "dnd-scenes" - }, - "plan_annotations": {}, - "chunks": [ - { - "id": "chunk-000001", - "index": 0, - "source_ref": { - "source_id": "session-7", - "start_unit_id": 1, - "end_unit_id": 18 - }, - "unit_count": 18, - "annotations": { - "dnd/scenes": { - "title": "At the city gate" - } - } - } - ] -} -``` - -Required top-level fields are: - -- `source_id`: the accepted source document identity; -- `source_digest`: the canonical digest of that document; -- `plan_digest`: the canonical digest of the accepted chunk plan; -- `requested_chunker`: the chunk module selected by the current resolved - pipeline; -- `producer`: the identity of the component that originally produced the - accepted plan; -- `plan_annotations`: the accepted plan-level annotation namespace map; and -- `chunks`: the non-empty ordered list used for lane execution. - -`producer` requires `input_module` and `chunk_module`. `llm_profile` is included -only when the producing chunker was LLM-backed. Producer references, module -metadata, warnings, creation timestamps, cache paths, and cache actions remain -in their existing provenance and diagnostic surfaces; they are not copied into -this artifact. - -Each chunk requires: - -- `id`: the stable materialized chunk ID; -- `index`: its zero-based position in execution order; -- `source_ref`: the exact inclusive current-source range; -- `unit_count`: the number of materialized source units in that range; and -- `annotations`: the accepted range-level annotation namespace map. - -Unknown fields are rejected at every fixed object level. Annotation namespaces -retain their canonical JSON values and may contain domain-specific JSON of any -type. Empty annotation maps are encoded as `{}` so the shape remains explicit. - -The durable artifact must guarantee that: - -- source and module identities are non-empty and contain no surrounding - whitespace; -- source and plan digests use the canonical `sha256:` representation; -- chunk IDs are non-empty and unique; -- chunk indexes are unique, contiguous, zero-based, and agree with array order; -- every chunk reference uses the top-level source identity; -- every range has positive endpoint IDs and was validated in source-document - order when the artifact was constructed; -- every unit count is positive; -- annotations contain valid JSON under non-empty canonical namespaces and are - canonicalized before digest calculation; and -- reconstructing the logical plan from the source digest, plan annotations, - chunk ranges, and chunk annotations reproduces `plan_digest`. - -The artifact represents chunk structure, not source content. It must not -contain materialized units, transcript bytes, source-unit metadata, chunk -content, private model responses, rejected boundary proposals, or debug -payloads. - -## Accepted-State And Failure Policy - -Construct the artifact only after plan materialization and the configured chunk -validator chain have accepted the chunks. The runner passes the same immutable -logical chunk identities, ranges, and annotations used for every lane; no -second chunking or model call occurs. - -When `include_chunk_map` is enabled: - -- emit the artifact even if one or more later artifact lanes are rejected; -- omit it when chunk validation rejects the candidate map, because no accepted - chunk map exists; -- retain the ordinary rejection record when it is omitted for that reason; and -- treat failure to construct, validate, serialize, index, or write an available - accepted chunk map as a run failure, consistent with any explicitly requested - durable output. - -A cache hit and a newly generated plan with the same accepted logical plan must -produce the same chunk ranges, annotations, plan digest, and materialized chunk -identities. The artifact records both the current `requested_chunker` and the -stored producer's `chunk_module`, since canonical plan reuse permits those -identities to differ. - -## Output Bundle Integration - -Extend `index.json` with an optional `chunk_map` descriptor: - -```json -{ - "chunk_map": { - "artifact_kind": "source/chunk-map", - "file": "chunk-map.json", - "media_type": "application/json", - "schema_id": "notarius.source.chunk_map", - "schema_name": "notarius_source_chunk_map_v1", - "schema_version": "v1" - } -} -``` - -The descriptor and file are both absent when export is disabled or no accepted -chunk map exists. The chunk map does not appear in lane-oriented -`output_files`, because it is pipeline-wide and has no lane, extractor, merger, -or normalizer identity. - -The existing manifest remains the canonical run-provenance index. Its -`chunk_plan` summary continues to own cache mode, lookup and publication -actions, producer reference provenance, module metadata, timestamps, and -validation status. The chunk-map artifact provides the accepted structure and -only the minimal producer identities needed to interpret it independently. - -## Ownership And Architecture - -Use a domain-neutral framework package for the durable chunk-map DTO, embedded -JSON Schema, invariant validation, cloning, and canonical serialization. Do not -marshal `source.Chunk` directly: its content, units, metadata, and internal -fields are intentionally broader than this external contract. - -Extend the output request with an optional cloned serialized chunk-map artifact. -The runner constructs that value from the accepted chunk-plan execution result -before invoking the output encoder. This keeps source and chunk-plan knowledge -out of the generic JSON encoder and allows future encoders to consume the same -framework-owned representation. - -The JSON encoder adds the file and index descriptor only when its -`include_chunk_map` option is true and the request contains an accepted -artifact. It applies the same logical-path validation and pretty-printed JSON -conventions as the rest of the bundle. - -## Security And Data Handling - -Chunk annotations may contain model-derived or source-derived information. -Treat `chunk-map.json` as durable user output with the same sensitivity and -retention expectations as lane artifacts. Opt-in export prevents new durable -content from appearing silently in existing pipelines. - -Do not copy annotation values, source ranges, or chunk IDs into the manifest. -Do not include external-reference content or filesystem paths in the chunk map. -Existing output-directory confinement, atomic writing, and permission policy -apply unchanged. - -## Quality And Documentation Policy - -The framework contract must remain strict, canonical, immutable across -ownership boundaries, and independent of source or D&D interpretation. The -runner must preserve accepted-state and cache-producer provenance, while the -JSON encoder must keep opt-in file selection separate from chunk semantics. - -Once implemented, the durable schema belongs in `docs/integrations/`; -configuration owns the selectable option; pipeline and state internals own the -framework handoff; and a maintained scene-chunking example should demonstrate -that namespaced annotations survive export. Tests should protect those -observable contracts without relying on exact payload lengths or private -helper structure. - -## Non-Goals - -This scope does not: - -- add a chunk-map extraction lane or artifact-lane registration; -- rerun chunking or ask an LLM to reconstruct accepted chunks; -- export transcript content, source units, source-unit metadata, or raw model - proposals; -- interpret D&D scene annotations as a generic contract; -- add scene descriptions, titles, summaries, kinds, or participant extraction; -- make chunk maps generated references or inputs to later ordered steps; -- change chunk planning, validation, cache selection, or publication behavior; -- add a chunk-map import or replay mechanism; -- enable export by default; or -- introduce a general output-plugin capability negotiation system. diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md deleted file mode 100644 index 5f1c641..0000000 --- a/docs/roadmap/implementation.md +++ /dev/null @@ -1,463 +0,0 @@ -# Accepted Chunk Map Export Implementation Plan - -Status: Completed - -## Objective - -Implement the accepted [Accepted Chunk Map Export](accepted-chunk-map-export.md) -roadmap as an opt-in logical file in the production JSON output bundle. The -finished feature must serialize the exact accepted materialized chunks used by -the run, preserve framework ownership of chunk semantics, and keep file -selection and placement in the output encoder. - -This document is the implementation authority for sequencing and concrete -engineering decisions. The feature roadmap remains authoritative for product -intent, durable field semantics, security policy, and non-goals. Implement the -stages in order and do not release or merge a partially completed sequence. - -## Fixed Decisions - -Use these durable identities: - -| Concern | Identity | -| --- | --- | -| Artifact kind | `source/chunk-map` | -| Logical file | `chunk-map.json` | -| Schema ID | `notarius.source.chunk_map` | -| Schema name | `notarius_source_chunk_map_v1` | -| Schema version | `v1` | -| Media type | `application/json` | -| JSON output option | `include_chunk_map` | - -Keep the schema at `v1`; Notarius is pre-release and no compatibility layer, -dual writer, or migration path is required. - -The framework package owns the durable DTO, embedded JSON Schema, construction -from accepted source state, invariant validation, cloning, and canonical JSON -serialization. The runner owns construction and passes a serialized artifact -through the generic output request. The JSON output encoder owns the opt-in -decision, logical filename, formatting, and `index.json` descriptor. - -Do not register the chunk map with the artifact-lane codec registry. That -registry describes lane artifacts; the chunk map is a pipeline-wide source -artifact with no extractor, merger, or normalizer. - -The runner constructs the serialized value whenever an accepted chunk map -exists, independent of the selected output encoder and its options. Output -encoders may ignore the optional value. This keeps output-specific -configuration out of the runner and leaves future encoders free to define -their own explicit behavior. - -Use canonical `sha256:` digests throughout. Annotation values must be -semantically canonicalized before plan-digest verification; decoding must -accept insignificant JSON whitespace rather than require annotation -`json.RawMessage` bytes to already use one formatting style. - -## Stage 1: Add The Framework-Owned Durable Contract - -### Goal - -Create a source-domain chunk-map contract that can be built, validated, -encoded, and decoded without depending on an output module or D&D package. - -### Work - -1. Add `internal/framework/chunkmap/` with: - - - `model.go` for the durable DTO and identity constants; - - `codec.go` for construction, validation, canonical encoding, and strict - decoding; - - `assets/schemas/source_chunk_map.v1.json` as the embedded external schema; - - `testdata/source_chunk_map.v1.json` as one valid contract fixture; and - - focused package tests. - -2. Define constants for the fixed artifact kind, schema identity, schema - version, and media type. Define DTOs matching the feature roadmap exactly: - - - `ChunkMap` with `source_id`, `source_digest`, `plan_digest`, - `requested_chunker`, `producer`, `plan_annotations`, and `chunks`; - - `Producer` with `input_module`, `chunk_module`, and optional - `llm_profile`; and - - `Chunk` with `id`, `index`, `source_ref`, `unit_count`, and - `annotations`. - - Do not expose source units, chunk content, source metadata, producer - references or metadata, warnings, timestamps, cache actions, or model - responses. - -3. Provide a small package API: - - ```go - type BuildRequest struct { - Source *source.SourceDocument - Plan source.ChunkPlan - Chunks []source.Chunk - RequestedChunker string - Producer Producer - } - - func Build(BuildRequest) (ChunkMap, error) - func Serialize(BuildRequest) (contracts.SerializedArtifact, error) - func New() *Codec - ``` - - `Codec` must expose the same identity and encode/decode operations used by - other durable codecs, but it is invoked directly by this package and the - JSON encoder rather than added to `pipeline.ArtifactCodecRegistry`. - -4. Make `Build` the trusted boundary between runtime source structures and the - external contract: - - - validate the source document and accepted plan with existing source - validators; - - calculate the source and plan digests with `source.DigestDocument` and - `source.DigestChunkPlan`, and require the calculated source digest to - equal both `Source.Digest` and `Plan.SourceDigest`; - - require a non-empty requested chunker and producer module identities; - - verify the supplied materialized chunks correspond exactly, in order, to - the accepted plan and source document, including IDs, indexes, source - ranges, unit membership/counts, plan annotations, and range annotations; - - validate range order against source-document position rather than numeric - unit-ID assumptions; - - deep-clone all references and annotation values; and - - encode absent annotation namespaces as non-nil empty maps so JSON contains - `{}`. - -5. Make codec decoding strict at fixed object levels: - - - accept one JSON value only and reject trailing content; - - reject unknown fields; - - validate trimmed, non-empty identities; - - require canonical lower-case `sha256:` digests; - - require a non-empty chunk array; - - require unique stable chunk IDs and contiguous zero-based indexes matching - array order; - - require every source reference to use the top-level source identity and - positive range endpoints; - - require positive unit counts; - - validate non-empty canonical annotation namespaces and valid JSON values; - - canonicalize annotation JSON before using the existing plan-digest helper - to reconstruct and verify `plan_digest`; and - - return defensively owned DTO data. - - Standalone decoding cannot prove range order against a source document it - does not contain. Construction owns that stronger invariant; decoding owns - the structural, identity, and digest invariants available from the durable - payload. - -6. Make serialization return a `contracts.SerializedArtifact` with the fixed - kind, schema, and media type, canonical content, and no duplicated metadata. - Validate the embedded schema defensively so the checked-in schema and Go - contract cannot silently diverge. - -7. Define a strict JSON Schema with: - - - `additionalProperties: false` at every fixed object level; - - all roadmap-required fields; - - `minItems: 1` for chunks; - - positive integer range endpoints and unit counts; - - zero-or-greater indexes; - - the pattern `^sha256:[0-9a-f]{64}$` for digests; - - non-empty strings for identities; - - optional, non-empty `llm_profile`; and - - annotation objects whose namespace values may be arbitrary JSON. - -### Tests - -Add focused behavioral tests proving: - -- the valid fixture round-trips through the codec; -- building from a real accepted source, plan, and materialized chunk list - produces the fixed envelope and exact durable fields; -- output is deterministic and annotation JSON formatting does not alter the - reconstructed plan digest; -- unknown fields, trailing JSON, malformed identities/digests, invalid - indexes, duplicate IDs, mismatched source identities, invalid ranges or - counts, malformed annotation namespaces/values, and plan-digest mismatches - are rejected; -- construction rejects materialized chunks that differ from the accepted plan - or document order; and -- mutating caller-owned inputs or returned values cannot mutate another owned - representation. - -Keep these tests semantic. Do not add exact byte-length, exact schema-length, -private-helper, or redundant field-by-field “change detector” tests. - -### Completion Criteria - -- The framework contract can represent and validate the roadmap payload - without importing output or D&D packages. -- Construction proves the artifact describes the exact accepted materialized - chunks. -- No production output behavior has changed yet. - -## Stage 2: Carry The Accepted Artifact Through The Runner - -### Goal - -Construct the durable artifact at the accepted chunk-plan boundary and make it -available to output encoders without changing default logical files. - -### Work - -1. Extend `contracts.OutputRequest` with: - - ```go - ChunkMap *contracts.SerializedArtifact `json:"chunk_map,omitempty"` - ``` - - Update its cloning/ownership path so the envelope, content bytes, and - metadata are defensively copied. Do not place chunk-map content in the run - manifest or generic request metadata. - -2. In `internal/framework/pipeline/runner.go`, construct the serialized chunk - map after materialization and configured chunk validation accept the plan, - and before the output encoder is invoked. Build it from: - - - the accepted source document; - - the accepted logical plan; - - the exact materialized chunks passed to lanes; - - the chunk module selected by the current resolved pipeline as - `requested_chunker`; and - - the original producer identities recorded by the accepted chunk-plan - record. - -3. Preserve cache semantics: - - - a generated or bypassed plan records the current producer; - - a cache hit keeps the stored producer's input module, chunk module, and - optional LLM profile; - - `requested_chunker` still records the current resolved chunk module; and - - producer references, metadata, warnings, and cache details remain only in - existing provenance surfaces. - -4. Apply the accepted-state policy: - - - pass no chunk map when chunk validation rejects the candidate; - - retain the artifact when a later extraction, merge, or normalization lane - is rejected; - - treat artifact construction or serialization failure after acceptance as - a framework run error; and - - do not rerun chunking or make another LLM request. - -5. If debug serialization records the output request, represent the new value - through the existing safe serialized-artifact envelope conventions. Do not - duplicate its content into the manifest or add a new default debug surface. - -### Tests - -Extend `internal/framework/pipeline/runner_chunk_plan_test.go` and the nearest -existing output-request ownership tests to prove: - -- accepted generated and cached plans reach a capturing output encoder as a - valid serialized chunk map; -- cache reuse distinguishes current `requested_chunker` from the stored - producer chunker; -- chunk rejection yields a nil chunk map; -- a later lane rejection still leaves the accepted chunk map available; -- captured requests do not alias runner, store, or caller-owned bytes/maps. - -Use existing fake modules and stores. Do not add an LLM-backed integration test -for behavior already deterministically owned by the runner, and do not add a -production injection seam solely to force an otherwise unreachable -construction failure; Stage 1 owns invalid-construction coverage. - -### Completion Criteria - -- Every accepted chunk plan yields one validated serialized chunk map in the - output request. -- Rejected chunk plans yield none, while later lane rejection does not discard - it. -- Existing output encoders remain behaviorally unchanged. - -## Stage 3: Add Opt-In JSON Bundle Export - -### Goal - -Expose the available framework artifact as `chunk-map.json` only when the JSON -output binding explicitly enables it. - -### Work - -1. Add an `Options` field: - - ```go - IncludeChunkMap bool - ``` - - Keep `New()` as the default-disabled constructor and add - `NewWithOptions(Options)` for configured construction and focused tests. - -2. Update the JSON output builder to decode only `include_chunk_map`: - - - omit or `false` means disabled; - - require a boolean when present; and - - continue rejecting unknown keys. - -3. When disabled, ignore an available `OutputRequest.ChunkMap` and preserve the - exact existing bundle shape. When enabled: - - - omit both file and descriptor if the request has no accepted chunk map; - - validate the artifact kind, schema identity/version, and media type; - - decode through `chunkmap.New()` so malformed or non-canonical payloads - fail the run; - - write valid content as pretty-printed JSON with the encoder's established - trailing-newline convention; and - - use the existing logical-path validation. - - Reuse or generalize the current serialized-artifact JSON helper rather than - adding a second subtly different validation and formatting path. - -4. Extend the output index model with an optional `chunk_map` descriptor - containing exactly: - - - `artifact_kind`; - - `file`; - - `media_type`; - - `schema_id`; - - `schema_name`; and - - `schema_version`. - - Do not add the chunk map to lane-oriented `output_files`, and do not change - the manifest. - -5. Treat a present but invalid accepted artifact as an output error. Do not - silently omit it when export was explicitly requested. - -### Tests - -Extend `internal/modules/generic/output/json/encoder_test.go` and existing -output registration tests to prove: - -- option omission and `false` retain the current default bundle even when an - artifact is available; -- `true` plus a valid artifact emits `chunk-map.json` and the exact optional - index descriptor; -- `true` plus no accepted artifact emits neither file nor descriptor; -- the chunk map never appears in lane `output_files`; -- wrong option types and unknown options are rejected; -- wrong kind, schema, media type, or invalid content returns an error; -- emitted JSON is valid, formatted consistently, and newline-terminated; and -- encoding does not mutate the request or serialized content. - -Prefer testing the observable logical file map and decoded index rather than -private helper calls or exact byte lengths. - -### Completion Criteria - -- Existing JSON configurations remain unchanged by default. -- An explicitly enabled JSON output produces exactly one validated - `chunk-map.json` file and one pipeline-wide index descriptor when an - accepted map exists. -- No lane identity or manifest structure is fabricated for the chunk map. - -## Stage 4: Prove The Assembled Behavior And Document It - -### Goal - -Demonstrate the feature with the production D&D scene chunker, publish all -current-behavior documentation, and close the roadmap scope. - -### Work - -1. Extend the representative assembled production contract in - `internal/cli/production_contract_test.go`: - - - configure `dnd/scenes` with JSON `include_chunk_map: true`; - - execute through normal production assembly with the existing deterministic - fake LLM; - - read `index.json` and `chunk-map.json`; - - decode the artifact through the framework codec; and - - assert the actual accepted scene ranges, stable IDs/order, source and plan - identity, requested/current producer provenance, and surviving namespaced - scene annotations. - - This is the single assembled proof. Do not duplicate all codec and encoder - edge cases at the CLI layer. - -2. Add a copyable maintained example such as - `examples/dnd-scene-chunk-map.config.yml` that uses `dnd/scenes`, one normal - D&D artifact lane, and the opt-in JSON output option. Add it to - `internal/cli/example_contract_test.go` so schema/config drift is caught by - the existing example contract. - -3. Add `docs/integrations/chunk-map.md` as the canonical durable contract - reference. Document fixed identities, every payload field, invariants, - accepted-state behavior, cache provenance semantics, sensitivity, and - excluded content. Link to the embedded schema without duplicating it in the - prose. - -4. Update current-behavior documentation: - - - `docs/integrations/json-output.md`: optional logical file and index - descriptor; - - `docs/config.md`: strict `include_chunk_map` option and copyable binding; - - `docs/operations.md`: opt-in sensitivity and retention implications; - - `docs/internal/pipeline.md`: accepted-plan construction and output-request - handoff; - - `docs/internal/state.md`: logical output and manifest/index ownership; - - `docs/internal/modules.md`: JSON encoder option and responsibility; and - - `docs/internal/overview.md`: framework chunk-map contract and encoder - boundary. - - Keep implementation details in internal docs, user-selectable behavior in - configuration docs, and the durable schema only in integrations docs. - -5. Close planning state: - - - mark `accepted-chunk-map-export.md` as implemented; - - remove the completed “Export Accepted Chunk Maps” entry from - `docs/roadmap/future.md`; and - - mark this implementation plan completed, preserving it until the user - chooses to retire completed roadmap documents. - -6. Review links, examples, terminology, and file identities across the - documentation. Do not add an ADR: this feature applies the existing fixed - pipeline, output/cache separation, and framework ownership decisions rather - than changing them. - -### Validation - -Run focused tests after each stage. At completion, run: - -```sh -go test ./... -go vet ./... -go build ./cmd/notarius -git diff --check -``` - -Also verify that maintained example tests pass, all new documentation links -resolve, and a default JSON output run still has no chunk-map file or index -descriptor. - -### Completion Criteria - -- The assembled scene-chunking run exports the exact accepted map and - namespaced annotations without a second model call. -- Current documentation and the maintained example describe only implemented - behavior. -- The future roadmap no longer lists the completed feature. -- Repository-wide tests, vet, build, formatting, and documentation checks pass. - -## Cross-Stage Guardrails - -- Do not expose internal `source.Chunk` directly as the durable DTO. -- Do not add transcript content, materialized units, source-unit metadata, - model proposals, or debug data to the artifact. -- Do not create an extractor, lane registration, normalizer, generated - reference, import path, or replay mechanism for chunk maps. -- Do not make D&D scene annotation fields part of the generic schema. -- Do not make export the default or add a CLI-global/filesystem option. -- Do not change chunk selection, validation, cache lookup/publication, or lane - execution semantics. -- Do not duplicate chunk-map payloads in the manifest or lane output indexes. -- Preserve defensive ownership at source, runner, request, codec, and encoder - boundaries. -- Keep tests offline and deterministic, and test each invariant at its stable - owning layer. - -## Open Questions - -None. The feature roadmap and the fixed decisions above resolve the -implementation-significant choices. diff --git a/internal/framework/chunkmap/codec.go b/internal/framework/chunkmap/codec.go index d05a0e6..721a4e1 100644 --- a/internal/framework/chunkmap/codec.go +++ b/internal/framework/chunkmap/codec.go @@ -8,9 +8,11 @@ import ( "io" "regexp" "strings" + "sync" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "github.com/santhosh-tekuri/jsonschema/v6" ) //go:embed assets/schemas/source_chunk_map.v1.json @@ -18,6 +20,13 @@ var schemaAssets embed.FS var digestPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`) +var ( + loadSchemaOnce sync.Once + loadedSchema []byte + compiledSchema *jsonschema.Schema + loadSchemaErr error +) + // Codec owns strict serialization for the durable chunk-map contract. type Codec struct{} @@ -123,7 +132,7 @@ func (c *Codec) Encode(value ChunkMap) ([]byte, error) { if _, err := c.schemaBytes(); err != nil { return nil, err } - canonical, err := canonicalize(value) + canonical, err := canonicalize(clone(value)) if err != nil { return nil, fmt.Errorf("encode source chunk map: %w", err) } @@ -131,6 +140,9 @@ func (c *Codec) Encode(value ChunkMap) ([]byte, error) { if err != nil { return nil, fmt.Errorf("encode source chunk map: %w", err) } + if err := validateSchemaInstance(content); err != nil { + return nil, fmt.Errorf("encode source chunk map: %w", err) + } return content, nil } @@ -138,6 +150,9 @@ func (c *Codec) Decode(content []byte) (ChunkMap, error) { if _, err := c.schemaBytes(); err != nil { return ChunkMap{}, err } + if err := validateSchemaInstance(content); err != nil { + return ChunkMap{}, fmt.Errorf("decode source chunk map: %w", err) + } decoder := json.NewDecoder(bytes.NewReader(content)) decoder.DisallowUnknownFields() var value ChunkMap @@ -156,23 +171,61 @@ func (c *Codec) Decode(content []byte) (ChunkMap, error) { } func (c *Codec) schemaBytes() ([]byte, error) { + loadSchemaOnce.Do(loadAndCompileSchema) + if loadSchemaErr != nil { + return nil, loadSchemaErr + } + return append([]byte(nil), loadedSchema...), nil +} + +func loadAndCompileSchema() { raw, err := schemaAssets.ReadFile("assets/schemas/source_chunk_map.v1.json") if err != nil { - return nil, fmt.Errorf("read source chunk map schema: %w", err) + loadSchemaErr = fmt.Errorf("read source chunk map schema: %w", err) + return } - var schema struct { + var identity struct { ID string `json:"$id"` Title string `json:"title"` Type string `json:"type"` Required []string `json:"required"` } - if err := json.Unmarshal(raw, &schema); err != nil { - return nil, fmt.Errorf("decode source chunk map schema: %w", err) + if err := json.Unmarshal(raw, &identity); err != nil { + loadSchemaErr = fmt.Errorf("decode source chunk map schema: %w", err) + return } - if schema.ID != SchemaID || schema.Title != SchemaName || schema.Type != "object" || !hasRequiredFields(schema.Required) { - return nil, fmt.Errorf("source chunk map schema identity or required fields are invalid") + if identity.ID != SchemaID || identity.Title != SchemaName || identity.Type != "object" || !hasRequiredFields(identity.Required) { + loadSchemaErr = fmt.Errorf("source chunk map schema identity or required fields are invalid") + return } - return append([]byte(nil), raw...), nil + schemaDocument, err := jsonschema.UnmarshalJSON(bytes.NewReader(raw)) + if err != nil { + loadSchemaErr = fmt.Errorf("parse source chunk map schema: %w", err) + return + } + compiler := jsonschema.NewCompiler() + if err := compiler.AddResource("source-chunk-map-schema.json", schemaDocument); err != nil { + loadSchemaErr = fmt.Errorf("load source chunk map schema: %w", err) + return + } + compiled, err := compiler.Compile("source-chunk-map-schema.json") + if err != nil { + loadSchemaErr = fmt.Errorf("compile source chunk map schema: %w", err) + return + } + loadedSchema = append([]byte(nil), raw...) + compiledSchema = compiled +} + +func validateSchemaInstance(content []byte) error { + instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(content)) + if err != nil { + return fmt.Errorf("payload is not valid JSON: %w", err) + } + if err := compiledSchema.Validate(instance); err != nil { + return fmt.Errorf("payload does not conform to source chunk map schema: %w", err) + } + return nil } func hasRequiredFields(required []string) bool { diff --git a/internal/framework/chunkmap/codec_test.go b/internal/framework/chunkmap/codec_test.go index a92a0ea..63f3e3e 100644 --- a/internal/framework/chunkmap/codec_test.go +++ b/internal/framework/chunkmap/codec_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "os" + "reflect" "strings" "testing" @@ -140,6 +141,62 @@ func TestCodecRejectsInvalidDurableBoundaries(t *testing.T) { } } +func TestDecodeEnforcesRequiredSchemaFieldsAndTypes(t *testing.T) { + request := acceptedBuildRequest(t) + request.Plan.Annotations = nil + var err error + request.Chunks, err = source.MaterializeChunkPlan(request.Source, request.Plan) + if err != nil { + t.Fatal(err) + } + artifact, err := Serialize(request) + if err != nil { + t.Fatal(err) + } + + for _, test := range []struct { + name string + mutate func(map[string]any) + }{ + {name: "missing plan annotations", mutate: func(value map[string]any) { delete(value, "plan_annotations") }}, + {name: "null plan annotations", mutate: func(value map[string]any) { value["plan_annotations"] = nil }}, + {name: "missing first index", mutate: func(value map[string]any) { delete(chunkDocument(value, 0), "index") }}, + {name: "null first index", mutate: func(value map[string]any) { chunkDocument(value, 0)["index"] = nil }}, + {name: "missing empty chunk annotations", mutate: func(value map[string]any) { delete(chunkDocument(value, 1), "annotations") }}, + {name: "null empty chunk annotations", mutate: func(value map[string]any) { chunkDocument(value, 1)["annotations"] = nil }}, + {name: "explicit empty llm profile", mutate: func(value map[string]any) { + value["producer"].(map[string]any)["llm_profile"] = "" + }}, + } { + t.Run(test.name, func(t *testing.T) { + value := decodeJSONDocument(t, artifact.Content) + test.mutate(value) + content, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + if _, err := New().Decode(content); err == nil { + t.Fatalf("Decode(%s) error = nil, want schema rejection", content) + } + }) + } +} + +func TestEncodeDoesNotMutateValue(t *testing.T) { + value, err := Build(acceptedBuildRequest(t)) + if err != nil { + t.Fatal(err) + } + value.Chunks[0].Annotations["dnd/scenes"] = json.RawMessage(" { \n \"kind\" : \"narrative\" \n } ") + before := clone(value) + if _, err := New().Encode(value); err != nil { + t.Fatalf("Encode() error = %v", err) + } + if !reflect.DeepEqual(value, before) { + t.Fatalf("Encode() mutated value:\nbefore: %#v\nafter: %#v", before, value) + } +} + func TestChunkMapOwnershipIsIndependent(t *testing.T) { request := acceptedBuildRequest(t) first, err := Build(request) @@ -157,6 +214,21 @@ func TestChunkMapOwnershipIsIndependent(t *testing.T) { } } +func decodeJSONDocument(t *testing.T, content []byte) map[string]any { + t.Helper() + decoder := json.NewDecoder(bytes.NewReader(content)) + decoder.UseNumber() + var value map[string]any + if err := decoder.Decode(&value); err != nil { + t.Fatal(err) + } + return value +} + +func chunkDocument(value map[string]any, index int) map[string]any { + return value["chunks"].([]any)[index].(map[string]any) +} + func acceptedBuildRequest(t *testing.T) BuildRequest { t.Helper() document := &source.SourceDocument{ diff --git a/internal/modules/generic/output/json/encoder.go b/internal/modules/generic/output/json/encoder.go index 1775b73..bfef442 100644 --- a/internal/modules/generic/output/json/encoder.go +++ b/internal/modules/generic/output/json/encoder.go @@ -1,9 +1,11 @@ package json import ( + "bytes" "context" stdjson "encoding/json" "fmt" + "io" "mime" "regexp" "sort" @@ -247,10 +249,16 @@ func serializedOutputFile(name string, artifact contracts.SerializedArtifact) (c if !isJSONMediaType(mediaType) { return contracts.OutputFile{}, encoderErrorf("normalized output %q has unsupported media type %q", name, mediaType) } + decoder := stdjson.NewDecoder(bytes.NewReader(content)) + decoder.UseNumber() var decoded any - if err := stdjson.Unmarshal(content, &decoded); err != nil { + if err := decoder.Decode(&decoded); err != nil { return contracts.OutputFile{}, encoderErrorf("normalized output %q contains invalid JSON: %w", name, err) } + var trailing any + if err := decoder.Decode(&trailing); err != io.EOF { + return contracts.OutputFile{}, encoderErrorf("normalized output %q contains multiple JSON values", name) + } pretty, err := marshalPretty(decoded) if err != nil { return contracts.OutputFile{}, err diff --git a/internal/modules/generic/output/json/encoder_test.go b/internal/modules/generic/output/json/encoder_test.go index 82a79b9..622b759 100644 --- a/internal/modules/generic/output/json/encoder_test.go +++ b/internal/modules/generic/output/json/encoder_test.go @@ -234,6 +234,23 @@ func TestEncodeIncludesValidatedChunkMap(t *testing.T) { } } +func TestEncodePreservesChunkMapAnnotationNumbers(t *testing.T) { + artifact := acceptedChunkMapArtifactWithPlanAnnotation(t, stdjson.RawMessage(`{"decimal":1.0,"large":9007199254740993}`)) + result, err := NewWithOptions(Options{IncludeChunkMap: true}).Encode(context.Background(), contracts.OutputRequest{ + ChunkMap: &artifact, + }) + if err != nil { + t.Fatalf("Encode() error = %v", err) + } + value, err := chunkmap.New().Decode(fileBytes(t, result.Files, chunkMapFileName)) + if err != nil { + t.Fatalf("Decode(emitted chunk map) error = %v", err) + } + if got, want := string(value.PlanAnnotations["test/numbers"]), `{"decimal":1.0,"large":9007199254740993}`; got != want { + t.Fatalf("numeric annotation = %s, want %s", got, want) + } +} + func TestEncodeOmitsChunkMapWithoutAcceptedArtifact(t *testing.T) { result, err := NewWithOptions(Options{IncludeChunkMap: true}).Encode(context.Background(), contracts.OutputRequest{ Manifest: artifacts.RunManifest{RunID: "run-1"}, @@ -544,6 +561,10 @@ func normalizeOutput(laneID string, content string) contracts.SerializedOutput { } func acceptedChunkMapArtifact(t *testing.T) contracts.SerializedArtifact { + return acceptedChunkMapArtifactWithPlanAnnotation(t, nil) +} + +func acceptedChunkMapArtifactWithPlanAnnotation(t *testing.T, annotation stdjson.RawMessage) contracts.SerializedArtifact { t.Helper() document := &source.SourceDocument{ ID: "source-1", @@ -562,6 +583,9 @@ func acceptedChunkMapArtifact(t *testing.T) contracts.SerializedArtifact { } document.Digest = digest plan := source.ChunkPlan{SourceDigest: digest, Ranges: []source.ChunkRange{{StartUnitID: 1, EndUnitID: 1}}} + if annotation != nil { + plan.Annotations = source.ChunkAnnotations{"test/numbers": append(stdjson.RawMessage(nil), annotation...)} + } chunks, err := source.MaterializeChunkPlan(document, plan) if err != nil { t.Fatal(err)