Compare commits
8 Commits
1c13e1d64a
...
96a49bb7cd
| Author | SHA1 | Date | |
|---|---|---|---|
| 96a49bb7cd | |||
| 6d0a19c94c | |||
| 6fc6ce0adb | |||
| 8ba5228c01 | |||
| 51a36efb6b | |||
| ebd449d847 | |||
| 7844c0a93f | |||
| 3bfac14397 |
@@ -1,6 +1,6 @@
|
|||||||
# ADR-0005: Cache one canonical chunk plan per source
|
# ADR-0005: Cache one canonical chunk plan per source
|
||||||
|
|
||||||
**Status:** Proposed
|
**Status:** Accepted
|
||||||
**Date:** 2026-07-17
|
**Date:** 2026-07-17
|
||||||
|
|
||||||
## Context
|
## Context
|
||||||
@@ -70,6 +70,9 @@ Notarius state.
|
|||||||
|
|
||||||
One mutable active plan is stored under the canonical source identity and
|
One mutable active plan is stored under the canonical source identity and
|
||||||
retains provenance for the module and relevant runtime inputs that produced it.
|
retains provenance for the module and relevant runtime inputs that produced it.
|
||||||
|
Refreshing the active plan atomically replaces that one mutable record; readers
|
||||||
|
must observe either the previous complete plan or the replacement complete
|
||||||
|
plan, never a partial update.
|
||||||
The effective plan producer is reported separately from the chunk module
|
The effective plan producer is reported separately from the chunk module
|
||||||
requested by the current pipeline; reuse must not attribute cached boundaries
|
requested by the current pipeline; reuse must not attribute cached boundaries
|
||||||
or annotations to a module that did not produce them.
|
or annotations to a module that did not produce them.
|
||||||
|
|||||||
26
docs/cli.md
26
docs/cli.md
@@ -9,7 +9,7 @@ For the minimal end-to-end invocation, see the [README](../README.md).
|
|||||||
|
|
||||||
```text
|
```text
|
||||||
notarius help
|
notarius help
|
||||||
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--output-dir path] [--diagnostics-dir path] [--llm-profile id] [--resume] [--session-id id] [--reference selector=path] [--without-reference selector]
|
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--chunk_cache auto|bypass|refresh] [--output-dir path] [--diagnostics-dir path] [--llm-profile id] [--resume] [--session-id id] [--reference selector=path] [--without-reference selector]
|
||||||
notarius config validate [--config path/to/config.yml] [--pipeline pipeline-id] [--only lane-a,lane-b]
|
notarius config validate [--config path/to/config.yml] [--pipeline pipeline-id] [--only lane-a,lane-b]
|
||||||
notarius pipelines list [--config path/to/config.yml] [--json]
|
notarius pipelines list [--config path/to/config.yml] [--json]
|
||||||
```
|
```
|
||||||
@@ -31,6 +31,12 @@ Flags:
|
|||||||
comma-separated and must be non-empty.
|
comma-separated and must be non-empty.
|
||||||
- `--resume`: request checkpoint reuse for this invocation. See
|
- `--resume`: request checkpoint reuse for this invocation. See
|
||||||
[Operations](operations.md#checkpoints) for prerequisites and reuse behavior.
|
[Operations](operations.md#checkpoints) for prerequisites and reuse behavior.
|
||||||
|
- `--chunk_cache auto|bypass|refresh`: select chunk-plan reuse for this
|
||||||
|
invocation. `auto` reuses a valid plan by canonical source digest, `bypass`
|
||||||
|
performs no plan-cache I/O, and `refresh` regenerates and replaces a valid
|
||||||
|
plan only after chunk validation succeeds. See
|
||||||
|
[Configuration](config.md#workspace) for the persistent setting, precedence,
|
||||||
|
and cache-root selection.
|
||||||
- `--output-dir path`: output root. Defaults to `./notarius-output`.
|
- `--output-dir path`: output root. Defaults to `./notarius-output`.
|
||||||
- `--diagnostics-dir path`: diagnostics work directory override for this
|
- `--diagnostics-dir path`: diagnostics work directory override for this
|
||||||
invocation. It does not change the workspace directory.
|
invocation. It does not change the workspace directory.
|
||||||
@@ -125,6 +131,24 @@ go run ./cmd/notarius run dnd-session \
|
|||||||
--resume
|
--resume
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Use `refresh` when intentionally replacing the cached plan for the same source:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/notarius run dnd-session \
|
||||||
|
--config examples/dnd-spells.config.yml \
|
||||||
|
--input examples/seriatim-minimal-transcript.json \
|
||||||
|
--chunk_cache refresh
|
||||||
|
```
|
||||||
|
|
||||||
|
Use `bypass` for a one-off run that must not inspect or create plan-cache state:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/notarius run dnd-session \
|
||||||
|
--config examples/dnd-spells.config.yml \
|
||||||
|
--input examples/seriatim-minimal-transcript.json \
|
||||||
|
--chunk_cache bypass
|
||||||
|
```
|
||||||
|
|
||||||
For checkpoint behavior, durable output, diagnostics, retention, and failure
|
For checkpoint behavior, durable output, diagnostics, retention, and failure
|
||||||
inspection, see [Operations](operations.md).
|
inspection, see [Operations](operations.md).
|
||||||
|
|
||||||
|
|||||||
@@ -48,6 +48,8 @@ Built-in defaults:
|
|||||||
- `workspace.diagnostics.enabled`: `true`
|
- `workspace.diagnostics.enabled`: `true`
|
||||||
- `workspace.resume.enabled`: `false`
|
- `workspace.resume.enabled`: `false`
|
||||||
- `workspace.debug.enabled`: `false`
|
- `workspace.debug.enabled`: `false`
|
||||||
|
- `workspace.chunk_cache.mode`: `auto`
|
||||||
|
- `workspace.chunk_cache.directory`: unset
|
||||||
|
|
||||||
No pipelines are built in. A run requires a configured pipeline.
|
No pipelines are built in. A run requires a configured pipeline.
|
||||||
|
|
||||||
@@ -97,6 +99,8 @@ These environment variables are applied after the config file:
|
|||||||
- `NOTARIUS_WORKSPACE_RESUME_ENABLED`: boolean resume checkpointing
|
- `NOTARIUS_WORKSPACE_RESUME_ENABLED`: boolean resume checkpointing
|
||||||
enablement.
|
enablement.
|
||||||
- `NOTARIUS_WORKSPACE_DEBUG_ENABLED`: boolean debug artifact enablement.
|
- `NOTARIUS_WORKSPACE_DEBUG_ENABLED`: boolean debug artifact enablement.
|
||||||
|
- `NOTARIUS_WORKSPACE_CHUNK_CACHE_MODE`: chunk-plan cache mode.
|
||||||
|
- `NOTARIUS_WORKSPACE_CHUNK_CACHE_DIR`: chunk-plan cache root.
|
||||||
- `NOTARIUS_WORK_DIR`: deprecated diagnostics work directory compatibility
|
- `NOTARIUS_WORK_DIR`: deprecated diagnostics work directory compatibility
|
||||||
override.
|
override.
|
||||||
- `NOTARIUS_DIAGNOSTICS_RETENTION`: deprecated diagnostics retention
|
- `NOTARIUS_DIAGNOSTICS_RETENTION`: deprecated diagnostics retention
|
||||||
@@ -344,11 +348,35 @@ casts still must be present in the source transcript.
|
|||||||
- `directory`: optional workspace root for Notarius-owned local state.
|
- `directory`: optional workspace root for Notarius-owned local state.
|
||||||
- `resume.enabled`: boolean resume checkpointing setting.
|
- `resume.enabled`: boolean resume checkpointing setting.
|
||||||
- `debug.enabled`: boolean debug artifact setting.
|
- `debug.enabled`: boolean debug artifact setting.
|
||||||
|
- `chunk_cache.mode`: persistent chunk-plan cache mode: `auto`, `bypass`, or
|
||||||
|
`refresh`. The default is `auto`.
|
||||||
|
- `chunk_cache.directory`: optional chunk-plan cache root. This value is the
|
||||||
|
root itself; Notarius does not append `chunk-plans` to it.
|
||||||
- `diagnostics`: optional diagnostics settings defined below.
|
- `diagnostics`: optional diagnostics settings defined below.
|
||||||
|
|
||||||
`workspace.resume.enabled` and `workspace.debug.enabled` are independent.
|
`workspace.resume.enabled`, `workspace.debug.enabled`, and
|
||||||
Enabling one does not enable the other. For directory layout, state lifecycle,
|
`workspace.chunk_cache` are independent. `workspace.directory` does not affect
|
||||||
permissions, and sensitive content, see [Operations](operations.md).
|
chunk-plan placement. For directory layout, state lifecycle, permissions, and
|
||||||
|
sensitive content, see [Operations](operations.md).
|
||||||
|
|
||||||
|
`chunk_cache.mode` accepts only `auto`, `bypass`, and `refresh`. In `auto`, a
|
||||||
|
valid source-addressed plan is reused and a missing or invalid record is
|
||||||
|
regenerated and published after chunk validation. `bypass` neither reads nor
|
||||||
|
writes plan-cache state. `refresh` always generates a new plan and publishes it
|
||||||
|
only after validation succeeds.
|
||||||
|
|
||||||
|
Configuration values are applied in file then environment order; an explicit
|
||||||
|
`--chunk_cache` CLI value has highest precedence for the mode. The cache root
|
||||||
|
is selected from environment, file, then the per-user default; there is no CLI
|
||||||
|
root override. Every supplied value is parsed strictly even when a higher
|
||||||
|
precedence value wins, so malformed configuration is still an error.
|
||||||
|
|
||||||
|
When `chunk_cache.directory` is unset, the root is
|
||||||
|
`<os.UserCacheDir>/notarius/chunk-plans`. On Unix this is ordinarily
|
||||||
|
`$XDG_CACHE_HOME/notarius/chunk-plans` when `XDG_CACHE_HOME` is an absolute
|
||||||
|
path, or `$HOME/.cache/notarius/chunk-plans` when it is unset. A relative
|
||||||
|
`XDG_CACHE_HOME` is rejected by `os.UserCacheDir`; Notarius reports that as a
|
||||||
|
configuration error and does not fall back to another directory.
|
||||||
|
|
||||||
## Diagnostics
|
## Diagnostics
|
||||||
|
|
||||||
@@ -390,6 +418,7 @@ Configuration validation checks:
|
|||||||
- supported stage-worker keys and an effective extract worker count in the
|
- supported stage-worker keys and an effective extract worker count in the
|
||||||
inclusive range `1..concurrency.total_llm`;
|
inclusive range `1..concurrency.total_llm`;
|
||||||
- supported diagnostics retention and non-empty work directory;
|
- supported diagnostics retention and non-empty work directory;
|
||||||
|
- a supported chunk-cache mode and a chunk-cache directory without NUL bytes;
|
||||||
- stale removed fields such as `llm_profiles`.
|
- stale removed fields such as `llm_profiles`.
|
||||||
|
|
||||||
Pipeline resolution additionally checks:
|
Pipeline resolution additionally checks:
|
||||||
|
|||||||
@@ -91,6 +91,16 @@ The manifest fields are:
|
|||||||
identity;
|
identity;
|
||||||
- `input_module`, `chunker`, `extractors`, `merger`, `normalizer`, and
|
- `input_module`, `chunker`, `extractors`, `merger`, `normalizer`, and
|
||||||
`output_encoder`: resolved module keys;
|
`output_encoder`: resolved module keys;
|
||||||
|
- `chunk_plan`: payload-free provenance for the effective chunk plan. `mode`
|
||||||
|
is the effective cache mode; `action` is `reused`, `generated`,
|
||||||
|
`refreshed`, or `bypassed` when a plan was materialized. `requested_module`
|
||||||
|
is the current pipeline chunker, while `producer_input_module`,
|
||||||
|
`producer_module`, `producer_llm_profile`, `producer_references`,
|
||||||
|
`producer_metadata`, `source_digest`, `plan_digest`, `plan_schema_version`,
|
||||||
|
and `created_at` describe the stored or generated producer when available.
|
||||||
|
A cached plan can therefore identify a producer different from the requested
|
||||||
|
module. This object never embeds ranges, units, annotations, prompts,
|
||||||
|
responses, or reference content;
|
||||||
- `module_metadata` and `artifact_lanes`: module and per-lane provenance,
|
- `module_metadata` and `artifact_lanes`: module and per-lane provenance,
|
||||||
including prompt and response-schema provenance when provided;
|
including prompt and response-schema provenance when provided;
|
||||||
- `validator_chains`: resolved validation points and validators;
|
- `validator_chains`: resolved validation points and validators;
|
||||||
@@ -115,6 +125,10 @@ references.
|
|||||||
`validation_status` is `approved` when no outputs were rejected and `rejected`
|
`validation_status` is `approved` when no outputs were rejected and `rejected`
|
||||||
when one or more outputs were rejected.
|
when one or more outputs were rejected.
|
||||||
|
|
||||||
|
Producer warnings and the current run's chunk-validation warnings remain in
|
||||||
|
`warnings.json`. The manifest records only provenance and decision summaries;
|
||||||
|
empty producer-only values are omitted for compatibility with existing readers.
|
||||||
|
|
||||||
`validator_chains` records the resolved validator chain for each validation
|
`validator_chains` records the resolved validator chain for each validation
|
||||||
point. Entries include stage, lane ID when applicable, module key, and validators
|
point. Entries include stage, lane ID when applicable, module key, and validators
|
||||||
with key and execution class. Empty chains are recorded with an empty
|
with key and execution class. Empty chains are recorded with an empty
|
||||||
|
|||||||
@@ -22,10 +22,18 @@ constructor.
|
|||||||
|
|
||||||
Typed methods on `RunDirectory` write invocation metadata, redacted effective
|
Typed methods on `RunDirectory` write invocation metadata, redacted effective
|
||||||
configuration, resolved pipeline/reference data, checkpoint events, source data
|
configuration, resolved pipeline/reference data, checkpoint events, source data
|
||||||
when explicitly requested, manifests, reports, warnings, and error text. The
|
when explicitly requested, manifests, reports, warnings, redacted chunk-plan
|
||||||
|
summaries, and error text. The
|
||||||
current filenames and their operator-facing contents are listed in
|
current filenames and their operator-facing contents are listed in
|
||||||
[Operations](../operations.md#diagnostics-directory).
|
[Operations](../operations.md#diagnostics-directory).
|
||||||
|
|
||||||
|
The chunk-plan summary records the effective mode, source and candidate
|
||||||
|
digests, requested module, lookup decision, materialization action, validation
|
||||||
|
decision, and publication decision. Its closed decision values make failures
|
||||||
|
and recoverable invalid records inspectable without serializing plan ranges,
|
||||||
|
annotations, source content, reference content, prompts, model responses, or
|
||||||
|
raw invalid-file bytes.
|
||||||
|
|
||||||
JSON methods indent their payload and append a newline. All artifact writes use
|
JSON methods indent their payload and append a newline. All artifact writes use
|
||||||
a temporary file in the target directory, apply the requested permissions, and
|
a temporary file in the target directory, apply the requested permissions, and
|
||||||
rename it into place. Artifact resolution accepts only a single relative base
|
rename it into place. Artifact resolution accepts only a single relative base
|
||||||
@@ -60,7 +68,8 @@ pipeline results, and the final report. This ordering permits later failures to
|
|||||||
retain the context already established.
|
retain the context already established.
|
||||||
|
|
||||||
Failures before construction have no `RunDirectory`. Later failures write an
|
Failures before construction have no `RunDirectory`. Later failures write an
|
||||||
error log, preserve any available partial manifest, and apply a failed-run
|
error log, preserve any available partial manifest and chunk-plan summary, and
|
||||||
|
apply a failed-run
|
||||||
retention decision. A diagnostics write failure is itself a command failure so
|
retention decision. A diagnostics write failure is itself a command failure so
|
||||||
the CLI does not report success after losing requested inspection data.
|
the CLI does not report success after losing requested inspection data.
|
||||||
|
|
||||||
|
|||||||
@@ -67,34 +67,41 @@ rules are defined in the
|
|||||||
|
|
||||||
## Chunkers
|
## Chunkers
|
||||||
|
|
||||||
|
Chunkers implement `contracts.Chunker.Plan`. A plan identifies ordered source
|
||||||
|
unit ranges and may carry optional namespaced JSON annotations; it does not
|
||||||
|
contain materialized chunk content. The framework canonicalizes annotations,
|
||||||
|
validates ranges against the current source, and materializes chunk IDs,
|
||||||
|
indexes, references, content, units, and generic metadata. Annotation
|
||||||
|
namespaces remain optional data: generic framework code and downstream modules
|
||||||
|
must not require D&D scene annotations or import `dnd/scenes`.
|
||||||
|
|
||||||
### `internal/modules/generic/chunk/units`
|
### `internal/modules/generic/chunk/units`
|
||||||
|
|
||||||
The generic chunker validates the source document, walks units in configured
|
The generic chunker validates the source document and returns ranges over units
|
||||||
windows, clones each selected unit, and emits deterministic ordered chunk IDs.
|
in configured windows. Overlap changes the next window start but never reorders
|
||||||
Overlap changes the next window start but never reorders units. It records the
|
units. Framework materialization derives the resulting chunk identity and
|
||||||
first and last unit and unit count in chunk metadata, and derives the chunk's
|
generic metadata from those ranges.
|
||||||
canonical source reference from those unit references.
|
|
||||||
|
|
||||||
The accepted options and defaults are defined in
|
The accepted options and defaults are defined in
|
||||||
[Configuration](../config.md#implemented-production-modules). Generic
|
[Configuration](../config.md#implemented-production-modules). Generic
|
||||||
framework validation canonicalizes the returned unit slices before extraction.
|
framework validation canonicalizes the returned unit slices before extraction.
|
||||||
The chunker decodes its options during construction and retains only the typed
|
The chunker decodes its options during construction and retains only the typed
|
||||||
window settings used by `Chunk`.
|
window settings used by `Plan`.
|
||||||
|
|
||||||
### `internal/modules/dnd/chunk/scenes`
|
### `internal/modules/dnd/chunk/scenes`
|
||||||
|
|
||||||
The scene chunker prepares a structured Scriptorium request from the full
|
The scene chunker prepares a structured Scriptorium request from the full
|
||||||
transcript, session, and optional D&D reference inputs. It validates the model's
|
transcript, session, and optional D&D reference inputs. It validates the model's
|
||||||
scene boundaries against source-unit IDs and converts them into deterministic
|
scene boundaries against source-unit IDs and converts them into deterministic
|
||||||
chunks with canonical source references spanning each scene's units.
|
plan ranges with optional scene annotations. Preparation injects the shared
|
||||||
Preparation injects the shared structured LLM client into the chunker; `Chunk`
|
structured LLM client into the chunker; `Plan`
|
||||||
supplies only the run-specific profile, session, source, references, and
|
supplies only the run-specific profile, session, source, references, and
|
||||||
metadata.
|
metadata.
|
||||||
|
|
||||||
Scene validation requires sequential, contiguous, non-overlapping coverage from
|
Scene validation requires sequential, contiguous, non-overlapping coverage from
|
||||||
the first source unit through the last. Each chunk contains JSON scene content
|
the first source unit through the last. Scene descriptions, boundaries,
|
||||||
and module-owned metadata for the scene description, boundaries, confidence,
|
confidence, and participants are module-owned annotations. Boundary caveats
|
||||||
participants, and unit count. Boundary caveats become warnings. Malformed
|
become warnings. Malformed
|
||||||
structured output is returned as an error; there is no fallback chunker.
|
structured output is returned as an error; there is no fallback chunker.
|
||||||
|
|
||||||
The package embeds its prompt and response schema and reports their non-secret
|
The package embeds its prompt and response schema and reports their non-secret
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ normalize continuations that may overlap across lanes.
|
|||||||
| `internal/core/config` | Defaults, YAML parsing, environment overrides, validation, redaction, and effective pipeline resolution. |
|
| `internal/core/config` | Defaults, YAML parsing, environment overrides, validation, redaction, and effective pipeline resolution. |
|
||||||
| `internal/core/diagnostics` | Scoped run directories, diagnostics writers, atomic writes, and retention decisions. |
|
| `internal/core/diagnostics` | Scoped run directories, diagnostics writers, atomic writes, and retention decisions. |
|
||||||
| `internal/core/source` | Generic source documents, units, chunks, canonical references, lookup, validation, and deterministic source digests. |
|
| `internal/core/source` | Generic source documents, units, chunks, canonical references, lookup, validation, and deterministic source digests. |
|
||||||
| `internal/core/workspace` | Effective workspace settings, confined paths and writes, checkpoint identity, and checkpoint manifest models. |
|
| `internal/core/workspace` | Effective workspace settings, confined paths and writes, and checkpoint identity and manifest models. |
|
||||||
|
|
||||||
## Framework Packages
|
## Framework Packages
|
||||||
|
|
||||||
@@ -47,6 +47,7 @@ normalize continuations that may overlap across lanes.
|
|||||||
| `internal/framework/llm` | Scriptorium-backed structured completions, prompt/schema registration, scheduling, profile recording, and secret redaction. |
|
| `internal/framework/llm` | Scriptorium-backed structured completions, prompt/schema registration, scheduling, profile recording, and secret redaction. |
|
||||||
| `internal/framework/promptfs` | Builds module prompt filesystems from module-owned and caller-provided shared prompt assets. |
|
| `internal/framework/promptfs` | Builds module prompt filesystems from module-owned and caller-provided shared prompt assets. |
|
||||||
| `internal/framework/checkpoint` | Workspace-backed checkpoint loading, recording, and payload serialization. |
|
| `internal/framework/checkpoint` | Workspace-backed checkpoint loading, recording, and payload serialization. |
|
||||||
|
| `internal/framework/chunkplan` | Source-addressed chunk-plan filesystem storage, envelope validation, and atomic publication. |
|
||||||
| `internal/framework/debug` | Workspace-backed framework and LLM debug recording. |
|
| `internal/framework/debug` | Workspace-backed framework and LLM debug recording. |
|
||||||
|
|
||||||
Framework contracts provide typed artifact, provenance-wrapper, chunk-validator,
|
Framework contracts provide typed artifact, provenance-wrapper, chunk-validator,
|
||||||
@@ -122,7 +123,8 @@ Implementation details for all production extensions are in
|
|||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| Durable output | Output module, pipeline runner, and CLI writer | Return logical consumer files and place them for a run. |
|
| Durable output | Output module, pipeline runner, and CLI writer | Return logical consumer files and place them for a run. |
|
||||||
| Diagnostics | `internal/core/diagnostics` and `internal/cli` | Record redacted invocation, resolution, result, and failure inspection data. |
|
| Diagnostics | `internal/core/diagnostics` and `internal/cli` | Record redacted invocation, resolution, result, and failure inspection data. |
|
||||||
| Checkpoints | `internal/framework/checkpoint` and `internal/core/workspace` | Validate and serialize reusable stage outcomes. |
|
| Checkpoints | `internal/framework/checkpoint` and `internal/core/workspace` | Validate and serialize reusable extract, merge, and normalize outcomes. |
|
||||||
|
| Chunk-plan cache | `internal/framework/chunkplan` and `internal/cli` | Persist and select source-addressed plans before framework materialization. |
|
||||||
| Debug artifacts | `internal/framework/debug` and pipeline instrumentation | Capture sensitive framework-boundary and LLM-call material. |
|
| Debug artifacts | `internal/framework/debug` and pipeline instrumentation | Capture sensitive framework-boundary and LLM-call material. |
|
||||||
|
|
||||||
Physical layout, retention, recovery, and sensitive-data handling are defined
|
Physical layout, retention, recovery, and sensitive-data handling are defined
|
||||||
|
|||||||
@@ -8,7 +8,8 @@ defaults, and selectable keys are defined in
|
|||||||
|
|
||||||
Resolution fixes the selected lanes and all stage bindings; preparation
|
Resolution fixes the selected lanes and all stage bindings; preparation
|
||||||
constructs every selected implementation before the runner begins source work.
|
constructs every selected implementation before the runner begins source work.
|
||||||
After serial input parsing and chunking, the runner dispatches extract work to
|
After serial input parsing and plan selection or generation, the runner
|
||||||
|
materializes chunks and dispatches extract work to
|
||||||
one bounded run-wide worker pool in chunk-first, lane-second order. Each lane's
|
one bounded run-wide worker pool in chunk-first, lane-second order. Each lane's
|
||||||
merge and normalize operations remain serial and may overlap other lanes once
|
merge and normalize operations remain serial and may overlap other lanes once
|
||||||
all extracts for that lane are terminal.
|
all extracts for that lane are terminal.
|
||||||
@@ -118,7 +119,8 @@ validator context as applicable. It never invokes an operation method.
|
|||||||
`PreparedPipeline` keeps private constructed executors and exposes cloned
|
`PreparedPipeline` keeps private constructed executors and exposes cloned
|
||||||
resolved input, chunk, lane, and output identities. `pipeline.RunInput` carries
|
resolved input, chunk, lane, and output identities. `pipeline.RunInput` carries
|
||||||
that prepared pipeline, raw source input, run identity and timing, optional
|
that prepared pipeline, raw source input, run identity and timing, optional
|
||||||
session and profile metadata, and checkpoint/debug collaborators. The runner
|
session and profile metadata, a chunk-plan store and mode, and checkpoint/debug
|
||||||
|
collaborators. The runner
|
||||||
parses source bytes through the already constructed input adapter. Later stage
|
parses source bytes through the already constructed input adapter. Later stage
|
||||||
requests receive the generic source model; extract requests receive
|
requests receive the generic source model; extract requests receive
|
||||||
chunk-scoped input material, while chunk, merge, and normalize requests retain
|
chunk-scoped input material, while chunk, merge, and normalize requests retain
|
||||||
@@ -151,8 +153,9 @@ The runner:
|
|||||||
1. validates its prepared input;
|
1. validates its prepared input;
|
||||||
2. parses the raw input with the prepared adapter and validates the generic
|
2. parses the raw input with the prepared adapter and validates the generic
|
||||||
source document;
|
source document;
|
||||||
3. obtains or executes the chunk result;
|
3. selects a stored plan or executes the configured chunker's `Plan` operation;
|
||||||
4. validates and canonicalizes chunks;
|
4. canonicalizes and materializes the plan, then validates the resulting
|
||||||
|
chunks;
|
||||||
5. dispatches extract jobs in source-chunk then resolved-lane order, starting a
|
5. dispatches extract jobs in source-chunk then resolved-lane order, starting a
|
||||||
bounded lane continuation when all extracts for that lane are terminal;
|
bounded lane continuation when all extracts for that lane are terminal;
|
||||||
6. invokes the prepared output encoder and validates its logical file results;
|
6. invokes the prepared output encoder and validates its logical file results;
|
||||||
@@ -173,6 +176,31 @@ and validators while performing these transitions:
|
|||||||
Module-provided warnings and payload warnings are promoted only from attempts
|
Module-provided warnings and payload warnings are promoted only from attempts
|
||||||
whose results are accepted and used.
|
whose results are accepted and used.
|
||||||
|
|
||||||
|
## Chunk Plans And Reuse
|
||||||
|
|
||||||
|
`Chunker.Plan` returns a `source.ChunkPlan`: the canonical source digest,
|
||||||
|
ordered unit-ID ranges, and optional plan or range annotations. The framework
|
||||||
|
owns plan canonicalization and materialization. It creates the generic chunks
|
||||||
|
and therefore owns their IDs, indexes, source references, JSON content, units,
|
||||||
|
media type, and generic metadata. Plan and range annotations are independently
|
||||||
|
owned raw JSON and become `Chunk.PlanAnnotations` and `Chunk.Annotations`.
|
||||||
|
|
||||||
|
In `auto`, the runner looks up the source digest before invoking the chunker. A
|
||||||
|
valid hit is materialized and sent through the current run's configured chunk
|
||||||
|
validators; it does not invoke the chunk module, consume its retry budget, or
|
||||||
|
make a chunk-stage LLM call. A missing, invalid, or unmaterializable record
|
||||||
|
generates a candidate. `refresh` generates without lookup; `bypass` generates
|
||||||
|
without cache access. Generated plans are published only after the full chunk
|
||||||
|
validator chain approves them. A validator rejection is a regular rejected
|
||||||
|
pipeline outcome and never replaces a cached plan.
|
||||||
|
|
||||||
|
The store is source-addressed, not pipeline-addressed. Changes to pipeline
|
||||||
|
configuration, requested chunker, options, references, lanes, validators, or
|
||||||
|
LLM profile do not prevent a source-digest hit. The manifest records both the
|
||||||
|
currently requested chunker and the effective plan producer. Cache state and
|
||||||
|
paths are configured and operated outside the runner; see
|
||||||
|
[Configuration](../config.md#workspace) and [Operations](../operations.md).
|
||||||
|
|
||||||
The extract job channel has the same capacity as the effective extract worker
|
The extract job channel has the same capacity as the effective extract worker
|
||||||
count, so dispatch applies backpressure. A fixed continuation executor prevents
|
count, so dispatch applies backpressure. A fixed continuation executor prevents
|
||||||
ready or checkpoint-reused lanes from creating one goroutine each. Workers and
|
ready or checkpoint-reused lanes from creating one goroutine each. Workers and
|
||||||
@@ -180,19 +208,19 @@ continuations publish lane-local results; the coordinator is the only writer of
|
|||||||
aggregate output and merges those results in resolved lane and source-chunk
|
aggregate output and merges those results in resolved lane and source-chunk
|
||||||
order.
|
order.
|
||||||
|
|
||||||
## Chunk Canonicalization
|
## Plan Canonicalization And Chunk Materialization
|
||||||
|
|
||||||
Before lane execution, generic validation requires unique chunk IDs, matching
|
Plan canonicalization requires canonical JSON annotations, a matching source
|
||||||
source identity, indexes matching returned order, a valid canonical reference,
|
digest, at least one range, existing ordered boundaries, and increasing range
|
||||||
non-empty content and media type, and at least one valid source unit per chunk.
|
starts. Ranges may overlap or leave gaps; a chunker may impose stricter policy.
|
||||||
Units may not repeat inside a chunk and must form a contiguous range in
|
Materialization deterministically reconstructs each range from the current
|
||||||
source-document order. The chunk reference must exactly match the source and
|
source document and copies annotations without interpreting their namespaces.
|
||||||
the first and last unit references.
|
|
||||||
|
|
||||||
The runner then rebuilds each chunk's unit slice from the source document by
|
Before lane execution, generic chunk validation checks the materialized chunks'
|
||||||
unit ID. It preserves the canonical reference, content, media type, and cloned
|
identities, order, source references, content, media type, units, and metadata.
|
||||||
metadata. The framework permits gaps and overlap between separate
|
No chunk checkpoint participates in plan selection: plan storage is the only
|
||||||
chunks; stricter coverage policy belongs to the chunk implementation.
|
chunk-reuse mechanism. Extract, merge, and normalize checkpoints continue to
|
||||||
|
use materialized chunk digests as their dependencies.
|
||||||
|
|
||||||
## Validation And Retries
|
## Validation And Retries
|
||||||
|
|
||||||
|
|||||||
@@ -57,6 +57,10 @@ Implemented diagnostics artifacts:
|
|||||||
binding source, without reference content.
|
binding source, without reference content.
|
||||||
- `checkpoint-events.json`: checkpoint steps that were reused or executed
|
- `checkpoint-events.json`: checkpoint steps that were reused or executed
|
||||||
during an explicit resume invocation.
|
during an explicit resume invocation.
|
||||||
|
- `chunk-plan.json`: redacted plan-cache lookup, validation, and publication
|
||||||
|
summary. It contains identifiers and decisions, never source units, plan
|
||||||
|
annotations, reference content, prompts, model responses, or invalid-file
|
||||||
|
bytes.
|
||||||
- `run-manifest.json`: the same run manifest written to durable output when it
|
- `run-manifest.json`: the same run manifest written to durable output when it
|
||||||
is available, including top-level module metadata when present.
|
is available, including top-level module metadata when present.
|
||||||
- `warnings.json`: warning list.
|
- `warnings.json`: warning list.
|
||||||
@@ -64,6 +68,54 @@ Implemented diagnostics artifacts:
|
|||||||
- `error.log`: failure message, written after diagnostics directory creation
|
- `error.log`: failure message, written after diagnostics directory creation
|
||||||
when a run fails.
|
when a run fails.
|
||||||
|
|
||||||
|
## Chunk-Plan Cache
|
||||||
|
|
||||||
|
The chunk-plan cache is independent of the workspace and checkpoints. Its
|
||||||
|
configuration and selection precedence are defined in
|
||||||
|
[Configuration](config.md#workspace); the invocation override is documented in
|
||||||
|
the [CLI reference](cli.md#run).
|
||||||
|
|
||||||
|
When no root is configured, a normal Linux user uses
|
||||||
|
`$XDG_CACHE_HOME/notarius/chunk-plans` when `XDG_CACHE_HOME` is a valid absolute
|
||||||
|
path, or `$HOME/.cache/notarius/chunk-plans` when it is unset. A relative
|
||||||
|
`XDG_CACHE_HOME` is a configuration error. A configured
|
||||||
|
`workspace.chunk_cache.directory` is the root itself, not a parent to which
|
||||||
|
Notarius adds a suffix.
|
||||||
|
|
||||||
|
Each source digest has one file:
|
||||||
|
|
||||||
|
```text
|
||||||
|
<chunk-plan-root>/<source-sha256-hex>/plan.json
|
||||||
|
```
|
||||||
|
|
||||||
|
Directories are created with `0700` permissions and plan files with `0600`.
|
||||||
|
`auto` reuses a complete valid plan or regenerates an absent or invalid one;
|
||||||
|
`refresh` deliberately regenerates; `bypass` performs no cache I/O. A stored
|
||||||
|
plan is still validated and materialized against the current source before use,
|
||||||
|
and the current run's chunk validators always run. Invalid state is recoverable:
|
||||||
|
an `auto` run regenerates and atomically replaces it only after validation
|
||||||
|
succeeds. Delete an exact cache root or digest directory only when regeneration
|
||||||
|
cost is acceptable.
|
||||||
|
|
||||||
|
Publication uses atomic replacement. Concurrent readers observe a complete old
|
||||||
|
or new plan, and concurrent writers leave one complete valid winner; there is
|
||||||
|
no history, lock protocol, or rollback facility. Do not share a cache root
|
||||||
|
between mutually untrusted users because plans can contain source-derived
|
||||||
|
structure and annotations.
|
||||||
|
|
||||||
|
For a system-wide Linux deployment under a dedicated service account, configure
|
||||||
|
and provision a separate restrictive root such as:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
workspace:
|
||||||
|
chunk_cache:
|
||||||
|
directory: /var/cache/notarius/chunk-plans
|
||||||
|
```
|
||||||
|
|
||||||
|
`/var/cache/notarius/chunk-plans` is a recommended configured service root, not
|
||||||
|
the unprivileged default. The operator or package installer must create it with
|
||||||
|
restrictive service-account ownership and permissions before use.
|
||||||
|
|
||||||
## Checkpoints
|
## Checkpoints
|
||||||
|
|
||||||
When checkpoint writing is enabled for a configured workspace, runs write
|
When checkpoint writing is enabled for a configured workspace, runs write
|
||||||
@@ -158,8 +210,12 @@ failure before a candidate exists has no candidate payload. If the envelope
|
|||||||
cannot be persisted, the run does not retry that module attempt and reports the
|
cannot be persisted, the run does not retry that module attempt and reports the
|
||||||
debug failure together with any primary attempt error.
|
debug failure together with any primary attempt error.
|
||||||
|
|
||||||
Checkpoint-reused chunk, extract, merge, and normalize work retains the
|
Chunk-plan candidates, materialized chunks, annotations, and chunk-attempt
|
||||||
stage-level input and output artifacts but has no retry-attempt artifacts
|
details appear only in these opt-in debug artifacts. They are intentionally not
|
||||||
|
included in normal manifests or the `chunk-plan.json` diagnostics summary.
|
||||||
|
|
||||||
|
Checkpoint-reused extract, merge, and normalize work retains the stage-level
|
||||||
|
input and output artifacts but has no retry-attempt artifacts
|
||||||
because no module attempt executed. Debug artifacts may contain source
|
because no module attempt executed. Debug artifacts may contain source
|
||||||
material, reference material, prompt inputs, model outputs, and other sensitive
|
material, reference material, prompt inputs, model outputs, and other sensitive
|
||||||
data. Typed artifact
|
data. Typed artifact
|
||||||
@@ -218,6 +274,10 @@ rm -rf /var/lib/notarius/checkpoints/dnd-session/seriatim-abcdef123456/7890abcd1
|
|||||||
rm -rf /var/lib/notarius/debug/run-1234567890
|
rm -rf /var/lib/notarius/debug/run-1234567890
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Chunk-plan cache entries can likewise be removed by exact digest directory or
|
||||||
|
configured root. Removal is recoverable, but the next non-bypass run may need
|
||||||
|
to regenerate plans and repeat any chunk-stage LLM work.
|
||||||
|
|
||||||
Use exact run-directory paths. Avoid broad cleanup commands against parent
|
Use exact run-directory paths. Avoid broad cleanup commands against parent
|
||||||
directories unless they are part of your own operational policy.
|
directories unless they are part of your own operational policy.
|
||||||
|
|
||||||
|
|||||||
@@ -105,6 +105,14 @@ Stage ownership is explicit:
|
|||||||
- normalize modules reconcile merged output;
|
- normalize modules reconcile merged output;
|
||||||
- output modules encode accepted results and run outcomes into logical files.
|
- output modules encode accepted results and run outcomes into logical files.
|
||||||
|
|
||||||
|
Chunk modules produce source-addressed chunk plans rather than materialized
|
||||||
|
chunks. The framework validates and materializes those plans into the generic
|
||||||
|
chunk representation before chunk validation and lane execution. Plan reuse is
|
||||||
|
therefore independent of the configured pipeline, module options, references,
|
||||||
|
lanes, validators, and LLM profile: the canonical source digest selects the
|
||||||
|
plan, while the current run still applies its configured chunk validators to
|
||||||
|
the materialized chunks.
|
||||||
|
|
||||||
The framework owns orchestration and handoff provenance. Modules return logical
|
The framework owns orchestration and handoff provenance. Modules return logical
|
||||||
results and warnings; they do not own CLI reporting, workspace paths, durable
|
results and warnings; they do not own CLI reporting, workspace paths, durable
|
||||||
file placement, checkpoints, or diagnostics.
|
file placement, checkpoints, or diagnostics.
|
||||||
@@ -191,6 +199,12 @@ surfaces with separate ownership:
|
|||||||
- debug artifacts are opt-in inspection data and may contain sensitive source,
|
- debug artifacts are opt-in inspection data and may contain sensitive source,
|
||||||
prompt, reference, and model-output content.
|
prompt, reference, and model-output content.
|
||||||
|
|
||||||
|
Chunk-plan cache state is an additional independent surface. It is keyed only
|
||||||
|
by canonical source digest, is not rooted under `workspace.directory`, and is
|
||||||
|
not a checkpoint or a diagnostic. A cache record is atomically replaced as one
|
||||||
|
complete plan envelope; it has no history, locking, or rollback interface.
|
||||||
|
Invalid records are recoverable cache misses rather than pipeline state.
|
||||||
|
|
||||||
Writes of durable state are atomic where practical. Paths for writes, moves,
|
Writes of durable state are atomic where practical. Paths for writes, moves,
|
||||||
overwrites, and deletion must be narrow and explicit. Cleanup that can lose data
|
overwrites, and deletion must be narrow and explicit. Cleanup that can lose data
|
||||||
is opt-in.
|
is opt-in.
|
||||||
|
|||||||
@@ -1,13 +1,12 @@
|
|||||||
# ADR-0005 Staged Implementation Plan
|
# ADR-0005 Implementation Record
|
||||||
|
|
||||||
This document is the executable implementation plan for the target state in
|
This document records the completed implementation of the target state in
|
||||||
[ADR-0005 Feature Roadmap](adr0005.md), governed by
|
[ADR-0005 Feature Roadmap](adr0005.md), governed by
|
||||||
[ADR-0005](../adr/0005-cache-canonical-chunk-plans-by-source.md). The feature is
|
[ADR-0005](../adr/0005-cache-canonical-chunk-plans-by-source.md).
|
||||||
not implemented.
|
|
||||||
|
|
||||||
The audience is an LLM coding agent. Implement the stages in order. Each stage
|
All stages below are complete. The plan remains as historical target-state
|
||||||
is deliberately scoped to finish with a compiling, tested repository and may be
|
context; current behavior is documented in the canonical references linked from
|
||||||
assigned as one implementation prompt.
|
[Development](../development.md).
|
||||||
|
|
||||||
## Execution Rules
|
## Execution Rules
|
||||||
|
|
||||||
@@ -284,7 +283,7 @@ then fills candidate and producer fields when a plan reaches materialization.
|
|||||||
|
|
||||||
## Stage 1: Finalize ADR and add the source-zone plan model
|
## Stage 1: Finalize ADR and add the source-zone plan model
|
||||||
|
|
||||||
**Status:** Not started
|
**Status:** Complete
|
||||||
|
|
||||||
### Objective
|
### Objective
|
||||||
|
|
||||||
@@ -351,7 +350,7 @@ tested; current modules and the runner still behave as before.
|
|||||||
|
|
||||||
## Stage 2: Replace the chunk operation with plan generation
|
## Stage 2: Replace the chunk operation with plan generation
|
||||||
|
|
||||||
**Status:** Not started
|
**Status:** Complete
|
||||||
|
|
||||||
### Objective
|
### Objective
|
||||||
|
|
||||||
@@ -432,7 +431,7 @@ is green.
|
|||||||
|
|
||||||
## Stage 3: Add dedicated cache configuration and the plan store
|
## Stage 3: Add dedicated cache configuration and the plan store
|
||||||
|
|
||||||
**Status:** Not started
|
**Status:** Complete
|
||||||
|
|
||||||
### Objective
|
### Objective
|
||||||
|
|
||||||
@@ -531,7 +530,7 @@ runs still use Stage 2’s generate-and-materialize behavior.
|
|||||||
|
|
||||||
## Stage 4: Integrate cache policy and remove chunk checkpoints
|
## Stage 4: Integrate cache policy and remove chunk checkpoints
|
||||||
|
|
||||||
**Status:** Not started
|
**Status:** Complete
|
||||||
|
|
||||||
### Objective
|
### Objective
|
||||||
|
|
||||||
@@ -617,7 +616,7 @@ runs are not yet wired to persistent plan storage.
|
|||||||
|
|
||||||
## Stage 5: Wire persistent policy into the CLI
|
## Stage 5: Wire persistent policy into the CLI
|
||||||
|
|
||||||
**Status:** Not started
|
**Status:** Complete
|
||||||
|
|
||||||
### Objective
|
### Objective
|
||||||
|
|
||||||
@@ -709,7 +708,7 @@ refresh have exact state semantics, and the repository is green.
|
|||||||
|
|
||||||
## Stage 6: Expose producer provenance and safe diagnostics
|
## Stage 6: Expose producer provenance and safe diagnostics
|
||||||
|
|
||||||
**Status:** Not started
|
**Status:** Complete
|
||||||
|
|
||||||
### Objective
|
### Objective
|
||||||
|
|
||||||
@@ -798,7 +797,7 @@ compatibility tests pass.
|
|||||||
|
|
||||||
## Stage 7: Add cross-run, corruption, and concurrency hardening
|
## Stage 7: Add cross-run, corruption, and concurrency hardening
|
||||||
|
|
||||||
**Status:** Not started
|
**Status:** Complete
|
||||||
|
|
||||||
### Objective
|
### Objective
|
||||||
|
|
||||||
@@ -870,7 +869,7 @@ and race checks pass.
|
|||||||
|
|
||||||
## Stage 8: Publish current-behavior documentation
|
## Stage 8: Publish current-behavior documentation
|
||||||
|
|
||||||
**Status:** Not started
|
**Status:** Complete
|
||||||
|
|
||||||
### Objective
|
### Objective
|
||||||
|
|
||||||
|
|||||||
@@ -12,6 +12,8 @@ workspace:
|
|||||||
enabled: false
|
enabled: false
|
||||||
debug:
|
debug:
|
||||||
enabled: false
|
enabled: false
|
||||||
|
chunk_cache:
|
||||||
|
directory: /var/cache/notarius/chunk-plans
|
||||||
pipelines:
|
pipelines:
|
||||||
dnd-session:
|
dnd-session:
|
||||||
input: seriatim
|
input: seriatim
|
||||||
|
|||||||
@@ -1,4 +1,7 @@
|
|||||||
version: 2
|
version: 2
|
||||||
|
workspace:
|
||||||
|
chunk_cache:
|
||||||
|
mode: bypass
|
||||||
pipelines:
|
pipelines:
|
||||||
dnd-session:
|
dnd-session:
|
||||||
input: seriatim
|
input: seriatim
|
||||||
|
|||||||
480
internal/cli/chunk_cache_test.go
Normal file
480
internal/cli/chunk_cache_test.go
Normal file
@@ -0,0 +1,480 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
)
|
||||||
|
|
||||||
|
type recordingChunkPlanStore struct {
|
||||||
|
record pipeline.ChunkPlanRecord
|
||||||
|
decision pipeline.ChunkPlanDecision
|
||||||
|
loadErr error
|
||||||
|
saveErr error
|
||||||
|
loads int
|
||||||
|
saves int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *recordingChunkPlanStore) Load(string) (pipeline.ChunkPlanRecord, pipeline.ChunkPlanDecision, error) {
|
||||||
|
s.loads++
|
||||||
|
return s.record, s.decision, s.loadErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *recordingChunkPlanStore) Save(record pipeline.ChunkPlanRecord) error {
|
||||||
|
s.saves++
|
||||||
|
s.record = record
|
||||||
|
return s.saveErr
|
||||||
|
}
|
||||||
|
|
||||||
|
type recordingChunkPlanFactory struct {
|
||||||
|
roots []string
|
||||||
|
store *recordingChunkPlanStore
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *recordingChunkPlanFactory) build(root string) (pipeline.ChunkPlanStore, error) {
|
||||||
|
f.roots = append(f.roots, root)
|
||||||
|
if f.err != nil {
|
||||||
|
return nil, f.err
|
||||||
|
}
|
||||||
|
if f.store == nil {
|
||||||
|
f.store = &recordingChunkPlanStore{decision: pipeline.ChunkPlanDecision{Status: pipeline.ChunkPlanMissing}}
|
||||||
|
}
|
||||||
|
return f.store, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunChunkCachePrecedence(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
fileMode string
|
||||||
|
envMode string
|
||||||
|
flagMode string
|
||||||
|
wantMode pipeline.ChunkCacheMode
|
||||||
|
wantBuild bool
|
||||||
|
}{
|
||||||
|
{name: "file refresh", fileMode: "refresh", wantMode: pipeline.ChunkCacheRefresh, wantBuild: true},
|
||||||
|
{name: "environment over file", fileMode: "bypass", envMode: "refresh", wantMode: pipeline.ChunkCacheRefresh, wantBuild: true},
|
||||||
|
{name: "flag over environment", fileMode: "refresh", envMode: "auto", flagMode: "bypass", wantMode: pipeline.ChunkCacheBypass},
|
||||||
|
{name: "explicit auto", fileMode: "bypass", envMode: "refresh", flagMode: "auto", wantMode: pipeline.ChunkCacheAuto, wantBuild: true},
|
||||||
|
}
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
root := filepath.Join(t.TempDir(), "plans")
|
||||||
|
configPath := writeTestConfig(t, cacheTestConfig(tc.fileMode, root, ""))
|
||||||
|
inputPath := writeFile(t, "input.txt", "input")
|
||||||
|
values := map[string]string{}
|
||||||
|
if tc.envMode != "" {
|
||||||
|
values["NOTARIUS_WORKSPACE_CHUNK_CACHE_MODE"] = tc.envMode
|
||||||
|
}
|
||||||
|
factory := &recordingChunkPlanFactory{}
|
||||||
|
args := []string{"run", "example", "--config", configPath, "--input", inputPath, "--output-dir", t.TempDir()}
|
||||||
|
if tc.flagMode != "" {
|
||||||
|
args = append(args, "--chunk_cache", tc.flagMode)
|
||||||
|
}
|
||||||
|
code, stderr := runCacheCommand(t, args, cacheTestOptions(t, values, factory))
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("code = %d stderr = %q", code, stderr)
|
||||||
|
}
|
||||||
|
if got := len(factory.roots) > 0; got != tc.wantBuild {
|
||||||
|
t.Fatalf("store built = %t roots = %#v, want %t", got, factory.roots, tc.wantBuild)
|
||||||
|
}
|
||||||
|
if !tc.wantBuild {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if tc.wantMode == pipeline.ChunkCacheAuto && (factory.store.loads != 1 || factory.store.saves != 1) {
|
||||||
|
t.Fatalf("auto calls = load %d save %d", factory.store.loads, factory.store.saves)
|
||||||
|
}
|
||||||
|
if tc.wantMode == pipeline.ChunkCacheRefresh && (factory.store.loads != 0 || factory.store.saves != 1) {
|
||||||
|
t.Fatalf("refresh calls = load %d save %d", factory.store.loads, factory.store.saves)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunChunkCacheInvalidValuesHaveEstablishedExitCodes(t *testing.T) {
|
||||||
|
validConfig := writeTestConfig(t, cacheTestConfig("bypass", "", ""))
|
||||||
|
invalidFile := writeTestConfig(t, cacheTestConfig("sometimes", "", ""))
|
||||||
|
inputPath := writeFile(t, "input.txt", "input")
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
config string
|
||||||
|
env map[string]string
|
||||||
|
flag string
|
||||||
|
want int
|
||||||
|
}{
|
||||||
|
{name: "flag", config: validConfig, flag: "sometimes", want: 2},
|
||||||
|
{name: "environment", config: validConfig, env: map[string]string{"NOTARIUS_WORKSPACE_CHUNK_CACHE_MODE": "sometimes"}, want: 1},
|
||||||
|
{name: "file", config: invalidFile, want: 1},
|
||||||
|
{name: "flag does not mask invalid file", config: invalidFile, flag: "bypass", want: 1},
|
||||||
|
{name: "flag does not mask invalid environment", config: validConfig, env: map[string]string{"NOTARIUS_WORKSPACE_CHUNK_CACHE_MODE": "sometimes"}, flag: "bypass", want: 1},
|
||||||
|
}
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
args := []string{"run", "example", "--config", tc.config, "--input", inputPath, "--output-dir", t.TempDir()}
|
||||||
|
if tc.flag != "" {
|
||||||
|
args = append(args, "--chunk_cache", tc.flag)
|
||||||
|
}
|
||||||
|
code, _ := runCacheCommand(t, args, cacheTestOptions(t, tc.env, &recordingChunkPlanFactory{}))
|
||||||
|
if code != tc.want {
|
||||||
|
t.Fatalf("code = %d, want %d", code, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunChunkPlanRootResolution(t *testing.T) {
|
||||||
|
t.Run("explicit file root", func(t *testing.T) {
|
||||||
|
factory := &recordingChunkPlanFactory{}
|
||||||
|
resolverCalls := 0
|
||||||
|
opts := cacheTestOptions(t, nil, factory)
|
||||||
|
opts.UserCacheDir = func() (string, error) { resolverCalls++; return "", errors.New("must not be called") }
|
||||||
|
configPath := writeTestConfig(t, cacheTestConfig("auto", "/var/cache/notarius/chunk-plans", ""))
|
||||||
|
code, stderr := runCacheCommand(t, cacheRunArgs(t, configPath), opts)
|
||||||
|
if code != 0 || stderr != "" || resolverCalls != 0 || len(factory.roots) != 1 || factory.roots[0] != "/var/cache/notarius/chunk-plans" {
|
||||||
|
t.Fatalf("code=%d stderr=%q resolver=%d roots=%#v", code, stderr, resolverCalls, factory.roots)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("environment root", func(t *testing.T) {
|
||||||
|
factory := &recordingChunkPlanFactory{}
|
||||||
|
environmentRoot := filepath.Join(t.TempDir(), "environment-plans")
|
||||||
|
configPath := writeTestConfig(t, cacheTestConfig("auto", filepath.Join(t.TempDir(), "file-plans"), ""))
|
||||||
|
code, stderr := runCacheCommand(t, cacheRunArgs(t, configPath), cacheTestOptions(t, map[string]string{"NOTARIUS_WORKSPACE_CHUNK_CACHE_DIR": environmentRoot}, factory))
|
||||||
|
if code != 0 || stderr != "" || len(factory.roots) != 1 || factory.roots[0] != environmentRoot {
|
||||||
|
t.Fatalf("code=%d stderr=%q roots=%#v", code, stderr, factory.roots)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("per-user default", func(t *testing.T) {
|
||||||
|
factory := &recordingChunkPlanFactory{}
|
||||||
|
cacheDir := filepath.Join(t.TempDir(), "user-cache")
|
||||||
|
opts := cacheTestOptions(t, nil, factory)
|
||||||
|
opts.UserCacheDir = func() (string, error) { return cacheDir, nil }
|
||||||
|
configPath := writeTestConfig(t, cacheTestConfig("auto", "", ""))
|
||||||
|
code, stderr := runCacheCommand(t, cacheRunArgs(t, configPath), opts)
|
||||||
|
want := filepath.Join(cacheDir, "notarius", "chunk-plans")
|
||||||
|
if code != 0 || stderr != "" || len(factory.roots) != 1 || factory.roots[0] != want {
|
||||||
|
t.Fatalf("code=%d stderr=%q roots=%#v want=%q", code, stderr, factory.roots, want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunBypassSkipsRootAndStore(t *testing.T) {
|
||||||
|
factory := &recordingChunkPlanFactory{err: errors.New("must not build")}
|
||||||
|
resolverCalls := 0
|
||||||
|
opts := cacheTestOptions(t, nil, factory)
|
||||||
|
opts.UserCacheDir = func() (string, error) { resolverCalls++; return "", errors.New("must not resolve") }
|
||||||
|
configPath := writeTestConfig(t, cacheTestConfig("bypass", "", ""))
|
||||||
|
code, stderr := runCacheCommand(t, cacheRunArgs(t, configPath), opts)
|
||||||
|
if code != 0 || stderr != "" || resolverCalls != 0 || len(factory.roots) != 0 {
|
||||||
|
t.Fatalf("code=%d stderr=%q resolver=%d roots=%#v", code, stderr, resolverCalls, factory.roots)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunChunkPlanSetupFailures(t *testing.T) {
|
||||||
|
for _, mode := range []string{"auto", "refresh"} {
|
||||||
|
t.Run(mode+" resolver", func(t *testing.T) {
|
||||||
|
factory := &recordingChunkPlanFactory{}
|
||||||
|
opts := cacheTestOptions(t, nil, factory)
|
||||||
|
opts.UserCacheDir = func() (string, error) { return "", errors.New("cache unavailable") }
|
||||||
|
configPath := writeTestConfig(t, cacheTestConfig(mode, "", ""))
|
||||||
|
code, stderr := runCacheCommand(t, cacheRunArgs(t, configPath), opts)
|
||||||
|
if code != 1 || !strings.Contains(stderr, "cache unavailable") || len(factory.roots) != 0 {
|
||||||
|
t.Fatalf("code=%d stderr=%q roots=%#v", code, stderr, factory.roots)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("store", func(t *testing.T) {
|
||||||
|
factory := &recordingChunkPlanFactory{err: errors.New("unwritable store")}
|
||||||
|
configPath := writeTestConfig(t, cacheTestConfig("auto", t.TempDir(), ""))
|
||||||
|
code, stderr := runCacheCommand(t, cacheRunArgs(t, configPath), cacheTestOptions(t, nil, factory))
|
||||||
|
if code != 1 || !strings.Contains(stderr, "unwritable store") {
|
||||||
|
t.Fatalf("code=%d stderr=%q", code, stderr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("unwritable filesystem store", func(t *testing.T) {
|
||||||
|
root := writeFile(t, "not-a-directory", "occupied")
|
||||||
|
configPath := writeTestConfig(t, `version: 2
|
||||||
|
workspace:
|
||||||
|
chunk_cache:
|
||||||
|
mode: auto
|
||||||
|
directory: `+root+`
|
||||||
|
pipelines:
|
||||||
|
dnd-session:
|
||||||
|
input: seriatim
|
||||||
|
artifacts:
|
||||||
|
spells:
|
||||||
|
extract: dnd/spells
|
||||||
|
`)
|
||||||
|
code, stderr := runCacheCommand(t, []string{"run", "dnd-session", "--config", configPath, "--input", writeSeriatimInput(t), "--output-dir", t.TempDir()}, Options{LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil), LookupEnv: mapLookup(nil)})
|
||||||
|
if code != 1 || (!strings.Contains(stderr, "load chunk plan") && !strings.Contains(stderr, "save chunk plan")) {
|
||||||
|
t.Fatalf("code=%d stderr=%q", code, stderr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigCommandsDoNotResolveOrCreateChunkPlanState(t *testing.T) {
|
||||||
|
configPath := writeTestConfig(t, cacheTestConfig("auto", "", ""))
|
||||||
|
for _, args := range [][]string{
|
||||||
|
{"config", "validate", "--config", configPath},
|
||||||
|
{"pipelines", "list", "--config", configPath},
|
||||||
|
} {
|
||||||
|
factory := &recordingChunkPlanFactory{err: errors.New("must not build")}
|
||||||
|
resolverCalls := 0
|
||||||
|
opts := cacheTestOptions(t, nil, factory)
|
||||||
|
opts.UserCacheDir = func() (string, error) { resolverCalls++; return "", errors.New("must not resolve") }
|
||||||
|
code, stderr := runCacheCommand(t, args, opts)
|
||||||
|
if code != 0 || stderr != "" || resolverCalls != 0 || len(factory.roots) != 0 {
|
||||||
|
t.Fatalf("args=%v code=%d stderr=%q resolver=%d roots=%#v", args, code, stderr, resolverCalls, factory.roots)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunRecordsExplicitChunkCacheOverrideAndEffectiveMode(t *testing.T) {
|
||||||
|
diagnosticsDir := t.TempDir()
|
||||||
|
configPath := writeTestConfig(t, cacheTestConfig("bypass", "", diagnosticsDir))
|
||||||
|
factory := &recordingChunkPlanFactory{}
|
||||||
|
args := append(cacheRunArgs(t, configPath), "--chunk_cache", "refresh")
|
||||||
|
code, stderr := runCacheCommand(t, args, cacheTestOptions(t, nil, factory))
|
||||||
|
if code != 0 || stderr != "" {
|
||||||
|
t.Fatalf("code=%d stderr=%q", code, stderr)
|
||||||
|
}
|
||||||
|
runDir := onlyChildDir(t, diagnosticsDir)
|
||||||
|
invocation := string(readFile(t, filepath.Join(runDir, diagnostics.ArtifactInvocationMetadata)))
|
||||||
|
effective := string(readFile(t, filepath.Join(runDir, diagnostics.ArtifactEffectiveConfig)))
|
||||||
|
if !strings.Contains(invocation, `"chunk_cache_override": "refresh"`) || !strings.Contains(effective, `"mode": "refresh"`) {
|
||||||
|
t.Fatalf("invocation=%s effective=%s", invocation, effective)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDefaultAutoReusesPlanAcrossIndependentInvocations(t *testing.T) {
|
||||||
|
cacheBase := filepath.Join(t.TempDir(), "cache")
|
||||||
|
workspaceDir := filepath.Join(t.TempDir(), "workspace")
|
||||||
|
configPath := writeTestConfig(t, `version: 2
|
||||||
|
workspace:
|
||||||
|
directory: `+workspaceDir+`
|
||||||
|
debug:
|
||||||
|
enabled: true
|
||||||
|
pipelines:
|
||||||
|
dnd-session:
|
||||||
|
input: seriatim
|
||||||
|
chunk: dnd/scenes
|
||||||
|
artifacts:
|
||||||
|
spells:
|
||||||
|
extract: dnd/spells
|
||||||
|
`)
|
||||||
|
inputPath := writeFile(t, "source.json", `{
|
||||||
|
"metadata": {"id": "session-alpha"},
|
||||||
|
"segments": [
|
||||||
|
{"id": 1, "start": 0, "end": 1, "speaker": "Aria", "text": "Aria casts Cure Wounds."},
|
||||||
|
{"id": 2, "start": 1, "end": 2, "speaker": "Borin", "text": "Borin recovers."}
|
||||||
|
]
|
||||||
|
}`)
|
||||||
|
client := newFakeRunLLMClient(false)
|
||||||
|
opts := Options{
|
||||||
|
LLMClientFactory: fakeLLMFactory(client, nil),
|
||||||
|
LookupEnv: mapLookup(nil),
|
||||||
|
UserCacheDir: func() (string, error) { return cacheBase, nil },
|
||||||
|
}
|
||||||
|
for i := 0; i < 2; i++ {
|
||||||
|
code, stderr := runCacheCommand(t, []string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", t.TempDir()}, opts)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("run %d code=%d stderr=%q", i+1, code, stderr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if client.calls != 3 {
|
||||||
|
t.Fatalf("LLM calls = %d, want chunk+extract then extract-only reuse", client.calls)
|
||||||
|
}
|
||||||
|
planRoot := filepath.Join(cacheBase, "notarius", "chunk-plans")
|
||||||
|
entries, err := os.ReadDir(planRoot)
|
||||||
|
if err != nil || len(entries) != 1 {
|
||||||
|
t.Fatalf("plan root entries = %v error=%v", entries, err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(planRoot, entries[0].Name(), "plan.json")); err != nil {
|
||||||
|
t.Fatalf("plan file: %v", err)
|
||||||
|
}
|
||||||
|
debugRuns := childDirs(t, filepath.Join(workspaceDir, "debug"))
|
||||||
|
if len(debugRuns) != 2 {
|
||||||
|
t.Fatalf("debug runs = %#v", debugRuns)
|
||||||
|
}
|
||||||
|
attemptCounts := 0
|
||||||
|
for _, runDir := range debugRuns {
|
||||||
|
if _, err := os.Stat(filepath.Join(runDir, "chunk", "attempt-01.json")); err == nil {
|
||||||
|
attemptCounts++
|
||||||
|
} else if !os.IsNotExist(err) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if attemptCounts != 1 {
|
||||||
|
t.Fatalf("chunk attempt files across runs = %d, want only generating run", attemptCounts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChunkPlanReuseDependsOnlyOnSourceDigest(t *testing.T) {
|
||||||
|
cacheRoot := filepath.Join(t.TempDir(), "plans")
|
||||||
|
inputPath := writeSeriatimInput(t)
|
||||||
|
referencePath := writeFile(t, "players.txt", "Alyx")
|
||||||
|
profilePath := writeScriptoriumProfileFile(t, "chunk-profile", "http://127.0.0.1:8080/v1", "test-model")
|
||||||
|
seedConfig := writeTestConfig(t, `version: 2
|
||||||
|
workspace:
|
||||||
|
chunk_cache:
|
||||||
|
mode: auto
|
||||||
|
directory: `+cacheRoot+`
|
||||||
|
pipelines:
|
||||||
|
seed:
|
||||||
|
input: seriatim
|
||||||
|
chunk:
|
||||||
|
module: generic
|
||||||
|
options:
|
||||||
|
max_units: 1
|
||||||
|
overlap_units: 0
|
||||||
|
artifacts:
|
||||||
|
spells:
|
||||||
|
extract: dnd/spells
|
||||||
|
`)
|
||||||
|
changedConfig := writeTestConfig(t, `version: 2
|
||||||
|
scriptorium:
|
||||||
|
profile_file: `+profilePath+`
|
||||||
|
workspace:
|
||||||
|
chunk_cache:
|
||||||
|
mode: auto
|
||||||
|
directory: `+cacheRoot+`
|
||||||
|
pipelines:
|
||||||
|
changed:
|
||||||
|
input: seriatim
|
||||||
|
chunk:
|
||||||
|
module: dnd/scenes
|
||||||
|
llm_profile: chunk-profile
|
||||||
|
references:
|
||||||
|
players: `+referencePath+`
|
||||||
|
validators:
|
||||||
|
- generic/always_accept
|
||||||
|
artifacts:
|
||||||
|
spells:
|
||||||
|
extract: dnd/spells
|
||||||
|
`)
|
||||||
|
client := newFakeRunLLMClient(false)
|
||||||
|
opts := Options{LLMClientFactory: fakeLLMFactory(client, nil), LookupEnv: mapLookup(nil)}
|
||||||
|
for _, run := range []struct {
|
||||||
|
pipeline string
|
||||||
|
config string
|
||||||
|
}{{pipeline: "seed", config: seedConfig}, {pipeline: "changed", config: changedConfig}} {
|
||||||
|
code, stderr := runCacheCommand(t, []string{"run", run.pipeline, "--config", run.config, "--input", inputPath, "--output-dir", t.TempDir()}, opts)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("pipeline %q code=%d stderr=%q", run.pipeline, code, stderr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if client.calls != 2 {
|
||||||
|
t.Fatalf("LLM calls = %d, want one extractor call per run and no changed chunker call", client.calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResumeAndChunkCacheModesRemainIndependent(t *testing.T) {
|
||||||
|
for _, mode := range []pipeline.ChunkCacheMode{pipeline.ChunkCacheAuto, pipeline.ChunkCacheBypass, pipeline.ChunkCacheRefresh} {
|
||||||
|
t.Run(string(mode), func(t *testing.T) {
|
||||||
|
workspaceDir := filepath.Join(t.TempDir(), "workspace")
|
||||||
|
cacheRoot := filepath.Join(t.TempDir(), "plans")
|
||||||
|
configPath := writeTestConfig(t, cacheTestConfigWithWorkspace(string(mode), cacheRoot, workspaceDir))
|
||||||
|
factory := &recordingChunkPlanFactory{}
|
||||||
|
args := append(cacheRunArgs(t, configPath), "--resume")
|
||||||
|
code, stderr := runCacheCommand(t, args, cacheTestOptions(t, nil, factory))
|
||||||
|
if code != 0 || stderr != "" {
|
||||||
|
t.Fatalf("code=%d stderr=%q", code, stderr)
|
||||||
|
}
|
||||||
|
if mode == pipeline.ChunkCacheBypass {
|
||||||
|
if len(factory.roots) != 0 {
|
||||||
|
t.Fatalf("bypass roots = %#v", factory.roots)
|
||||||
|
}
|
||||||
|
} else if len(factory.roots) != 1 || factory.roots[0] != cacheRoot {
|
||||||
|
t.Fatalf("roots = %#v, want %q", factory.roots, cacheRoot)
|
||||||
|
}
|
||||||
|
if entries := childDirs(t, filepath.Join(workspaceDir, "checkpoints")); len(entries) != 1 {
|
||||||
|
t.Fatalf("checkpoint roots = %#v", entries)
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(cacheRoot, workspaceDir+string(filepath.Separator)) || strings.HasPrefix(workspaceDir, cacheRoot+string(filepath.Separator)) {
|
||||||
|
t.Fatalf("cache root %q and workspace root %q overlap", cacheRoot, workspaceDir)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWorkspaceDirectoryDoesNotSelectChunkPlanRoot(t *testing.T) {
|
||||||
|
cacheBase := filepath.Join(t.TempDir(), "user-cache")
|
||||||
|
factory := &recordingChunkPlanFactory{}
|
||||||
|
for _, workspaceDir := range []string{filepath.Join(t.TempDir(), "workspace-one"), filepath.Join(t.TempDir(), "workspace-two")} {
|
||||||
|
configPath := writeTestConfig(t, cacheTestConfigWithWorkspace("auto", "", workspaceDir))
|
||||||
|
opts := cacheTestOptions(t, nil, factory)
|
||||||
|
opts.UserCacheDir = func() (string, error) { return cacheBase, nil }
|
||||||
|
code, stderr := runCacheCommand(t, cacheRunArgs(t, configPath), opts)
|
||||||
|
if code != 0 || stderr != "" {
|
||||||
|
t.Fatalf("workspace=%q code=%d stderr=%q", workspaceDir, code, stderr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
want := filepath.Join(cacheBase, "notarius", "chunk-plans")
|
||||||
|
if len(factory.roots) != 2 || factory.roots[0] != want || factory.roots[1] != want {
|
||||||
|
t.Fatalf("roots = %#v, want %q twice", factory.roots, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func cacheTestOptions(t *testing.T, env map[string]string, factory *recordingChunkPlanFactory) Options {
|
||||||
|
t.Helper()
|
||||||
|
registries := fakeExecutionRegistries(t)
|
||||||
|
return Options{
|
||||||
|
Catalog: catalogFromRegistries(registries),
|
||||||
|
Registries: registries,
|
||||||
|
LLMClientFactory: fakeLLMFactory(nil, nil),
|
||||||
|
LookupEnv: mapLookup(env),
|
||||||
|
UserCacheDir: func() (string, error) { return filepath.Join(t.TempDir(), "cache"), nil },
|
||||||
|
ChunkPlanStoreFactory: factory.build,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func cacheRunArgs(t *testing.T, configPath string) []string {
|
||||||
|
t.Helper()
|
||||||
|
return []string{"run", "example", "--config", configPath, "--input", writeFile(t, "input.txt", "input"), "--output-dir", t.TempDir()}
|
||||||
|
}
|
||||||
|
|
||||||
|
func runCacheCommand(t *testing.T, args []string, opts Options) (int, string) {
|
||||||
|
t.Helper()
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := RunWithOptions(args, &stdout, &stderr, opts)
|
||||||
|
return code, stderr.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func cacheTestConfig(mode, directory, diagnosticsDir string) string {
|
||||||
|
var b strings.Builder
|
||||||
|
b.WriteString("version: 2\n")
|
||||||
|
if mode != "" || directory != "" {
|
||||||
|
b.WriteString("workspace:\n chunk_cache:\n")
|
||||||
|
if mode != "" {
|
||||||
|
b.WriteString(" mode: " + mode + "\n")
|
||||||
|
}
|
||||||
|
if directory != "" {
|
||||||
|
b.WriteString(" directory: " + directory + "\n")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if diagnosticsDir != "" {
|
||||||
|
b.WriteString("diagnostics:\n work_dir: " + diagnosticsDir + "\n retention: always\n")
|
||||||
|
}
|
||||||
|
b.WriteString("pipelines:\n example:\n input: fake/input\n artifacts:\n spells:\n extract: fake/extract\n")
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func cacheTestConfigWithWorkspace(mode, cacheRoot, workspaceDir string) string {
|
||||||
|
var b strings.Builder
|
||||||
|
b.WriteString("version: 2\nworkspace:\n directory: " + workspaceDir + "\n resume:\n enabled: true\n chunk_cache:\n mode: " + mode + "\n")
|
||||||
|
if cacheRoot != "" {
|
||||||
|
b.WriteString(" directory: " + cacheRoot + "\n")
|
||||||
|
}
|
||||||
|
b.WriteString("pipelines:\n example:\n input: fake/input\n artifacts:\n spells:\n extract: fake/extract\n")
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
@@ -341,7 +341,7 @@ func TestProductionLLMCallersShareScheduledClient(t *testing.T) {
|
|||||||
started.Done()
|
started.Done()
|
||||||
chunker, err := scenes.New(client, scenes.Options{})
|
chunker, err := scenes.New(client, scenes.Options{})
|
||||||
if err == nil {
|
if err == nil {
|
||||||
_, err = chunker.Chunk(context.Background(), contracts.ChunkRequest{Source: doc})
|
_, err = chunker.Plan(context.Background(), contracts.ChunkRequest{Source: doc})
|
||||||
}
|
}
|
||||||
errs <- err
|
errs <- err
|
||||||
}()
|
}()
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/workspace"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/workspace"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkplan"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug"
|
frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
@@ -30,7 +31,7 @@ const defaultOutputRoot = "./notarius-output"
|
|||||||
|
|
||||||
const usage = `Usage:
|
const usage = `Usage:
|
||||||
notarius help
|
notarius help
|
||||||
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--resume] [--session-id id] [--reference selector=path] [--without-reference selector]
|
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--chunk_cache auto|bypass|refresh] [--resume] [--session-id id] [--reference selector=path] [--without-reference selector]
|
||||||
notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b]
|
notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b]
|
||||||
notarius pipelines list --config path/to/config.yml [--json]
|
notarius pipelines list --config path/to/config.yml [--json]
|
||||||
`
|
`
|
||||||
@@ -41,6 +42,8 @@ type Options struct {
|
|||||||
LLMClientFactory LLMClientFactory
|
LLMClientFactory LLMClientFactory
|
||||||
LookupEnv func(string) (string, bool)
|
LookupEnv func(string) (string, bool)
|
||||||
Now func() time.Time
|
Now func() time.Time
|
||||||
|
UserCacheDir func() (string, error)
|
||||||
|
ChunkPlanStoreFactory pipeline.ChunkPlanStoreFactory
|
||||||
}
|
}
|
||||||
|
|
||||||
type LLMClientFactory func(ctx context.Context, cfg config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error)
|
type LLMClientFactory func(ctx context.Context, cfg config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error)
|
||||||
@@ -90,6 +93,12 @@ func normalizeOptions(opts Options) (Options, error) {
|
|||||||
if opts.Now == nil {
|
if opts.Now == nil {
|
||||||
opts.Now = time.Now
|
opts.Now = time.Now
|
||||||
}
|
}
|
||||||
|
if opts.UserCacheDir == nil {
|
||||||
|
opts.UserCacheDir = os.UserCacheDir
|
||||||
|
}
|
||||||
|
if opts.ChunkPlanStoreFactory == nil {
|
||||||
|
opts.ChunkPlanStoreFactory = chunkplan.NewFilesystemStore
|
||||||
|
}
|
||||||
if isEmptyCatalog(opts.Catalog) && isEmptyRegistries(opts.Registries) {
|
if isEmptyCatalog(opts.Catalog) && isEmptyRegistries(opts.Registries) {
|
||||||
components, err := newProductionComponents()
|
components, err := newProductionComponents()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -117,10 +126,12 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
|||||||
diagnosticsDir := fs.String("diagnostics-dir", "", "diagnostics directory")
|
diagnosticsDir := fs.String("diagnostics-dir", "", "diagnostics directory")
|
||||||
llmProfile := fs.String("llm-profile", "", "LLM profile override")
|
llmProfile := fs.String("llm-profile", "", "LLM profile override")
|
||||||
resume := fs.Bool("resume", false, "reuse valid workspace checkpoints")
|
resume := fs.Bool("resume", false, "reuse valid workspace checkpoints")
|
||||||
|
chunkCache := chunkCacheFlag{}
|
||||||
sessionID := sessionIDFlag{}
|
sessionID := sessionIDFlag{}
|
||||||
referenceFlags := stringListFlag{}
|
referenceFlags := stringListFlag{}
|
||||||
withoutReferenceFlags := stringListFlag{}
|
withoutReferenceFlags := stringListFlag{}
|
||||||
fs.Var(&sessionID, "session-id", "prompt session identifier")
|
fs.Var(&sessionID, "session-id", "prompt session identifier")
|
||||||
|
fs.Var(&chunkCache, "chunk_cache", "chunk plan cache mode: auto, bypass, or refresh")
|
||||||
fs.Var(&referenceFlags, "reference", "reference binding, as slot=path, chunk.slot=path, merge.slot=path, lane.slot=path, lane.extract.slot=path, lane.merge.slot=path, or lane.normalize.slot=path")
|
fs.Var(&referenceFlags, "reference", "reference binding, as slot=path, chunk.slot=path, merge.slot=path, lane.slot=path, lane.extract.slot=path, lane.merge.slot=path, or lane.normalize.slot=path")
|
||||||
fs.Var(&withoutReferenceFlags, "without-reference", "unbind a reference, using the same selector forms as --reference")
|
fs.Var(&withoutReferenceFlags, "without-reference", "unbind a reference, using the same selector forms as --reference")
|
||||||
if err := validateRunFlagValues(args); err != nil {
|
if err := validateRunFlagValues(args); err != nil {
|
||||||
@@ -173,6 +184,9 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
|||||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
if chunkCache.set {
|
||||||
|
cfg.Workspace.ChunkCache.Mode = chunkCache.value
|
||||||
|
}
|
||||||
workspaceSettings := workspace.FromConfig(cfg)
|
workspaceSettings := workspace.FromConfig(cfg)
|
||||||
if dir := strings.TrimSpace(*diagnosticsDir); dir != "" {
|
if dir := strings.TrimSpace(*diagnosticsDir); dir != "" {
|
||||||
workspaceSettings.DiagnosticsRoot = dir
|
workspaceSettings.DiagnosticsRoot = dir
|
||||||
@@ -197,6 +211,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
|||||||
ConfigPath: loadedConfigPath,
|
ConfigPath: loadedConfigPath,
|
||||||
ConfigSource: configSource(*configPath),
|
ConfigSource: configSource(*configPath),
|
||||||
OnlyLanes: append([]string(nil), only...),
|
OnlyLanes: append([]string(nil), only...),
|
||||||
|
ChunkCacheOverride: chunkCache.explicitValue(),
|
||||||
Resume: *resume,
|
Resume: *resume,
|
||||||
RunID: runID,
|
RunID: runID,
|
||||||
StartedAt: startedAt,
|
StartedAt: startedAt,
|
||||||
@@ -287,6 +302,10 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("read input %q: %w", strings.TrimSpace(*inputPath), err))
|
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("read input %q: %w", strings.TrimSpace(*inputPath), err))
|
||||||
}
|
}
|
||||||
|
chunkPlans, err := chunkPlanStoreForRun(effective.Config.Workspace.ChunkCache, opts)
|
||||||
|
if err != nil {
|
||||||
|
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
|
||||||
|
}
|
||||||
checkpointRecorder, checkpointLoader, err := checkpointHandlersForRun(workspaceSettings, effective.ResolvedPipeline, rawInput, only, llmProfiles, strings.TrimSpace(*llmProfile), strings.TrimSpace(sessionID.value), *resume)
|
checkpointRecorder, checkpointLoader, err := checkpointHandlersForRun(workspaceSettings, effective.ResolvedPipeline, rawInput, only, llmProfiles, strings.TrimSpace(*llmProfile), strings.TrimSpace(sessionID.value), *resume)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
|
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
|
||||||
@@ -302,6 +321,8 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
|||||||
LLMProfiles: llmProfiles,
|
LLMProfiles: llmProfiles,
|
||||||
Metadata: runMetadata(*outputDir, *diagnosticsDir),
|
Metadata: runMetadata(*outputDir, *diagnosticsDir),
|
||||||
Warnings: referenceWarnings,
|
Warnings: referenceWarnings,
|
||||||
|
ChunkCacheMode: effective.Config.Workspace.ChunkCache.Mode,
|
||||||
|
ChunkPlans: chunkPlans,
|
||||||
Checkpoints: checkpointRecorder,
|
Checkpoints: checkpointRecorder,
|
||||||
Checkpoint: checkpointLoader,
|
Checkpoint: checkpointLoader,
|
||||||
Debug: debugRecorder,
|
Debug: debugRecorder,
|
||||||
@@ -310,6 +331,9 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
if output.Manifest.PipelineID != "" && runDir != nil {
|
if output.Manifest.PipelineID != "" && runDir != nil {
|
||||||
_ = runDir.WriteRunManifest(output.Manifest)
|
_ = runDir.WriteRunManifest(output.Manifest)
|
||||||
|
if output.ChunkPlan != nil {
|
||||||
|
_ = runDir.WriteChunkPlan(*output.ChunkPlan)
|
||||||
|
}
|
||||||
_ = runDir.WriteCheckpointEvents(output.CheckpointEvents)
|
_ = runDir.WriteCheckpointEvents(output.CheckpointEvents)
|
||||||
}
|
}
|
||||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("run pipeline %q: %w", pipelineID, err))
|
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("run pipeline %q: %w", pipelineID, err))
|
||||||
@@ -319,6 +343,11 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
|||||||
if err := writeDiagnostics(runDir, func() error { return runDir.WriteRunManifest(output.Manifest) }); err != nil {
|
if err := writeDiagnostics(runDir, func() error { return runDir.WriteRunManifest(output.Manifest) }); err != nil {
|
||||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics run manifest: %w", err))
|
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics run manifest: %w", err))
|
||||||
}
|
}
|
||||||
|
if output.ChunkPlan != nil {
|
||||||
|
if err := writeDiagnostics(runDir, func() error { return runDir.WriteChunkPlan(*output.ChunkPlan) }); err != nil {
|
||||||
|
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics chunk plan: %w", err))
|
||||||
|
}
|
||||||
|
}
|
||||||
if err := writeDiagnostics(runDir, func() error { return runDir.WriteWarnings(output.Warnings) }); err != nil {
|
if err := writeDiagnostics(runDir, func() error { return runDir.WriteWarnings(output.Warnings) }); err != nil {
|
||||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics warnings: %w", err))
|
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics warnings: %w", err))
|
||||||
}
|
}
|
||||||
@@ -598,13 +627,64 @@ func reorderRunArgs(args []string) []string {
|
|||||||
|
|
||||||
func runFlagTakesValue(arg string) bool {
|
func runFlagTakesValue(arg string) bool {
|
||||||
switch arg {
|
switch arg {
|
||||||
case "--config", "--input", "--only", "--output-dir", "--diagnostics-dir", "--llm-profile", "--session-id", "--reference", "--without-reference":
|
case "--config", "--input", "--only", "--output-dir", "--diagnostics-dir", "--llm-profile", "--session-id", "--chunk_cache", "--reference", "--without-reference":
|
||||||
return true
|
return true
|
||||||
default:
|
default:
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type chunkCacheFlag struct {
|
||||||
|
value pipeline.ChunkCacheMode
|
||||||
|
set bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *chunkCacheFlag) String() string {
|
||||||
|
if f == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return string(f.value)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *chunkCacheFlag) Set(raw string) error {
|
||||||
|
mode, err := pipeline.ParseChunkCacheMode(raw)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
f.value = mode
|
||||||
|
f.set = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f chunkCacheFlag) explicitValue() string {
|
||||||
|
if !f.set {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return string(f.value)
|
||||||
|
}
|
||||||
|
|
||||||
|
func chunkPlanStoreForRun(cfg config.WorkspaceChunkCacheConfig, opts Options) (pipeline.ChunkPlanStore, error) {
|
||||||
|
if cfg.Mode == pipeline.ChunkCacheBypass {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
root := strings.TrimSpace(cfg.Directory)
|
||||||
|
if root == "" {
|
||||||
|
var err error
|
||||||
|
root, err = workspace.DefaultChunkPlanRoot(opts.UserCacheDir)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("resolve chunk plan root: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
store, err := opts.ChunkPlanStoreFactory(root)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("create chunk plan store at %q: %w", root, err)
|
||||||
|
}
|
||||||
|
if store == nil {
|
||||||
|
return nil, fmt.Errorf("create chunk plan store at %q: factory returned nil", root)
|
||||||
|
}
|
||||||
|
return store, nil
|
||||||
|
}
|
||||||
|
|
||||||
func validateRunFlagValues(args []string) error {
|
func validateRunFlagValues(args []string) error {
|
||||||
for i, arg := range args {
|
for i, arg := range args {
|
||||||
if arg != "--session-id" {
|
if arg != "--session-id" {
|
||||||
|
|||||||
@@ -2333,8 +2333,6 @@ func TestRunPipelineWritesCheckpointsWhenWorkspaceResumeEnabled(t *testing.T) {
|
|||||||
for _, name := range []string{
|
for _, name := range []string{
|
||||||
"source/manifest.json",
|
"source/manifest.json",
|
||||||
"source/source-document.json",
|
"source/source-document.json",
|
||||||
"chunk/manifest.json",
|
|
||||||
"chunk/chunks.json",
|
|
||||||
"extract/spells/manifest.json",
|
"extract/spells/manifest.json",
|
||||||
"extract/spells/outputs.json",
|
"extract/spells/outputs.json",
|
||||||
"merge/spells/manifest.json",
|
"merge/spells/manifest.json",
|
||||||
@@ -2346,6 +2344,7 @@ func TestRunPipelineWritesCheckpointsWhenWorkspaceResumeEnabled(t *testing.T) {
|
|||||||
t.Fatalf("expected checkpoint artifact %q: %v", name, err)
|
t.Fatalf("expected checkpoint artifact %q: %v", name, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
assertPathNotExist(t, filepath.Join(checkpointDir, "chunk"))
|
||||||
assertPathNotExist(t, filepath.Join(workspaceDir, "debug"))
|
assertPathNotExist(t, filepath.Join(workspaceDir, "debug"))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -3742,11 +3741,9 @@ func (fakeRunChunker) ReferenceSlots() []contracts.ReferenceSlot {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (fakeRunChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
func (fakeRunChunker) Plan(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
|
||||||
return contracts.ChunkResult{
|
return contracts.ChunkPlanResult{
|
||||||
Chunks: []source.Chunk{
|
Plan: source.ChunkPlan{SourceDigest: req.Source.Digest, Ranges: []source.ChunkRange{{StartUnitID: 1, EndUnitID: 1}}},
|
||||||
{ID: "chunk-1", SourceID: req.Source.ID, Index: 0, Ref: source.SourceRef{SourceID: req.Source.ID, StartUnitID: 1, EndUnitID: 1}, Content: []byte(`{"units":[1]}`), MediaType: "application/json", Units: req.Source.Units},
|
|
||||||
},
|
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
21
internal/cli/test_main_test.go
Normal file
21
internal/cli/test_main_test.go
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMain(m *testing.M) {
|
||||||
|
const name = "NOTARIUS_WORKSPACE_CHUNK_CACHE_MODE"
|
||||||
|
previous, existed := os.LookupEnv(name)
|
||||||
|
if err := os.Setenv(name, "bypass"); err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
code := m.Run()
|
||||||
|
if existed {
|
||||||
|
_ = os.Setenv(name, previous)
|
||||||
|
} else {
|
||||||
|
_ = os.Unsetenv(name)
|
||||||
|
}
|
||||||
|
os.Exit(code)
|
||||||
|
}
|
||||||
@@ -69,12 +69,42 @@ type RejectedOutputManifest struct {
|
|||||||
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
|
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type ChunkPlanManifest struct {
|
||||||
|
Mode string `json:"mode"`
|
||||||
|
Action string `json:"action,omitempty"`
|
||||||
|
SourceDigest string `json:"source_digest,omitempty"`
|
||||||
|
PlanDigest string `json:"plan_digest,omitempty"`
|
||||||
|
PlanSchemaVersion string `json:"plan_schema_version,omitempty"`
|
||||||
|
RequestedModule string `json:"requested_module"`
|
||||||
|
ProducerInputModule string `json:"producer_input_module,omitempty"`
|
||||||
|
ProducerModule string `json:"producer_module,omitempty"`
|
||||||
|
ProducerLLMProfile string `json:"producer_llm_profile,omitempty"`
|
||||||
|
ProducerReferences []ReferenceProvenance `json:"producer_references,omitempty"`
|
||||||
|
ProducerMetadata map[string]any `json:"producer_metadata,omitempty"`
|
||||||
|
CreatedAt *time.Time `json:"created_at,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ChunkPlanSummary is deliberately limited to cache and validation decisions.
|
||||||
|
// It must never contain plan units, source content, annotations, or model I/O.
|
||||||
|
type ChunkPlanSummary struct {
|
||||||
|
Mode string `json:"mode"`
|
||||||
|
SourceDigest string `json:"source_digest,omitempty"`
|
||||||
|
CandidateDigest string `json:"candidate_digest,omitempty"`
|
||||||
|
RequestedModule string `json:"requested_module"`
|
||||||
|
LookupStatus string `json:"lookup_status"`
|
||||||
|
LookupReason string `json:"lookup_reason,omitempty"`
|
||||||
|
Action string `json:"action,omitempty"`
|
||||||
|
ValidationStatus string `json:"validation_status"`
|
||||||
|
PublicationStatus string `json:"publication_status"`
|
||||||
|
}
|
||||||
|
|
||||||
type RunManifest struct {
|
type RunManifest struct {
|
||||||
RunID string `json:"run_id,omitempty"`
|
RunID string `json:"run_id,omitempty"`
|
||||||
PipelineID string `json:"pipeline_id,omitempty"`
|
PipelineID string `json:"pipeline_id,omitempty"`
|
||||||
PipelineDigest string `json:"pipeline_digest,omitempty"`
|
PipelineDigest string `json:"pipeline_digest,omitempty"`
|
||||||
InputModule string `json:"input_module,omitempty"`
|
InputModule string `json:"input_module,omitempty"`
|
||||||
Chunker string `json:"chunker,omitempty"`
|
Chunker string `json:"chunker,omitempty"`
|
||||||
|
ChunkPlan *ChunkPlanManifest `json:"chunk_plan,omitempty"`
|
||||||
SourceDigests []string `json:"source_digests,omitempty"`
|
SourceDigests []string `json:"source_digests,omitempty"`
|
||||||
Extractors []string `json:"extractors,omitempty"`
|
Extractors []string `json:"extractors,omitempty"`
|
||||||
Merger string `json:"merger,omitempty"`
|
Merger string `json:"merger,omitempty"`
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package artifacts
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -16,6 +17,37 @@ func TestRunManifestOmitsEmptyOptionalFields(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunManifestChunkPlanIsAdditiveAndOmitsPlanContent(t *testing.T) {
|
||||||
|
manifest := RunManifest{ChunkPlan: &ChunkPlanManifest{
|
||||||
|
Mode: "auto", Action: "reused", SourceDigest: "sha256:source", PlanDigest: "sha256:plan",
|
||||||
|
PlanSchemaVersion: "notarius.chunk-plan.v1", RequestedModule: "chunk/current",
|
||||||
|
ProducerInputModule: "input/original", ProducerModule: "chunk/original",
|
||||||
|
}}
|
||||||
|
encoded, err := json.Marshal(manifest)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
text := string(encoded)
|
||||||
|
for _, want := range []string{`"chunk_plan"`, `"action":"reused"`, `"requested_module":"chunk/current"`, `"producer_module":"chunk/original"`} {
|
||||||
|
if !strings.Contains(text, want) {
|
||||||
|
t.Fatalf("manifest JSON %s does not contain %s", text, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, forbidden := range []string{`"plan"`, `"units"`, `"annotations"`} {
|
||||||
|
if strings.Contains(text, forbidden) {
|
||||||
|
t.Fatalf("manifest JSON contains forbidden field %s: %s", forbidden, text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var legacy RunManifest
|
||||||
|
if err := json.Unmarshal([]byte(`{"pipeline_id":"legacy"}`), &legacy); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if legacy.PipelineID != "legacy" || legacy.ChunkPlan != nil {
|
||||||
|
t.Fatalf("legacy manifest = %#v", legacy)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
|
func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
|
||||||
manifest := RunManifest{
|
manifest := RunManifest{
|
||||||
PipelineID: "pipeline-1",
|
PipelineID: "pipeline-1",
|
||||||
|
|||||||
126
internal/core/config/chunk_cache_test.go
Normal file
126
internal/core/config/chunk_cache_test.go
Normal file
@@ -0,0 +1,126 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import (
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestChunkCacheDefaults(t *testing.T) {
|
||||||
|
cfg := Default()
|
||||||
|
if cfg.Workspace.ChunkCache.Mode != pipeline.ChunkCacheAuto || cfg.Workspace.ChunkCache.Directory != "" {
|
||||||
|
t.Fatalf("chunk cache defaults = %#v", cfg.Workspace.ChunkCache)
|
||||||
|
}
|
||||||
|
if err := cfg.Validate(); err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChunkCacheFileConfiguration(t *testing.T) {
|
||||||
|
cfg := parseAndApplyConfig(t, `
|
||||||
|
version: 2
|
||||||
|
workspace:
|
||||||
|
chunk_cache:
|
||||||
|
mode: refresh
|
||||||
|
directory: " ./state/../plans "
|
||||||
|
`)
|
||||||
|
if cfg.Workspace.ChunkCache.Mode != pipeline.ChunkCacheRefresh {
|
||||||
|
t.Fatalf("mode = %q", cfg.Workspace.ChunkCache.Mode)
|
||||||
|
}
|
||||||
|
if got, want := cfg.Workspace.ChunkCache.Directory, filepath.Clean("./state/../plans"); got != want {
|
||||||
|
t.Fatalf("directory = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
if err := cfg.Validate(); err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChunkCacheEnvironmentOverridesFile(t *testing.T) {
|
||||||
|
fileCfg, err := ParseFileConfigYAML([]byte(`
|
||||||
|
version: 2
|
||||||
|
workspace:
|
||||||
|
chunk_cache:
|
||||||
|
mode: bypass
|
||||||
|
directory: /file/plans
|
||||||
|
`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cfg := Default()
|
||||||
|
if err := cfg.ApplyFileConfigWithLookup(fileCfg, emptyLookup); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := cfg.ApplyEnvOverridesWithLookup(mapLookup(map[string]string{
|
||||||
|
"NOTARIUS_WORKSPACE_CHUNK_CACHE_MODE": "refresh",
|
||||||
|
"NOTARIUS_WORKSPACE_CHUNK_CACHE_DIR": " /environment/../cache/plans ",
|
||||||
|
})); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if cfg.Workspace.ChunkCache.Mode != pipeline.ChunkCacheRefresh || cfg.Workspace.ChunkCache.Directory != filepath.Clean("/environment/../cache/plans") {
|
||||||
|
t.Fatalf("effective chunk cache = %#v", cfg.Workspace.ChunkCache)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChunkCacheEmptyDirectoryEnvironmentSelectsDefault(t *testing.T) {
|
||||||
|
cfg := Default()
|
||||||
|
cfg.Workspace.ChunkCache.Directory = "/file/plans"
|
||||||
|
if err := cfg.ApplyEnvOverridesWithLookup(mapLookup(map[string]string{"NOTARIUS_WORKSPACE_CHUNK_CACHE_DIR": " \t "})); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if cfg.Workspace.ChunkCache.Directory != "" {
|
||||||
|
t.Fatalf("directory = %q, want unset", cfg.Workspace.ChunkCache.Directory)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChunkCacheRejectsInvalidSuppliedModes(t *testing.T) {
|
||||||
|
fileCfg, err := ParseFileConfigYAML([]byte("version: 2\nworkspace:\n chunk_cache:\n mode: sometimes\n"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cfg := Default()
|
||||||
|
if err := cfg.ApplyFileConfigWithLookup(fileCfg, emptyLookup); err == nil || !strings.Contains(err.Error(), "workspace.chunk_cache.mode") {
|
||||||
|
t.Fatalf("file mode error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg = Default()
|
||||||
|
if err := cfg.ApplyEnvOverridesWithLookup(mapLookup(map[string]string{"NOTARIUS_WORKSPACE_CHUNK_CACHE_MODE": "sometimes"})); err == nil || !strings.Contains(err.Error(), "NOTARIUS_WORKSPACE_CHUNK_CACHE_MODE") {
|
||||||
|
t.Fatalf("environment mode error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg = Default()
|
||||||
|
cfg.Workspace.ChunkCache.Mode = "sometimes"
|
||||||
|
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "chunk cache") {
|
||||||
|
t.Fatalf("Validate() error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChunkCacheConfigurationClonesAndRedacts(t *testing.T) {
|
||||||
|
cfg := Default()
|
||||||
|
cfg.Workspace.ChunkCache = WorkspaceChunkCacheConfig{Mode: pipeline.ChunkCacheRefresh, Directory: "/var/cache/notarius/chunk-plans"}
|
||||||
|
cloned := cloneConfig(cfg)
|
||||||
|
redacted := cfg.Redacted()
|
||||||
|
if cloned.Workspace.ChunkCache != cfg.Workspace.ChunkCache || redacted.Workspace.ChunkCache != cfg.Workspace.ChunkCache {
|
||||||
|
t.Fatalf("cloned=%#v redacted=%#v", cloned.Workspace.ChunkCache, redacted.Workspace.ChunkCache)
|
||||||
|
}
|
||||||
|
redacted.Workspace.ChunkCache.Directory = "/changed"
|
||||||
|
if cfg.Workspace.ChunkCache.Directory != "/var/cache/notarius/chunk-plans" {
|
||||||
|
t.Fatal("redacted mutation changed original")
|
||||||
|
}
|
||||||
|
if err := cfg.Validate(); err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChunkCacheDirectoryValidation(t *testing.T) {
|
||||||
|
cfg := Default()
|
||||||
|
cfg.Workspace.ChunkCache.Directory = "/var/cache/notarius/chunk-plans"
|
||||||
|
if err := cfg.Validate(); err != nil {
|
||||||
|
t.Fatalf("Validate(system root) error = %v", err)
|
||||||
|
}
|
||||||
|
cfg.Workspace.ChunkCache.Directory = "bad\x00path"
|
||||||
|
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "NUL") {
|
||||||
|
t.Fatalf("Validate(NUL directory) error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -38,11 +38,17 @@ type DiagnosticsConfig struct {
|
|||||||
|
|
||||||
type WorkspaceConfig struct {
|
type WorkspaceConfig struct {
|
||||||
Directory string `json:"directory,omitempty"`
|
Directory string `json:"directory,omitempty"`
|
||||||
|
ChunkCache WorkspaceChunkCacheConfig `json:"chunk_cache"`
|
||||||
Diagnostics WorkspaceDiagnosticsConfig `json:"diagnostics"`
|
Diagnostics WorkspaceDiagnosticsConfig `json:"diagnostics"`
|
||||||
Resume WorkspaceResumeConfig `json:"resume"`
|
Resume WorkspaceResumeConfig `json:"resume"`
|
||||||
Debug WorkspaceDebugConfig `json:"debug"`
|
Debug WorkspaceDebugConfig `json:"debug"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type WorkspaceChunkCacheConfig struct {
|
||||||
|
Mode pipeline.ChunkCacheMode `json:"mode"`
|
||||||
|
Directory string `json:"directory,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
type WorkspaceDiagnosticsConfig struct {
|
type WorkspaceDiagnosticsConfig struct {
|
||||||
Enabled bool `json:"enabled"`
|
Enabled bool `json:"enabled"`
|
||||||
Retention diagnostics.RetentionMode `json:"retention,omitempty"`
|
Retention diagnostics.RetentionMode `json:"retention,omitempty"`
|
||||||
@@ -71,6 +77,7 @@ func Default() Config {
|
|||||||
Retention: diagnostics.RetentionAuto,
|
Retention: diagnostics.RetentionAuto,
|
||||||
},
|
},
|
||||||
Workspace: WorkspaceConfig{
|
Workspace: WorkspaceConfig{
|
||||||
|
ChunkCache: WorkspaceChunkCacheConfig{Mode: pipeline.ChunkCacheAuto},
|
||||||
Diagnostics: WorkspaceDiagnosticsConfig{
|
Diagnostics: WorkspaceDiagnosticsConfig{
|
||||||
Enabled: true,
|
Enabled: true,
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -135,7 +135,7 @@ func TestResolveCanBindSceneChunkerFromCatalog(t *testing.T) {
|
|||||||
Key: "dnd/scenes",
|
Key: "dnd/scenes",
|
||||||
Stage: pipeline.StageChunk,
|
Stage: pipeline.StageChunk,
|
||||||
Requires: []string{"source.transcript"},
|
Requires: []string{"source.transcript"},
|
||||||
Provides: []string{"chunks", "chunks.scenes"},
|
Provides: []string{"chunks"},
|
||||||
})
|
})
|
||||||
|
|
||||||
effective, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: catalog})
|
effective, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: catalog})
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
)
|
)
|
||||||
|
|
||||||
func LoadFromEnv() (Config, error) {
|
func LoadFromEnv() (Config, error) {
|
||||||
@@ -57,6 +58,16 @@ func (c *Config) applyEnvOverridesWithLookup(lookup func(string) (string, bool))
|
|||||||
if raw, ok := lookup("NOTARIUS_WORKSPACE_DIR"); ok {
|
if raw, ok := lookup("NOTARIUS_WORKSPACE_DIR"); ok {
|
||||||
c.Workspace.Directory = strings.TrimSpace(raw)
|
c.Workspace.Directory = strings.TrimSpace(raw)
|
||||||
}
|
}
|
||||||
|
if raw, ok := lookup("NOTARIUS_WORKSPACE_CHUNK_CACHE_MODE"); ok {
|
||||||
|
mode, err := pipeline.ParseChunkCacheMode(raw)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("NOTARIUS_WORKSPACE_CHUNK_CACHE_MODE: %w", err)
|
||||||
|
}
|
||||||
|
c.Workspace.ChunkCache.Mode = mode
|
||||||
|
}
|
||||||
|
if raw, ok := lookup("NOTARIUS_WORKSPACE_CHUNK_CACHE_DIR"); ok {
|
||||||
|
c.Workspace.ChunkCache.Directory = cleanOptionalPath(raw)
|
||||||
|
}
|
||||||
if raw, ok := lookup("NOTARIUS_WORKSPACE_DIAGNOSTICS_ENABLED"); ok {
|
if raw, ok := lookup("NOTARIUS_WORKSPACE_DIAGNOSTICS_ENABLED"); ok {
|
||||||
value, err := parseBoolEnv("NOTARIUS_WORKSPACE_DIAGNOSTICS_ENABLED", raw)
|
value, err := parseBoolEnv("NOTARIUS_WORKSPACE_DIAGNOSTICS_ENABLED", raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
@@ -54,11 +55,17 @@ type FileDiagnosticsConfig struct {
|
|||||||
|
|
||||||
type FileWorkspaceConfig struct {
|
type FileWorkspaceConfig struct {
|
||||||
Directory *string `yaml:"directory,omitempty"`
|
Directory *string `yaml:"directory,omitempty"`
|
||||||
|
ChunkCache *FileWorkspaceChunkCacheConfig `yaml:"chunk_cache,omitempty"`
|
||||||
Diagnostics *FileWorkspaceDiagnosticsConfig `yaml:"diagnostics,omitempty"`
|
Diagnostics *FileWorkspaceDiagnosticsConfig `yaml:"diagnostics,omitempty"`
|
||||||
Resume *FileWorkspaceEnabledConfig `yaml:"resume,omitempty"`
|
Resume *FileWorkspaceEnabledConfig `yaml:"resume,omitempty"`
|
||||||
Debug *FileWorkspaceEnabledConfig `yaml:"debug,omitempty"`
|
Debug *FileWorkspaceEnabledConfig `yaml:"debug,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type FileWorkspaceChunkCacheConfig struct {
|
||||||
|
Mode *string `yaml:"mode,omitempty"`
|
||||||
|
Directory *string `yaml:"directory,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
type FileWorkspaceDiagnosticsConfig struct {
|
type FileWorkspaceDiagnosticsConfig struct {
|
||||||
Enabled *bool `yaml:"enabled,omitempty"`
|
Enabled *bool `yaml:"enabled,omitempty"`
|
||||||
Retention *string `yaml:"retention,omitempty"`
|
Retention *string `yaml:"retention,omitempty"`
|
||||||
@@ -338,6 +345,18 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
|
|||||||
if fileCfg.Workspace.Directory != nil {
|
if fileCfg.Workspace.Directory != nil {
|
||||||
c.Workspace.Directory = strings.TrimSpace(*fileCfg.Workspace.Directory)
|
c.Workspace.Directory = strings.TrimSpace(*fileCfg.Workspace.Directory)
|
||||||
}
|
}
|
||||||
|
if fileCfg.Workspace.ChunkCache != nil {
|
||||||
|
if fileCfg.Workspace.ChunkCache.Mode != nil {
|
||||||
|
mode, err := pipeline.ParseChunkCacheMode(*fileCfg.Workspace.ChunkCache.Mode)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("workspace.chunk_cache.mode: %w", err)
|
||||||
|
}
|
||||||
|
c.Workspace.ChunkCache.Mode = mode
|
||||||
|
}
|
||||||
|
if fileCfg.Workspace.ChunkCache.Directory != nil {
|
||||||
|
c.Workspace.ChunkCache.Directory = cleanOptionalPath(*fileCfg.Workspace.ChunkCache.Directory)
|
||||||
|
}
|
||||||
|
}
|
||||||
if fileCfg.Workspace.Diagnostics != nil {
|
if fileCfg.Workspace.Diagnostics != nil {
|
||||||
if fileCfg.Workspace.Diagnostics.Enabled != nil {
|
if fileCfg.Workspace.Diagnostics.Enabled != nil {
|
||||||
c.Workspace.Diagnostics.Enabled = *fileCfg.Workspace.Diagnostics.Enabled
|
c.Workspace.Diagnostics.Enabled = *fileCfg.Workspace.Diagnostics.Enabled
|
||||||
@@ -360,6 +379,14 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func cleanOptionalPath(value string) string {
|
||||||
|
value = strings.TrimSpace(value)
|
||||||
|
if value == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return filepath.Clean(value)
|
||||||
|
}
|
||||||
|
|
||||||
func normalizeStageWorkers(values map[string]int) (map[string]int, bool, error) {
|
func normalizeStageWorkers(values map[string]int) (map[string]int, bool, error) {
|
||||||
workers := make(map[string]int, len(values))
|
workers := make(map[string]int, len(values))
|
||||||
configured := false
|
configured := false
|
||||||
|
|||||||
@@ -61,6 +61,12 @@ func validateScriptorium(cfg ScriptoriumConfig) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func validateWorkspace(cfg WorkspaceConfig) error {
|
func validateWorkspace(cfg WorkspaceConfig) error {
|
||||||
|
if err := cfg.ChunkCache.Mode.Validate(); err != nil {
|
||||||
|
return fmt.Errorf("workspace chunk cache: %w", err)
|
||||||
|
}
|
||||||
|
if strings.ContainsRune(cfg.ChunkCache.Directory, '\x00') {
|
||||||
|
return fmt.Errorf("workspace chunk cache directory must not contain NUL")
|
||||||
|
}
|
||||||
if cfg.Diagnostics.retentionSet {
|
if cfg.Diagnostics.retentionSet {
|
||||||
switch cfg.Diagnostics.Retention {
|
switch cfg.Diagnostics.Retention {
|
||||||
case "", diagnostics.RetentionAuto, diagnostics.RetentionAlways, diagnostics.RetentionNever:
|
case "", diagnostics.RetentionAuto, diagnostics.RetentionAlways, diagnostics.RetentionNever:
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ const (
|
|||||||
ArtifactCheckpointEvents = "checkpoint-events.json"
|
ArtifactCheckpointEvents = "checkpoint-events.json"
|
||||||
ArtifactSourceDocument = "source-document.json"
|
ArtifactSourceDocument = "source-document.json"
|
||||||
ArtifactRunManifest = "run-manifest.json"
|
ArtifactRunManifest = "run-manifest.json"
|
||||||
|
ArtifactChunkPlan = "chunk-plan.json"
|
||||||
ArtifactRunReport = "run-report.json"
|
ArtifactRunReport = "run-report.json"
|
||||||
ArtifactWarnings = "warnings.json"
|
ArtifactWarnings = "warnings.json"
|
||||||
ArtifactErrorLog = "error.log"
|
ArtifactErrorLog = "error.log"
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ func TestArtifactNamesUseExtractionOrientedNames(t *testing.T) {
|
|||||||
ArtifactResolvedReferences,
|
ArtifactResolvedReferences,
|
||||||
ArtifactSourceDocument,
|
ArtifactSourceDocument,
|
||||||
ArtifactRunManifest,
|
ArtifactRunManifest,
|
||||||
|
ArtifactChunkPlan,
|
||||||
ArtifactRunReport,
|
ArtifactRunReport,
|
||||||
ArtifactWarnings,
|
ArtifactWarnings,
|
||||||
ArtifactErrorLog,
|
ArtifactErrorLog,
|
||||||
|
|||||||
@@ -56,6 +56,7 @@ type InvocationMetadata struct {
|
|||||||
ConfigPath string `json:"config_path,omitempty"`
|
ConfigPath string `json:"config_path,omitempty"`
|
||||||
ConfigSource string `json:"config_source,omitempty"`
|
ConfigSource string `json:"config_source,omitempty"`
|
||||||
OnlyLanes []string `json:"only_lanes,omitempty"`
|
OnlyLanes []string `json:"only_lanes,omitempty"`
|
||||||
|
ChunkCacheOverride string `json:"chunk_cache_override,omitempty"`
|
||||||
RunID string `json:"run_id"`
|
RunID string `json:"run_id"`
|
||||||
StartedAt time.Time `json:"started_at"`
|
StartedAt time.Time `json:"started_at"`
|
||||||
}
|
}
|
||||||
@@ -166,6 +167,10 @@ func (r *RunDirectory) WriteRunManifest(manifest artifacts.RunManifest) error {
|
|||||||
return r.WriteJSONArtifact(ArtifactRunManifest, manifest)
|
return r.WriteJSONArtifact(ArtifactRunManifest, manifest)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (r *RunDirectory) WriteChunkPlan(summary artifacts.ChunkPlanSummary) error {
|
||||||
|
return r.WriteJSONArtifact(ArtifactChunkPlan, summary)
|
||||||
|
}
|
||||||
|
|
||||||
func (r *RunDirectory) WriteRunReport(payload any) error {
|
func (r *RunDirectory) WriteRunReport(payload any) error {
|
||||||
return r.WriteJSONArtifact(ArtifactRunReport, payload)
|
return r.WriteJSONArtifact(ArtifactRunReport, payload)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -200,6 +200,9 @@ func TestWriteTypedArtifacts(t *testing.T) {
|
|||||||
if err := runDir.WriteRunManifest(artifacts.RunManifest{RunID: "run-1"}); err != nil {
|
if err := runDir.WriteRunManifest(artifacts.RunManifest{RunID: "run-1"}); err != nil {
|
||||||
t.Fatalf("WriteRunManifest: %v", err)
|
t.Fatalf("WriteRunManifest: %v", err)
|
||||||
}
|
}
|
||||||
|
if err := runDir.WriteChunkPlan(artifacts.ChunkPlanSummary{Mode: "auto", RequestedModule: "chunk/test", LookupStatus: "invalid", LookupReason: "stored chunk plan failed validation", ValidationStatus: "not_run", PublicationStatus: "not_published"}); err != nil {
|
||||||
|
t.Fatalf("WriteChunkPlan: %v", err)
|
||||||
|
}
|
||||||
if err := runDir.WriteRunReport(map[string]any{"ok": true}); err != nil {
|
if err := runDir.WriteRunReport(map[string]any{"ok": true}); err != nil {
|
||||||
t.Fatalf("WriteRunReport: %v", err)
|
t.Fatalf("WriteRunReport: %v", err)
|
||||||
}
|
}
|
||||||
@@ -213,6 +216,7 @@ func TestWriteTypedArtifacts(t *testing.T) {
|
|||||||
ArtifactResolvedReferences,
|
ArtifactResolvedReferences,
|
||||||
ArtifactSourceDocument,
|
ArtifactSourceDocument,
|
||||||
ArtifactRunManifest,
|
ArtifactRunManifest,
|
||||||
|
ArtifactChunkPlan,
|
||||||
ArtifactRunReport,
|
ArtifactRunReport,
|
||||||
ArtifactWarnings,
|
ArtifactWarnings,
|
||||||
} {
|
} {
|
||||||
@@ -220,6 +224,15 @@ func TestWriteTypedArtifacts(t *testing.T) {
|
|||||||
t.Fatalf("expected artifact %q: %v", name, err)
|
t.Fatalf("expected artifact %q: %v", name, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
chunkSummary, err := os.ReadFile(filepath.Join(runDir.Path(), ArtifactChunkPlan))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, forbidden := range []string{"units", "annotations", "source content", "prompt", "response", "raw invalid"} {
|
||||||
|
if strings.Contains(string(chunkSummary), forbidden) {
|
||||||
|
t.Fatalf("chunk plan summary leaked %q: %s", forbidden, chunkSummary)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestWriteRedactedEffectiveConfigWritesPayloadReturnedByProvider(t *testing.T) {
|
func TestWriteRedactedEffectiveConfigWritesPayloadReturnedByProvider(t *testing.T) {
|
||||||
|
|||||||
240
internal/core/source/chunk_plan.go
Normal file
240
internal/core/source/chunk_plan.go
Normal file
@@ -0,0 +1,240 @@
|
|||||||
|
package source
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CanonicalizeChunkAnnotations validates annotation namespaces and JSON values
|
||||||
|
// and returns an independently owned map whose values use canonical JSON bytes.
|
||||||
|
func CanonicalizeChunkAnnotations(annotations ChunkAnnotations) (ChunkAnnotations, error) {
|
||||||
|
if len(annotations) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
canonical := make(ChunkAnnotations, len(annotations))
|
||||||
|
for namespace, raw := range annotations {
|
||||||
|
if strings.TrimSpace(namespace) == "" {
|
||||||
|
return nil, fmt.Errorf("chunk annotation namespace must not be empty")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(namespace) != namespace {
|
||||||
|
return nil, fmt.Errorf("chunk annotation namespace %q must not contain leading or trailing whitespace", namespace)
|
||||||
|
}
|
||||||
|
value, err := decodeAnnotation(raw)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("chunk annotation %q: %w", namespace, err)
|
||||||
|
}
|
||||||
|
encoded, err := json.Marshal(value)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("chunk annotation %q contains an unsupported value: %w", namespace, err)
|
||||||
|
}
|
||||||
|
canonical[namespace] = encoded
|
||||||
|
}
|
||||||
|
return canonical, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func decodeAnnotation(raw json.RawMessage) (any, error) {
|
||||||
|
decoder := json.NewDecoder(bytes.NewReader(raw))
|
||||||
|
decoder.UseNumber()
|
||||||
|
var value any
|
||||||
|
if err := decoder.Decode(&value); err != nil {
|
||||||
|
return nil, fmt.Errorf("must contain valid JSON: %w", err)
|
||||||
|
}
|
||||||
|
var trailing any
|
||||||
|
if err := decoder.Decode(&trailing); err != io.EOF {
|
||||||
|
if err == nil {
|
||||||
|
return nil, fmt.Errorf("must contain exactly one JSON value")
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("must contain exactly one JSON value: %w", err)
|
||||||
|
}
|
||||||
|
return value, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidateChunkAnnotations requires annotations to already contain canonical
|
||||||
|
// JSON. CanonicalizeChunkAnnotations can be used at producer boundaries.
|
||||||
|
func ValidateChunkAnnotations(annotations ChunkAnnotations) error {
|
||||||
|
canonical, err := CanonicalizeChunkAnnotations(annotations)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for namespace, raw := range annotations {
|
||||||
|
if !bytes.Equal(raw, canonical[namespace]) {
|
||||||
|
return fmt.Errorf("chunk annotation %q must use canonical JSON", namespace)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// CloneChunkAnnotations returns a deep clone, including every raw JSON value.
|
||||||
|
func CloneChunkAnnotations(annotations ChunkAnnotations) ChunkAnnotations {
|
||||||
|
if len(annotations) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
cloned := make(ChunkAnnotations, len(annotations))
|
||||||
|
for namespace, raw := range annotations {
|
||||||
|
cloned[namespace] = append(json.RawMessage(nil), raw...)
|
||||||
|
}
|
||||||
|
return cloned
|
||||||
|
}
|
||||||
|
|
||||||
|
// CloneChunkPlan returns a deep clone of a chunk plan.
|
||||||
|
func CloneChunkPlan(plan ChunkPlan) ChunkPlan {
|
||||||
|
cloned := ChunkPlan{
|
||||||
|
SourceDigest: plan.SourceDigest,
|
||||||
|
Ranges: make([]ChunkRange, len(plan.Ranges)),
|
||||||
|
Annotations: CloneChunkAnnotations(plan.Annotations),
|
||||||
|
}
|
||||||
|
for i, chunkRange := range plan.Ranges {
|
||||||
|
cloned.Ranges[i] = ChunkRange{
|
||||||
|
StartUnitID: chunkRange.StartUnitID,
|
||||||
|
EndUnitID: chunkRange.EndUnitID,
|
||||||
|
Annotations: CloneChunkAnnotations(chunkRange.Annotations),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return cloned
|
||||||
|
}
|
||||||
|
|
||||||
|
// CanonicalizeChunkPlan returns a deep clone with canonical annotation bytes.
|
||||||
|
func CanonicalizeChunkPlan(plan ChunkPlan) (ChunkPlan, error) {
|
||||||
|
canonical := CloneChunkPlan(plan)
|
||||||
|
annotations, err := CanonicalizeChunkAnnotations(plan.Annotations)
|
||||||
|
if err != nil {
|
||||||
|
return ChunkPlan{}, fmt.Errorf("chunk plan annotations: %w", err)
|
||||||
|
}
|
||||||
|
canonical.Annotations = annotations
|
||||||
|
for i := range plan.Ranges {
|
||||||
|
annotations, err := CanonicalizeChunkAnnotations(plan.Ranges[i].Annotations)
|
||||||
|
if err != nil {
|
||||||
|
return ChunkPlan{}, fmt.Errorf("chunk plan range[%d] annotations: %w", i, err)
|
||||||
|
}
|
||||||
|
canonical.Ranges[i].Annotations = annotations
|
||||||
|
}
|
||||||
|
return canonical, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidateChunkPlan validates a canonical plan against the current source.
|
||||||
|
// Ranges may contain gaps or overlap, but their start positions must increase.
|
||||||
|
func ValidateChunkPlan(doc *SourceDocument, plan ChunkPlan) error {
|
||||||
|
if err := ValidateDocument(doc); err != nil {
|
||||||
|
return fmt.Errorf("source document: %w", err)
|
||||||
|
}
|
||||||
|
if plan.SourceDigest != doc.Digest {
|
||||||
|
return fmt.Errorf("chunk plan source_digest %q does not match source document digest %q", plan.SourceDigest, doc.Digest)
|
||||||
|
}
|
||||||
|
if len(plan.Ranges) == 0 {
|
||||||
|
return fmt.Errorf("chunk plan ranges must not be empty")
|
||||||
|
}
|
||||||
|
if err := ValidateChunkAnnotations(plan.Annotations); err != nil {
|
||||||
|
return fmt.Errorf("chunk plan annotations: %w", err)
|
||||||
|
}
|
||||||
|
previousStart := -1
|
||||||
|
for i, chunkRange := range plan.Ranges {
|
||||||
|
start, ok := UnitIndex(doc, chunkRange.StartUnitID)
|
||||||
|
if !ok {
|
||||||
|
return fmt.Errorf("chunk plan range[%d] start_unit_id %d was not found", i, chunkRange.StartUnitID)
|
||||||
|
}
|
||||||
|
end, ok := UnitIndex(doc, chunkRange.EndUnitID)
|
||||||
|
if !ok {
|
||||||
|
return fmt.Errorf("chunk plan range[%d] end_unit_id %d was not found", i, chunkRange.EndUnitID)
|
||||||
|
}
|
||||||
|
if start > end {
|
||||||
|
return fmt.Errorf("chunk plan range[%d] start_unit_id %d appears after end_unit_id %d", i, chunkRange.StartUnitID, chunkRange.EndUnitID)
|
||||||
|
}
|
||||||
|
if start <= previousStart {
|
||||||
|
return fmt.Errorf("chunk plan range[%d] start_unit_id %d does not appear after the previous range start", i, chunkRange.StartUnitID)
|
||||||
|
}
|
||||||
|
if err := ValidateChunkAnnotations(chunkRange.Annotations); err != nil {
|
||||||
|
return fmt.Errorf("chunk plan range[%d] annotations: %w", i, err)
|
||||||
|
}
|
||||||
|
previousStart = start
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// MaterializeChunkPlan deterministically expands a validated plan into chunks.
|
||||||
|
func MaterializeChunkPlan(doc *SourceDocument, plan ChunkPlan) ([]Chunk, error) {
|
||||||
|
if err := ValidateChunkPlan(doc, plan); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
chunks := make([]Chunk, 0, len(plan.Ranges))
|
||||||
|
for index, chunkRange := range plan.Ranges {
|
||||||
|
start, _ := UnitIndex(doc, chunkRange.StartUnitID)
|
||||||
|
end, _ := UnitIndex(doc, chunkRange.EndUnitID)
|
||||||
|
units := cloneSourceUnits(doc.Units[start : end+1])
|
||||||
|
content, err := json.Marshal(struct {
|
||||||
|
Units []SourceUnit `json:"units"`
|
||||||
|
}{Units: units})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("encode chunk plan range[%d]: %w", index, err)
|
||||||
|
}
|
||||||
|
chunks = append(chunks, Chunk{
|
||||||
|
ID: fmt.Sprintf("chunk-%06d", index+1),
|
||||||
|
SourceID: doc.ID,
|
||||||
|
Index: index,
|
||||||
|
Ref: SourceRef{SourceID: doc.ID, StartUnitID: chunkRange.StartUnitID, EndUnitID: chunkRange.EndUnitID},
|
||||||
|
Content: content,
|
||||||
|
MediaType: "application/json",
|
||||||
|
Units: units,
|
||||||
|
Metadata: map[string]any{
|
||||||
|
"start_unit_id": chunkRange.StartUnitID,
|
||||||
|
"end_unit_id": chunkRange.EndUnitID,
|
||||||
|
"unit_count": len(units),
|
||||||
|
},
|
||||||
|
Annotations: CloneChunkAnnotations(chunkRange.Annotations),
|
||||||
|
PlanAnnotations: CloneChunkAnnotations(plan.Annotations),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return chunks, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneSourceUnits(units []SourceUnit) []SourceUnit {
|
||||||
|
if len(units) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
cloned := make([]SourceUnit, len(units))
|
||||||
|
for i, unit := range units {
|
||||||
|
cloned[i] = unit
|
||||||
|
cloned[i].Metadata = cloneJSONMap(unit.Metadata)
|
||||||
|
}
|
||||||
|
return cloned
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneJSONMap(values map[string]any) map[string]any {
|
||||||
|
if len(values) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
cloned := make(map[string]any, len(values))
|
||||||
|
for key, value := range values {
|
||||||
|
cloned[key] = cloneJSONValue(value)
|
||||||
|
}
|
||||||
|
return cloned
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneJSONValue(value any) any {
|
||||||
|
switch typed := value.(type) {
|
||||||
|
case map[string]any:
|
||||||
|
return cloneJSONMap(typed)
|
||||||
|
case []any:
|
||||||
|
cloned := make([]any, len(typed))
|
||||||
|
for i := range typed {
|
||||||
|
cloned[i] = cloneJSONValue(typed[i])
|
||||||
|
}
|
||||||
|
return cloned
|
||||||
|
case json.RawMessage:
|
||||||
|
return append(json.RawMessage(nil), typed...)
|
||||||
|
case []byte:
|
||||||
|
return append([]byte(nil), typed...)
|
||||||
|
case []string:
|
||||||
|
return append([]string(nil), typed...)
|
||||||
|
case map[string]string:
|
||||||
|
cloned := make(map[string]string, len(typed))
|
||||||
|
for key, item := range typed {
|
||||||
|
cloned[key] = item
|
||||||
|
}
|
||||||
|
return cloned
|
||||||
|
default:
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
}
|
||||||
246
internal/core/source/chunk_plan_test.go
Normal file
246
internal/core/source/chunk_plan_test.go
Normal file
@@ -0,0 +1,246 @@
|
|||||||
|
package source
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCanonicalizeChunkAnnotations(t *testing.T) {
|
||||||
|
original := ChunkAnnotations{
|
||||||
|
"domain/items": json.RawMessage(` { "z": [3, 2, 1], "a": 1.0 } `),
|
||||||
|
}
|
||||||
|
canonical, err := CanonicalizeChunkAnnotations(original)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("CanonicalizeChunkAnnotations() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if got, want := string(canonical["domain/items"]), `{"a":1.0,"z":[3,2,1]}`; got != want {
|
||||||
|
t.Fatalf("canonical annotation = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
original["domain/items"][0] = '['
|
||||||
|
if got := string(canonical["domain/items"]); got != `{"a":1.0,"z":[3,2,1]}` {
|
||||||
|
t.Fatalf("canonical annotation changed after input mutation: %q", got)
|
||||||
|
}
|
||||||
|
canonical["domain/items"][0] = '['
|
||||||
|
if original["domain/items"][0] == '[' && bytes.Equal(original["domain/items"], canonical["domain/items"]) {
|
||||||
|
t.Fatal("input and canonical annotation share value storage")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCanonicalizeChunkAnnotationsRejectsInvalidValues(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
annotations ChunkAnnotations
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "blank namespace", annotations: ChunkAnnotations{" \t": json.RawMessage(`true`)}, want: "namespace must not be empty"},
|
||||||
|
{name: "untrimmed namespace", annotations: ChunkAnnotations{" items ": json.RawMessage(`true`)}, want: "leading or trailing whitespace"},
|
||||||
|
{name: "invalid JSON", annotations: ChunkAnnotations{"items": json.RawMessage(`{"x":`)}, want: "valid JSON"},
|
||||||
|
{name: "trailing JSON", annotations: ChunkAnnotations{"items": json.RawMessage(`true false`)}, want: "exactly one JSON value"},
|
||||||
|
{name: "non-finite number", annotations: ChunkAnnotations{"items": json.RawMessage(`NaN`)}, want: "valid JSON"},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
_, err := CanonicalizeChunkAnnotations(tt.annotations)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), tt.want) {
|
||||||
|
t.Fatalf("CanonicalizeChunkAnnotations() error = %v, want containing %q", err, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateChunkAnnotationsRequiresCanonicalJSON(t *testing.T) {
|
||||||
|
if err := ValidateChunkAnnotations(ChunkAnnotations{"items": json.RawMessage(` {"b":2,"a":1}`)}); err == nil || !strings.Contains(err.Error(), "canonical JSON") {
|
||||||
|
t.Fatalf("ValidateChunkAnnotations() error = %v, want canonical JSON error", err)
|
||||||
|
}
|
||||||
|
if err := ValidateChunkAnnotations(ChunkAnnotations{"items": json.RawMessage(`{"a":1,"b":2}`)}); err != nil {
|
||||||
|
t.Fatalf("ValidateChunkAnnotations(canonical) error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCloneChunkPlanDoesNotShareAnnotationBytes(t *testing.T) {
|
||||||
|
plan := validChunkPlan(planDocument())
|
||||||
|
cloned := CloneChunkPlan(plan)
|
||||||
|
cloned.Annotations["plan"][0] = '['
|
||||||
|
cloned.Ranges[0].Annotations["range"][0] = '['
|
||||||
|
if string(plan.Annotations["plan"]) != `{"value":1}` || string(plan.Ranges[0].Annotations["range"]) != `{"value":2}` {
|
||||||
|
t.Fatal("CloneChunkPlan() shares annotation value storage")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateChunkPlanRanges(t *testing.T) {
|
||||||
|
doc := planDocument()
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
mutate func(*ChunkPlan)
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "source mismatch", mutate: func(plan *ChunkPlan) { plan.SourceDigest = "sha256:other" }, want: "does not match"},
|
||||||
|
{name: "missing ranges", mutate: func(plan *ChunkPlan) { plan.Ranges = nil }, want: "ranges must not be empty"},
|
||||||
|
{name: "missing start", mutate: func(plan *ChunkPlan) { plan.Ranges[0].StartUnitID = 99 }, want: "start_unit_id 99 was not found"},
|
||||||
|
{name: "missing end", mutate: func(plan *ChunkPlan) { plan.Ranges[0].EndUnitID = 99 }, want: "end_unit_id 99 was not found"},
|
||||||
|
{name: "backward range", mutate: func(plan *ChunkPlan) { plan.Ranges[0] = ChunkRange{StartUnitID: 30, EndUnitID: 10} }, want: "appears after end_unit_id"},
|
||||||
|
{name: "duplicate start", mutate: func(plan *ChunkPlan) { plan.Ranges[1].StartUnitID = plan.Ranges[0].StartUnitID }, want: "does not appear after"},
|
||||||
|
{name: "backward starts", mutate: func(plan *ChunkPlan) {
|
||||||
|
plan.Ranges = []ChunkRange{{StartUnitID: 30, EndUnitID: 50}, {StartUnitID: 20, EndUnitID: 40}}
|
||||||
|
}, want: "does not appear after"},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
plan := validChunkPlan(doc)
|
||||||
|
tt.mutate(&plan)
|
||||||
|
err := ValidateChunkPlan(doc, plan)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), tt.want) {
|
||||||
|
t.Fatalf("ValidateChunkPlan() error = %v, want containing %q", err, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, ranges := range map[string][]ChunkRange{
|
||||||
|
"gap": {{StartUnitID: 10, EndUnitID: 20}, {StartUnitID: 40, EndUnitID: 50}},
|
||||||
|
"overlap": {{StartUnitID: 10, EndUnitID: 30}, {StartUnitID: 20, EndUnitID: 50}},
|
||||||
|
} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
plan := validChunkPlan(doc)
|
||||||
|
plan.Ranges = ranges
|
||||||
|
if err := ValidateChunkPlan(doc, plan); err != nil {
|
||||||
|
t.Fatalf("ValidateChunkPlan() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDigestChunkPlanIsStableAndCoversLogicalPlan(t *testing.T) {
|
||||||
|
doc := planDocument()
|
||||||
|
plan := validChunkPlan(doc)
|
||||||
|
first, err := DigestChunkPlan(plan)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("DigestChunkPlan() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
reformatted := CloneChunkPlan(plan)
|
||||||
|
reformatted.Annotations["plan"] = json.RawMessage(` { "value" : 1 } `)
|
||||||
|
second, err := DigestChunkPlan(reformatted)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("DigestChunkPlan(reformatted) error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if first != second {
|
||||||
|
t.Fatalf("digests = %q and %q, want stable canonical annotation digest", first, second)
|
||||||
|
}
|
||||||
|
|
||||||
|
changes := []func(*ChunkPlan){
|
||||||
|
func(value *ChunkPlan) { value.Ranges[0].EndUnitID = 30 },
|
||||||
|
func(value *ChunkPlan) { value.Annotations["plan"] = json.RawMessage(`{"value":2}`) },
|
||||||
|
func(value *ChunkPlan) { value.Ranges[0].Annotations["range"] = json.RawMessage(`{"value":3}`) },
|
||||||
|
}
|
||||||
|
for i, change := range changes {
|
||||||
|
changed := CloneChunkPlan(plan)
|
||||||
|
change(&changed)
|
||||||
|
digest, err := DigestChunkPlan(changed)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("DigestChunkPlan(change %d) error = %v", i, err)
|
||||||
|
}
|
||||||
|
if digest == first {
|
||||||
|
t.Fatalf("DigestChunkPlan(change %d) = %q, want changed digest", i, digest)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMaterializeChunkPlanExactOutputAndMutationSafety(t *testing.T) {
|
||||||
|
doc := planDocument()
|
||||||
|
plan := validChunkPlan(doc)
|
||||||
|
plan.Ranges = []ChunkRange{
|
||||||
|
{StartUnitID: 10, EndUnitID: 30, Annotations: ChunkAnnotations{"range": json.RawMessage(`{"value":2}`)}},
|
||||||
|
{StartUnitID: 20, EndUnitID: 50, Annotations: ChunkAnnotations{"range": json.RawMessage(`{"value":3}`)}},
|
||||||
|
}
|
||||||
|
chunks, err := MaterializeChunkPlan(doc, plan)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("MaterializeChunkPlan() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if len(chunks) != 2 {
|
||||||
|
t.Fatalf("chunks = %d, want 2", len(chunks))
|
||||||
|
}
|
||||||
|
first := chunks[0]
|
||||||
|
if first.ID != "chunk-000001" || first.SourceID != doc.ID || first.Index != 0 || first.MediaType != "application/json" {
|
||||||
|
t.Fatalf("first chunk identity = %#v", first)
|
||||||
|
}
|
||||||
|
if want := (SourceRef{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 30}); first.Ref != want {
|
||||||
|
t.Fatalf("first ref = %#v, want %#v", first.Ref, want)
|
||||||
|
}
|
||||||
|
if got, want := unitIDs(first.Units), []int{10, 20, 30}; !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("first unit ids = %#v, want %#v", got, want)
|
||||||
|
}
|
||||||
|
wantContent, _ := json.Marshal(struct {
|
||||||
|
Units []SourceUnit `json:"units"`
|
||||||
|
}{Units: doc.Units[:3]})
|
||||||
|
if !bytes.Equal(first.Content, wantContent) {
|
||||||
|
t.Fatalf("first content = %s, want %s", first.Content, wantContent)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(first.Metadata, map[string]any{"start_unit_id": 10, "end_unit_id": 30, "unit_count": 3}) {
|
||||||
|
t.Fatalf("first metadata = %#v", first.Metadata)
|
||||||
|
}
|
||||||
|
if string(first.Annotations["range"]) != `{"value":2}` || string(first.PlanAnnotations["plan"]) != `{"value":1}` {
|
||||||
|
t.Fatalf("first annotations = %#v / %#v", first.Annotations, first.PlanAnnotations)
|
||||||
|
}
|
||||||
|
if got, want := unitIDs(chunks[1].Units), []int{20, 30, 40, 50}; !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("overlapping unit ids = %#v, want %#v", got, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
again, err := MaterializeChunkPlan(doc, plan)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("MaterializeChunkPlan(repeated) error = %v", err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(chunks, again) {
|
||||||
|
t.Fatalf("repeated materialization differs:\nfirst: %#v\nagain: %#v", chunks, again)
|
||||||
|
}
|
||||||
|
chunks[0].Units[0].Text = "mutated"
|
||||||
|
chunks[0].Annotations["range"][0] = '['
|
||||||
|
chunks[0].PlanAnnotations["plan"][0] = '['
|
||||||
|
if doc.Units[0].Text == "mutated" || string(plan.Ranges[0].Annotations["range"]) != `{"value":2}` || string(plan.Annotations["plan"]) != `{"value":1}` {
|
||||||
|
t.Fatal("materialized chunk shares owned plan or source storage")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMaterializeChunkPlanDeepClonesUnitMetadata(t *testing.T) {
|
||||||
|
doc := planDocument()
|
||||||
|
doc.Units[0].Metadata = map[string]any{"nested": map[string]any{"values": []any{json.RawMessage(`{"ok":true}`)}}}
|
||||||
|
chunks, err := MaterializeChunkPlan(doc, validChunkPlan(doc))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
nested := chunks[0].Units[0].Metadata["nested"].(map[string]any)
|
||||||
|
nested["values"].([]any)[0].(json.RawMessage)[0] = '['
|
||||||
|
nested["changed"] = true
|
||||||
|
original := doc.Units[0].Metadata["nested"].(map[string]any)
|
||||||
|
if _, exists := original["changed"]; exists || string(original["values"].([]any)[0].(json.RawMessage)) != `{"ok":true}` {
|
||||||
|
t.Fatalf("source metadata changed through materialized chunk: %#v", doc.Units[0].Metadata)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func planDocument() *SourceDocument {
|
||||||
|
doc := &SourceDocument{ID: "source-plan", Kind: "test", Format: "application/test", Digest: "sha256:source-plan"}
|
||||||
|
for _, id := range []int{10, 20, 30, 40, 50} {
|
||||||
|
doc.Units = append(doc.Units, SourceUnit{ID: id, Kind: "line", Text: "unit", Ref: SourceRef{SourceID: doc.ID, StartUnitID: id, EndUnitID: id}})
|
||||||
|
}
|
||||||
|
return doc
|
||||||
|
}
|
||||||
|
|
||||||
|
func validChunkPlan(doc *SourceDocument) ChunkPlan {
|
||||||
|
return ChunkPlan{
|
||||||
|
SourceDigest: doc.Digest,
|
||||||
|
Ranges: []ChunkRange{
|
||||||
|
{StartUnitID: 10, EndUnitID: 20, Annotations: ChunkAnnotations{"range": json.RawMessage(`{"value":2}`)}},
|
||||||
|
{StartUnitID: 30, EndUnitID: 50},
|
||||||
|
},
|
||||||
|
Annotations: ChunkAnnotations{"plan": json.RawMessage(`{"value":1}`)},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func unitIDs(units []SourceUnit) []int {
|
||||||
|
ids := make([]int, len(units))
|
||||||
|
for i, unit := range units {
|
||||||
|
ids[i] = unit.ID
|
||||||
|
}
|
||||||
|
return ids
|
||||||
|
}
|
||||||
@@ -37,6 +37,14 @@ func DigestDocument(doc *SourceDocument) (string, error) {
|
|||||||
// DigestChunk returns a deterministic digest of a chunk, including its source
|
// DigestChunk returns a deterministic digest of a chunk, including its source
|
||||||
// provenance, content, units, and metadata.
|
// provenance, content, units, and metadata.
|
||||||
func DigestChunk(chunk Chunk) (string, error) {
|
func DigestChunk(chunk Chunk) (string, error) {
|
||||||
|
annotations, err := CanonicalizeChunkAnnotations(chunk.Annotations)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("canonicalize source chunk annotations: %w", err)
|
||||||
|
}
|
||||||
|
planAnnotations, err := CanonicalizeChunkAnnotations(chunk.PlanAnnotations)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("canonicalize source chunk plan annotations: %w", err)
|
||||||
|
}
|
||||||
payload := struct {
|
payload := struct {
|
||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
SourceID string `json:"source_id"`
|
SourceID string `json:"source_id"`
|
||||||
@@ -46,6 +54,8 @@ func DigestChunk(chunk Chunk) (string, error) {
|
|||||||
MediaType string `json:"media_type"`
|
MediaType string `json:"media_type"`
|
||||||
Units []SourceUnit `json:"units"`
|
Units []SourceUnit `json:"units"`
|
||||||
Metadata map[string]any `json:"metadata,omitempty"`
|
Metadata map[string]any `json:"metadata,omitempty"`
|
||||||
|
Annotations ChunkAnnotations `json:"annotations,omitempty"`
|
||||||
|
PlanAnnotations ChunkAnnotations `json:"plan_annotations,omitempty"`
|
||||||
}{
|
}{
|
||||||
ID: chunk.ID,
|
ID: chunk.ID,
|
||||||
SourceID: chunk.SourceID,
|
SourceID: chunk.SourceID,
|
||||||
@@ -55,6 +65,8 @@ func DigestChunk(chunk Chunk) (string, error) {
|
|||||||
MediaType: chunk.MediaType,
|
MediaType: chunk.MediaType,
|
||||||
Units: chunk.Units,
|
Units: chunk.Units,
|
||||||
Metadata: chunk.Metadata,
|
Metadata: chunk.Metadata,
|
||||||
|
Annotations: annotations,
|
||||||
|
PlanAnnotations: planAnnotations,
|
||||||
}
|
}
|
||||||
encoded, err := json.Marshal(payload)
|
encoded, err := json.Marshal(payload)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -63,3 +75,28 @@ func DigestChunk(chunk Chunk) (string, error) {
|
|||||||
sum := sha256.Sum256(encoded)
|
sum := sha256.Sum256(encoded)
|
||||||
return "sha256:" + hex.EncodeToString(sum[:]), nil
|
return "sha256:" + hex.EncodeToString(sum[:]), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DigestChunkPlan returns a deterministic digest of the logical plan. Storage
|
||||||
|
// schema, producer provenance, warnings, and timestamps are intentionally not
|
||||||
|
// part of the digest.
|
||||||
|
func DigestChunkPlan(plan ChunkPlan) (string, error) {
|
||||||
|
canonical, err := CanonicalizeChunkPlan(plan)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if isBlank(canonical.SourceDigest) {
|
||||||
|
return "", fmt.Errorf("chunk plan source_digest must not be empty")
|
||||||
|
}
|
||||||
|
if hasSurroundingWhitespace(canonical.SourceDigest) {
|
||||||
|
return "", fmt.Errorf("chunk plan source_digest must not contain leading or trailing whitespace")
|
||||||
|
}
|
||||||
|
if len(canonical.Ranges) == 0 {
|
||||||
|
return "", fmt.Errorf("chunk plan ranges must not be empty")
|
||||||
|
}
|
||||||
|
encoded, err := json.Marshal(canonical)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("encode chunk plan for digest: %w", err)
|
||||||
|
}
|
||||||
|
sum := sha256.Sum256(encoded)
|
||||||
|
return "sha256:" + hex.EncodeToString(sum[:]), nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
package source
|
package source
|
||||||
|
|
||||||
|
import "encoding/json"
|
||||||
|
|
||||||
type SourceDocument struct {
|
type SourceDocument struct {
|
||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
Kind string `json:"kind"`
|
Kind string `json:"kind"`
|
||||||
@@ -23,6 +25,20 @@ type SourceRef struct {
|
|||||||
EndUnitID int `json:"end_unit_id"`
|
EndUnitID int `json:"end_unit_id"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type ChunkAnnotations map[string]json.RawMessage
|
||||||
|
|
||||||
|
type ChunkPlan struct {
|
||||||
|
SourceDigest string `json:"source_digest"`
|
||||||
|
Ranges []ChunkRange `json:"ranges"`
|
||||||
|
Annotations ChunkAnnotations `json:"annotations,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ChunkRange struct {
|
||||||
|
StartUnitID int `json:"start_unit_id"`
|
||||||
|
EndUnitID int `json:"end_unit_id"`
|
||||||
|
Annotations ChunkAnnotations `json:"annotations,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
type Chunk struct {
|
type Chunk struct {
|
||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
SourceID string `json:"source_id"`
|
SourceID string `json:"source_id"`
|
||||||
@@ -32,4 +48,6 @@ type Chunk struct {
|
|||||||
MediaType string `json:"media_type"`
|
MediaType string `json:"media_type"`
|
||||||
Units []SourceUnit `json:"units"`
|
Units []SourceUnit `json:"units"`
|
||||||
Metadata map[string]any `json:"metadata,omitempty"`
|
Metadata map[string]any `json:"metadata,omitempty"`
|
||||||
|
Annotations ChunkAnnotations `json:"annotations,omitempty"`
|
||||||
|
PlanAnnotations ChunkAnnotations `json:"plan_annotations,omitempty"`
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package source
|
package source
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/json"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
@@ -258,6 +259,28 @@ func TestDigestChunkIsDeterministicAndIncludesReference(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestDigestChunkIncludesAnnotationScopes(t *testing.T) {
|
||||||
|
doc := validDocument()
|
||||||
|
chunk := Chunk{
|
||||||
|
ID: "chunk-1", SourceID: doc.ID, Ref: SourceRef{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 2},
|
||||||
|
Content: []byte("content"), MediaType: "text/plain", Units: doc.Units,
|
||||||
|
Annotations: ChunkAnnotations{"scope": json.RawMessage(`{"value":1}`)},
|
||||||
|
PlanAnnotations: ChunkAnnotations{"scope": json.RawMessage(`{"value":2}`)},
|
||||||
|
}
|
||||||
|
base, err := DigestChunk(chunk)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("DigestChunk() error = %v", err)
|
||||||
|
}
|
||||||
|
chunk.Annotations["scope"] = json.RawMessage(`{"value":3}`)
|
||||||
|
rangeChanged, _ := DigestChunk(chunk)
|
||||||
|
chunk.Annotations["scope"] = json.RawMessage(`{"value":1}`)
|
||||||
|
chunk.PlanAnnotations["scope"] = json.RawMessage(`{"value":3}`)
|
||||||
|
planChanged, _ := DigestChunk(chunk)
|
||||||
|
if base == rangeChanged || base == planChanged || rangeChanged == planChanged {
|
||||||
|
t.Fatalf("annotation scope digests did not change distinctly: %q %q %q", base, rangeChanged, planChanged)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestValidateRefValid(t *testing.T) {
|
func TestValidateRefValid(t *testing.T) {
|
||||||
doc := validDocument()
|
doc := validDocument()
|
||||||
ref := SourceRef{
|
ref := SourceRef{
|
||||||
|
|||||||
22
internal/core/workspace/chunk_plan.go
Normal file
22
internal/core/workspace/chunk_plan.go
Normal file
@@ -0,0 +1,22 @@
|
|||||||
|
package workspace
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
func DefaultChunkPlanRoot(userCacheDir func() (string, error)) (string, error) {
|
||||||
|
if userCacheDir == nil {
|
||||||
|
return "", fmt.Errorf("user cache directory resolver must not be nil")
|
||||||
|
}
|
||||||
|
root, err := userCacheDir()
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("resolve user cache directory: %w", err)
|
||||||
|
}
|
||||||
|
root = strings.TrimSpace(root)
|
||||||
|
if root == "" {
|
||||||
|
return "", fmt.Errorf("user cache directory must not be empty")
|
||||||
|
}
|
||||||
|
return filepath.Join(filepath.Clean(root), "notarius", "chunk-plans"), nil
|
||||||
|
}
|
||||||
57
internal/core/workspace/chunk_plan_test.go
Normal file
57
internal/core/workspace/chunk_plan_test.go
Normal file
@@ -0,0 +1,57 @@
|
|||||||
|
package workspace
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDefaultChunkPlanRoot(t *testing.T) {
|
||||||
|
got, err := DefaultChunkPlanRoot(func() (string, error) { return "/cache/user", nil })
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("DefaultChunkPlanRoot() error = %v", err)
|
||||||
|
}
|
||||||
|
if want := filepath.Join("/cache/user", "notarius", "chunk-plans"); got != want {
|
||||||
|
t.Fatalf("root = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDefaultChunkPlanRootRejectsResolverFailures(t *testing.T) {
|
||||||
|
boom := errors.New("resolver failed")
|
||||||
|
if _, err := DefaultChunkPlanRoot(func() (string, error) { return "", boom }); !errors.Is(err, boom) {
|
||||||
|
t.Fatalf("resolver error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := DefaultChunkPlanRoot(func() (string, error) { return " \t ", nil }); err == nil || !strings.Contains(err.Error(), "empty") {
|
||||||
|
t.Fatalf("empty result error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := DefaultChunkPlanRoot(nil); err == nil {
|
||||||
|
t.Fatal("nil resolver error = nil")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChunkPlanDirectoryIsIndependentFromWorkspaceSettings(t *testing.T) {
|
||||||
|
base := config.Default()
|
||||||
|
base.Workspace.Directory = "/workspace/one"
|
||||||
|
base.Workspace.ChunkCache.Directory = "/cache/plans"
|
||||||
|
first := FromConfig(base)
|
||||||
|
|
||||||
|
changedWorkspace := base
|
||||||
|
changedWorkspace.Workspace.Directory = "/workspace/two"
|
||||||
|
second := FromConfig(changedWorkspace)
|
||||||
|
if base.Workspace.ChunkCache.Directory != changedWorkspace.Workspace.ChunkCache.Directory {
|
||||||
|
t.Fatal("workspace directory changed chunk plan directory")
|
||||||
|
}
|
||||||
|
if first.RootDir == second.RootDir || first.CheckpointsRoot == second.CheckpointsRoot || first.DebugRoot == second.DebugRoot {
|
||||||
|
t.Fatalf("workspace settings did not follow workspace directory: %#v %#v", first, second)
|
||||||
|
}
|
||||||
|
|
||||||
|
changedCache := base
|
||||||
|
changedCache.Workspace.ChunkCache.Directory = "/cache/other"
|
||||||
|
third := FromConfig(changedCache)
|
||||||
|
if first != third {
|
||||||
|
t.Fatalf("chunk plan directory changed workspace settings: %#v %#v", first, third)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -11,7 +11,6 @@ type StageName string
|
|||||||
|
|
||||||
const (
|
const (
|
||||||
StageSource StageName = "source"
|
StageSource StageName = "source"
|
||||||
StageChunk StageName = "chunk"
|
|
||||||
StageExtract StageName = "extract"
|
StageExtract StageName = "extract"
|
||||||
StageMerge StageName = "merge"
|
StageMerge StageName = "merge"
|
||||||
StageNormalize StageName = "normalize"
|
StageNormalize StageName = "normalize"
|
||||||
@@ -55,11 +54,6 @@ type SourceManifest struct {
|
|||||||
SourceID string `json:"source_id,omitempty"`
|
SourceID string `json:"source_id,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type ChunkManifest struct {
|
|
||||||
StageManifest
|
|
||||||
ChunkCount int `json:"chunk_count,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type ExtractLaneManifest struct {
|
type ExtractLaneManifest struct {
|
||||||
StageManifest
|
StageManifest
|
||||||
ChunkCount int `json:"chunk_count,omitempty"`
|
ChunkCount int `json:"chunk_count,omitempty"`
|
||||||
|
|||||||
@@ -42,18 +42,6 @@ func TestManifestJSONRoundTrips(t *testing.T) {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("chunk", func(t *testing.T) {
|
|
||||||
manifest := ChunkManifest{
|
|
||||||
StageManifest: populatedManifest(StageChunk, "", "generic", started, completed),
|
|
||||||
ChunkCount: 3,
|
|
||||||
}
|
|
||||||
var got ChunkManifest
|
|
||||||
roundTripManifest(t, manifest, &got)
|
|
||||||
if got.ChunkCount != manifest.ChunkCount || got.Stage != StageChunk {
|
|
||||||
t.Fatalf("round trip chunk manifest = %+v", got)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("extract", func(t *testing.T) {
|
t.Run("extract", func(t *testing.T) {
|
||||||
manifest := ExtractLaneManifest{
|
manifest := ExtractLaneManifest{
|
||||||
StageManifest: populatedManifest(StageExtract, "spells", "dnd/spells", started, completed),
|
StageManifest: populatedManifest(StageExtract, "spells", "dnd/spells", started, completed),
|
||||||
|
|||||||
@@ -58,36 +58,6 @@ func (l *WorkspaceLoader) Source(moduleKey string) (pipeline.SourceCheckpoint, p
|
|||||||
return pipeline.SourceCheckpoint{Document: &doc}, reusedDecision()
|
return pipeline.SourceCheckpoint{Document: &doc}, reusedDecision()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (l *WorkspaceLoader) Chunk(moduleKey string, sourceDigest string) (pipeline.ChunkCheckpoint, pipeline.CheckpointDecision) {
|
|
||||||
expectedDependencies := digestFingerprints("source_document", sourceDigest)
|
|
||||||
var manifest coreworkspace.ChunkManifest
|
|
||||||
if decision := l.readJSON("chunk/manifest.json", &manifest); !decision.Reused {
|
|
||||||
return pipeline.ChunkCheckpoint{}, decision
|
|
||||||
}
|
|
||||||
if decision := l.validateManifest(manifest.StageManifest, coreworkspace.StageChunk, "", moduleKey, coreworkspace.StatusSucceeded, expectedDependencies); !decision.Reused {
|
|
||||||
return pipeline.ChunkCheckpoint{}, decision
|
|
||||||
}
|
|
||||||
var payload chunksEnvelope
|
|
||||||
if decision := l.readJSON("chunk/chunks.json", &payload); !decision.Reused {
|
|
||||||
return pipeline.ChunkCheckpoint{}, decision
|
|
||||||
}
|
|
||||||
chunks, err := sourceChunksFromEnvelope(payload.Chunks)
|
|
||||||
if err != nil {
|
|
||||||
return pipeline.ChunkCheckpoint{}, invalidDecision("chunk checkpoint payload is invalid: %v", err)
|
|
||||||
}
|
|
||||||
if len(chunks) == 0 {
|
|
||||||
return pipeline.ChunkCheckpoint{}, invalidDecision("chunk checkpoint payload has no chunks")
|
|
||||||
}
|
|
||||||
outputDigests, err := chunkOutputDigests(chunks)
|
|
||||||
if err != nil {
|
|
||||||
return pipeline.ChunkCheckpoint{}, invalidDecision("chunk checkpoint output cannot be digested: %v", err)
|
|
||||||
}
|
|
||||||
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), outputDigests) {
|
|
||||||
return pipeline.ChunkCheckpoint{}, invalidDecision("chunk checkpoint output digests do not match payload")
|
|
||||||
}
|
|
||||||
return pipeline.ChunkCheckpoint{Chunks: chunks, Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (l *WorkspaceLoader) Extract(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.ExtractCheckpoint, pipeline.CheckpointDecision) {
|
func (l *WorkspaceLoader) Extract(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.ExtractCheckpoint, pipeline.CheckpointDecision) {
|
||||||
var manifest coreworkspace.ExtractLaneManifest
|
var manifest coreworkspace.ExtractLaneManifest
|
||||||
if d := l.readJSON(laneManifestPath("extract", laneID), &manifest); !d.Reused {
|
if d := l.readJSON(laneManifestPath("extract", laneID), &manifest); !d.Reused {
|
||||||
@@ -232,30 +202,6 @@ func (l *WorkspaceLoader) validateLaneManifest(manifest coreworkspace.StageManif
|
|||||||
return reusedDecision()
|
return reusedDecision()
|
||||||
}
|
}
|
||||||
|
|
||||||
func sourceChunksFromEnvelope(values []chunkEnvelope) ([]source.Chunk, error) {
|
|
||||||
if len(values) == 0 {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
out := make([]source.Chunk, 0, len(values))
|
|
||||||
for _, value := range values {
|
|
||||||
content, err := contentFromEnvelope(value.Content)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
out = append(out, source.Chunk{
|
|
||||||
ID: value.ID,
|
|
||||||
SourceID: value.SourceID,
|
|
||||||
Index: value.Index,
|
|
||||||
Ref: value.Ref,
|
|
||||||
Content: content,
|
|
||||||
MediaType: value.Content.MediaType,
|
|
||||||
Units: cloneSourceUnits(value.Units),
|
|
||||||
Metadata: cloneMetadata(value.Metadata),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return out, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func contentFromEnvelope(value binaryEnvelope) ([]byte, error) {
|
func contentFromEnvelope(value binaryEnvelope) ([]byte, error) {
|
||||||
content, err := base64.StdEncoding.DecodeString(value.ContentBase64)
|
content, err := base64.StdEncoding.DecodeString(value.ContentBase64)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -65,54 +65,6 @@ func (r *WorkspaceRecorder) SourceFailed(moduleKey string, err error) error {
|
|||||||
return r.writeManifest("source/manifest.json", coreworkspace.SourceManifest{StageManifest: manifest})
|
return r.writeManifest("source/manifest.json", coreworkspace.SourceManifest{StageManifest: manifest})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *WorkspaceRecorder) ChunkRunning(moduleKey string, sourceDigest string) error {
|
|
||||||
manifest := r.newStageManifest(coreworkspace.StageChunk, coreworkspace.StatusRunning)
|
|
||||||
manifest.ModuleKey = moduleKey
|
|
||||||
manifest.DependencyFingerprints = workspaceFingerprints(digestFingerprints("source_document", sourceDigest))
|
|
||||||
manifest.StartedAt = timePtr(r.timestamp())
|
|
||||||
return r.writeManifest("chunk/manifest.json", coreworkspace.ChunkManifest{StageManifest: manifest})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *WorkspaceRecorder) ChunkSucceeded(moduleKey string, sourceDigest string, chunks []source.Chunk, warnings []contracts.Warning) error {
|
|
||||||
outputDigests, err := chunkOutputDigests(chunks)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("digest chunk checkpoint output: %w", err)
|
|
||||||
}
|
|
||||||
payload := chunksEnvelope{Chunks: chunkEnvelopes(chunks), Warnings: cloneWarnings(warnings)}
|
|
||||||
if err := r.writePayload("chunk/chunks.json", payload); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
manifest := r.newStageManifest(coreworkspace.StageChunk, coreworkspace.StatusSucceeded)
|
|
||||||
manifest.ModuleKey = moduleKey
|
|
||||||
manifest.DependencyFingerprints = workspaceFingerprints(digestFingerprints("source_document", sourceDigest))
|
|
||||||
manifest.OutputDigests = workspaceFingerprints(outputDigests)
|
|
||||||
manifest.ValidationStatus = validationStatusString(warnings, nil)
|
|
||||||
manifest.CompletedAt = timePtr(r.timestamp())
|
|
||||||
return r.writeManifest("chunk/manifest.json", coreworkspace.ChunkManifest{
|
|
||||||
StageManifest: manifest,
|
|
||||||
ChunkCount: len(chunks),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *WorkspaceRecorder) ChunkRejected(moduleKey string, sourceDigest string, rejected contracts.RejectedOutput) error {
|
|
||||||
manifest := r.newStageManifest(coreworkspace.StageChunk, coreworkspace.StatusSucceededWithRejections)
|
|
||||||
manifest.ModuleKey = moduleKey
|
|
||||||
manifest.DependencyFingerprints = workspaceFingerprints(digestFingerprints("source_document", sourceDigest))
|
|
||||||
manifest.ValidationStatus = "rejected"
|
|
||||||
manifest.Rejections = rejectionSummaries([]contracts.RejectedOutput{rejected})
|
|
||||||
manifest.CompletedAt = timePtr(r.timestamp())
|
|
||||||
return r.writeManifest("chunk/manifest.json", coreworkspace.ChunkManifest{StageManifest: manifest})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *WorkspaceRecorder) ChunkFailed(moduleKey string, sourceDigest string, err error) error {
|
|
||||||
manifest := r.newStageManifest(coreworkspace.StageChunk, coreworkspace.StatusFailed)
|
|
||||||
manifest.ModuleKey = moduleKey
|
|
||||||
manifest.DependencyFingerprints = workspaceFingerprints(digestFingerprints("source_document", sourceDigest))
|
|
||||||
manifest.CompletedAt = timePtr(r.timestamp())
|
|
||||||
manifest.Metadata = errorMetadata(err)
|
|
||||||
return r.writeManifest("chunk/manifest.json", coreworkspace.ChunkManifest{StageManifest: manifest})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *WorkspaceRecorder) ExtractRunning(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) error {
|
func (r *WorkspaceRecorder) ExtractRunning(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) error {
|
||||||
manifest := r.laneManifest(coreworkspace.StageExtract, coreworkspace.StatusRunning, laneID, moduleKey, dependencies)
|
manifest := r.laneManifest(coreworkspace.StageExtract, coreworkspace.StatusRunning, laneID, moduleKey, dependencies)
|
||||||
manifest.StartedAt = timePtr(r.timestamp())
|
manifest.StartedAt = timePtr(r.timestamp())
|
||||||
@@ -245,21 +197,6 @@ type sourceDocumentEnvelope struct {
|
|||||||
Document source.SourceDocument `json:"document"`
|
Document source.SourceDocument `json:"document"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type chunksEnvelope struct {
|
|
||||||
Chunks []chunkEnvelope `json:"chunks"`
|
|
||||||
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type chunkEnvelope struct {
|
|
||||||
ID string `json:"id"`
|
|
||||||
SourceID string `json:"source_id"`
|
|
||||||
Index int `json:"index"`
|
|
||||||
Ref source.SourceRef `json:"ref"`
|
|
||||||
Content binaryEnvelope `json:"content"`
|
|
||||||
Units []source.SourceUnit `json:"units,omitempty"`
|
|
||||||
Metadata map[string]any `json:"metadata,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type binaryEnvelope struct {
|
type binaryEnvelope struct {
|
||||||
ContentBase64 string `json:"content_base64,omitempty"`
|
ContentBase64 string `json:"content_base64,omitempty"`
|
||||||
ContentDigest string `json:"content_digest,omitempty"`
|
ContentDigest string `json:"content_digest,omitempty"`
|
||||||
@@ -313,25 +250,6 @@ func artifactOutputDigests(outputs []pipeline.CheckpointArtifact) []pipeline.Che
|
|||||||
return normalizeFingerprints(values)
|
return normalizeFingerprints(values)
|
||||||
}
|
}
|
||||||
|
|
||||||
func chunkEnvelopes(chunks []source.Chunk) []chunkEnvelope {
|
|
||||||
if len(chunks) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
out := make([]chunkEnvelope, 0, len(chunks))
|
|
||||||
for _, chunk := range chunks {
|
|
||||||
out = append(out, chunkEnvelope{
|
|
||||||
ID: chunk.ID,
|
|
||||||
SourceID: chunk.SourceID,
|
|
||||||
Index: chunk.Index,
|
|
||||||
Ref: chunk.Ref,
|
|
||||||
Content: binaryEnvelopeFromContent(chunk.Content, chunk.MediaType, chunk.Metadata, nil),
|
|
||||||
Units: cloneSourceUnits(chunk.Units),
|
|
||||||
Metadata: cloneMetadata(chunk.Metadata),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func binaryEnvelopeFromContent(content []byte, mediaType string, metadata map[string]any, warnings []contracts.Warning) binaryEnvelope {
|
func binaryEnvelopeFromContent(content []byte, mediaType string, metadata map[string]any, warnings []contracts.Warning) binaryEnvelope {
|
||||||
return binaryEnvelope{
|
return binaryEnvelope{
|
||||||
ContentBase64: base64.StdEncoding.EncodeToString(content),
|
ContentBase64: base64.StdEncoding.EncodeToString(content),
|
||||||
@@ -390,21 +308,6 @@ func cloneMetadata(metadata map[string]any) map[string]any {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
func chunkOutputDigests(chunks []source.Chunk) ([]pipeline.CheckpointFingerprint, error) {
|
|
||||||
values := make([]pipeline.CheckpointFingerprint, 0, len(chunks))
|
|
||||||
for _, chunk := range chunks {
|
|
||||||
digest, err := source.DigestChunk(chunk)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("chunk %q: %w", chunk.ID, err)
|
|
||||||
}
|
|
||||||
values = append(values, pipeline.CheckpointFingerprint{
|
|
||||||
Name: chunk.ID,
|
|
||||||
Value: digest,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return normalizeFingerprints(values), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func digestFingerprints(name string, digest string) []pipeline.CheckpointFingerprint {
|
func digestFingerprints(name string, digest string) []pipeline.CheckpointFingerprint {
|
||||||
digest = strings.TrimSpace(digest)
|
digest = strings.TrimSpace(digest)
|
||||||
if digest == "" {
|
if digest == "" {
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
package checkpoint
|
package checkpoint
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/base64"
|
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -24,18 +23,6 @@ func TestWorkspaceRecorderWritesSuccessfulCheckpointFiles(t *testing.T) {
|
|||||||
Digest: "sha256:source",
|
Digest: "sha256:source",
|
||||||
Units: []source.SourceUnit{{ID: 1, Kind: "line", Text: "hello", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}}},
|
Units: []source.SourceUnit{{ID: 1, Kind: "line", Text: "hello", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}}},
|
||||||
}
|
}
|
||||||
chunks := []source.Chunk{
|
|
||||||
{
|
|
||||||
ID: "chunk-1",
|
|
||||||
SourceID: "source-1",
|
|
||||||
Index: 0,
|
|
||||||
Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1},
|
|
||||||
Content: []byte("chunk content"),
|
|
||||||
MediaType: "text/plain",
|
|
||||||
Units: doc.Units,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := recorder.SourceRunning("seriatim"); err != nil {
|
if err := recorder.SourceRunning("seriatim"); err != nil {
|
||||||
t.Fatalf("SourceRunning: %v", err)
|
t.Fatalf("SourceRunning: %v", err)
|
||||||
}
|
}
|
||||||
@@ -47,36 +34,6 @@ func TestWorkspaceRecorderWritesSuccessfulCheckpointFiles(t *testing.T) {
|
|||||||
if _, err := os.Stat(filepath.Join(root, "source", "source-document.json")); err != nil {
|
if _, err := os.Stat(filepath.Join(root, "source", "source-document.json")); err != nil {
|
||||||
t.Fatalf("expected source checkpoint payload: %v", err)
|
t.Fatalf("expected source checkpoint payload: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := recorder.ChunkRunning("generic", doc.Digest); err != nil {
|
|
||||||
t.Fatalf("ChunkRunning: %v", err)
|
|
||||||
}
|
|
||||||
if err := recorder.ChunkSucceeded("generic", doc.Digest, chunks, nil); err != nil {
|
|
||||||
t.Fatalf("ChunkSucceeded: %v", err)
|
|
||||||
}
|
|
||||||
assertManifestStatus(t, filepath.Join(root, "chunk", "manifest.json"), coreworkspace.StatusSucceeded)
|
|
||||||
var chunkPayload struct {
|
|
||||||
Chunks []struct {
|
|
||||||
Content struct {
|
|
||||||
ContentBase64 string `json:"content_base64"`
|
|
||||||
ContentDigest string `json:"content_digest"`
|
|
||||||
} `json:"content"`
|
|
||||||
} `json:"chunks"`
|
|
||||||
}
|
|
||||||
readJSON(t, filepath.Join(root, "chunk", "chunks.json"), &chunkPayload)
|
|
||||||
if len(chunkPayload.Chunks) != 1 {
|
|
||||||
t.Fatalf("checkpoint chunks = %#v, want one", chunkPayload.Chunks)
|
|
||||||
}
|
|
||||||
decoded, err := base64.StdEncoding.DecodeString(chunkPayload.Chunks[0].Content.ContentBase64)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("decode chunk content: %v", err)
|
|
||||||
}
|
|
||||||
if string(decoded) != "chunk content" {
|
|
||||||
t.Fatalf("chunk content = %q, want original content", decoded)
|
|
||||||
}
|
|
||||||
if got, want := chunkPayload.Chunks[0].Content.ContentDigest, contentDigest([]byte("chunk content")); got != want {
|
|
||||||
t.Fatalf("content digest = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestWorkspaceArtifactCheckpointsRoundTripCodecIdentityAndBytes(t *testing.T) {
|
func TestWorkspaceArtifactCheckpointsRoundTripCodecIdentityAndBytes(t *testing.T) {
|
||||||
@@ -127,24 +84,6 @@ func TestWorkspaceLoaderInvalidatesMissingCorruptAndMismatchedCheckpoints(t *tes
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("dependency mismatch", func(t *testing.T) {
|
|
||||||
root := t.TempDir()
|
|
||||||
recorder := newTestRecorder(t, root)
|
|
||||||
chunks := []source.Chunk{{
|
|
||||||
ID: "chunk-1",
|
|
||||||
SourceID: "source-1",
|
|
||||||
Content: []byte("chunk content"),
|
|
||||||
MediaType: "text/plain",
|
|
||||||
}}
|
|
||||||
if err := recorder.ChunkSucceeded("generic", "sha256:source-a", chunks, nil); err != nil {
|
|
||||||
t.Fatalf("ChunkSucceeded: %v", err)
|
|
||||||
}
|
|
||||||
loader := &WorkspaceLoader{root: root}
|
|
||||||
if _, decision := loader.Chunk("generic", "sha256:source-b"); decision.Reused || !strings.Contains(decision.Reason, "dependency") {
|
|
||||||
t.Fatalf("decision = %#v, want dependency invalidation", decision)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("incompatible workspace schema remains untouched", func(t *testing.T) {
|
t.Run("incompatible workspace schema remains untouched", func(t *testing.T) {
|
||||||
root := t.TempDir()
|
root := t.TempDir()
|
||||||
recorder := newTestRecorder(t, root)
|
recorder := newTestRecorder(t, root)
|
||||||
@@ -176,28 +115,41 @@ func TestWorkspaceLoaderInvalidatesMissingCorruptAndMismatchedCheckpoints(t *tes
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("corrupt payload", func(t *testing.T) {
|
}
|
||||||
|
|
||||||
|
func TestWorkspaceCheckpointsIgnoreLegacyChunkFiles(t *testing.T) {
|
||||||
root := t.TempDir()
|
root := t.TempDir()
|
||||||
|
legacyManifest := []byte(`{"legacy":"manifest"}`)
|
||||||
|
legacyPayload := []byte(`{"legacy":"chunks"}`)
|
||||||
|
legacyDir := filepath.Join(root, "chunk")
|
||||||
|
if err := os.MkdirAll(legacyDir, 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
manifestPath := filepath.Join(legacyDir, "manifest.json")
|
||||||
|
payloadPath := filepath.Join(legacyDir, "chunks.json")
|
||||||
|
if err := os.WriteFile(manifestPath, legacyManifest, 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(payloadPath, legacyPayload, 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
doc := &source.SourceDocument{ID: "source-1", Kind: "document", Format: "text/plain", Units: []source.SourceUnit{{ID: 1, Kind: "line", Text: "hello", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}}}}
|
||||||
|
doc.Digest, _ = source.DigestDocument(doc)
|
||||||
recorder := newTestRecorder(t, root)
|
recorder := newTestRecorder(t, root)
|
||||||
chunks := []source.Chunk{{
|
if err := recorder.SourceSucceeded("seriatim", doc); err != nil {
|
||||||
ID: "chunk-1",
|
t.Fatal(err)
|
||||||
SourceID: "source-1",
|
|
||||||
Content: []byte("chunk content"),
|
|
||||||
MediaType: "text/plain",
|
|
||||||
}}
|
|
||||||
if err := recorder.ChunkSucceeded("generic", "sha256:source", chunks, nil); err != nil {
|
|
||||||
t.Fatalf("ChunkSucceeded: %v", err)
|
|
||||||
}
|
}
|
||||||
payloadPath := filepath.Join(root, "chunk", "chunks.json")
|
loaded, decision := (&WorkspaceLoader{root: root}).Source("seriatim")
|
||||||
data := strings.ReplaceAll(string(readFile(t, payloadPath)), contentDigest([]byte("chunk content")), "sha256:bad")
|
if !decision.Reused || loaded.Document == nil || loaded.Document.Digest != doc.Digest {
|
||||||
if err := os.WriteFile(payloadPath, []byte(data), 0o644); err != nil {
|
t.Fatalf("source checkpoint = %#v decision = %#v", loaded, decision)
|
||||||
t.Fatalf("corrupt chunk payload: %v", err)
|
|
||||||
}
|
}
|
||||||
loader := &WorkspaceLoader{root: root}
|
if got := readFile(t, manifestPath); string(got) != string(legacyManifest) {
|
||||||
if _, decision := loader.Chunk("generic", "sha256:source"); decision.Reused || !strings.Contains(decision.Reason, "invalid") {
|
t.Fatalf("legacy manifest changed: %s", got)
|
||||||
t.Fatalf("decision = %#v, want corrupt payload invalidation", decision)
|
}
|
||||||
|
if got := readFile(t, payloadPath); string(got) != string(legacyPayload) {
|
||||||
|
t.Fatalf("legacy payload changed: %s", got)
|
||||||
}
|
}
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestWorkspaceRecorderRecordsRejectedExtractOutputs(t *testing.T) {
|
func TestWorkspaceRecorderRecordsRejectedExtractOutputs(t *testing.T) {
|
||||||
|
|||||||
62
internal/framework/chunkplan/import_boundaries_test.go
Normal file
62
internal/framework/chunkplan/import_boundaries_test.go
Normal file
@@ -0,0 +1,62 @@
|
|||||||
|
package chunkplan
|
||||||
|
|
||||||
|
import (
|
||||||
|
"go/parser"
|
||||||
|
"go/token"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
const repositoryImportPrefix = "gitea.maximumdirect.net/eric/notarius/internal/"
|
||||||
|
|
||||||
|
func TestPlanStoreAndSourceImportBoundaries(t *testing.T) {
|
||||||
|
repositoryRoot := repositoryRoot(t)
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
directory string
|
||||||
|
forbidden []string
|
||||||
|
}{
|
||||||
|
{name: "source is framework and module independent", directory: "internal/core/source", forbidden: []string{"framework/", "modules/"}},
|
||||||
|
{name: "plan store is module independent", directory: "internal/framework/chunkplan", forbidden: []string{"modules/"}},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
files, err := filepath.Glob(filepath.Join(repositoryRoot, tc.directory, "*.go"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, filename := range files {
|
||||||
|
if strings.HasSuffix(filename, "_test.go") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
parsed, err := parser.ParseFile(token.NewFileSet(), filename, nil, parser.ImportsOnly)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, item := range parsed.Imports {
|
||||||
|
path, err := strconv.Unquote(item.Path.Value)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
path = strings.TrimPrefix(path, repositoryImportPrefix)
|
||||||
|
for _, prefix := range tc.forbidden {
|
||||||
|
if strings.HasPrefix(path, prefix) {
|
||||||
|
t.Fatalf("%s imports %q, forbidden by %s boundary", filepath.Base(filename), path, tc.name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func repositoryRoot(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
_, filename, _, ok := runtime.Caller(0)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("resolve test location")
|
||||||
|
}
|
||||||
|
return filepath.Clean(filepath.Join(filepath.Dir(filename), "..", "..", ".."))
|
||||||
|
}
|
||||||
231
internal/framework/chunkplan/store.go
Normal file
231
internal/framework/chunkplan/store.go
Normal file
@@ -0,0 +1,231 @@
|
|||||||
|
package chunkplan
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
)
|
||||||
|
|
||||||
|
const SchemaVersion = pipeline.ChunkPlanSchemaVersion
|
||||||
|
|
||||||
|
type filesystemStore struct {
|
||||||
|
root string
|
||||||
|
write func(string, []byte) error
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewFilesystemStore(root string) (pipeline.ChunkPlanStore, error) {
|
||||||
|
root = strings.TrimSpace(root)
|
||||||
|
if root == "" {
|
||||||
|
return nil, fmt.Errorf("chunk plan root must not be empty")
|
||||||
|
}
|
||||||
|
if strings.ContainsRune(root, '\x00') {
|
||||||
|
return nil, fmt.Errorf("chunk plan root must not contain NUL")
|
||||||
|
}
|
||||||
|
return &filesystemStore{root: filepath.Clean(root), write: writeAtomic}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *filesystemStore) Load(sourceDigest string) (pipeline.ChunkPlanRecord, pipeline.ChunkPlanDecision, error) {
|
||||||
|
target, err := s.planPath(sourceDigest)
|
||||||
|
if err != nil {
|
||||||
|
return pipeline.ChunkPlanRecord{}, pipeline.ChunkPlanDecision{}, err
|
||||||
|
}
|
||||||
|
data, err := os.ReadFile(target)
|
||||||
|
if err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return pipeline.ChunkPlanRecord{}, pipeline.ChunkPlanDecision{Status: pipeline.ChunkPlanMissing, Reason: "chunk plan not found"}, nil
|
||||||
|
}
|
||||||
|
return pipeline.ChunkPlanRecord{}, pipeline.ChunkPlanDecision{}, fmt.Errorf("read chunk plan: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var record pipeline.ChunkPlanRecord
|
||||||
|
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||||
|
decoder.DisallowUnknownFields()
|
||||||
|
decoder.UseNumber()
|
||||||
|
if err := decoder.Decode(&record); err != nil {
|
||||||
|
return invalidDecision(fmt.Sprintf("decode stored chunk plan: %v", err))
|
||||||
|
}
|
||||||
|
var trailing any
|
||||||
|
if err := decoder.Decode(&trailing); err != io.EOF {
|
||||||
|
if err == nil {
|
||||||
|
return invalidDecision("stored chunk plan contains trailing JSON")
|
||||||
|
}
|
||||||
|
return invalidDecision(fmt.Sprintf("decode stored chunk plan trailer: %v", err))
|
||||||
|
}
|
||||||
|
if err := validateRecord(record, sourceDigest); err != nil {
|
||||||
|
return invalidDecision(err.Error())
|
||||||
|
}
|
||||||
|
return record, pipeline.ChunkPlanDecision{Status: pipeline.ChunkPlanHit, Reason: "stored chunk plan is valid"}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *filesystemStore) Save(record pipeline.ChunkPlanRecord) error {
|
||||||
|
if err := validateRecord(record, record.SourceDigest); err != nil {
|
||||||
|
return fmt.Errorf("validate chunk plan record: %w", err)
|
||||||
|
}
|
||||||
|
target, err := s.planPath(record.SourceDigest)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
data, err := json.Marshal(record)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("encode chunk plan record: %w", err)
|
||||||
|
}
|
||||||
|
data = append(data, '\n')
|
||||||
|
writer := s.write
|
||||||
|
if writer == nil {
|
||||||
|
writer = writeAtomic
|
||||||
|
}
|
||||||
|
if err := writer(target, data); err != nil {
|
||||||
|
return fmt.Errorf("write chunk plan: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *filesystemStore) planPath(sourceDigest string) (string, error) {
|
||||||
|
if s == nil || strings.TrimSpace(s.root) == "" {
|
||||||
|
return "", fmt.Errorf("chunk plan store must not be nil")
|
||||||
|
}
|
||||||
|
hexDigest, err := digestPathSegment(sourceDigest)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return filepath.Join(s.root, hexDigest, "plan.json"), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func digestPathSegment(digest string) (string, error) {
|
||||||
|
if !strings.HasPrefix(digest, "sha256:") {
|
||||||
|
return "", fmt.Errorf("source digest must use sha256:<64 lowercase hex> format")
|
||||||
|
}
|
||||||
|
hexDigest := strings.TrimPrefix(digest, "sha256:")
|
||||||
|
if len(hexDigest) != 64 || strings.ToLower(hexDigest) != hexDigest {
|
||||||
|
return "", fmt.Errorf("source digest must use sha256:<64 lowercase hex> format")
|
||||||
|
}
|
||||||
|
decoded, err := hex.DecodeString(hexDigest)
|
||||||
|
if err != nil || len(decoded) != 32 {
|
||||||
|
return "", fmt.Errorf("source digest must use sha256:<64 lowercase hex> format")
|
||||||
|
}
|
||||||
|
return hexDigest, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateRecord(record pipeline.ChunkPlanRecord, requestedDigest string) error {
|
||||||
|
if record.SchemaVersion != SchemaVersion {
|
||||||
|
return fmt.Errorf("schema_version %q is not supported", record.SchemaVersion)
|
||||||
|
}
|
||||||
|
if _, err := digestPathSegment(requestedDigest); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if record.SourceDigest != requestedDigest {
|
||||||
|
return fmt.Errorf("source_digest does not match requested source")
|
||||||
|
}
|
||||||
|
if record.Plan.SourceDigest != record.SourceDigest {
|
||||||
|
return fmt.Errorf("plan source_digest does not match record source_digest")
|
||||||
|
}
|
||||||
|
if len(record.Plan.Ranges) == 0 {
|
||||||
|
return fmt.Errorf("plan ranges must not be empty")
|
||||||
|
}
|
||||||
|
if err := source.ValidateChunkAnnotations(record.Plan.Annotations); err != nil {
|
||||||
|
return fmt.Errorf("plan annotations: %w", err)
|
||||||
|
}
|
||||||
|
seenStarts := make(map[int]struct{}, len(record.Plan.Ranges))
|
||||||
|
for i, chunkRange := range record.Plan.Ranges {
|
||||||
|
if chunkRange.StartUnitID <= 0 || chunkRange.EndUnitID <= 0 {
|
||||||
|
return fmt.Errorf("plan range[%d] boundaries must be positive", i)
|
||||||
|
}
|
||||||
|
if _, exists := seenStarts[chunkRange.StartUnitID]; exists {
|
||||||
|
return fmt.Errorf("plan range[%d] duplicates start_unit_id %d", i, chunkRange.StartUnitID)
|
||||||
|
}
|
||||||
|
seenStarts[chunkRange.StartUnitID] = struct{}{}
|
||||||
|
if err := source.ValidateChunkAnnotations(chunkRange.Annotations); err != nil {
|
||||||
|
return fmt.Errorf("plan range[%d] annotations: %w", i, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
wantDigest, err := source.DigestChunkPlan(record.Plan)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("digest plan: %w", err)
|
||||||
|
}
|
||||||
|
if record.PlanDigest != wantDigest {
|
||||||
|
return fmt.Errorf("plan_digest does not match plan")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(record.Producer.InputModule) == "" {
|
||||||
|
return fmt.Errorf("producer input_module must not be empty")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(record.Producer.ChunkModule) == "" {
|
||||||
|
return fmt.Errorf("producer chunk_module must not be empty")
|
||||||
|
}
|
||||||
|
if record.CreatedAt.IsZero() {
|
||||||
|
return fmt.Errorf("created_at must not be zero")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func invalidDecision(reason string) (pipeline.ChunkPlanRecord, pipeline.ChunkPlanDecision, error) {
|
||||||
|
return pipeline.ChunkPlanRecord{}, pipeline.ChunkPlanDecision{Status: pipeline.ChunkPlanInvalid, Reason: reason}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type atomicWriteHooks struct {
|
||||||
|
BeforeCreateTemp func() error
|
||||||
|
BeforeRename func() error
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeAtomic(target string, data []byte) error {
|
||||||
|
return writeAtomicWithHooks(target, data, atomicWriteHooks{})
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeAtomicWithHooks(target string, data []byte, hooks atomicWriteHooks) error {
|
||||||
|
dir := filepath.Dir(target)
|
||||||
|
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := os.Chmod(dir, 0o700); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if hooks.BeforeCreateTemp != nil {
|
||||||
|
if err := hooks.BeforeCreateTemp(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
temp, err := os.CreateTemp(dir, ".plan.json.tmp-*")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
tempPath := temp.Name()
|
||||||
|
removeTemp := true
|
||||||
|
defer func() {
|
||||||
|
if removeTemp {
|
||||||
|
_ = os.Remove(tempPath)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
if err := temp.Chmod(0o600); err != nil {
|
||||||
|
_ = temp.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := temp.Write(data); err != nil {
|
||||||
|
_ = temp.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := temp.Sync(); err != nil {
|
||||||
|
_ = temp.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := temp.Close(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if hooks.BeforeRename != nil {
|
||||||
|
if err := hooks.BeforeRename(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := os.Rename(tempPath, target); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
removeTemp = false
|
||||||
|
return nil
|
||||||
|
}
|
||||||
360
internal/framework/chunkplan/store_test.go
Normal file
360
internal/framework/chunkplan/store_test.go
Normal file
@@ -0,0 +1,360 @@
|
|||||||
|
package chunkplan
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
)
|
||||||
|
|
||||||
|
const testSourceDigest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||||
|
|
||||||
|
func TestFilesystemStoreRoundTripAndExactPath(t *testing.T) {
|
||||||
|
root := filepath.Join(t.TempDir(), "plans")
|
||||||
|
store := newStore(t, root)
|
||||||
|
record := testRecord(t, 1)
|
||||||
|
if err := store.Save(record); err != nil {
|
||||||
|
t.Fatalf("Save() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
target := filepath.Join(root, strings.TrimPrefix(testSourceDigest, "sha256:"), "plan.json")
|
||||||
|
data, err := os.ReadFile(target)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read exact plan path: %v", err)
|
||||||
|
}
|
||||||
|
if bytes.Contains(data, []byte("RAW REFERENCE CONTENT")) {
|
||||||
|
t.Fatal("stored record contains raw reference content")
|
||||||
|
}
|
||||||
|
var envelope struct {
|
||||||
|
Producer struct {
|
||||||
|
References []map[string]any `json:"references"`
|
||||||
|
} `json:"producer"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(data, &envelope); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(envelope.Producer.References) != 1 {
|
||||||
|
t.Fatalf("stored references = %#v", envelope.Producer.References)
|
||||||
|
}
|
||||||
|
if _, exists := envelope.Producer.References[0]["content"]; exists {
|
||||||
|
t.Fatalf("stored reference contains content field: %#v", envelope.Producer.References[0])
|
||||||
|
}
|
||||||
|
|
||||||
|
got, decision, err := store.Load(testSourceDigest)
|
||||||
|
if err != nil || decision.Status != pipeline.ChunkPlanHit {
|
||||||
|
t.Fatalf("Load() decision=%#v error=%v", decision, err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got, record) {
|
||||||
|
t.Fatalf("round trip record = %#v, want %#v", got, record)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFilesystemStorePermissions(t *testing.T) {
|
||||||
|
root := filepath.Join(t.TempDir(), "plans")
|
||||||
|
store := newStore(t, root)
|
||||||
|
if err := store.Save(testRecord(t, 1)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
digestDir := filepath.Join(root, strings.TrimPrefix(testSourceDigest, "sha256:"))
|
||||||
|
for path, want := range map[string]os.FileMode{
|
||||||
|
root: 0o700,
|
||||||
|
digestDir: 0o700,
|
||||||
|
filepath.Join(digestDir, "plan.json"): 0o600,
|
||||||
|
} {
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := info.Mode().Perm(); got != want {
|
||||||
|
t.Fatalf("%s mode = %04o, want %04o", path, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFilesystemStoreMissingAndOperationalErrors(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
store := newStore(t, root)
|
||||||
|
_, decision, err := store.Load(testSourceDigest)
|
||||||
|
if err != nil || decision.Status != pipeline.ChunkPlanMissing {
|
||||||
|
t.Fatalf("missing decision=%#v error=%v", decision, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
target := filepath.Join(root, strings.TrimPrefix(testSourceDigest, "sha256:"), "plan.json")
|
||||||
|
if err := os.MkdirAll(target, 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, _, err := store.Load(testSourceDigest); err == nil {
|
||||||
|
t.Fatal("Load(plan.json directory) error = nil")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFilesystemStoreRejectsMalformedSourceDigests(t *testing.T) {
|
||||||
|
store := newStore(t, t.TempDir())
|
||||||
|
for _, digest := range []string{"", "sha1:" + strings.Repeat("a", 64), "sha256:../escape", "sha256:" + strings.Repeat("A", 64), "sha256:" + strings.Repeat("a", 63)} {
|
||||||
|
t.Run(digest, func(t *testing.T) {
|
||||||
|
if _, _, err := store.Load(digest); err == nil {
|
||||||
|
t.Fatal("Load() error = nil")
|
||||||
|
}
|
||||||
|
record := testRecord(t, 1)
|
||||||
|
record.SourceDigest = digest
|
||||||
|
record.Plan.SourceDigest = digest
|
||||||
|
record.PlanDigest, _ = source.DigestChunkPlan(record.Plan)
|
||||||
|
if err := store.Save(record); err == nil {
|
||||||
|
t.Fatal("Save() error = nil")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFilesystemStoreReportsInvalidRecordsAsRecoverable(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
mutate func([]byte) []byte
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "unknown field", mutate: func(data []byte) []byte {
|
||||||
|
return bytes.Replace(data, []byte(`{"schema_version"`), []byte(`{"unknown":true,"schema_version"`), 1)
|
||||||
|
}, want: "unknown"},
|
||||||
|
{name: "truncated JSON", mutate: func(data []byte) []byte { return data[:len(data)/2] }, want: "decode"},
|
||||||
|
{name: "schema mismatch", mutate: replaceJSON(`notarius.chunk-plan.v1`, `notarius.chunk-plan.v2`), want: "schema_version"},
|
||||||
|
{name: "source mismatch", mutate: replaceJSON(testSourceDigest, "sha256:"+strings.Repeat("b", 64)), want: "source_digest"},
|
||||||
|
{name: "plan digest mismatch", mutate: func(data []byte) []byte {
|
||||||
|
prefix := []byte(`"plan_digest":"sha256:`)
|
||||||
|
index := bytes.Index(data, prefix)
|
||||||
|
if index >= 0 {
|
||||||
|
data[index+len(prefix)] = '0'
|
||||||
|
}
|
||||||
|
return data
|
||||||
|
}, want: "plan_digest"},
|
||||||
|
{name: "noncanonical annotation", mutate: func(data []byte) []byte {
|
||||||
|
return bytes.Replace(data, []byte(`{"value":1}`), []byte(`{ "value": 1 }`), 1)
|
||||||
|
}, want: "canonical JSON"},
|
||||||
|
{name: "bad boundary", mutate: func(data []byte) []byte {
|
||||||
|
return bytes.Replace(data, []byte(`"start_unit_id":1`), []byte(`"start_unit_id":0`), 1)
|
||||||
|
}, want: "boundaries must be positive"},
|
||||||
|
{name: "trailing JSON", mutate: func(data []byte) []byte { return append(data, []byte(` {}`)...) }, want: "trailing"},
|
||||||
|
}
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
store := newStore(t, root)
|
||||||
|
record := testRecord(t, 1)
|
||||||
|
if err := store.Save(record); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
path := filepath.Join(root, strings.TrimPrefix(testSourceDigest, "sha256:"), "plan.json")
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path, tc.mutate(data), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, decision, err := store.Load(testSourceDigest)
|
||||||
|
if err != nil || decision.Status != pipeline.ChunkPlanInvalid || !reflect.DeepEqual(got, pipeline.ChunkPlanRecord{}) || !strings.Contains(decision.Reason, tc.want) {
|
||||||
|
t.Fatalf("record=%#v decision=%#v error=%v", got, decision, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFilesystemStoreAtomicallyReplacesAndPreservesValidRecordOnFailure(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
store := newStore(t, root)
|
||||||
|
first := testRecord(t, 1)
|
||||||
|
second := testRecord(t, 2)
|
||||||
|
if err := store.Save(first); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
invalid := second
|
||||||
|
invalid.PlanDigest = "sha256:" + strings.Repeat("0", 64)
|
||||||
|
if err := store.Save(invalid); err == nil {
|
||||||
|
t.Fatal("Save(invalid) error = nil")
|
||||||
|
}
|
||||||
|
got, _, _ := store.Load(testSourceDigest)
|
||||||
|
if !reflect.DeepEqual(got, first) {
|
||||||
|
t.Fatalf("record after failed replacement = %#v", got)
|
||||||
|
}
|
||||||
|
if err := store.Save(second); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, _, _ = store.Load(testSourceDigest)
|
||||||
|
if !reflect.DeepEqual(got, second) {
|
||||||
|
t.Fatalf("record after replacement = %#v", got)
|
||||||
|
}
|
||||||
|
assertNoTemps(t, filepath.Join(root, strings.TrimPrefix(testSourceDigest, "sha256:")))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFilesystemStoreConcurrentWritersExposeCompleteRecord(t *testing.T) {
|
||||||
|
store := newStore(t, t.TempDir())
|
||||||
|
const writers = 24
|
||||||
|
records := make([]pipeline.ChunkPlanRecord, writers)
|
||||||
|
for i := range records {
|
||||||
|
records[i] = testRecord(t, i+1)
|
||||||
|
}
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
errs := make(chan error, writers)
|
||||||
|
for i := range records {
|
||||||
|
wg.Add(1)
|
||||||
|
go func(record pipeline.ChunkPlanRecord) {
|
||||||
|
defer wg.Done()
|
||||||
|
errs <- store.Save(record)
|
||||||
|
}(records[i])
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
close(errs)
|
||||||
|
for err := range errs {
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("concurrent Save() error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
got, decision, err := store.Load(testSourceDigest)
|
||||||
|
if err != nil || decision.Status != pipeline.ChunkPlanHit {
|
||||||
|
t.Fatalf("Load() decision=%#v error=%v", decision, err)
|
||||||
|
}
|
||||||
|
var annotation struct {
|
||||||
|
Value int `json:"value"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(got.Plan.Annotations["test/value"], &annotation); err != nil || annotation.Value < 1 || annotation.Value > writers {
|
||||||
|
t.Fatalf("final annotation=%#v error=%v", annotation, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFilesystemStoreReadersObserveOnlyCompleteRecordsDuringWrites(t *testing.T) {
|
||||||
|
store := newStore(t, t.TempDir())
|
||||||
|
if err := store.Save(testRecord(t, 1)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
const writers = 12
|
||||||
|
const readers = 12
|
||||||
|
errs := make(chan error, writers+readers)
|
||||||
|
start := make(chan struct{})
|
||||||
|
var writersDone sync.WaitGroup
|
||||||
|
for i := 0; i < writers; i++ {
|
||||||
|
writersDone.Add(1)
|
||||||
|
go func(value int) {
|
||||||
|
defer writersDone.Done()
|
||||||
|
<-start
|
||||||
|
errs <- store.Save(testRecord(t, value+2))
|
||||||
|
}(i)
|
||||||
|
}
|
||||||
|
for i := 0; i < readers; i++ {
|
||||||
|
go func() {
|
||||||
|
<-start
|
||||||
|
for attempt := 0; attempt < 50; attempt++ {
|
||||||
|
record, decision, err := store.Load(testSourceDigest)
|
||||||
|
if err != nil || decision.Status != pipeline.ChunkPlanHit {
|
||||||
|
errs <- fmt.Errorf("Load() decision=%#v error=%v", decision, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := validateRecord(record, testSourceDigest); err != nil {
|
||||||
|
errs <- fmt.Errorf("reader observed invalid record: %w", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
errs <- nil
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
close(start)
|
||||||
|
writersDone.Wait()
|
||||||
|
for i := 0; i < readers+writers; i++ {
|
||||||
|
if err := <-errs; err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFilesystemStoreInterruptedWritesPreservePreviousRecord(t *testing.T) {
|
||||||
|
store := newStore(t, t.TempDir()).(*filesystemStore)
|
||||||
|
first := testRecord(t, 1)
|
||||||
|
if err := store.Save(first); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
hooks atomicWriteHooks
|
||||||
|
}{
|
||||||
|
{name: "before temporary file", hooks: atomicWriteHooks{BeforeCreateTemp: func() error { return errors.New("interrupted before temporary file") }}},
|
||||||
|
{name: "before rename", hooks: atomicWriteHooks{BeforeRename: func() error { return errors.New("interrupted before rename") }}},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
store.write = func(target string, data []byte) error { return writeAtomicWithHooks(target, data, tc.hooks) }
|
||||||
|
if err := store.Save(testRecord(t, 2)); err == nil {
|
||||||
|
t.Fatal("Save() error = nil")
|
||||||
|
}
|
||||||
|
store.write = writeAtomic
|
||||||
|
got, decision, err := store.Load(testSourceDigest)
|
||||||
|
if err != nil || decision.Status != pipeline.ChunkPlanHit || !reflect.DeepEqual(got, first) {
|
||||||
|
t.Fatalf("record after interruption=%#v decision=%#v error=%v", got, decision, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newStore(t *testing.T, root string) pipeline.ChunkPlanStore {
|
||||||
|
t.Helper()
|
||||||
|
store, err := NewFilesystemStore(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewFilesystemStore() error = %v", err)
|
||||||
|
}
|
||||||
|
return store
|
||||||
|
}
|
||||||
|
|
||||||
|
func testRecord(t *testing.T, value int) pipeline.ChunkPlanRecord {
|
||||||
|
t.Helper()
|
||||||
|
annotation, err := json.Marshal(map[string]int{"value": value})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
plan := source.ChunkPlan{
|
||||||
|
SourceDigest: testSourceDigest,
|
||||||
|
Ranges: []source.ChunkRange{{StartUnitID: 1, EndUnitID: 2, Annotations: source.ChunkAnnotations{"test/range": json.RawMessage(`{"range":true}`)}}},
|
||||||
|
Annotations: source.ChunkAnnotations{"test/value": annotation},
|
||||||
|
}
|
||||||
|
planDigest, err := source.DigestChunkPlan(plan)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return pipeline.ChunkPlanRecord{
|
||||||
|
SchemaVersion: SchemaVersion,
|
||||||
|
SourceDigest: testSourceDigest,
|
||||||
|
PlanDigest: planDigest,
|
||||||
|
Plan: plan,
|
||||||
|
Producer: pipeline.ChunkPlanProducer{
|
||||||
|
InputModule: "input/test", ChunkModule: "chunk/test", LLMProfile: "profile/test",
|
||||||
|
References: []artifacts.ReferenceProvenance{{Stage: "chunk", SlotName: "guide", OriginType: "file", OriginURI: "file:///guide.txt", Digest: "sha256:reference"}},
|
||||||
|
Metadata: map[string]any{"prompt_id": "test/prompt", "enabled": true},
|
||||||
|
},
|
||||||
|
Warnings: []contracts.Warning{{Scope: "chunk/test", ReasonCode: "observed", Message: "warning"}},
|
||||||
|
CreatedAt: time.Date(2026, 7, 18, 12, 0, 0, 0, time.UTC),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func replaceJSON(old, replacement string) func([]byte) []byte {
|
||||||
|
return func(data []byte) []byte { return bytes.Replace(data, []byte(old), []byte(replacement), 1) }
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertNoTemps(t *testing.T, dir string) {
|
||||||
|
t.Helper()
|
||||||
|
entries, err := os.ReadDir(dir)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, entry := range entries {
|
||||||
|
if strings.Contains(entry.Name(), ".tmp-") {
|
||||||
|
t.Fatalf("temporary file remains: %s", entry.Name())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -146,15 +146,15 @@ type ChunkRequest struct {
|
|||||||
Metadata map[string]any `json:"metadata,omitempty"`
|
Metadata map[string]any `json:"metadata,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type ChunkResult struct {
|
type ChunkPlanResult struct {
|
||||||
Chunks []source.Chunk `json:"chunks"`
|
Plan source.ChunkPlan `json:"plan"`
|
||||||
Warnings []Warning `json:"warnings,omitempty"`
|
Warnings []Warning `json:"warnings,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type Chunker interface {
|
type Chunker interface {
|
||||||
Key() string
|
Key() string
|
||||||
ReferenceSlots() []ReferenceSlot
|
ReferenceSlots() []ReferenceSlot
|
||||||
Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error)
|
Plan(ctx context.Context, req ChunkRequest) (ChunkPlanResult, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -214,6 +214,10 @@ const (
|
|||||||
ExecutionClassLLMBacked ExecutionClass = "llm_backed"
|
ExecutionClassLLMBacked ExecutionClass = "llm_backed"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type ChunkExecutionClassProvider interface {
|
||||||
|
ExecutionClass() ExecutionClass
|
||||||
|
}
|
||||||
|
|
||||||
type ValidationResult struct {
|
type ValidationResult struct {
|
||||||
Approved bool `json:"approved"`
|
Approved bool `json:"approved"`
|
||||||
ReasonCode string `json:"reason_code,omitempty"`
|
ReasonCode string `json:"reason_code,omitempty"`
|
||||||
|
|||||||
@@ -45,7 +45,7 @@ func TestFakeExtractorReturnsTypedOutput(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFakeChunkerReturnsSourceChunks(t *testing.T) {
|
func TestFakeChunkerReturnsSourcePlan(t *testing.T) {
|
||||||
doc := &source.SourceDocument{
|
doc := &source.SourceDocument{
|
||||||
ID: "source-1",
|
ID: "source-1",
|
||||||
Kind: "document",
|
Kind: "document",
|
||||||
@@ -57,36 +57,19 @@ func TestFakeChunkerReturnsSourceChunks(t *testing.T) {
|
|||||||
}
|
}
|
||||||
chunker := fakeChunker{key: "generic-chunker"}
|
chunker := fakeChunker{key: "generic-chunker"}
|
||||||
|
|
||||||
result, err := chunker.Chunk(context.Background(), ChunkRequest{Source: doc})
|
result, err := chunker.Plan(context.Background(), ChunkRequest{Source: doc})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Chunk() error = %v, want nil", err)
|
t.Fatalf("Plan() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if chunker.Key() != "generic-chunker" {
|
if chunker.Key() != "generic-chunker" {
|
||||||
t.Fatalf("Key() = %q, want generic-chunker", chunker.Key())
|
t.Fatalf("Key() = %q, want generic-chunker", chunker.Key())
|
||||||
}
|
}
|
||||||
if len(result.Chunks) != 1 {
|
if len(result.Plan.Ranges) != 1 {
|
||||||
t.Fatalf("len(Chunks) = %d, want 1", len(result.Chunks))
|
t.Fatalf("len(Ranges) = %d, want 1", len(result.Plan.Ranges))
|
||||||
}
|
}
|
||||||
|
if result.Plan.SourceDigest != doc.Digest || result.Plan.Ranges[0].StartUnitID != 1 || result.Plan.Ranges[0].EndUnitID != 1 {
|
||||||
chunk := result.Chunks[0]
|
t.Fatalf("Plan = %#v, want source digest and unit range", result.Plan)
|
||||||
if chunk.ID != "source-1:chunk:0" {
|
|
||||||
t.Fatalf("source.Chunk.ID = %q, want source-1:chunk:0", chunk.ID)
|
|
||||||
}
|
|
||||||
if chunk.SourceID != doc.ID {
|
|
||||||
t.Fatalf("source.Chunk.SourceID = %q, want %q", chunk.SourceID, doc.ID)
|
|
||||||
}
|
|
||||||
if chunk.Index != 0 {
|
|
||||||
t.Fatalf("source.Chunk.Index = %d, want 0", chunk.Index)
|
|
||||||
}
|
|
||||||
if chunk.Ref.StartUnitID != 1 || chunk.Ref.EndUnitID != 1 {
|
|
||||||
t.Fatalf("source.Chunk.Ref = %#v, want source-1:1-1", chunk.Ref)
|
|
||||||
}
|
|
||||||
if chunk.MediaType != "application/json" || string(chunk.Content) != `{"units":[{"id":1,"kind":"section","text":"Source text."}]}` {
|
|
||||||
t.Fatalf("source.Chunk payload = %q %s, want JSON units", chunk.MediaType, chunk.Content)
|
|
||||||
}
|
|
||||||
if len(chunk.Units) != 1 {
|
|
||||||
t.Fatalf("len(source.Chunk.Units) = %d, want 1", len(chunk.Units))
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -102,8 +85,8 @@ func TestFakeChunkerReceivesPerRunContext(t *testing.T) {
|
|||||||
}
|
}
|
||||||
chunker := &recordingChunker{key: "llm-chunker"}
|
chunker := &recordingChunker{key: "llm-chunker"}
|
||||||
|
|
||||||
if _, err := chunker.Chunk(context.Background(), ChunkRequest{Source: doc, SessionID: "session", LLMProfile: "profile"}); err != nil {
|
if _, err := chunker.Plan(context.Background(), ChunkRequest{Source: doc, SessionID: "session", LLMProfile: "profile"}); err != nil {
|
||||||
t.Fatalf("Chunk() error = %v, want nil", err)
|
t.Fatalf("Plan() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
if chunker.request.SessionID != "session" || chunker.request.LLMProfile != "profile" {
|
if chunker.request.SessionID != "session" || chunker.request.LLMProfile != "profile" {
|
||||||
t.Fatalf("ChunkRequest = %#v, want per-run session and profile", chunker.request)
|
t.Fatalf("ChunkRequest = %#v, want per-run session and profile", chunker.request)
|
||||||
@@ -446,22 +429,14 @@ func (chunker fakeChunker) ReferenceSlots() []ReferenceSlot {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (chunker fakeChunker) Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error) {
|
func (chunker fakeChunker) Plan(ctx context.Context, req ChunkRequest) (ChunkPlanResult, error) {
|
||||||
return ChunkResult{
|
return ChunkPlanResult{
|
||||||
Chunks: []source.Chunk{
|
Plan: source.ChunkPlan{
|
||||||
{
|
SourceDigest: req.Source.Digest,
|
||||||
ID: req.Source.ID + ":chunk:0",
|
Ranges: []source.ChunkRange{{
|
||||||
SourceID: req.Source.ID,
|
|
||||||
Index: 0,
|
|
||||||
Ref: source.SourceRef{
|
|
||||||
SourceID: req.Source.ID,
|
|
||||||
StartUnitID: req.Source.Units[0].ID,
|
StartUnitID: req.Source.Units[0].ID,
|
||||||
EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID,
|
EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID,
|
||||||
},
|
}},
|
||||||
Content: []byte(`{"units":[{"id":1,"kind":"section","text":"Source text."}]}`),
|
|
||||||
MediaType: "application/json",
|
|
||||||
Units: append([]source.SourceUnit(nil), req.Source.Units...),
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
@@ -479,9 +454,9 @@ func (chunker *recordingChunker) ReferenceSlots() []ReferenceSlot {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (chunker *recordingChunker) Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error) {
|
func (chunker *recordingChunker) Plan(ctx context.Context, req ChunkRequest) (ChunkPlanResult, error) {
|
||||||
chunker.request = req
|
chunker.request = req
|
||||||
return fakeChunker{key: chunker.key}.Chunk(ctx, req)
|
return fakeChunker{key: chunker.key}.Plan(ctx, req)
|
||||||
}
|
}
|
||||||
|
|
||||||
type fakeExtractor struct {
|
type fakeExtractor struct {
|
||||||
|
|||||||
@@ -20,10 +20,6 @@ type CheckpointRecorder interface {
|
|||||||
SourceRunning(moduleKey string) error
|
SourceRunning(moduleKey string) error
|
||||||
SourceSucceeded(moduleKey string, doc *source.SourceDocument) error
|
SourceSucceeded(moduleKey string, doc *source.SourceDocument) error
|
||||||
SourceFailed(moduleKey string, err error) error
|
SourceFailed(moduleKey string, err error) error
|
||||||
ChunkRunning(moduleKey string, sourceDigest string) error
|
|
||||||
ChunkSucceeded(moduleKey string, sourceDigest string, chunks []source.Chunk, warnings []contracts.Warning) error
|
|
||||||
ChunkRejected(moduleKey string, sourceDigest string, rejected contracts.RejectedOutput) error
|
|
||||||
ChunkFailed(moduleKey string, sourceDigest string, err error) error
|
|
||||||
ExtractRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
|
ExtractRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
|
||||||
ExtractSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error
|
ExtractSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error
|
||||||
ExtractFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
|
ExtractFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
|
||||||
@@ -54,11 +50,6 @@ type SourceCheckpoint struct {
|
|||||||
Document *source.SourceDocument
|
Document *source.SourceDocument
|
||||||
}
|
}
|
||||||
|
|
||||||
type ChunkCheckpoint struct {
|
|
||||||
Chunks []source.Chunk
|
|
||||||
Warnings []contracts.Warning
|
|
||||||
}
|
|
||||||
|
|
||||||
// CheckpointArtifact is the durable, domain-neutral value stored at a lane
|
// CheckpointArtifact is the durable, domain-neutral value stored at a lane
|
||||||
// checkpoint boundary.
|
// checkpoint boundary.
|
||||||
type CheckpointArtifact struct {
|
type CheckpointArtifact struct {
|
||||||
@@ -90,7 +81,6 @@ type NormalizeCheckpoint struct {
|
|||||||
type CheckpointLoader interface {
|
type CheckpointLoader interface {
|
||||||
Enabled() bool
|
Enabled() bool
|
||||||
Source(moduleKey string) (SourceCheckpoint, CheckpointDecision)
|
Source(moduleKey string) (SourceCheckpoint, CheckpointDecision)
|
||||||
Chunk(moduleKey string, sourceDigest string) (ChunkCheckpoint, CheckpointDecision)
|
|
||||||
Extract(laneID string, moduleKey string, dependencies []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision)
|
Extract(laneID string, moduleKey string, dependencies []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision)
|
||||||
Merge(laneID string, moduleKey string, dependencies []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision)
|
Merge(laneID string, moduleKey string, dependencies []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision)
|
||||||
Normalize(laneID string, moduleKey string, dependencies []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision)
|
Normalize(laneID string, moduleKey string, dependencies []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision)
|
||||||
@@ -105,14 +95,6 @@ func NoopCheckpointLoader() CheckpointLoader { return noopCheckpointLoader{}
|
|||||||
func (noopCheckpointRecorder) SourceRunning(string) error { return nil }
|
func (noopCheckpointRecorder) SourceRunning(string) error { return nil }
|
||||||
func (noopCheckpointRecorder) SourceSucceeded(string, *source.SourceDocument) error { return nil }
|
func (noopCheckpointRecorder) SourceSucceeded(string, *source.SourceDocument) error { return nil }
|
||||||
func (noopCheckpointRecorder) SourceFailed(string, error) error { return nil }
|
func (noopCheckpointRecorder) SourceFailed(string, error) error { return nil }
|
||||||
func (noopCheckpointRecorder) ChunkRunning(string, string) error { return nil }
|
|
||||||
func (noopCheckpointRecorder) ChunkSucceeded(string, string, []source.Chunk, []contracts.Warning) error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
func (noopCheckpointRecorder) ChunkRejected(string, string, contracts.RejectedOutput) error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
func (noopCheckpointRecorder) ChunkFailed(string, string, error) error { return nil }
|
|
||||||
func (noopCheckpointRecorder) ExtractRunning(string, string, []CheckpointFingerprint) error {
|
func (noopCheckpointRecorder) ExtractRunning(string, string, []CheckpointFingerprint) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -149,9 +131,6 @@ func (noopCheckpointLoader) Enabled() bool { return false }
|
|||||||
func (noopCheckpointLoader) Source(string) (SourceCheckpoint, CheckpointDecision) {
|
func (noopCheckpointLoader) Source(string) (SourceCheckpoint, CheckpointDecision) {
|
||||||
return SourceCheckpoint{}, CheckpointDecision{Reason: "checkpoint loading disabled"}
|
return SourceCheckpoint{}, CheckpointDecision{Reason: "checkpoint loading disabled"}
|
||||||
}
|
}
|
||||||
func (noopCheckpointLoader) Chunk(string, string) (ChunkCheckpoint, CheckpointDecision) {
|
|
||||||
return ChunkCheckpoint{}, CheckpointDecision{Reason: "checkpoint loading disabled"}
|
|
||||||
}
|
|
||||||
func (noopCheckpointLoader) Extract(string, string, []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
|
func (noopCheckpointLoader) Extract(string, string, []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
|
||||||
return ExtractCheckpoint{}, CheckpointDecision{Reason: "checkpoint loading disabled"}
|
return ExtractCheckpoint{}, CheckpointDecision{Reason: "checkpoint loading disabled"}
|
||||||
}
|
}
|
||||||
|
|||||||
72
internal/framework/pipeline/chunk_annotations_test.go
Normal file
72
internal/framework/pipeline/chunk_annotations_test.go
Normal file
@@ -0,0 +1,72 @@
|
|||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestChunkCanonicalizationAndClonePreserveAnnotationScopes(t *testing.T) {
|
||||||
|
doc := validSourceDocument()
|
||||||
|
plan := source.ChunkPlan{
|
||||||
|
SourceDigest: doc.Digest,
|
||||||
|
Ranges: []source.ChunkRange{{
|
||||||
|
StartUnitID: doc.Units[0].ID, EndUnitID: doc.Units[0].ID,
|
||||||
|
Annotations: source.ChunkAnnotations{"shared": json.RawMessage(` {"range": true} `)},
|
||||||
|
}},
|
||||||
|
Annotations: source.ChunkAnnotations{"shared": json.RawMessage(` {"plan": true} `)},
|
||||||
|
}
|
||||||
|
canonical, chunks, err := validateAndMaterializeChunkPlan(doc, plan)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("validateAndMaterializeChunkPlan() error = %v", err)
|
||||||
|
}
|
||||||
|
if got := string(canonical.Ranges[0].Annotations["shared"]); got != `{"range":true}` {
|
||||||
|
t.Fatalf("range annotation = %q", got)
|
||||||
|
}
|
||||||
|
if got := string(canonical.Annotations["shared"]); got != `{"plan":true}` {
|
||||||
|
t.Fatalf("plan annotation = %q", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
cloned := cloneSourceChunk(chunks[0])
|
||||||
|
cloned.Annotations["shared"][0] = '['
|
||||||
|
cloned.PlanAnnotations["shared"][0] = '['
|
||||||
|
if string(chunks[0].Annotations["shared"]) != `{"range":true}` || string(chunks[0].PlanAnnotations["shared"]) != `{"plan":true}` {
|
||||||
|
t.Fatal("cloneSourceChunk() shares annotation bytes")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSerializedChunkValidationRepresentationIncludesAnnotationScopes(t *testing.T) {
|
||||||
|
doc := validSourceDocument()
|
||||||
|
chunks := []source.Chunk{{
|
||||||
|
ID: "chunk-1", SourceID: doc.ID, Ref: doc.Units[0].Ref, MediaType: "application/json", Units: doc.Units[:1],
|
||||||
|
Annotations: source.ChunkAnnotations{"same": json.RawMessage(`{"range":1}`)},
|
||||||
|
PlanAnnotations: source.ChunkAnnotations{"same": json.RawMessage(`{"plan":2}`)},
|
||||||
|
}}
|
||||||
|
encoded, err := json.Marshal(chunks)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("json.Marshal(chunks) error = %v", err)
|
||||||
|
}
|
||||||
|
got := string(encoded)
|
||||||
|
if !strings.Contains(got, `"annotations":{"same":{"range":1}}`) || !strings.Contains(got, `"plan_annotations":{"same":{"plan":2}}`) {
|
||||||
|
t.Fatalf("serialized chunks = %s, want both annotation scopes", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDebugChunkEnvelopeClonesAnnotationScopes(t *testing.T) {
|
||||||
|
chunk := source.Chunk{
|
||||||
|
ID: "chunk-1", SourceID: "source-1",
|
||||||
|
Annotations: source.ChunkAnnotations{"same": json.RawMessage(`{"range":1}`)},
|
||||||
|
PlanAnnotations: source.ChunkAnnotations{"same": json.RawMessage(`{"plan":2}`)},
|
||||||
|
}
|
||||||
|
envelope := debugSourceChunkEnvelope(chunk)
|
||||||
|
if string(envelope.Annotations["same"]) != `{"range":1}` || string(envelope.PlanAnnotations["same"]) != `{"plan":2}` {
|
||||||
|
t.Fatalf("debug annotations = %#v / %#v", envelope.Annotations, envelope.PlanAnnotations)
|
||||||
|
}
|
||||||
|
envelope.Annotations["same"][0] = '['
|
||||||
|
envelope.PlanAnnotations["same"][0] = '['
|
||||||
|
if string(chunk.Annotations["same"]) != `{"range":1}` || string(chunk.PlanAnnotations["same"]) != `{"plan":2}` {
|
||||||
|
t.Fatal("debugSourceChunkEnvelope() shares annotation bytes")
|
||||||
|
}
|
||||||
|
}
|
||||||
29
internal/framework/pipeline/chunk_cache.go
Normal file
29
internal/framework/pipeline/chunk_cache.go
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
type ChunkCacheMode string
|
||||||
|
|
||||||
|
const (
|
||||||
|
ChunkCacheAuto ChunkCacheMode = "auto"
|
||||||
|
ChunkCacheBypass ChunkCacheMode = "bypass"
|
||||||
|
ChunkCacheRefresh ChunkCacheMode = "refresh"
|
||||||
|
)
|
||||||
|
|
||||||
|
func ParseChunkCacheMode(raw string) (ChunkCacheMode, error) {
|
||||||
|
mode := ChunkCacheMode(strings.TrimSpace(raw))
|
||||||
|
switch mode {
|
||||||
|
case ChunkCacheAuto, ChunkCacheBypass, ChunkCacheRefresh:
|
||||||
|
return mode, nil
|
||||||
|
default:
|
||||||
|
return "", fmt.Errorf("chunk cache mode %q is not supported", strings.TrimSpace(raw))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m ChunkCacheMode) Validate() error {
|
||||||
|
_, err := ParseChunkCacheMode(string(m))
|
||||||
|
return err
|
||||||
|
}
|
||||||
20
internal/framework/pipeline/chunk_cache_test.go
Normal file
20
internal/framework/pipeline/chunk_cache_test.go
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
package pipeline
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestParseChunkCacheMode(t *testing.T) {
|
||||||
|
for _, value := range []ChunkCacheMode{ChunkCacheAuto, ChunkCacheBypass, ChunkCacheRefresh} {
|
||||||
|
got, err := ParseChunkCacheMode(string(value))
|
||||||
|
if err != nil || got != value {
|
||||||
|
t.Fatalf("ParseChunkCacheMode(%q) = %q, %v", value, got, err)
|
||||||
|
}
|
||||||
|
if err := value.Validate(); err != nil {
|
||||||
|
t.Fatalf("%q.Validate() error = %v", value, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, value := range []string{"", "enabled", "AUTO", "auto,refresh"} {
|
||||||
|
if _, err := ParseChunkCacheMode(value); err == nil {
|
||||||
|
t.Fatalf("ParseChunkCacheMode(%q) error = nil, want error", value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
49
internal/framework/pipeline/chunk_plan_store.go
Normal file
49
internal/framework/pipeline/chunk_plan_store.go
Normal file
@@ -0,0 +1,49 @@
|
|||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
)
|
||||||
|
|
||||||
|
const ChunkPlanSchemaVersion = "notarius.chunk-plan.v1"
|
||||||
|
|
||||||
|
type ChunkPlanProducer struct {
|
||||||
|
InputModule string `json:"input_module"`
|
||||||
|
ChunkModule string `json:"chunk_module"`
|
||||||
|
LLMProfile string `json:"llm_profile,omitempty"`
|
||||||
|
References []artifacts.ReferenceProvenance `json:"references,omitempty"`
|
||||||
|
Metadata map[string]any `json:"metadata,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ChunkPlanRecord struct {
|
||||||
|
SchemaVersion string `json:"schema_version"`
|
||||||
|
SourceDigest string `json:"source_digest"`
|
||||||
|
PlanDigest string `json:"plan_digest"`
|
||||||
|
Plan source.ChunkPlan `json:"plan"`
|
||||||
|
Producer ChunkPlanProducer `json:"producer"`
|
||||||
|
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
||||||
|
CreatedAt time.Time `json:"created_at"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ChunkPlanStore interface {
|
||||||
|
Load(sourceDigest string) (ChunkPlanRecord, ChunkPlanDecision, error)
|
||||||
|
Save(ChunkPlanRecord) error
|
||||||
|
}
|
||||||
|
|
||||||
|
type ChunkPlanStoreFactory func(root string) (ChunkPlanStore, error)
|
||||||
|
|
||||||
|
type ChunkPlanDecision struct {
|
||||||
|
Status ChunkPlanStatus `json:"status"`
|
||||||
|
Reason string `json:"reason,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ChunkPlanStatus string
|
||||||
|
|
||||||
|
const (
|
||||||
|
ChunkPlanHit ChunkPlanStatus = "hit"
|
||||||
|
ChunkPlanMissing ChunkPlanStatus = "missing"
|
||||||
|
ChunkPlanInvalid ChunkPlanStatus = "invalid"
|
||||||
|
)
|
||||||
@@ -2,100 +2,23 @@ package pipeline
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
)
|
)
|
||||||
|
|
||||||
func validateAndCanonicalizeChunkResult(doc *source.SourceDocument, chunks []source.Chunk) ([]source.Chunk, error) {
|
func validateAndMaterializeChunkPlan(doc *source.SourceDocument, plan source.ChunkPlan) (source.ChunkPlan, []source.Chunk, error) {
|
||||||
if len(chunks) == 0 {
|
canonical, err := source.CanonicalizeChunkPlan(plan)
|
||||||
return nil, fmt.Errorf("chunks must not be empty")
|
if err != nil {
|
||||||
|
return source.ChunkPlan{}, nil, fmt.Errorf("canonicalize chunk plan: %w", err)
|
||||||
}
|
}
|
||||||
|
if err := source.ValidateChunkPlan(doc, canonical); err != nil {
|
||||||
sourceUnitIndexes := make(map[int]int, len(doc.Units))
|
return source.ChunkPlan{}, nil, err
|
||||||
sourceUnits := make(map[int]source.SourceUnit, len(doc.Units))
|
|
||||||
for index, unit := range doc.Units {
|
|
||||||
sourceUnitIndexes[unit.ID] = index
|
|
||||||
sourceUnits[unit.ID] = unit
|
|
||||||
}
|
}
|
||||||
|
chunks, err := source.MaterializeChunkPlan(doc, canonical)
|
||||||
canonicalChunks := make([]source.Chunk, 0, len(chunks))
|
if err != nil {
|
||||||
seenChunkIDs := make(map[string]struct{}, len(chunks))
|
return source.ChunkPlan{}, nil, err
|
||||||
for chunkIndex, chunk := range chunks {
|
|
||||||
if strings.TrimSpace(chunk.ID) == "" {
|
|
||||||
return nil, fmt.Errorf("chunk[%d].id must not be empty", chunkIndex)
|
|
||||||
}
|
}
|
||||||
if _, ok := seenChunkIDs[chunk.ID]; ok {
|
return canonical, chunks, nil
|
||||||
return nil, fmt.Errorf("chunk id %q is duplicated", chunk.ID)
|
|
||||||
}
|
|
||||||
seenChunkIDs[chunk.ID] = struct{}{}
|
|
||||||
|
|
||||||
if chunk.SourceID != doc.ID {
|
|
||||||
return nil, fmt.Errorf("chunk %q source_id %q does not match source document id %q", chunk.ID, chunk.SourceID, doc.ID)
|
|
||||||
}
|
|
||||||
if chunk.Index != chunkIndex {
|
|
||||||
return nil, fmt.Errorf("chunk %q index %d does not match returned order %d", chunk.ID, chunk.Index, chunkIndex)
|
|
||||||
}
|
|
||||||
if len(chunk.Units) == 0 {
|
|
||||||
return nil, fmt.Errorf("chunk %q units must not be empty", chunk.ID)
|
|
||||||
}
|
|
||||||
if len(chunk.Content) == 0 {
|
|
||||||
return nil, fmt.Errorf("chunk %q content must not be empty", chunk.ID)
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(chunk.MediaType) == "" {
|
|
||||||
return nil, fmt.Errorf("chunk %q media_type must not be empty", chunk.ID)
|
|
||||||
}
|
|
||||||
if err := source.ValidateRef(doc, chunk.Ref); err != nil {
|
|
||||||
return nil, fmt.Errorf("chunk %q ref: %w", chunk.ID, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
seenUnitIDs := make(map[int]struct{}, len(chunk.Units))
|
|
||||||
previousSourceIndex := -1
|
|
||||||
canonicalUnits := make([]source.SourceUnit, 0, len(chunk.Units))
|
|
||||||
for unitIndex, unit := range chunk.Units {
|
|
||||||
if unit.ID <= 0 {
|
|
||||||
return nil, fmt.Errorf("chunk %q unit[%d].id must be positive", chunk.ID, unitIndex)
|
|
||||||
}
|
|
||||||
if _, ok := seenUnitIDs[unit.ID]; ok {
|
|
||||||
return nil, fmt.Errorf("chunk %q repeats source unit %d", chunk.ID, unit.ID)
|
|
||||||
}
|
|
||||||
seenUnitIDs[unit.ID] = struct{}{}
|
|
||||||
|
|
||||||
sourceIndex, ok := sourceUnitIndexes[unit.ID]
|
|
||||||
if !ok {
|
|
||||||
return nil, fmt.Errorf("chunk %q source unit %d was not found in source document %q", chunk.ID, unit.ID, doc.ID)
|
|
||||||
}
|
|
||||||
if previousSourceIndex >= 0 && sourceIndex != previousSourceIndex+1 {
|
|
||||||
return nil, fmt.Errorf("chunk %q source units must form a contiguous range in source document order", chunk.ID)
|
|
||||||
}
|
|
||||||
if unit.Ref != sourceUnits[unit.ID].Ref {
|
|
||||||
return nil, fmt.Errorf("chunk %q source unit %d ref does not match source document", chunk.ID, unit.ID)
|
|
||||||
}
|
|
||||||
previousSourceIndex = sourceIndex
|
|
||||||
canonicalUnits = append(canonicalUnits, cloneSourceUnit(sourceUnits[unit.ID]))
|
|
||||||
}
|
|
||||||
expectedRef := source.SourceRef{
|
|
||||||
SourceID: doc.ID,
|
|
||||||
StartUnitID: canonicalUnits[0].Ref.StartUnitID,
|
|
||||||
EndUnitID: canonicalUnits[len(canonicalUnits)-1].Ref.EndUnitID,
|
|
||||||
}
|
|
||||||
if chunk.Ref != expectedRef {
|
|
||||||
return nil, fmt.Errorf("chunk %q ref %#v does not match unit span %#v", chunk.ID, chunk.Ref, expectedRef)
|
|
||||||
}
|
|
||||||
|
|
||||||
canonicalChunks = append(canonicalChunks, source.Chunk{
|
|
||||||
ID: chunk.ID,
|
|
||||||
SourceID: chunk.SourceID,
|
|
||||||
Index: chunk.Index,
|
|
||||||
Ref: expectedRef,
|
|
||||||
Content: append([]byte(nil), chunk.Content...),
|
|
||||||
MediaType: chunk.MediaType,
|
|
||||||
Units: canonicalUnits,
|
|
||||||
Metadata: cloneMetadata(chunk.Metadata),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return canonicalChunks, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func cloneSourceUnit(unit source.SourceUnit) source.SourceUnit {
|
func cloneSourceUnit(unit source.SourceUnit) source.SourceUnit {
|
||||||
|
|||||||
@@ -334,8 +334,8 @@ func (chunker registryChunker) ReferenceSlots() []contracts.ReferenceSlot {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (chunker registryChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
func (chunker registryChunker) Plan(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
|
||||||
return contracts.ChunkResult{}, nil
|
return contracts.ChunkPlanResult{}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type registryOutputEncoder struct {
|
type registryOutputEncoder struct {
|
||||||
|
|||||||
@@ -107,6 +107,20 @@ type debugSourceChunk struct {
|
|||||||
Content debugBinaryEnvelope `json:"content"`
|
Content debugBinaryEnvelope `json:"content"`
|
||||||
Units []source.SourceUnit `json:"units,omitempty"`
|
Units []source.SourceUnit `json:"units,omitempty"`
|
||||||
Metadata map[string]any `json:"metadata,omitempty"`
|
Metadata map[string]any `json:"metadata,omitempty"`
|
||||||
|
Annotations source.ChunkAnnotations `json:"annotations,omitempty"`
|
||||||
|
PlanAnnotations source.ChunkAnnotations `json:"plan_annotations,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type debugChunkPlan struct {
|
||||||
|
SourceDigest string `json:"source_digest,omitempty"`
|
||||||
|
Ranges []debugChunkRange `json:"ranges,omitempty"`
|
||||||
|
Annotations map[string]any `json:"annotations,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type debugChunkRange struct {
|
||||||
|
StartUnitID int `json:"start_unit_id"`
|
||||||
|
EndUnitID int `json:"end_unit_id"`
|
||||||
|
Annotations map[string]any `json:"annotations,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type debugSerializedOutput struct {
|
type debugSerializedOutput struct {
|
||||||
@@ -471,6 +485,8 @@ func debugSourceChunkEnvelope(chunk source.Chunk) debugSourceChunk {
|
|||||||
Content: debugContentEnvelope(chunk.Content, chunk.MediaType, chunk.Metadata, nil),
|
Content: debugContentEnvelope(chunk.Content, chunk.MediaType, chunk.Metadata, nil),
|
||||||
Units: cloneSourceUnits(chunk.Units),
|
Units: cloneSourceUnits(chunk.Units),
|
||||||
Metadata: redactSensitiveMap(chunk.Metadata),
|
Metadata: redactSensitiveMap(chunk.Metadata),
|
||||||
|
Annotations: source.CloneChunkAnnotations(chunk.Annotations),
|
||||||
|
PlanAnnotations: source.CloneChunkAnnotations(chunk.PlanAnnotations),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -485,6 +501,37 @@ func debugSourceChunkEnvelopes(chunks []source.Chunk) []debugSourceChunk {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func debugChunkPlanEnvelope(plan source.ChunkPlan) debugChunkPlan {
|
||||||
|
envelope := debugChunkPlan{
|
||||||
|
SourceDigest: plan.SourceDigest,
|
||||||
|
Ranges: make([]debugChunkRange, len(plan.Ranges)),
|
||||||
|
Annotations: debugChunkAnnotations(plan.Annotations),
|
||||||
|
}
|
||||||
|
for i, chunkRange := range plan.Ranges {
|
||||||
|
envelope.Ranges[i] = debugChunkRange{
|
||||||
|
StartUnitID: chunkRange.StartUnitID,
|
||||||
|
EndUnitID: chunkRange.EndUnitID,
|
||||||
|
Annotations: debugChunkAnnotations(chunkRange.Annotations),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return envelope
|
||||||
|
}
|
||||||
|
|
||||||
|
func debugChunkAnnotations(annotations source.ChunkAnnotations) map[string]any {
|
||||||
|
if len(annotations) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make(map[string]any, len(annotations))
|
||||||
|
for namespace, raw := range annotations {
|
||||||
|
if json.Valid(raw) {
|
||||||
|
out[namespace] = append(json.RawMessage(nil), raw...)
|
||||||
|
} else {
|
||||||
|
out[namespace] = string(raw)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
func debugSerializedOutputEnvelope(output contracts.SerializedOutput) debugSerializedOutput {
|
func debugSerializedOutputEnvelope(output contracts.SerializedOutput) debugSerializedOutput {
|
||||||
schema := contracts.CloneArtifactSchema(output.Artifact.Schema)
|
schema := contracts.CloneArtifactSchema(output.Artifact.Schema)
|
||||||
digest := contracts.DigestArtifactSchema(schema)
|
digest := contracts.DigestArtifactSchema(schema)
|
||||||
|
|||||||
@@ -48,6 +48,8 @@ type RunInput struct {
|
|||||||
LLMProfiles []artifacts.LLMProfileManifest
|
LLMProfiles []artifacts.LLMProfileManifest
|
||||||
Metadata map[string]any
|
Metadata map[string]any
|
||||||
Warnings []contracts.Warning
|
Warnings []contracts.Warning
|
||||||
|
ChunkCacheMode ChunkCacheMode
|
||||||
|
ChunkPlans ChunkPlanStore
|
||||||
Checkpoints CheckpointRecorder
|
Checkpoints CheckpointRecorder
|
||||||
Checkpoint CheckpointLoader
|
Checkpoint CheckpointLoader
|
||||||
Debug DebugRecorder
|
Debug DebugRecorder
|
||||||
@@ -61,6 +63,7 @@ type RunInput struct {
|
|||||||
|
|
||||||
type RunOutput struct {
|
type RunOutput struct {
|
||||||
Manifest artifacts.RunManifest `json:"manifest"`
|
Manifest artifacts.RunManifest `json:"manifest"`
|
||||||
|
ChunkPlan *artifacts.ChunkPlanSummary `json:"chunk_plan,omitempty"`
|
||||||
NormalizeOutputs []contracts.SerializedOutput `json:"normalize_outputs,omitempty"`
|
NormalizeOutputs []contracts.SerializedOutput `json:"normalize_outputs,omitempty"`
|
||||||
Rejected []contracts.RejectedOutput `json:"rejected,omitempty"`
|
Rejected []contracts.RejectedOutput `json:"rejected,omitempty"`
|
||||||
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
||||||
@@ -79,6 +82,11 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
|
|||||||
input.llmClient = input.Prepared.dependencies.LLM
|
input.llmClient = input.Prepared.dependencies.LLM
|
||||||
|
|
||||||
output.Manifest = manifestFromPipeline(input)
|
output.Manifest = manifestFromPipeline(input)
|
||||||
|
output.ChunkPlan = &artifacts.ChunkPlanSummary{
|
||||||
|
Mode: string(effectiveChunkCacheMode(input.ChunkCacheMode)), RequestedModule: input.pipeline.Chunk.Module,
|
||||||
|
LookupStatus: "skipped", LookupReason: "chunk plan lookup skipped",
|
||||||
|
ValidationStatus: "not_run", PublicationStatus: "not_requested",
|
||||||
|
}
|
||||||
checkpoints := input.Checkpoints
|
checkpoints := input.Checkpoints
|
||||||
if checkpoints == nil {
|
if checkpoints == nil {
|
||||||
checkpoints = NoopCheckpointRecorder()
|
checkpoints = NoopCheckpointRecorder()
|
||||||
@@ -174,18 +182,14 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
|
|||||||
|
|
||||||
chunker := input.Prepared.chunker
|
chunker := input.Prepared.chunker
|
||||||
attachModuleManifestMetadata(&output, "chunker", chunker)
|
attachModuleManifestMetadata(&output, "chunker", chunker)
|
||||||
var canonicalChunks []source.Chunk
|
|
||||||
var chunkWarnings []contracts.Warning
|
|
||||||
chunkCheckpoint, chunkDecision := checkpointLoader.Chunk(chunker.Key(), doc.Digest)
|
|
||||||
recordCheckpointEvent(&output, checkpointLoader, string(StageChunk), "", chunker.Key(), chunkDecision)
|
|
||||||
chunkStarted := time.Now().UTC()
|
chunkStarted := time.Now().UTC()
|
||||||
|
chunkMode := effectiveChunkCacheMode(input.ChunkCacheMode)
|
||||||
if err := writeDebugTimed(debugRecorder, "chunk/input.json", debugTimedEnvelope{
|
if err := writeDebugTimed(debugRecorder, "chunk/input.json", debugTimedEnvelope{
|
||||||
Stage: string(StageChunk),
|
Stage: string(StageChunk),
|
||||||
ModuleKey: chunker.Key(),
|
ModuleKey: chunker.Key(),
|
||||||
StartedAt: chunkStarted,
|
StartedAt: chunkStarted,
|
||||||
Payload: map[string]any{
|
Payload: map[string]any{
|
||||||
"reused": chunkDecision.Reused,
|
"cache_mode": chunkMode,
|
||||||
"decision": chunkDecision,
|
|
||||||
"source": debugSourceDocumentEnvelope(doc),
|
"source": debugSourceDocumentEnvelope(doc),
|
||||||
"source_input": debugContentEnvelope(sourceInput.Content, sourceInput.MediaType, nil, nil),
|
"source_input": debugContentEnvelope(sourceInput.Content, sourceInput.MediaType, nil, nil),
|
||||||
"options": redactSensitiveMap(input.pipeline.Chunk.Options),
|
"options": redactSensitiveMap(input.pipeline.Chunk.Options),
|
||||||
@@ -194,84 +198,29 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
|
|||||||
}); err != nil {
|
}); err != nil {
|
||||||
return failOutput(output), fmt.Errorf("write chunk debug artifact: %w", err)
|
return failOutput(output), fmt.Errorf("write chunk debug artifact: %w", err)
|
||||||
}
|
}
|
||||||
chunksAccepted := chunkDecision.Reused
|
chunkResult, err := r.runChunkPlan(ctx, input, doc, sourceInput, sessionID)
|
||||||
var chunkRejection *contracts.RejectedOutput
|
applyChunkPlanExecution(&output, chunkResult)
|
||||||
if chunkDecision.Reused {
|
|
||||||
canonicalChunks = cloneSourceChunks(chunkCheckpoint.Chunks)
|
|
||||||
chunkWarnings = cloneWarnings(chunkCheckpoint.Warnings)
|
|
||||||
output.Warnings = append(output.Warnings, chunkWarnings...)
|
|
||||||
} else {
|
|
||||||
if err := checkpoints.ChunkRunning(chunker.Key(), doc.Digest); err != nil {
|
|
||||||
return failOutput(output), fmt.Errorf("write chunk checkpoint: %w", err)
|
|
||||||
}
|
|
||||||
chunksAccepted, chunkRejection, err = runWithRetry(ctx, input.pipeline.Chunk.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
|
|
||||||
attemptStarted := time.Now().UTC()
|
|
||||||
attemptPath := path.Join("chunk", fmt.Sprintf("attempt-%02d", attempt))
|
|
||||||
attemptCtx, llmScope := withDebugLLMScope(ctx, attemptPath)
|
|
||||||
terminal := newAttemptTerminalRecorder(debugRecorder, attemptPath, "chunk", llmScope, debugTimedEnvelope{Stage: string(StageChunk), ModuleKey: chunker.Key(), Attempt: attempt, StartedAt: attemptStarted})
|
|
||||||
chunkResult, err := chunker.Chunk(attemptCtx, contracts.ChunkRequest{
|
|
||||||
Source: doc,
|
|
||||||
SourceInput: sourceInput.Clone(),
|
|
||||||
SessionID: sessionID,
|
|
||||||
References: CloneReferenceSet(input.pipeline.ChunkReferences.ReferenceSet),
|
|
||||||
LLMProfile: input.pipeline.Chunk.LLMProfile,
|
|
||||||
Metadata: input.Metadata,
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
attemptErr := fmt.Errorf("chunk source with chunker %q: %w", chunker.Key(), err)
|
|
||||||
return false, nil, terminal.record(nil, attemptErr)
|
|
||||||
}
|
|
||||||
if len(chunkResult.Chunks) == 0 {
|
|
||||||
attemptErr := fmt.Errorf("chunker %q returned no chunks", chunker.Key())
|
|
||||||
return false, nil, terminal.record(nil, attemptErr)
|
|
||||||
}
|
|
||||||
chunks, err := validateAndCanonicalizeChunkResult(doc, chunkResult.Chunks)
|
|
||||||
if err != nil {
|
|
||||||
attemptErr := fmt.Errorf("validate chunks from chunker %q: %w", chunker.Key(), err)
|
|
||||||
payload := map[string]any{"warnings": debugWarningEnvelopes(chunkResult.Warnings)}
|
|
||||||
return false, nil, terminal.record(payload, attemptErr)
|
|
||||||
}
|
|
||||||
validationWarnings, rejection, err := r.validateChunks(attemptCtx, doc, chunker.Key(), chunks, sourceInput, sessionID, input.pipeline.ChunkReferences.ReferenceSet, input.Metadata, input.Prepared.chunkValidators, attempt, input.Debug)
|
|
||||||
attemptWarnings := append(cloneWarnings(chunkResult.Warnings), validationWarnings...)
|
|
||||||
payload := map[string]any{
|
|
||||||
"chunks": debugSourceChunkEnvelopes(chunks),
|
|
||||||
"warnings": debugWarningEnvelopes(attemptWarnings),
|
|
||||||
"rejection": debugRejectedOutputPtr(rejection),
|
|
||||||
}
|
|
||||||
if err != nil || rejection != nil {
|
|
||||||
return false, rejection, terminal.record(payload, err)
|
|
||||||
}
|
|
||||||
canonicalChunks = chunks
|
|
||||||
chunkWarnings = attemptWarnings
|
|
||||||
if err := terminal.record(payload, nil); err != nil {
|
|
||||||
return false, nil, err
|
|
||||||
}
|
|
||||||
return true, nil, nil
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
_ = checkpoints.ChunkFailed(chunker.Key(), doc.Digest, err)
|
|
||||||
return failOutput(output), err
|
return failOutput(output), err
|
||||||
}
|
}
|
||||||
if !chunksAccepted {
|
if chunkResult.rejection != nil {
|
||||||
output.Rejected = append(output.Rejected, *chunkRejection)
|
output.Rejected = append(output.Rejected, *chunkResult.rejection)
|
||||||
if err := checkpoints.ChunkRejected(chunker.Key(), doc.Digest, *chunkRejection); err != nil {
|
|
||||||
return failOutput(output), fmt.Errorf("write chunk checkpoint: %w", err)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
output.Warnings = append(output.Warnings, chunkWarnings...)
|
|
||||||
if err := checkpoints.ChunkSucceeded(chunker.Key(), doc.Digest, canonicalChunks, chunkWarnings); err != nil {
|
|
||||||
return failOutput(output), fmt.Errorf("write chunk checkpoint: %w", err)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
if chunkResult.accepted || chunkResult.lookup.Status == ChunkPlanHit {
|
||||||
|
output.Warnings = append(output.Warnings, chunkResult.warnings...)
|
||||||
}
|
}
|
||||||
chunkDebugPayload := map[string]any{
|
chunkDebugPayload := map[string]any{
|
||||||
"reused": chunkDecision.Reused,
|
"cache_mode": chunkMode,
|
||||||
"accepted": chunksAccepted,
|
"lookup": chunkResult.lookup,
|
||||||
"chunks": debugSourceChunkEnvelopes(canonicalChunks),
|
"accepted": chunkResult.accepted,
|
||||||
"warnings": chunkWarnings,
|
"materialized_chunks": debugSourceChunkEnvelopes(chunkResult.chunks),
|
||||||
|
"warnings": chunkResult.warnings,
|
||||||
}
|
}
|
||||||
if chunkRejection != nil {
|
if chunkResult.plan != nil {
|
||||||
chunkDebugPayload["rejection"] = debugRejectedOutputEnvelope(*chunkRejection)
|
chunkDebugPayload["plan"] = debugChunkPlanEnvelope(*chunkResult.plan)
|
||||||
|
}
|
||||||
|
if chunkResult.rejection != nil {
|
||||||
|
chunkDebugPayload["rejection"] = debugRejectedOutputEnvelope(*chunkResult.rejection)
|
||||||
}
|
}
|
||||||
if err := writeDebugTimed(debugRecorder, "chunk/output.json", debugTimedEnvelope{
|
if err := writeDebugTimed(debugRecorder, "chunk/output.json", debugTimedEnvelope{
|
||||||
Stage: string(StageChunk),
|
Stage: string(StageChunk),
|
||||||
@@ -282,8 +231,8 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
|
|||||||
return failOutput(output), fmt.Errorf("write chunk debug artifact: %w", err)
|
return failOutput(output), fmt.Errorf("write chunk debug artifact: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if chunksAccepted {
|
if chunkResult.accepted {
|
||||||
laneOutput, laneErr := r.runLanes(ctx, input, checkpoints, checkpointLoader, doc, sourceInput, sessionID, canonicalChunks)
|
laneOutput, laneErr := r.runLanes(ctx, input, checkpoints, checkpointLoader, doc, sourceInput, sessionID, chunkResult.chunks)
|
||||||
mergeLaneOutput(&output, laneOutput)
|
mergeLaneOutput(&output, laneOutput)
|
||||||
if laneErr != nil {
|
if laneErr != nil {
|
||||||
return failOutput(output), laneErr
|
return failOutput(output), laneErr
|
||||||
@@ -471,6 +420,13 @@ func validateRunInput(input RunInput) error {
|
|||||||
if input.Prepared == nil {
|
if input.Prepared == nil {
|
||||||
return fmt.Errorf("prepared pipeline must not be nil")
|
return fmt.Errorf("prepared pipeline must not be nil")
|
||||||
}
|
}
|
||||||
|
mode := effectiveChunkCacheMode(input.ChunkCacheMode)
|
||||||
|
if err := mode.Validate(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if mode != ChunkCacheBypass && input.ChunkPlans == nil {
|
||||||
|
return fmt.Errorf("chunk plan store is required for %q mode", mode)
|
||||||
|
}
|
||||||
return validateResolvedPipeline(input.Prepared.resolved)
|
return validateResolvedPipeline(input.Prepared.resolved)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -536,6 +492,10 @@ func manifestFromPipeline(input RunInput) artifacts.RunManifest {
|
|||||||
PipelineDigest: pipeline.Digest,
|
PipelineDigest: pipeline.Digest,
|
||||||
InputModule: pipeline.Input.Module,
|
InputModule: pipeline.Input.Module,
|
||||||
Chunker: pipeline.Chunk.Module,
|
Chunker: pipeline.Chunk.Module,
|
||||||
|
ChunkPlan: &artifacts.ChunkPlanManifest{
|
||||||
|
Mode: string(effectiveChunkCacheMode(input.ChunkCacheMode)),
|
||||||
|
RequestedModule: pipeline.Chunk.Module,
|
||||||
|
},
|
||||||
OutputEncoder: pipeline.Output.Module,
|
OutputEncoder: pipeline.Output.Module,
|
||||||
ArtifactLanes: make([]artifacts.ArtifactLaneManifest, 0, len(pipeline.ArtifactLanes)),
|
ArtifactLanes: make([]artifacts.ArtifactLaneManifest, 0, len(pipeline.ArtifactLanes)),
|
||||||
ValidatorChains: validatorChainManifests(pipeline.ValidatorChains),
|
ValidatorChains: validatorChainManifests(pipeline.ValidatorChains),
|
||||||
@@ -544,10 +504,6 @@ func manifestFromPipeline(input RunInput) artifacts.RunManifest {
|
|||||||
References: ReferenceProvenance(pipeline),
|
References: ReferenceProvenance(pipeline),
|
||||||
LLMProfiles: cloneLLMProfiles(input.LLMProfiles),
|
LLMProfiles: cloneLLMProfiles(input.LLMProfiles),
|
||||||
}
|
}
|
||||||
// The runner does not currently maintain a cache or idempotency key. Reference
|
|
||||||
// digests are recorded in manifest provenance and intentionally kept separate
|
|
||||||
// from source_digests.
|
|
||||||
|
|
||||||
for _, lane := range pipeline.ArtifactLanes {
|
for _, lane := range pipeline.ArtifactLanes {
|
||||||
laneManifest := artifacts.ArtifactLaneManifest{
|
laneManifest := artifacts.ArtifactLaneManifest{
|
||||||
ID: lane.ID,
|
ID: lane.ID,
|
||||||
@@ -728,11 +684,38 @@ func cloneMetadata(metadata map[string]any) map[string]any {
|
|||||||
}
|
}
|
||||||
out := make(map[string]any, len(metadata))
|
out := make(map[string]any, len(metadata))
|
||||||
for key, value := range metadata {
|
for key, value := range metadata {
|
||||||
out[key] = value
|
out[key] = cloneMetadataValue(value)
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func cloneMetadataValue(value any) any {
|
||||||
|
switch typed := value.(type) {
|
||||||
|
case map[string]any:
|
||||||
|
return cloneMetadata(typed)
|
||||||
|
case []any:
|
||||||
|
out := make([]any, len(typed))
|
||||||
|
for i := range typed {
|
||||||
|
out[i] = cloneMetadataValue(typed[i])
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
case json.RawMessage:
|
||||||
|
return append(json.RawMessage(nil), typed...)
|
||||||
|
case []byte:
|
||||||
|
return append([]byte(nil), typed...)
|
||||||
|
case []string:
|
||||||
|
return append([]string(nil), typed...)
|
||||||
|
case map[string]string:
|
||||||
|
out := make(map[string]string, len(typed))
|
||||||
|
for key, item := range typed {
|
||||||
|
out[key] = item
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
default:
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func cloneLLMProfiles(profiles []artifacts.LLMProfileManifest) []artifacts.LLMProfileManifest {
|
func cloneLLMProfiles(profiles []artifacts.LLMProfileManifest) []artifacts.LLMProfileManifest {
|
||||||
if len(profiles) == 0 {
|
if len(profiles) == 0 {
|
||||||
return nil
|
return nil
|
||||||
@@ -863,6 +846,8 @@ func cloneSourceChunk(chunk source.Chunk) source.Chunk {
|
|||||||
chunk.Content = append([]byte(nil), chunk.Content...)
|
chunk.Content = append([]byte(nil), chunk.Content...)
|
||||||
chunk.Units = cloneSourceUnits(chunk.Units)
|
chunk.Units = cloneSourceUnits(chunk.Units)
|
||||||
chunk.Metadata = cloneMetadata(chunk.Metadata)
|
chunk.Metadata = cloneMetadata(chunk.Metadata)
|
||||||
|
chunk.Annotations = source.CloneChunkAnnotations(chunk.Annotations)
|
||||||
|
chunk.PlanAnnotations = source.CloneChunkAnnotations(chunk.PlanAnnotations)
|
||||||
return chunk
|
return chunk
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -209,7 +209,7 @@ func TestRunnerFinalEncodesAcceptedCandidatesOnce(t *testing.T) {
|
|||||||
for i, event := range output.CheckpointEvents {
|
for i, event := range output.CheckpointEvents {
|
||||||
gotStages[i] = event.Stage
|
gotStages[i] = event.Stage
|
||||||
}
|
}
|
||||||
wantStages := []string{"source", string(StageChunk), string(StageExtract), string(StageMerge), string(StageNormalize)}
|
wantStages := []string{"source", string(StageExtract), string(StageMerge), string(StageNormalize)}
|
||||||
if !reflect.DeepEqual(gotStages, wantStages) {
|
if !reflect.DeepEqual(gotStages, wantStages) {
|
||||||
t.Fatalf("checkpoint event stages = %#v, want %#v", gotStages, wantStages)
|
t.Fatalf("checkpoint event stages = %#v, want %#v", gotStages, wantStages)
|
||||||
}
|
}
|
||||||
|
|||||||
229
internal/framework/pipeline/runner_chunk_plan.go
Normal file
229
internal/framework/pipeline/runner_chunk_plan.go
Normal file
@@ -0,0 +1,229 @@
|
|||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"path"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
)
|
||||||
|
|
||||||
|
type chunkPlanExecution struct {
|
||||||
|
accepted bool
|
||||||
|
chunks []source.Chunk
|
||||||
|
plan *source.ChunkPlan
|
||||||
|
warnings []contracts.Warning
|
||||||
|
rejection *contracts.RejectedOutput
|
||||||
|
lookup ChunkPlanDecision
|
||||||
|
record *ChunkPlanRecord
|
||||||
|
action string
|
||||||
|
summary artifacts.ChunkPlanSummary
|
||||||
|
}
|
||||||
|
|
||||||
|
func effectiveChunkCacheMode(mode ChunkCacheMode) ChunkCacheMode {
|
||||||
|
if mode == "" {
|
||||||
|
return ChunkCacheBypass
|
||||||
|
}
|
||||||
|
parsed, _ := ParseChunkCacheMode(string(mode))
|
||||||
|
return parsed
|
||||||
|
}
|
||||||
|
|
||||||
|
// Chunk-plan lookup and publication happen serially before lane workers start.
|
||||||
|
// The runner therefore does not add synchronization around the store.
|
||||||
|
func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string) (chunkPlanExecution, error) {
|
||||||
|
mode := effectiveChunkCacheMode(input.ChunkCacheMode)
|
||||||
|
chunker := input.Prepared.chunker
|
||||||
|
result := chunkPlanExecution{
|
||||||
|
lookup: ChunkPlanDecision{Reason: "chunk plan lookup skipped"},
|
||||||
|
summary: artifacts.ChunkPlanSummary{
|
||||||
|
Mode: string(mode), SourceDigest: doc.Digest, RequestedModule: chunker.Key(), LookupStatus: "skipped",
|
||||||
|
LookupReason: "chunk plan lookup skipped", ValidationStatus: "not_run", PublicationStatus: "not_requested",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
if mode == ChunkCacheAuto {
|
||||||
|
record, decision, err := input.ChunkPlans.Load(doc.Digest)
|
||||||
|
result.lookup = decision
|
||||||
|
result.summary.LookupStatus = string(decision.Status)
|
||||||
|
result.summary.LookupReason = decision.Reason
|
||||||
|
if err != nil {
|
||||||
|
result.summary.LookupStatus = "skipped"
|
||||||
|
result.summary.LookupReason = "chunk plan lookup failed"
|
||||||
|
return result, fmt.Errorf("load chunk plan: %w", err)
|
||||||
|
}
|
||||||
|
switch decision.Status {
|
||||||
|
case ChunkPlanHit:
|
||||||
|
plan, chunks, validationErr := validateAndMaterializeChunkPlan(doc, record.Plan)
|
||||||
|
if validationErr == nil {
|
||||||
|
result.setCandidate(record, "reused")
|
||||||
|
validationWarnings, rejection, err := r.validateChunks(ctx, doc, chunker.Key(), chunks, sourceInput, sessionID, input.pipeline.ChunkReferences.ReferenceSet, input.Metadata, input.Prepared.chunkValidators, 1, input.Debug)
|
||||||
|
result.plan = &plan
|
||||||
|
result.chunks = chunks
|
||||||
|
result.warnings = append(cloneWarnings(record.Warnings), validationWarnings...)
|
||||||
|
result.rejection = rejection
|
||||||
|
result.accepted = rejection == nil && err == nil
|
||||||
|
result.setValidation(validationWarnings, rejection, err)
|
||||||
|
return result, err
|
||||||
|
}
|
||||||
|
result.lookup = ChunkPlanDecision{Status: ChunkPlanInvalid, Reason: "stored chunk plan cannot be materialized against the current source"}
|
||||||
|
result.summary.LookupStatus = "invalid"
|
||||||
|
result.summary.LookupReason = result.lookup.Reason
|
||||||
|
case ChunkPlanMissing, ChunkPlanInvalid:
|
||||||
|
// Generate below.
|
||||||
|
default:
|
||||||
|
return result, fmt.Errorf("load chunk plan returned unsupported status %q", decision.Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if mode == ChunkCacheAuto || mode == ChunkCacheRefresh {
|
||||||
|
result.summary.PublicationStatus = "not_published"
|
||||||
|
}
|
||||||
|
|
||||||
|
var producerWarnings []contracts.Warning
|
||||||
|
accepted, rejection, err := runWithRetry(ctx, input.pipeline.Chunk.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
|
||||||
|
attemptStarted := time.Now().UTC()
|
||||||
|
attemptPath := path.Join("chunk", fmt.Sprintf("attempt-%02d", attempt))
|
||||||
|
attemptCtx, llmScope := withDebugLLMScope(ctx, attemptPath)
|
||||||
|
terminal := newAttemptTerminalRecorder(input.Debug, attemptPath, "chunk", llmScope, debugTimedEnvelope{Stage: string(StageChunk), ModuleKey: chunker.Key(), Attempt: attempt, StartedAt: attemptStarted})
|
||||||
|
chunkResult, callErr := chunker.Plan(attemptCtx, contracts.ChunkRequest{
|
||||||
|
Source: doc, SourceInput: sourceInput.Clone(), SessionID: sessionID,
|
||||||
|
References: CloneReferenceSet(input.pipeline.ChunkReferences.ReferenceSet),
|
||||||
|
LLMProfile: input.pipeline.Chunk.LLMProfile, Metadata: input.Metadata,
|
||||||
|
})
|
||||||
|
if callErr != nil {
|
||||||
|
return false, nil, terminal.record(nil, fmt.Errorf("chunk source with chunker %q: %w", chunker.Key(), callErr))
|
||||||
|
}
|
||||||
|
plan, chunks, validationErr := validateAndMaterializeChunkPlan(doc, chunkResult.Plan)
|
||||||
|
if validationErr != nil {
|
||||||
|
attemptErr := fmt.Errorf("validate chunk plan from chunker %q: %w", chunker.Key(), validationErr)
|
||||||
|
payload := map[string]any{"plan": debugChunkPlanEnvelope(chunkResult.Plan), "warnings": debugWarningEnvelopes(chunkResult.Warnings)}
|
||||||
|
return false, nil, terminal.record(payload, attemptErr)
|
||||||
|
}
|
||||||
|
planDigest, digestErr := source.DigestChunkPlan(plan)
|
||||||
|
if digestErr != nil {
|
||||||
|
return false, nil, terminal.record(nil, fmt.Errorf("digest generated chunk plan: %w", digestErr))
|
||||||
|
}
|
||||||
|
producerMetadata, _ := moduleManifestMetadata(chunker)
|
||||||
|
profile := ""
|
||||||
|
if provider, ok := chunker.(contracts.ChunkExecutionClassProvider); ok && provider.ExecutionClass() == contracts.ExecutionClassLLMBacked {
|
||||||
|
profile = input.pipeline.Chunk.LLMProfile
|
||||||
|
}
|
||||||
|
candidate := ChunkPlanRecord{
|
||||||
|
SchemaVersion: ChunkPlanSchemaVersion, SourceDigest: doc.Digest, PlanDigest: planDigest,
|
||||||
|
Plan: source.CloneChunkPlan(plan),
|
||||||
|
Producer: ChunkPlanProducer{
|
||||||
|
InputModule: input.Prepared.input.Key(), ChunkModule: chunker.Key(), LLMProfile: profile,
|
||||||
|
References: append([]artifacts.ReferenceProvenance(nil), referenceTargetProvenance(input.pipeline.ChunkReferences)...),
|
||||||
|
Metadata: cloneMetadata(producerMetadata),
|
||||||
|
},
|
||||||
|
Warnings: cloneWarnings(chunkResult.Warnings), CreatedAt: time.Now().UTC(),
|
||||||
|
}
|
||||||
|
action := "generated"
|
||||||
|
if mode == ChunkCacheRefresh {
|
||||||
|
action = "refreshed"
|
||||||
|
}
|
||||||
|
if mode == ChunkCacheBypass {
|
||||||
|
action = "bypassed"
|
||||||
|
}
|
||||||
|
result.setCandidate(candidate, action)
|
||||||
|
validationWarnings, rejected, validationErr := r.validateChunks(attemptCtx, doc, chunker.Key(), chunks, sourceInput, sessionID, input.pipeline.ChunkReferences.ReferenceSet, input.Metadata, input.Prepared.chunkValidators, attempt, input.Debug)
|
||||||
|
attemptWarnings := append(cloneWarnings(chunkResult.Warnings), validationWarnings...)
|
||||||
|
payload := map[string]any{
|
||||||
|
"plan": debugChunkPlanEnvelope(plan), "materialized_chunks": debugSourceChunkEnvelopes(chunks),
|
||||||
|
"warnings": debugWarningEnvelopes(attemptWarnings), "rejection": debugRejectedOutputPtr(rejected),
|
||||||
|
}
|
||||||
|
if validationErr != nil || rejected != nil {
|
||||||
|
result.setValidation(validationWarnings, rejected, validationErr)
|
||||||
|
return false, rejected, terminal.record(payload, validationErr)
|
||||||
|
}
|
||||||
|
result.chunks = chunks
|
||||||
|
result.plan = &plan
|
||||||
|
result.warnings = attemptWarnings
|
||||||
|
producerWarnings = cloneWarnings(chunkResult.Warnings)
|
||||||
|
result.setValidation(validationWarnings, nil, nil)
|
||||||
|
if debugErr := terminal.record(payload, nil); debugErr != nil {
|
||||||
|
return false, nil, debugErr
|
||||||
|
}
|
||||||
|
return true, nil, nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
if result.summary.ValidationStatus == "not_run" {
|
||||||
|
result.summary.ValidationStatus = "error"
|
||||||
|
}
|
||||||
|
return result, err
|
||||||
|
}
|
||||||
|
result.accepted = accepted
|
||||||
|
result.rejection = rejection
|
||||||
|
if !accepted {
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if mode == ChunkCacheAuto || mode == ChunkCacheRefresh {
|
||||||
|
record := cloneChunkPlanRecord(*result.record)
|
||||||
|
record.Warnings = cloneWarnings(producerWarnings)
|
||||||
|
if err := input.ChunkPlans.Save(record); err != nil {
|
||||||
|
return result, fmt.Errorf("save chunk plan: %w", err)
|
||||||
|
}
|
||||||
|
result.summary.PublicationStatus = "published"
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (result *chunkPlanExecution) setCandidate(record ChunkPlanRecord, action string) {
|
||||||
|
cloned := cloneChunkPlanRecord(record)
|
||||||
|
result.record = &cloned
|
||||||
|
result.action = action
|
||||||
|
result.summary.Action = action
|
||||||
|
result.summary.SourceDigest = record.SourceDigest
|
||||||
|
result.summary.CandidateDigest = record.PlanDigest
|
||||||
|
}
|
||||||
|
|
||||||
|
func (result *chunkPlanExecution) setValidation(warnings []contracts.Warning, rejection *contracts.RejectedOutput, err error) {
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
result.summary.ValidationStatus = "error"
|
||||||
|
case rejection != nil:
|
||||||
|
result.summary.ValidationStatus = "rejected"
|
||||||
|
case len(warnings) > 0:
|
||||||
|
result.summary.ValidationStatus = "approved_with_warnings"
|
||||||
|
default:
|
||||||
|
result.summary.ValidationStatus = "approved"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneChunkPlanRecord(record ChunkPlanRecord) ChunkPlanRecord {
|
||||||
|
record.Plan = source.CloneChunkPlan(record.Plan)
|
||||||
|
record.Producer.References = append([]artifacts.ReferenceProvenance(nil), record.Producer.References...)
|
||||||
|
record.Producer.Metadata = cloneMetadata(record.Producer.Metadata)
|
||||||
|
record.Warnings = cloneWarnings(record.Warnings)
|
||||||
|
return record
|
||||||
|
}
|
||||||
|
|
||||||
|
func applyChunkPlanExecution(output *RunOutput, result chunkPlanExecution) {
|
||||||
|
if output == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
summary := result.summary
|
||||||
|
output.ChunkPlan = &summary
|
||||||
|
if output.Manifest.ChunkPlan == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
manifest := output.Manifest.ChunkPlan
|
||||||
|
manifest.Action = result.action
|
||||||
|
if result.record == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
record := result.record
|
||||||
|
manifest.SourceDigest = record.SourceDigest
|
||||||
|
manifest.PlanDigest = record.PlanDigest
|
||||||
|
manifest.PlanSchemaVersion = record.SchemaVersion
|
||||||
|
manifest.ProducerInputModule = record.Producer.InputModule
|
||||||
|
manifest.ProducerModule = record.Producer.ChunkModule
|
||||||
|
manifest.ProducerLLMProfile = record.Producer.LLMProfile
|
||||||
|
manifest.ProducerReferences = append([]artifacts.ReferenceProvenance(nil), record.Producer.References...)
|
||||||
|
manifest.ProducerMetadata = cloneMetadata(record.Producer.Metadata)
|
||||||
|
createdAt := record.CreatedAt
|
||||||
|
manifest.CreatedAt = &createdAt
|
||||||
|
}
|
||||||
455
internal/framework/pipeline/runner_chunk_plan_test.go
Normal file
455
internal/framework/pipeline/runner_chunk_plan_test.go
Normal file
@@ -0,0 +1,455 @@
|
|||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
)
|
||||||
|
|
||||||
|
type recordingChunkPlanStore struct {
|
||||||
|
record ChunkPlanRecord
|
||||||
|
decision ChunkPlanDecision
|
||||||
|
loadErr error
|
||||||
|
saveErr error
|
||||||
|
loads int
|
||||||
|
saves int
|
||||||
|
saved ChunkPlanRecord
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *recordingChunkPlanStore) Load(string) (ChunkPlanRecord, ChunkPlanDecision, error) {
|
||||||
|
s.loads++
|
||||||
|
return s.record, s.decision, s.loadErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *recordingChunkPlanStore) Save(record ChunkPlanRecord) error {
|
||||||
|
s.saves++
|
||||||
|
s.saved = record
|
||||||
|
return s.saveErr
|
||||||
|
}
|
||||||
|
|
||||||
|
type countingChunkValidator struct {
|
||||||
|
calls int
|
||||||
|
result contracts.ValidationResult
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (*countingChunkValidator) Name() string { return "test/counting-chunks" }
|
||||||
|
func (*countingChunkValidator) ExecutionClass() contracts.ExecutionClass {
|
||||||
|
return contracts.ExecutionClassDeterministic
|
||||||
|
}
|
||||||
|
func (v *countingChunkValidator) Validate(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
|
||||||
|
v.calls++
|
||||||
|
return v.result, v.err
|
||||||
|
}
|
||||||
|
|
||||||
|
type manifestChunker struct {
|
||||||
|
terminalChunker
|
||||||
|
metadata map[string]any
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c manifestChunker) ManifestMetadata() map[string]any { return c.metadata }
|
||||||
|
func (manifestChunker) ExecutionClass() contracts.ExecutionClass {
|
||||||
|
return contracts.ExecutionClassLLMBacked
|
||||||
|
}
|
||||||
|
|
||||||
|
type llmCountingChunker struct {
|
||||||
|
terminalChunker
|
||||||
|
llmCalls *int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (llmCountingChunker) ExecutionClass() contracts.ExecutionClass {
|
||||||
|
return contracts.ExecutionClassLLMBacked
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c llmCountingChunker) Plan(ctx context.Context, request contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
|
||||||
|
(*c.llmCalls)++
|
||||||
|
return c.terminalChunker.Plan(ctx, request)
|
||||||
|
}
|
||||||
|
|
||||||
|
type deterministicChunker struct{ terminalChunker }
|
||||||
|
|
||||||
|
func (deterministicChunker) ExecutionClass() contracts.ExecutionClass {
|
||||||
|
return contracts.ExecutionClassDeterministic
|
||||||
|
}
|
||||||
|
|
||||||
|
type retryingChunker struct {
|
||||||
|
key string
|
||||||
|
plan source.ChunkPlan
|
||||||
|
calls int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *retryingChunker) Key() string { return c.key }
|
||||||
|
func (*retryingChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||||
|
func (c *retryingChunker) Plan(context.Context, contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
|
||||||
|
c.calls++
|
||||||
|
if c.calls == 1 {
|
||||||
|
return contracts.ChunkPlanResult{Warnings: []contracts.Warning{{Scope: "discarded", ReasonCode: "retry", Message: "discarded warning"}}}, errors.New("retry generation")
|
||||||
|
}
|
||||||
|
return contracts.ChunkPlanResult{Plan: source.CloneChunkPlan(c.plan), Warnings: []contracts.Warning{{Scope: "accepted", ReasonCode: "observed", Message: "accepted warning"}}}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type dependencyLoader struct {
|
||||||
|
CheckpointLoader
|
||||||
|
extractDependencies []CheckpointFingerprint
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *dependencyLoader) Extract(_ string, _ string, dependencies []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
|
||||||
|
l.extractDependencies = append([]CheckpointFingerprint(nil), dependencies...)
|
||||||
|
return ExtractCheckpoint{}, CheckpointDecision{Reason: "not found"}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerChunkPlanModeMatrix(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
mode ChunkCacheMode
|
||||||
|
decision ChunkPlanDecision
|
||||||
|
wantLoads int
|
||||||
|
wantSaves int
|
||||||
|
wantCalls int
|
||||||
|
wantAction string
|
||||||
|
}{
|
||||||
|
{name: "auto hit", mode: ChunkCacheAuto, decision: ChunkPlanDecision{Status: ChunkPlanHit}, wantLoads: 1, wantAction: "reused"},
|
||||||
|
{name: "auto missing", mode: ChunkCacheAuto, decision: ChunkPlanDecision{Status: ChunkPlanMissing}, wantLoads: 1, wantSaves: 1, wantCalls: 1, wantAction: "generated"},
|
||||||
|
{name: "auto invalid", mode: ChunkCacheAuto, decision: ChunkPlanDecision{Status: ChunkPlanInvalid}, wantLoads: 1, wantSaves: 1, wantCalls: 1, wantAction: "generated"},
|
||||||
|
{name: "bypass", mode: ChunkCacheBypass, wantCalls: 1, wantAction: "bypassed"},
|
||||||
|
{name: "empty bypass", wantCalls: 1, wantAction: "bypassed"},
|
||||||
|
{name: "refresh", mode: ChunkCacheRefresh, wantSaves: 1, wantCalls: 1, wantAction: "refreshed"},
|
||||||
|
}
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
prepared, plan := preparedTerminalDebugPipeline(t)
|
||||||
|
calls := 0
|
||||||
|
llmCalls := 0
|
||||||
|
prepared.chunker = llmCountingChunker{terminalChunker: terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls}, llmCalls: &llmCalls}
|
||||||
|
store := &recordingChunkPlanStore{record: chunkPlanRecord(t, prepared, plan), decision: tc.decision}
|
||||||
|
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: tc.mode, ChunkPlans: store})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if store.loads != tc.wantLoads || store.saves != tc.wantSaves || calls != tc.wantCalls {
|
||||||
|
t.Fatalf("calls = load %d save %d module %d, want %d %d %d", store.loads, store.saves, calls, tc.wantLoads, tc.wantSaves, tc.wantCalls)
|
||||||
|
}
|
||||||
|
if output.Manifest.ChunkPlan == nil || output.Manifest.ChunkPlan.Action != tc.wantAction || output.ChunkPlan == nil || output.ChunkPlan.Action != tc.wantAction {
|
||||||
|
t.Fatalf("chunk plan manifest = %#v summary = %#v, want action %q", output.Manifest.ChunkPlan, output.ChunkPlan, tc.wantAction)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerChunkPlanHitUsesStoredProducerProvenance(t *testing.T) {
|
||||||
|
prepared, plan := preparedTerminalDebugPipeline(t)
|
||||||
|
prepared.resolved.Chunk.Module = "chunk/requested"
|
||||||
|
prepared.resolved.Chunk.LLMProfile = "current-profile"
|
||||||
|
record := chunkPlanRecord(t, prepared, plan)
|
||||||
|
record.Producer.InputModule = "input/original"
|
||||||
|
record.Producer.ChunkModule = "chunk/original"
|
||||||
|
record.Producer.LLMProfile = "original-profile"
|
||||||
|
record.Producer.Metadata = map[string]any{"prompt": map[string]any{"id": "original"}}
|
||||||
|
store := &recordingChunkPlanStore{record: record, decision: ChunkPlanDecision{Status: ChunkPlanHit}}
|
||||||
|
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
manifest := output.Manifest.ChunkPlan
|
||||||
|
if manifest.RequestedModule != "chunk/requested" || manifest.ProducerModule != "chunk/original" || manifest.ProducerLLMProfile != "original-profile" {
|
||||||
|
t.Fatalf("chunk plan manifest = %#v", manifest)
|
||||||
|
}
|
||||||
|
storedNested := record.Producer.Metadata["prompt"].(map[string]any)
|
||||||
|
storedNested["id"] = "mutated"
|
||||||
|
if got := manifest.ProducerMetadata["prompt"].(map[string]any)["id"]; got != "original" {
|
||||||
|
t.Fatalf("producer metadata changed through stored alias: %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerChunkPlanManifestRetainsCandidateOnRejection(t *testing.T) {
|
||||||
|
prepared, _ := preparedTerminalDebugPipeline(t)
|
||||||
|
validator := &countingChunkValidator{result: contracts.ValidationResult{Approved: false, ReasonCode: "policy", Message: "no"}}
|
||||||
|
prepared.chunkValidators.validators = []preparedValidator{{resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk}, chunk: validator}}
|
||||||
|
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheRefresh, ChunkPlans: &recordingChunkPlanStore{}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if output.Manifest.ChunkPlan.Action != "refreshed" || output.Manifest.ChunkPlan.PlanDigest == "" || output.ChunkPlan.ValidationStatus != "rejected" || output.ChunkPlan.PublicationStatus != "not_published" {
|
||||||
|
t.Fatalf("manifest = %#v summary = %#v", output.Manifest.ChunkPlan, output.ChunkPlan)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerValidatesChunkPlanPolicyInputs(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
mode ChunkCacheMode
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "auto requires store", mode: ChunkCacheAuto, want: "store is required"},
|
||||||
|
{name: "refresh requires store", mode: ChunkCacheRefresh, want: "store is required"},
|
||||||
|
{name: "invalid mode", mode: "sometimes", want: "not supported"},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
prepared, _ := preparedTerminalDebugPipeline(t)
|
||||||
|
_, err := New().Run(context.Background(), RunInput{Prepared: prepared, ChunkCacheMode: tc.mode})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), tc.want) {
|
||||||
|
t.Fatalf("Run() error = %v, want %q", err, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerChunkPlanStoreErrorsAreFrameworkErrors(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
mode ChunkCacheMode
|
||||||
|
decision ChunkPlanDecision
|
||||||
|
loadErr error
|
||||||
|
saveErr error
|
||||||
|
wantError string
|
||||||
|
wantCalls int
|
||||||
|
}{
|
||||||
|
{name: "load", mode: ChunkCacheAuto, loadErr: errors.New("read failed"), wantError: "load chunk plan", wantCalls: 0},
|
||||||
|
{name: "save", mode: ChunkCacheRefresh, saveErr: errors.New("write failed"), wantError: "save chunk plan", wantCalls: 1},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
prepared, plan := preparedTerminalDebugPipeline(t)
|
||||||
|
calls := 0
|
||||||
|
llmCalls := 0
|
||||||
|
prepared.chunker = llmCountingChunker{terminalChunker: terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls}, llmCalls: &llmCalls}
|
||||||
|
store := &recordingChunkPlanStore{decision: tc.decision, loadErr: tc.loadErr, saveErr: tc.saveErr}
|
||||||
|
_, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: tc.mode, ChunkPlans: store})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), tc.wantError) || calls != tc.wantCalls {
|
||||||
|
t.Fatalf("Run() error = %v, module calls = %d, want %q and %d", err, calls, tc.wantError, tc.wantCalls)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerRegeneratesStructurallyInvalidHit(t *testing.T) {
|
||||||
|
prepared, plan := preparedTerminalDebugPipeline(t)
|
||||||
|
calls := 0
|
||||||
|
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls}
|
||||||
|
invalid := chunkPlanRecord(t, prepared, plan)
|
||||||
|
invalid.Plan.Ranges[0].StartUnitID = 999
|
||||||
|
store := &recordingChunkPlanStore{record: invalid, decision: ChunkPlanDecision{Status: ChunkPlanHit}}
|
||||||
|
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if calls != 1 || store.loads != 1 || store.saves != 1 {
|
||||||
|
t.Fatalf("calls = module %d load %d save %d, want 1 1 1", calls, store.loads, store.saves)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerReusesCrossDomainAnnotationsAsOptionalData(t *testing.T) {
|
||||||
|
prepared, plan := preparedTerminalDebugPipeline(t)
|
||||||
|
plan.Annotations = source.ChunkAnnotations{"dnd/scenes": json.RawMessage(`{"boundary_caveats":["uncertain"]}`)}
|
||||||
|
plan.Ranges[0].Annotations = source.ChunkAnnotations{"dnd/scenes": json.RawMessage(`{"title":"Opening"}`)}
|
||||||
|
record := chunkPlanRecord(t, prepared, plan)
|
||||||
|
record.Producer.ChunkModule = "dnd/scenes"
|
||||||
|
store := &recordingChunkPlanStore{record: record, decision: ChunkPlanDecision{Status: ChunkPlanHit}}
|
||||||
|
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if output.Manifest.ChunkPlan.ProducerModule != "dnd/scenes" || output.Manifest.ChunkPlan.Action != "reused" {
|
||||||
|
t.Fatalf("cross-domain annotation plan was not reused: %#v", output.Manifest.ChunkPlan)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerRetriesBeforePublishingAcceptedPlan(t *testing.T) {
|
||||||
|
prepared, plan := preparedTerminalDebugPipeline(t)
|
||||||
|
prepared.resolved.Chunk.Retries = 1
|
||||||
|
chunker := &retryingChunker{key: prepared.resolved.Chunk.Module, plan: plan}
|
||||||
|
prepared.chunker = chunker
|
||||||
|
store := &recordingChunkPlanStore{decision: ChunkPlanDecision{Status: ChunkPlanMissing}}
|
||||||
|
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if chunker.calls != 2 || store.saves != 1 {
|
||||||
|
t.Fatalf("module calls = %d saves = %d, want 2 and 1", chunker.calls, store.saves)
|
||||||
|
}
|
||||||
|
if len(output.Warnings) != 1 || output.Warnings[0].Scope != "accepted" || len(store.saved.Warnings) != 1 || store.saved.Warnings[0].Scope != "accepted" {
|
||||||
|
t.Fatalf("output warnings = %#v stored warnings = %#v", output.Warnings, store.saved.Warnings)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerAutoHitValidatesOnceWithoutRegenerationOrMutation(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
result contracts.ValidationResult
|
||||||
|
validatorErr error
|
||||||
|
wantError string
|
||||||
|
wantReject bool
|
||||||
|
}{
|
||||||
|
{name: "warning", result: contracts.ValidationResult{Approved: true, Warnings: []contracts.Warning{{Scope: "current", ReasonCode: "observed", Message: "current warning"}}}},
|
||||||
|
{name: "rejection", result: contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "rejected hit"}, wantReject: true},
|
||||||
|
{name: "error", validatorErr: errors.New("validator failed"), wantError: "validator failed"},
|
||||||
|
}
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
prepared, plan := preparedTerminalDebugPipeline(t)
|
||||||
|
calls := 0
|
||||||
|
llmCalls := 0
|
||||||
|
prepared.chunker = llmCountingChunker{terminalChunker: terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls}, llmCalls: &llmCalls}
|
||||||
|
validator := &countingChunkValidator{result: tc.result, err: tc.validatorErr}
|
||||||
|
prepared.chunkValidators.validators = []preparedValidator{{resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk}, chunk: validator}}
|
||||||
|
record := chunkPlanRecord(t, prepared, plan)
|
||||||
|
record.Warnings = []contracts.Warning{{Scope: "stored", ReasonCode: "observed", Message: "stored warning"}}
|
||||||
|
store := &recordingChunkPlanStore{record: record, decision: ChunkPlanDecision{Status: ChunkPlanHit}}
|
||||||
|
debug := newCapturedDebugRecorder()
|
||||||
|
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store, Debug: debug})
|
||||||
|
if tc.wantError != "" {
|
||||||
|
if err == nil || !strings.Contains(err.Error(), tc.wantError) {
|
||||||
|
t.Fatalf("Run() error = %v, want %q", err, tc.wantError)
|
||||||
|
}
|
||||||
|
} else if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if validator.calls != 1 || calls != 0 || llmCalls != 0 || store.loads != 1 || store.saves != 0 {
|
||||||
|
t.Fatalf("calls = validator %d module %d llm %d load %d save %d", validator.calls, calls, llmCalls, store.loads, store.saves)
|
||||||
|
}
|
||||||
|
assertAttemptEnvelopeSequence(t, debug, "chunk")
|
||||||
|
if tc.wantError == "" {
|
||||||
|
encoded := string(debug.json["chunk/output.json"])
|
||||||
|
if !strings.Contains(encoded, `"status":"hit"`) || !strings.Contains(encoded, `"plan":`) || !strings.Contains(encoded, `"materialized_chunks":`) {
|
||||||
|
t.Fatalf("chunk hit debug = %s", encoded)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if tc.wantReject && (len(output.Rejected) != 1 || output.Rejected[0].ReasonCode != "rejected") {
|
||||||
|
t.Fatalf("rejected = %#v", output.Rejected)
|
||||||
|
}
|
||||||
|
if tc.wantError == "" && len(output.Warnings) != 1+len(tc.result.Warnings) {
|
||||||
|
t.Fatalf("warnings = %#v, want stored warning once plus current warnings", output.Warnings)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerPublishesOnlyAcceptedGeneratedPlans(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
moduleErr error
|
||||||
|
validator contracts.ValidationResult
|
||||||
|
cancel bool
|
||||||
|
wantCalls int
|
||||||
|
wantReject bool
|
||||||
|
}{
|
||||||
|
{name: "module error", moduleErr: errors.New("generation failed"), wantCalls: 2},
|
||||||
|
{name: "validator rejection", validator: contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "rejected generated plan"}, wantCalls: 2, wantReject: true},
|
||||||
|
{name: "cancellation", cancel: true},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
prepared, plan := preparedTerminalDebugPipeline(t)
|
||||||
|
prepared.resolved.Chunk.Retries = 1
|
||||||
|
calls := 0
|
||||||
|
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls, err: tc.moduleErr}
|
||||||
|
if tc.wantReject {
|
||||||
|
validator := &countingChunkValidator{result: tc.validator}
|
||||||
|
prepared.chunkValidators.validators = []preparedValidator{{resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk}, chunk: validator}}
|
||||||
|
}
|
||||||
|
ctx := context.Background()
|
||||||
|
if tc.cancel {
|
||||||
|
cancelled, cancel := context.WithCancel(ctx)
|
||||||
|
cancel()
|
||||||
|
ctx = cancelled
|
||||||
|
}
|
||||||
|
store := &recordingChunkPlanStore{decision: ChunkPlanDecision{Status: ChunkPlanInvalid}}
|
||||||
|
output, err := New().Run(ctx, RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store})
|
||||||
|
if tc.wantReject {
|
||||||
|
if err != nil || len(output.Rejected) != 1 {
|
||||||
|
t.Fatalf("Run() error = %v rejected = %#v", err, output.Rejected)
|
||||||
|
}
|
||||||
|
} else if err == nil {
|
||||||
|
t.Fatal("Run() error = nil")
|
||||||
|
}
|
||||||
|
if calls != tc.wantCalls || store.saves != 0 {
|
||||||
|
t.Fatalf("module calls = %d, saves = %d; want %d, 0", calls, store.saves, tc.wantCalls)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerStoresProducerProvenanceAndProducerWarnings(t *testing.T) {
|
||||||
|
prepared, plan := preparedTerminalDebugPipeline(t)
|
||||||
|
prepared.resolved.Chunk.LLMProfile = "chunk-profile"
|
||||||
|
prepared.resolved.ChunkReferences = ResolvedReferenceTarget{
|
||||||
|
Stage: StageChunk,
|
||||||
|
ReferenceSet: contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
|
||||||
|
"guide": {Items: []contracts.ReferenceItem{{SlotName: "guide", Digest: "sha256:guide", Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///guide.txt"}, Content: []byte("sensitive")}}},
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
prepared.chunker = manifestChunker{terminalChunker: terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, warnings: []contracts.Warning{{Scope: "producer", ReasonCode: "observed", Message: "producer warning"}}}, metadata: map[string]any{"prompt_id": "chunk/prompt"}}
|
||||||
|
validator := &countingChunkValidator{result: contracts.ValidationResult{Approved: true, Warnings: []contracts.Warning{{Scope: "validator", ReasonCode: "observed", Message: "validator warning"}}}}
|
||||||
|
prepared.chunkValidators.validators = []preparedValidator{{resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk}, chunk: validator}}
|
||||||
|
store := &recordingChunkPlanStore{}
|
||||||
|
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheRefresh, ChunkPlans: store}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
producer := store.saved.Producer
|
||||||
|
if producer.InputModule != prepared.input.Key() || producer.ChunkModule != prepared.chunker.Key() || producer.LLMProfile != "chunk-profile" || producer.Metadata["prompt_id"] != "chunk/prompt" {
|
||||||
|
t.Fatalf("producer = %#v", producer)
|
||||||
|
}
|
||||||
|
if len(producer.References) != 1 || producer.References[0].Digest != "sha256:guide" {
|
||||||
|
t.Fatalf("producer references = %#v", producer.References)
|
||||||
|
}
|
||||||
|
if len(store.saved.Warnings) != 1 || store.saved.Warnings[0].Scope != "producer" {
|
||||||
|
t.Fatalf("stored warnings = %#v, want only producer warning", store.saved.Warnings)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerOmitsProducerProfileForDeterministicChunker(t *testing.T) {
|
||||||
|
prepared, plan := preparedTerminalDebugPipeline(t)
|
||||||
|
prepared.resolved.Chunk.LLMProfile = "configured-but-unused"
|
||||||
|
prepared.chunker = deterministicChunker{terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan}}
|
||||||
|
store := &recordingChunkPlanStore{}
|
||||||
|
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheRefresh, ChunkPlans: store})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if store.saved.Producer.LLMProfile != "" || output.Manifest.ChunkPlan.ProducerLLMProfile != "" {
|
||||||
|
t.Fatalf("deterministic producer profile = stored %q manifest %q", store.saved.Producer.LLMProfile, output.Manifest.ChunkPlan.ProducerLLMProfile)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerRefreshChangesDownstreamChunkFingerprint(t *testing.T) {
|
||||||
|
doc := typedTestDocumentWithUnits(2)
|
||||||
|
prepared := preparedConcurrentPipeline(t, 1)
|
||||||
|
prepared.lanes = prepared.lanes[:1]
|
||||||
|
prepared.resolved.ArtifactLanes = prepared.resolved.ArtifactLanes[:1]
|
||||||
|
prepared.ArtifactLanes = prepared.ArtifactLanes[:1]
|
||||||
|
prepared.input = &typedTestInput{key: prepared.resolved.Input.Module, doc: doc}
|
||||||
|
|
||||||
|
run := func(plan source.ChunkPlan) []CheckpointFingerprint {
|
||||||
|
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan}
|
||||||
|
loader := &dependencyLoader{CheckpointLoader: NoopCheckpointLoader()}
|
||||||
|
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, ChunkCacheMode: ChunkCacheRefresh, ChunkPlans: &recordingChunkPlanStore{}, Checkpoint: loader}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return loader.extractDependencies
|
||||||
|
}
|
||||||
|
separate := typedTestPlan(doc)
|
||||||
|
combined := source.ChunkPlan{SourceDigest: doc.Digest, Ranges: []source.ChunkRange{{StartUnitID: 1, EndUnitID: 2}}}
|
||||||
|
if first, second := run(separate), run(combined); reflect.DeepEqual(first, second) {
|
||||||
|
t.Fatalf("downstream fingerprints did not change: %#v", first)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func chunkPlanRecord(t *testing.T, prepared *PreparedPipeline, plan source.ChunkPlan) ChunkPlanRecord {
|
||||||
|
t.Helper()
|
||||||
|
digest, err := source.DigestChunkPlan(plan)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return ChunkPlanRecord{
|
||||||
|
SchemaVersion: ChunkPlanSchemaVersion,
|
||||||
|
SourceDigest: plan.SourceDigest,
|
||||||
|
PlanDigest: digest,
|
||||||
|
Plan: source.CloneChunkPlan(plan),
|
||||||
|
Producer: ChunkPlanProducer{InputModule: prepared.input.Key(), ChunkModule: prepared.chunker.Key()},
|
||||||
|
CreatedAt: time.Now().UTC(),
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -11,7 +11,6 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
frameworkllm "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
frameworkllm "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||||
)
|
)
|
||||||
@@ -51,12 +50,13 @@ func preparedConcurrentPipeline(t *testing.T, chunkCount int) *PreparedPipeline
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Prepare() error = %v", err)
|
t.Fatalf("Prepare() error = %v", err)
|
||||||
}
|
}
|
||||||
doc := typedTestDocument()
|
doc := typedTestDocumentWithUnits(chunkCount)
|
||||||
chunks := make([]source.Chunk, chunkCount)
|
if adapter, ok := prepared.input.(*typedTestInput); ok {
|
||||||
for i := range chunks {
|
adapter.doc = doc
|
||||||
chunks[i] = source.Chunk{ID: fmt.Sprintf("chunk-%d", i+1), SourceID: doc.ID, Index: i, Ref: doc.Units[0].Ref, Content: []byte(fmt.Sprintf(`{"chunk":%d}`, i+1)), MediaType: "application/json", Units: []source.SourceUnit{doc.Units[0]}}
|
} else {
|
||||||
|
t.Fatalf("prepared input = %T, want *typedTestInput", prepared.input)
|
||||||
}
|
}
|
||||||
prepared.chunker = &typedTestChunker{key: "typed/chunk", chunks: chunks}
|
prepared.chunker = &typedTestChunker{key: "typed/chunk", plan: typedTestPlan(doc)}
|
||||||
return prepared
|
return prepared
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -258,7 +258,7 @@ func prepareLaneExtract(input RunInput, loader CheckpointLoader, doc *source.Sou
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, job extractJob) extractJobResult {
|
func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, job extractJob) extractJobResult {
|
||||||
state, chunk := job.lane, job.chunk
|
state, chunk := job.lane, cloneSourceChunk(job.chunk)
|
||||||
lane, typed := state.prepared.resolved, state.prepared.typed
|
lane, typed := state.prepared.resolved, state.prepared.typed
|
||||||
result := extractJobResult{laneIndex: state.index, chunkIndex: chunk.Index}
|
result := extractJobResult{laneIndex: state.index, chunkIndex: chunk.Index}
|
||||||
var accepted erasedExtractArtifact
|
var accepted erasedExtractArtifact
|
||||||
|
|||||||
@@ -2,10 +2,12 @@ package pipeline
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"reflect"
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -50,6 +52,31 @@ func cloneCheckpointArtifacts(values []CheckpointArtifact) []CheckpointArtifact
|
|||||||
return cloned
|
return cloned
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunnerPassesIndependentAnnotationScopesToExtractors(t *testing.T) {
|
||||||
|
prepared := preparedAttemptDebugPipeline(t)
|
||||||
|
chunker := prepared.chunker.(*typedTestChunker)
|
||||||
|
chunker.plan.Ranges[0].Annotations = source.ChunkAnnotations{"same": json.RawMessage(`{"range":1}`)}
|
||||||
|
chunker.plan.Annotations = source.ChunkAnnotations{"same": json.RawMessage(`{"plan":2}`)}
|
||||||
|
|
||||||
|
var gotRange, gotPlan string
|
||||||
|
installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||||
|
gotRange = string(request.Chunk.Annotations["same"])
|
||||||
|
gotPlan = string(request.Chunk.PlanAnnotations["same"])
|
||||||
|
request.Chunk.Annotations["same"][0] = '['
|
||||||
|
request.Chunk.PlanAnnotations["same"][0] = '['
|
||||||
|
return erasedTypedResult{Value: typedValueForLane(0, request.Chunk.Index)}, nil
|
||||||
|
})
|
||||||
|
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}); err != nil {
|
||||||
|
t.Fatalf("Run() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if gotRange != `{"range":1}` || gotPlan != `{"plan":2}` {
|
||||||
|
t.Fatalf("extract annotations = %q / %q", gotRange, gotPlan)
|
||||||
|
}
|
||||||
|
if string(chunker.plan.Ranges[0].Annotations["same"]) != `{"range":1}` || string(chunker.plan.Annotations["same"]) != `{"plan":2}` {
|
||||||
|
t.Fatal("extract request shares annotation bytes with chunker output")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunnerContinuesFromFreshAndReusedExtractResults(t *testing.T) {
|
func TestRunnerContinuesFromFreshAndReusedExtractResults(t *testing.T) {
|
||||||
prepared := preparedAttemptDebugPipeline(t)
|
prepared := preparedAttemptDebugPipeline(t)
|
||||||
extractCalls := 0
|
extractCalls := 0
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ import (
|
|||||||
|
|
||||||
type terminalChunker struct {
|
type terminalChunker struct {
|
||||||
key string
|
key string
|
||||||
chunks []source.Chunk
|
plan source.ChunkPlan
|
||||||
warnings []contracts.Warning
|
warnings []contracts.Warning
|
||||||
err error
|
err error
|
||||||
calls *int
|
calls *int
|
||||||
@@ -25,11 +25,11 @@ func (c terminalChunker) Key() string { return c.key }
|
|||||||
|
|
||||||
func (terminalChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
func (terminalChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||||
|
|
||||||
func (c terminalChunker) Chunk(context.Context, contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
func (c terminalChunker) Plan(context.Context, contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
|
||||||
if c.calls != nil {
|
if c.calls != nil {
|
||||||
(*c.calls)++
|
(*c.calls)++
|
||||||
}
|
}
|
||||||
return contracts.ChunkResult{Chunks: cloneSourceChunks(c.chunks), Warnings: cloneWarnings(c.warnings)}, c.err
|
return contracts.ChunkPlanResult{Plan: source.CloneChunkPlan(c.plan), Warnings: cloneWarnings(c.warnings)}, c.err
|
||||||
}
|
}
|
||||||
|
|
||||||
type terminalChunkValidator struct {
|
type terminalChunkValidator struct {
|
||||||
@@ -37,6 +37,19 @@ type terminalChunkValidator struct {
|
|||||||
err error
|
err error
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type observingChunkValidator struct {
|
||||||
|
request contracts.ChunkValidationRequest
|
||||||
|
}
|
||||||
|
|
||||||
|
func (*observingChunkValidator) Name() string { return "observing/chunk-validator" }
|
||||||
|
func (*observingChunkValidator) ExecutionClass() contracts.ExecutionClass {
|
||||||
|
return contracts.ExecutionClassDeterministic
|
||||||
|
}
|
||||||
|
func (v *observingChunkValidator) Validate(_ context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
|
||||||
|
v.request = request
|
||||||
|
return contracts.ValidationResult{Approved: true}, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (terminalChunkValidator) Name() string { return "terminal/chunk-validator" }
|
func (terminalChunkValidator) Name() string { return "terminal/chunk-validator" }
|
||||||
|
|
||||||
func (terminalChunkValidator) ExecutionClass() contracts.ExecutionClass {
|
func (terminalChunkValidator) ExecutionClass() contracts.ExecutionClass {
|
||||||
@@ -71,14 +84,14 @@ func assertAttemptEnvelopeSequence(t *testing.T, debug *capturedDebugRecorder, p
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func preparedTerminalDebugPipeline(t *testing.T) (*PreparedPipeline, []source.Chunk) {
|
func preparedTerminalDebugPipeline(t *testing.T) (*PreparedPipeline, source.ChunkPlan) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
prepared := preparedAttemptDebugPipeline(t)
|
prepared := preparedAttemptDebugPipeline(t)
|
||||||
chunker, ok := prepared.chunker.(*typedTestChunker)
|
chunker, ok := prepared.chunker.(*typedTestChunker)
|
||||||
if !ok {
|
if !ok {
|
||||||
t.Fatalf("prepared chunker = %T, want *typedTestChunker", prepared.chunker)
|
t.Fatalf("prepared chunker = %T, want *typedTestChunker", prepared.chunker)
|
||||||
}
|
}
|
||||||
return prepared, cloneSourceChunks(chunker.chunks)
|
return prepared, source.CloneChunkPlan(chunker.plan)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerRecordsChunkTerminalOutcomes(t *testing.T) {
|
func TestRunnerRecordsChunkTerminalOutcomes(t *testing.T) {
|
||||||
@@ -96,8 +109,8 @@ func TestRunnerRecordsChunkTerminalOutcomes(t *testing.T) {
|
|||||||
}
|
}
|
||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
prepared, chunks := preparedTerminalDebugPipeline(t)
|
prepared, plan := preparedTerminalDebugPipeline(t)
|
||||||
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, chunks: chunks, warnings: []contracts.Warning{{Scope: "chunk", ReasonCode: "observed", Message: "chunk warning"}}, err: tc.moduleError}
|
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, warnings: []contracts.Warning{{Scope: "chunk", ReasonCode: "observed", Message: "chunk warning"}}, err: tc.moduleError}
|
||||||
prepared.chunkValidators.validators = []preparedValidator{{resolved: ResolvedValidator{Binding: Binding("terminal/chunk-validator"), Target: ValidatorTargetChunk}, chunk: tc.validator}}
|
prepared.chunkValidators.validators = []preparedValidator{{resolved: ResolvedValidator{Binding: Binding("terminal/chunk-validator"), Target: ValidatorTargetChunk}, chunk: tc.validator}}
|
||||||
debug := newCapturedDebugRecorder()
|
debug := newCapturedDebugRecorder()
|
||||||
|
|
||||||
@@ -116,10 +129,63 @@ func TestRunnerRecordsChunkTerminalOutcomes(t *testing.T) {
|
|||||||
t.Fatalf("chunk rejection = envelope %#v, outputs %#v", envelope, output.Rejected)
|
t.Fatalf("chunk rejection = envelope %#v, outputs %#v", envelope, output.Rejected)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if tc.name == "accepted" {
|
||||||
|
encoded := string(debug.json["chunk/attempt-01.json"])
|
||||||
|
if !strings.Contains(encoded, `"plan":`) || !strings.Contains(encoded, `"materialized_chunks":`) || strings.Contains(encoded, `"chunks":`) {
|
||||||
|
t.Fatalf("chunk attempt debug = %s, want separate plan and materialized_chunks", encoded)
|
||||||
|
}
|
||||||
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunnerMaterializesAnnotatedPlanBeforeChunkValidation(t *testing.T) {
|
||||||
|
prepared, plan := preparedTerminalDebugPipeline(t)
|
||||||
|
plan.Annotations = source.ChunkAnnotations{"same": []byte(`{"plan":1}`)}
|
||||||
|
plan.Ranges[0].Annotations = source.ChunkAnnotations{"same": []byte(`{"range":2}`)}
|
||||||
|
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan}
|
||||||
|
validator := &observingChunkValidator{}
|
||||||
|
prepared.chunkValidators.validators = []preparedValidator{{
|
||||||
|
resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk},
|
||||||
|
chunk: validator,
|
||||||
|
}}
|
||||||
|
|
||||||
|
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}); err != nil {
|
||||||
|
t.Fatalf("Run() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if len(validator.request.Chunks) != 1 {
|
||||||
|
t.Fatalf("validator chunks = %#v, want one", validator.request.Chunks)
|
||||||
|
}
|
||||||
|
chunk := validator.request.Chunks[0]
|
||||||
|
if chunk.ID != "chunk-000001" || chunk.Index != 0 || chunk.MediaType != "application/json" {
|
||||||
|
t.Fatalf("materialized chunk identity = %#v", chunk)
|
||||||
|
}
|
||||||
|
if string(chunk.Annotations["same"]) != `{"range":2}` || string(chunk.PlanAnnotations["same"]) != `{"plan":1}` {
|
||||||
|
t.Fatalf("materialized annotations = %s / %s", chunk.Annotations["same"], chunk.PlanAnnotations["same"])
|
||||||
|
}
|
||||||
|
if chunk.Metadata["start_unit_id"] != 1 || chunk.Metadata["end_unit_id"] != 1 || chunk.Metadata["unit_count"] != 1 {
|
||||||
|
t.Fatalf("materialized metadata = %#v", chunk.Metadata)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerRetriesMalformedPlanWithDebugEnabled(t *testing.T) {
|
||||||
|
prepared, plan := preparedTerminalDebugPipeline(t)
|
||||||
|
plan.Ranges[0].Annotations = source.ChunkAnnotations{"broken": []byte(`{"value":`)}
|
||||||
|
prepared.resolved.Chunk.Retries = 1
|
||||||
|
calls := 0
|
||||||
|
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls}
|
||||||
|
debug := newCapturedDebugRecorder()
|
||||||
|
|
||||||
|
_, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "valid JSON") {
|
||||||
|
t.Fatalf("Run() error = %v, want malformed annotation error", err)
|
||||||
|
}
|
||||||
|
if calls != 2 {
|
||||||
|
t.Fatalf("plan calls = %d, want two attempts", calls)
|
||||||
|
}
|
||||||
|
assertAttemptEnvelopeSequence(t, debug, "chunk", 1, 2)
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunnerRecordsExtractTerminalOutcomes(t *testing.T) {
|
func TestRunnerRecordsExtractTerminalOutcomes(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
@@ -187,9 +253,9 @@ func TestRunnerRecordsExtractTerminalOutcomes(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunnerJoinsPrimaryAndAttemptWriteErrors(t *testing.T) {
|
func TestRunnerJoinsPrimaryAndAttemptWriteErrors(t *testing.T) {
|
||||||
t.Run("chunk", func(t *testing.T) {
|
t.Run("chunk", func(t *testing.T) {
|
||||||
prepared, chunks := preparedTerminalDebugPipeline(t)
|
prepared, plan := preparedTerminalDebugPipeline(t)
|
||||||
primary := errors.New("chunk operation failed")
|
primary := errors.New("chunk operation failed")
|
||||||
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, chunks: chunks, err: primary}
|
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, err: primary}
|
||||||
debug := newCapturedDebugRecorder()
|
debug := newCapturedDebugRecorder()
|
||||||
debug.failPath = "chunk/attempt-01.json"
|
debug.failPath = "chunk/attempt-01.json"
|
||||||
|
|
||||||
@@ -216,10 +282,10 @@ func TestRunnerJoinsPrimaryAndAttemptWriteErrors(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerDoesNotRetryAfterTerminalAttemptWriteFailure(t *testing.T) {
|
func TestRunnerDoesNotRetryAfterTerminalAttemptWriteFailure(t *testing.T) {
|
||||||
prepared, chunks := preparedTerminalDebugPipeline(t)
|
prepared, plan := preparedTerminalDebugPipeline(t)
|
||||||
prepared.resolved.Chunk.Retries = 1
|
prepared.resolved.Chunk.Retries = 1
|
||||||
calls := 0
|
calls := 0
|
||||||
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, chunks: chunks, calls: &calls}
|
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls}
|
||||||
prepared.chunkValidators.validators = []preparedValidator{{
|
prepared.chunkValidators.validators = []preparedValidator{{
|
||||||
resolved: ResolvedValidator{Binding: Binding("terminal/chunk-validator"), Target: ValidatorTargetChunk},
|
resolved: ResolvedValidator{Binding: Binding("terminal/chunk-validator"), Target: ValidatorTargetChunk},
|
||||||
chunk: terminalChunkValidator{result: contracts.ValidationResult{Approved: true}},
|
chunk: terminalChunkValidator{result: contracts.ValidationResult{Approved: true}},
|
||||||
|
|||||||
@@ -61,11 +61,6 @@ func (l *lockedCheckpointLoader) Source(key string) (SourceCheckpoint, Checkpoin
|
|||||||
defer l.mu.Unlock()
|
defer l.mu.Unlock()
|
||||||
return l.inner.Source(key)
|
return l.inner.Source(key)
|
||||||
}
|
}
|
||||||
func (l *lockedCheckpointLoader) Chunk(key, digest string) (ChunkCheckpoint, CheckpointDecision) {
|
|
||||||
l.mu.Lock()
|
|
||||||
defer l.mu.Unlock()
|
|
||||||
return l.inner.Chunk(key, digest)
|
|
||||||
}
|
|
||||||
func (l *lockedCheckpointLoader) Extract(lane, key string, deps []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
|
func (l *lockedCheckpointLoader) Extract(lane, key string, deps []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
|
||||||
l.mu.Lock()
|
l.mu.Lock()
|
||||||
defer l.mu.Unlock()
|
defer l.mu.Unlock()
|
||||||
@@ -107,18 +102,6 @@ func (r *lockedCheckpointRecorder) SourceSucceeded(key string, doc *source.Sourc
|
|||||||
func (r *lockedCheckpointRecorder) SourceFailed(key string, err error) error {
|
func (r *lockedCheckpointRecorder) SourceFailed(key string, err error) error {
|
||||||
return r.call(func() error { return r.inner.SourceFailed(key, err) })
|
return r.call(func() error { return r.inner.SourceFailed(key, err) })
|
||||||
}
|
}
|
||||||
func (r *lockedCheckpointRecorder) ChunkRunning(key, digest string) error {
|
|
||||||
return r.call(func() error { return r.inner.ChunkRunning(key, digest) })
|
|
||||||
}
|
|
||||||
func (r *lockedCheckpointRecorder) ChunkSucceeded(key, digest string, chunks []source.Chunk, warnings []contracts.Warning) error {
|
|
||||||
return r.call(func() error { return r.inner.ChunkSucceeded(key, digest, chunks, warnings) })
|
|
||||||
}
|
|
||||||
func (r *lockedCheckpointRecorder) ChunkRejected(key, digest string, rejected contracts.RejectedOutput) error {
|
|
||||||
return r.call(func() error { return r.inner.ChunkRejected(key, digest, rejected) })
|
|
||||||
}
|
|
||||||
func (r *lockedCheckpointRecorder) ChunkFailed(key, digest string, err error) error {
|
|
||||||
return r.call(func() error { return r.inner.ChunkFailed(key, digest, err) })
|
|
||||||
}
|
|
||||||
func (r *lockedCheckpointRecorder) ExtractRunning(lane, key string, deps []CheckpointFingerprint) error {
|
func (r *lockedCheckpointRecorder) ExtractRunning(lane, key string, deps []CheckpointFingerprint) error {
|
||||||
return r.call(func() error { return r.inner.ExtractRunning(lane, key, deps) })
|
return r.call(func() error { return r.inner.ExtractRunning(lane, key, deps) })
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -68,13 +68,13 @@ func (v *typedTestInput) Parse(context.Context, contracts.ParseRequest) (*source
|
|||||||
|
|
||||||
type typedTestChunker struct {
|
type typedTestChunker struct {
|
||||||
key string
|
key string
|
||||||
chunks []source.Chunk
|
plan source.ChunkPlan
|
||||||
}
|
}
|
||||||
|
|
||||||
func (v *typedTestChunker) Key() string { return v.key }
|
func (v *typedTestChunker) Key() string { return v.key }
|
||||||
func (v *typedTestChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
func (v *typedTestChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||||
func (v *typedTestChunker) Chunk(context.Context, contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
func (v *typedTestChunker) Plan(context.Context, contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
|
||||||
return contracts.ChunkResult{Chunks: v.chunks}, nil
|
return contracts.ChunkPlanResult{Plan: source.CloneChunkPlan(v.plan)}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type typedTestOutput struct{ key string }
|
type typedTestOutput struct{ key string }
|
||||||
@@ -84,12 +84,27 @@ func (v *typedTestOutput) Encode(context.Context, contracts.OutputRequest) (cont
|
|||||||
return contracts.OutputResult{}, nil
|
return contracts.OutputResult{}, nil
|
||||||
}
|
}
|
||||||
func typedTestDocument() *source.SourceDocument {
|
func typedTestDocument() *source.SourceDocument {
|
||||||
ref := source.SourceRef{SourceID: "source", StartUnitID: 1, EndUnitID: 1}
|
return typedTestDocumentWithUnits(1)
|
||||||
doc := &source.SourceDocument{ID: "source", Kind: "document", Format: "text/plain", Units: []source.SourceUnit{{ID: 1, Kind: "line", Text: "text", Ref: ref}}}
|
}
|
||||||
|
|
||||||
|
func typedTestDocumentWithUnits(count int) *source.SourceDocument {
|
||||||
|
doc := &source.SourceDocument{ID: "source", Kind: "document", Format: "text/plain"}
|
||||||
|
for id := 1; id <= count; id++ {
|
||||||
|
ref := source.SourceRef{SourceID: doc.ID, StartUnitID: id, EndUnitID: id}
|
||||||
|
doc.Units = append(doc.Units, source.SourceUnit{ID: id, Kind: "line", Text: fmt.Sprintf("text-%d", id), Ref: ref})
|
||||||
|
}
|
||||||
doc.Digest, _ = source.DigestDocument(doc)
|
doc.Digest, _ = source.DigestDocument(doc)
|
||||||
return doc
|
return doc
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func typedTestPlan(doc *source.SourceDocument) source.ChunkPlan {
|
||||||
|
plan := source.ChunkPlan{SourceDigest: doc.Digest, Ranges: make([]source.ChunkRange, len(doc.Units))}
|
||||||
|
for i, unit := range doc.Units {
|
||||||
|
plan.Ranges[i] = source.ChunkRange{StartUnitID: unit.ID, EndUnitID: unit.ID}
|
||||||
|
}
|
||||||
|
return plan
|
||||||
|
}
|
||||||
|
|
||||||
func (v typedTestSerializedValidator) Name() string { return v.key }
|
func (v typedTestSerializedValidator) Name() string { return v.key }
|
||||||
func (typedTestSerializedValidator) ExecutionClass() contracts.ExecutionClass {
|
func (typedTestSerializedValidator) ExecutionClass() contracts.ExecutionClass {
|
||||||
return contracts.ExecutionClassDeterministic
|
return contracts.ExecutionClassDeterministic
|
||||||
@@ -466,7 +481,7 @@ func mustRegisterTypedTestBase(t *testing.T, catalog ModuleCatalog) {
|
|||||||
}
|
}
|
||||||
if err := catalog.Chunkers.RegisterWithSpec(ModuleSpec{Key: "typed/chunk", Stage: StageChunk}, func() (contracts.Chunker, error) {
|
if err := catalog.Chunkers.RegisterWithSpec(ModuleSpec{Key: "typed/chunk", Stage: StageChunk}, func() (contracts.Chunker, error) {
|
||||||
doc := typedTestDocument()
|
doc := typedTestDocument()
|
||||||
return &typedTestChunker{key: "typed/chunk", chunks: []source.Chunk{{ID: "chunk-1", SourceID: doc.ID, Index: 0, Ref: doc.Units[0].Ref, Content: []byte(`{"chunk":1}`), MediaType: "application/json", Units: []source.SourceUnit{doc.Units[0]}}}}, nil
|
return &typedTestChunker{key: "typed/chunk", plan: typedTestPlan(doc)}, nil
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
t.Fatalf("register chunker: %v", err)
|
t.Fatalf("register chunker: %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -20,9 +20,10 @@ var requiredCapabilities = []string{
|
|||||||
|
|
||||||
var providedCapabilities = []string{
|
var providedCapabilities = []string{
|
||||||
"chunks",
|
"chunks",
|
||||||
"chunks.scenes",
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const annotationNamespace = "dnd/scenes"
|
||||||
|
|
||||||
var referenceSlotDescriptions = shared.ReferenceSlotDescriptions{
|
var referenceSlotDescriptions = shared.ReferenceSlotDescriptions{
|
||||||
Glossary: "Optional campaign glossary reference material used only for scene disambiguation.",
|
Glossary: "Optional campaign glossary reference material used only for scene disambiguation.",
|
||||||
Party: "Optional party roster reference material used only for scene disambiguation.",
|
Party: "Optional party roster reference material used only for scene disambiguation.",
|
||||||
@@ -50,6 +51,10 @@ func (c *Chunker) Key() string {
|
|||||||
return Key
|
return Key
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (*Chunker) ExecutionClass() contracts.ExecutionClass {
|
||||||
|
return contracts.ExecutionClassLLMBacked
|
||||||
|
}
|
||||||
|
|
||||||
func (c *Chunker) ReferenceSlots() []contracts.ReferenceSlot {
|
func (c *Chunker) ReferenceSlots() []contracts.ReferenceSlot {
|
||||||
return shared.ReferenceSlots(referenceSlotDescriptions)
|
return shared.ReferenceSlots(referenceSlotDescriptions)
|
||||||
}
|
}
|
||||||
@@ -74,27 +79,27 @@ func (c *Chunker) ManifestMetadata() map[string]any {
|
|||||||
return metadata
|
return metadata
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Chunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
func (c *Chunker) Plan(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
|
||||||
if c == nil {
|
if c == nil {
|
||||||
return contracts.ChunkResult{}, chunkerErrorf("chunker must not be nil")
|
return contracts.ChunkPlanResult{}, chunkerErrorf("chunker must not be nil")
|
||||||
}
|
}
|
||||||
if c.llm == nil {
|
if c.llm == nil {
|
||||||
return contracts.ChunkResult{}, chunkerErrorf("LLM client must not be nil")
|
return contracts.ChunkPlanResult{}, chunkerErrorf("LLM client must not be nil")
|
||||||
}
|
}
|
||||||
if ctx == nil {
|
if ctx == nil {
|
||||||
return contracts.ChunkResult{}, chunkerErrorf("context must not be nil")
|
return contracts.ChunkPlanResult{}, chunkerErrorf("context must not be nil")
|
||||||
}
|
}
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return contracts.ChunkResult{}, chunkerErrorf("context error before chunking: %w", err)
|
return contracts.ChunkPlanResult{}, chunkerErrorf("context error before chunking: %w", err)
|
||||||
}
|
}
|
||||||
if req.Source == nil {
|
if req.Source == nil {
|
||||||
return contracts.ChunkResult{}, chunkerErrorf("source must not be nil")
|
return contracts.ChunkPlanResult{}, chunkerErrorf("source must not be nil")
|
||||||
}
|
}
|
||||||
if len(req.Source.Units) == 0 {
|
if len(req.Source.Units) == 0 {
|
||||||
return contracts.ChunkResult{}, chunkerErrorf("source units must not be empty")
|
return contracts.ChunkPlanResult{}, chunkerErrorf("source units must not be empty")
|
||||||
}
|
}
|
||||||
if err := source.ValidateDocument(req.Source); err != nil {
|
if err := source.ValidateDocument(req.Source); err != nil {
|
||||||
return contracts.ChunkResult{}, chunkerErrorf("validate source document: %w", err)
|
return contracts.ChunkPlanResult{}, chunkerErrorf("validate source document: %w", err)
|
||||||
}
|
}
|
||||||
var response chunkResponse
|
var response chunkResponse
|
||||||
if _, err := c.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
if _, err := c.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||||
@@ -105,19 +110,19 @@ func (c *Chunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contra
|
|||||||
SessionID: req.SessionID,
|
SessionID: req.SessionID,
|
||||||
Inputs: shared.PromptInputs(req.SourceInput, req.References),
|
Inputs: shared.PromptInputs(req.SourceInput, req.References),
|
||||||
}, &response); err != nil {
|
}, &response); err != nil {
|
||||||
return contracts.ChunkResult{}, chunkerErrorf("complete structured output: %w", err)
|
return contracts.ChunkPlanResult{}, chunkerErrorf("complete structured output: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
warnings, err := warningsFromCaveats(response.BoundaryCaveats)
|
warnings, err := warningsFromCaveats(response.BoundaryCaveats)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return contracts.ChunkResult{}, chunkerErrorf("malformed structured output: %w", err)
|
return contracts.ChunkPlanResult{}, chunkerErrorf("malformed structured output: %w", err)
|
||||||
}
|
}
|
||||||
chunks, err := chunksFromResponse(req.Source, response)
|
plan, err := planFromResponse(req.Source, response, warnings)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return contracts.ChunkResult{}, chunkerErrorf("malformed structured output: %w", err)
|
return contracts.ChunkPlanResult{}, chunkerErrorf("malformed structured output: %w", err)
|
||||||
}
|
}
|
||||||
return contracts.ChunkResult{
|
return contracts.ChunkPlanResult{
|
||||||
Chunks: chunks,
|
Plan: plan,
|
||||||
Warnings: warnings,
|
Warnings: warnings,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
@@ -154,12 +159,12 @@ func DecodeOptions(options map[string]any) (Options, error) {
|
|||||||
return Options{}, nil
|
return Options{}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func chunksFromResponse(doc *source.SourceDocument, response chunkResponse) ([]source.Chunk, error) {
|
func planFromResponse(doc *source.SourceDocument, response chunkResponse, warnings []contracts.Warning) (source.ChunkPlan, error) {
|
||||||
if response.Scenes == nil {
|
if response.Scenes == nil {
|
||||||
return nil, fmt.Errorf("scenes must be present")
|
return source.ChunkPlan{}, fmt.Errorf("scenes must be present")
|
||||||
}
|
}
|
||||||
if len(response.Scenes) == 0 {
|
if len(response.Scenes) == 0 {
|
||||||
return nil, fmt.Errorf("scenes must not be empty")
|
return source.ChunkPlan{}, fmt.Errorf("scenes must not be empty")
|
||||||
}
|
}
|
||||||
|
|
||||||
unitIndexes := make(map[int]int, len(doc.Units))
|
unitIndexes := make(map[int]int, len(doc.Units))
|
||||||
@@ -167,86 +172,75 @@ func chunksFromResponse(doc *source.SourceDocument, response chunkResponse) ([]s
|
|||||||
unitIndexes[unit.ID] = i
|
unitIndexes[unit.ID] = i
|
||||||
}
|
}
|
||||||
|
|
||||||
chunks := make([]source.Chunk, 0, len(response.Scenes))
|
ranges := make([]source.ChunkRange, 0, len(response.Scenes))
|
||||||
previousEnd := -1
|
previousEnd := -1
|
||||||
for i, scene := range response.Scenes {
|
for i, scene := range response.Scenes {
|
||||||
normalized, err := normalizeScene(doc, i, scene)
|
normalized, err := normalizeScene(doc, i, scene)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return source.ChunkPlan{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
startIndex, ok := unitIndexes[normalized.StartUnitID]
|
startIndex, ok := unitIndexes[normalized.StartUnitID]
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, fmt.Errorf("scene[%d] start_unit_id %d was not found", i, normalized.StartUnitID)
|
return source.ChunkPlan{}, fmt.Errorf("scene[%d] start_unit_id %d was not found", i, normalized.StartUnitID)
|
||||||
}
|
}
|
||||||
endIndex, ok := unitIndexes[normalized.EndUnitID]
|
endIndex, ok := unitIndexes[normalized.EndUnitID]
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, fmt.Errorf("scene[%d] end_unit_id %d was not found", i, normalized.EndUnitID)
|
return source.ChunkPlan{}, fmt.Errorf("scene[%d] end_unit_id %d was not found", i, normalized.EndUnitID)
|
||||||
}
|
}
|
||||||
if startIndex > endIndex {
|
if startIndex > endIndex {
|
||||||
return nil, fmt.Errorf("scene[%d] start_unit_id %d appears after end_unit_id %d", i, normalized.StartUnitID, normalized.EndUnitID)
|
return source.ChunkPlan{}, fmt.Errorf("scene[%d] start_unit_id %d appears after end_unit_id %d", i, normalized.StartUnitID, normalized.EndUnitID)
|
||||||
}
|
}
|
||||||
|
|
||||||
if i == 0 && startIndex != 0 {
|
if i == 0 && startIndex != 0 {
|
||||||
return nil, fmt.Errorf("first scene must start at first source unit %d", doc.Units[0].ID)
|
return source.ChunkPlan{}, fmt.Errorf("first scene must start at first source unit %d", doc.Units[0].ID)
|
||||||
}
|
}
|
||||||
if i > 0 {
|
if i > 0 {
|
||||||
if startIndex <= previousEnd {
|
if startIndex <= previousEnd {
|
||||||
return nil, fmt.Errorf("scene[%d] overlaps previous scene", i)
|
return source.ChunkPlan{}, fmt.Errorf("scene[%d] overlaps previous scene", i)
|
||||||
}
|
}
|
||||||
if startIndex > previousEnd+1 {
|
if startIndex > previousEnd+1 {
|
||||||
return nil, fmt.Errorf("scene[%d] leaves a gap after previous scene", i)
|
return source.ChunkPlan{}, fmt.Errorf("scene[%d] leaves a gap after previous scene", i)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
previousEnd = endIndex
|
previousEnd = endIndex
|
||||||
|
|
||||||
units := cloneUnits(doc.Units[startIndex : endIndex+1])
|
annotation, err := json.Marshal(struct {
|
||||||
content, err := chunkContent(units)
|
ShortTitle string `json:"short_title"`
|
||||||
|
PrimaryMode string `json:"primary_mode"`
|
||||||
|
MainParticipants []string `json:"main_participants"`
|
||||||
|
Summary string `json:"summary"`
|
||||||
|
BoundaryNote string `json:"boundary_note"`
|
||||||
|
BoundaryConfidence string `json:"boundary_confidence"`
|
||||||
|
}{normalized.ShortTitle, normalized.PrimaryMode, normalized.MainParticipants, normalized.Summary, normalized.BoundaryNote, normalized.BoundaryConfidence})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return source.ChunkPlan{}, fmt.Errorf("encode scene[%d] annotation: %w", i, err)
|
||||||
}
|
}
|
||||||
chunks = append(chunks, source.Chunk{
|
ranges = append(ranges, source.ChunkRange{
|
||||||
ID: fmt.Sprintf("scene-%06d", i+1),
|
StartUnitID: normalized.StartUnitID,
|
||||||
SourceID: doc.ID,
|
EndUnitID: normalized.EndUnitID,
|
||||||
Index: i,
|
Annotations: source.ChunkAnnotations{annotationNamespace: annotation},
|
||||||
Ref: source.SourceRef{
|
|
||||||
SourceID: doc.ID,
|
|
||||||
StartUnitID: units[0].Ref.StartUnitID,
|
|
||||||
EndUnitID: units[len(units)-1].Ref.EndUnitID,
|
|
||||||
},
|
|
||||||
Content: content,
|
|
||||||
MediaType: "application/json",
|
|
||||||
Units: units,
|
|
||||||
Metadata: map[string]any{
|
|
||||||
"scene_title": normalized.ShortTitle,
|
|
||||||
"primary_mode": normalized.PrimaryMode,
|
|
||||||
"main_participants": append([]string(nil), normalized.MainParticipants...),
|
|
||||||
"summary": normalized.Summary,
|
|
||||||
"boundary_note": normalized.BoundaryNote,
|
|
||||||
"boundary_confidence": normalized.BoundaryConfidence,
|
|
||||||
"start_unit_id": normalized.StartUnitID,
|
|
||||||
"end_unit_id": normalized.EndUnitID,
|
|
||||||
"unit_count": len(units),
|
|
||||||
},
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
if previousEnd != len(doc.Units)-1 {
|
if previousEnd != len(doc.Units)-1 {
|
||||||
return nil, fmt.Errorf("final scene must end at final source unit %d", doc.Units[len(doc.Units)-1].ID)
|
return source.ChunkPlan{}, fmt.Errorf("final scene must end at final source unit %d", doc.Units[len(doc.Units)-1].ID)
|
||||||
}
|
}
|
||||||
return chunks, nil
|
caveats := make([]string, 0, len(warnings))
|
||||||
|
for _, warning := range warnings {
|
||||||
|
caveats = append(caveats, warning.Message)
|
||||||
}
|
}
|
||||||
|
annotation, err := json.Marshal(struct {
|
||||||
func chunkContent(units []source.SourceUnit) ([]byte, error) {
|
BoundaryCaveats []string `json:"boundary_caveats"`
|
||||||
content, err := json.Marshal(struct {
|
}{BoundaryCaveats: caveats})
|
||||||
Units []source.SourceUnit `json:"units"`
|
|
||||||
}{
|
|
||||||
Units: units,
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("encode chunk content: %w", err)
|
return source.ChunkPlan{}, fmt.Errorf("encode plan annotation: %w", err)
|
||||||
}
|
}
|
||||||
return content, nil
|
return source.ChunkPlan{
|
||||||
|
SourceDigest: doc.Digest,
|
||||||
|
Ranges: ranges,
|
||||||
|
Annotations: source.ChunkAnnotations{annotationNamespace: annotation},
|
||||||
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func normalizeScene(doc *source.SourceDocument, index int, scene sceneResponse) (normalizedScene, error) {
|
func normalizeScene(doc *source.SourceDocument, index int, scene sceneResponse) (normalizedScene, error) {
|
||||||
@@ -347,31 +341,6 @@ func warningsFromCaveats(caveats []string) ([]contracts.Warning, error) {
|
|||||||
return warnings, nil
|
return warnings, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func cloneUnits(units []source.SourceUnit) []source.SourceUnit {
|
|
||||||
out := make([]source.SourceUnit, 0, len(units))
|
|
||||||
for _, unit := range units {
|
|
||||||
out = append(out, source.SourceUnit{
|
|
||||||
ID: unit.ID,
|
|
||||||
Kind: unit.Kind,
|
|
||||||
Text: unit.Text,
|
|
||||||
Ref: unit.Ref,
|
|
||||||
Metadata: cloneMetadata(unit.Metadata),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func cloneMetadata(metadata map[string]any) map[string]any {
|
|
||||||
if len(metadata) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
out := make(map[string]any, len(metadata))
|
|
||||||
for key, value := range metadata {
|
|
||||||
out[key] = value
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func chunkerErrorf(format string, args ...any) error {
|
func chunkerErrorf(format string, args ...any) error {
|
||||||
return fmt.Errorf("dnd scenes chunker: "+format, args...)
|
return fmt.Errorf("dnd scenes chunker: "+format, args...)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ func TestNewModuleSpecAndRegister(t *testing.T) {
|
|||||||
Key: Key,
|
Key: Key,
|
||||||
Stage: pipeline.StageChunk,
|
Stage: pipeline.StageChunk,
|
||||||
Requires: []string{"source.transcript"},
|
Requires: []string{"source.transcript"},
|
||||||
Provides: []string{"chunks", "chunks.scenes"},
|
Provides: []string{"chunks"},
|
||||||
ReferenceSlots: wantReferenceSlots(),
|
ReferenceSlots: wantReferenceSlots(),
|
||||||
}
|
}
|
||||||
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
|
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
|
||||||
@@ -110,7 +110,7 @@ func wantReferenceSlots() []contracts.ReferenceSlot {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestChunkReturnsSceneChunksFromStructuredOutput(t *testing.T) {
|
func TestPlanReturnsSceneRangesAndAnnotationsFromStructuredOutput(t *testing.T) {
|
||||||
client := &fakeScenesLLMClient{
|
client := &fakeScenesLLMClient{
|
||||||
response: chunkResponse{
|
response: chunkResponse{
|
||||||
Scenes: []sceneResponse{
|
Scenes: []sceneResponse{
|
||||||
@@ -139,9 +139,9 @@ func TestChunkReturnsSceneChunksFromStructuredOutput(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
result, err := newChunker(t, client).Chunk(context.Background(), chunkRequest())
|
result, err := newChunker(t, client).Plan(context.Background(), chunkRequest())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Chunk() error = %v, want nil", err)
|
t.Fatalf("Plan() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(client.requests) != 1 {
|
if len(client.requests) != 1 {
|
||||||
@@ -177,36 +177,41 @@ func TestChunkReturnsSceneChunksFromStructuredOutput(t *testing.T) {
|
|||||||
t.Fatalf("glossary input = %q, want empty reference placeholder", got)
|
t.Fatalf("glossary input = %q, want empty reference placeholder", got)
|
||||||
}
|
}
|
||||||
|
|
||||||
if got := chunkIDs(result.Chunks); !reflect.DeepEqual(got, []string{"scene-000001", "scene-000002"}) {
|
if result.Plan.SourceDigest != "sha256:source" {
|
||||||
t.Fatalf("chunk IDs = %#v, want deterministic scene IDs", got)
|
t.Fatalf("SourceDigest = %q, want source digest", result.Plan.SourceDigest)
|
||||||
}
|
}
|
||||||
gotUnits := [][]int{unitIDs(result.Chunks[0].Units), unitIDs(result.Chunks[1].Units)}
|
wantRanges := []source.ChunkRange{{StartUnitID: 1, EndUnitID: 2}, {StartUnitID: 3, EndUnitID: 4}}
|
||||||
wantUnits := [][]int{{1, 2}, {3, 4}}
|
if len(result.Plan.Ranges) != len(wantRanges) {
|
||||||
if !reflect.DeepEqual(gotUnits, wantUnits) {
|
t.Fatalf("ranges = %#v, want two", result.Plan.Ranges)
|
||||||
t.Fatalf("chunk units = %#v, want %#v", gotUnits, wantUnits)
|
|
||||||
}
|
}
|
||||||
first := result.Chunks[0]
|
for i, want := range wantRanges {
|
||||||
if first.SourceID != "session-alpha" || first.Index != 0 {
|
if result.Plan.Ranges[i].StartUnitID != want.StartUnitID || result.Plan.Ranges[i].EndUnitID != want.EndUnitID {
|
||||||
t.Fatalf("first chunk = %#v, want source and index fields", first)
|
t.Fatalf("range[%d] = %#v, want %#v", i, result.Plan.Ranges[i], want)
|
||||||
}
|
}
|
||||||
if first.Ref != (source.SourceRef{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 2}) {
|
|
||||||
t.Fatalf("first ref = %#v, want session-alpha:1-2", first.Ref)
|
|
||||||
}
|
}
|
||||||
if first.MediaType != "application/json" || len(first.Content) == 0 {
|
var firstAnnotation map[string]any
|
||||||
t.Fatalf("first payload = media type %q length %d, want JSON content", first.MediaType, len(first.Content))
|
if err := json.Unmarshal(result.Plan.Ranges[0].Annotations[annotationNamespace], &firstAnnotation); err != nil {
|
||||||
|
t.Fatalf("decode first scene annotation: %v", err)
|
||||||
}
|
}
|
||||||
if first.Metadata["scene_title"] != "Goblin parley" ||
|
wantFirst := map[string]any{
|
||||||
first.Metadata["primary_mode"] != "Discussion" ||
|
"short_title": "Goblin parley", "primary_mode": "Discussion",
|
||||||
first.Metadata["summary"] != "The party negotiates with a scout." ||
|
"main_participants": []any{"Aria", "Goblin scout"},
|
||||||
first.Metadata["boundary_note"] != "The scene covers the discussion before fighting starts." ||
|
"summary": "The party negotiates with a scout.",
|
||||||
first.Metadata["boundary_confidence"] != "High" ||
|
"boundary_note": "The scene covers the discussion before fighting starts.",
|
||||||
first.Metadata["start_unit_id"] != 1 ||
|
"boundary_confidence": "High",
|
||||||
first.Metadata["end_unit_id"] != 2 ||
|
|
||||||
first.Metadata["unit_count"] != 2 {
|
|
||||||
t.Fatalf("first metadata = %#v, want scene metadata", first.Metadata)
|
|
||||||
}
|
}
|
||||||
if got, ok := first.Metadata["main_participants"].([]string); !ok || !reflect.DeepEqual(got, []string{"Aria", "Goblin scout"}) {
|
if !reflect.DeepEqual(firstAnnotation, wantFirst) {
|
||||||
t.Fatalf("main_participants = %#v, want trimmed participant slice", first.Metadata["main_participants"])
|
t.Fatalf("first scene annotation = %#v, want %#v", firstAnnotation, wantFirst)
|
||||||
|
}
|
||||||
|
if len(firstAnnotation) != 6 {
|
||||||
|
t.Fatalf("first scene annotation keys = %#v, want exact six fields", firstAnnotation)
|
||||||
|
}
|
||||||
|
var planAnnotation map[string]any
|
||||||
|
if err := json.Unmarshal(result.Plan.Annotations[annotationNamespace], &planAnnotation); err != nil {
|
||||||
|
t.Fatalf("decode plan annotation: %v", err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(planAnnotation, map[string]any{"boundary_caveats": []any{"The transition into combat is gradual."}}) || len(planAnnotation) != 1 {
|
||||||
|
t.Fatalf("plan annotation = %#v, want normalized caveats only", planAnnotation)
|
||||||
}
|
}
|
||||||
if got := result.Warnings; len(got) != 1 ||
|
if got := result.Warnings; len(got) != 1 ||
|
||||||
got[0].Scope != Key ||
|
got[0].Scope != Key ||
|
||||||
@@ -216,7 +221,7 @@ func TestChunkReturnsSceneChunksFromStructuredOutput(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestChunkPassesReferencesAsPromptInputs(t *testing.T) {
|
func TestPlanPassesReferencesAsPromptInputs(t *testing.T) {
|
||||||
client := &fakeScenesLLMClient{response: chunkResponse{
|
client := &fakeScenesLLMClient{response: chunkResponse{
|
||||||
Scenes: []sceneResponse{
|
Scenes: []sceneResponse{
|
||||||
{
|
{
|
||||||
@@ -255,8 +260,8 @@ func TestChunkPassesReferencesAsPromptInputs(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, err := newChunker(t, client).Chunk(context.Background(), req); err != nil {
|
if _, err := newChunker(t, client).Plan(context.Background(), req); err != nil {
|
||||||
t.Fatalf("Chunk() error = %v, want nil", err)
|
t.Fatalf("Plan() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
request := client.requests[0]
|
request := client.requests[0]
|
||||||
if got := string(request.Inputs["players"].Content); got != "Alice: Aria" {
|
if got := string(request.Inputs["players"].Content); got != "Alice: Aria" {
|
||||||
@@ -293,7 +298,7 @@ func TestPromptInputsMapLegacyRosterReferenceToParty(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestChunkRejectsWhitespaceOnlyBoundaryCaveats(t *testing.T) {
|
func TestPlanRejectsWhitespaceOnlyBoundaryCaveats(t *testing.T) {
|
||||||
client := &fakeScenesLLMClient{
|
client := &fakeScenesLLMClient{
|
||||||
response: chunkResponse{
|
response: chunkResponse{
|
||||||
Scenes: validSceneResponse().Scenes,
|
Scenes: validSceneResponse().Scenes,
|
||||||
@@ -303,46 +308,38 @@ func TestChunkRejectsWhitespaceOnlyBoundaryCaveats(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err := newChunker(t, client).Chunk(context.Background(), chunkRequest())
|
_, err := newChunker(t, client).Plan(context.Background(), chunkRequest())
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("Chunk() error = nil, want malformed structured output error")
|
t.Fatal("Plan() error = nil, want malformed structured output error")
|
||||||
}
|
}
|
||||||
if !strings.Contains(err.Error(), "dnd scenes chunker") || !strings.Contains(err.Error(), "malformed structured output") || !strings.Contains(err.Error(), "boundary_caveats[0]") {
|
if !strings.Contains(err.Error(), "dnd scenes chunker") || !strings.Contains(err.Error(), "malformed structured output") || !strings.Contains(err.Error(), "boundary_caveats[0]") {
|
||||||
t.Fatalf("Chunk() error = %q, want malformed boundary caveat context", err.Error())
|
t.Fatalf("Plan() error = %q, want malformed boundary caveat context", err.Error())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestChunkDefensivelyCopiesSourceUnitsAndMetadata(t *testing.T) {
|
func TestPlanDefensivelyCopiesAnnotationValues(t *testing.T) {
|
||||||
doc := sceneSourceDocument()
|
doc := sceneSourceDocument()
|
||||||
client := &fakeScenesLLMClient{response: validSceneResponse()}
|
client := &fakeScenesLLMClient{response: validSceneResponse()}
|
||||||
|
|
||||||
result, err := newChunker(t, client).Chunk(context.Background(), contracts.ChunkRequest{
|
result, err := newChunker(t, client).Plan(context.Background(), contracts.ChunkRequest{
|
||||||
Source: doc,
|
Source: doc,
|
||||||
SourceInput: sceneSourceInput(),
|
SourceInput: sceneSourceInput(),
|
||||||
SessionID: "session-123",
|
SessionID: "session-123",
|
||||||
LLMProfile: "profile-scenes",
|
LLMProfile: "profile-scenes",
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Chunk() error = %v, want nil", err)
|
t.Fatalf("Plan() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
doc.Units[0].ID = 99
|
|
||||||
doc.Units[0].Ref.SourceID = "mutated"
|
|
||||||
doc.Units[0].Metadata["speaker"] = "mutated"
|
|
||||||
client.response.Scenes[0].MainParticipants[0] = "mutated"
|
client.response.Scenes[0].MainParticipants[0] = "mutated"
|
||||||
|
var annotation struct {
|
||||||
if result.Chunks[0].Units[0].ID != 1 {
|
MainParticipants []string `json:"main_participants"`
|
||||||
t.Fatalf("chunk unit ID changed after source mutation: %#v", result.Chunks[0].Units[0])
|
|
||||||
}
|
}
|
||||||
if got := result.Chunks[0].Units[0].Ref.SourceID; got != "session-alpha" {
|
if err := json.Unmarshal(result.Plan.Ranges[0].Annotations[annotationNamespace], &annotation); err != nil {
|
||||||
t.Fatalf("chunk unit ref changed after source mutation: %#v", result.Chunks[0].Units[0].Ref)
|
t.Fatalf("decode annotation: %v", err)
|
||||||
}
|
}
|
||||||
if result.Chunks[0].Units[0].Metadata["speaker"] != "Alice" {
|
if !reflect.DeepEqual(annotation.MainParticipants, []string{"Aria"}) {
|
||||||
t.Fatalf("chunk unit metadata changed after source mutation: %#v", result.Chunks[0].Units[0].Metadata)
|
t.Fatalf("participants = %#v, want defensive copy", annotation.MainParticipants)
|
||||||
}
|
|
||||||
participants, ok := result.Chunks[0].Metadata["main_participants"].([]string)
|
|
||||||
if !ok || participants[0] != "Aria" {
|
|
||||||
t.Fatalf("participants = %#v, want defensive copy", result.Chunks[0].Metadata["main_participants"])
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -375,7 +372,7 @@ func TestChunkerManifestMetadataIncludesPromptAndSchemaProvenance(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestChunkRejectsInvalidRequests(t *testing.T) {
|
func TestPlanRejectsInvalidRequests(t *testing.T) {
|
||||||
validClient := &fakeScenesLLMClient{response: validSceneResponse()}
|
validClient := &fakeScenesLLMClient{response: validSceneResponse()}
|
||||||
validReq := chunkRequest()
|
validReq := chunkRequest()
|
||||||
canceledCtx, cancel := context.WithCancel(context.Background())
|
canceledCtx, cancel := context.WithCancel(context.Background())
|
||||||
@@ -402,18 +399,18 @@ func TestChunkRejectsInvalidRequests(t *testing.T) {
|
|||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
_, err := tt.chunker.Chunk(tt.ctx, tt.req)
|
_, err := tt.chunker.Plan(tt.ctx, tt.req)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("Chunk() error = nil, want error")
|
t.Fatal("Plan() error = nil, want error")
|
||||||
}
|
}
|
||||||
if !strings.Contains(err.Error(), "dnd scenes") || !strings.Contains(err.Error(), tt.want) {
|
if !strings.Contains(err.Error(), "dnd scenes") || !strings.Contains(err.Error(), tt.want) {
|
||||||
t.Fatalf("Chunk() error = %q, want module context and %q", err.Error(), tt.want)
|
t.Fatalf("Plan() error = %q, want module context and %q", err.Error(), tt.want)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestChunkRejectsMalformedStructuredOutput(t *testing.T) {
|
func TestPlanRejectsMalformedStructuredOutput(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
response chunkResponse
|
response chunkResponse
|
||||||
@@ -495,26 +492,26 @@ func TestChunkRejectsMalformedStructuredOutput(t *testing.T) {
|
|||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
client := &fakeScenesLLMClient{response: tt.response}
|
client := &fakeScenesLLMClient{response: tt.response}
|
||||||
_, err := newChunker(t, client).Chunk(context.Background(), chunkRequest())
|
_, err := newChunker(t, client).Plan(context.Background(), chunkRequest())
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("Chunk() error = nil, want error")
|
t.Fatal("Plan() error = nil, want error")
|
||||||
}
|
}
|
||||||
if !strings.Contains(err.Error(), "dnd scenes") || !strings.Contains(err.Error(), tt.want) {
|
if !strings.Contains(err.Error(), "dnd scenes") || !strings.Contains(err.Error(), tt.want) {
|
||||||
t.Fatalf("Chunk() error = %q, want module context and %q", err.Error(), tt.want)
|
t.Fatalf("Plan() error = %q, want module context and %q", err.Error(), tt.want)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestChunkWrapsLLMClientError(t *testing.T) {
|
func TestPlanWrapsLLMClientError(t *testing.T) {
|
||||||
client := &fakeScenesLLMClient{err: errors.New("provider unavailable")}
|
client := &fakeScenesLLMClient{err: errors.New("provider unavailable")}
|
||||||
|
|
||||||
_, err := newChunker(t, client).Chunk(context.Background(), chunkRequest())
|
_, err := newChunker(t, client).Plan(context.Background(), chunkRequest())
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("Chunk() error = nil, want LLM error")
|
t.Fatal("Plan() error = nil, want LLM error")
|
||||||
}
|
}
|
||||||
if !strings.Contains(err.Error(), "dnd scenes") || !strings.Contains(err.Error(), "provider unavailable") {
|
if !strings.Contains(err.Error(), "dnd scenes") || !strings.Contains(err.Error(), "provider unavailable") {
|
||||||
t.Fatalf("Chunk() error = %q, want wrapped LLM context", err.Error())
|
t.Fatalf("Plan() error = %q, want wrapped LLM context", err.Error())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -38,81 +38,54 @@ func (c *Chunker) Key() string {
|
|||||||
return Key
|
return Key
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (*Chunker) ExecutionClass() contracts.ExecutionClass {
|
||||||
|
return contracts.ExecutionClassDeterministic
|
||||||
|
}
|
||||||
|
|
||||||
func (c *Chunker) ReferenceSlots() []contracts.ReferenceSlot {
|
func (c *Chunker) ReferenceSlots() []contracts.ReferenceSlot {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Chunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
func (c *Chunker) Plan(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
|
||||||
if c == nil {
|
if c == nil {
|
||||||
return contracts.ChunkResult{}, chunkerErrorf("chunker must not be nil")
|
return contracts.ChunkPlanResult{}, chunkerErrorf("chunker must not be nil")
|
||||||
}
|
}
|
||||||
if c.options.MaxUnits <= 0 || c.options.OverlapUnits < 0 || c.options.OverlapUnits >= c.options.MaxUnits {
|
if c.options.MaxUnits <= 0 || c.options.OverlapUnits < 0 || c.options.OverlapUnits >= c.options.MaxUnits {
|
||||||
return contracts.ChunkResult{}, chunkerErrorf("chunker options must be initialized by construction")
|
return contracts.ChunkPlanResult{}, chunkerErrorf("chunker options must be initialized by construction")
|
||||||
}
|
}
|
||||||
if ctx == nil {
|
if ctx == nil {
|
||||||
return contracts.ChunkResult{}, chunkerErrorf("context must not be nil")
|
return contracts.ChunkPlanResult{}, chunkerErrorf("context must not be nil")
|
||||||
}
|
}
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return contracts.ChunkResult{}, chunkerErrorf("context error before chunking: %w", err)
|
return contracts.ChunkPlanResult{}, chunkerErrorf("context error before chunking: %w", err)
|
||||||
}
|
}
|
||||||
if req.Source == nil {
|
if req.Source == nil {
|
||||||
return contracts.ChunkResult{}, chunkerErrorf("source must not be nil")
|
return contracts.ChunkPlanResult{}, chunkerErrorf("source must not be nil")
|
||||||
}
|
}
|
||||||
if len(req.Source.Units) == 0 {
|
if len(req.Source.Units) == 0 {
|
||||||
return contracts.ChunkResult{}, chunkerErrorf("source units must not be empty")
|
return contracts.ChunkPlanResult{}, chunkerErrorf("source units must not be empty")
|
||||||
}
|
}
|
||||||
if err := source.ValidateDocument(req.Source); err != nil {
|
if err := source.ValidateDocument(req.Source); err != nil {
|
||||||
return contracts.ChunkResult{}, chunkerErrorf("validate source document: %w", err)
|
return contracts.ChunkPlanResult{}, chunkerErrorf("validate source document: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
step := c.options.MaxUnits - c.options.OverlapUnits
|
step := c.options.MaxUnits - c.options.OverlapUnits
|
||||||
chunks := make([]source.Chunk, 0, (len(req.Source.Units)+step-1)/step)
|
ranges := make([]source.ChunkRange, 0, (len(req.Source.Units)+step-1)/step)
|
||||||
for start := 0; start < len(req.Source.Units); start += step {
|
for start := 0; start < len(req.Source.Units); start += step {
|
||||||
end := start + c.options.MaxUnits
|
end := start + c.options.MaxUnits
|
||||||
if end > len(req.Source.Units) {
|
if end > len(req.Source.Units) {
|
||||||
end = len(req.Source.Units)
|
end = len(req.Source.Units)
|
||||||
}
|
}
|
||||||
units := cloneUnits(req.Source.Units[start:end])
|
ranges = append(ranges, source.ChunkRange{
|
||||||
content, err := chunkContent(units)
|
StartUnitID: req.Source.Units[start].ID,
|
||||||
if err != nil {
|
EndUnitID: req.Source.Units[end-1].ID,
|
||||||
return contracts.ChunkResult{}, err
|
|
||||||
}
|
|
||||||
chunks = append(chunks, source.Chunk{
|
|
||||||
ID: fmt.Sprintf("chunk-%06d", len(chunks)+1),
|
|
||||||
SourceID: req.Source.ID,
|
|
||||||
Index: len(chunks),
|
|
||||||
Ref: source.SourceRef{
|
|
||||||
SourceID: req.Source.ID,
|
|
||||||
StartUnitID: units[0].Ref.StartUnitID,
|
|
||||||
EndUnitID: units[len(units)-1].Ref.EndUnitID,
|
|
||||||
},
|
|
||||||
Content: content,
|
|
||||||
MediaType: "application/json",
|
|
||||||
Units: units,
|
|
||||||
Metadata: map[string]any{
|
|
||||||
"start_unit_id": units[0].ID,
|
|
||||||
"end_unit_id": units[len(units)-1].ID,
|
|
||||||
"unit_count": len(units),
|
|
||||||
},
|
|
||||||
})
|
})
|
||||||
if end == len(req.Source.Units) {
|
if end == len(req.Source.Units) {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return contracts.ChunkResult{Chunks: chunks}, nil
|
return contracts.ChunkPlanResult{Plan: source.ChunkPlan{SourceDigest: req.Source.Digest, Ranges: ranges}}, nil
|
||||||
}
|
|
||||||
|
|
||||||
func chunkContent(units []source.SourceUnit) ([]byte, error) {
|
|
||||||
content, err := json.Marshal(struct {
|
|
||||||
Units []source.SourceUnit `json:"units"`
|
|
||||||
}{
|
|
||||||
Units: units,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return nil, chunkerErrorf("encode chunk content: %w", err)
|
|
||||||
}
|
|
||||||
return content, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func ModuleSpec() pipeline.ModuleSpec {
|
func ModuleSpec() pipeline.ModuleSpec {
|
||||||
@@ -251,31 +224,6 @@ func minInt() int64 {
|
|||||||
return -maxInt() - 1
|
return -maxInt() - 1
|
||||||
}
|
}
|
||||||
|
|
||||||
func cloneUnits(units []source.SourceUnit) []source.SourceUnit {
|
|
||||||
out := make([]source.SourceUnit, 0, len(units))
|
|
||||||
for _, unit := range units {
|
|
||||||
out = append(out, source.SourceUnit{
|
|
||||||
ID: unit.ID,
|
|
||||||
Kind: unit.Kind,
|
|
||||||
Text: unit.Text,
|
|
||||||
Ref: unit.Ref,
|
|
||||||
Metadata: cloneMetadata(unit.Metadata),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func cloneMetadata(metadata map[string]any) map[string]any {
|
|
||||||
if len(metadata) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
out := make(map[string]any, len(metadata))
|
|
||||||
for key, value := range metadata {
|
|
||||||
out[key] = value
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func chunkerErrorf(format string, args ...any) error {
|
func chunkerErrorf(format string, args ...any) error {
|
||||||
return fmt.Errorf("generic chunker: "+format, args...)
|
return fmt.Errorf("generic chunker: "+format, args...)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -48,72 +48,51 @@ func TestModuleSpecAndRegister(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("BuildWithRequest(%q) error = %v, want nil", Key, err)
|
t.Fatalf("BuildWithRequest(%q) error = %v, want nil", Key, err)
|
||||||
}
|
}
|
||||||
result, err := configured.Chunk(context.Background(), contracts.ChunkRequest{Source: testSource(2)})
|
result, err := configured.Plan(context.Background(), contracts.ChunkRequest{Source: testSource(2)})
|
||||||
if err != nil || len(result.Chunks) != 2 {
|
if err != nil || len(result.Plan.Ranges) != 2 {
|
||||||
t.Fatalf("constructed chunker result = %#v, %v; want two chunks", result, err)
|
t.Fatalf("constructed chunker result = %#v, %v; want two ranges", result, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestChunkUsesDefaultsForSingleChunk(t *testing.T) {
|
func TestPlanUsesDefaultsForSingleRange(t *testing.T) {
|
||||||
result, err := newChunker(t, nil).Chunk(context.Background(), contracts.ChunkRequest{Source: testSource(3)})
|
result, err := newChunker(t, nil).Plan(context.Background(), contracts.ChunkRequest{Source: testSource(3)})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Chunk() error = %v, want nil", err)
|
t.Fatalf("Plan() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if got := chunkIDs(result.Chunks); !reflect.DeepEqual(got, []string{"chunk-000001"}) {
|
if result.Plan.SourceDigest != "sha256:source" || !reflect.DeepEqual(result.Plan.Ranges, []source.ChunkRange{{StartUnitID: 1, EndUnitID: 3}}) {
|
||||||
t.Fatalf("chunk IDs = %#v, want one stable ID", got)
|
t.Fatalf("Plan = %#v, want source digest and one complete range", result.Plan)
|
||||||
}
|
}
|
||||||
chunk := result.Chunks[0]
|
if len(result.Plan.Annotations) != 0 || len(result.Plan.Ranges[0].Annotations) != 0 {
|
||||||
if chunk.Index != 0 || chunk.SourceID != "source-1" {
|
t.Fatalf("Plan annotations = %#v / %#v, want none", result.Plan.Annotations, result.Plan.Ranges[0].Annotations)
|
||||||
t.Fatalf("chunk = %#v, want source and index fields", chunk)
|
|
||||||
}
|
|
||||||
if got := unitIDs(chunk.Units); !reflect.DeepEqual(got, []int{1, 2, 3}) {
|
|
||||||
t.Fatalf("unit IDs = %#v, want all units", got)
|
|
||||||
}
|
|
||||||
if chunk.Ref != (source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 3}) {
|
|
||||||
t.Fatalf("chunk ref = %#v, want source-1:1-3", chunk.Ref)
|
|
||||||
}
|
|
||||||
if chunk.MediaType != "application/json" || len(chunk.Content) == 0 {
|
|
||||||
t.Fatalf("chunk payload = media type %q length %d, want JSON content", chunk.MediaType, len(chunk.Content))
|
|
||||||
}
|
|
||||||
if chunk.Metadata["start_unit_id"] != 1 || chunk.Metadata["end_unit_id"] != 3 || chunk.Metadata["unit_count"] != 3 {
|
|
||||||
t.Fatalf("metadata = %#v, want chunk bounds", chunk.Metadata)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestChunkExactBoundaries(t *testing.T) {
|
func TestPlanExactBoundaries(t *testing.T) {
|
||||||
result, err := newChunker(t, map[string]any{"max_units": 2}).Chunk(context.Background(), contracts.ChunkRequest{Source: testSource(6)})
|
result, err := newChunker(t, map[string]any{"max_units": 2}).Plan(context.Background(), contracts.ChunkRequest{Source: testSource(6)})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Chunk() error = %v, want nil", err)
|
t.Fatalf("Plan() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if got := chunkIDs(result.Chunks); !reflect.DeepEqual(got, []string{"chunk-000001", "chunk-000002", "chunk-000003"}) {
|
want := []source.ChunkRange{{StartUnitID: 1, EndUnitID: 2}, {StartUnitID: 3, EndUnitID: 4}, {StartUnitID: 5, EndUnitID: 6}}
|
||||||
t.Fatalf("chunk IDs = %#v, want stable IDs", got)
|
if !reflect.DeepEqual(result.Plan.Ranges, want) {
|
||||||
}
|
t.Fatalf("ranges = %#v, want %#v", result.Plan.Ranges, want)
|
||||||
gotUnits := [][]int{unitIDs(result.Chunks[0].Units), unitIDs(result.Chunks[1].Units), unitIDs(result.Chunks[2].Units)}
|
|
||||||
wantUnits := [][]int{{1, 2}, {3, 4}, {5, 6}}
|
|
||||||
if !reflect.DeepEqual(gotUnits, wantUnits) {
|
|
||||||
t.Fatalf("chunk units = %#v, want %#v", gotUnits, wantUnits)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestChunkOverlap(t *testing.T) {
|
func TestPlanOverlap(t *testing.T) {
|
||||||
result, err := newChunker(t, map[string]any{"max_units": 3, "overlap_units": 1}).Chunk(context.Background(), contracts.ChunkRequest{Source: testSource(7)})
|
result, err := newChunker(t, map[string]any{"max_units": 3, "overlap_units": 1}).Plan(context.Background(), contracts.ChunkRequest{Source: testSource(7)})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Chunk() error = %v, want nil", err)
|
t.Fatalf("Plan() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
gotUnits := make([][]int, 0, len(result.Chunks))
|
want := []source.ChunkRange{{StartUnitID: 1, EndUnitID: 3}, {StartUnitID: 3, EndUnitID: 5}, {StartUnitID: 5, EndUnitID: 7}}
|
||||||
for _, chunk := range result.Chunks {
|
if !reflect.DeepEqual(result.Plan.Ranges, want) {
|
||||||
gotUnits = append(gotUnits, unitIDs(chunk.Units))
|
t.Fatalf("ranges = %#v, want %#v", result.Plan.Ranges, want)
|
||||||
}
|
|
||||||
wantUnits := [][]int{{1, 2, 3}, {3, 4, 5}, {5, 6, 7}}
|
|
||||||
if !reflect.DeepEqual(gotUnits, wantUnits) {
|
|
||||||
t.Fatalf("chunk units = %#v, want %#v", gotUnits, wantUnits)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestChunkRejectsInvalidOptions(t *testing.T) {
|
func TestPlanRejectsInvalidOptions(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
options map[string]any
|
options map[string]any
|
||||||
@@ -141,42 +120,36 @@ func TestChunkRejectsInvalidOptions(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestChunkRejectsEmptySource(t *testing.T) {
|
func TestPlanRejectsEmptySource(t *testing.T) {
|
||||||
doc := testSource(1)
|
doc := testSource(1)
|
||||||
doc.Units = nil
|
doc.Units = nil
|
||||||
|
|
||||||
_, err := newChunker(t, nil).Chunk(context.Background(), contracts.ChunkRequest{Source: doc})
|
_, err := newChunker(t, nil).Plan(context.Background(), contracts.ChunkRequest{Source: doc})
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("Chunk() error = nil, want empty source error")
|
t.Fatal("Plan() error = nil, want empty source error")
|
||||||
}
|
}
|
||||||
if !strings.Contains(err.Error(), "generic chunker") || !strings.Contains(err.Error(), "units") {
|
if !strings.Contains(err.Error(), "generic chunker") || !strings.Contains(err.Error(), "units") {
|
||||||
t.Fatalf("Chunk() error = %q, want empty source context", err.Error())
|
t.Fatalf("Plan() error = %q, want empty source context", err.Error())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestChunkDefensivelyCopiesUnits(t *testing.T) {
|
func TestPlanDoesNotRetainSourceUnits(t *testing.T) {
|
||||||
doc := testSource(2)
|
doc := testSource(2)
|
||||||
|
|
||||||
result, err := newChunker(t, map[string]any{"max_units": 1}).Chunk(context.Background(), contracts.ChunkRequest{Source: doc})
|
result, err := newChunker(t, map[string]any{"max_units": 1}).Plan(context.Background(), contracts.ChunkRequest{Source: doc})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Chunk() error = %v, want nil", err)
|
t.Fatalf("Plan() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
if len(result.Chunks) != 2 {
|
if len(result.Plan.Ranges) != 2 {
|
||||||
t.Fatalf("len(Chunks) = %d, want 2", len(result.Chunks))
|
t.Fatalf("len(Ranges) = %d, want 2", len(result.Plan.Ranges))
|
||||||
}
|
}
|
||||||
|
|
||||||
doc.Units[0].ID = 99
|
doc.Units[0].ID = 99
|
||||||
doc.Units[0].Ref.SourceID = "changed"
|
doc.Units[0].Ref.SourceID = "changed"
|
||||||
doc.Units[0].Metadata["speaker"] = "changed"
|
doc.Units[0].Metadata["speaker"] = "changed"
|
||||||
|
|
||||||
if result.Chunks[0].Units[0].ID != 1 {
|
if result.Plan.Ranges[0].StartUnitID != 1 || result.Plan.Ranges[0].EndUnitID != 1 {
|
||||||
t.Fatalf("chunk unit ID changed after source mutation: %#v", result.Chunks[0].Units[0])
|
t.Fatalf("first range changed after source mutation: %#v", result.Plan.Ranges[0])
|
||||||
}
|
|
||||||
if got := result.Chunks[0].Units[0].Ref.SourceID; got != "source-1" {
|
|
||||||
t.Fatalf("chunk unit ref changed after source mutation: %#v", result.Chunks[0].Units[0].Ref)
|
|
||||||
}
|
|
||||||
if result.Chunks[0].Units[0].Metadata["speaker"] != "speaker-001" {
|
|
||||||
t.Fatalf("chunk unit metadata changed after source mutation: %#v", result.Chunks[0].Units[0].Metadata)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -214,19 +187,3 @@ func testSource(count int) *source.SourceDocument {
|
|||||||
func zeroPad3(value int) string {
|
func zeroPad3(value int) string {
|
||||||
return fmt.Sprintf("%03d", value)
|
return fmt.Sprintf("%03d", value)
|
||||||
}
|
}
|
||||||
|
|
||||||
func chunkIDs(chunks []source.Chunk) []string {
|
|
||||||
ids := make([]string, 0, len(chunks))
|
|
||||||
for _, chunk := range chunks {
|
|
||||||
ids = append(ids, chunk.ID)
|
|
||||||
}
|
|
||||||
return ids
|
|
||||||
}
|
|
||||||
|
|
||||||
func unitIDs(units []source.SourceUnit) []int {
|
|
||||||
ids := make([]int, 0, len(units))
|
|
||||||
for _, unit := range units {
|
|
||||||
ids = append(ids, unit.ID)
|
|
||||||
}
|
|
||||||
return ids
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -273,11 +273,32 @@ func cloneMetadata(metadata map[string]any) map[string]any {
|
|||||||
}
|
}
|
||||||
out := make(map[string]any, len(metadata))
|
out := make(map[string]any, len(metadata))
|
||||||
for key, value := range metadata {
|
for key, value := range metadata {
|
||||||
out[key] = value
|
out[key] = cloneJSONMetadataValue(value)
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func cloneJSONMetadataValue(value any) any {
|
||||||
|
switch typed := value.(type) {
|
||||||
|
case map[string]any:
|
||||||
|
return cloneMetadata(typed)
|
||||||
|
case []any:
|
||||||
|
out := make([]any, len(typed))
|
||||||
|
for i := range typed {
|
||||||
|
out[i] = cloneJSONMetadataValue(typed[i])
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
case stdjson.RawMessage:
|
||||||
|
return append(stdjson.RawMessage(nil), typed...)
|
||||||
|
case []byte:
|
||||||
|
return append([]byte(nil), typed...)
|
||||||
|
case []string:
|
||||||
|
return append([]string(nil), typed...)
|
||||||
|
default:
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func encoderErrorf(format string, args ...any) error {
|
func encoderErrorf(format string, args ...any) error {
|
||||||
return fmt.Errorf("json output encoder: "+format, args...)
|
return fmt.Errorf("json output encoder: "+format, args...)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,7 +4,6 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -68,12 +67,12 @@ type concurrentChunker struct{}
|
|||||||
|
|
||||||
func (concurrentChunker) Key() string { return concurrentChunkerKey }
|
func (concurrentChunker) Key() string { return concurrentChunkerKey }
|
||||||
func (concurrentChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
func (concurrentChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||||
func (concurrentChunker) Chunk(_ context.Context, request contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
func (concurrentChunker) Plan(_ context.Context, request contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
|
||||||
chunks := make([]source.Chunk, len(request.Source.Units))
|
ranges := make([]source.ChunkRange, len(request.Source.Units))
|
||||||
for i, unit := range request.Source.Units {
|
for i, unit := range request.Source.Units {
|
||||||
chunks[i] = source.Chunk{ID: fmt.Sprintf("%s:chunk:%d", request.Source.ID, i), SourceID: request.Source.ID, Index: i, Ref: unit.Ref, Content: []byte(fmt.Sprintf(`{"unit":%d}`, unit.ID)), MediaType: "application/json", Units: []source.SourceUnit{unit}}
|
ranges[i] = source.ChunkRange{StartUnitID: unit.ID, EndUnitID: unit.ID}
|
||||||
}
|
}
|
||||||
return contracts.ChunkResult{Chunks: chunks}, nil
|
return contracts.ChunkPlanResult{Plan: source.ChunkPlan{SourceDigest: request.Source.Digest, Ranges: ranges}}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type concurrentExtractor struct {
|
type concurrentExtractor struct {
|
||||||
|
|||||||
@@ -271,19 +271,9 @@ func (dndSpellsChunker) ReferenceSlots() []contracts.ReferenceSlot {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (dndSpellsChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
func (dndSpellsChunker) Plan(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
|
||||||
return contracts.ChunkResult{
|
return contracts.ChunkPlanResult{
|
||||||
Chunks: []source.Chunk{
|
Plan: source.ChunkPlan{SourceDigest: req.Source.Digest, Ranges: []source.ChunkRange{{StartUnitID: req.Source.Units[0].ID, EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID}}},
|
||||||
{
|
|
||||||
ID: req.Source.ID + ":chunk:0",
|
|
||||||
SourceID: req.Source.ID,
|
|
||||||
Index: 0,
|
|
||||||
Ref: source.SourceRef{SourceID: req.Source.ID, StartUnitID: req.Source.Units[0].ID, EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID},
|
|
||||||
Content: []byte(`{"units":[1,2,3]}`),
|
|
||||||
MediaType: "application/json",
|
|
||||||
Units: append([]source.SourceUnit(nil), req.Source.Units...),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -220,8 +220,8 @@ func (fakeChunker) Key() string { return "fake/chunk" }
|
|||||||
|
|
||||||
func (fakeChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
func (fakeChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||||
|
|
||||||
func (fakeChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
func (fakeChunker) Plan(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
|
||||||
return contracts.ChunkResult{}, nil
|
return contracts.ChunkPlanResult{}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type fakeExtractor struct{}
|
type fakeExtractor struct{}
|
||||||
|
|||||||
@@ -177,19 +177,9 @@ func (runnerSeriatimChunker) ReferenceSlots() []contracts.ReferenceSlot {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (runnerSeriatimChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
func (runnerSeriatimChunker) Plan(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
|
||||||
return contracts.ChunkResult{
|
return contracts.ChunkPlanResult{
|
||||||
Chunks: []source.Chunk{
|
Plan: source.ChunkPlan{SourceDigest: req.Source.Digest, Ranges: []source.ChunkRange{{StartUnitID: req.Source.Units[0].ID, EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID}}},
|
||||||
{
|
|
||||||
ID: req.Source.ID + ":chunk:0",
|
|
||||||
SourceID: req.Source.ID,
|
|
||||||
Index: 0,
|
|
||||||
Ref: source.SourceRef{SourceID: req.Source.ID, StartUnitID: req.Source.Units[0].ID, EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID},
|
|
||||||
Content: []byte(`{"units":[1,2]}`),
|
|
||||||
MediaType: "application/json",
|
|
||||||
Units: append([]source.SourceUnit(nil), req.Source.Units...),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user