16 Commits

Author SHA1 Message Date
b7cc5fb980 Added the standard documentation policy and a roadmap for adding documentation 2026-05-24 08:19:39 -05:00
b20438acf0 Updated the normalize command to correct common errors in WhisperX-generated input transcripts
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
2026-05-16 23:05:42 -05:00
6dbb7ab17e Review normalize command architecture
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
2026-05-09 12:38:06 +00:00
3591041fa8 Document normalize command 2026-05-09 12:35:48 +00:00
5b008e272c Add normalize report diagnostics 2026-05-09 12:34:37 +00:00
6c780f6293 Implement normalize output conversion 2026-05-09 12:32:18 +00:00
c132f3fd5d Add normalize input parsing 2026-05-09 12:29:12 +00:00
3679435063 Add normalize command scaffold 2026-05-09 12:26:47 +00:00
e6d3b4a46e Harden trim integration
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
2026-05-08 15:00:46 +00:00
54f7717de8 Document trim command 2026-05-08 14:57:52 +00:00
c48b02d2ec Add trim report output 2026-05-08 14:56:24 +00:00
ac3dcf2557 Add trim CLI command 2026-05-08 14:53:59 +00:00
1c0e4438ae Recompute overlap groups during trim 2026-05-08 14:47:52 +00:00
52f7729100 Add artifact trim transformation 2026-05-08 14:44:31 +00:00
2c82f8bf5c Add trim selector parsing 2026-05-08 14:41:47 +00:00
d865bda4a9 Updated .gitignore to ignore .codex and related files 2026-05-08 14:36:00 +00:00
24 changed files with 6003 additions and 5 deletions

4
.gitignore vendored
View File

@@ -1,3 +1,7 @@
# ---> Codex
.codex
AGENTS.md
# ---> Go # ---> Go
# If you prefer the allow list template instead of the deny list, see community template: # If you prefer the allow list template instead of the deny list, see community template:
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore

View File

@@ -1,4 +1,4 @@
Copyright (c) 2026 eric. Copyright (c) 2026 Eric Rakestraw.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

114
README.md
View File

@@ -1,8 +1,8 @@
# seriatim # seriatim
`seriatim` merges per-speaker WhisperX-style JSON transcripts into a single JSON transcript that preserves speaker identity and chronological order. `seriatim` merges per-speaker WhisperX-style JSON transcripts into a single JSON transcript that preserves speaker identity and chronological order. It also trims existing seriatim output artifacts by segment ID and normalizes external transcript-like JSON into standard seriatim output schemas.
The current implementation supports the `merge` command. It reads one or more input JSON files, optionally maps each input file to a canonical speaker using `speakers.yml`, sorts all segments by timestamp, detects and resolves overlaps when word-level timing is available, assigns consecutive numeric `id` values, and writes a merged JSON artifact. The current implementation supports the `merge`, `trim`, and `normalize` commands. `merge` reads one or more input JSON files, optionally maps each input file to a canonical speaker using `speakers.yml`, sorts all segments by timestamp, detects and resolves overlaps when word-level timing is available, assigns consecutive numeric `id` values, and writes a merged JSON artifact. `trim` reads an existing seriatim output artifact and projects it to a retained segment subset. `normalize` reads transcript-like JSON input, validates required segment fields, sorts deterministically, assigns fresh IDs, and emits a selected seriatim output schema.
## Usage ## Usage
@@ -25,10 +25,39 @@ go run ./cmd/seriatim merge \
--report-file report.json --report-file report.json
``` ```
Trim an existing seriatim artifact:
```sh
go run ./cmd/seriatim trim \
--input-file merged.json \
--output-file trimmed.json \
--keep "1-10, 15, 20-25"
```
Normalize external transcript-style JSON:
```sh
go run ./cmd/seriatim normalize \
--input-file transcript.json \
--output-file normalized.json
```
Normalize an Audita-style bare segment array to full schema with report output:
```sh
go run ./cmd/seriatim normalize \
--input-file audita-segments.json \
--output-file normalized-full.json \
--output-schema seriatim-full \
--report-file normalize-report.json
```
## CLI ## CLI
```text ```text
seriatim merge [flags] seriatim merge [flags]
seriatim trim [flags]
seriatim normalize [flags]
``` ```
Global flags: Global flags:
@@ -54,6 +83,87 @@ Global flags:
| `--postprocessing-modules` | No | `detect-overlaps,resolve-overlaps,backchannel,filler,resolve-danglers,coalesce,detect-overlaps,autocorrect,assign-ids,validate-output` | Comma-separated postprocessing modules, evaluated in order. | | `--postprocessing-modules` | No | `detect-overlaps,resolve-overlaps,backchannel,filler,resolve-danglers,coalesce,detect-overlaps,autocorrect,assign-ids,validate-output` | Comma-separated postprocessing modules, evaluated in order. |
| `--coalesce-gap` | No | `3.0` | Maximum same-speaker gap in seconds for `coalesce`; also used as the `resolve-overlaps` context window. Must be a non-negative float. | | `--coalesce-gap` | No | `3.0` | Maximum same-speaker gap in seconds for `coalesce`; also used as the `resolve-overlaps` context window. Must be a non-negative float. |
`trim` flags:
| Flag | Required | Default | Description |
| --- | --- | --- | --- |
| `--input-file` | Yes | none | Input seriatim output artifact JSON file. |
| `--output-file` | Yes | none | Trimmed transcript JSON output path. |
| `--keep` | Exactly one of `--keep` or `--remove` is required | none | Segment ID selector to retain. |
| `--remove` | Exactly one of `--keep` or `--remove` is required | none | Segment ID selector to drop. |
| `--output-schema` | No | preserve input artifact schema | Optional output schema override: `seriatim-minimal`, `seriatim-intermediate`, or `seriatim-full`. |
| `--report-file` | No | none | Optional report JSON output path. |
| `--allow-empty` | No | `false` | Allow trimming to zero retained segments. |
`trim` selection rules:
- `--keep` and `--remove` are mutually exclusive.
- Exactly one of `--keep` or `--remove` is required.
- Selection is by segment ID only.
- Invalid selected segment IDs fail the command by default.
`trim` selector syntax:
- Segment IDs are positive 1-based integers.
- Inclusive ranges are supported: `1-10`.
- Comma-separated selectors are supported: `1-10,15,20-25`.
- Whitespace around numbers, commas, and hyphens is allowed: `1 - 10, 15, 20 - 25`.
- Duplicate and overlapping ranges are accepted and normalized as a union.
- Descending ranges (for example `10-1`) are rejected.
`trim` behavior:
- `trim` consumes existing seriatim JSON output artifacts only.
- `trim` does not accept raw WhisperX transcript JSON as input.
- Retained output segment IDs are renumbered sequentially from `1` to `N`.
- Transcript order is preserved from input transcript order; selector order does not reorder output.
- When output schema is `seriatim-full`, overlap groups are recomputed from retained segments.
- `--output-schema seriatim-full` is supported when trim has full-schema artifact data to emit; trim does not synthesize missing full-schema provenance from minimal/intermediate input artifacts.
- `trim` does not run merge postprocessors such as `resolve-overlaps`, `coalesce`, or `autocorrect`.
`trim` report output:
- When `--report-file` is provided, the report includes standard trim/validation/output events.
- The report includes a `trim-audit` event containing trim operation metadata, including selected IDs, retained/removed counts, removed IDs, and old-to-new segment ID mapping.
- Old-to-new ID mapping is emitted as a deterministic ordered array of `{old_id, new_id}` pairs.
`normalize` flags:
| Flag | Required | Default | Description |
| --- | --- | --- | --- |
| `--input-file` | Yes | none | Input transcript JSON file. |
| `--output-file` | Yes | none | Normalized transcript JSON output path. |
| `--output-schema` | No | `seriatim-intermediate` (resolved via `SERIATIM_OUTPUT_SCHEMA` when set) | Output JSON schema: `seriatim-minimal`, `seriatim-intermediate`, or `seriatim-full`. |
| `--output-modules` | No | `json` | Comma-separated output modules. Current normalize support is `json` only. |
| `--report-file` | No | none | Optional report JSON output path. |
`normalize` input shapes:
- Top-level object with a `segments` array.
- Bare top-level array of segment objects (for example, Audita-style output).
`normalize` behavior:
- Repairs missing timing fields deterministically:
if one of `start`/`end` is present, sets both to that value;
if both are missing, uses midpoint of previous `end` and next `start`,
with edge fallback to available neighbor and `0.0` for single-segment inputs.
- If `end < start`, swaps them.
- Fills missing/empty `speaker` with `Unknown_Speaker`.
- Drops segments with missing, empty, or whitespace-only `text`.
- Validates repaired timing with `start >= 0`.
- Accepts existing input `id` values as provenance only.
- Reassigns output segment IDs sequentially from `1` to `N`.
- Sorts deterministically by `(start, end, original_input_index, speaker)`.
- Uses original input order only as a tie-breaker.
- Does not run merge postprocessors such as overlap detection, overlap resolution, coalescing, or autocorrect.
- Useful for converting external transcript outputs into standard seriatim artifacts.
`normalize` report output:
- When `--report-file` is provided, normalize emits deterministic report events with input shape detection, segment counts, schema/module selections, sorting/ID diagnostics, and output write/validation summaries.
- A machine-readable `normalize-audit` event is included for downstream tooling.
Environment variables: Environment variables:
| Environment Variable | Default | Description | | Environment Variable | Default | Description |

View File

@@ -1,6 +1,10 @@
# seriatim Architecture # seriatim Architecture
`seriatim` is a deterministic transcript merge utility for combining multiple per-speaker transcript inputs into a single chronologically ordered diarized transcript. `seriatim` is a deterministic transcript utility for:
- merging multiple per-speaker transcript inputs into a single chronologically ordered diarized transcript, and
- projecting existing seriatim transcript artifacts through deterministic segment-ID trimming, and
- canonicalizing external transcript-style JSON inputs into standard seriatim output schemas.
The initial use case is merging independently transcribed speaker audio tracks from the same recorded session, such as a weekly tabletop RPG session. The architecture should also support meetings, podcasts, interviews, and other multi-speaker events. The initial use case is merging independently transcribed speaker audio tracks from the same recorded session, such as a weekly tabletop RPG session. The architecture should also support meetings, podcasts, interviews, and other multi-speaker events.
@@ -20,6 +24,7 @@ The initial use case is merging independently transcribed speaker audio tracks f
8. Detect and annotate overlapping speech regions. 8. Detect and annotate overlapping speech regions.
9. Emit one or more output artifacts through output writers. 9. Emit one or more output artifacts through output writers.
10. Produce report data for validation findings, corrections, and transformations. 10. Produce report data for validation findings, corrections, and transformations.
11. Support artifact-level transcript projection commands that operate on existing seriatim output.
## Non-goals ## Non-goals
@@ -56,6 +61,8 @@ configuration check
Each stage has an explicit data contract. Input and output stages perform I/O. Processing stages should be deterministic transformations over in-memory models and should record report events for validation findings, corrections, and transformations. Each stage has an explicit data contract. Input and output stages perform I/O. Processing stages should be deterministic transformations over in-memory models and should record report events for validation findings, corrections, and transformations.
`merge` runs this pipeline. `trim` and `normalize` are intentionally separate from this pipeline and operate at the artifact layer.
## Stage Contracts ## Stage Contracts
### 1. Configuration Check ### 1. Configuration Check
@@ -191,6 +198,41 @@ Future output formats may include:
Output writers should be selected from an explicit registry and should consume the final transcript model read-only. Multiple output writers may run for a single invocation. Output writers should be selected from an explicit registry and should consume the final transcript model read-only. Multiple output writers may run for a single invocation.
### 7. Artifact Projection Stage (`trim` command)
`trim` is an artifact-level command that reads an existing seriatim output artifact and emits a projected artifact containing a segment-ID subset.
Design constraints:
- `trim` runs after `merge`, not as a merge postprocessor.
- `trim` validates the input artifact against supported seriatim output schemas.
- `trim` performs deterministic keep/remove selection by segment ID.
- `trim` renumbers retained IDs to `1..N` in transcript order.
- `trim` validates the final output against the selected output schema before writing.
- `trim` records audit metadata in report output.
`trim` is intentionally separate from merge postprocessing because it consumes already-emitted public artifacts. This separation keeps merge semantics stable and avoids rerunning merge-only transforms on projected artifacts.
`trim` must not rerun merge postprocessors such as `resolve-overlaps`, `coalesce`, or `autocorrect`.
### 8. Artifact Canonicalization Stage (`normalize` command)
`normalize` is an artifact-level command that reads transcript-like JSON and emits a standard seriatim output artifact in a selected schema.
Design constraints:
- `normalize` runs outside the merge pipeline and does not invoke merge preprocessing or postprocessing modules.
- `normalize` accepts two input shapes: object-with-`segments` and bare segment arrays.
- `normalize` applies deterministic repair rules for missing/irregular `start`, `end`, and `speaker`, and drops segments with missing/empty `text`.
- `normalize` sorts segments deterministically by chronological keys and stable input-index tie-breakers.
- `normalize` assigns fresh sequential output IDs (`1..N`) after sorting.
- `normalize` validates final output against the selected schema before writing.
- `normalize` writes optional deterministic report diagnostics when `--report-file` is requested.
`normalize` is intended for canonicalizing external transcript outputs (including Audita-style bare arrays) into seriatim contracts, not for running merge-time language or overlap transformations.
`normalize` must not run merge postprocessors such as overlap detection, overlap resolution, coalescing, or autocorrect.
## Module Classification ## Module Classification
Modules should be classified by their contract and allowed effects. Modules should be classified by their contract and allowed effects.
@@ -397,6 +439,8 @@ A valid merged transcript should satisfy:
- Every referenced segment exists. - Every referenced segment exists.
- Output validates against the selected output schema. - Output validates against the selected output schema.
For full-schema trim output, overlap groups are recomputed from retained segments so overlap annotations and group references remain internally consistent after projection.
## Determinism Requirements ## Determinism Requirements
Given the same inputs, config, and application version, `seriatim` should produce byte-stable JSON output where practical. Given the same inputs, config, and application version, `seriatim` should produce byte-stable JSON output where practical.
@@ -411,6 +455,19 @@ To support this:
- Record application version in output metadata. - Record application version in output metadata.
- Record enabled module names and module order in output metadata or report data. - Record enabled module names and module order in output metadata or report data.
Trim-specific determinism requirements:
- Selector normalization and retained IDs are deterministic.
- Old-to-new ID mapping in trim reports is emitted in deterministic order.
- Full-schema overlap recomputation is deterministic for the same input artifact and selector.
Normalize-specific determinism requirements:
- Input-shape detection is deterministic.
- Segment ordering is deterministic for identical input data.
- Output IDs are always reassigned sequentially after deterministic sorting.
- Normalize diagnostic reports are deterministic for identical inputs and configuration.
## Go Package Layout ## Go Package Layout
```text ```text
@@ -419,6 +476,8 @@ internal/config/ CLI/env/config loading and validation
internal/pipeline/ Pipeline orchestration and module registry internal/pipeline/ Pipeline orchestration and module registry
internal/builtin/ Built-in pipeline modules internal/builtin/ Built-in pipeline modules
internal/artifact/ Conversion from internal model to public output schema internal/artifact/ Conversion from internal model to public output schema
internal/normalize/ Normalize input parsing, validation, deterministic sorting, schema conversion, and diagnostics
internal/trim/ Artifact parsing, trim selection, schema conversion, overlap recomputation for full schema
internal/buildinfo/ Build-time version metadata internal/buildinfo/ Build-time version metadata
internal/speaker/ Speaker map parsing and lookup internal/speaker/ Speaker map parsing and lookup
internal/model/ Canonical and merged transcript models internal/model/ Canonical and merged transcript models
@@ -430,6 +489,18 @@ schema/ Public output contract and JSON Schema validation
Package boundaries should follow data ownership. Shared models belong in `internal/model`; stage-specific behavior belongs in the relevant stage package. Package boundaries should follow data ownership. Shared models belong in `internal/model`; stage-specific behavior belongs in the relevant stage package.
For trim:
- `internal/trim` contains pure transformation logic over artifact structs.
- CLI command code handles only flag parsing, file I/O, and report emission.
- Transform logic is deterministic and pure except for command-layer I/O.
For normalize:
- `internal/normalize` contains parsing/validation and deterministic schema conversion logic.
- CLI command code handles flag parsing and delegates execution.
- Normalize remains artifact-level and does not compose merge pipeline modules.
## Default Modules ## Default Modules
The default pipeline is equivalent to explicit module lists. The default pipeline is equivalent to explicit module lists.

219
docs/policy/architecture.md Normal file
View File

@@ -0,0 +1,219 @@
# Architecture Policy
## Purpose
This document defines seriatim's development architecture and invariants for
maintainers and automated coding agents. It describes how the implemented
system is intended to be built and changed. It is not a user manual, CLI
reference, config reference, or roadmap.
Keep this document aligned with [documentation policy](documentation.md). It
must describe current behavior only; planned or speculative work belongs under
`docs/roadmap/`.
## Project Shape
seriatim is a Go CLI for transcript artifact processing. The implemented
commands are `merge`, `trim`, and `normalize`.
`merge` reads one or more JSON transcript files, optionally maps input files to
canonical speakers, runs a registry-selected preprocessing chain, merges
canonical segments into deterministic chronological order, runs a
registry-selected postprocessing chain, validates the selected output schema,
and writes JSON output plus an optional JSON report.
`trim` and `normalize` are artifact-level commands outside the merge pipeline.
`trim` reads an existing seriatim output artifact and projects it by segment ID.
`normalize` reads transcript-like JSON and emits one of seriatim's supported
output schemas. Neither command runs merge preprocessing or postprocessing
modules.
The supported public output schemas are `seriatim-minimal`,
`seriatim-intermediate`, and `seriatim-full`. For current command and flag
details, use [the README](../../README.md) until dedicated `docs/cli.md` and
`docs/config.md` files exist.
## Core Design Principles
- Keep a hexagonal architecture boundary. Domain models, stage contracts, and
deterministic transformations must stay separate from CLI parsing,
filesystem access, config loading, reporting, and other external adapters.
- Keep stages and modules composable. Built-in modules are selected by
canonical registry names and implement explicit interfaces for their pipeline
role.
- Preserve deterministic behavior. Given the same inputs, configuration, and
version, output ordering, segment IDs, schema validation, and report event
ordering should remain stable.
- Current command execution is sequential. There is no scheduler, worker pool,
or concurrent module execution in the implemented pipeline. Any concurrency
added later must be bounded, observable, and must not make output handling
nondeterministic.
- Prefer the Go standard library. Third-party dependencies should remain narrow
and justified, such as Cobra for CLI structure, YAML parsing, and JSON Schema
validation.
- Document current behavior. Architecture, user, and internal docs must not
describe planned features as implemented behavior.
## Architectural Boundaries
Core transcript data belongs in `internal/model` and public artifact contracts
belong in `schema`. Conversion from internal merged data to public JSON shapes
belongs at the artifact boundary, not inside CLI code or transformation
packages.
Pipeline orchestration belongs in `internal/pipeline`. It resolves registered
modules, validates preprocessing state transitions, executes stages in order,
collects report events, converts the final transcript, and writes optional
reports. Built-in adapters and modules are registered from `internal/builtin`.
CLI code in `internal/cli` should parse flags, build validated config values,
and delegate. `merge` delegates to `pipeline.Run`; `trim` and `normalize`
perform artifact-level orchestration and delegate deterministic parsing,
validation, and transformation work to their internal packages.
Config loading and validation belongs in `internal/config`. Filesystem reads and
writes are adapter concerns and should not spread into pure transformation
helpers. Existing built-in modules that load configured YAML files must keep
that I/O narrow and explicit.
Reports belong in `internal/report`. Modules and commands should emit concise
events for validation findings, corrections, and transformations without
turning report messages into a duplicate output artifact.
Tests and samples are supporting evidence for behavior. Tests should verify
stable contracts and edge cases; samples should remain valid examples, not
hidden architecture dependencies.
## Modules or Stages
The merge pipeline has these implemented stages:
- `InputReader`: reads configured external input into raw transcripts.
- `Preprocessor`: transforms `PreprocessState` from raw to canonical state.
- `Merger`: combines canonical transcripts into one merged transcript.
- `Postprocessor`: transforms or annotates the merged transcript.
- `OutputWriter`: writes the selected output artifact.
Modules must keep narrow responsibilities, declare their stage through the
interface they implement, and use explicit config values. Preprocessors must
declare `Requires()` and `Produces()` states; the runner rejects invalid
raw/canonical ordering before processing completes.
Modules run in the configured order. Order-affecting modules must run before
`assign-ids`, and `validate-output` must see final IDs that match the selected
schema. Accepted and rejected transformations should be deterministic and, when
observable, recorded through report events.
Transformation helpers should avoid hidden global state. Shared caches, such as
compiled JSON schemas, must be protected and must not affect output ordering.
## State, Inputs, and Outputs
seriatim is file-based. It reads JSON inputs and optional YAML rule files, then
writes JSON transcript artifacts and optional JSON reports.
The implemented application has no durable database, daemon state, resume
state, remote storage, or background job state. Runtime state is held in memory
for the current command invocation and serialized only through requested output
and report files.
Input file paths are normalized and validated during config construction.
`merge` sorts input file paths before processing, then uses stable segment sort
keys. `trim` preserves transcript order while renumbering retained IDs.
`normalize` sorts by implemented deterministic keys and assigns fresh IDs.
## Configuration and CLI Boundaries
The CLI surface is an adapter over validated config structs. Cobra command code
should stay thin: parse flags, account for flag/default precedence, call config
constructors, and delegate.
Config constructors validate required paths, output parent directories, module
lists, selected schemas, mutually exclusive trim selector options, and supported
environment-derived settings. Module name validation is split between config
where command-specific names are fixed and the pipeline registry where module
composition is resolved.
Do not duplicate full CLI or config reference material here. Use
[the README](../../README.md) for the current user-facing reference until the
canonical `docs/cli.md` and `docs/config.md` files exist.
## Errors, Logging, and Diagnostics
Commands return errors instead of printing inside deep logic. The root command
silences Cobra usage/error output, and `cmd/seriatim/main.go` prints one error
to stderr and exits with status `1`.
Validation failures should fail fast with contextual errors. Correctable
conditions should be deterministic and, where reports are requested, reflected
as report events. Optional reports contain metadata and ordered events; they are
not required for command success unless the report file itself cannot be
written.
The implemented code does not use a logging subsystem. Diagnostics are returned
as errors or written to optional report JSON. Normalize report events avoid
embedding transcript text; keep that privacy-oriented behavior when changing
normalize diagnostics.
## Testing Expectations
`go test ./...` is the repository-wide check. There is currently no Makefile,
taskfile, linter config, or dedicated documentation check.
When changing config or CLI behavior, inspect `internal/config` and
`internal/cli` tests. When changing pipeline composition or stage contracts,
inspect `internal/pipeline` and `internal/builtin` tests. When changing
correction or annotation modules, inspect the package tests for overlap,
coalesce, danglers, backchannel, filler, and autocorrect behavior.
When changing artifact-level commands, inspect `internal/trim`,
`internal/normalize`, and their CLI tests. When changing public output shape or
schema validation, inspect `schema` and `internal/artifact` tests. Report and
diagnostic changes should be covered through the command or package tests that
emit the affected events.
## Dependency Policy
Prefer the Go standard library for parsing, data transformation, concurrency
primitives, filesystem work, and testing wherever it is reasonable.
Third-party dependencies must be narrow, justified, and preferably de facto
standard for their purpose. Existing examples include Cobra for CLI structure,
`gopkg.in/yaml.v3` for YAML files, and `jsonschema/v6` for validating embedded
public JSON schemas. Avoid broad framework dependencies for behavior that is
already simple and local.
## Documentation Expectations
Architecture docs must stay aligned with [documentation policy](documentation.md).
Current-behavior docs must not become aspirational. If code and docs disagree,
fix the inaccurate current-behavior doc or put planned work under
`docs/roadmap/`.
Prefer links to canonical docs instead of repeating full CLI, config, schema, or
operations reference material. Keep examples real, tested where practical, and
free of secrets or private transcript data.
## Architectural Invariants
- Keep core/domain logic separate from CLI, config, filesystem, reporting, and
other adapter concerns.
- Keep modules narrowly scoped, explicitly configured, and composable by
registry name.
- Preserve deterministic ordering, final segment ID assignment, and schema
validation before output acceptance.
- Keep `trim` and `normalize` artifact-level; do not run merge modules from
those commands.
- Keep public output schemas validated through `schema`.
- Keep optional reports ordered, concise, and diagnostic.
- Avoid broad dependencies without a concrete maintainability benefit.
- Do not document unimplemented behavior outside `docs/roadmap/`.
## Non-Goals
The implemented application does not perform transcription, audio diarization,
speaker inference from audio or text, summarization, daemon operation, remote
storage, dynamic external plugin loading, or concurrent pipeline execution.
The architecture policy is not a package-by-package reference, CLI manual,
config reference, schema reference, or roadmap.

View File

@@ -0,0 +1,356 @@
# Go Project Documentation Policy
## Purpose
Project documentation must help four audiences:
1. users who need to run the application;
2. administrators/operators who need to configure and operate it;
3. developers who need to understand and change it safely;
4. LLM coding agents that need clear scope, boundaries, and invariants.
Docs should be accurate, concise, task-oriented, and organized by audience. Prefer links to canonical docs over repetition.
## Core Rules
### 1. Keep docs concise
Each document should cover a defined scope and only the essentials for that scope.
Avoid:
- long background explanations;
- repeated reference material;
- implementation detail in user-facing docs;
- aspirational language outside roadmap docs;
- verbose examples where one minimal example is clearer.
### 2. Document only implemented behavior outside roadmap files
Unimplemented, planned, aspirational, experimental, or future work may be described only under:
- `docs/roadmap/`
No other documentation file, including `README.md`, should describe code, features, modules, stages, commands, config fields, or behaviors that do not currently exist.
If a feature is partial, non-roadmap docs may describe only the implemented portion and its current boundary.
### 3. Use canonical homes
Each type of information should have one canonical location.
Canonical homes:
- project purpose and quickstart: `README.md`
- development principles: `docs/policy/architecture.md`
- configuration reference: `docs/config.md`
- CLI reference: `docs/cli.md`
- operations and recovery: `docs/operations.md`
- troubleshooting: `docs/troubleshooting.md`
- implemented internals: `docs/internal/`
- future work: `docs/roadmap/`
- contributor workflow: `docs/policy/development.md`
- copyable examples: `examples/`
Other files should summarize briefly and link to the canonical source.
### 4. Keep examples real
Examples should be valid, maintained, and free of secrets.
Where practical:
- example configs should load successfully;
- example commands should match real CLI syntax;
- important examples should be covered by tests.
## Documentation Profiles
All projects require:
- `README.md`
- `docs/policy/architecture.md`
Additional docs depend on the project.
### Small library
Recommended:
- `docs/policy/development.md`, if contributor conventions are non-obvious
### Simple CLI
Required:
- `docs/cli.md`
Recommended:
- `docs/policy/development.md`
### Config-driven CLI
Required:
- `docs/cli.md`
- `docs/config.md`
Recommended:
- `examples/`
- `docs/policy/development.md`
### Stateful or operator-facing application
Required:
- `docs/cli.md`, if CLI-based
- `docs/config.md`, if config-driven
- `docs/operations.md`
Recommended:
- `docs/troubleshooting.md`
- `examples/`
- `docs/policy/development.md`
### Modular, staged, service-oriented, or orchestration application
Required:
- `docs/cli.md`, if CLI-based
- `docs/config.md`, if config-driven
- `docs/operations.md`
- `docs/internal/`
- `docs/policy/development.md`
Recommended:
- `docs/troubleshooting.md`
- validated examples under `examples/`
## Required Documents
### README.md
**Audience:** users, administrators, operators
The README is the outward-facing project orientation page.
It should include, in order:
1. concise description;
2. elevator pitch;
3. shortest useful command or usage example;
4. links to targeted docs.
The README should be short. It is not a manual.
The “shortest useful command” means the simplest command that performs the projects core use case. (It does not mean `app --help`.)
### docs/policy/architecture.md
**Audience:** developers, LLM coding agents
`docs/policy/architecture.md` is required for every project.
It is an inward-facing development policy document. It should describe how the project is intended to be built and changed.
It should include:
- project shape;
- core design principles;
- package and boundary philosophy;
- state/persistence philosophy, if applicable;
- external integration philosophy, if applicable;
- error-handling and logging principles;
- testing expectations;
- documentation expectations;
- architectural invariants;
- explicit non-goals, if useful.
For small projects, this file may be brief. It may simply state that the project is intentionally narrow, monolithic, and dependency-light.
### docs/policy/development.md
**Audience:** developers, LLM coding agents
Required for projects maintained by humans and LLM coding agents.
It should include:
- repository layout;
- build/test commands;
- coding conventions;
- dependency policy;
- how to add config fields;
- how to add CLI flags;
- how to add stages/modules/adapters, if applicable;
- how to update examples;
- documentation update expectations.
### docs/config.md
**Audience:** administrators, operators, advanced users
Required for applications with configuration files.
It should include, in order:
1. config file locations and discovery precedence;
2. minimal working config;
3. production-oriented config;
4. full configuration reference;
5. secrets handling, if applicable;
6. links to maintained examples.
The full configuration reference should be canonical.
### docs/cli.md
**Audience:** users, administrators, operators
Required for CLI applications.
It should include, in order:
1. shortest useful command;
2. command overview;
3. complete flag reference;
4. common workflows;
5. diagnostic or recovery commands, if applicable.
Explain when commands are useful, not just their syntax.
### docs/operations.md
**Audience:** administrators, operators
Required for applications that maintain state, support resume behavior, run multiple stages, write durable artifacts, use remote storage, or require recovery procedures.
It should cover:
- normal workflow;
- filesystem layout;
- remote storage layout, if applicable;
- logs and manifests;
- resume/retry behavior;
- cleanup behavior;
- archive/backup behavior;
- safe recovery procedures;
- operational caveats.
### docs/troubleshooting.md
**Audience:** administrators, operators
Recommended once recurring failure modes exist.
Each entry should include:
- symptom;
- likely cause;
- diagnostic command or inspection step;
- safe fix;
- relevant links.
### docs/internal/
**Audience:** developers, LLM coding agents
Required for modular, staged, service-oriented, or orchestration projects.
This directory describes implemented internal components. It is not the roadmap.
Use one file per major component where useful.
Each component doc should include:
1. purpose;
2. inputs and outputs;
3. boundaries;
4. config fields used;
5. external adapters used;
6. state or manifest behavior, if applicable;
7. skip/resume behavior, if applicable;
8. failure behavior;
9. tests to inspect before changing;
10. architectural invariants.
### docs/roadmap/
**Audience:** maintainers, developers, LLM coding agents
This is the only place for planned, future, aspirational, experimental, or unimplemented work.
Roadmap docs should clearly distinguish:
- proposed work;
- accepted plans;
- deferred ideas;
- rejected ideas;
- implementation prompts or task breakdowns, if useful.
Roadmap docs should not be confused with current behavior.
### docs/integrations/
**Audience:** developers, LLM coding agents
Required for projects that depend on external CLIs, APIs, services, protocols, or file formats where the integration contract is important to maintain.
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses.
Use one file per integration where useful.
## Examples Directory
Projects with non-trivial configuration or workflows should include `examples/`.
Useful examples include:
- minimal working config;
- production-oriented config;
- full annotated config;
- local development config;
- remote/object-storage config;
- minimal session/input file.
Examples should be valid, maintained, tested when practical, and linked from relevant docs.
## Security and Privacy
Docs and examples must not include:
- real API keys;
- tokens;
- passwords;
- private keys;
- private environment dumps;
- sensitive user data;
- raw private transcripts;
- private infrastructure details unless intentionally public.
Document secret-handling mechanisms, not actual secret values.
## Maintenance Rules
When docs change, verify the affected behavior.
Where practical:
- load example config files in tests;
- test CLI examples or command parser behavior;
- validate documented flags against real flags;
- remove stale references;
- update links after renames;
- keep roadmap content out of non-roadmap docs.
If documentation and code disagree, fix the documentation and/or open a roadmap item; do not leave aspirational behavior in current-behavior docs.
Documentation is complete only when it matches the current code.
## Documentation Change Checklist
Before merging documentation changes, verify:
- README is concise and orientation-focused.
- `docs/policy/architecture.md` describes development principles.
- Future work appears only under `docs/roadmap/`.
- User-facing docs avoid unnecessary internals.
- Developer-facing docs preserve boundaries and invariants.
- Config examples match the schema.
- CLI examples match real commands and flags.
- Defaults appear in the canonical config reference.
- No secrets or private data are included.
- Links are accurate.

View File

@@ -0,0 +1,579 @@
# Documentation Roadmap
## Purpose
This roadmap defines the work required to bring seriatim's documentation into
compliance with `docs/policy/documentation.md` and the current implementation.
It is grounded in the repository as it exists now: the Go CLI, config loading,
pipeline modules, artifact commands, schemas, reports, samples, and tests.
Outside `docs/roadmap/`, documentation must describe only implemented
behavior. Planned, future, deprecated, experimental, or unimplemented work must
remain in roadmap documents until the code exists.
## Repository Documentation Inventory
- `README.md` - keep and rewrite. It currently mixes project orientation,
quickstart, full CLI reference, config/env reference, file formats, module
internals, limitations, and release build notes. Policy says README should be
concise and link to canonical docs.
- `docs/policy/documentation.md` - keep and lightly update only if the policy
itself changes. It is the controlling documentation layout and maintenance
policy.
- `docs/policy/architecture.md` - keep and lightly update as implementation
changes. It is the canonical development architecture policy.
- Root `architecture.md` - delete after salvage, or move only truly roadmap
material into `docs/roadmap/`. It is in the wrong canonical home and contains
future-oriented and aspirational claims.
- `docs/roadmap/documentation.md` - create new. This file is the planning
artifact for the documentation migration.
- `samples/` - split or move after audit. It contains sample raw transcripts,
merged artifacts, reports, `speakers.yml`, and `autocorrect.yml`, but
copyable examples belong under `examples/`. The raw sample data is large and
should be reviewed for privacy and maintainability before linking from docs.
- `schema/*.schema.json` - keep. These are public output contracts and should
be linked from documentation instead of duplicated in full.
- Missing canonical docs - create `docs/cli.md`, `docs/config.md`,
`docs/operations.md`, `docs/policy/development.md`, `docs/internal/`, and
likely `docs/troubleshooting.md`, `docs/integrations/`, and `examples/`.
## Policy Compliance Assessment
Required documents missing for seriatim's current shape as a modular, staged,
CLI/config-driven project:
- `docs/cli.md`
- `docs/config.md`
- `docs/operations.md`
- `docs/internal/`
- `docs/policy/development.md`
Recommended documents and directories missing:
- `docs/troubleshooting.md`
- maintained copyable examples under `examples/`
- concise integration notes under `docs/integrations/`
Existing compliance issues:
- `README.md` is too broad for its canonical scope. It should keep project
purpose, quickstart, and links, then delegate CLI, config, operations,
internals, and schema details.
- Root `architecture.md` is stale and in the wrong home. It includes future
input methods and formats, future output formats, dynamic plugin speculation,
an LLM non-goal, interface sketches that diverge from code, and other
development-policy content now covered by `docs/policy/architecture.md`.
- Non-roadmap docs should not carry forward claims about future defaults,
future formats, unimplemented plugin systems, or unimplemented alternate
input/output methods.
- Historical or deprecated wording, such as the old speaker map format, should
move out of the README unless it is still needed in troubleshooting or a
narrow migration note.
- There is no `examples/` directory. `samples/` exists but is not the canonical
examples home and should not be treated as copyable public examples without a
privacy and size audit.
- Links need verification after migration: README should link to all new
canonical docs, docs should link to schema files and maintained examples, and
no doc should link to the deleted root `architecture.md`.
## Target Documentation Set
### `README.md`
- Audience: users, administrators, and operators.
- Purpose: project orientation and shortest useful quickstart.
- Canonical scope: concise project purpose, elevator pitch, one minimal command,
and links to targeted docs.
- Recommended outline: project description; shortest merge command; command
summary; links to CLI, config, operations, architecture, development, schemas,
examples, and troubleshooting.
- Source of truth: current `README.md`, `internal/cli`, `internal/config`,
`cmd/seriatim/main.go`, and CLI tests.
- Acceptance criteria: no full flag tables, no full config reference, no module
manual, no future-feature claims, and all links resolve.
### `docs/cli.md`
- Audience: users, administrators, and operators.
- Purpose: canonical CLI reference and workflows.
- Canonical scope: shortest useful command, command overview, complete flag
reference, common workflows, diagnostics and report flags.
- Recommended outline: shortest useful command; global flags; `merge`; `trim`;
`normalize`; common workflows; exit/error behavior; links to config,
operations, examples, and schemas.
- Source of truth: `internal/cli/root.go`, `internal/cli/merge.go`,
`internal/cli/trim.go`, `internal/cli/normalize.go`, `internal/config`, and
`internal/cli/*_test.go`.
- Acceptance criteria: every documented flag, default, and required/mutually
exclusive rule matches code; package internals are linked rather than
explained in depth.
### `docs/config.md`
- Audience: administrators, operators, and advanced users.
- Purpose: canonical runtime configuration reference.
- Canonical scope: environment variables, default module lists, output schema
selection, `speakers.yml`, `autocorrect.yml`, path validation, and precedence.
- Recommended outline: config surfaces; output schema precedence; merge module
defaults; environment variables; speaker map YAML; autocorrect YAML; path and
validation rules; links to examples.
- Source of truth: `internal/config/config.go`, `internal/speaker/map.go`,
`internal/autocorrect/autocorrect.go`, `internal/config/config_test.go`,
`internal/speaker/map_test.go`, and `internal/autocorrect/autocorrect_test.go`.
- Acceptance criteria: all config fields and `SERIATIM_*` env vars match code;
unsupported config files or unimplemented formats are not described.
### `docs/operations.md`
- Audience: administrators and operators.
- Purpose: operational behavior for running commands safely.
- Canonical scope: file workflow, filesystem layout expectations, output and
report files, retry behavior, cleanup, validation failures, and operational
caveats.
- Recommended outline: normal workflow; input/output/report files; no durable
state; failure and retry behavior; reports and diagnostics; cleanup; privacy
considerations for transcript artifacts.
- Source of truth: `cmd/seriatim/main.go`, `internal/cli`, `internal/config`,
`internal/report`, `internal/builtin/output.go`, `internal/normalize`, and
trim/merge/normalize CLI tests.
- Acceptance criteria: clearly states there is no daemon, database, resume
state, remote storage, or background job state; does not invent recovery
workflows.
### `docs/policy/development.md`
- Audience: developers and coding agents.
- Purpose: contributor workflow and change guidance.
- Canonical scope: repository layout, build/test commands, coding conventions,
dependency policy, adding flags/config fields/modules/docs/examples.
- Recommended outline: repo layout; local checks; coding conventions; adding
CLI flags; adding config/env vars; adding modules/stages; schema changes;
examples and documentation updates.
- Source of truth: `docs/policy/documentation.md`,
`docs/policy/architecture.md`, `go.mod`, package layout, and test layout.
- Acceptance criteria: includes `go test ./...`; states there is no current
Makefile, taskfile, linter config, or automated doc checker; aligns with the
architecture policy.
### `docs/internal/pipeline.md`
- Audience: developers and coding agents.
- Purpose: implemented merge pipeline internals.
- Canonical scope: registry, stage interfaces, preprocessing state transitions,
module order, report event accumulation, final output/report writing.
- Recommended outline: purpose; inputs and outputs; stage contracts; registry
resolution; execution order; config fields used; adapters; failure behavior;
tests; invariants.
- Source of truth: `internal/pipeline`, `internal/builtin`, `internal/model`,
`internal/report`, `internal/pipeline/runner_test.go`,
`internal/builtin/*_test.go`, and `internal/cli/merge_test.go`.
- Acceptance criteria: describes only implemented sequential execution; does
not document concurrency, plugins, or future formats.
### `docs/internal/artifacts.md`
- Audience: developers and coding agents.
- Purpose: public artifact conversion and validation internals.
- Canonical scope: schema structs, embedded JSON Schemas, conversion from merged
model, trim/normalize artifact handling, and output validation.
- Recommended outline: artifact contracts; schema selection; conversion;
validation; trim projection; normalize canonicalization; tests; invariants.
- Source of truth: `schema`, `internal/artifact`, `internal/trim`,
`internal/normalize`, and related tests.
- Acceptance criteria: links to `schema/*.schema.json`; does not duplicate full
schemas or describe unavailable output formats.
### `docs/internal/modules.md`
- Audience: developers and coding agents.
- Purpose: implemented built-in module behavior and boundaries.
- Canonical scope: `json-files`, preprocessing modules, chronological merge,
postprocessing modules, and JSON output writer.
- Recommended outline: module list; inputs/outputs; config fields used; allowed
side effects; ordering constraints; failure behavior; tests; invariants.
- Source of truth: `internal/builtin`, `internal/overlap`, `internal/coalesce`,
`internal/danglers`, `internal/backchannel`, `internal/filler`,
`internal/autocorrect`, and package tests.
- Acceptance criteria: avoids full CLI/config duplication; identifies
order-sensitive transforms that must run before `assign-ids`.
### `docs/troubleshooting.md`
- Audience: users, administrators, and operators.
- Purpose: common failure symptoms and safe fixes.
- Canonical scope: implemented validation and runtime failures observed in
error paths and tests.
- Recommended outline: invalid JSON/input shape; missing required flags; invalid
output parent directory; invalid speaker/autocorrect YAML; unknown module;
invalid output schema; invalid trim selector; schema validation failure;
report write failure.
- Source of truth: `internal/config`, `internal/cli/*_test.go`,
`internal/trim/*_test.go`, `internal/normalize/*_test.go`,
`internal/speaker/*_test.go`, and `internal/autocorrect/*_test.go`.
- Acceptance criteria: each entry has symptom, likely cause, inspection step,
safe fix, and link; no speculative failure modes.
### `docs/integrations/whisperx-json.md`
- Audience: developers and coding agents.
- Purpose: external input JSON contract used by `merge`.
- Canonical scope: the supported WhisperX-like subset only.
- Recommended outline: top-level shape; required segment fields; optional word
timing fields; validation/failure behavior; how word timing affects overlap
resolution; links to CLI and examples.
- Source of truth: `internal/builtin/input.go`, merge CLI tests, and README
input-format material.
- Acceptance criteria: does not attempt to document full WhisperX behavior or
unsupported input formats.
### `docs/integrations/output-schemas.md`
- Audience: developers, coding agents, and artifact consumers.
- Purpose: orientation to public JSON output contracts.
- Canonical scope: minimal/intermediate/full schema roles and links to schema
files.
- Recommended outline: schema selection; minimal; intermediate; full; semantic
invariants; validation APIs; links to `schema/*.schema.json`.
- Source of truth: `schema/output.go`, `schema/*.schema.json`,
`schema/output_test.go`, and `internal/artifact`.
- Acceptance criteria: links to machine-readable schemas instead of copying
them in full.
### `examples/`
- Audience: users, administrators, operators, developers, and coding agents.
- Purpose: maintained copyable examples.
- Canonical scope: small synthetic inputs and config files for implemented
commands only.
- Source of truth: examples created during the documentation migration and
validated through actual command invocations.
- Acceptance criteria: examples are valid, free of secrets/private transcript
data, and linked from README, CLI, config, and operations docs.
## File-by-File Rewrite Guidance
### README
Cover what seriatim is, the shortest useful `merge` command, a brief command
summary, and links to canonical docs. Avoid full flag tables, config/env
reference, module internals, schema examples, troubleshooting details, future
formats, or release-history narrative. Inspect `internal/cli`, `internal/config`,
and CLI tests before updating commands.
### CLI Reference
Document actual `merge`, `trim`, and `normalize` flags from `internal/cli`.
Include required flags, defaults, mutually exclusive selector rules, schema
selection, report flags, and common workflows. Link to `docs/config.md` for
environment variables and YAML formats. Avoid internal package explanations.
Inspect `internal/cli/*_test.go` for edge cases and examples.
### Config Reference
Document all implemented config surfaces: flags that become config values,
`SERIATIM_OUTPUT_SCHEMA`, `SERIATIM_OVERLAP_WORD_RUN_GAP`,
`SERIATIM_OVERLAP_WORD_RUN_REORDER_WINDOW`,
`SERIATIM_BACKCHANNEL_MAX_DURATION`, `SERIATIM_FILLER_MAX_DURATION`, module
lists, output schemas, `speakers.yml`, and `autocorrect.yml`. Avoid command
tutorials and unimplemented config files. Inspect `internal/config`,
`internal/speaker`, `internal/autocorrect`, and tests.
### Operations
Document filesystem-only command execution, output/report artifacts, validation
failures, retry behavior, and cleanup. Explicitly say there is no daemon,
database, remote storage, resume state, or background job state. Avoid
unimplemented recovery procedures.
### Development Policy
Document repository layout, `go test ./...`, package conventions,
standard-library-first dependency guidance, how to add flags/config/modules,
and documentation update expectations. State that no Makefile, taskfile,
linter config, or automated documentation checker currently exists.
### Internal Docs
Keep internal docs behavior-level and concise. Describe implemented inputs,
outputs, boundaries, config fields used, adapters, failure behavior, tests, and
invariants. Avoid future plugins, future input/output formats, concurrency, or
duplicating CLI/config reference material.
### Root `architecture.md`
Do not carry forward future input methods, future formats, future output
formats, LLM text, dynamic plugin speculation, or interface sketches that
diverge from code. Salvage only current-behavior details that are not already
covered in `docs/policy/architecture.md` and move any legitimate future ideas
under `docs/roadmap/`.
## Examples Plan
Create small synthetic examples under `examples/` rather than relying on the
current large `samples/raw` data.
- `examples/minimal-merge/`
- Purpose: shortest complete merge workflow with two small raw JSON files and
optional `speakers.yml`.
- Expected validity check: run `go run ./cmd/seriatim merge` with the example
files and validate JSON output is produced.
- Docs to link: README, `docs/cli.md`, `docs/config.md`,
`docs/operations.md`.
- `examples/normalize/`
- Purpose: normalize object-with-`segments` and bare segment array inputs.
- Expected validity check: run `go run ./cmd/seriatim normalize` for both
shapes.
- Docs to link: `docs/cli.md`, `docs/operations.md`, and any Audita/bare
array integration note if created.
- `examples/trim/`
- Purpose: trim a small existing seriatim artifact by `--keep` and/or
`--remove`.
- Expected validity check: run `go run ./cmd/seriatim trim` and validate
sequential retained IDs.
- Docs to link: `docs/cli.md`, `docs/operations.md`.
- `examples/speakers.yml` and `examples/autocorrect.yml`
- Purpose: copyable YAML rule examples if linked from `docs/config.md`.
- Expected validity check: load through merge command or package tests.
- Docs to link: `docs/config.md`, `docs/cli.md`.
Do not invent examples for unimplemented input methods, output formats,
services, or plugin systems. Do not reuse `samples/raw` as public examples
without privacy and size review.
## Internal Documentation Plan
### Pipeline
- Path: `docs/internal/pipeline.md`
- Purpose: document implemented merge pipeline orchestration.
- Inputs and outputs: `config.Config`, raw transcripts, canonical transcripts,
merged transcript, selected public artifact, optional report.
- Boundaries: registry and runner orchestration; no CLI flag parsing; no schema
details beyond output selection.
- Config fields used: input reader, module lists, output modules, output schema,
input/output/report files, timing thresholds passed through modules.
- Adapters used: input reader, output writer, report writer.
- Failure behavior: unknown modules, invalid preprocessing state, stage errors,
output/report write failures.
- Tests to inspect: `internal/pipeline/runner_test.go`,
`internal/builtin/*_test.go`, `internal/cli/merge_test.go`.
- Architectural invariants: deterministic sequential stage order, explicit
raw-to-canonical preprocessing state, output validation before acceptance.
### Artifacts and Schemas
- Path: `docs/internal/artifacts.md`
- Purpose: document public artifact conversion and validation internals.
- Inputs and outputs: merged model, schema structs, serialized JSON artifacts,
parsed trim/normalize artifacts.
- Boundaries: conversion and validation only; CLI docs own user-facing flags.
- Config fields used: output schema, output modules, input files for metadata.
- Adapters used: embedded JSON Schema files and JSON encoders/decoders.
- Failure behavior: schema validation errors, unsupported artifact/schema
conversion, invalid IDs/timing.
- Tests to inspect: `schema/output_test.go`,
`internal/artifact/transcript_test.go`, `internal/trim/*_test.go`,
`internal/normalize/*_test.go`.
- Architectural invariants: sequential IDs, selected schema validation, no
internal-only fields in public schemas.
### Built-In Modules
- Path: `docs/internal/modules.md`
- Purpose: document implemented module responsibilities and ordering
constraints.
- Inputs and outputs: raw transcripts, preprocess state, merged transcript,
report events, selected JSON output.
- Boundaries: module behavior only; no full CLI/config reference.
- Config fields used: speaker file, autocorrect file, coalesce gap, overlap word
gap, word run reorder window, backchannel/filler max durations.
- Adapters used: JSON input/output, speaker YAML, autocorrect YAML, report
events.
- Failure behavior: input validation errors, invalid YAML, unknown module names,
invalid output schema before write.
- Tests to inspect: `internal/builtin`, `internal/overlap`,
`internal/coalesce`, `internal/danglers`, `internal/backchannel`,
`internal/filler`, `internal/autocorrect`, and CLI merge tests.
- Architectural invariants: order-sensitive transforms run before `assign-ids`;
modules stay narrow and explicitly configured.
### Trim
- Path: include in `docs/internal/artifacts.md` or create
`docs/internal/trim.md` if artifacts doc grows too large.
- Purpose: document artifact-level segment projection.
- Inputs and outputs: existing seriatim artifact, selector, selected output
schema, optional report.
- Boundaries: no merge postprocessors; no raw WhisperX input.
- Config fields used: input/output/report files, keep/remove selector,
optional output schema, allow-empty.
- Adapters used: file I/O in CLI, artifact parsing/validation, report writer.
- Failure behavior: malformed selector, invalid artifact, missing selected IDs,
non-sequential input IDs, empty output unless allowed, unsupported schema
up-conversion.
- Tests to inspect: `internal/trim/*_test.go`, `internal/cli/trim_test.go`.
- Architectural invariants: preserve transcript order, renumber retained IDs,
recompute full-schema overlap groups, never run merge modules.
### Normalize
- Path: include in `docs/internal/artifacts.md` or create
`docs/internal/normalize.md` if artifacts doc grows too large.
- Purpose: document artifact-level transcript canonicalization.
- Inputs and outputs: transcript-like JSON object or bare array, selected
seriatim output schema, optional report.
- Boundaries: no merge preprocessing or postprocessing modules.
- Config fields used: input/output/report files, output schema, output modules.
- Adapters used: file I/O, JSON parsing, schema validation, report writer.
- Failure behavior: invalid JSON, unsupported top-level shape, invalid timing
after repair, unsupported output module/schema, report write failure.
- Tests to inspect: `internal/normalize/*_test.go`,
`internal/cli/normalize_test.go`.
- Architectural invariants: deterministic repair/sort/ID assignment, no
transcript text in normalize report events, no merge modules.
## Integration Documentation Plan
- `docs/integrations/whisperx-json.md`
- External system or contract: WhisperX-like JSON transcript subset.
- Current usage: `merge` reads a top-level `segments` array with required
segment timing/text and optional word timing.
- Version or compatibility notes: no explicit WhisperX version is encoded in
the repository; document only the accepted subset.
- Document: supported fields, validation, word timing behavior, errors.
- Do not document: full WhisperX schema, audio diarization, non-JSON formats.
- `docs/integrations/output-schemas.md`
- External system or contract: seriatim public JSON output contracts.
- Current usage: `merge`, `trim`, and `normalize` emit
`seriatim-minimal`, `seriatim-intermediate`, or `seriatim-full`.
- Version or compatibility notes: schemas are embedded from `schema/`; release
version metadata is injected through build info.
- Document: schema roles, semantic invariants, validation APIs, links to
schema files.
- Do not document: unimplemented output formats or full schema copies.
- YAML rule files
- Prefer documenting speaker and autocorrect YAML contracts in
`docs/config.md`. Create `docs/integrations/yaml-rule-files.md` only if the
config reference becomes too large.
- Audita-style bare arrays
- Cover under `docs/cli.md` normalize behavior unless maintainers need a
separate integration note. Do not generalize beyond implemented bare segment
arrays.
- No external CLI/API/service docs are needed now. The repository implements no
external CLI, network API, daemon, remote storage, or service integration.
## Recommended Implementation Sequence
### Stage 1: Write Documentation Roadmap
- Goal: create this roadmap.
- Files: `docs/roadmap/documentation.md`.
- Repository areas to inspect: documentation policy, architecture policy,
README, root `architecture.md`, CLI/config/pipeline/schema/report/tests.
- Acceptance criteria: roadmap exists, no other files changed by this stage,
and the roadmap is action-oriented.
- Suggested validation commands: `go test ./...`; `git status --short`.
- Prompt size: one implementation prompt.
### Stage 2: User-Facing Canonical Docs and Slim README
- Goal: move user reference material out of README into canonical docs.
- Files: update `README.md`; create `docs/cli.md` and `docs/config.md`.
- Repository areas to inspect: `internal/cli`, `internal/config`,
`internal/speaker`, `internal/autocorrect`, CLI/config tests.
- Acceptance criteria: README is concise; CLI/config docs match flags, defaults,
env vars, YAML formats, and validation; no roadmap-only content appears.
- Suggested validation commands: `go test ./...`;
`go run ./cmd/seriatim --help`;
`go run ./cmd/seriatim merge --help`;
`go run ./cmd/seriatim trim --help`;
`go run ./cmd/seriatim normalize --help`;
stale-term grep from the validation plan.
- Prompt size: one prompt if concise; split if README rewrite or config
reference grows too large.
### Stage 3: Operations and Troubleshooting
- Goal: document runtime operation, reports, failure behavior, and common fixes.
- Files: create `docs/operations.md` and `docs/troubleshooting.md`.
- Repository areas to inspect: `cmd/seriatim/main.go`, `internal/cli`,
`internal/config`, `internal/report`, output writer, normalize/trim/merge
tests.
- Acceptance criteria: docs describe filesystem-only operation and current
failure modes; no daemon, resume, remote storage, or recovery behavior is
invented.
- Suggested validation commands: `go test ./...`; manual link review.
- Prompt size: one prompt.
### Stage 4: Developer and Internal Docs
- Goal: create developer workflow and implemented internal component docs.
- Files: create `docs/policy/development.md`,
`docs/internal/pipeline.md`, `docs/internal/artifacts.md`, and
`docs/internal/modules.md`.
- Repository areas to inspect: architecture policy, pipeline, modules, schema,
artifact conversion, trim/normalize packages, tests.
- Acceptance criteria: docs preserve boundaries, avoid CLI/config duplication,
and identify tests/invariants for future changes.
- Suggested validation commands: `go test ./...`; grep for unimplemented
future-format/plugin/concurrency claims outside roadmap.
- Prompt size: split into development policy and internal docs if needed.
### Stage 5: Integrations and Examples
- Goal: add concise integration notes and maintained synthetic examples.
- Files: create `docs/integrations/whisperx-json.md`,
`docs/integrations/output-schemas.md`, and `examples/*`; decide whether
`samples/` should remain separate.
- Repository areas to inspect: `internal/builtin/input.go`, `schema`,
`internal/artifact`, CLI tests, existing `samples/`.
- Acceptance criteria: examples are small, synthetic, valid, and linked from
relevant docs; integration docs document only implemented contracts.
- Suggested validation commands: `go test ./...`; run documented example
`go run` commands; validate example YAML through command paths.
- Prompt size: split if examples need tests or sample cleanup decisions.
### Stage 6: Stale Documentation Cleanup
- Goal: remove wrong-home and stale documentation after canonical replacements
exist.
- Files: delete or relocate root `architecture.md`; remove stale material from
README; update links across docs.
- Repository areas to inspect: all docs, README, roadmap, root files.
- Acceptance criteria: no links to deleted root `architecture.md`; no
unimplemented behavior outside `docs/roadmap/`; canonical homes are respected.
- Suggested validation commands: `go test ./...`; stale-term grep; manual link
check; `git status --short`.
- Prompt size: one prompt.
## Validation Plan
Use these checks during or after documentation migration:
- Run `go test ./...`.
- Run `go run ./cmd/seriatim --help`.
- Run `go run ./cmd/seriatim merge --help`.
- Run `go run ./cmd/seriatim trim --help`.
- Run `go run ./cmd/seriatim normalize --help`.
- Once examples exist, run each documented example command and verify output is
produced in a temporary path.
- Load example YAML through the merge command or package tests.
- Validate example JSON through existing CLI/schema paths where practical.
- Grep outside `docs/roadmap/` for stale or roadmap-only terms:
`Future input`, `Future output`, `LLM`, `plugin`, `SRT`, `VTT`, `.tar.gz`,
`URI`, `old format`, `not implemented yet`, and
`runtime default may change`.
- Manually check links unless a link checker is added. No automated
documentation checker currently exists.
- Verify docs and examples contain no secrets, private transcript data, API
keys, tokens, passwords, or private infrastructure details.
## Open Questions
- Should `samples/` be removed, kept as non-doc sample data, or replaced by
small synthetic `examples/`? Recommendation: create small synthetic examples
first, then audit `samples/` for privacy, size, and ongoing maintenance before
deleting or linking it.
- Should Audita-style bare-array normalization have a separate integration doc?
Recommendation: cover it in `docs/cli.md` normalize behavior unless a
stronger external-contract requirement emerges.

39
internal/cli/normalize.go Normal file
View File

@@ -0,0 +1,39 @@
package cli
import (
"github.com/spf13/cobra"
"gitea.maximumdirect.net/eric/seriatim/internal/config"
"gitea.maximumdirect.net/eric/seriatim/internal/normalize"
)
func newNormalizeCommand() *cobra.Command {
var opts config.NormalizeOptions
cmd := &cobra.Command{
Use: "normalize",
Short: "Normalize a transcript artifact into a standard seriatim output shape",
RunE: func(cmd *cobra.Command, args []string) error {
normalizeOpts := opts
if !cmd.Flags().Changed("output-schema") {
normalizeOpts.OutputSchema = ""
}
cfg, err := config.NewNormalizeConfig(normalizeOpts)
if err != nil {
return err
}
return normalize.Run(cmd.Context(), cfg)
},
}
flags := cmd.Flags()
flags.StringVar(&opts.InputFile, "input-file", "", "input transcript JSON file")
flags.StringVar(&opts.OutputFile, "output-file", "", "output transcript JSON file")
flags.StringVar(&opts.ReportFile, "report-file", "", "optional report JSON file")
flags.StringVar(&opts.OutputSchema, "output-schema", config.DefaultOutputSchema, "output JSON schema: seriatim-minimal, seriatim-intermediate, or seriatim-full")
flags.StringVar(&opts.OutputModules, "output-modules", config.DefaultOutputModules, "comma-separated output modules")
return cmd
}

View File

@@ -0,0 +1,522 @@
package cli
import (
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/seriatim/internal/config"
"gitea.maximumdirect.net/eric/seriatim/internal/report"
"gitea.maximumdirect.net/eric/seriatim/schema"
)
func TestNormalizeCommandIsRecognized(t *testing.T) {
cmd := NewRootCommand()
cmd.SetArgs([]string{"normalize", "--help"})
if err := cmd.Execute(); err != nil {
t.Fatalf("normalize command should be recognized: %v", err)
}
}
func TestNormalizeMissingInputFileFails(t *testing.T) {
dir := t.TempDir()
output := filepath.Join(dir, "normalized.json")
err := executeNormalize(
"--output-file", output,
)
if err == nil {
t.Fatal("expected missing input-file error")
}
if !strings.Contains(err.Error(), "--input-file is required") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestNormalizeMissingOutputFileFails(t *testing.T) {
dir := t.TempDir()
input := writeJSONFile(t, dir, "input.json", `{"segments":[]}`)
err := executeNormalize(
"--input-file", input,
)
if err == nil {
t.Fatal("expected missing output-file error")
}
if !strings.Contains(err.Error(), "--output-file is required") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestNormalizeInvalidOutputSchemaFails(t *testing.T) {
dir := t.TempDir()
input := writeJSONFile(t, dir, "input.json", `{"segments":[]}`)
output := filepath.Join(dir, "normalized.json")
err := executeNormalize(
"--input-file", input,
"--output-file", output,
"--output-schema", "compact",
)
if err == nil {
t.Fatal("expected invalid output schema error")
}
if !strings.Contains(err.Error(), "--output-schema must be one of") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestNormalizeInvalidOutputModuleFails(t *testing.T) {
dir := t.TempDir()
input := writeJSONFile(t, dir, "input.json", `{"segments":[]}`)
output := filepath.Join(dir, "normalized.json")
err := executeNormalize(
"--input-file", input,
"--output-file", output,
"--output-modules", "yaml",
)
if err == nil {
t.Fatal("expected invalid output module error")
}
if !strings.Contains(err.Error(), "unknown output module") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestNormalizeDefaultOutputSchemaIsIntermediate(t *testing.T) {
dir := t.TempDir()
input := writeJSONFile(t, dir, "input.json", `{
"segments": [
{"id": 99, "start": 5, "end": 6, "speaker": "Bob", "text": "second", "categories": ["filler"]},
{"id": 10, "start": 1, "end": 2, "speaker": "Alice", "text": "first", "categories": ["backchannel"]}
]
}`)
output := filepath.Join(dir, "normalized.json")
err := executeNormalize(
"--input-file", input,
"--output-file", output,
)
if err != nil {
t.Fatalf("normalize failed: %v", err)
}
var transcript schema.IntermediateTranscript
readJSON(t, output, &transcript)
if transcript.Metadata.OutputSchema != config.OutputSchemaIntermediate {
t.Fatalf("output schema = %q, want %q", transcript.Metadata.OutputSchema, config.OutputSchemaIntermediate)
}
if len(transcript.Segments) != 2 {
t.Fatalf("segment count = %d, want 2", len(transcript.Segments))
}
if transcript.Segments[0].ID != 1 || transcript.Segments[1].ID != 2 {
t.Fatalf("segment IDs = %d,%d, want 1,2", transcript.Segments[0].ID, transcript.Segments[1].ID)
}
if transcript.Segments[0].Text != "first" || transcript.Segments[1].Text != "second" {
t.Fatalf("unexpected sort order: %#v", transcript.Segments)
}
if len(transcript.Segments[0].Categories) != 1 || transcript.Segments[0].Categories[0] != "backchannel" {
t.Fatalf("expected categories preserved on first segment, got %#v", transcript.Segments[0].Categories)
}
}
func TestNormalizeBareArrayInputToIntermediateOutput(t *testing.T) {
dir := t.TempDir()
input := writeJSONFile(t, dir, "input.json", `[
{"start": 2, "end": 3, "speaker": "Bob", "text": "second"},
{"start": 1, "end": 2, "speaker": "Alice", "text": "first"}
]`)
output := filepath.Join(dir, "normalized.json")
err := executeNormalize(
"--input-file", input,
"--output-file", output,
"--output-schema", config.OutputSchemaIntermediate,
)
if err != nil {
t.Fatalf("normalize failed: %v", err)
}
var transcript schema.IntermediateTranscript
readJSON(t, output, &transcript)
if len(transcript.Segments) != 2 {
t.Fatalf("segment count = %d, want 2", len(transcript.Segments))
}
if transcript.Segments[0].Speaker != "Alice" || transcript.Segments[1].Speaker != "Bob" {
t.Fatalf("unexpected sorted speakers: %#v", transcript.Segments)
}
}
func TestNormalizeInputIndexTieBreakerIsDeterministic(t *testing.T) {
dir := t.TempDir()
input := writeJSONFile(t, dir, "input.json", `[
{"start": 1, "end": 2, "speaker": "Zulu", "text": "first in"},
{"start": 1, "end": 2, "speaker": "Alpha", "text": "second in"}
]`)
output := filepath.Join(dir, "normalized.json")
err := executeNormalize(
"--input-file", input,
"--output-file", output,
)
if err != nil {
t.Fatalf("normalize failed: %v", err)
}
var transcript schema.IntermediateTranscript
readJSON(t, output, &transcript)
if transcript.Segments[0].Speaker != "Zulu" || transcript.Segments[1].Speaker != "Alpha" {
t.Fatalf("tie-break order mismatch: %#v", transcript.Segments)
}
}
func TestNormalizeMinimalSchemaOmitsCategories(t *testing.T) {
dir := t.TempDir()
input := writeJSONFile(t, dir, "input.json", `{
"segments": [
{"start": 1, "end": 2, "speaker": "Alice", "text": "first", "categories": ["filler"]}
]
}`)
output := filepath.Join(dir, "normalized.json")
err := executeNormalize(
"--input-file", input,
"--output-file", output,
"--output-schema", config.OutputSchemaMinimal,
)
if err != nil {
t.Fatalf("normalize failed: %v", err)
}
var transcript schema.MinimalTranscript
readJSON(t, output, &transcript)
if transcript.Metadata.OutputSchema != config.OutputSchemaMinimal {
t.Fatalf("output schema = %q, want %q", transcript.Metadata.OutputSchema, config.OutputSchemaMinimal)
}
if len(transcript.Segments) != 1 || transcript.Segments[0].ID != 1 {
t.Fatalf("unexpected minimal output: %#v", transcript.Segments)
}
bytes, readErr := os.ReadFile(output)
if readErr != nil {
t.Fatalf("read output: %v", readErr)
}
if strings.Contains(string(bytes), "categories") {
t.Fatalf("minimal output unexpectedly contains categories:\n%s", string(bytes))
}
}
func TestNormalizeFullSchemaOutputValidatesAndHasProvenanceFallback(t *testing.T) {
dir := t.TempDir()
input := writeJSONFile(t, dir, "input.json", `[
{"start": 1, "end": 2, "speaker": "Alice", "text": "first"},
{"start": 3, "end": 4, "speaker": "Bob", "text": "second", "source":"custom.json", "source_segment_index": 7}
]`)
output := filepath.Join(dir, "normalized.json")
err := executeNormalize(
"--input-file", input,
"--output-file", output,
"--output-schema", config.OutputSchemaFull,
)
if err != nil {
t.Fatalf("normalize failed: %v", err)
}
var transcript schema.Transcript
readJSON(t, output, &transcript)
if err := schema.ValidateTranscript(transcript); err != nil {
t.Fatalf("full output should validate: %v", err)
}
if len(transcript.Segments) != 2 {
t.Fatalf("segment count = %d, want 2", len(transcript.Segments))
}
if transcript.Segments[0].Source != filepath.Base(input) {
t.Fatalf("source fallback = %q, want %q", transcript.Segments[0].Source, filepath.Base(input))
}
if transcript.Segments[0].SourceSegmentIndex == nil || *transcript.Segments[0].SourceSegmentIndex != 0 {
t.Fatalf("source_segment_index fallback = %v, want 0", transcript.Segments[0].SourceSegmentIndex)
}
if transcript.Segments[1].Source != "custom.json" {
t.Fatalf("explicit source preserved = %q, want custom.json", transcript.Segments[1].Source)
}
if transcript.Segments[1].SourceSegmentIndex == nil || *transcript.Segments[1].SourceSegmentIndex != 7 {
t.Fatalf("explicit source_segment_index preserved = %v, want 7", transcript.Segments[1].SourceSegmentIndex)
}
if transcript.OverlapGroups == nil || len(transcript.OverlapGroups) != 0 {
t.Fatalf("overlap_groups = %#v, want empty array", transcript.OverlapGroups)
}
}
func TestNormalizeEmptySegmentsArrayProducesValidOutput(t *testing.T) {
dir := t.TempDir()
input := writeJSONFile(t, dir, "input.json", `{"segments":[]}`)
output := filepath.Join(dir, "normalized.json")
err := executeNormalize(
"--input-file", input,
"--output-file", output,
)
if err != nil {
t.Fatalf("normalize failed: %v", err)
}
var transcript schema.IntermediateTranscript
readJSON(t, output, &transcript)
if len(transcript.Segments) != 0 {
t.Fatalf("segment count = %d, want 0", len(transcript.Segments))
}
if err := schema.ValidateIntermediateTranscript(transcript); err != nil {
t.Fatalf("intermediate output should validate: %v", err)
}
}
func TestNormalizeRepairsAndDropsDefectiveSegments(t *testing.T) {
dir := t.TempDir()
input := writeJSONFile(t, dir, "input.json", `[
{"start": 5, "speaker": "", "text": "keep-a"},
{"end": 3, "speaker": " ", "text": "keep-b"},
{"speaker": "A"},
{"speaker": "A", "text": " "},
{"start": 9, "end": 4, "speaker": "B", "text": "keep-c"}
]`)
output := filepath.Join(dir, "normalized.json")
reportPath := filepath.Join(dir, "report.json")
err := executeNormalize(
"--input-file", input,
"--output-file", output,
"--report-file", reportPath,
"--output-schema", config.OutputSchemaIntermediate,
)
if err != nil {
t.Fatalf("normalize failed: %v", err)
}
var transcript schema.IntermediateTranscript
readJSON(t, output, &transcript)
if len(transcript.Segments) != 3 {
t.Fatalf("segment count = %d, want 3", len(transcript.Segments))
}
if transcript.Segments[0].Start != 3 || transcript.Segments[0].End != 3 {
t.Fatalf("segment[0] timing = %v..%v, want 3..3", transcript.Segments[0].Start, transcript.Segments[0].End)
}
if transcript.Segments[1].Start != 4 || transcript.Segments[1].End != 9 {
t.Fatalf("segment[1] timing = %v..%v, want 4..9", transcript.Segments[1].Start, transcript.Segments[1].End)
}
if transcript.Segments[2].Start != 5 || transcript.Segments[2].End != 5 {
t.Fatalf("segment[2] timing = %v..%v, want 5..5", transcript.Segments[2].Start, transcript.Segments[2].End)
}
if transcript.Segments[0].Speaker != "Unknown_Speaker" || transcript.Segments[2].Speaker != "Unknown_Speaker" {
t.Fatalf("expected Unknown_Speaker placeholders, got %#v", transcript.Segments)
}
var rpt report.Report
readJSON(t, reportPath, &rpt)
audit := extractNormalizeAudit(t, rpt)
if audit.InputSegmentCount != 5 || audit.OutputSegmentCount != 3 {
t.Fatalf("audit counts = in:%d out:%d, want in:5 out:3", audit.InputSegmentCount, audit.OutputSegmentCount)
}
if audit.TimingFieldsRepaired != 2 {
t.Fatalf("timing fields repaired = %d, want 2", audit.TimingFieldsRepaired)
}
if audit.TimingOrderSwapped != 1 {
t.Fatalf("timing order swapped = %d, want 1", audit.TimingOrderSwapped)
}
if audit.SpeakerFilled != 2 {
t.Fatalf("speaker filled = %d, want 2", audit.SpeakerFilled)
}
if audit.SegmentsDroppedText != 2 {
t.Fatalf("segments dropped text = %d, want 2", audit.SegmentsDroppedText)
}
}
func TestNormalizeSelectedOutputSchemaIsHonored(t *testing.T) {
dir := t.TempDir()
input := writeJSONFile(t, dir, "input.json", `{"segments":[{"start":1,"end":2,"speaker":"A","text":"one"}]}`)
output := filepath.Join(dir, "normalized.json")
err := executeNormalize(
"--input-file", input,
"--output-file", output,
"--output-schema", config.OutputSchemaMinimal,
)
if err != nil {
t.Fatalf("normalize failed: %v", err)
}
var transcript schema.MinimalTranscript
readJSON(t, output, &transcript)
if transcript.Metadata.OutputSchema != config.OutputSchemaMinimal {
t.Fatalf("output schema = %q, want %q", transcript.Metadata.OutputSchema, config.OutputSchemaMinimal)
}
}
func TestNormalizeReportFileWrittenAndContainsObjectInputShape(t *testing.T) {
dir := t.TempDir()
input := writeJSONFile(t, dir, "input.json", `{"segments":[{"start":1,"end":2,"speaker":"A","text":"one"}]}`)
output := filepath.Join(dir, "normalized.json")
reportPath := filepath.Join(dir, "report.json")
err := executeNormalize(
"--input-file", input,
"--output-file", output,
"--report-file", reportPath,
)
if err != nil {
t.Fatalf("normalize failed: %v", err)
}
var rpt report.Report
readJSON(t, reportPath, &rpt)
audit := extractNormalizeAudit(t, rpt)
if audit.InputShape != "object_with_segments" {
t.Fatalf("input shape = %q, want object_with_segments", audit.InputShape)
}
if audit.InputSegmentCount != 1 {
t.Fatalf("input segment count = %d, want 1", audit.InputSegmentCount)
}
if audit.OutputSchema != config.OutputSchemaIntermediate {
t.Fatalf("output schema = %q, want %q", audit.OutputSchema, config.OutputSchemaIntermediate)
}
if len(audit.OutputModules) != 1 || audit.OutputModules[0] != "json" {
t.Fatalf("output modules = %v, want [json]", audit.OutputModules)
}
}
func TestNormalizeReportIncludesBareArrayShape(t *testing.T) {
dir := t.TempDir()
input := writeJSONFile(t, dir, "input.json", `[{"start":1,"end":2,"speaker":"A","text":"one"}]`)
output := filepath.Join(dir, "normalized.json")
reportPath := filepath.Join(dir, "report.json")
err := executeNormalize(
"--input-file", input,
"--output-file", output,
"--report-file", reportPath,
)
if err != nil {
t.Fatalf("normalize failed: %v", err)
}
var rpt report.Report
readJSON(t, reportPath, &rpt)
audit := extractNormalizeAudit(t, rpt)
if audit.InputShape != "bare_segments_array" {
t.Fatalf("input shape = %q, want bare_segments_array", audit.InputShape)
}
}
func TestNormalizeReportDoesNotIncludeTranscriptText(t *testing.T) {
dir := t.TempDir()
const segmentText = "normalize-report-secret-text"
input := writeJSONFile(t, dir, "input.json", `[{"start":1,"end":2,"speaker":"A","text":"`+segmentText+`"}]`)
output := filepath.Join(dir, "normalized.json")
reportPath := filepath.Join(dir, "report.json")
err := executeNormalize(
"--input-file", input,
"--output-file", output,
"--report-file", reportPath,
)
if err != nil {
t.Fatalf("normalize failed: %v", err)
}
var rpt report.Report
readJSON(t, reportPath, &rpt)
for _, event := range rpt.Events {
if strings.Contains(event.Message, segmentText) {
t.Fatalf("report unexpectedly contained transcript text in event %#v", event)
}
}
}
func TestNormalizeReportEmptyInputEmitsWarning(t *testing.T) {
dir := t.TempDir()
input := writeJSONFile(t, dir, "input.json", `{"segments":[]}`)
output := filepath.Join(dir, "normalized.json")
reportPath := filepath.Join(dir, "report.json")
err := executeNormalize(
"--input-file", input,
"--output-file", output,
"--report-file", reportPath,
)
if err != nil {
t.Fatalf("normalize failed: %v", err)
}
var rpt report.Report
readJSON(t, reportPath, &rpt)
found := false
for _, event := range rpt.Events {
if event.Stage == "normalize" && event.Module == "normalize" && event.Severity == report.SeverityWarning &&
strings.Contains(event.Message, "zero segments") {
found = true
break
}
}
if !found {
t.Fatalf("expected empty transcript warning event, got %#v", rpt.Events)
}
}
func TestNormalizeReportWriteFailureReturnsClearError(t *testing.T) {
dir := t.TempDir()
input := writeJSONFile(t, dir, "input.json", `{"segments":[{"start":1,"end":2,"speaker":"A","text":"one"}]}`)
output := filepath.Join(dir, "normalized.json")
err := executeNormalize(
"--input-file", input,
"--output-file", output,
"--report-file", dir,
)
if err == nil {
t.Fatal("expected report write failure")
}
if !strings.Contains(err.Error(), "write --report-file") {
t.Fatalf("unexpected error: %v", err)
}
}
func executeNormalize(args ...string) error {
cmd := NewRootCommand()
cmd.SetArgs(append([]string{"normalize"}, args...))
return cmd.Execute()
}
type normalizeAudit struct {
Command string `json:"command"`
InputFile string `json:"input_file"`
OutputFile string `json:"output_file"`
InputShape string `json:"input_shape"`
InputSegmentCount int `json:"input_segment_count"`
OutputSegmentCount int `json:"output_segment_count"`
OutputSchema string `json:"output_schema"`
OutputModules []string `json:"output_modules"`
IDsReassigned bool `json:"ids_reassigned"`
SortingChangedInput bool `json:"sorting_changed_input_order"`
SegmentsWithCategories int `json:"segments_with_categories"`
TimingFieldsRepaired int `json:"timing_fields_repaired"`
TimingOrderSwapped int `json:"timing_order_swapped"`
SpeakerFilled int `json:"speaker_filled"`
SegmentsDroppedText int `json:"segments_dropped_text"`
}
func extractNormalizeAudit(t *testing.T, rpt report.Report) normalizeAudit {
t.Helper()
for _, event := range rpt.Events {
if event.Stage == "normalize" && event.Module == "normalize-audit" {
var audit normalizeAudit
if err := json.Unmarshal([]byte(event.Message), &audit); err != nil {
t.Fatalf("decode normalize audit: %v", err)
}
return audit
}
}
t.Fatalf("missing normalize-audit event: %#v", rpt.Events)
return normalizeAudit{}
}

View File

@@ -10,12 +10,14 @@ import (
func NewRootCommand() *cobra.Command { func NewRootCommand() *cobra.Command {
cmd := &cobra.Command{ cmd := &cobra.Command{
Use: "seriatim", Use: "seriatim",
Short: "Merge per-speaker transcripts into a chronological transcript", Short: "Merge, trim, and normalize transcript artifacts",
Version: buildinfo.Version, Version: buildinfo.Version,
SilenceErrors: true, SilenceErrors: true,
SilenceUsage: true, SilenceUsage: true,
} }
cmd.AddCommand(newMergeCommand()) cmd.AddCommand(newMergeCommand())
cmd.AddCommand(newNormalizeCommand())
cmd.AddCommand(newTrimCommand())
return cmd return cmd
} }

191
internal/cli/trim.go Normal file
View File

@@ -0,0 +1,191 @@
package cli
import (
"encoding/json"
"fmt"
"os"
"sort"
"github.com/spf13/cobra"
"gitea.maximumdirect.net/eric/seriatim/internal/config"
"gitea.maximumdirect.net/eric/seriatim/internal/report"
triminternal "gitea.maximumdirect.net/eric/seriatim/internal/trim"
)
type trimAuditReport struct {
Operation string `json:"operation"`
InputFile string `json:"input_file"`
OutputFile string `json:"output_file"`
InputSchema string `json:"input_schema"`
OutputSchema string `json:"output_schema"`
Mode string `json:"mode"`
Selector string `json:"selector"`
SelectedIDs []int `json:"selected_ids"`
AllowEmpty bool `json:"allow_empty"`
InputSegmentCount int `json:"input_segment_count"`
RetainedSegmentCount int `json:"retained_segment_count"`
RemovedSegmentCount int `json:"removed_segment_count"`
RemovedInputIDs []int `json:"removed_input_ids"`
OldToNewIDMapping []trimIDMapping `json:"old_to_new_id_mapping"`
OverlapGroupsRecomputed bool `json:"overlap_groups_recomputed"`
}
type trimIDMapping struct {
OldID int `json:"old_id"`
NewID int `json:"new_id"`
}
func newTrimCommand() *cobra.Command {
var opts config.TrimOptions
cmd := &cobra.Command{
Use: "trim",
Short: "Trim an existing seriatim transcript artifact by segment ID",
RunE: func(cmd *cobra.Command, args []string) error {
trimOpts := opts
if !cmd.Flags().Changed("output-schema") {
trimOpts.OutputSchema = ""
}
cfg, err := config.NewTrimConfig(trimOpts)
if err != nil {
return err
}
selector, err := triminternal.ParseSelector(cfg.Selector)
if err != nil {
return fmt.Errorf("invalid selector %q: %w", cfg.Selector, err)
}
data, err := os.ReadFile(cfg.InputFile)
if err != nil {
return fmt.Errorf("read --input-file %q: %w", cfg.InputFile, err)
}
artifact, err := triminternal.ParseArtifactJSON(data)
if err != nil {
return fmt.Errorf("--input-file %q: %w", cfg.InputFile, err)
}
inputSegmentCount := artifact.SegmentCount()
inputSchema := artifact.Schema
mode := triminternal.ModeKeep
if cfg.Mode == "remove" {
mode = triminternal.ModeRemove
}
trimmed, err := triminternal.ApplyArtifact(artifact, triminternal.Options{
Mode: mode,
Selector: selector,
AllowEmpty: cfg.AllowEmpty,
})
if err != nil {
return err
}
outputSchema := artifact.Schema
if cfg.OutputSchema != "" {
outputSchema = cfg.OutputSchema
}
outputArtifact, err := triminternal.ConvertArtifact(trimmed.Artifact, outputSchema)
if err != nil {
return err
}
if err := triminternal.ValidateArtifact(outputArtifact); err != nil {
return fmt.Errorf("validate trimmed output: %w", err)
}
if err := writeOutputJSON(cfg.OutputFile, outputArtifact.Value()); err != nil {
return err
}
if cfg.ReportFile != "" {
audit := trimAuditReport{
Operation: "trim",
InputFile: cfg.InputFile,
OutputFile: cfg.OutputFile,
InputSchema: inputSchema,
OutputSchema: outputArtifact.Schema,
Mode: cfg.Mode,
Selector: cfg.Selector,
SelectedIDs: selector.IDs(),
AllowEmpty: cfg.AllowEmpty,
InputSegmentCount: inputSegmentCount,
RetainedSegmentCount: len(trimmed.OldToNewID),
RemovedSegmentCount: len(trimmed.RemovedIDs),
RemovedInputIDs: append([]int(nil), trimmed.RemovedIDs...),
OldToNewIDMapping: orderedIDMapping(trimmed.OldToNewID),
OverlapGroupsRecomputed: trimmed.OverlapGroupsRecomputed,
}
auditJSON, err := json.Marshal(audit)
if err != nil {
return fmt.Errorf("marshal trim audit report: %w", err)
}
rpt := report.Report{
Metadata: report.Metadata{
Application: outputArtifact.Application(),
Version: outputArtifact.Version(),
InputReader: "trim-artifact",
InputFiles: []string{cfg.InputFile},
OutputModules: []string{"json"},
},
Events: []report.Event{
report.Info("trim", "trim", fmt.Sprintf("trimmed %d input segment(s) into %d output segment(s) with mode=%s", inputSegmentCount, outputArtifact.SegmentCount(), cfg.Mode)),
report.Info("trim", "trim-audit", string(auditJSON)),
report.Info("trim", "validate-output", fmt.Sprintf("validated %d output segment(s)", outputArtifact.SegmentCount())),
report.Info("output", "json", "wrote transcript JSON"),
},
}
if err := report.WriteJSON(cfg.ReportFile, rpt); err != nil {
return err
}
}
return nil
},
}
flags := cmd.Flags()
flags.StringVar(&opts.InputFile, "input-file", "", "input seriatim transcript artifact JSON file")
flags.StringVar(&opts.OutputFile, "output-file", "", "output transcript JSON file")
flags.StringVar(&opts.ReportFile, "report-file", "", "optional report JSON file")
flags.StringVar(&opts.Keep, "keep", "", "segment ID selector to keep (for example: 1-10,15)")
flags.StringVar(&opts.Remove, "remove", "", "segment ID selector to remove (for example: 1-10,15)")
flags.StringVar(&opts.OutputSchema, "output-schema", "", "optional output JSON schema override: seriatim-minimal, seriatim-intermediate, or seriatim-full")
flags.BoolVar(&opts.AllowEmpty, "allow-empty", false, "allow trimming to an empty transcript")
return cmd
}
func writeOutputJSON(path string, value any) error {
file, err := os.Create(path)
if err != nil {
return err
}
defer file.Close()
enc := json.NewEncoder(file)
enc.SetIndent("", " ")
return enc.Encode(value)
}
func orderedIDMapping(mapping map[int]int) []trimIDMapping {
keys := make([]int, 0, len(mapping))
for oldID := range mapping {
keys = append(keys, oldID)
}
sort.Ints(keys)
pairs := make([]trimIDMapping, 0, len(keys))
for _, oldID := range keys {
pairs = append(pairs, trimIDMapping{
OldID: oldID,
NewID: mapping[oldID],
})
}
return pairs
}

758
internal/cli/trim_test.go Normal file
View File

@@ -0,0 +1,758 @@
package cli
import (
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/seriatim/internal/config"
"gitea.maximumdirect.net/eric/seriatim/internal/report"
"gitea.maximumdirect.net/eric/seriatim/schema"
)
func TestTrimKeepModeEndToEnd(t *testing.T) {
dir := t.TempDir()
input := writeTrimFullFixture(t, dir, "input.json")
output := filepath.Join(dir, "trimmed.json")
err := executeTrim(
"--input-file", input,
"--output-file", output,
"--keep", "2,4",
)
if err != nil {
t.Fatalf("trim failed: %v", err)
}
var transcript schema.Transcript
readJSON(t, output, &transcript)
if len(transcript.Segments) != 2 {
t.Fatalf("segment count = %d, want 2", len(transcript.Segments))
}
if transcript.Segments[0].Text != "two" || transcript.Segments[1].Text != "four" {
t.Fatalf("unexpected kept text order: %#v", transcript.Segments)
}
assertSequentialIDs(t, []int{transcript.Segments[0].ID, transcript.Segments[1].ID})
}
func TestTrimRemoveModeEndToEnd(t *testing.T) {
dir := t.TempDir()
input := writeTrimFullFixture(t, dir, "input.json")
output := filepath.Join(dir, "trimmed.json")
err := executeTrim(
"--input-file", input,
"--output-file", output,
"--remove", "2,4",
)
if err != nil {
t.Fatalf("trim failed: %v", err)
}
var transcript schema.Transcript
readJSON(t, output, &transcript)
if len(transcript.Segments) != 2 {
t.Fatalf("segment count = %d, want 2", len(transcript.Segments))
}
if transcript.Segments[0].Text != "one" || transcript.Segments[1].Text != "three" {
t.Fatalf("unexpected remaining text order: %#v", transcript.Segments)
}
assertSequentialIDs(t, []int{transcript.Segments[0].ID, transcript.Segments[1].ID})
}
func TestTrimMutualExclusionFailure(t *testing.T) {
dir := t.TempDir()
input := writeTrimFullFixture(t, dir, "input.json")
output := filepath.Join(dir, "trimmed.json")
err := executeTrim(
"--input-file", input,
"--output-file", output,
"--keep", "1",
"--remove", "2",
)
if err == nil {
t.Fatal("expected mutual exclusion error")
}
if !strings.Contains(err.Error(), "mutually exclusive") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestTrimMissingSelectionFailure(t *testing.T) {
dir := t.TempDir()
input := writeTrimFullFixture(t, dir, "input.json")
output := filepath.Join(dir, "trimmed.json")
err := executeTrim(
"--input-file", input,
"--output-file", output,
)
if err == nil {
t.Fatal("expected selection flag error")
}
if !strings.Contains(err.Error(), "exactly one of --keep or --remove is required") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestTrimInvalidSelectedIDFailure(t *testing.T) {
dir := t.TempDir()
input := writeTrimFullFixture(t, dir, "input.json")
output := filepath.Join(dir, "trimmed.json")
err := executeTrim(
"--input-file", input,
"--output-file", output,
"--keep", "99",
)
if err == nil {
t.Fatal("expected missing selected ID error")
}
if !strings.Contains(err.Error(), "does not exist") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestTrimOmittedOutputSchemaPreservesInputSchema(t *testing.T) {
dir := t.TempDir()
input := writeTrimMinimalFixture(t, dir, "input-minimal.json")
output := filepath.Join(dir, "trimmed.json")
err := executeTrim(
"--input-file", input,
"--output-file", output,
"--keep", "1",
)
if err != nil {
t.Fatalf("trim failed: %v", err)
}
var transcript schema.MinimalTranscript
readJSON(t, output, &transcript)
if transcript.Metadata.OutputSchema != config.OutputSchemaMinimal {
t.Fatalf("output_schema = %q, want %q", transcript.Metadata.OutputSchema, config.OutputSchemaMinimal)
}
if len(transcript.Segments) != 1 || transcript.Segments[0].ID != 1 {
t.Fatalf("unexpected minimal trim output: %#v", transcript.Segments)
}
}
func TestTrimExplicitOutputSchemaChangesOutputSchema(t *testing.T) {
dir := t.TempDir()
input := writeTrimFullFixture(t, dir, "input.json")
output := filepath.Join(dir, "trimmed.json")
err := executeTrim(
"--input-file", input,
"--output-file", output,
"--keep", "1,3",
"--output-schema", config.OutputSchemaMinimal,
)
if err != nil {
t.Fatalf("trim failed: %v", err)
}
var transcript schema.MinimalTranscript
readJSON(t, output, &transcript)
if transcript.Metadata.OutputSchema != config.OutputSchemaMinimal {
t.Fatalf("output_schema = %q, want %q", transcript.Metadata.OutputSchema, config.OutputSchemaMinimal)
}
if len(transcript.Segments) != 2 {
t.Fatalf("segment count = %d, want 2", len(transcript.Segments))
}
assertSequentialIDs(t, []int{transcript.Segments[0].ID, transcript.Segments[1].ID})
}
func TestTrimExplicitOutputSchemaConvertsMinimalToIntermediate(t *testing.T) {
dir := t.TempDir()
input := writeTrimMinimalFixture(t, dir, "input-minimal.json")
output := filepath.Join(dir, "trimmed.json")
err := executeTrim(
"--input-file", input,
"--output-file", output,
"--keep", "1-2",
"--output-schema", config.OutputSchemaIntermediate,
)
if err != nil {
t.Fatalf("trim failed: %v", err)
}
var transcript schema.IntermediateTranscript
readJSON(t, output, &transcript)
if transcript.Metadata.OutputSchema != config.OutputSchemaIntermediate {
t.Fatalf("output_schema = %q, want %q", transcript.Metadata.OutputSchema, config.OutputSchemaIntermediate)
}
if len(transcript.Segments) != 2 {
t.Fatalf("segment count = %d, want 2", len(transcript.Segments))
}
assertSequentialIDs(t, []int{transcript.Segments[0].ID, transcript.Segments[1].ID})
}
func TestTrimIntermediateInputPreservesIntermediateOutputAndCategories(t *testing.T) {
dir := t.TempDir()
input := writeTrimIntermediateFixture(t, dir, "input-intermediate.json")
output := filepath.Join(dir, "trimmed.json")
err := executeTrim(
"--input-file", input,
"--output-file", output,
"--keep", "2",
)
if err != nil {
t.Fatalf("trim failed: %v", err)
}
var transcript schema.IntermediateTranscript
readJSON(t, output, &transcript)
if transcript.Metadata.OutputSchema != config.OutputSchemaIntermediate {
t.Fatalf("output_schema = %q, want %q", transcript.Metadata.OutputSchema, config.OutputSchemaIntermediate)
}
if len(transcript.Segments) != 1 {
t.Fatalf("segment count = %d, want 1", len(transcript.Segments))
}
if transcript.Segments[0].ID != 1 {
t.Fatalf("segment ID = %d, want 1", transcript.Segments[0].ID)
}
assertIntSliceEqual(t, []int{len(transcript.Segments[0].Categories)}, []int{2})
if transcript.Segments[0].Categories[0] != "filler" || transcript.Segments[0].Categories[1] != "backchannel" {
t.Fatalf("categories = %v, want [filler backchannel]", transcript.Segments[0].Categories)
}
}
func TestTrimFullInputPreservesFullShapeAndRecomputesOverlapGroups(t *testing.T) {
dir := t.TempDir()
input := writeTrimFullOverlapFixture(t, dir, "input-full-overlap.json")
output := filepath.Join(dir, "trimmed.json")
err := executeTrim(
"--input-file", input,
"--output-file", output,
"--keep", "1,2",
)
if err != nil {
t.Fatalf("trim failed: %v", err)
}
var transcript schema.Transcript
readJSON(t, output, &transcript)
if len(transcript.Segments) != 2 {
t.Fatalf("segment count = %d, want 2", len(transcript.Segments))
}
assertSequentialIDs(t, []int{transcript.Segments[0].ID, transcript.Segments[1].ID})
if len(transcript.OverlapGroups) != 1 {
t.Fatalf("overlap group count = %d, want 1", len(transcript.OverlapGroups))
}
if transcript.OverlapGroups[0].ID != 1 {
t.Fatalf("overlap group id = %d, want 1", transcript.OverlapGroups[0].ID)
}
if transcript.Segments[0].OverlapGroupID != 1 || transcript.Segments[1].OverlapGroupID != 1 {
t.Fatalf("segment overlap IDs = %d,%d, want 1,1", transcript.Segments[0].OverlapGroupID, transcript.Segments[1].OverlapGroupID)
}
}
func TestTrimMalformedSelectorFailsWithClearError(t *testing.T) {
dir := t.TempDir()
input := writeTrimFullFixture(t, dir, "input.json")
output := filepath.Join(dir, "trimmed.json")
err := executeTrim(
"--input-file", input,
"--output-file", output,
"--keep", "1-",
)
if err == nil {
t.Fatal("expected malformed selector error")
}
if !strings.Contains(err.Error(), "invalid selector") || !strings.Contains(err.Error(), "malformed element") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestTrimMalformedInputArtifactFailsClearly(t *testing.T) {
dir := t.TempDir()
input := writeJSONFile(t, dir, "broken.json", `{"metadata":`)
output := filepath.Join(dir, "trimmed.json")
err := executeTrim(
"--input-file", input,
"--output-file", output,
"--keep", "1",
)
if err == nil {
t.Fatal("expected malformed artifact error")
}
if !strings.Contains(err.Error(), "input JSON is malformed") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestTrimDuplicateInputSegmentIDsFail(t *testing.T) {
dir := t.TempDir()
input := writeTrimMinimalWithIDsFixture(t, dir, "input-dup.json", []int{1, 1})
output := filepath.Join(dir, "trimmed.json")
err := executeTrim(
"--input-file", input,
"--output-file", output,
"--keep", "1",
)
if err == nil {
t.Fatal("expected duplicate segment ID failure")
}
if !strings.Contains(err.Error(), "not a valid seriatim output artifact") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestTrimNonSequentialInputSegmentIDsFail(t *testing.T) {
dir := t.TempDir()
input := writeTrimMinimalWithIDsFixture(t, dir, "input-nonseq.json", []int{1, 3})
output := filepath.Join(dir, "trimmed.json")
err := executeTrim(
"--input-file", input,
"--output-file", output,
"--keep", "1",
)
if err == nil {
t.Fatal("expected non-sequential segment ID failure")
}
if !strings.Contains(err.Error(), "not a valid seriatim output artifact") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestTrimKeepSelectorWithOverlappingRanges(t *testing.T) {
dir := t.TempDir()
input := writeTrimFullFixture(t, dir, "input.json")
output := filepath.Join(dir, "trimmed.json")
err := executeTrim(
"--input-file", input,
"--output-file", output,
"--keep", "1-3,2-4",
)
if err != nil {
t.Fatalf("trim failed: %v", err)
}
var transcript schema.Transcript
readJSON(t, output, &transcript)
if len(transcript.Segments) != 4 {
t.Fatalf("segment count = %d, want 4", len(transcript.Segments))
}
assertSequentialIDs(t, []int{
transcript.Segments[0].ID,
transcript.Segments[1].ID,
transcript.Segments[2].ID,
transcript.Segments[3].ID,
})
}
func TestTrimRemoveSelectorWithOverlappingRanges(t *testing.T) {
dir := t.TempDir()
input := writeTrimFullFixture(t, dir, "input.json")
output := filepath.Join(dir, "trimmed.json")
err := executeTrim(
"--input-file", input,
"--output-file", output,
"--remove", "2-3,3-4",
)
if err != nil {
t.Fatalf("trim failed: %v", err)
}
var transcript schema.Transcript
readJSON(t, output, &transcript)
if len(transcript.Segments) != 1 {
t.Fatalf("segment count = %d, want 1", len(transcript.Segments))
}
if transcript.Segments[0].Text != "one" {
t.Fatalf("remaining segment = %#v, want one", transcript.Segments[0])
}
}
func TestTrimSelectorOrderDoesNotAffectTranscriptOrder(t *testing.T) {
dir := t.TempDir()
input := writeTrimFullFixture(t, dir, "input.json")
output := filepath.Join(dir, "trimmed.json")
err := executeTrim(
"--input-file", input,
"--output-file", output,
"--keep", "4,1,3",
)
if err != nil {
t.Fatalf("trim failed: %v", err)
}
var transcript schema.Transcript
readJSON(t, output, &transcript)
if len(transcript.Segments) != 3 {
t.Fatalf("segment count = %d, want 3", len(transcript.Segments))
}
got := []string{
transcript.Segments[0].Text,
transcript.Segments[1].Text,
transcript.Segments[2].Text,
}
want := []string{"one", "three", "four"}
if got[0] != want[0] || got[1] != want[1] || got[2] != want[2] {
t.Fatalf("segment text order = %v, want %v", got, want)
}
}
func TestTrimAllowEmptyBehavior(t *testing.T) {
dir := t.TempDir()
input := writeTrimFullFixture(t, dir, "input.json")
output := filepath.Join(dir, "trimmed.json")
err := executeTrim(
"--input-file", input,
"--output-file", output,
"--remove", "1-4",
)
if err == nil {
t.Fatal("expected empty-output error")
}
if !strings.Contains(err.Error(), "empty transcript") {
t.Fatalf("unexpected error: %v", err)
}
err = executeTrim(
"--input-file", input,
"--output-file", output,
"--remove", "1-4",
"--allow-empty",
)
if err != nil {
t.Fatalf("trim with --allow-empty failed: %v", err)
}
var transcript schema.Transcript
readJSON(t, output, &transcript)
if len(transcript.Segments) != 0 {
t.Fatalf("segment count = %d, want 0", len(transcript.Segments))
}
}
func TestTrimRejectsNonSeriatimInputArtifacts(t *testing.T) {
dir := t.TempDir()
input := writeJSONFile(t, dir, "raw-whisperx.json", `{
"segments": [
{"start": 1, "end": 2, "text": "hello"}
]
}`)
output := filepath.Join(dir, "trimmed.json")
err := executeTrim(
"--input-file", input,
"--output-file", output,
"--keep", "1",
)
if err == nil {
t.Fatal("expected invalid artifact error")
}
if !strings.Contains(err.Error(), "not a valid seriatim output artifact") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestTrimReportFileContainsAuditFields(t *testing.T) {
dir := t.TempDir()
input := writeTrimFullFixture(t, dir, "input.json")
output := filepath.Join(dir, "trimmed.json")
reportPath := filepath.Join(dir, "trim-report.json")
err := executeTrim(
"--input-file", input,
"--output-file", output,
"--report-file", reportPath,
"--remove", "4,2",
)
if err != nil {
t.Fatalf("trim failed: %v", err)
}
var rpt report.Report
readJSON(t, reportPath, &rpt)
if len(rpt.Events) == 0 {
t.Fatal("expected report events")
}
if !hasReportEvent(rpt, "trim", "trim", "trimmed 4 input segment(s) into 2 output segment(s) with mode=remove") {
t.Fatal("expected trim summary event")
}
if !hasReportEvent(rpt, "trim", "validate-output", "validated 2 output segment(s)") {
t.Fatal("expected validation event")
}
audit := extractTrimAuditEvent(t, rpt)
if audit.Operation != "trim" {
t.Fatalf("operation = %q, want trim", audit.Operation)
}
if audit.InputFile != input {
t.Fatalf("input_file = %q, want %q", audit.InputFile, input)
}
if audit.OutputFile != output {
t.Fatalf("output_file = %q, want %q", audit.OutputFile, output)
}
if audit.InputSchema != config.OutputSchemaFull || audit.OutputSchema != config.OutputSchemaFull {
t.Fatalf("schemas = %q -> %q, want full -> full", audit.InputSchema, audit.OutputSchema)
}
if audit.Mode != "remove" {
t.Fatalf("mode = %q, want remove", audit.Mode)
}
if audit.Selector != "4,2" {
t.Fatalf("selector = %q, want %q", audit.Selector, "4,2")
}
assertIntSliceEqual(t, audit.SelectedIDs, []int{2, 4})
if audit.AllowEmpty {
t.Fatal("allow_empty should be false")
}
if audit.InputSegmentCount != 4 || audit.RetainedSegmentCount != 2 || audit.RemovedSegmentCount != 2 {
t.Fatalf("counts = input:%d retained:%d removed:%d, want 4/2/2", audit.InputSegmentCount, audit.RetainedSegmentCount, audit.RemovedSegmentCount)
}
assertIntSliceEqual(t, audit.RemovedInputIDs, []int{2, 4})
if len(audit.OldToNewIDMapping) != 2 {
t.Fatalf("mapping length = %d, want 2", len(audit.OldToNewIDMapping))
}
if audit.OldToNewIDMapping[0].OldID != 1 || audit.OldToNewIDMapping[0].NewID != 1 {
t.Fatalf("mapping[0] = %#v, want old_id=1 new_id=1", audit.OldToNewIDMapping[0])
}
if audit.OldToNewIDMapping[1].OldID != 3 || audit.OldToNewIDMapping[1].NewID != 2 {
t.Fatalf("mapping[1] = %#v, want old_id=3 new_id=2", audit.OldToNewIDMapping[1])
}
if !audit.OverlapGroupsRecomputed {
t.Fatal("expected overlap_groups_recomputed=true for full schema trim")
}
}
func TestTrimReportOldToNewMappingIsDeterministicSorted(t *testing.T) {
dir := t.TempDir()
input := writeTrimFullFixture(t, dir, "input.json")
output := filepath.Join(dir, "trimmed.json")
reportPath := filepath.Join(dir, "trim-report.json")
err := executeTrim(
"--input-file", input,
"--output-file", output,
"--report-file", reportPath,
"--keep", "4,1,3",
)
if err != nil {
t.Fatalf("trim failed: %v", err)
}
var rpt report.Report
readJSON(t, reportPath, &rpt)
audit := extractTrimAuditEvent(t, rpt)
if len(audit.OldToNewIDMapping) != 3 {
t.Fatalf("mapping length = %d, want 3", len(audit.OldToNewIDMapping))
}
for index, expectedOld := range []int{1, 3, 4} {
if audit.OldToNewIDMapping[index].OldID != expectedOld {
t.Fatalf("mapping[%d].old_id = %d, want %d", index, audit.OldToNewIDMapping[index].OldID, expectedOld)
}
}
}
func TestTrimNoReportFileWhenOmitted(t *testing.T) {
dir := t.TempDir()
input := writeTrimFullFixture(t, dir, "input.json")
output := filepath.Join(dir, "trimmed.json")
reportPath := filepath.Join(dir, "trim-report.json")
err := executeTrim(
"--input-file", input,
"--output-file", output,
"--keep", "1",
)
if err != nil {
t.Fatalf("trim failed: %v", err)
}
_, statErr := os.Stat(reportPath)
if !os.IsNotExist(statErr) {
t.Fatalf("expected no report file at %q, got err=%v", reportPath, statErr)
}
}
func executeTrim(args ...string) error {
cmd := NewRootCommand()
cmd.SetArgs(append([]string{"trim"}, args...))
return cmd.Execute()
}
func writeTrimFullFixture(t *testing.T, dir string, name string) string {
t.Helper()
first := 10
second := 20
third := 30
fourth := 40
value := schema.Transcript{
Metadata: schema.Metadata{
Application: "seriatim",
Version: "v-test",
InputReader: "json-files",
InputFiles: []string{"a.json"},
PreprocessingModules: []string{"validate-raw"},
PostprocessingModules: []string{"assign-ids"},
OutputModules: []string{"json"},
},
Segments: []schema.Segment{
{ID: 1, Source: "a.json", SourceSegmentIndex: &first, SourceRef: "a.json#10", Speaker: "A", Start: 1, End: 2, Text: "one", OverlapGroupID: 9},
{ID: 2, Source: "a.json", SourceSegmentIndex: &second, SourceRef: "a.json#20", Speaker: "B", Start: 2, End: 3, Text: "two", OverlapGroupID: 9},
{ID: 3, Source: "a.json", SourceSegmentIndex: &third, SourceRef: "a.json#30", Speaker: "C", Start: 4, End: 5, Text: "three", OverlapGroupID: 10},
{ID: 4, Source: "a.json", SourceSegmentIndex: &fourth, SourceRef: "a.json#40", Speaker: "D", Start: 5, End: 6, Text: "four", OverlapGroupID: 10},
},
OverlapGroups: []schema.OverlapGroup{
{ID: 9, Start: 1, End: 3, Segments: []string{"a.json#10", "a.json#20"}, Speakers: []string{"A", "B"}, Class: "unknown", Resolution: "unresolved"},
},
}
return writeTrimArtifactFile(t, dir, name, value)
}
func writeTrimMinimalFixture(t *testing.T, dir string, name string) string {
t.Helper()
value := schema.MinimalTranscript{
Metadata: schema.MinimalMetadata{
Application: "seriatim",
Version: "v-test",
OutputSchema: config.OutputSchemaMinimal,
},
Segments: []schema.MinimalSegment{
{ID: 1, Start: 1, End: 2, Speaker: "A", Text: "one"},
{ID: 2, Start: 2, End: 3, Speaker: "B", Text: "two"},
},
}
return writeTrimArtifactFile(t, dir, name, value)
}
func writeTrimIntermediateFixture(t *testing.T, dir string, name string) string {
t.Helper()
value := schema.IntermediateTranscript{
Metadata: schema.IntermediateMetadata{
Application: "seriatim",
Version: "v-test",
OutputSchema: config.OutputSchemaIntermediate,
},
Segments: []schema.IntermediateSegment{
{ID: 1, Start: 1, End: 2, Speaker: "A", Text: "one", Categories: []string{"word-run"}},
{ID: 2, Start: 2, End: 3, Speaker: "B", Text: "two", Categories: []string{"filler", "backchannel"}},
},
}
return writeTrimArtifactFile(t, dir, name, value)
}
func writeTrimMinimalWithIDsFixture(t *testing.T, dir string, name string, ids []int) string {
t.Helper()
if len(ids) < 2 {
t.Fatalf("need at least two IDs, got %d", len(ids))
}
value := schema.MinimalTranscript{
Metadata: schema.MinimalMetadata{
Application: "seriatim",
Version: "v-test",
OutputSchema: config.OutputSchemaMinimal,
},
Segments: []schema.MinimalSegment{
{ID: ids[0], Start: 1, End: 2, Speaker: "A", Text: "one"},
{ID: ids[1], Start: 2, End: 3, Speaker: "B", Text: "two"},
},
}
return writeTrimArtifactFile(t, dir, name, value)
}
func writeTrimFullOverlapFixture(t *testing.T, dir string, name string) string {
t.Helper()
first := 10
second := 20
third := 30
value := schema.Transcript{
Metadata: schema.Metadata{
Application: "seriatim",
Version: "v-test",
InputReader: "json-files",
InputFiles: []string{"a.json"},
PreprocessingModules: []string{"validate-raw"},
PostprocessingModules: []string{"detect-overlaps", "assign-ids"},
OutputModules: []string{"json"},
},
Segments: []schema.Segment{
{ID: 1, Source: "a.json", SourceSegmentIndex: &first, SourceRef: "a.json#10", Speaker: "A", Start: 1, End: 3, Text: "one", OverlapGroupID: 5},
{ID: 2, Source: "a.json", SourceSegmentIndex: &second, SourceRef: "a.json#20", Speaker: "B", Start: 2, End: 4, Text: "two", OverlapGroupID: 5},
{ID: 3, Source: "a.json", SourceSegmentIndex: &third, SourceRef: "a.json#30", Speaker: "C", Start: 6, End: 7, Text: "three", OverlapGroupID: 6},
},
OverlapGroups: []schema.OverlapGroup{
{ID: 99, Start: 0, End: 100, Segments: []string{"stale"}, Speakers: []string{"stale"}, Class: "unknown", Resolution: "unresolved"},
},
}
return writeTrimArtifactFile(t, dir, name, value)
}
func writeTrimArtifactFile(t *testing.T, dir string, name string, value any) string {
t.Helper()
data, err := json.MarshalIndent(value, "", " ")
if err != nil {
t.Fatalf("marshal fixture: %v", err)
}
path := filepath.Join(dir, name)
if err := os.WriteFile(path, append(data, '\n'), 0o600); err != nil {
t.Fatalf("write fixture: %v", err)
}
return path
}
func assertSequentialIDs(t *testing.T, ids []int) {
t.Helper()
for index, id := range ids {
want := index + 1
if id != want {
t.Fatalf("id at index %d = %d, want %d", index, id, want)
}
}
}
func extractTrimAuditEvent(t *testing.T, rpt report.Report) trimAuditReport {
t.Helper()
for _, event := range rpt.Events {
if event.Stage == "trim" && event.Module == "trim-audit" {
var audit trimAuditReport
if err := json.Unmarshal([]byte(event.Message), &audit); err != nil {
t.Fatalf("decode trim audit event: %v", err)
}
return audit
}
}
t.Fatal("missing trim-audit event")
return trimAuditReport{}
}
func assertIntSliceEqual(t *testing.T, got []int, want []int) {
t.Helper()
if len(got) != len(want) {
t.Fatalf("slice length = %d, want %d", len(got), len(want))
}
for index := range got {
if got[index] != want[index] {
t.Fatalf("slice[%d] = %d, want %d (full got=%v, want=%v)", index, got[index], want[index], got, want)
}
}
}

View File

@@ -47,6 +47,26 @@ type MergeOptions struct {
CoalesceGap string CoalesceGap string
} }
// TrimOptions captures raw CLI option values before validation.
type TrimOptions struct {
InputFile string
OutputFile string
ReportFile string
Keep string
Remove string
OutputSchema string
AllowEmpty bool
}
// NormalizeOptions captures raw CLI option values before validation.
type NormalizeOptions struct {
InputFile string
OutputFile string
ReportFile string
OutputSchema string
OutputModules string
}
// Config is the validated runtime configuration for a merge invocation. // Config is the validated runtime configuration for a merge invocation.
type Config struct { type Config struct {
InputFiles []string InputFiles []string
@@ -66,6 +86,26 @@ type Config struct {
FillerMaxDuration float64 FillerMaxDuration float64
} }
// TrimConfig is the validated runtime configuration for a trim invocation.
type TrimConfig struct {
InputFile string
OutputFile string
ReportFile string
Mode string
Selector string
OutputSchema string
AllowEmpty bool
}
// NormalizeConfig is the validated runtime configuration for a normalize invocation.
type NormalizeConfig struct {
InputFile string
OutputFile string
ReportFile string
OutputSchema string
OutputModules []string
}
// NewMergeConfig validates raw merge options and returns normalized config. // NewMergeConfig validates raw merge options and returns normalized config.
func NewMergeConfig(opts MergeOptions) (Config, error) { func NewMergeConfig(opts MergeOptions) (Config, error) {
cfg := Config{ cfg := Config{
@@ -168,6 +208,111 @@ func NewMergeConfig(opts MergeOptions) (Config, error) {
return cfg, nil return cfg, nil
} }
// NewTrimConfig validates raw trim options and returns normalized config.
func NewTrimConfig(opts TrimOptions) (TrimConfig, error) {
inputFile := filepath.Clean(strings.TrimSpace(opts.InputFile))
if strings.TrimSpace(opts.InputFile) == "" {
return TrimConfig{}, errors.New("--input-file is required")
}
if err := requireFile(inputFile, "--input-file"); err != nil {
return TrimConfig{}, err
}
outputFile, err := normalizeOutputPath(opts.OutputFile, "--output-file")
if err != nil {
return TrimConfig{}, err
}
reportFile := ""
if strings.TrimSpace(opts.ReportFile) != "" {
reportFile, err = normalizeOutputPath(opts.ReportFile, "--report-file")
if err != nil {
return TrimConfig{}, err
}
}
keep := strings.TrimSpace(opts.Keep)
remove := strings.TrimSpace(opts.Remove)
if keep == "" && remove == "" {
return TrimConfig{}, errors.New("exactly one of --keep or --remove is required")
}
if keep != "" && remove != "" {
return TrimConfig{}, errors.New("--keep and --remove are mutually exclusive")
}
mode := "keep"
selector := keep
if remove != "" {
mode = "remove"
selector = remove
}
outputSchema := strings.TrimSpace(opts.OutputSchema)
if outputSchema != "" {
if err := validateOutputSchema(outputSchema); err != nil {
return TrimConfig{}, err
}
}
return TrimConfig{
InputFile: inputFile,
OutputFile: outputFile,
ReportFile: reportFile,
Mode: mode,
Selector: selector,
OutputSchema: outputSchema,
AllowEmpty: opts.AllowEmpty,
}, nil
}
// NewNormalizeConfig validates raw normalize options and returns normalized config.
func NewNormalizeConfig(opts NormalizeOptions) (NormalizeConfig, error) {
inputFile := filepath.Clean(strings.TrimSpace(opts.InputFile))
if strings.TrimSpace(opts.InputFile) == "" {
return NormalizeConfig{}, errors.New("--input-file is required")
}
if err := requireFile(inputFile, "--input-file"); err != nil {
return NormalizeConfig{}, err
}
outputFile, err := normalizeOutputPath(opts.OutputFile, "--output-file")
if err != nil {
return NormalizeConfig{}, err
}
reportFile := ""
if strings.TrimSpace(opts.ReportFile) != "" {
reportFile, err = normalizeOutputPath(opts.ReportFile, "--report-file")
if err != nil {
return NormalizeConfig{}, err
}
}
outputSchema, err := resolveOutputSchema(opts.OutputSchema)
if err != nil {
return NormalizeConfig{}, err
}
outputModules, err := parseModuleList(opts.OutputModules)
if err != nil {
return NormalizeConfig{}, fmt.Errorf("--output-modules: %w", err)
}
if len(outputModules) == 0 {
return NormalizeConfig{}, errors.New("--output-modules must include at least one module")
}
if err := validateNormalizeOutputModules(outputModules); err != nil {
return NormalizeConfig{}, err
}
return NormalizeConfig{
InputFile: inputFile,
OutputFile: outputFile,
ReportFile: reportFile,
OutputSchema: outputSchema,
OutputModules: outputModules,
}, nil
}
func parseModuleList(value string) ([]string, error) { func parseModuleList(value string) ([]string, error) {
value = strings.TrimSpace(value) value = strings.TrimSpace(value)
if value == "" { if value == "" {
@@ -321,3 +466,12 @@ func contains(values []string, target string) bool {
} }
return false return false
} }
func validateNormalizeOutputModules(modules []string) error {
for _, module := range modules {
if module != "json" {
return fmt.Errorf("unknown output module %q", module)
}
}
return nil
}

View File

@@ -612,6 +612,206 @@ func TestCoalesceGapRejectsInvalidOverride(t *testing.T) {
} }
} }
func TestNewTrimConfigRequiresInputAndOutput(t *testing.T) {
dir := t.TempDir()
input := writeTempFile(t, dir, "input.json")
output := filepath.Join(dir, "trimmed.json")
_, err := NewTrimConfig(TrimOptions{
OutputFile: output,
Keep: "1",
})
if err == nil || !strings.Contains(err.Error(), "--input-file is required") {
t.Fatalf("expected input-file required error, got %v", err)
}
_, err = NewTrimConfig(TrimOptions{
InputFile: input,
Keep: "1",
})
if err == nil || !strings.Contains(err.Error(), "--output-file is required") {
t.Fatalf("expected output-file required error, got %v", err)
}
}
func TestNewTrimConfigRequiresExactlyOneSelectorFlag(t *testing.T) {
dir := t.TempDir()
input := writeTempFile(t, dir, "input.json")
output := filepath.Join(dir, "trimmed.json")
_, err := NewTrimConfig(TrimOptions{
InputFile: input,
OutputFile: output,
})
if err == nil || !strings.Contains(err.Error(), "exactly one of --keep or --remove is required") {
t.Fatalf("expected missing selector error, got %v", err)
}
_, err = NewTrimConfig(TrimOptions{
InputFile: input,
OutputFile: output,
Keep: "1",
Remove: "2",
})
if err == nil || !strings.Contains(err.Error(), "mutually exclusive") {
t.Fatalf("expected mutually exclusive selector error, got %v", err)
}
}
func TestNewTrimConfigAcceptsOutputSchemaOverride(t *testing.T) {
dir := t.TempDir()
input := writeTempFile(t, dir, "input.json")
output := filepath.Join(dir, "trimmed.json")
reportPath := filepath.Join(dir, "report.json")
cfg, err := NewTrimConfig(TrimOptions{
InputFile: input,
OutputFile: output,
ReportFile: reportPath,
Remove: "3-5",
OutputSchema: OutputSchemaMinimal,
AllowEmpty: true,
})
if err != nil {
t.Fatalf("config failed: %v", err)
}
if cfg.Mode != "remove" {
t.Fatalf("mode = %q, want remove", cfg.Mode)
}
if cfg.Selector != "3-5" {
t.Fatalf("selector = %q, want 3-5", cfg.Selector)
}
if cfg.OutputSchema != OutputSchemaMinimal {
t.Fatalf("output schema = %q, want %q", cfg.OutputSchema, OutputSchemaMinimal)
}
if !cfg.AllowEmpty {
t.Fatal("allow empty should be true")
}
if cfg.ReportFile != reportPath {
t.Fatalf("report file = %q, want %q", cfg.ReportFile, reportPath)
}
}
func TestNewTrimConfigRejectsInvalidOutputSchemaOverride(t *testing.T) {
dir := t.TempDir()
input := writeTempFile(t, dir, "input.json")
output := filepath.Join(dir, "trimmed.json")
_, err := NewTrimConfig(TrimOptions{
InputFile: input,
OutputFile: output,
Keep: "1",
OutputSchema: "compact",
})
if err == nil {
t.Fatal("expected output schema validation error")
}
if !strings.Contains(err.Error(), "--output-schema must be one of") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestNewNormalizeConfigRequiresInputFile(t *testing.T) {
dir := t.TempDir()
output := filepath.Join(dir, "normalized.json")
_, err := NewNormalizeConfig(NormalizeOptions{
OutputFile: output,
OutputModules: DefaultOutputModules,
})
if err == nil {
t.Fatal("expected input-file required error")
}
if !strings.Contains(err.Error(), "--input-file is required") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestNewNormalizeConfigRequiresOutputFile(t *testing.T) {
dir := t.TempDir()
input := writeTempFile(t, dir, "input.json")
_, err := NewNormalizeConfig(NormalizeOptions{
InputFile: input,
OutputModules: DefaultOutputModules,
})
if err == nil {
t.Fatal("expected output-file required error")
}
if !strings.Contains(err.Error(), "--output-file is required") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestNewNormalizeConfigResolvesOutputSchemaDefaultAndEnv(t *testing.T) {
dir := t.TempDir()
input := writeTempFile(t, dir, "input.json")
output := filepath.Join(dir, "normalized.json")
t.Setenv(OutputSchemaEnv, "")
cfg, err := NewNormalizeConfig(NormalizeOptions{
InputFile: input,
OutputFile: output,
OutputModules: DefaultOutputModules,
})
if err != nil {
t.Fatalf("config failed: %v", err)
}
if cfg.OutputSchema != DefaultOutputSchema {
t.Fatalf("output schema = %q, want %q", cfg.OutputSchema, DefaultOutputSchema)
}
t.Setenv(OutputSchemaEnv, OutputSchemaMinimal)
cfg, err = NewNormalizeConfig(NormalizeOptions{
InputFile: input,
OutputFile: output,
OutputModules: DefaultOutputModules,
})
if err != nil {
t.Fatalf("config failed: %v", err)
}
if cfg.OutputSchema != OutputSchemaMinimal {
t.Fatalf("output schema = %q, want %q", cfg.OutputSchema, OutputSchemaMinimal)
}
}
func TestNewNormalizeConfigRejectsInvalidOutputSchema(t *testing.T) {
dir := t.TempDir()
input := writeTempFile(t, dir, "input.json")
output := filepath.Join(dir, "normalized.json")
_, err := NewNormalizeConfig(NormalizeOptions{
InputFile: input,
OutputFile: output,
OutputSchema: "compact",
OutputModules: DefaultOutputModules,
})
if err == nil {
t.Fatal("expected output schema error")
}
if !strings.Contains(err.Error(), "--output-schema must be one of") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestNewNormalizeConfigRejectsUnknownOutputModule(t *testing.T) {
dir := t.TempDir()
input := writeTempFile(t, dir, "input.json")
output := filepath.Join(dir, "normalized.json")
_, err := NewNormalizeConfig(NormalizeOptions{
InputFile: input,
OutputFile: output,
OutputModules: "json,yaml",
})
if err == nil {
t.Fatal("expected output module error")
}
if !strings.Contains(err.Error(), "unknown output module") {
t.Fatalf("unexpected error: %v", err)
}
}
func assertPositiveFloatEnvValidation(t *testing.T, envName string) { func assertPositiveFloatEnvValidation(t *testing.T, envName string) {
t.Helper() t.Helper()

220
internal/normalize/build.go Normal file
View File

@@ -0,0 +1,220 @@
package normalize
import (
"fmt"
"path/filepath"
"sort"
"strings"
"gitea.maximumdirect.net/eric/seriatim/internal/artifact"
"gitea.maximumdirect.net/eric/seriatim/internal/buildinfo"
"gitea.maximumdirect.net/eric/seriatim/internal/config"
"gitea.maximumdirect.net/eric/seriatim/schema"
)
// BuildResult contains normalize output plus deterministic transformation diagnostics.
type BuildResult struct {
Output any
OutputSegmentCount int
SortingChanged bool
IDsReassigned bool
SegmentsWithCategories int
}
// Build converts parsed normalize input into a selected seriatim output schema.
func Build(parsed ParsedTranscript, cfg config.NormalizeConfig) (BuildResult, error) {
ordered := sortedSegments(parsed.Segments)
sortingChanged := didSortingChangeOrder(ordered)
idsReassigned := didReassignIDs(ordered)
segmentsWithCategories := countSegmentsWithCategories(ordered)
switch cfg.OutputSchema {
case config.OutputSchemaMinimal:
output := buildMinimal(ordered)
if err := schema.ValidateMinimalTranscript(output); err != nil {
return BuildResult{}, fmt.Errorf("validate normalize output: %w", err)
}
return BuildResult{
Output: output,
OutputSegmentCount: len(ordered),
SortingChanged: sortingChanged,
IDsReassigned: idsReassigned,
SegmentsWithCategories: segmentsWithCategories,
}, nil
case config.OutputSchemaIntermediate:
output := buildIntermediate(ordered)
if err := schema.ValidateIntermediateTranscript(output); err != nil {
return BuildResult{}, fmt.Errorf("validate normalize output: %w", err)
}
return BuildResult{
Output: output,
OutputSegmentCount: len(ordered),
SortingChanged: sortingChanged,
IDsReassigned: idsReassigned,
SegmentsWithCategories: segmentsWithCategories,
}, nil
case config.OutputSchemaFull:
output := buildFull(ordered, cfg)
if err := schema.ValidateTranscript(output); err != nil {
return BuildResult{}, fmt.Errorf("validate normalize output: %w", err)
}
return BuildResult{
Output: output,
OutputSegmentCount: len(ordered),
SortingChanged: sortingChanged,
IDsReassigned: idsReassigned,
SegmentsWithCategories: segmentsWithCategories,
}, nil
default:
return BuildResult{}, fmt.Errorf("unsupported output schema %q", cfg.OutputSchema)
}
}
func sortedSegments(input []InputSegment) []InputSegment {
ordered := make([]InputSegment, len(input))
copy(ordered, input)
sort.SliceStable(ordered, func(i, j int) bool {
left := ordered[i]
right := ordered[j]
if left.Start != right.Start {
return left.Start < right.Start
}
if left.End != right.End {
return left.End < right.End
}
if left.InputIndex != right.InputIndex {
return left.InputIndex < right.InputIndex
}
return left.Speaker < right.Speaker
})
return ordered
}
func buildMinimal(segments []InputSegment) schema.MinimalTranscript {
outputSegments := make([]schema.MinimalSegment, len(segments))
for index, segment := range segments {
outputSegments[index] = schema.MinimalSegment{
ID: index + 1,
Start: segment.Start,
End: segment.End,
Speaker: segment.Speaker,
Text: segment.Text,
}
}
return schema.MinimalTranscript{
Metadata: schema.MinimalMetadata{
Application: artifact.ApplicationName,
Version: buildinfo.Version,
OutputSchema: config.OutputSchemaMinimal,
},
Segments: outputSegments,
}
}
func buildIntermediate(segments []InputSegment) schema.IntermediateTranscript {
outputSegments := make([]schema.IntermediateSegment, len(segments))
for index, segment := range segments {
outputSegments[index] = schema.IntermediateSegment{
ID: index + 1,
Start: segment.Start,
End: segment.End,
Speaker: segment.Speaker,
Text: segment.Text,
Categories: append([]string(nil), segment.Categories...),
}
}
return schema.IntermediateTranscript{
Metadata: schema.IntermediateMetadata{
Application: artifact.ApplicationName,
Version: buildinfo.Version,
OutputSchema: config.OutputSchemaIntermediate,
},
Segments: outputSegments,
}
}
func buildFull(segments []InputSegment, cfg config.NormalizeConfig) schema.Transcript {
defaultSource := filepath.Base(cfg.InputFile)
outputSegments := make([]schema.Segment, len(segments))
for index, segment := range segments {
source := strings.TrimSpace(segment.Source)
if source == "" {
source = defaultSource
}
sourceSegmentIndex := copyIntPtr(segment.SourceSegmentIndex)
if sourceSegmentIndex == nil {
fallback := segment.InputIndex
sourceSegmentIndex = &fallback
}
outputSegments[index] = schema.Segment{
ID: index + 1,
Source: source,
SourceSegmentIndex: sourceSegmentIndex,
SourceRef: segment.SourceRef,
DerivedFrom: append([]string(nil), segment.DerivedFrom...),
Speaker: segment.Speaker,
Start: segment.Start,
End: segment.End,
Text: segment.Text,
Categories: append([]string(nil), segment.Categories...),
}
}
return schema.Transcript{
Metadata: schema.Metadata{
Application: artifact.ApplicationName,
Version: buildinfo.Version,
InputReader: "normalize-input",
InputFiles: []string{cfg.InputFile},
PreprocessingModules: []string{},
PostprocessingModules: []string{},
OutputModules: append([]string(nil), cfg.OutputModules...),
},
Segments: outputSegments,
OverlapGroups: []schema.OverlapGroup{},
}
}
func copyIntPtr(value *int) *int {
if value == nil {
return nil
}
copied := *value
return &copied
}
func didSortingChangeOrder(segments []InputSegment) bool {
for index, segment := range segments {
if segment.InputIndex != index {
return true
}
}
return false
}
func didReassignIDs(segments []InputSegment) bool {
if len(segments) == 0 {
return false
}
for index, segment := range segments {
newID := index + 1
if segment.OriginalID == nil || *segment.OriginalID != newID {
return true
}
}
return false
}
func countSegmentsWithCategories(segments []InputSegment) int {
count := 0
for _, segment := range segments {
if len(segment.Categories) > 0 {
count++
}
}
return count
}

View File

@@ -0,0 +1,135 @@
package normalize
import (
"context"
"encoding/json"
"fmt"
"os"
"strings"
"gitea.maximumdirect.net/eric/seriatim/internal/artifact"
"gitea.maximumdirect.net/eric/seriatim/internal/buildinfo"
"gitea.maximumdirect.net/eric/seriatim/internal/config"
"gitea.maximumdirect.net/eric/seriatim/internal/report"
)
type normalizeAudit struct {
Command string `json:"command"`
InputFile string `json:"input_file"`
OutputFile string `json:"output_file"`
InputShape string `json:"input_shape"`
InputSegmentCount int `json:"input_segment_count"`
OutputSegmentCount int `json:"output_segment_count"`
OutputSchema string `json:"output_schema"`
OutputModules []string `json:"output_modules"`
IDsReassigned bool `json:"ids_reassigned"`
SortingChangedInput bool `json:"sorting_changed_input_order"`
SegmentsWithCategories int `json:"segments_with_categories"`
TimingFieldsRepaired int `json:"timing_fields_repaired"`
TimingOrderSwapped int `json:"timing_order_swapped"`
SpeakerFilled int `json:"speaker_filled"`
SegmentsDroppedText int `json:"segments_dropped_text"`
}
// Run executes artifact-level normalization.
func Run(ctx context.Context, cfg config.NormalizeConfig) error {
if err := ctx.Err(); err != nil {
return err
}
parsed, err := ParseFile(cfg.InputFile)
if err != nil {
return err
}
built, err := Build(parsed, cfg)
if err != nil {
return err
}
if err := writeOutputJSON(cfg.OutputFile, built.Output); err != nil {
return err
}
if cfg.ReportFile != "" {
audit := normalizeAudit{
Command: "normalize",
InputFile: cfg.InputFile,
OutputFile: cfg.OutputFile,
InputShape: string(parsed.Shape),
InputSegmentCount: parsed.InputSegmentCount,
OutputSegmentCount: built.OutputSegmentCount,
OutputSchema: cfg.OutputSchema,
OutputModules: append([]string(nil), cfg.OutputModules...),
IDsReassigned: built.IDsReassigned,
SortingChangedInput: built.SortingChanged,
SegmentsWithCategories: built.SegmentsWithCategories,
TimingFieldsRepaired: parsed.Stats.TimingFieldsRepaired,
TimingOrderSwapped: parsed.Stats.TimingOrderSwapped,
SpeakerFilled: parsed.Stats.SpeakerFilled,
SegmentsDroppedText: parsed.Stats.SegmentsDroppedText,
}
auditJSON, err := json.Marshal(audit)
if err != nil {
return fmt.Errorf("marshal normalize audit: %w", err)
}
events := []report.Event{
report.Info("normalize", "normalize", "started normalize command"),
report.Info("normalize", "normalize", fmt.Sprintf("input file: %s", cfg.InputFile)),
report.Info("normalize", "normalize", fmt.Sprintf("detected input shape: %s", parsed.Shape)),
report.Info("normalize", "normalize", fmt.Sprintf("input segment count: %d", parsed.InputSegmentCount)),
report.Info("normalize", "normalize", fmt.Sprintf("selected output schema: %s", cfg.OutputSchema)),
report.Info("normalize", "normalize", fmt.Sprintf("selected output modules: %s", strings.Join(cfg.OutputModules, ","))),
report.Info("normalize", "normalize", fmt.Sprintf("output file: %s", cfg.OutputFile)),
report.Info("normalize", "normalize", fmt.Sprintf("ids reassigned: %t", built.IDsReassigned)),
report.Info("normalize", "normalize", fmt.Sprintf("sorting changed input order: %t", built.SortingChanged)),
report.Info("normalize", "normalize", fmt.Sprintf("segments with categories: %d", built.SegmentsWithCategories)),
report.Info("normalize", "normalize", fmt.Sprintf("timing fields repaired: %d", parsed.Stats.TimingFieldsRepaired)),
report.Info("normalize", "normalize", fmt.Sprintf("timing order swapped: %d", parsed.Stats.TimingOrderSwapped)),
report.Info("normalize", "normalize", fmt.Sprintf("speaker placeholders added: %d", parsed.Stats.SpeakerFilled)),
report.Info("normalize", "normalize", fmt.Sprintf("segments dropped for empty text: %d", parsed.Stats.SegmentsDroppedText)),
report.Info("normalize", "normalize-audit", string(auditJSON)),
}
if parsed.InputSegmentCount == 0 {
events = append(events, report.Warning("normalize", "normalize", "input transcript contains zero segments"))
}
events = append(events,
report.Info("normalize", "validate-output", fmt.Sprintf("validated %d output segment(s)", built.OutputSegmentCount)),
report.Info("output", "json", "wrote transcript JSON"),
)
rpt := report.Report{
Metadata: report.Metadata{
Application: artifact.ApplicationName,
Version: buildinfo.Version,
InputReader: "normalize-input",
InputFiles: []string{cfg.InputFile},
PreprocessingModules: []string{},
PostprocessingModules: []string{},
OutputModules: append([]string(nil), cfg.OutputModules...),
},
Events: events,
}
if err := report.WriteJSON(cfg.ReportFile, rpt); err != nil {
return fmt.Errorf("write --report-file %q: %w", cfg.ReportFile, err)
}
}
return nil
}
func writeOutputJSON(path string, value any) error {
file, err := os.Create(path)
if err != nil {
return err
}
defer file.Close()
encoder := json.NewEncoder(file)
encoder.SetIndent("", " ")
if err := encoder.Encode(value); err != nil {
return fmt.Errorf("encode normalize output JSON: %w", err)
}
return nil
}

307
internal/normalize/parse.go Normal file
View File

@@ -0,0 +1,307 @@
package normalize
import (
"bytes"
"encoding/json"
"fmt"
"io"
"os"
"strings"
)
// InputShape identifies which top-level input shape was parsed.
type InputShape string
const (
ShapeObjectWithSegments InputShape = "object_with_segments"
ShapeBareSegmentsArray InputShape = "bare_segments_array"
)
// ParsedTranscript is the validated normalize input model.
type ParsedTranscript struct {
Shape InputShape
InputSegmentCount int
Stats NormalizeStats
Segments []InputSegment
}
// InputSegment is a validated segment from normalize input.
type InputSegment struct {
InputIndex int
OriginalID *int
StartPresent bool
EndPresent bool
SpeakerPresent bool
TextPresent bool
Start float64
End float64
Speaker string
Text string
Categories []string
Source string
SourceSegmentIndex *int
SourceRef string
DerivedFrom []string
OverlapGroupID *int
}
// NormalizeStats captures deterministic repair/drop outcomes from normalize input processing.
type NormalizeStats struct {
TimingFieldsRepaired int
TimingOrderSwapped int
SpeakerFilled int
SegmentsDroppedText int
}
type inputSegmentPayload struct {
ID *int `json:"id"`
Start *float64 `json:"start"`
End *float64 `json:"end"`
Speaker *string `json:"speaker"`
Text *string `json:"text"`
Categories []string `json:"categories"`
Source string `json:"source"`
SourceSegmentIndex *int `json:"source_segment_index"`
SourceRef string `json:"source_ref"`
DerivedFrom []string `json:"derived_from"`
OverlapGroupID *int `json:"overlap_group_id"`
}
// ParseFile parses normalize input JSON from file path.
func ParseFile(path string) (ParsedTranscript, error) {
file, err := os.Open(path)
if err != nil {
return ParsedTranscript{}, err
}
defer file.Close()
return ParseReader(file)
}
// ParseReader parses normalize input JSON from a reader.
func ParseReader(reader io.Reader) (ParsedTranscript, error) {
var raw json.RawMessage
decoder := json.NewDecoder(reader)
decoder.UseNumber()
if err := decoder.Decode(&raw); err != nil {
return ParsedTranscript{}, fmt.Errorf("decode normalize input JSON: %w", err)
}
if err := ensureSingleValue(decoder); err != nil {
return ParsedTranscript{}, err
}
trimmed := bytes.TrimSpace(raw)
if len(trimmed) == 0 {
return ParsedTranscript{}, fmt.Errorf("normalize input is empty")
}
switch trimmed[0] {
case '{':
return parseObjectShape(trimmed)
case '[':
segments, stats, inputCount, err := parseSegmentsArray(trimmed)
if err != nil {
return ParsedTranscript{}, err
}
return ParsedTranscript{
Shape: ShapeBareSegmentsArray,
InputSegmentCount: inputCount,
Stats: stats,
Segments: segments,
}, nil
default:
return ParsedTranscript{}, fmt.Errorf("normalize input must be a top-level object with \"segments\" or a top-level segment array")
}
}
func ensureSingleValue(decoder *json.Decoder) error {
var extra json.RawMessage
err := decoder.Decode(&extra)
if err == io.EOF {
return nil
}
if err == nil {
return fmt.Errorf("normalize input must contain exactly one top-level JSON value")
}
return fmt.Errorf("decode normalize input JSON: %w", err)
}
func parseObjectShape(raw []byte) (ParsedTranscript, error) {
var object map[string]json.RawMessage
if err := json.Unmarshal(raw, &object); err != nil {
return ParsedTranscript{}, fmt.Errorf("decode normalize object input: %w", err)
}
segmentsRaw, exists := object["segments"]
if !exists {
return ParsedTranscript{}, fmt.Errorf("normalize object input must contain a \"segments\" field")
}
segments, stats, inputCount, err := parseSegmentsArray(segmentsRaw)
if err != nil {
return ParsedTranscript{}, err
}
return ParsedTranscript{
Shape: ShapeObjectWithSegments,
InputSegmentCount: inputCount,
Stats: stats,
Segments: segments,
}, nil
}
func parseSegmentsArray(raw []byte) ([]InputSegment, NormalizeStats, int, error) {
var segmentValues []json.RawMessage
if err := json.Unmarshal(raw, &segmentValues); err != nil {
return nil, NormalizeStats{}, 0, fmt.Errorf("normalize input \"segments\" must be an array")
}
segments := make([]InputSegment, len(segmentValues))
for index, segmentRaw := range segmentValues {
segment, err := decodeSegment(index, segmentRaw)
if err != nil {
return nil, NormalizeStats{}, 0, err
}
segments[index] = segment
}
normalized, stats, err := normalizeSegments(segments)
return normalized, stats, len(segmentValues), err
}
func decodeSegment(index int, raw []byte) (InputSegment, error) {
var payload inputSegmentPayload
if err := json.Unmarshal(raw, &payload); err != nil {
return InputSegment{}, fmt.Errorf("segment %d: invalid segment object: %w", index, err)
}
var start float64
if payload.Start != nil {
start = *payload.Start
}
var end float64
if payload.End != nil {
end = *payload.End
}
speaker := ""
if payload.Speaker != nil {
speaker = strings.TrimSpace(*payload.Speaker)
}
text := ""
if payload.Text != nil {
text = *payload.Text
}
return InputSegment{
InputIndex: index,
OriginalID: payload.ID,
StartPresent: payload.Start != nil,
EndPresent: payload.End != nil,
SpeakerPresent: payload.Speaker != nil,
TextPresent: payload.Text != nil,
Start: start,
End: end,
Speaker: speaker,
Text: text,
Categories: append([]string(nil), payload.Categories...),
Source: payload.Source,
SourceSegmentIndex: payload.SourceSegmentIndex,
SourceRef: payload.SourceRef,
DerivedFrom: append([]string(nil), payload.DerivedFrom...),
OverlapGroupID: payload.OverlapGroupID,
}, nil
}
func normalizeSegments(segments []InputSegment) ([]InputSegment, NormalizeStats, error) {
stats := NormalizeStats{}
filtered := make([]InputSegment, 0, len(segments))
for _, segment := range segments {
if strings.TrimSpace(segment.Text) == "" {
stats.SegmentsDroppedText++
continue
}
filtered = append(filtered, segment)
}
if len(filtered) == 0 {
return filtered, stats, nil
}
hasStart := make([]bool, len(filtered))
hasEnd := make([]bool, len(filtered))
for index, segment := range filtered {
hasStart[index] = segment.StartPresent
hasEnd[index] = segment.EndPresent
}
for index := range filtered {
if !hasStart[index] && !hasEnd[index] {
midpoint := inferTimestampFromNeighbors(filtered, hasStart, hasEnd, index)
filtered[index].Start = midpoint
filtered[index].End = midpoint
stats.TimingFieldsRepaired += 2
hasStart[index] = true
hasEnd[index] = true
continue
}
if hasStart[index] && !hasEnd[index] {
filtered[index].End = filtered[index].Start
stats.TimingFieldsRepaired++
hasEnd[index] = true
}
if !hasStart[index] && hasEnd[index] {
filtered[index].Start = filtered[index].End
stats.TimingFieldsRepaired++
hasStart[index] = true
}
}
for index := range filtered {
if filtered[index].Start < 0 {
return nil, NormalizeStats{}, fmt.Errorf("segment %d has start %v; start must be >= 0", filtered[index].InputIndex, filtered[index].Start)
}
if filtered[index].End < filtered[index].Start {
filtered[index].Start, filtered[index].End = filtered[index].End, filtered[index].Start
stats.TimingOrderSwapped++
}
if strings.TrimSpace(filtered[index].Speaker) == "" {
filtered[index].Speaker = "Unknown_Speaker"
stats.SpeakerFilled++
}
}
return filtered, stats, nil
}
func inferTimestampFromNeighbors(segments []InputSegment, hasStart []bool, hasEnd []bool, index int) float64 {
prevEnd, hasPrev := nearestPreviousEnd(segments, hasEnd, index)
nextStart, hasNext := nearestNextStart(segments, hasStart, index)
if hasPrev && hasNext {
return (prevEnd + nextStart) / 2
}
if hasPrev {
return prevEnd
}
if hasNext {
return nextStart
}
return 0
}
func nearestPreviousEnd(segments []InputSegment, hasEnd []bool, index int) (float64, bool) {
for i := index - 1; i >= 0; i-- {
if hasEnd[i] {
return segments[i].End, true
}
}
return 0, false
}
func nearestNextStart(segments []InputSegment, hasStart []bool, index int) (float64, bool) {
for i := index + 1; i < len(segments); i++ {
if hasStart[i] {
return segments[i].Start, true
}
}
return 0, false
}

View File

@@ -0,0 +1,279 @@
package normalize
import (
"strings"
"testing"
)
func TestParseReaderObjectWithSegmentsParses(t *testing.T) {
input := `{
"segments": [
{"start": 1.0, "end": 2.0, "speaker": " Alice ", "text": "hello", "id": 100}
]
}`
parsed, err := ParseReader(strings.NewReader(input))
if err != nil {
t.Fatalf("parse failed: %v", err)
}
if parsed.Shape != ShapeObjectWithSegments {
t.Fatalf("shape = %q, want %q", parsed.Shape, ShapeObjectWithSegments)
}
if len(parsed.Segments) != 1 {
t.Fatalf("segment count = %d, want 1", len(parsed.Segments))
}
segment := parsed.Segments[0]
if segment.Speaker != "Alice" {
t.Fatalf("speaker = %q, want %q", segment.Speaker, "Alice")
}
if segment.OriginalID == nil || *segment.OriginalID != 100 {
t.Fatalf("original id = %v, want 100", segment.OriginalID)
}
}
func TestParseReaderBareSegmentArrayParses(t *testing.T) {
input := `[
{"start": 1.0, "end": 2.0, "speaker": "Alice", "text": "hello"},
{"start": 3.0, "end": 4.0, "speaker": "Bob", "text": "world"}
]`
parsed, err := ParseReader(strings.NewReader(input))
if err != nil {
t.Fatalf("parse failed: %v", err)
}
if parsed.Shape != ShapeBareSegmentsArray {
t.Fatalf("shape = %q, want %q", parsed.Shape, ShapeBareSegmentsArray)
}
if len(parsed.Segments) != 2 {
t.Fatalf("segment count = %d, want 2", len(parsed.Segments))
}
}
func TestParseReaderInvalidJSONFails(t *testing.T) {
_, err := ParseReader(strings.NewReader(`{"segments":`))
if err == nil {
t.Fatal("expected parse error")
}
if !strings.Contains(err.Error(), "decode normalize input JSON") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestParseReaderObjectMissingSegmentsFails(t *testing.T) {
_, err := ParseReader(strings.NewReader(`{"items":[]}`))
if err == nil {
t.Fatal("expected missing segments error")
}
if !strings.Contains(err.Error(), "must contain a \"segments\" field") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestParseReaderSegmentsNotArrayFails(t *testing.T) {
_, err := ParseReader(strings.NewReader(`{"segments": {}}`))
if err == nil {
t.Fatal("expected segments not array error")
}
if !strings.Contains(err.Error(), "\"segments\" must be an array") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestParseReaderTopLevelScalarShapesFail(t *testing.T) {
tests := []string{`"text"`, `42`, `null`, `true`}
for _, input := range tests {
_, err := ParseReader(strings.NewReader(input))
if err == nil {
t.Fatalf("expected top-level shape error for %s", input)
}
if !strings.Contains(err.Error(), "top-level object") {
t.Fatalf("unexpected error for %s: %v", input, err)
}
}
}
func TestParseReaderMissingStartUsesEndValue(t *testing.T) {
parsed, err := ParseReader(strings.NewReader(`[{"end":2,"speaker":"A","text":"t"}]`))
if err != nil {
t.Fatalf("parse failed: %v", err)
}
if len(parsed.Segments) != 1 {
t.Fatalf("segment count = %d, want 1", len(parsed.Segments))
}
if parsed.Segments[0].Start != 2 || parsed.Segments[0].End != 2 {
t.Fatalf("segment timing = %v..%v, want 2..2", parsed.Segments[0].Start, parsed.Segments[0].End)
}
}
func TestParseReaderMissingEndUsesStartValue(t *testing.T) {
parsed, err := ParseReader(strings.NewReader(`[{"start":1,"speaker":"A","text":"t"}]`))
if err != nil {
t.Fatalf("parse failed: %v", err)
}
if len(parsed.Segments) != 1 {
t.Fatalf("segment count = %d, want 1", len(parsed.Segments))
}
if parsed.Segments[0].Start != 1 || parsed.Segments[0].End != 1 {
t.Fatalf("segment timing = %v..%v, want 1..1", parsed.Segments[0].Start, parsed.Segments[0].End)
}
}
func TestParseReaderMissingSpeakerUsesUnknownPlaceholder(t *testing.T) {
parsed, err := ParseReader(strings.NewReader(`[{"start":1,"end":2,"text":"t"}]`))
if err != nil {
t.Fatalf("parse failed: %v", err)
}
if parsed.Segments[0].Speaker != "Unknown_Speaker" {
t.Fatalf("speaker = %q, want Unknown_Speaker", parsed.Segments[0].Speaker)
}
}
func TestParseReaderEmptySpeakerUsesUnknownPlaceholder(t *testing.T) {
parsed, err := ParseReader(strings.NewReader(`[{"start":1,"end":2,"speaker":" ","text":"t"}]`))
if err != nil {
t.Fatalf("parse failed: %v", err)
}
if parsed.Segments[0].Speaker != "Unknown_Speaker" {
t.Fatalf("speaker = %q, want Unknown_Speaker", parsed.Segments[0].Speaker)
}
}
func TestParseReaderMissingTextDropsSegment(t *testing.T) {
parsed, err := ParseReader(strings.NewReader(`[{"start":1,"end":2,"speaker":"A"}]`))
if err != nil {
t.Fatalf("parse failed: %v", err)
}
if len(parsed.Segments) != 0 {
t.Fatalf("segment count = %d, want 0", len(parsed.Segments))
}
}
func TestParseReaderEndBeforeStartSwapsValues(t *testing.T) {
parsed, err := ParseReader(strings.NewReader(`[{"start":3,"end":2,"speaker":"A","text":"t"}]`))
if err != nil {
t.Fatalf("parse failed: %v", err)
}
if parsed.Segments[0].Start != 2 || parsed.Segments[0].End != 3 {
t.Fatalf("segment timing = %v..%v, want 2..3", parsed.Segments[0].Start, parsed.Segments[0].End)
}
}
func TestParseReaderNegativeStartFails(t *testing.T) {
_, err := ParseReader(strings.NewReader(`[{"start":-1,"end":2,"speaker":"A","text":"t"}]`))
assertContains(t, err, "start must be >= 0")
}
func TestParseReaderEmptySegmentsArrayAccepted(t *testing.T) {
parsed, err := ParseReader(strings.NewReader(`{"segments":[]}`))
if err != nil {
t.Fatalf("parse failed: %v", err)
}
if len(parsed.Segments) != 0 {
t.Fatalf("segment count = %d, want 0", len(parsed.Segments))
}
}
func TestParseReaderCategoriesPreservedWhenValid(t *testing.T) {
parsed, err := ParseReader(strings.NewReader(`[{"start":1,"end":2,"speaker":"A","text":"t","categories":["filler","backchannel"]}]`))
if err != nil {
t.Fatalf("parse failed: %v", err)
}
if len(parsed.Segments) != 1 {
t.Fatalf("segment count = %d, want 1", len(parsed.Segments))
}
if len(parsed.Segments[0].Categories) != 2 {
t.Fatalf("categories length = %d, want 2", len(parsed.Segments[0].Categories))
}
if parsed.Segments[0].Categories[0] != "filler" || parsed.Segments[0].Categories[1] != "backchannel" {
t.Fatalf("categories = %v", parsed.Segments[0].Categories)
}
}
func TestParseReaderMissingBothTimesMiddleUsesNeighborMidpoint(t *testing.T) {
parsed, err := ParseReader(strings.NewReader(`[
{"start":1,"end":2,"speaker":"A","text":"left"},
{"speaker":"B","text":"middle"},
{"start":6,"end":7,"speaker":"C","text":"right"}
]`))
if err != nil {
t.Fatalf("parse failed: %v", err)
}
if parsed.Segments[1].Start != 4 || parsed.Segments[1].End != 4 {
t.Fatalf("middle timing = %v..%v, want 4..4", parsed.Segments[1].Start, parsed.Segments[1].End)
}
}
func TestParseReaderMissingBothTimesEdgeFallbacks(t *testing.T) {
parsedFirst, err := ParseReader(strings.NewReader(`[
{"speaker":"A","text":"first"},
{"start":5,"end":6,"speaker":"B","text":"second"}
]`))
if err != nil {
t.Fatalf("parse first failed: %v", err)
}
if parsedFirst.Segments[0].Start != 5 || parsedFirst.Segments[0].End != 5 {
t.Fatalf("first timing = %v..%v, want 5..5", parsedFirst.Segments[0].Start, parsedFirst.Segments[0].End)
}
parsedLast, err := ParseReader(strings.NewReader(`[
{"start":1,"end":2,"speaker":"A","text":"first"},
{"speaker":"B","text":"last"}
]`))
if err != nil {
t.Fatalf("parse last failed: %v", err)
}
if parsedLast.Segments[1].Start != 2 || parsedLast.Segments[1].End != 2 {
t.Fatalf("last timing = %v..%v, want 2..2", parsedLast.Segments[1].Start, parsedLast.Segments[1].End)
}
}
func TestParseReaderMissingBothTimesSingleSegmentUsesZero(t *testing.T) {
parsed, err := ParseReader(strings.NewReader(`[{"speaker":"A","text":"only"}]`))
if err != nil {
t.Fatalf("parse failed: %v", err)
}
if parsed.Segments[0].Start != 0 || parsed.Segments[0].End != 0 {
t.Fatalf("timing = %v..%v, want 0..0", parsed.Segments[0].Start, parsed.Segments[0].End)
}
}
func TestParseReaderEmptyOrWhitespaceTextDropped(t *testing.T) {
parsed, err := ParseReader(strings.NewReader(`[
{"start":1,"end":2,"speaker":"A","text":"ok"},
{"start":2,"end":3,"speaker":"A","text":""},
{"start":3,"end":4,"speaker":"A","text":" "}
]`))
if err != nil {
t.Fatalf("parse failed: %v", err)
}
if len(parsed.Segments) != 1 {
t.Fatalf("segment count = %d, want 1", len(parsed.Segments))
}
}
func TestParseReaderOriginalInputIndexPreserved(t *testing.T) {
input := `[
{"start":1,"end":2,"speaker":"A","text":"one"},
{"start":2,"end":3,"speaker":"B","text":"two"},
{"start":3,"end":4,"speaker":"C","text":"three"}
]`
parsed, err := ParseReader(strings.NewReader(input))
if err != nil {
t.Fatalf("parse failed: %v", err)
}
for index, segment := range parsed.Segments {
if segment.InputIndex != index {
t.Fatalf("segment %d input index = %d, want %d", index, segment.InputIndex, index)
}
}
}
func assertContains(t *testing.T, err error, fragment string) {
t.Helper()
if err == nil {
t.Fatalf("expected error containing %q", fragment)
}
if !strings.Contains(err.Error(), fragment) {
t.Fatalf("error = %q, want substring %q", err.Error(), fragment)
}
}

367
internal/trim/apply.go Normal file
View File

@@ -0,0 +1,367 @@
package trim
import (
"fmt"
"gitea.maximumdirect.net/eric/seriatim/internal/model"
"gitea.maximumdirect.net/eric/seriatim/internal/overlap"
"gitea.maximumdirect.net/eric/seriatim/schema"
)
// Mode controls how selector IDs are applied.
type Mode string
const (
ModeKeep Mode = "keep"
ModeRemove Mode = "remove"
)
// Options configures transcript trimming.
type Options struct {
Mode Mode
Selector Selector
AllowEmpty bool
}
// Result contains trimming output and ID mapping metadata.
type Result struct {
Transcript schema.Transcript
OldToNewID map[int]int
RemovedIDs []int
}
// IntermediateResult contains trimming output for intermediate schema artifacts.
type IntermediateResult struct {
Transcript schema.IntermediateTranscript
OldToNewID map[int]int
RemovedIDs []int
}
// MinimalResult contains trimming output for minimal schema artifacts.
type MinimalResult struct {
Transcript schema.MinimalTranscript
OldToNewID map[int]int
RemovedIDs []int
}
// Apply trims a full seriatim output transcript by segment ID.
func Apply(input schema.Transcript, opts Options) (Result, error) {
if err := validateMode(opts.Mode); err != nil {
return Result{}, err
}
selected := opts.Selector.IDs()
if len(selected) == 0 {
return Result{}, fmt.Errorf("selector cannot be empty")
}
inputIDs := make([]int, len(input.Segments))
for index, segment := range input.Segments {
inputIDs[index] = segment.ID
}
idIndex, err := validateInputIDs(inputIDs)
if err != nil {
return Result{}, err
}
if err := validateSelectedIDsExist(selected, idIndex); err != nil {
return Result{}, err
}
kept := make([]schema.Segment, 0, len(input.Segments))
removed := make([]int, 0, len(input.Segments))
oldToNew := make(map[int]int, len(input.Segments))
for _, segment := range input.Segments {
keep := opts.Mode == ModeKeep && opts.Selector.Contains(segment.ID)
if opts.Mode == ModeRemove {
keep = !opts.Selector.Contains(segment.ID)
}
if !keep {
removed = append(removed, segment.ID)
continue
}
rewritten := copySegment(segment)
rewritten.ID = len(kept) + 1
rewritten.OverlapGroupID = 0
kept = append(kept, rewritten)
oldToNew[segment.ID] = rewritten.ID
}
if len(kept) == 0 && !opts.AllowEmpty {
return Result{}, fmt.Errorf("trim operation produced an empty transcript; set AllowEmpty to true to permit this")
}
kept, groups := recomputeOverlapGroups(kept)
if groups == nil {
groups = make([]schema.OverlapGroup, 0)
}
out := copyTranscript(input)
out.Segments = kept
out.OverlapGroups = groups
return Result{
Transcript: out,
OldToNewID: oldToNew,
RemovedIDs: removed,
}, nil
}
// ApplyIntermediate trims an intermediate seriatim output transcript by
// segment ID.
func ApplyIntermediate(input schema.IntermediateTranscript, opts Options) (IntermediateResult, error) {
if err := validateMode(opts.Mode); err != nil {
return IntermediateResult{}, err
}
selected := opts.Selector.IDs()
if len(selected) == 0 {
return IntermediateResult{}, fmt.Errorf("selector cannot be empty")
}
inputIDs := make([]int, len(input.Segments))
for index, segment := range input.Segments {
inputIDs[index] = segment.ID
}
idIndex, err := validateInputIDs(inputIDs)
if err != nil {
return IntermediateResult{}, err
}
if err := validateSelectedIDsExist(selected, idIndex); err != nil {
return IntermediateResult{}, err
}
kept := make([]schema.IntermediateSegment, 0, len(input.Segments))
removed := make([]int, 0, len(input.Segments))
oldToNew := make(map[int]int, len(input.Segments))
for _, segment := range input.Segments {
keep := opts.Mode == ModeKeep && opts.Selector.Contains(segment.ID)
if opts.Mode == ModeRemove {
keep = !opts.Selector.Contains(segment.ID)
}
if !keep {
removed = append(removed, segment.ID)
continue
}
rewritten := schema.IntermediateSegment{
ID: len(kept) + 1,
Start: segment.Start,
End: segment.End,
Speaker: segment.Speaker,
Text: segment.Text,
Categories: append([]string(nil), segment.Categories...),
}
kept = append(kept, rewritten)
oldToNew[segment.ID] = rewritten.ID
}
if len(kept) == 0 && !opts.AllowEmpty {
return IntermediateResult{}, fmt.Errorf("trim operation produced an empty transcript; set AllowEmpty to true to permit this")
}
return IntermediateResult{
Transcript: schema.IntermediateTranscript{
Metadata: schema.IntermediateMetadata{
Application: input.Metadata.Application,
Version: input.Metadata.Version,
OutputSchema: input.Metadata.OutputSchema,
},
Segments: kept,
},
OldToNewID: oldToNew,
RemovedIDs: removed,
}, nil
}
// ApplyMinimal trims a minimal seriatim output transcript by segment ID.
func ApplyMinimal(input schema.MinimalTranscript, opts Options) (MinimalResult, error) {
if err := validateMode(opts.Mode); err != nil {
return MinimalResult{}, err
}
selected := opts.Selector.IDs()
if len(selected) == 0 {
return MinimalResult{}, fmt.Errorf("selector cannot be empty")
}
inputIDs := make([]int, len(input.Segments))
for index, segment := range input.Segments {
inputIDs[index] = segment.ID
}
idIndex, err := validateInputIDs(inputIDs)
if err != nil {
return MinimalResult{}, err
}
if err := validateSelectedIDsExist(selected, idIndex); err != nil {
return MinimalResult{}, err
}
kept := make([]schema.MinimalSegment, 0, len(input.Segments))
removed := make([]int, 0, len(input.Segments))
oldToNew := make(map[int]int, len(input.Segments))
for _, segment := range input.Segments {
keep := opts.Mode == ModeKeep && opts.Selector.Contains(segment.ID)
if opts.Mode == ModeRemove {
keep = !opts.Selector.Contains(segment.ID)
}
if !keep {
removed = append(removed, segment.ID)
continue
}
rewritten := schema.MinimalSegment{
ID: len(kept) + 1,
Start: segment.Start,
End: segment.End,
Speaker: segment.Speaker,
Text: segment.Text,
}
kept = append(kept, rewritten)
oldToNew[segment.ID] = rewritten.ID
}
if len(kept) == 0 && !opts.AllowEmpty {
return MinimalResult{}, fmt.Errorf("trim operation produced an empty transcript; set AllowEmpty to true to permit this")
}
return MinimalResult{
Transcript: schema.MinimalTranscript{
Metadata: schema.MinimalMetadata{
Application: input.Metadata.Application,
Version: input.Metadata.Version,
OutputSchema: input.Metadata.OutputSchema,
},
Segments: kept,
},
OldToNewID: oldToNew,
RemovedIDs: removed,
}, nil
}
func validateMode(mode Mode) error {
switch mode {
case ModeKeep, ModeRemove:
return nil
default:
return fmt.Errorf("invalid trim mode %q", mode)
}
}
func validateInputIDs(ids []int) (map[int]int, error) {
seen := make(map[int]int, len(ids))
for index, id := range ids {
if id <= 0 {
return nil, fmt.Errorf("input transcript has non-positive segment ID %d at index %d", id, index)
}
if firstIndex, exists := seen[id]; exists {
return nil, fmt.Errorf("input transcript has duplicate segment ID %d at indexes %d and %d", id, firstIndex, index)
}
seen[id] = index
}
for id := 1; id <= len(ids); id++ {
if _, exists := seen[id]; !exists {
return nil, fmt.Errorf("input transcript segment IDs must be sequential 1..%d; missing ID %d", len(ids), id)
}
}
return seen, nil
}
func validateSelectedIDsExist(selected []int, idIndex map[int]int) error {
for _, id := range selected {
if _, exists := idIndex[id]; !exists {
return fmt.Errorf("selected segment ID %d does not exist in input transcript", id)
}
}
return nil
}
func recomputeOverlapGroups(segments []schema.Segment) ([]schema.Segment, []schema.OverlapGroup) {
if len(segments) == 0 {
return segments, make([]schema.OverlapGroup, 0)
}
modelSegments := make([]model.Segment, len(segments))
for index, segment := range segments {
modelSegments[index] = model.Segment{
ID: segment.ID,
Source: segment.Source,
SourceSegmentIndex: copyIntPtr(segment.SourceSegmentIndex),
SourceRef: segment.SourceRef,
DerivedFrom: append([]string(nil), segment.DerivedFrom...),
Speaker: segment.Speaker,
Start: segment.Start,
End: segment.End,
Text: segment.Text,
Categories: append([]string(nil), segment.Categories...),
OverlapGroupID: segment.OverlapGroupID,
}
}
detected := overlap.Detect(model.MergedTranscript{
Segments: modelSegments,
})
rewrittenSegments := make([]schema.Segment, len(segments))
for index, segment := range segments {
rewritten := copySegment(segment)
rewritten.OverlapGroupID = detected.Segments[index].OverlapGroupID
rewrittenSegments[index] = rewritten
}
groups := make([]schema.OverlapGroup, len(detected.OverlapGroups))
for index, group := range detected.OverlapGroups {
groups[index] = schema.OverlapGroup{
ID: group.ID,
Start: group.Start,
End: group.End,
Segments: append([]string(nil), group.Segments...),
Speakers: append([]string(nil), group.Speakers...),
Class: group.Class,
Resolution: group.Resolution,
}
}
return rewrittenSegments, groups
}
func copyTranscript(input schema.Transcript) schema.Transcript {
return schema.Transcript{
Metadata: schema.Metadata{
Application: input.Metadata.Application,
Version: input.Metadata.Version,
InputReader: input.Metadata.InputReader,
InputFiles: append([]string(nil), input.Metadata.InputFiles...),
PreprocessingModules: append([]string(nil), input.Metadata.PreprocessingModules...),
PostprocessingModules: append([]string(nil), input.Metadata.PostprocessingModules...),
OutputModules: append([]string(nil), input.Metadata.OutputModules...),
},
Segments: append([]schema.Segment(nil), input.Segments...),
OverlapGroups: append([]schema.OverlapGroup(nil), input.OverlapGroups...),
}
}
func copySegment(input schema.Segment) schema.Segment {
return schema.Segment{
ID: input.ID,
Source: input.Source,
SourceSegmentIndex: copyIntPtr(input.SourceSegmentIndex),
SourceRef: input.SourceRef,
DerivedFrom: append([]string(nil), input.DerivedFrom...),
Speaker: input.Speaker,
Start: input.Start,
End: input.End,
Text: input.Text,
Categories: append([]string(nil), input.Categories...),
OverlapGroupID: input.OverlapGroupID,
}
}
func copyIntPtr(value *int) *int {
if value == nil {
return nil
}
copied := *value
return &copied
}

668
internal/trim/apply_test.go Normal file
View File

@@ -0,0 +1,668 @@
package trim
import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/seriatim/schema"
)
func TestApplyKeepModeRenumbersFromOne(t *testing.T) {
input := fullTranscriptFixture()
selector := mustParseSelector(t, "2,4")
result, err := Apply(input, Options{
Mode: ModeKeep,
Selector: selector,
})
if err != nil {
t.Fatalf("apply failed: %v", err)
}
if len(result.Transcript.Segments) != 2 {
t.Fatalf("segment count = %d, want 2", len(result.Transcript.Segments))
}
assertSegmentIDs(t, result.Transcript.Segments, []int{1, 2})
assertSegmentTexts(t, result.Transcript.Segments, []string{"beta", "delta"})
assertIntMap(t, result.OldToNewID, map[int]int{2: 1, 4: 2})
assertIntSlice(t, result.RemovedIDs, []int{1, 3})
}
func TestApplyRemoveModeRenumbersFromOne(t *testing.T) {
input := fullTranscriptFixture()
selector := mustParseSelector(t, "2,4")
result, err := Apply(input, Options{
Mode: ModeRemove,
Selector: selector,
})
if err != nil {
t.Fatalf("apply failed: %v", err)
}
assertSegmentIDs(t, result.Transcript.Segments, []int{1, 2})
assertSegmentTexts(t, result.Transcript.Segments, []string{"alpha", "gamma"})
assertIntMap(t, result.OldToNewID, map[int]int{1: 1, 3: 2})
assertIntSlice(t, result.RemovedIDs, []int{2, 4})
}
func TestApplySelectorOrderDoesNotChangeTranscriptOrder(t *testing.T) {
input := fullTranscriptFixture()
selector := mustParseSelector(t, "4,1,3")
result, err := Apply(input, Options{
Mode: ModeKeep,
Selector: selector,
})
if err != nil {
t.Fatalf("apply failed: %v", err)
}
assertSegmentIDs(t, result.Transcript.Segments, []int{1, 2, 3})
assertSegmentTexts(t, result.Transcript.Segments, []string{"alpha", "gamma", "delta"})
}
func TestApplyFailsWhenSelectedIDDoesNotExist(t *testing.T) {
input := fullTranscriptFixture()
selector := mustParseSelector(t, "2,99")
_, err := Apply(input, Options{
Mode: ModeKeep,
Selector: selector,
})
if err == nil {
t.Fatal("expected missing selected ID error")
}
if !strings.Contains(err.Error(), "does not exist") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestApplyFailsOnDuplicateInputIDs(t *testing.T) {
input := fullTranscriptFixture()
input.Segments[2].ID = 2
selector := mustParseSelector(t, "2")
_, err := Apply(input, Options{
Mode: ModeKeep,
Selector: selector,
})
if err == nil {
t.Fatal("expected duplicate input ID error")
}
if !strings.Contains(err.Error(), "duplicate segment ID") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestApplyFailsOnMissingOrNonSequentialInputIDs(t *testing.T) {
input := fullTranscriptFixture()
input.Segments[1].ID = 5
selector := mustParseSelector(t, "1")
_, err := Apply(input, Options{
Mode: ModeKeep,
Selector: selector,
})
if err == nil {
t.Fatal("expected non-sequential input ID error")
}
if !strings.Contains(err.Error(), "must be sequential") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestApplyFailsOnNonPositiveInputIDs(t *testing.T) {
input := fullTranscriptFixture()
input.Segments[0].ID = 0
selector := mustParseSelector(t, "1")
_, err := Apply(input, Options{
Mode: ModeKeep,
Selector: selector,
})
if err == nil {
t.Fatal("expected non-positive input ID error")
}
if !strings.Contains(err.Error(), "non-positive") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestApplyEmptyOutputFailsUnlessAllowEmpty(t *testing.T) {
input := fullTranscriptFixture()
selector := mustParseSelector(t, "1-4")
_, err := Apply(input, Options{
Mode: ModeRemove,
Selector: selector,
})
if err == nil {
t.Fatal("expected empty-output error")
}
if !strings.Contains(err.Error(), "empty transcript") {
t.Fatalf("unexpected error: %v", err)
}
allowed, err := Apply(input, Options{
Mode: ModeRemove,
Selector: selector,
AllowEmpty: true,
})
if err != nil {
t.Fatalf("apply with AllowEmpty failed: %v", err)
}
if len(allowed.Transcript.Segments) != 0 {
t.Fatalf("segment count = %d, want 0", len(allowed.Transcript.Segments))
}
assertIntMap(t, allowed.OldToNewID, map[int]int{})
assertIntSlice(t, allowed.RemovedIDs, []int{1, 2, 3, 4})
}
func TestApplyPreservesRetainedSegmentFieldsAndClearsOverlapIDs(t *testing.T) {
input := fullTranscriptFixture()
selector := mustParseSelector(t, "2")
result, err := Apply(input, Options{
Mode: ModeKeep,
Selector: selector,
})
if err != nil {
t.Fatalf("apply failed: %v", err)
}
if len(result.Transcript.Segments) != 1 {
t.Fatalf("segment count = %d, want 1", len(result.Transcript.Segments))
}
segment := result.Transcript.Segments[0]
if segment.ID != 1 {
t.Fatalf("segment ID = %d, want 1", segment.ID)
}
if segment.Source != "b.json" {
t.Fatalf("source = %q, want %q", segment.Source, "b.json")
}
if segment.SourceSegmentIndex == nil || *segment.SourceSegmentIndex != 20 {
t.Fatalf("source_segment_index = %v, want 20", segment.SourceSegmentIndex)
}
if segment.SourceRef != "b.json#20" {
t.Fatalf("source_ref = %q, want %q", segment.SourceRef, "b.json#20")
}
if !equalStringSlices(segment.DerivedFrom, []string{"b.json#19", "b.json#20"}) {
t.Fatalf("derived_from = %v, want %v", segment.DerivedFrom, []string{"b.json#19", "b.json#20"})
}
if !equalStringSlices(segment.Categories, []string{"filler", "backchannel"}) {
t.Fatalf("categories = %v, want %v", segment.Categories, []string{"filler", "backchannel"})
}
if segment.Speaker != "Bob" {
t.Fatalf("speaker = %q, want Bob", segment.Speaker)
}
if segment.Start != 2 || segment.End != 3 {
t.Fatalf("times = %.3f-%.3f, want 2.000-3.000", segment.Start, segment.End)
}
if segment.Text != "beta" {
t.Fatalf("text = %q, want beta", segment.Text)
}
if segment.OverlapGroupID != 0 {
t.Fatalf("overlap_group_id = %d, want 0", segment.OverlapGroupID)
}
if len(result.Transcript.OverlapGroups) != 0 {
t.Fatalf("overlap_groups count = %d, want 0", len(result.Transcript.OverlapGroups))
}
}
func TestApplyFullSchemaRemovesStaleOverlapGroups(t *testing.T) {
input := overlapTranscriptFixture()
selector := mustParseSelector(t, "1,3")
result, err := Apply(input, Options{
Mode: ModeKeep,
Selector: selector,
})
if err != nil {
t.Fatalf("apply failed: %v", err)
}
if len(result.Transcript.OverlapGroups) != 0 {
t.Fatalf("overlap_groups count = %d, want 0", len(result.Transcript.OverlapGroups))
}
for index, segment := range result.Transcript.Segments {
if segment.OverlapGroupID != 0 {
t.Fatalf("segment %d overlap_group_id = %d, want 0", index, segment.OverlapGroupID)
}
}
}
func TestApplyFullSchemaRecomputesOverlapGroup(t *testing.T) {
input := overlapTranscriptFixture()
selector := mustParseSelector(t, "1,2")
result, err := Apply(input, Options{
Mode: ModeKeep,
Selector: selector,
})
if err != nil {
t.Fatalf("apply failed: %v", err)
}
assertSegmentIDs(t, result.Transcript.Segments, []int{1, 2})
assertIntSlice(t, []int{
result.Transcript.Segments[0].OverlapGroupID,
result.Transcript.Segments[1].OverlapGroupID,
}, []int{1, 1})
if len(result.Transcript.OverlapGroups) != 1 {
t.Fatalf("overlap_groups count = %d, want 1", len(result.Transcript.OverlapGroups))
}
group := result.Transcript.OverlapGroups[0]
if group.ID != 1 {
t.Fatalf("group ID = %d, want 1", group.ID)
}
if group.Start != 1 || group.End != 4 {
t.Fatalf("group times = %.3f-%.3f, want 1.000-4.000", group.Start, group.End)
}
if !equalStringSlices(group.Segments, []string{"a.json#10", "b.json#20"}) {
t.Fatalf("group segments = %v, want %v", group.Segments, []string{"a.json#10", "b.json#20"})
}
if !equalStringSlices(group.Speakers, []string{"Alice", "Bob"}) {
t.Fatalf("group speakers = %v, want %v", group.Speakers, []string{"Alice", "Bob"})
}
}
func TestApplyFullSchemaDropsGroupWhenFewerThanTwoSpeakersRemain(t *testing.T) {
input := overlapTranscriptFixture()
selector := mustParseSelector(t, "1")
result, err := Apply(input, Options{
Mode: ModeKeep,
Selector: selector,
})
if err != nil {
t.Fatalf("apply failed: %v", err)
}
if len(result.Transcript.OverlapGroups) != 0 {
t.Fatalf("overlap_groups count = %d, want 0", len(result.Transcript.OverlapGroups))
}
if len(result.Transcript.Segments) != 1 {
t.Fatalf("segment count = %d, want 1", len(result.Transcript.Segments))
}
if result.Transcript.Segments[0].OverlapGroupID != 0 {
t.Fatalf("segment overlap_group_id = %d, want 0", result.Transcript.Segments[0].OverlapGroupID)
}
}
func TestApplyFullSchemaHandlesTransitiveOverlaps(t *testing.T) {
input := transitiveOverlapFixture()
selector := mustParseSelector(t, "1-3")
result, err := Apply(input, Options{
Mode: ModeKeep,
Selector: selector,
})
if err != nil {
t.Fatalf("apply failed: %v", err)
}
if len(result.Transcript.OverlapGroups) != 1 {
t.Fatalf("overlap_groups count = %d, want 1", len(result.Transcript.OverlapGroups))
}
assertIntSlice(t, []int{
result.Transcript.Segments[0].OverlapGroupID,
result.Transcript.Segments[1].OverlapGroupID,
result.Transcript.Segments[2].OverlapGroupID,
}, []int{1, 1, 1})
group := result.Transcript.OverlapGroups[0]
if group.Start != 10 || group.End != 15 {
t.Fatalf("group times = %.3f-%.3f, want 10.000-15.000", group.Start, group.End)
}
}
func TestApplyFullSchemaBoundaryTouchingNotGrouped(t *testing.T) {
input := boundaryFixture()
selector := mustParseSelector(t, "1-2")
result, err := Apply(input, Options{
Mode: ModeKeep,
Selector: selector,
})
if err != nil {
t.Fatalf("apply failed: %v", err)
}
if len(result.Transcript.OverlapGroups) != 0 {
t.Fatalf("overlap_groups count = %d, want 0", len(result.Transcript.OverlapGroups))
}
assertIntSlice(t, []int{
result.Transcript.Segments[0].OverlapGroupID,
result.Transcript.Segments[1].OverlapGroupID,
}, []int{0, 0})
}
func TestApplyIntermediateDoesNotIncludeOverlapGroups(t *testing.T) {
input := schema.IntermediateTranscript{
Metadata: schema.IntermediateMetadata{
Application: "seriatim",
Version: "v-test",
OutputSchema: "seriatim-intermediate",
},
Segments: []schema.IntermediateSegment{
{ID: 1, Start: 1, End: 3, Speaker: "Alice", Text: "alpha", Categories: []string{"word-run"}},
{ID: 2, Start: 2, End: 4, Speaker: "Bob", Text: "beta", Categories: []string{"filler"}},
},
}
selector := mustParseSelector(t, "1")
result, err := ApplyIntermediate(input, Options{
Mode: ModeKeep,
Selector: selector,
})
if err != nil {
t.Fatalf("apply intermediate failed: %v", err)
}
if len(result.Transcript.Segments) != 1 {
t.Fatalf("segment count = %d, want 1", len(result.Transcript.Segments))
}
if result.Transcript.Segments[0].ID != 1 {
t.Fatalf("segment id = %d, want 1", result.Transcript.Segments[0].ID)
}
if err := schema.ValidateIntermediateTranscript(result.Transcript); err != nil {
t.Fatalf("intermediate output should remain valid: %v", err)
}
}
func TestApplyMinimalDoesNotIncludeOverlapGroups(t *testing.T) {
input := schema.MinimalTranscript{
Metadata: schema.MinimalMetadata{
Application: "seriatim",
Version: "v-test",
OutputSchema: "seriatim-minimal",
},
Segments: []schema.MinimalSegment{
{ID: 1, Start: 1, End: 3, Speaker: "Alice", Text: "alpha"},
{ID: 2, Start: 2, End: 4, Speaker: "Bob", Text: "beta"},
},
}
selector := mustParseSelector(t, "2")
result, err := ApplyMinimal(input, Options{
Mode: ModeKeep,
Selector: selector,
})
if err != nil {
t.Fatalf("apply minimal failed: %v", err)
}
if len(result.Transcript.Segments) != 1 {
t.Fatalf("segment count = %d, want 1", len(result.Transcript.Segments))
}
if result.Transcript.Segments[0].ID != 1 {
t.Fatalf("segment id = %d, want 1", result.Transcript.Segments[0].ID)
}
if err := schema.ValidateMinimalTranscript(result.Transcript); err != nil {
t.Fatalf("minimal output should remain valid: %v", err)
}
}
func TestApplyOutputInvariantsValidAfterRenumberAndOverlapRecompute(t *testing.T) {
input := overlapTranscriptFixture()
selector := mustParseSelector(t, "2,1")
result, err := Apply(input, Options{
Mode: ModeKeep,
Selector: selector,
})
if err != nil {
t.Fatalf("apply failed: %v", err)
}
if err := schema.ValidateTranscript(result.Transcript); err != nil {
t.Fatalf("trim output should remain valid: %v", err)
}
}
func mustParseSelector(t *testing.T, value string) Selector {
t.Helper()
selector, err := ParseSelector(value)
if err != nil {
t.Fatalf("selector parse failed for %q: %v", value, err)
}
return selector
}
func fullTranscriptFixture() schema.Transcript {
firstIndex := 10
secondIndex := 20
thirdIndex := 30
fourthIndex := 40
return schema.Transcript{
Metadata: schema.Metadata{
Application: "seriatim",
Version: "v-test",
InputReader: "json-files",
InputFiles: []string{"a.json", "b.json"},
PreprocessingModules: []string{"validate-raw"},
PostprocessingModules: []string{"detect-overlaps"},
OutputModules: []string{"json"},
},
Segments: []schema.Segment{
{
ID: 1,
Source: "a.json",
SourceSegmentIndex: &firstIndex,
SourceRef: "a.json#10",
DerivedFrom: []string{"a.json#10"},
Speaker: "Alice",
Start: 1,
End: 2,
Text: "alpha",
Categories: []string{"word-run"},
OverlapGroupID: 7,
},
{
ID: 2,
Source: "b.json",
SourceSegmentIndex: &secondIndex,
SourceRef: "b.json#20",
DerivedFrom: []string{"b.json#19", "b.json#20"},
Speaker: "Bob",
Start: 2,
End: 3,
Text: "beta",
Categories: []string{"filler", "backchannel"},
OverlapGroupID: 7,
},
{
ID: 3,
Source: "c.json",
SourceSegmentIndex: &thirdIndex,
SourceRef: "c.json#30",
DerivedFrom: []string{"c.json#30"},
Speaker: "Carol",
Start: 3,
End: 4,
Text: "gamma",
Categories: []string{"normal"},
OverlapGroupID: 8,
},
{
ID: 4,
Source: "d.json",
SourceSegmentIndex: &fourthIndex,
SourceRef: "d.json#40",
DerivedFrom: []string{"d.json#40"},
Speaker: "Dan",
Start: 4,
End: 5,
Text: "delta",
Categories: []string{"normal"},
OverlapGroupID: 9,
},
},
OverlapGroups: []schema.OverlapGroup{
{
ID: 7,
Start: 1.5,
End: 3.1,
Segments: []string{"a.json#10", "b.json#20"},
Speakers: []string{"Alice", "Bob"},
Class: "unknown",
Resolution: "unresolved",
},
},
}
}
func overlapTranscriptFixture() schema.Transcript {
first := 10
second := 20
third := 30
return schema.Transcript{
Metadata: schema.Metadata{
Application: "seriatim",
Version: "v-test",
InputReader: "json-files",
InputFiles: []string{"a.json", "b.json", "c.json"},
PreprocessingModules: []string{"validate-raw"},
PostprocessingModules: []string{"detect-overlaps"},
OutputModules: []string{"json"},
},
Segments: []schema.Segment{
{
ID: 1,
Source: "a.json",
SourceSegmentIndex: &first,
SourceRef: "a.json#10",
Speaker: "Alice",
Start: 1,
End: 4,
Text: "a",
OverlapGroupID: 99,
},
{
ID: 2,
Source: "b.json",
SourceSegmentIndex: &second,
SourceRef: "b.json#20",
Speaker: "Bob",
Start: 2,
End: 3,
Text: "b",
OverlapGroupID: 99,
},
{
ID: 3,
Source: "c.json",
SourceSegmentIndex: &third,
SourceRef: "c.json#30",
Speaker: "Carol",
Start: 10,
End: 11,
Text: "c",
OverlapGroupID: 100,
},
},
OverlapGroups: []schema.OverlapGroup{
{
ID: 99,
Start: 0,
End: 100,
Segments: []string{"stale#1", "stale#2"},
Speakers: []string{"stale"},
Class: "unknown",
Resolution: "unresolved",
},
},
}
}
func transitiveOverlapFixture() schema.Transcript {
one := 1
two := 2
three := 3
return schema.Transcript{
Metadata: schema.Metadata{
Application: "seriatim",
Version: "v-test",
},
Segments: []schema.Segment{
{ID: 1, Source: "a.json", SourceSegmentIndex: &one, Speaker: "Alice", Start: 10, End: 14, Text: "a"},
{ID: 2, Source: "b.json", SourceSegmentIndex: &two, Speaker: "Bob", Start: 12, End: 13, Text: "b"},
{ID: 3, Source: "c.json", SourceSegmentIndex: &three, Speaker: "Carol", Start: 13.5, End: 15, Text: "c"},
},
OverlapGroups: []schema.OverlapGroup{{ID: 77}},
}
}
func boundaryFixture() schema.Transcript {
one := 1
two := 2
return schema.Transcript{
Metadata: schema.Metadata{
Application: "seriatim",
Version: "v-test",
},
Segments: []schema.Segment{
{ID: 1, Source: "a.json", SourceSegmentIndex: &one, Speaker: "Alice", Start: 1, End: 2, Text: "a", OverlapGroupID: 7},
{ID: 2, Source: "b.json", SourceSegmentIndex: &two, Speaker: "Bob", Start: 2, End: 3, Text: "b", OverlapGroupID: 7},
},
OverlapGroups: []schema.OverlapGroup{{ID: 7, Start: 1, End: 3}},
}
}
func assertSegmentIDs(t *testing.T, segments []schema.Segment, want []int) {
t.Helper()
got := make([]int, len(segments))
for index, segment := range segments {
got[index] = segment.ID
}
assertIntSlice(t, got, want)
}
func assertSegmentTexts(t *testing.T, segments []schema.Segment, want []string) {
t.Helper()
got := make([]string, len(segments))
for index, segment := range segments {
got[index] = segment.Text
}
if !equalStringSlices(got, want) {
t.Fatalf("segment texts = %v, want %v", got, want)
}
}
func assertIntSlice(t *testing.T, got []int, want []int) {
t.Helper()
if len(got) != len(want) {
t.Fatalf("slice length = %d, want %d", len(got), len(want))
}
for index := range got {
if got[index] != want[index] {
t.Fatalf("slice[%d] = %d, want %d (full got=%v, want=%v)", index, got[index], want[index], got, want)
}
}
}
func assertIntMap(t *testing.T, got map[int]int, want map[int]int) {
t.Helper()
if len(got) != len(want) {
t.Fatalf("map length = %d, want %d", len(got), len(want))
}
for key, wantValue := range want {
gotValue, exists := got[key]
if !exists {
t.Fatalf("missing map key %d", key)
}
if gotValue != wantValue {
t.Fatalf("map[%d] = %d, want %d", key, gotValue, wantValue)
}
}
}
func equalStringSlices(got []string, want []string) bool {
if len(got) != len(want) {
return false
}
for index := range got {
if got[index] != want[index] {
return false
}
}
return true
}

396
internal/trim/artifact.go Normal file
View File

@@ -0,0 +1,396 @@
package trim
import (
"encoding/json"
"fmt"
"gitea.maximumdirect.net/eric/seriatim/schema"
)
const (
SchemaMinimal = "seriatim-minimal"
SchemaIntermediate = "seriatim-intermediate"
SchemaFull = "seriatim-full"
)
// Artifact stores a parsed seriatim output artifact of one supported schema.
type Artifact struct {
Schema string
Full *schema.Transcript
Intermediate *schema.IntermediateTranscript
Minimal *schema.MinimalTranscript
}
// ApplyArtifactResult contains trimmed artifact output and ID mapping metadata.
type ApplyArtifactResult struct {
Artifact Artifact
OldToNewID map[int]int
RemovedIDs []int
OverlapGroupsRecomputed bool
}
// ParseArtifactJSON parses and validates a serialized seriatim output artifact.
func ParseArtifactJSON(data []byte) (Artifact, error) {
var decoded any
if err := json.Unmarshal(data, &decoded); err != nil {
return Artifact{}, fmt.Errorf("input JSON is malformed: %w", err)
}
var full schema.Transcript
if err := json.Unmarshal(data, &full); err == nil {
if err := schema.ValidateTranscript(full); err == nil {
return Artifact{
Schema: SchemaFull,
Full: &full,
}, nil
}
}
var intermediate schema.IntermediateTranscript
if err := json.Unmarshal(data, &intermediate); err == nil {
if err := schema.ValidateIntermediateTranscript(intermediate); err == nil {
return Artifact{
Schema: SchemaIntermediate,
Intermediate: &intermediate,
}, nil
}
}
var minimal schema.MinimalTranscript
if err := json.Unmarshal(data, &minimal); err == nil {
if err := schema.ValidateMinimalTranscript(minimal); err == nil {
return Artifact{
Schema: SchemaMinimal,
Minimal: &minimal,
}, nil
}
}
return Artifact{}, fmt.Errorf("input JSON is not a valid seriatim output artifact")
}
// ValidateArtifact validates an artifact against its declared schema.
func ValidateArtifact(artifact Artifact) error {
switch artifact.Schema {
case SchemaFull:
if artifact.Full == nil {
return fmt.Errorf("full artifact payload is missing")
}
return schema.ValidateTranscript(*artifact.Full)
case SchemaIntermediate:
if artifact.Intermediate == nil {
return fmt.Errorf("intermediate artifact payload is missing")
}
return schema.ValidateIntermediateTranscript(*artifact.Intermediate)
case SchemaMinimal:
if artifact.Minimal == nil {
return fmt.Errorf("minimal artifact payload is missing")
}
return schema.ValidateMinimalTranscript(*artifact.Minimal)
default:
return fmt.Errorf("unsupported artifact schema %q", artifact.Schema)
}
}
// Value returns the artifact value for JSON serialization.
func (artifact Artifact) Value() any {
switch artifact.Schema {
case SchemaFull:
if artifact.Full == nil {
return schema.Transcript{}
}
return *artifact.Full
case SchemaIntermediate:
if artifact.Intermediate == nil {
return schema.IntermediateTranscript{}
}
return *artifact.Intermediate
case SchemaMinimal:
if artifact.Minimal == nil {
return schema.MinimalTranscript{}
}
return *artifact.Minimal
default:
return nil
}
}
// SegmentCount returns the number of segments in the artifact.
func (artifact Artifact) SegmentCount() int {
switch artifact.Schema {
case SchemaFull:
if artifact.Full == nil {
return 0
}
return len(artifact.Full.Segments)
case SchemaIntermediate:
if artifact.Intermediate == nil {
return 0
}
return len(artifact.Intermediate.Segments)
case SchemaMinimal:
if artifact.Minimal == nil {
return 0
}
return len(artifact.Minimal.Segments)
default:
return 0
}
}
// Application returns artifact metadata application name.
func (artifact Artifact) Application() string {
switch artifact.Schema {
case SchemaFull:
if artifact.Full == nil {
return ""
}
return artifact.Full.Metadata.Application
case SchemaIntermediate:
if artifact.Intermediate == nil {
return ""
}
return artifact.Intermediate.Metadata.Application
case SchemaMinimal:
if artifact.Minimal == nil {
return ""
}
return artifact.Minimal.Metadata.Application
default:
return ""
}
}
// Version returns artifact metadata version.
func (artifact Artifact) Version() string {
switch artifact.Schema {
case SchemaFull:
if artifact.Full == nil {
return ""
}
return artifact.Full.Metadata.Version
case SchemaIntermediate:
if artifact.Intermediate == nil {
return ""
}
return artifact.Intermediate.Metadata.Version
case SchemaMinimal:
if artifact.Minimal == nil {
return ""
}
return artifact.Minimal.Metadata.Version
default:
return ""
}
}
// ApplyArtifact trims a parsed artifact while preserving its input schema.
func ApplyArtifact(input Artifact, opts Options) (ApplyArtifactResult, error) {
switch input.Schema {
case SchemaFull:
if input.Full == nil {
return ApplyArtifactResult{}, fmt.Errorf("full artifact payload is missing")
}
result, err := Apply(*input.Full, opts)
if err != nil {
return ApplyArtifactResult{}, err
}
out := result.Transcript
return ApplyArtifactResult{
Artifact: Artifact{
Schema: SchemaFull,
Full: &out,
},
OldToNewID: result.OldToNewID,
RemovedIDs: result.RemovedIDs,
OverlapGroupsRecomputed: true,
}, nil
case SchemaIntermediate:
if input.Intermediate == nil {
return ApplyArtifactResult{}, fmt.Errorf("intermediate artifact payload is missing")
}
result, err := ApplyIntermediate(*input.Intermediate, opts)
if err != nil {
return ApplyArtifactResult{}, err
}
out := result.Transcript
return ApplyArtifactResult{
Artifact: Artifact{
Schema: SchemaIntermediate,
Intermediate: &out,
},
OldToNewID: result.OldToNewID,
RemovedIDs: result.RemovedIDs,
OverlapGroupsRecomputed: false,
}, nil
case SchemaMinimal:
if input.Minimal == nil {
return ApplyArtifactResult{}, fmt.Errorf("minimal artifact payload is missing")
}
result, err := ApplyMinimal(*input.Minimal, opts)
if err != nil {
return ApplyArtifactResult{}, err
}
out := result.Transcript
return ApplyArtifactResult{
Artifact: Artifact{
Schema: SchemaMinimal,
Minimal: &out,
},
OldToNewID: result.OldToNewID,
RemovedIDs: result.RemovedIDs,
OverlapGroupsRecomputed: false,
}, nil
default:
return ApplyArtifactResult{}, fmt.Errorf("unsupported artifact schema %q", input.Schema)
}
}
// ConvertArtifact converts a parsed artifact to another supported output schema.
func ConvertArtifact(input Artifact, outputSchema string) (Artifact, error) {
if outputSchema == "" || outputSchema == input.Schema {
return input, nil
}
switch input.Schema {
case SchemaFull:
if input.Full == nil {
return Artifact{}, fmt.Errorf("full artifact payload is missing")
}
switch outputSchema {
case SchemaIntermediate:
out := intermediateFromFull(*input.Full)
return Artifact{
Schema: SchemaIntermediate,
Intermediate: &out,
}, nil
case SchemaMinimal:
out := minimalFromFull(*input.Full)
return Artifact{
Schema: SchemaMinimal,
Minimal: &out,
}, nil
default:
return Artifact{}, fmt.Errorf("unsupported output schema %q", outputSchema)
}
case SchemaIntermediate:
if input.Intermediate == nil {
return Artifact{}, fmt.Errorf("intermediate artifact payload is missing")
}
switch outputSchema {
case SchemaMinimal:
out := minimalFromIntermediate(*input.Intermediate)
return Artifact{
Schema: SchemaMinimal,
Minimal: &out,
}, nil
case SchemaFull:
return Artifact{}, fmt.Errorf("cannot emit %q from %q input artifact", SchemaFull, SchemaIntermediate)
default:
return Artifact{}, fmt.Errorf("unsupported output schema %q", outputSchema)
}
case SchemaMinimal:
if input.Minimal == nil {
return Artifact{}, fmt.Errorf("minimal artifact payload is missing")
}
switch outputSchema {
case SchemaIntermediate:
out := intermediateFromMinimal(*input.Minimal)
return Artifact{
Schema: SchemaIntermediate,
Intermediate: &out,
}, nil
case SchemaFull:
return Artifact{}, fmt.Errorf("cannot emit %q from %q input artifact", SchemaFull, SchemaMinimal)
default:
return Artifact{}, fmt.Errorf("unsupported output schema %q", outputSchema)
}
default:
return Artifact{}, fmt.Errorf("unsupported input schema %q", input.Schema)
}
}
func intermediateFromFull(input schema.Transcript) schema.IntermediateTranscript {
segments := make([]schema.IntermediateSegment, len(input.Segments))
for index, segment := range input.Segments {
segments[index] = schema.IntermediateSegment{
ID: segment.ID,
Start: segment.Start,
End: segment.End,
Speaker: segment.Speaker,
Text: segment.Text,
Categories: append([]string(nil), segment.Categories...),
}
}
return schema.IntermediateTranscript{
Metadata: schema.IntermediateMetadata{
Application: input.Metadata.Application,
Version: input.Metadata.Version,
OutputSchema: SchemaIntermediate,
},
Segments: segments,
}
}
func minimalFromFull(input schema.Transcript) schema.MinimalTranscript {
segments := make([]schema.MinimalSegment, len(input.Segments))
for index, segment := range input.Segments {
segments[index] = schema.MinimalSegment{
ID: segment.ID,
Start: segment.Start,
End: segment.End,
Speaker: segment.Speaker,
Text: segment.Text,
}
}
return schema.MinimalTranscript{
Metadata: schema.MinimalMetadata{
Application: input.Metadata.Application,
Version: input.Metadata.Version,
OutputSchema: SchemaMinimal,
},
Segments: segments,
}
}
func minimalFromIntermediate(input schema.IntermediateTranscript) schema.MinimalTranscript {
segments := make([]schema.MinimalSegment, len(input.Segments))
for index, segment := range input.Segments {
segments[index] = schema.MinimalSegment{
ID: segment.ID,
Start: segment.Start,
End: segment.End,
Speaker: segment.Speaker,
Text: segment.Text,
}
}
return schema.MinimalTranscript{
Metadata: schema.MinimalMetadata{
Application: input.Metadata.Application,
Version: input.Metadata.Version,
OutputSchema: SchemaMinimal,
},
Segments: segments,
}
}
func intermediateFromMinimal(input schema.MinimalTranscript) schema.IntermediateTranscript {
segments := make([]schema.IntermediateSegment, len(input.Segments))
for index, segment := range input.Segments {
segments[index] = schema.IntermediateSegment{
ID: segment.ID,
Start: segment.Start,
End: segment.End,
Speaker: segment.Speaker,
Text: segment.Text,
}
}
return schema.IntermediateTranscript{
Metadata: schema.IntermediateMetadata{
Application: input.Metadata.Application,
Version: input.Metadata.Version,
OutputSchema: SchemaIntermediate,
},
Segments: segments,
}
}

View File

@@ -0,0 +1,138 @@
package trim
import (
"encoding/json"
"strings"
"testing"
"gitea.maximumdirect.net/eric/seriatim/schema"
)
func TestParseArtifactJSONRejectsMalformedJSON(t *testing.T) {
_, err := ParseArtifactJSON([]byte(`{"metadata":`))
if err == nil {
t.Fatal("expected malformed JSON error")
}
if !strings.Contains(err.Error(), "input JSON is malformed") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestParseArtifactJSONRejectsDuplicateSegmentIDs(t *testing.T) {
first := 10
second := 20
value := schema.Transcript{
Metadata: schema.Metadata{
Application: "seriatim",
Version: "v-test",
},
Segments: []schema.Segment{
{ID: 1, Source: "a.json", SourceSegmentIndex: &first, Speaker: "A", Start: 1, End: 2, Text: "one"},
{ID: 1, Source: "a.json", SourceSegmentIndex: &second, Speaker: "B", Start: 2, End: 3, Text: "two"},
},
OverlapGroups: []schema.OverlapGroup{},
}
data := mustMarshalJSON(t, value)
_, err := ParseArtifactJSON(data)
if err == nil {
t.Fatal("expected invalid artifact error")
}
if !strings.Contains(err.Error(), "not a valid seriatim output artifact") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestParseArtifactJSONRejectsNonSequentialSegmentIDs(t *testing.T) {
first := 10
second := 20
value := schema.Transcript{
Metadata: schema.Metadata{
Application: "seriatim",
Version: "v-test",
},
Segments: []schema.Segment{
{ID: 1, Source: "a.json", SourceSegmentIndex: &first, Speaker: "A", Start: 1, End: 2, Text: "one"},
{ID: 3, Source: "a.json", SourceSegmentIndex: &second, Speaker: "B", Start: 2, End: 3, Text: "two"},
},
OverlapGroups: []schema.OverlapGroup{},
}
data := mustMarshalJSON(t, value)
_, err := ParseArtifactJSON(data)
if err == nil {
t.Fatal("expected invalid artifact error")
}
if !strings.Contains(err.Error(), "not a valid seriatim output artifact") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestConvertArtifactMinimalToIntermediate(t *testing.T) {
value := schema.MinimalTranscript{
Metadata: schema.MinimalMetadata{
Application: "seriatim",
Version: "v-test",
OutputSchema: SchemaMinimal,
},
Segments: []schema.MinimalSegment{
{ID: 1, Start: 1, End: 2, Speaker: "A", Text: "one"},
{ID: 2, Start: 2, End: 3, Speaker: "B", Text: "two"},
},
}
artifact := Artifact{
Schema: SchemaMinimal,
Minimal: &value,
}
converted, err := ConvertArtifact(artifact, SchemaIntermediate)
if err != nil {
t.Fatalf("convert failed: %v", err)
}
if converted.Schema != SchemaIntermediate {
t.Fatalf("schema = %q, want %q", converted.Schema, SchemaIntermediate)
}
if converted.Intermediate == nil {
t.Fatal("expected intermediate artifact")
}
if len(converted.Intermediate.Segments) != 2 {
t.Fatalf("segment count = %d, want 2", len(converted.Intermediate.Segments))
}
if converted.Intermediate.Segments[0].ID != 1 || converted.Intermediate.Segments[1].ID != 2 {
t.Fatalf("unexpected IDs: %#v", converted.Intermediate.Segments)
}
}
func TestConvertArtifactMinimalToFullFails(t *testing.T) {
value := schema.MinimalTranscript{
Metadata: schema.MinimalMetadata{
Application: "seriatim",
Version: "v-test",
OutputSchema: SchemaMinimal,
},
Segments: []schema.MinimalSegment{
{ID: 1, Start: 1, End: 2, Speaker: "A", Text: "one"},
},
}
artifact := Artifact{
Schema: SchemaMinimal,
Minimal: &value,
}
_, err := ConvertArtifact(artifact, SchemaFull)
if err == nil {
t.Fatal("expected conversion error")
}
if !strings.Contains(err.Error(), "cannot emit") {
t.Fatalf("unexpected error: %v", err)
}
}
func mustMarshalJSON(t *testing.T, value any) []byte {
t.Helper()
data, err := json.Marshal(value)
if err != nil {
t.Fatalf("marshal: %v", err)
}
return data
}

156
internal/trim/selector.go Normal file
View File

@@ -0,0 +1,156 @@
package trim
import (
"fmt"
"regexp"
"sort"
"strconv"
"strings"
)
var selectorElementPattern = regexp.MustCompile(`^([+-]?\d+)(?:\s*-\s*([+-]?\d+))?$`)
// Selector represents a normalized union of segment IDs.
type Selector struct {
ranges []idRange
}
type idRange struct {
start int
end int
}
// ParseSelector parses an inline segment selector expression.
func ParseSelector(input string) (Selector, error) {
if strings.TrimSpace(input) == "" {
return Selector{}, fmt.Errorf("selector cannot be empty")
}
parts := strings.Split(input, ",")
ranges := make([]idRange, 0, len(parts))
for index, raw := range parts {
element := strings.TrimSpace(raw)
if element == "" {
return Selector{}, fmt.Errorf("selector element %d cannot be empty", index+1)
}
rangeValue, err := parseElement(element)
if err != nil {
return Selector{}, fmt.Errorf("selector element %d %q: %w", index+1, element, err)
}
ranges = append(ranges, rangeValue)
}
normalized := normalizeRanges(ranges)
if len(normalized) == 0 {
return Selector{}, fmt.Errorf("selector cannot be empty")
}
return Selector{ranges: normalized}, nil
}
// Contains returns true when id is included by this selector.
func (s Selector) Contains(id int) bool {
if id <= 0 {
return false
}
index := sort.Search(len(s.ranges), func(i int) bool {
return s.ranges[i].end >= id
})
if index == len(s.ranges) {
return false
}
rangeValue := s.ranges[index]
return id >= rangeValue.start && id <= rangeValue.end
}
// IDs returns a deterministic ascending list of unique segment IDs.
func (s Selector) IDs() []int {
total := 0
for _, rangeValue := range s.ranges {
total += rangeValue.end - rangeValue.start + 1
}
ids := make([]int, 0, total)
for _, rangeValue := range s.ranges {
for id := rangeValue.start; id <= rangeValue.end; id++ {
ids = append(ids, id)
}
}
return ids
}
func parseElement(element string) (idRange, error) {
matches := selectorElementPattern.FindStringSubmatch(element)
if matches == nil {
return idRange{}, fmt.Errorf("malformed element")
}
start, err := parseID(matches[1])
if err != nil {
return idRange{}, err
}
if matches[2] == "" {
return idRange{start: start, end: start}, nil
}
end, err := parseID(matches[2])
if err != nil {
return idRange{}, fmt.Errorf("invalid range end: %w", err)
}
if start > end {
return idRange{}, fmt.Errorf("descending range %d-%d is invalid", start, end)
}
return idRange{start: start, end: end}, nil
}
func parseID(value string) (int, error) {
value = strings.TrimSpace(value)
if value == "" {
return 0, fmt.Errorf("missing segment ID")
}
id, err := strconv.Atoi(value)
if err != nil {
return 0, fmt.Errorf("segment ID must be an integer")
}
if id <= 0 {
return 0, fmt.Errorf("segment ID must be positive")
}
return id, nil
}
func normalizeRanges(in []idRange) []idRange {
if len(in) == 0 {
return nil
}
sorted := make([]idRange, len(in))
copy(sorted, in)
sort.Slice(sorted, func(i, j int) bool {
if sorted[i].start == sorted[j].start {
return sorted[i].end < sorted[j].end
}
return sorted[i].start < sorted[j].start
})
merged := make([]idRange, 0, len(sorted))
for _, next := range sorted {
if len(merged) == 0 {
merged = append(merged, next)
continue
}
last := &merged[len(merged)-1]
if next.start <= last.end+1 {
if next.end > last.end {
last.end = next.end
}
continue
}
merged = append(merged, next)
}
return merged
}

View File

@@ -0,0 +1,127 @@
package trim
import (
"strings"
"testing"
)
func TestParseSelectorSingleID(t *testing.T) {
selector, err := ParseSelector("1")
if err != nil {
t.Fatalf("parse failed: %v", err)
}
assertIDs(t, selector, []int{1})
assertContains(t, selector, map[int]bool{1: true, 2: false, 0: false, -1: false})
}
func TestParseSelectorInclusiveRange(t *testing.T) {
selector, err := ParseSelector("1-3")
if err != nil {
t.Fatalf("parse failed: %v", err)
}
assertIDs(t, selector, []int{1, 2, 3})
}
func TestParseSelectorCommaSeparatedCombination(t *testing.T) {
selector, err := ParseSelector("1-3,8,10-12")
if err != nil {
t.Fatalf("parse failed: %v", err)
}
assertIDs(t, selector, []int{1, 2, 3, 8, 10, 11, 12})
}
func TestParseSelectorWhitespaceTolerance(t *testing.T) {
selector, err := ParseSelector(" 1 - 3 , 8 , 10 - 12 ")
if err != nil {
t.Fatalf("parse failed: %v", err)
}
assertIDs(t, selector, []int{1, 2, 3, 8, 10, 11, 12})
}
func TestParseSelectorDuplicatesAndOverlapsNormalizeUnion(t *testing.T) {
selector, err := ParseSelector("1-4,2,4,3-6,6")
if err != nil {
t.Fatalf("parse failed: %v", err)
}
assertIDs(t, selector, []int{1, 2, 3, 4, 5, 6})
assertContains(t, selector, map[int]bool{1: true, 5: true, 6: true, 7: false})
}
func TestParseSelectorDeterministicNormalizedOutput(t *testing.T) {
left, err := ParseSelector("8,1-3,2,10-12")
if err != nil {
t.Fatalf("parse left failed: %v", err)
}
right, err := ParseSelector("10-12,3,2,1,8")
if err != nil {
t.Fatalf("parse right failed: %v", err)
}
leftIDs := left.IDs()
rightIDs := right.IDs()
if !equalInts(leftIDs, rightIDs) {
t.Fatalf("normalized IDs mismatch: %v vs %v", leftIDs, rightIDs)
}
}
func TestParseSelectorFailures(t *testing.T) {
tests := []struct {
name string
selector string
wantError string
}{
{name: "empty", selector: "", wantError: "cannot be empty"},
{name: "whitespace only", selector: " ", wantError: "cannot be empty"},
{name: "zero", selector: "0", wantError: "must be positive"},
{name: "negative", selector: "-1", wantError: "must be positive"},
{name: "range includes zero", selector: "0-2", wantError: "must be positive"},
{name: "descending range", selector: "10-1", wantError: "descending range"},
{name: "empty element", selector: "1,,2", wantError: "cannot be empty"},
{name: "trailing comma", selector: "1,", wantError: "cannot be empty"},
{name: "malformed alpha", selector: "abc", wantError: "malformed element"},
{name: "malformed range", selector: "1-2-3", wantError: "malformed element"},
{name: "missing end", selector: "1-", wantError: "malformed element"},
{name: "missing start", selector: "-2", wantError: "must be positive"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
_, err := ParseSelector(test.selector)
if err == nil {
t.Fatalf("expected error for %q", test.selector)
}
if !strings.Contains(err.Error(), test.wantError) {
t.Fatalf("error = %q, want substring %q", err.Error(), test.wantError)
}
})
}
}
func assertIDs(t *testing.T, selector Selector, want []int) {
t.Helper()
got := selector.IDs()
if !equalInts(got, want) {
t.Fatalf("IDs = %v, want %v", got, want)
}
}
func assertContains(t *testing.T, selector Selector, checks map[int]bool) {
t.Helper()
for id, want := range checks {
if got := selector.Contains(id); got != want {
t.Fatalf("Contains(%d) = %t, want %t", id, got, want)
}
}
}
func equalInts(left []int, right []int) bool {
if len(left) != len(right) {
return false
}
for index := range left {
if left[index] != right[index] {
return false
}
}
return true
}