Compare commits
16 Commits
v1.0.0
...
b7cc5fb980
| Author | SHA1 | Date | |
|---|---|---|---|
| b7cc5fb980 | |||
| b20438acf0 | |||
| 6dbb7ab17e | |||
| 3591041fa8 | |||
| 5b008e272c | |||
| 6c780f6293 | |||
| c132f3fd5d | |||
| 3679435063 | |||
| e6d3b4a46e | |||
| 54f7717de8 | |||
| c48b02d2ec | |||
| ac3dcf2557 | |||
| 1c0e4438ae | |||
| 52f7729100 | |||
| 2c82f8bf5c | |||
| d865bda4a9 |
4
.gitignore
vendored
4
.gitignore
vendored
@@ -1,3 +1,7 @@
|
||||
# ---> Codex
|
||||
.codex
|
||||
AGENTS.md
|
||||
|
||||
# ---> Go
|
||||
# If you prefer the allow list template instead of the deny list, see community template:
|
||||
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
|
||||
|
||||
2
LICENSE
2
LICENSE
@@ -1,4 +1,4 @@
|
||||
Copyright (c) 2026 eric.
|
||||
Copyright (c) 2026 Eric Rakestraw.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
|
||||
|
||||
|
||||
114
README.md
114
README.md
@@ -1,8 +1,8 @@
|
||||
# seriatim
|
||||
|
||||
`seriatim` merges per-speaker WhisperX-style JSON transcripts into a single JSON transcript that preserves speaker identity and chronological order.
|
||||
`seriatim` merges per-speaker WhisperX-style JSON transcripts into a single JSON transcript that preserves speaker identity and chronological order. It also trims existing seriatim output artifacts by segment ID and normalizes external transcript-like JSON into standard seriatim output schemas.
|
||||
|
||||
The current implementation supports the `merge` command. It reads one or more input JSON files, optionally maps each input file to a canonical speaker using `speakers.yml`, sorts all segments by timestamp, detects and resolves overlaps when word-level timing is available, assigns consecutive numeric `id` values, and writes a merged JSON artifact.
|
||||
The current implementation supports the `merge`, `trim`, and `normalize` commands. `merge` reads one or more input JSON files, optionally maps each input file to a canonical speaker using `speakers.yml`, sorts all segments by timestamp, detects and resolves overlaps when word-level timing is available, assigns consecutive numeric `id` values, and writes a merged JSON artifact. `trim` reads an existing seriatim output artifact and projects it to a retained segment subset. `normalize` reads transcript-like JSON input, validates required segment fields, sorts deterministically, assigns fresh IDs, and emits a selected seriatim output schema.
|
||||
|
||||
## Usage
|
||||
|
||||
@@ -25,10 +25,39 @@ go run ./cmd/seriatim merge \
|
||||
--report-file report.json
|
||||
```
|
||||
|
||||
Trim an existing seriatim artifact:
|
||||
|
||||
```sh
|
||||
go run ./cmd/seriatim trim \
|
||||
--input-file merged.json \
|
||||
--output-file trimmed.json \
|
||||
--keep "1-10, 15, 20-25"
|
||||
```
|
||||
|
||||
Normalize external transcript-style JSON:
|
||||
|
||||
```sh
|
||||
go run ./cmd/seriatim normalize \
|
||||
--input-file transcript.json \
|
||||
--output-file normalized.json
|
||||
```
|
||||
|
||||
Normalize an Audita-style bare segment array to full schema with report output:
|
||||
|
||||
```sh
|
||||
go run ./cmd/seriatim normalize \
|
||||
--input-file audita-segments.json \
|
||||
--output-file normalized-full.json \
|
||||
--output-schema seriatim-full \
|
||||
--report-file normalize-report.json
|
||||
```
|
||||
|
||||
## CLI
|
||||
|
||||
```text
|
||||
seriatim merge [flags]
|
||||
seriatim trim [flags]
|
||||
seriatim normalize [flags]
|
||||
```
|
||||
|
||||
Global flags:
|
||||
@@ -54,6 +83,87 @@ Global flags:
|
||||
| `--postprocessing-modules` | No | `detect-overlaps,resolve-overlaps,backchannel,filler,resolve-danglers,coalesce,detect-overlaps,autocorrect,assign-ids,validate-output` | Comma-separated postprocessing modules, evaluated in order. |
|
||||
| `--coalesce-gap` | No | `3.0` | Maximum same-speaker gap in seconds for `coalesce`; also used as the `resolve-overlaps` context window. Must be a non-negative float. |
|
||||
|
||||
`trim` flags:
|
||||
|
||||
| Flag | Required | Default | Description |
|
||||
| --- | --- | --- | --- |
|
||||
| `--input-file` | Yes | none | Input seriatim output artifact JSON file. |
|
||||
| `--output-file` | Yes | none | Trimmed transcript JSON output path. |
|
||||
| `--keep` | Exactly one of `--keep` or `--remove` is required | none | Segment ID selector to retain. |
|
||||
| `--remove` | Exactly one of `--keep` or `--remove` is required | none | Segment ID selector to drop. |
|
||||
| `--output-schema` | No | preserve input artifact schema | Optional output schema override: `seriatim-minimal`, `seriatim-intermediate`, or `seriatim-full`. |
|
||||
| `--report-file` | No | none | Optional report JSON output path. |
|
||||
| `--allow-empty` | No | `false` | Allow trimming to zero retained segments. |
|
||||
|
||||
`trim` selection rules:
|
||||
|
||||
- `--keep` and `--remove` are mutually exclusive.
|
||||
- Exactly one of `--keep` or `--remove` is required.
|
||||
- Selection is by segment ID only.
|
||||
- Invalid selected segment IDs fail the command by default.
|
||||
|
||||
`trim` selector syntax:
|
||||
|
||||
- Segment IDs are positive 1-based integers.
|
||||
- Inclusive ranges are supported: `1-10`.
|
||||
- Comma-separated selectors are supported: `1-10,15,20-25`.
|
||||
- Whitespace around numbers, commas, and hyphens is allowed: `1 - 10, 15, 20 - 25`.
|
||||
- Duplicate and overlapping ranges are accepted and normalized as a union.
|
||||
- Descending ranges (for example `10-1`) are rejected.
|
||||
|
||||
`trim` behavior:
|
||||
|
||||
- `trim` consumes existing seriatim JSON output artifacts only.
|
||||
- `trim` does not accept raw WhisperX transcript JSON as input.
|
||||
- Retained output segment IDs are renumbered sequentially from `1` to `N`.
|
||||
- Transcript order is preserved from input transcript order; selector order does not reorder output.
|
||||
- When output schema is `seriatim-full`, overlap groups are recomputed from retained segments.
|
||||
- `--output-schema seriatim-full` is supported when trim has full-schema artifact data to emit; trim does not synthesize missing full-schema provenance from minimal/intermediate input artifacts.
|
||||
- `trim` does not run merge postprocessors such as `resolve-overlaps`, `coalesce`, or `autocorrect`.
|
||||
|
||||
`trim` report output:
|
||||
|
||||
- When `--report-file` is provided, the report includes standard trim/validation/output events.
|
||||
- The report includes a `trim-audit` event containing trim operation metadata, including selected IDs, retained/removed counts, removed IDs, and old-to-new segment ID mapping.
|
||||
- Old-to-new ID mapping is emitted as a deterministic ordered array of `{old_id, new_id}` pairs.
|
||||
|
||||
`normalize` flags:
|
||||
|
||||
| Flag | Required | Default | Description |
|
||||
| --- | --- | --- | --- |
|
||||
| `--input-file` | Yes | none | Input transcript JSON file. |
|
||||
| `--output-file` | Yes | none | Normalized transcript JSON output path. |
|
||||
| `--output-schema` | No | `seriatim-intermediate` (resolved via `SERIATIM_OUTPUT_SCHEMA` when set) | Output JSON schema: `seriatim-minimal`, `seriatim-intermediate`, or `seriatim-full`. |
|
||||
| `--output-modules` | No | `json` | Comma-separated output modules. Current normalize support is `json` only. |
|
||||
| `--report-file` | No | none | Optional report JSON output path. |
|
||||
|
||||
`normalize` input shapes:
|
||||
|
||||
- Top-level object with a `segments` array.
|
||||
- Bare top-level array of segment objects (for example, Audita-style output).
|
||||
|
||||
`normalize` behavior:
|
||||
|
||||
- Repairs missing timing fields deterministically:
|
||||
if one of `start`/`end` is present, sets both to that value;
|
||||
if both are missing, uses midpoint of previous `end` and next `start`,
|
||||
with edge fallback to available neighbor and `0.0` for single-segment inputs.
|
||||
- If `end < start`, swaps them.
|
||||
- Fills missing/empty `speaker` with `Unknown_Speaker`.
|
||||
- Drops segments with missing, empty, or whitespace-only `text`.
|
||||
- Validates repaired timing with `start >= 0`.
|
||||
- Accepts existing input `id` values as provenance only.
|
||||
- Reassigns output segment IDs sequentially from `1` to `N`.
|
||||
- Sorts deterministically by `(start, end, original_input_index, speaker)`.
|
||||
- Uses original input order only as a tie-breaker.
|
||||
- Does not run merge postprocessors such as overlap detection, overlap resolution, coalescing, or autocorrect.
|
||||
- Useful for converting external transcript outputs into standard seriatim artifacts.
|
||||
|
||||
`normalize` report output:
|
||||
|
||||
- When `--report-file` is provided, normalize emits deterministic report events with input shape detection, segment counts, schema/module selections, sorting/ID diagnostics, and output write/validation summaries.
|
||||
- A machine-readable `normalize-audit` event is included for downstream tooling.
|
||||
|
||||
Environment variables:
|
||||
|
||||
| Environment Variable | Default | Description |
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
# seriatim Architecture
|
||||
|
||||
`seriatim` is a deterministic transcript merge utility for combining multiple per-speaker transcript inputs into a single chronologically ordered diarized transcript.
|
||||
`seriatim` is a deterministic transcript utility for:
|
||||
|
||||
- merging multiple per-speaker transcript inputs into a single chronologically ordered diarized transcript, and
|
||||
- projecting existing seriatim transcript artifacts through deterministic segment-ID trimming, and
|
||||
- canonicalizing external transcript-style JSON inputs into standard seriatim output schemas.
|
||||
|
||||
The initial use case is merging independently transcribed speaker audio tracks from the same recorded session, such as a weekly tabletop RPG session. The architecture should also support meetings, podcasts, interviews, and other multi-speaker events.
|
||||
|
||||
@@ -20,6 +24,7 @@ The initial use case is merging independently transcribed speaker audio tracks f
|
||||
8. Detect and annotate overlapping speech regions.
|
||||
9. Emit one or more output artifacts through output writers.
|
||||
10. Produce report data for validation findings, corrections, and transformations.
|
||||
11. Support artifact-level transcript projection commands that operate on existing seriatim output.
|
||||
|
||||
## Non-goals
|
||||
|
||||
@@ -56,6 +61,8 @@ configuration check
|
||||
|
||||
Each stage has an explicit data contract. Input and output stages perform I/O. Processing stages should be deterministic transformations over in-memory models and should record report events for validation findings, corrections, and transformations.
|
||||
|
||||
`merge` runs this pipeline. `trim` and `normalize` are intentionally separate from this pipeline and operate at the artifact layer.
|
||||
|
||||
## Stage Contracts
|
||||
|
||||
### 1. Configuration Check
|
||||
@@ -191,6 +198,41 @@ Future output formats may include:
|
||||
|
||||
Output writers should be selected from an explicit registry and should consume the final transcript model read-only. Multiple output writers may run for a single invocation.
|
||||
|
||||
### 7. Artifact Projection Stage (`trim` command)
|
||||
|
||||
`trim` is an artifact-level command that reads an existing seriatim output artifact and emits a projected artifact containing a segment-ID subset.
|
||||
|
||||
Design constraints:
|
||||
|
||||
- `trim` runs after `merge`, not as a merge postprocessor.
|
||||
- `trim` validates the input artifact against supported seriatim output schemas.
|
||||
- `trim` performs deterministic keep/remove selection by segment ID.
|
||||
- `trim` renumbers retained IDs to `1..N` in transcript order.
|
||||
- `trim` validates the final output against the selected output schema before writing.
|
||||
- `trim` records audit metadata in report output.
|
||||
|
||||
`trim` is intentionally separate from merge postprocessing because it consumes already-emitted public artifacts. This separation keeps merge semantics stable and avoids rerunning merge-only transforms on projected artifacts.
|
||||
|
||||
`trim` must not rerun merge postprocessors such as `resolve-overlaps`, `coalesce`, or `autocorrect`.
|
||||
|
||||
### 8. Artifact Canonicalization Stage (`normalize` command)
|
||||
|
||||
`normalize` is an artifact-level command that reads transcript-like JSON and emits a standard seriatim output artifact in a selected schema.
|
||||
|
||||
Design constraints:
|
||||
|
||||
- `normalize` runs outside the merge pipeline and does not invoke merge preprocessing or postprocessing modules.
|
||||
- `normalize` accepts two input shapes: object-with-`segments` and bare segment arrays.
|
||||
- `normalize` applies deterministic repair rules for missing/irregular `start`, `end`, and `speaker`, and drops segments with missing/empty `text`.
|
||||
- `normalize` sorts segments deterministically by chronological keys and stable input-index tie-breakers.
|
||||
- `normalize` assigns fresh sequential output IDs (`1..N`) after sorting.
|
||||
- `normalize` validates final output against the selected schema before writing.
|
||||
- `normalize` writes optional deterministic report diagnostics when `--report-file` is requested.
|
||||
|
||||
`normalize` is intended for canonicalizing external transcript outputs (including Audita-style bare arrays) into seriatim contracts, not for running merge-time language or overlap transformations.
|
||||
|
||||
`normalize` must not run merge postprocessors such as overlap detection, overlap resolution, coalescing, or autocorrect.
|
||||
|
||||
## Module Classification
|
||||
|
||||
Modules should be classified by their contract and allowed effects.
|
||||
@@ -397,6 +439,8 @@ A valid merged transcript should satisfy:
|
||||
- Every referenced segment exists.
|
||||
- Output validates against the selected output schema.
|
||||
|
||||
For full-schema trim output, overlap groups are recomputed from retained segments so overlap annotations and group references remain internally consistent after projection.
|
||||
|
||||
## Determinism Requirements
|
||||
|
||||
Given the same inputs, config, and application version, `seriatim` should produce byte-stable JSON output where practical.
|
||||
@@ -411,6 +455,19 @@ To support this:
|
||||
- Record application version in output metadata.
|
||||
- Record enabled module names and module order in output metadata or report data.
|
||||
|
||||
Trim-specific determinism requirements:
|
||||
|
||||
- Selector normalization and retained IDs are deterministic.
|
||||
- Old-to-new ID mapping in trim reports is emitted in deterministic order.
|
||||
- Full-schema overlap recomputation is deterministic for the same input artifact and selector.
|
||||
|
||||
Normalize-specific determinism requirements:
|
||||
|
||||
- Input-shape detection is deterministic.
|
||||
- Segment ordering is deterministic for identical input data.
|
||||
- Output IDs are always reassigned sequentially after deterministic sorting.
|
||||
- Normalize diagnostic reports are deterministic for identical inputs and configuration.
|
||||
|
||||
## Go Package Layout
|
||||
|
||||
```text
|
||||
@@ -419,6 +476,8 @@ internal/config/ CLI/env/config loading and validation
|
||||
internal/pipeline/ Pipeline orchestration and module registry
|
||||
internal/builtin/ Built-in pipeline modules
|
||||
internal/artifact/ Conversion from internal model to public output schema
|
||||
internal/normalize/ Normalize input parsing, validation, deterministic sorting, schema conversion, and diagnostics
|
||||
internal/trim/ Artifact parsing, trim selection, schema conversion, overlap recomputation for full schema
|
||||
internal/buildinfo/ Build-time version metadata
|
||||
internal/speaker/ Speaker map parsing and lookup
|
||||
internal/model/ Canonical and merged transcript models
|
||||
@@ -430,6 +489,18 @@ schema/ Public output contract and JSON Schema validation
|
||||
|
||||
Package boundaries should follow data ownership. Shared models belong in `internal/model`; stage-specific behavior belongs in the relevant stage package.
|
||||
|
||||
For trim:
|
||||
|
||||
- `internal/trim` contains pure transformation logic over artifact structs.
|
||||
- CLI command code handles only flag parsing, file I/O, and report emission.
|
||||
- Transform logic is deterministic and pure except for command-layer I/O.
|
||||
|
||||
For normalize:
|
||||
|
||||
- `internal/normalize` contains parsing/validation and deterministic schema conversion logic.
|
||||
- CLI command code handles flag parsing and delegates execution.
|
||||
- Normalize remains artifact-level and does not compose merge pipeline modules.
|
||||
|
||||
## Default Modules
|
||||
|
||||
The default pipeline is equivalent to explicit module lists.
|
||||
|
||||
219
docs/policy/architecture.md
Normal file
219
docs/policy/architecture.md
Normal file
@@ -0,0 +1,219 @@
|
||||
# Architecture Policy
|
||||
|
||||
## Purpose
|
||||
|
||||
This document defines seriatim's development architecture and invariants for
|
||||
maintainers and automated coding agents. It describes how the implemented
|
||||
system is intended to be built and changed. It is not a user manual, CLI
|
||||
reference, config reference, or roadmap.
|
||||
|
||||
Keep this document aligned with [documentation policy](documentation.md). It
|
||||
must describe current behavior only; planned or speculative work belongs under
|
||||
`docs/roadmap/`.
|
||||
|
||||
## Project Shape
|
||||
|
||||
seriatim is a Go CLI for transcript artifact processing. The implemented
|
||||
commands are `merge`, `trim`, and `normalize`.
|
||||
|
||||
`merge` reads one or more JSON transcript files, optionally maps input files to
|
||||
canonical speakers, runs a registry-selected preprocessing chain, merges
|
||||
canonical segments into deterministic chronological order, runs a
|
||||
registry-selected postprocessing chain, validates the selected output schema,
|
||||
and writes JSON output plus an optional JSON report.
|
||||
|
||||
`trim` and `normalize` are artifact-level commands outside the merge pipeline.
|
||||
`trim` reads an existing seriatim output artifact and projects it by segment ID.
|
||||
`normalize` reads transcript-like JSON and emits one of seriatim's supported
|
||||
output schemas. Neither command runs merge preprocessing or postprocessing
|
||||
modules.
|
||||
|
||||
The supported public output schemas are `seriatim-minimal`,
|
||||
`seriatim-intermediate`, and `seriatim-full`. For current command and flag
|
||||
details, use [the README](../../README.md) until dedicated `docs/cli.md` and
|
||||
`docs/config.md` files exist.
|
||||
|
||||
## Core Design Principles
|
||||
|
||||
- Keep a hexagonal architecture boundary. Domain models, stage contracts, and
|
||||
deterministic transformations must stay separate from CLI parsing,
|
||||
filesystem access, config loading, reporting, and other external adapters.
|
||||
- Keep stages and modules composable. Built-in modules are selected by
|
||||
canonical registry names and implement explicit interfaces for their pipeline
|
||||
role.
|
||||
- Preserve deterministic behavior. Given the same inputs, configuration, and
|
||||
version, output ordering, segment IDs, schema validation, and report event
|
||||
ordering should remain stable.
|
||||
- Current command execution is sequential. There is no scheduler, worker pool,
|
||||
or concurrent module execution in the implemented pipeline. Any concurrency
|
||||
added later must be bounded, observable, and must not make output handling
|
||||
nondeterministic.
|
||||
- Prefer the Go standard library. Third-party dependencies should remain narrow
|
||||
and justified, such as Cobra for CLI structure, YAML parsing, and JSON Schema
|
||||
validation.
|
||||
- Document current behavior. Architecture, user, and internal docs must not
|
||||
describe planned features as implemented behavior.
|
||||
|
||||
## Architectural Boundaries
|
||||
|
||||
Core transcript data belongs in `internal/model` and public artifact contracts
|
||||
belong in `schema`. Conversion from internal merged data to public JSON shapes
|
||||
belongs at the artifact boundary, not inside CLI code or transformation
|
||||
packages.
|
||||
|
||||
Pipeline orchestration belongs in `internal/pipeline`. It resolves registered
|
||||
modules, validates preprocessing state transitions, executes stages in order,
|
||||
collects report events, converts the final transcript, and writes optional
|
||||
reports. Built-in adapters and modules are registered from `internal/builtin`.
|
||||
|
||||
CLI code in `internal/cli` should parse flags, build validated config values,
|
||||
and delegate. `merge` delegates to `pipeline.Run`; `trim` and `normalize`
|
||||
perform artifact-level orchestration and delegate deterministic parsing,
|
||||
validation, and transformation work to their internal packages.
|
||||
|
||||
Config loading and validation belongs in `internal/config`. Filesystem reads and
|
||||
writes are adapter concerns and should not spread into pure transformation
|
||||
helpers. Existing built-in modules that load configured YAML files must keep
|
||||
that I/O narrow and explicit.
|
||||
|
||||
Reports belong in `internal/report`. Modules and commands should emit concise
|
||||
events for validation findings, corrections, and transformations without
|
||||
turning report messages into a duplicate output artifact.
|
||||
|
||||
Tests and samples are supporting evidence for behavior. Tests should verify
|
||||
stable contracts and edge cases; samples should remain valid examples, not
|
||||
hidden architecture dependencies.
|
||||
|
||||
## Modules or Stages
|
||||
|
||||
The merge pipeline has these implemented stages:
|
||||
|
||||
- `InputReader`: reads configured external input into raw transcripts.
|
||||
- `Preprocessor`: transforms `PreprocessState` from raw to canonical state.
|
||||
- `Merger`: combines canonical transcripts into one merged transcript.
|
||||
- `Postprocessor`: transforms or annotates the merged transcript.
|
||||
- `OutputWriter`: writes the selected output artifact.
|
||||
|
||||
Modules must keep narrow responsibilities, declare their stage through the
|
||||
interface they implement, and use explicit config values. Preprocessors must
|
||||
declare `Requires()` and `Produces()` states; the runner rejects invalid
|
||||
raw/canonical ordering before processing completes.
|
||||
|
||||
Modules run in the configured order. Order-affecting modules must run before
|
||||
`assign-ids`, and `validate-output` must see final IDs that match the selected
|
||||
schema. Accepted and rejected transformations should be deterministic and, when
|
||||
observable, recorded through report events.
|
||||
|
||||
Transformation helpers should avoid hidden global state. Shared caches, such as
|
||||
compiled JSON schemas, must be protected and must not affect output ordering.
|
||||
|
||||
## State, Inputs, and Outputs
|
||||
|
||||
seriatim is file-based. It reads JSON inputs and optional YAML rule files, then
|
||||
writes JSON transcript artifacts and optional JSON reports.
|
||||
|
||||
The implemented application has no durable database, daemon state, resume
|
||||
state, remote storage, or background job state. Runtime state is held in memory
|
||||
for the current command invocation and serialized only through requested output
|
||||
and report files.
|
||||
|
||||
Input file paths are normalized and validated during config construction.
|
||||
`merge` sorts input file paths before processing, then uses stable segment sort
|
||||
keys. `trim` preserves transcript order while renumbering retained IDs.
|
||||
`normalize` sorts by implemented deterministic keys and assigns fresh IDs.
|
||||
|
||||
## Configuration and CLI Boundaries
|
||||
|
||||
The CLI surface is an adapter over validated config structs. Cobra command code
|
||||
should stay thin: parse flags, account for flag/default precedence, call config
|
||||
constructors, and delegate.
|
||||
|
||||
Config constructors validate required paths, output parent directories, module
|
||||
lists, selected schemas, mutually exclusive trim selector options, and supported
|
||||
environment-derived settings. Module name validation is split between config
|
||||
where command-specific names are fixed and the pipeline registry where module
|
||||
composition is resolved.
|
||||
|
||||
Do not duplicate full CLI or config reference material here. Use
|
||||
[the README](../../README.md) for the current user-facing reference until the
|
||||
canonical `docs/cli.md` and `docs/config.md` files exist.
|
||||
|
||||
## Errors, Logging, and Diagnostics
|
||||
|
||||
Commands return errors instead of printing inside deep logic. The root command
|
||||
silences Cobra usage/error output, and `cmd/seriatim/main.go` prints one error
|
||||
to stderr and exits with status `1`.
|
||||
|
||||
Validation failures should fail fast with contextual errors. Correctable
|
||||
conditions should be deterministic and, where reports are requested, reflected
|
||||
as report events. Optional reports contain metadata and ordered events; they are
|
||||
not required for command success unless the report file itself cannot be
|
||||
written.
|
||||
|
||||
The implemented code does not use a logging subsystem. Diagnostics are returned
|
||||
as errors or written to optional report JSON. Normalize report events avoid
|
||||
embedding transcript text; keep that privacy-oriented behavior when changing
|
||||
normalize diagnostics.
|
||||
|
||||
## Testing Expectations
|
||||
|
||||
`go test ./...` is the repository-wide check. There is currently no Makefile,
|
||||
taskfile, linter config, or dedicated documentation check.
|
||||
|
||||
When changing config or CLI behavior, inspect `internal/config` and
|
||||
`internal/cli` tests. When changing pipeline composition or stage contracts,
|
||||
inspect `internal/pipeline` and `internal/builtin` tests. When changing
|
||||
correction or annotation modules, inspect the package tests for overlap,
|
||||
coalesce, danglers, backchannel, filler, and autocorrect behavior.
|
||||
|
||||
When changing artifact-level commands, inspect `internal/trim`,
|
||||
`internal/normalize`, and their CLI tests. When changing public output shape or
|
||||
schema validation, inspect `schema` and `internal/artifact` tests. Report and
|
||||
diagnostic changes should be covered through the command or package tests that
|
||||
emit the affected events.
|
||||
|
||||
## Dependency Policy
|
||||
|
||||
Prefer the Go standard library for parsing, data transformation, concurrency
|
||||
primitives, filesystem work, and testing wherever it is reasonable.
|
||||
|
||||
Third-party dependencies must be narrow, justified, and preferably de facto
|
||||
standard for their purpose. Existing examples include Cobra for CLI structure,
|
||||
`gopkg.in/yaml.v3` for YAML files, and `jsonschema/v6` for validating embedded
|
||||
public JSON schemas. Avoid broad framework dependencies for behavior that is
|
||||
already simple and local.
|
||||
|
||||
## Documentation Expectations
|
||||
|
||||
Architecture docs must stay aligned with [documentation policy](documentation.md).
|
||||
Current-behavior docs must not become aspirational. If code and docs disagree,
|
||||
fix the inaccurate current-behavior doc or put planned work under
|
||||
`docs/roadmap/`.
|
||||
|
||||
Prefer links to canonical docs instead of repeating full CLI, config, schema, or
|
||||
operations reference material. Keep examples real, tested where practical, and
|
||||
free of secrets or private transcript data.
|
||||
|
||||
## Architectural Invariants
|
||||
|
||||
- Keep core/domain logic separate from CLI, config, filesystem, reporting, and
|
||||
other adapter concerns.
|
||||
- Keep modules narrowly scoped, explicitly configured, and composable by
|
||||
registry name.
|
||||
- Preserve deterministic ordering, final segment ID assignment, and schema
|
||||
validation before output acceptance.
|
||||
- Keep `trim` and `normalize` artifact-level; do not run merge modules from
|
||||
those commands.
|
||||
- Keep public output schemas validated through `schema`.
|
||||
- Keep optional reports ordered, concise, and diagnostic.
|
||||
- Avoid broad dependencies without a concrete maintainability benefit.
|
||||
- Do not document unimplemented behavior outside `docs/roadmap/`.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
The implemented application does not perform transcription, audio diarization,
|
||||
speaker inference from audio or text, summarization, daemon operation, remote
|
||||
storage, dynamic external plugin loading, or concurrent pipeline execution.
|
||||
|
||||
The architecture policy is not a package-by-package reference, CLI manual,
|
||||
config reference, schema reference, or roadmap.
|
||||
356
docs/policy/documentation.md
Normal file
356
docs/policy/documentation.md
Normal file
@@ -0,0 +1,356 @@
|
||||
# Go Project Documentation Policy
|
||||
|
||||
## Purpose
|
||||
|
||||
Project documentation must help four audiences:
|
||||
|
||||
1. users who need to run the application;
|
||||
2. administrators/operators who need to configure and operate it;
|
||||
3. developers who need to understand and change it safely;
|
||||
4. LLM coding agents that need clear scope, boundaries, and invariants.
|
||||
|
||||
Docs should be accurate, concise, task-oriented, and organized by audience. Prefer links to canonical docs over repetition.
|
||||
|
||||
## Core Rules
|
||||
|
||||
### 1. Keep docs concise
|
||||
|
||||
Each document should cover a defined scope and only the essentials for that scope.
|
||||
|
||||
Avoid:
|
||||
- long background explanations;
|
||||
- repeated reference material;
|
||||
- implementation detail in user-facing docs;
|
||||
- aspirational language outside roadmap docs;
|
||||
- verbose examples where one minimal example is clearer.
|
||||
|
||||
### 2. Document only implemented behavior outside roadmap files
|
||||
|
||||
Unimplemented, planned, aspirational, experimental, or future work may be described only under:
|
||||
|
||||
- `docs/roadmap/`
|
||||
|
||||
No other documentation file, including `README.md`, should describe code, features, modules, stages, commands, config fields, or behaviors that do not currently exist.
|
||||
|
||||
If a feature is partial, non-roadmap docs may describe only the implemented portion and its current boundary.
|
||||
|
||||
### 3. Use canonical homes
|
||||
|
||||
Each type of information should have one canonical location.
|
||||
|
||||
Canonical homes:
|
||||
|
||||
- project purpose and quickstart: `README.md`
|
||||
- development principles: `docs/policy/architecture.md`
|
||||
- configuration reference: `docs/config.md`
|
||||
- CLI reference: `docs/cli.md`
|
||||
- operations and recovery: `docs/operations.md`
|
||||
- troubleshooting: `docs/troubleshooting.md`
|
||||
- implemented internals: `docs/internal/`
|
||||
- future work: `docs/roadmap/`
|
||||
- contributor workflow: `docs/policy/development.md`
|
||||
- copyable examples: `examples/`
|
||||
|
||||
Other files should summarize briefly and link to the canonical source.
|
||||
|
||||
### 4. Keep examples real
|
||||
|
||||
Examples should be valid, maintained, and free of secrets.
|
||||
|
||||
Where practical:
|
||||
- example configs should load successfully;
|
||||
- example commands should match real CLI syntax;
|
||||
- important examples should be covered by tests.
|
||||
|
||||
## Documentation Profiles
|
||||
|
||||
All projects require:
|
||||
|
||||
- `README.md`
|
||||
- `docs/policy/architecture.md`
|
||||
|
||||
Additional docs depend on the project.
|
||||
|
||||
### Small library
|
||||
|
||||
Recommended:
|
||||
- `docs/policy/development.md`, if contributor conventions are non-obvious
|
||||
|
||||
### Simple CLI
|
||||
|
||||
Required:
|
||||
- `docs/cli.md`
|
||||
|
||||
Recommended:
|
||||
- `docs/policy/development.md`
|
||||
|
||||
### Config-driven CLI
|
||||
|
||||
Required:
|
||||
- `docs/cli.md`
|
||||
- `docs/config.md`
|
||||
|
||||
Recommended:
|
||||
- `examples/`
|
||||
- `docs/policy/development.md`
|
||||
|
||||
### Stateful or operator-facing application
|
||||
|
||||
Required:
|
||||
- `docs/cli.md`, if CLI-based
|
||||
- `docs/config.md`, if config-driven
|
||||
- `docs/operations.md`
|
||||
|
||||
Recommended:
|
||||
- `docs/troubleshooting.md`
|
||||
- `examples/`
|
||||
- `docs/policy/development.md`
|
||||
|
||||
### Modular, staged, service-oriented, or orchestration application
|
||||
|
||||
Required:
|
||||
- `docs/cli.md`, if CLI-based
|
||||
- `docs/config.md`, if config-driven
|
||||
- `docs/operations.md`
|
||||
- `docs/internal/`
|
||||
- `docs/policy/development.md`
|
||||
|
||||
Recommended:
|
||||
- `docs/troubleshooting.md`
|
||||
- validated examples under `examples/`
|
||||
|
||||
## Required Documents
|
||||
|
||||
### README.md
|
||||
|
||||
**Audience:** users, administrators, operators
|
||||
|
||||
The README is the outward-facing project orientation page.
|
||||
|
||||
It should include, in order:
|
||||
|
||||
1. concise description;
|
||||
2. elevator pitch;
|
||||
3. shortest useful command or usage example;
|
||||
4. links to targeted docs.
|
||||
|
||||
The README should be short. It is not a manual.
|
||||
|
||||
The “shortest useful command” means the simplest command that performs the project’s core use case. (It does not mean `app --help`.)
|
||||
|
||||
### docs/policy/architecture.md
|
||||
|
||||
**Audience:** developers, LLM coding agents
|
||||
|
||||
`docs/policy/architecture.md` is required for every project.
|
||||
|
||||
It is an inward-facing development policy document. It should describe how the project is intended to be built and changed.
|
||||
|
||||
It should include:
|
||||
|
||||
- project shape;
|
||||
- core design principles;
|
||||
- package and boundary philosophy;
|
||||
- state/persistence philosophy, if applicable;
|
||||
- external integration philosophy, if applicable;
|
||||
- error-handling and logging principles;
|
||||
- testing expectations;
|
||||
- documentation expectations;
|
||||
- architectural invariants;
|
||||
- explicit non-goals, if useful.
|
||||
|
||||
For small projects, this file may be brief. It may simply state that the project is intentionally narrow, monolithic, and dependency-light.
|
||||
|
||||
### docs/policy/development.md
|
||||
|
||||
**Audience:** developers, LLM coding agents
|
||||
|
||||
Required for projects maintained by humans and LLM coding agents.
|
||||
|
||||
It should include:
|
||||
|
||||
- repository layout;
|
||||
- build/test commands;
|
||||
- coding conventions;
|
||||
- dependency policy;
|
||||
- how to add config fields;
|
||||
- how to add CLI flags;
|
||||
- how to add stages/modules/adapters, if applicable;
|
||||
- how to update examples;
|
||||
- documentation update expectations.
|
||||
|
||||
### docs/config.md
|
||||
|
||||
**Audience:** administrators, operators, advanced users
|
||||
|
||||
Required for applications with configuration files.
|
||||
|
||||
It should include, in order:
|
||||
|
||||
1. config file locations and discovery precedence;
|
||||
2. minimal working config;
|
||||
3. production-oriented config;
|
||||
4. full configuration reference;
|
||||
5. secrets handling, if applicable;
|
||||
6. links to maintained examples.
|
||||
|
||||
The full configuration reference should be canonical.
|
||||
|
||||
### docs/cli.md
|
||||
|
||||
**Audience:** users, administrators, operators
|
||||
|
||||
Required for CLI applications.
|
||||
|
||||
It should include, in order:
|
||||
|
||||
1. shortest useful command;
|
||||
2. command overview;
|
||||
3. complete flag reference;
|
||||
4. common workflows;
|
||||
5. diagnostic or recovery commands, if applicable.
|
||||
|
||||
Explain when commands are useful, not just their syntax.
|
||||
|
||||
### docs/operations.md
|
||||
|
||||
**Audience:** administrators, operators
|
||||
|
||||
Required for applications that maintain state, support resume behavior, run multiple stages, write durable artifacts, use remote storage, or require recovery procedures.
|
||||
|
||||
It should cover:
|
||||
|
||||
- normal workflow;
|
||||
- filesystem layout;
|
||||
- remote storage layout, if applicable;
|
||||
- logs and manifests;
|
||||
- resume/retry behavior;
|
||||
- cleanup behavior;
|
||||
- archive/backup behavior;
|
||||
- safe recovery procedures;
|
||||
- operational caveats.
|
||||
|
||||
### docs/troubleshooting.md
|
||||
|
||||
**Audience:** administrators, operators
|
||||
|
||||
Recommended once recurring failure modes exist.
|
||||
|
||||
Each entry should include:
|
||||
|
||||
- symptom;
|
||||
- likely cause;
|
||||
- diagnostic command or inspection step;
|
||||
- safe fix;
|
||||
- relevant links.
|
||||
|
||||
### docs/internal/
|
||||
|
||||
**Audience:** developers, LLM coding agents
|
||||
|
||||
Required for modular, staged, service-oriented, or orchestration projects.
|
||||
|
||||
This directory describes implemented internal components. It is not the roadmap.
|
||||
|
||||
Use one file per major component where useful.
|
||||
|
||||
Each component doc should include:
|
||||
|
||||
1. purpose;
|
||||
2. inputs and outputs;
|
||||
3. boundaries;
|
||||
4. config fields used;
|
||||
5. external adapters used;
|
||||
6. state or manifest behavior, if applicable;
|
||||
7. skip/resume behavior, if applicable;
|
||||
8. failure behavior;
|
||||
9. tests to inspect before changing;
|
||||
10. architectural invariants.
|
||||
|
||||
### docs/roadmap/
|
||||
|
||||
**Audience:** maintainers, developers, LLM coding agents
|
||||
|
||||
This is the only place for planned, future, aspirational, experimental, or unimplemented work.
|
||||
|
||||
Roadmap docs should clearly distinguish:
|
||||
|
||||
- proposed work;
|
||||
- accepted plans;
|
||||
- deferred ideas;
|
||||
- rejected ideas;
|
||||
- implementation prompts or task breakdowns, if useful.
|
||||
|
||||
Roadmap docs should not be confused with current behavior.
|
||||
|
||||
### docs/integrations/
|
||||
|
||||
**Audience:** developers, LLM coding agents
|
||||
|
||||
Required for projects that depend on external CLIs, APIs, services, protocols, or file formats where the integration contract is important to maintain.
|
||||
|
||||
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses.
|
||||
|
||||
Use one file per integration where useful.
|
||||
|
||||
## Examples Directory
|
||||
|
||||
Projects with non-trivial configuration or workflows should include `examples/`.
|
||||
|
||||
Useful examples include:
|
||||
|
||||
- minimal working config;
|
||||
- production-oriented config;
|
||||
- full annotated config;
|
||||
- local development config;
|
||||
- remote/object-storage config;
|
||||
- minimal session/input file.
|
||||
|
||||
Examples should be valid, maintained, tested when practical, and linked from relevant docs.
|
||||
|
||||
## Security and Privacy
|
||||
|
||||
Docs and examples must not include:
|
||||
|
||||
- real API keys;
|
||||
- tokens;
|
||||
- passwords;
|
||||
- private keys;
|
||||
- private environment dumps;
|
||||
- sensitive user data;
|
||||
- raw private transcripts;
|
||||
- private infrastructure details unless intentionally public.
|
||||
|
||||
Document secret-handling mechanisms, not actual secret values.
|
||||
|
||||
## Maintenance Rules
|
||||
|
||||
When docs change, verify the affected behavior.
|
||||
|
||||
Where practical:
|
||||
|
||||
- load example config files in tests;
|
||||
- test CLI examples or command parser behavior;
|
||||
- validate documented flags against real flags;
|
||||
- remove stale references;
|
||||
- update links after renames;
|
||||
- keep roadmap content out of non-roadmap docs.
|
||||
|
||||
If documentation and code disagree, fix the documentation and/or open a roadmap item; do not leave aspirational behavior in current-behavior docs.
|
||||
|
||||
Documentation is complete only when it matches the current code.
|
||||
|
||||
## Documentation Change Checklist
|
||||
|
||||
Before merging documentation changes, verify:
|
||||
|
||||
- README is concise and orientation-focused.
|
||||
- `docs/policy/architecture.md` describes development principles.
|
||||
- Future work appears only under `docs/roadmap/`.
|
||||
- User-facing docs avoid unnecessary internals.
|
||||
- Developer-facing docs preserve boundaries and invariants.
|
||||
- Config examples match the schema.
|
||||
- CLI examples match real commands and flags.
|
||||
- Defaults appear in the canonical config reference.
|
||||
- No secrets or private data are included.
|
||||
- Links are accurate.
|
||||
579
docs/roadmap/documentation.md
Normal file
579
docs/roadmap/documentation.md
Normal file
@@ -0,0 +1,579 @@
|
||||
# Documentation Roadmap
|
||||
|
||||
## Purpose
|
||||
|
||||
This roadmap defines the work required to bring seriatim's documentation into
|
||||
compliance with `docs/policy/documentation.md` and the current implementation.
|
||||
It is grounded in the repository as it exists now: the Go CLI, config loading,
|
||||
pipeline modules, artifact commands, schemas, reports, samples, and tests.
|
||||
|
||||
Outside `docs/roadmap/`, documentation must describe only implemented
|
||||
behavior. Planned, future, deprecated, experimental, or unimplemented work must
|
||||
remain in roadmap documents until the code exists.
|
||||
|
||||
## Repository Documentation Inventory
|
||||
|
||||
- `README.md` - keep and rewrite. It currently mixes project orientation,
|
||||
quickstart, full CLI reference, config/env reference, file formats, module
|
||||
internals, limitations, and release build notes. Policy says README should be
|
||||
concise and link to canonical docs.
|
||||
- `docs/policy/documentation.md` - keep and lightly update only if the policy
|
||||
itself changes. It is the controlling documentation layout and maintenance
|
||||
policy.
|
||||
- `docs/policy/architecture.md` - keep and lightly update as implementation
|
||||
changes. It is the canonical development architecture policy.
|
||||
- Root `architecture.md` - delete after salvage, or move only truly roadmap
|
||||
material into `docs/roadmap/`. It is in the wrong canonical home and contains
|
||||
future-oriented and aspirational claims.
|
||||
- `docs/roadmap/documentation.md` - create new. This file is the planning
|
||||
artifact for the documentation migration.
|
||||
- `samples/` - split or move after audit. It contains sample raw transcripts,
|
||||
merged artifacts, reports, `speakers.yml`, and `autocorrect.yml`, but
|
||||
copyable examples belong under `examples/`. The raw sample data is large and
|
||||
should be reviewed for privacy and maintainability before linking from docs.
|
||||
- `schema/*.schema.json` - keep. These are public output contracts and should
|
||||
be linked from documentation instead of duplicated in full.
|
||||
- Missing canonical docs - create `docs/cli.md`, `docs/config.md`,
|
||||
`docs/operations.md`, `docs/policy/development.md`, `docs/internal/`, and
|
||||
likely `docs/troubleshooting.md`, `docs/integrations/`, and `examples/`.
|
||||
|
||||
## Policy Compliance Assessment
|
||||
|
||||
Required documents missing for seriatim's current shape as a modular, staged,
|
||||
CLI/config-driven project:
|
||||
|
||||
- `docs/cli.md`
|
||||
- `docs/config.md`
|
||||
- `docs/operations.md`
|
||||
- `docs/internal/`
|
||||
- `docs/policy/development.md`
|
||||
|
||||
Recommended documents and directories missing:
|
||||
|
||||
- `docs/troubleshooting.md`
|
||||
- maintained copyable examples under `examples/`
|
||||
- concise integration notes under `docs/integrations/`
|
||||
|
||||
Existing compliance issues:
|
||||
|
||||
- `README.md` is too broad for its canonical scope. It should keep project
|
||||
purpose, quickstart, and links, then delegate CLI, config, operations,
|
||||
internals, and schema details.
|
||||
- Root `architecture.md` is stale and in the wrong home. It includes future
|
||||
input methods and formats, future output formats, dynamic plugin speculation,
|
||||
an LLM non-goal, interface sketches that diverge from code, and other
|
||||
development-policy content now covered by `docs/policy/architecture.md`.
|
||||
- Non-roadmap docs should not carry forward claims about future defaults,
|
||||
future formats, unimplemented plugin systems, or unimplemented alternate
|
||||
input/output methods.
|
||||
- Historical or deprecated wording, such as the old speaker map format, should
|
||||
move out of the README unless it is still needed in troubleshooting or a
|
||||
narrow migration note.
|
||||
- There is no `examples/` directory. `samples/` exists but is not the canonical
|
||||
examples home and should not be treated as copyable public examples without a
|
||||
privacy and size audit.
|
||||
- Links need verification after migration: README should link to all new
|
||||
canonical docs, docs should link to schema files and maintained examples, and
|
||||
no doc should link to the deleted root `architecture.md`.
|
||||
|
||||
## Target Documentation Set
|
||||
|
||||
### `README.md`
|
||||
|
||||
- Audience: users, administrators, and operators.
|
||||
- Purpose: project orientation and shortest useful quickstart.
|
||||
- Canonical scope: concise project purpose, elevator pitch, one minimal command,
|
||||
and links to targeted docs.
|
||||
- Recommended outline: project description; shortest merge command; command
|
||||
summary; links to CLI, config, operations, architecture, development, schemas,
|
||||
examples, and troubleshooting.
|
||||
- Source of truth: current `README.md`, `internal/cli`, `internal/config`,
|
||||
`cmd/seriatim/main.go`, and CLI tests.
|
||||
- Acceptance criteria: no full flag tables, no full config reference, no module
|
||||
manual, no future-feature claims, and all links resolve.
|
||||
|
||||
### `docs/cli.md`
|
||||
|
||||
- Audience: users, administrators, and operators.
|
||||
- Purpose: canonical CLI reference and workflows.
|
||||
- Canonical scope: shortest useful command, command overview, complete flag
|
||||
reference, common workflows, diagnostics and report flags.
|
||||
- Recommended outline: shortest useful command; global flags; `merge`; `trim`;
|
||||
`normalize`; common workflows; exit/error behavior; links to config,
|
||||
operations, examples, and schemas.
|
||||
- Source of truth: `internal/cli/root.go`, `internal/cli/merge.go`,
|
||||
`internal/cli/trim.go`, `internal/cli/normalize.go`, `internal/config`, and
|
||||
`internal/cli/*_test.go`.
|
||||
- Acceptance criteria: every documented flag, default, and required/mutually
|
||||
exclusive rule matches code; package internals are linked rather than
|
||||
explained in depth.
|
||||
|
||||
### `docs/config.md`
|
||||
|
||||
- Audience: administrators, operators, and advanced users.
|
||||
- Purpose: canonical runtime configuration reference.
|
||||
- Canonical scope: environment variables, default module lists, output schema
|
||||
selection, `speakers.yml`, `autocorrect.yml`, path validation, and precedence.
|
||||
- Recommended outline: config surfaces; output schema precedence; merge module
|
||||
defaults; environment variables; speaker map YAML; autocorrect YAML; path and
|
||||
validation rules; links to examples.
|
||||
- Source of truth: `internal/config/config.go`, `internal/speaker/map.go`,
|
||||
`internal/autocorrect/autocorrect.go`, `internal/config/config_test.go`,
|
||||
`internal/speaker/map_test.go`, and `internal/autocorrect/autocorrect_test.go`.
|
||||
- Acceptance criteria: all config fields and `SERIATIM_*` env vars match code;
|
||||
unsupported config files or unimplemented formats are not described.
|
||||
|
||||
### `docs/operations.md`
|
||||
|
||||
- Audience: administrators and operators.
|
||||
- Purpose: operational behavior for running commands safely.
|
||||
- Canonical scope: file workflow, filesystem layout expectations, output and
|
||||
report files, retry behavior, cleanup, validation failures, and operational
|
||||
caveats.
|
||||
- Recommended outline: normal workflow; input/output/report files; no durable
|
||||
state; failure and retry behavior; reports and diagnostics; cleanup; privacy
|
||||
considerations for transcript artifacts.
|
||||
- Source of truth: `cmd/seriatim/main.go`, `internal/cli`, `internal/config`,
|
||||
`internal/report`, `internal/builtin/output.go`, `internal/normalize`, and
|
||||
trim/merge/normalize CLI tests.
|
||||
- Acceptance criteria: clearly states there is no daemon, database, resume
|
||||
state, remote storage, or background job state; does not invent recovery
|
||||
workflows.
|
||||
|
||||
### `docs/policy/development.md`
|
||||
|
||||
- Audience: developers and coding agents.
|
||||
- Purpose: contributor workflow and change guidance.
|
||||
- Canonical scope: repository layout, build/test commands, coding conventions,
|
||||
dependency policy, adding flags/config fields/modules/docs/examples.
|
||||
- Recommended outline: repo layout; local checks; coding conventions; adding
|
||||
CLI flags; adding config/env vars; adding modules/stages; schema changes;
|
||||
examples and documentation updates.
|
||||
- Source of truth: `docs/policy/documentation.md`,
|
||||
`docs/policy/architecture.md`, `go.mod`, package layout, and test layout.
|
||||
- Acceptance criteria: includes `go test ./...`; states there is no current
|
||||
Makefile, taskfile, linter config, or automated doc checker; aligns with the
|
||||
architecture policy.
|
||||
|
||||
### `docs/internal/pipeline.md`
|
||||
|
||||
- Audience: developers and coding agents.
|
||||
- Purpose: implemented merge pipeline internals.
|
||||
- Canonical scope: registry, stage interfaces, preprocessing state transitions,
|
||||
module order, report event accumulation, final output/report writing.
|
||||
- Recommended outline: purpose; inputs and outputs; stage contracts; registry
|
||||
resolution; execution order; config fields used; adapters; failure behavior;
|
||||
tests; invariants.
|
||||
- Source of truth: `internal/pipeline`, `internal/builtin`, `internal/model`,
|
||||
`internal/report`, `internal/pipeline/runner_test.go`,
|
||||
`internal/builtin/*_test.go`, and `internal/cli/merge_test.go`.
|
||||
- Acceptance criteria: describes only implemented sequential execution; does
|
||||
not document concurrency, plugins, or future formats.
|
||||
|
||||
### `docs/internal/artifacts.md`
|
||||
|
||||
- Audience: developers and coding agents.
|
||||
- Purpose: public artifact conversion and validation internals.
|
||||
- Canonical scope: schema structs, embedded JSON Schemas, conversion from merged
|
||||
model, trim/normalize artifact handling, and output validation.
|
||||
- Recommended outline: artifact contracts; schema selection; conversion;
|
||||
validation; trim projection; normalize canonicalization; tests; invariants.
|
||||
- Source of truth: `schema`, `internal/artifact`, `internal/trim`,
|
||||
`internal/normalize`, and related tests.
|
||||
- Acceptance criteria: links to `schema/*.schema.json`; does not duplicate full
|
||||
schemas or describe unavailable output formats.
|
||||
|
||||
### `docs/internal/modules.md`
|
||||
|
||||
- Audience: developers and coding agents.
|
||||
- Purpose: implemented built-in module behavior and boundaries.
|
||||
- Canonical scope: `json-files`, preprocessing modules, chronological merge,
|
||||
postprocessing modules, and JSON output writer.
|
||||
- Recommended outline: module list; inputs/outputs; config fields used; allowed
|
||||
side effects; ordering constraints; failure behavior; tests; invariants.
|
||||
- Source of truth: `internal/builtin`, `internal/overlap`, `internal/coalesce`,
|
||||
`internal/danglers`, `internal/backchannel`, `internal/filler`,
|
||||
`internal/autocorrect`, and package tests.
|
||||
- Acceptance criteria: avoids full CLI/config duplication; identifies
|
||||
order-sensitive transforms that must run before `assign-ids`.
|
||||
|
||||
### `docs/troubleshooting.md`
|
||||
|
||||
- Audience: users, administrators, and operators.
|
||||
- Purpose: common failure symptoms and safe fixes.
|
||||
- Canonical scope: implemented validation and runtime failures observed in
|
||||
error paths and tests.
|
||||
- Recommended outline: invalid JSON/input shape; missing required flags; invalid
|
||||
output parent directory; invalid speaker/autocorrect YAML; unknown module;
|
||||
invalid output schema; invalid trim selector; schema validation failure;
|
||||
report write failure.
|
||||
- Source of truth: `internal/config`, `internal/cli/*_test.go`,
|
||||
`internal/trim/*_test.go`, `internal/normalize/*_test.go`,
|
||||
`internal/speaker/*_test.go`, and `internal/autocorrect/*_test.go`.
|
||||
- Acceptance criteria: each entry has symptom, likely cause, inspection step,
|
||||
safe fix, and link; no speculative failure modes.
|
||||
|
||||
### `docs/integrations/whisperx-json.md`
|
||||
|
||||
- Audience: developers and coding agents.
|
||||
- Purpose: external input JSON contract used by `merge`.
|
||||
- Canonical scope: the supported WhisperX-like subset only.
|
||||
- Recommended outline: top-level shape; required segment fields; optional word
|
||||
timing fields; validation/failure behavior; how word timing affects overlap
|
||||
resolution; links to CLI and examples.
|
||||
- Source of truth: `internal/builtin/input.go`, merge CLI tests, and README
|
||||
input-format material.
|
||||
- Acceptance criteria: does not attempt to document full WhisperX behavior or
|
||||
unsupported input formats.
|
||||
|
||||
### `docs/integrations/output-schemas.md`
|
||||
|
||||
- Audience: developers, coding agents, and artifact consumers.
|
||||
- Purpose: orientation to public JSON output contracts.
|
||||
- Canonical scope: minimal/intermediate/full schema roles and links to schema
|
||||
files.
|
||||
- Recommended outline: schema selection; minimal; intermediate; full; semantic
|
||||
invariants; validation APIs; links to `schema/*.schema.json`.
|
||||
- Source of truth: `schema/output.go`, `schema/*.schema.json`,
|
||||
`schema/output_test.go`, and `internal/artifact`.
|
||||
- Acceptance criteria: links to machine-readable schemas instead of copying
|
||||
them in full.
|
||||
|
||||
### `examples/`
|
||||
|
||||
- Audience: users, administrators, operators, developers, and coding agents.
|
||||
- Purpose: maintained copyable examples.
|
||||
- Canonical scope: small synthetic inputs and config files for implemented
|
||||
commands only.
|
||||
- Source of truth: examples created during the documentation migration and
|
||||
validated through actual command invocations.
|
||||
- Acceptance criteria: examples are valid, free of secrets/private transcript
|
||||
data, and linked from README, CLI, config, and operations docs.
|
||||
|
||||
## File-by-File Rewrite Guidance
|
||||
|
||||
### README
|
||||
|
||||
Cover what seriatim is, the shortest useful `merge` command, a brief command
|
||||
summary, and links to canonical docs. Avoid full flag tables, config/env
|
||||
reference, module internals, schema examples, troubleshooting details, future
|
||||
formats, or release-history narrative. Inspect `internal/cli`, `internal/config`,
|
||||
and CLI tests before updating commands.
|
||||
|
||||
### CLI Reference
|
||||
|
||||
Document actual `merge`, `trim`, and `normalize` flags from `internal/cli`.
|
||||
Include required flags, defaults, mutually exclusive selector rules, schema
|
||||
selection, report flags, and common workflows. Link to `docs/config.md` for
|
||||
environment variables and YAML formats. Avoid internal package explanations.
|
||||
Inspect `internal/cli/*_test.go` for edge cases and examples.
|
||||
|
||||
### Config Reference
|
||||
|
||||
Document all implemented config surfaces: flags that become config values,
|
||||
`SERIATIM_OUTPUT_SCHEMA`, `SERIATIM_OVERLAP_WORD_RUN_GAP`,
|
||||
`SERIATIM_OVERLAP_WORD_RUN_REORDER_WINDOW`,
|
||||
`SERIATIM_BACKCHANNEL_MAX_DURATION`, `SERIATIM_FILLER_MAX_DURATION`, module
|
||||
lists, output schemas, `speakers.yml`, and `autocorrect.yml`. Avoid command
|
||||
tutorials and unimplemented config files. Inspect `internal/config`,
|
||||
`internal/speaker`, `internal/autocorrect`, and tests.
|
||||
|
||||
### Operations
|
||||
|
||||
Document filesystem-only command execution, output/report artifacts, validation
|
||||
failures, retry behavior, and cleanup. Explicitly say there is no daemon,
|
||||
database, remote storage, resume state, or background job state. Avoid
|
||||
unimplemented recovery procedures.
|
||||
|
||||
### Development Policy
|
||||
|
||||
Document repository layout, `go test ./...`, package conventions,
|
||||
standard-library-first dependency guidance, how to add flags/config/modules,
|
||||
and documentation update expectations. State that no Makefile, taskfile,
|
||||
linter config, or automated documentation checker currently exists.
|
||||
|
||||
### Internal Docs
|
||||
|
||||
Keep internal docs behavior-level and concise. Describe implemented inputs,
|
||||
outputs, boundaries, config fields used, adapters, failure behavior, tests, and
|
||||
invariants. Avoid future plugins, future input/output formats, concurrency, or
|
||||
duplicating CLI/config reference material.
|
||||
|
||||
### Root `architecture.md`
|
||||
|
||||
Do not carry forward future input methods, future formats, future output
|
||||
formats, LLM text, dynamic plugin speculation, or interface sketches that
|
||||
diverge from code. Salvage only current-behavior details that are not already
|
||||
covered in `docs/policy/architecture.md` and move any legitimate future ideas
|
||||
under `docs/roadmap/`.
|
||||
|
||||
## Examples Plan
|
||||
|
||||
Create small synthetic examples under `examples/` rather than relying on the
|
||||
current large `samples/raw` data.
|
||||
|
||||
- `examples/minimal-merge/`
|
||||
- Purpose: shortest complete merge workflow with two small raw JSON files and
|
||||
optional `speakers.yml`.
|
||||
- Expected validity check: run `go run ./cmd/seriatim merge` with the example
|
||||
files and validate JSON output is produced.
|
||||
- Docs to link: README, `docs/cli.md`, `docs/config.md`,
|
||||
`docs/operations.md`.
|
||||
- `examples/normalize/`
|
||||
- Purpose: normalize object-with-`segments` and bare segment array inputs.
|
||||
- Expected validity check: run `go run ./cmd/seriatim normalize` for both
|
||||
shapes.
|
||||
- Docs to link: `docs/cli.md`, `docs/operations.md`, and any Audita/bare
|
||||
array integration note if created.
|
||||
- `examples/trim/`
|
||||
- Purpose: trim a small existing seriatim artifact by `--keep` and/or
|
||||
`--remove`.
|
||||
- Expected validity check: run `go run ./cmd/seriatim trim` and validate
|
||||
sequential retained IDs.
|
||||
- Docs to link: `docs/cli.md`, `docs/operations.md`.
|
||||
- `examples/speakers.yml` and `examples/autocorrect.yml`
|
||||
- Purpose: copyable YAML rule examples if linked from `docs/config.md`.
|
||||
- Expected validity check: load through merge command or package tests.
|
||||
- Docs to link: `docs/config.md`, `docs/cli.md`.
|
||||
|
||||
Do not invent examples for unimplemented input methods, output formats,
|
||||
services, or plugin systems. Do not reuse `samples/raw` as public examples
|
||||
without privacy and size review.
|
||||
|
||||
## Internal Documentation Plan
|
||||
|
||||
### Pipeline
|
||||
|
||||
- Path: `docs/internal/pipeline.md`
|
||||
- Purpose: document implemented merge pipeline orchestration.
|
||||
- Inputs and outputs: `config.Config`, raw transcripts, canonical transcripts,
|
||||
merged transcript, selected public artifact, optional report.
|
||||
- Boundaries: registry and runner orchestration; no CLI flag parsing; no schema
|
||||
details beyond output selection.
|
||||
- Config fields used: input reader, module lists, output modules, output schema,
|
||||
input/output/report files, timing thresholds passed through modules.
|
||||
- Adapters used: input reader, output writer, report writer.
|
||||
- Failure behavior: unknown modules, invalid preprocessing state, stage errors,
|
||||
output/report write failures.
|
||||
- Tests to inspect: `internal/pipeline/runner_test.go`,
|
||||
`internal/builtin/*_test.go`, `internal/cli/merge_test.go`.
|
||||
- Architectural invariants: deterministic sequential stage order, explicit
|
||||
raw-to-canonical preprocessing state, output validation before acceptance.
|
||||
|
||||
### Artifacts and Schemas
|
||||
|
||||
- Path: `docs/internal/artifacts.md`
|
||||
- Purpose: document public artifact conversion and validation internals.
|
||||
- Inputs and outputs: merged model, schema structs, serialized JSON artifacts,
|
||||
parsed trim/normalize artifacts.
|
||||
- Boundaries: conversion and validation only; CLI docs own user-facing flags.
|
||||
- Config fields used: output schema, output modules, input files for metadata.
|
||||
- Adapters used: embedded JSON Schema files and JSON encoders/decoders.
|
||||
- Failure behavior: schema validation errors, unsupported artifact/schema
|
||||
conversion, invalid IDs/timing.
|
||||
- Tests to inspect: `schema/output_test.go`,
|
||||
`internal/artifact/transcript_test.go`, `internal/trim/*_test.go`,
|
||||
`internal/normalize/*_test.go`.
|
||||
- Architectural invariants: sequential IDs, selected schema validation, no
|
||||
internal-only fields in public schemas.
|
||||
|
||||
### Built-In Modules
|
||||
|
||||
- Path: `docs/internal/modules.md`
|
||||
- Purpose: document implemented module responsibilities and ordering
|
||||
constraints.
|
||||
- Inputs and outputs: raw transcripts, preprocess state, merged transcript,
|
||||
report events, selected JSON output.
|
||||
- Boundaries: module behavior only; no full CLI/config reference.
|
||||
- Config fields used: speaker file, autocorrect file, coalesce gap, overlap word
|
||||
gap, word run reorder window, backchannel/filler max durations.
|
||||
- Adapters used: JSON input/output, speaker YAML, autocorrect YAML, report
|
||||
events.
|
||||
- Failure behavior: input validation errors, invalid YAML, unknown module names,
|
||||
invalid output schema before write.
|
||||
- Tests to inspect: `internal/builtin`, `internal/overlap`,
|
||||
`internal/coalesce`, `internal/danglers`, `internal/backchannel`,
|
||||
`internal/filler`, `internal/autocorrect`, and CLI merge tests.
|
||||
- Architectural invariants: order-sensitive transforms run before `assign-ids`;
|
||||
modules stay narrow and explicitly configured.
|
||||
|
||||
### Trim
|
||||
|
||||
- Path: include in `docs/internal/artifacts.md` or create
|
||||
`docs/internal/trim.md` if artifacts doc grows too large.
|
||||
- Purpose: document artifact-level segment projection.
|
||||
- Inputs and outputs: existing seriatim artifact, selector, selected output
|
||||
schema, optional report.
|
||||
- Boundaries: no merge postprocessors; no raw WhisperX input.
|
||||
- Config fields used: input/output/report files, keep/remove selector,
|
||||
optional output schema, allow-empty.
|
||||
- Adapters used: file I/O in CLI, artifact parsing/validation, report writer.
|
||||
- Failure behavior: malformed selector, invalid artifact, missing selected IDs,
|
||||
non-sequential input IDs, empty output unless allowed, unsupported schema
|
||||
up-conversion.
|
||||
- Tests to inspect: `internal/trim/*_test.go`, `internal/cli/trim_test.go`.
|
||||
- Architectural invariants: preserve transcript order, renumber retained IDs,
|
||||
recompute full-schema overlap groups, never run merge modules.
|
||||
|
||||
### Normalize
|
||||
|
||||
- Path: include in `docs/internal/artifacts.md` or create
|
||||
`docs/internal/normalize.md` if artifacts doc grows too large.
|
||||
- Purpose: document artifact-level transcript canonicalization.
|
||||
- Inputs and outputs: transcript-like JSON object or bare array, selected
|
||||
seriatim output schema, optional report.
|
||||
- Boundaries: no merge preprocessing or postprocessing modules.
|
||||
- Config fields used: input/output/report files, output schema, output modules.
|
||||
- Adapters used: file I/O, JSON parsing, schema validation, report writer.
|
||||
- Failure behavior: invalid JSON, unsupported top-level shape, invalid timing
|
||||
after repair, unsupported output module/schema, report write failure.
|
||||
- Tests to inspect: `internal/normalize/*_test.go`,
|
||||
`internal/cli/normalize_test.go`.
|
||||
- Architectural invariants: deterministic repair/sort/ID assignment, no
|
||||
transcript text in normalize report events, no merge modules.
|
||||
|
||||
## Integration Documentation Plan
|
||||
|
||||
- `docs/integrations/whisperx-json.md`
|
||||
- External system or contract: WhisperX-like JSON transcript subset.
|
||||
- Current usage: `merge` reads a top-level `segments` array with required
|
||||
segment timing/text and optional word timing.
|
||||
- Version or compatibility notes: no explicit WhisperX version is encoded in
|
||||
the repository; document only the accepted subset.
|
||||
- Document: supported fields, validation, word timing behavior, errors.
|
||||
- Do not document: full WhisperX schema, audio diarization, non-JSON formats.
|
||||
- `docs/integrations/output-schemas.md`
|
||||
- External system or contract: seriatim public JSON output contracts.
|
||||
- Current usage: `merge`, `trim`, and `normalize` emit
|
||||
`seriatim-minimal`, `seriatim-intermediate`, or `seriatim-full`.
|
||||
- Version or compatibility notes: schemas are embedded from `schema/`; release
|
||||
version metadata is injected through build info.
|
||||
- Document: schema roles, semantic invariants, validation APIs, links to
|
||||
schema files.
|
||||
- Do not document: unimplemented output formats or full schema copies.
|
||||
- YAML rule files
|
||||
- Prefer documenting speaker and autocorrect YAML contracts in
|
||||
`docs/config.md`. Create `docs/integrations/yaml-rule-files.md` only if the
|
||||
config reference becomes too large.
|
||||
- Audita-style bare arrays
|
||||
- Cover under `docs/cli.md` normalize behavior unless maintainers need a
|
||||
separate integration note. Do not generalize beyond implemented bare segment
|
||||
arrays.
|
||||
- No external CLI/API/service docs are needed now. The repository implements no
|
||||
external CLI, network API, daemon, remote storage, or service integration.
|
||||
|
||||
## Recommended Implementation Sequence
|
||||
|
||||
### Stage 1: Write Documentation Roadmap
|
||||
|
||||
- Goal: create this roadmap.
|
||||
- Files: `docs/roadmap/documentation.md`.
|
||||
- Repository areas to inspect: documentation policy, architecture policy,
|
||||
README, root `architecture.md`, CLI/config/pipeline/schema/report/tests.
|
||||
- Acceptance criteria: roadmap exists, no other files changed by this stage,
|
||||
and the roadmap is action-oriented.
|
||||
- Suggested validation commands: `go test ./...`; `git status --short`.
|
||||
- Prompt size: one implementation prompt.
|
||||
|
||||
### Stage 2: User-Facing Canonical Docs and Slim README
|
||||
|
||||
- Goal: move user reference material out of README into canonical docs.
|
||||
- Files: update `README.md`; create `docs/cli.md` and `docs/config.md`.
|
||||
- Repository areas to inspect: `internal/cli`, `internal/config`,
|
||||
`internal/speaker`, `internal/autocorrect`, CLI/config tests.
|
||||
- Acceptance criteria: README is concise; CLI/config docs match flags, defaults,
|
||||
env vars, YAML formats, and validation; no roadmap-only content appears.
|
||||
- Suggested validation commands: `go test ./...`;
|
||||
`go run ./cmd/seriatim --help`;
|
||||
`go run ./cmd/seriatim merge --help`;
|
||||
`go run ./cmd/seriatim trim --help`;
|
||||
`go run ./cmd/seriatim normalize --help`;
|
||||
stale-term grep from the validation plan.
|
||||
- Prompt size: one prompt if concise; split if README rewrite or config
|
||||
reference grows too large.
|
||||
|
||||
### Stage 3: Operations and Troubleshooting
|
||||
|
||||
- Goal: document runtime operation, reports, failure behavior, and common fixes.
|
||||
- Files: create `docs/operations.md` and `docs/troubleshooting.md`.
|
||||
- Repository areas to inspect: `cmd/seriatim/main.go`, `internal/cli`,
|
||||
`internal/config`, `internal/report`, output writer, normalize/trim/merge
|
||||
tests.
|
||||
- Acceptance criteria: docs describe filesystem-only operation and current
|
||||
failure modes; no daemon, resume, remote storage, or recovery behavior is
|
||||
invented.
|
||||
- Suggested validation commands: `go test ./...`; manual link review.
|
||||
- Prompt size: one prompt.
|
||||
|
||||
### Stage 4: Developer and Internal Docs
|
||||
|
||||
- Goal: create developer workflow and implemented internal component docs.
|
||||
- Files: create `docs/policy/development.md`,
|
||||
`docs/internal/pipeline.md`, `docs/internal/artifacts.md`, and
|
||||
`docs/internal/modules.md`.
|
||||
- Repository areas to inspect: architecture policy, pipeline, modules, schema,
|
||||
artifact conversion, trim/normalize packages, tests.
|
||||
- Acceptance criteria: docs preserve boundaries, avoid CLI/config duplication,
|
||||
and identify tests/invariants for future changes.
|
||||
- Suggested validation commands: `go test ./...`; grep for unimplemented
|
||||
future-format/plugin/concurrency claims outside roadmap.
|
||||
- Prompt size: split into development policy and internal docs if needed.
|
||||
|
||||
### Stage 5: Integrations and Examples
|
||||
|
||||
- Goal: add concise integration notes and maintained synthetic examples.
|
||||
- Files: create `docs/integrations/whisperx-json.md`,
|
||||
`docs/integrations/output-schemas.md`, and `examples/*`; decide whether
|
||||
`samples/` should remain separate.
|
||||
- Repository areas to inspect: `internal/builtin/input.go`, `schema`,
|
||||
`internal/artifact`, CLI tests, existing `samples/`.
|
||||
- Acceptance criteria: examples are small, synthetic, valid, and linked from
|
||||
relevant docs; integration docs document only implemented contracts.
|
||||
- Suggested validation commands: `go test ./...`; run documented example
|
||||
`go run` commands; validate example YAML through command paths.
|
||||
- Prompt size: split if examples need tests or sample cleanup decisions.
|
||||
|
||||
### Stage 6: Stale Documentation Cleanup
|
||||
|
||||
- Goal: remove wrong-home and stale documentation after canonical replacements
|
||||
exist.
|
||||
- Files: delete or relocate root `architecture.md`; remove stale material from
|
||||
README; update links across docs.
|
||||
- Repository areas to inspect: all docs, README, roadmap, root files.
|
||||
- Acceptance criteria: no links to deleted root `architecture.md`; no
|
||||
unimplemented behavior outside `docs/roadmap/`; canonical homes are respected.
|
||||
- Suggested validation commands: `go test ./...`; stale-term grep; manual link
|
||||
check; `git status --short`.
|
||||
- Prompt size: one prompt.
|
||||
|
||||
## Validation Plan
|
||||
|
||||
Use these checks during or after documentation migration:
|
||||
|
||||
- Run `go test ./...`.
|
||||
- Run `go run ./cmd/seriatim --help`.
|
||||
- Run `go run ./cmd/seriatim merge --help`.
|
||||
- Run `go run ./cmd/seriatim trim --help`.
|
||||
- Run `go run ./cmd/seriatim normalize --help`.
|
||||
- Once examples exist, run each documented example command and verify output is
|
||||
produced in a temporary path.
|
||||
- Load example YAML through the merge command or package tests.
|
||||
- Validate example JSON through existing CLI/schema paths where practical.
|
||||
- Grep outside `docs/roadmap/` for stale or roadmap-only terms:
|
||||
`Future input`, `Future output`, `LLM`, `plugin`, `SRT`, `VTT`, `.tar.gz`,
|
||||
`URI`, `old format`, `not implemented yet`, and
|
||||
`runtime default may change`.
|
||||
- Manually check links unless a link checker is added. No automated
|
||||
documentation checker currently exists.
|
||||
- Verify docs and examples contain no secrets, private transcript data, API
|
||||
keys, tokens, passwords, or private infrastructure details.
|
||||
|
||||
## Open Questions
|
||||
|
||||
- Should `samples/` be removed, kept as non-doc sample data, or replaced by
|
||||
small synthetic `examples/`? Recommendation: create small synthetic examples
|
||||
first, then audit `samples/` for privacy, size, and ongoing maintenance before
|
||||
deleting or linking it.
|
||||
- Should Audita-style bare-array normalization have a separate integration doc?
|
||||
Recommendation: cover it in `docs/cli.md` normalize behavior unless a
|
||||
stronger external-contract requirement emerges.
|
||||
39
internal/cli/normalize.go
Normal file
39
internal/cli/normalize.go
Normal file
@@ -0,0 +1,39 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.maximumdirect.net/eric/seriatim/internal/config"
|
||||
"gitea.maximumdirect.net/eric/seriatim/internal/normalize"
|
||||
)
|
||||
|
||||
func newNormalizeCommand() *cobra.Command {
|
||||
var opts config.NormalizeOptions
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "normalize",
|
||||
Short: "Normalize a transcript artifact into a standard seriatim output shape",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
normalizeOpts := opts
|
||||
if !cmd.Flags().Changed("output-schema") {
|
||||
normalizeOpts.OutputSchema = ""
|
||||
}
|
||||
|
||||
cfg, err := config.NewNormalizeConfig(normalizeOpts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return normalize.Run(cmd.Context(), cfg)
|
||||
},
|
||||
}
|
||||
|
||||
flags := cmd.Flags()
|
||||
flags.StringVar(&opts.InputFile, "input-file", "", "input transcript JSON file")
|
||||
flags.StringVar(&opts.OutputFile, "output-file", "", "output transcript JSON file")
|
||||
flags.StringVar(&opts.ReportFile, "report-file", "", "optional report JSON file")
|
||||
flags.StringVar(&opts.OutputSchema, "output-schema", config.DefaultOutputSchema, "output JSON schema: seriatim-minimal, seriatim-intermediate, or seriatim-full")
|
||||
flags.StringVar(&opts.OutputModules, "output-modules", config.DefaultOutputModules, "comma-separated output modules")
|
||||
|
||||
return cmd
|
||||
}
|
||||
522
internal/cli/normalize_test.go
Normal file
522
internal/cli/normalize_test.go
Normal file
@@ -0,0 +1,522 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/seriatim/internal/config"
|
||||
"gitea.maximumdirect.net/eric/seriatim/internal/report"
|
||||
"gitea.maximumdirect.net/eric/seriatim/schema"
|
||||
)
|
||||
|
||||
func TestNormalizeCommandIsRecognized(t *testing.T) {
|
||||
cmd := NewRootCommand()
|
||||
cmd.SetArgs([]string{"normalize", "--help"})
|
||||
if err := cmd.Execute(); err != nil {
|
||||
t.Fatalf("normalize command should be recognized: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeMissingInputFileFails(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
output := filepath.Join(dir, "normalized.json")
|
||||
|
||||
err := executeNormalize(
|
||||
"--output-file", output,
|
||||
)
|
||||
if err == nil {
|
||||
t.Fatal("expected missing input-file error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "--input-file is required") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeMissingOutputFileFails(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeJSONFile(t, dir, "input.json", `{"segments":[]}`)
|
||||
|
||||
err := executeNormalize(
|
||||
"--input-file", input,
|
||||
)
|
||||
if err == nil {
|
||||
t.Fatal("expected missing output-file error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "--output-file is required") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeInvalidOutputSchemaFails(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeJSONFile(t, dir, "input.json", `{"segments":[]}`)
|
||||
output := filepath.Join(dir, "normalized.json")
|
||||
|
||||
err := executeNormalize(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
"--output-schema", "compact",
|
||||
)
|
||||
if err == nil {
|
||||
t.Fatal("expected invalid output schema error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "--output-schema must be one of") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeInvalidOutputModuleFails(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeJSONFile(t, dir, "input.json", `{"segments":[]}`)
|
||||
output := filepath.Join(dir, "normalized.json")
|
||||
|
||||
err := executeNormalize(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
"--output-modules", "yaml",
|
||||
)
|
||||
if err == nil {
|
||||
t.Fatal("expected invalid output module error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "unknown output module") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeDefaultOutputSchemaIsIntermediate(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeJSONFile(t, dir, "input.json", `{
|
||||
"segments": [
|
||||
{"id": 99, "start": 5, "end": 6, "speaker": "Bob", "text": "second", "categories": ["filler"]},
|
||||
{"id": 10, "start": 1, "end": 2, "speaker": "Alice", "text": "first", "categories": ["backchannel"]}
|
||||
]
|
||||
}`)
|
||||
output := filepath.Join(dir, "normalized.json")
|
||||
|
||||
err := executeNormalize(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("normalize failed: %v", err)
|
||||
}
|
||||
|
||||
var transcript schema.IntermediateTranscript
|
||||
readJSON(t, output, &transcript)
|
||||
if transcript.Metadata.OutputSchema != config.OutputSchemaIntermediate {
|
||||
t.Fatalf("output schema = %q, want %q", transcript.Metadata.OutputSchema, config.OutputSchemaIntermediate)
|
||||
}
|
||||
if len(transcript.Segments) != 2 {
|
||||
t.Fatalf("segment count = %d, want 2", len(transcript.Segments))
|
||||
}
|
||||
if transcript.Segments[0].ID != 1 || transcript.Segments[1].ID != 2 {
|
||||
t.Fatalf("segment IDs = %d,%d, want 1,2", transcript.Segments[0].ID, transcript.Segments[1].ID)
|
||||
}
|
||||
if transcript.Segments[0].Text != "first" || transcript.Segments[1].Text != "second" {
|
||||
t.Fatalf("unexpected sort order: %#v", transcript.Segments)
|
||||
}
|
||||
if len(transcript.Segments[0].Categories) != 1 || transcript.Segments[0].Categories[0] != "backchannel" {
|
||||
t.Fatalf("expected categories preserved on first segment, got %#v", transcript.Segments[0].Categories)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeBareArrayInputToIntermediateOutput(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeJSONFile(t, dir, "input.json", `[
|
||||
{"start": 2, "end": 3, "speaker": "Bob", "text": "second"},
|
||||
{"start": 1, "end": 2, "speaker": "Alice", "text": "first"}
|
||||
]`)
|
||||
output := filepath.Join(dir, "normalized.json")
|
||||
|
||||
err := executeNormalize(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
"--output-schema", config.OutputSchemaIntermediate,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("normalize failed: %v", err)
|
||||
}
|
||||
|
||||
var transcript schema.IntermediateTranscript
|
||||
readJSON(t, output, &transcript)
|
||||
if len(transcript.Segments) != 2 {
|
||||
t.Fatalf("segment count = %d, want 2", len(transcript.Segments))
|
||||
}
|
||||
if transcript.Segments[0].Speaker != "Alice" || transcript.Segments[1].Speaker != "Bob" {
|
||||
t.Fatalf("unexpected sorted speakers: %#v", transcript.Segments)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeInputIndexTieBreakerIsDeterministic(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeJSONFile(t, dir, "input.json", `[
|
||||
{"start": 1, "end": 2, "speaker": "Zulu", "text": "first in"},
|
||||
{"start": 1, "end": 2, "speaker": "Alpha", "text": "second in"}
|
||||
]`)
|
||||
output := filepath.Join(dir, "normalized.json")
|
||||
|
||||
err := executeNormalize(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("normalize failed: %v", err)
|
||||
}
|
||||
|
||||
var transcript schema.IntermediateTranscript
|
||||
readJSON(t, output, &transcript)
|
||||
if transcript.Segments[0].Speaker != "Zulu" || transcript.Segments[1].Speaker != "Alpha" {
|
||||
t.Fatalf("tie-break order mismatch: %#v", transcript.Segments)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeMinimalSchemaOmitsCategories(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeJSONFile(t, dir, "input.json", `{
|
||||
"segments": [
|
||||
{"start": 1, "end": 2, "speaker": "Alice", "text": "first", "categories": ["filler"]}
|
||||
]
|
||||
}`)
|
||||
output := filepath.Join(dir, "normalized.json")
|
||||
|
||||
err := executeNormalize(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
"--output-schema", config.OutputSchemaMinimal,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("normalize failed: %v", err)
|
||||
}
|
||||
|
||||
var transcript schema.MinimalTranscript
|
||||
readJSON(t, output, &transcript)
|
||||
if transcript.Metadata.OutputSchema != config.OutputSchemaMinimal {
|
||||
t.Fatalf("output schema = %q, want %q", transcript.Metadata.OutputSchema, config.OutputSchemaMinimal)
|
||||
}
|
||||
if len(transcript.Segments) != 1 || transcript.Segments[0].ID != 1 {
|
||||
t.Fatalf("unexpected minimal output: %#v", transcript.Segments)
|
||||
}
|
||||
bytes, readErr := os.ReadFile(output)
|
||||
if readErr != nil {
|
||||
t.Fatalf("read output: %v", readErr)
|
||||
}
|
||||
if strings.Contains(string(bytes), "categories") {
|
||||
t.Fatalf("minimal output unexpectedly contains categories:\n%s", string(bytes))
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeFullSchemaOutputValidatesAndHasProvenanceFallback(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeJSONFile(t, dir, "input.json", `[
|
||||
{"start": 1, "end": 2, "speaker": "Alice", "text": "first"},
|
||||
{"start": 3, "end": 4, "speaker": "Bob", "text": "second", "source":"custom.json", "source_segment_index": 7}
|
||||
]`)
|
||||
output := filepath.Join(dir, "normalized.json")
|
||||
|
||||
err := executeNormalize(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
"--output-schema", config.OutputSchemaFull,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("normalize failed: %v", err)
|
||||
}
|
||||
|
||||
var transcript schema.Transcript
|
||||
readJSON(t, output, &transcript)
|
||||
if err := schema.ValidateTranscript(transcript); err != nil {
|
||||
t.Fatalf("full output should validate: %v", err)
|
||||
}
|
||||
if len(transcript.Segments) != 2 {
|
||||
t.Fatalf("segment count = %d, want 2", len(transcript.Segments))
|
||||
}
|
||||
if transcript.Segments[0].Source != filepath.Base(input) {
|
||||
t.Fatalf("source fallback = %q, want %q", transcript.Segments[0].Source, filepath.Base(input))
|
||||
}
|
||||
if transcript.Segments[0].SourceSegmentIndex == nil || *transcript.Segments[0].SourceSegmentIndex != 0 {
|
||||
t.Fatalf("source_segment_index fallback = %v, want 0", transcript.Segments[0].SourceSegmentIndex)
|
||||
}
|
||||
if transcript.Segments[1].Source != "custom.json" {
|
||||
t.Fatalf("explicit source preserved = %q, want custom.json", transcript.Segments[1].Source)
|
||||
}
|
||||
if transcript.Segments[1].SourceSegmentIndex == nil || *transcript.Segments[1].SourceSegmentIndex != 7 {
|
||||
t.Fatalf("explicit source_segment_index preserved = %v, want 7", transcript.Segments[1].SourceSegmentIndex)
|
||||
}
|
||||
if transcript.OverlapGroups == nil || len(transcript.OverlapGroups) != 0 {
|
||||
t.Fatalf("overlap_groups = %#v, want empty array", transcript.OverlapGroups)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeEmptySegmentsArrayProducesValidOutput(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeJSONFile(t, dir, "input.json", `{"segments":[]}`)
|
||||
output := filepath.Join(dir, "normalized.json")
|
||||
|
||||
err := executeNormalize(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("normalize failed: %v", err)
|
||||
}
|
||||
|
||||
var transcript schema.IntermediateTranscript
|
||||
readJSON(t, output, &transcript)
|
||||
if len(transcript.Segments) != 0 {
|
||||
t.Fatalf("segment count = %d, want 0", len(transcript.Segments))
|
||||
}
|
||||
if err := schema.ValidateIntermediateTranscript(transcript); err != nil {
|
||||
t.Fatalf("intermediate output should validate: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeRepairsAndDropsDefectiveSegments(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeJSONFile(t, dir, "input.json", `[
|
||||
{"start": 5, "speaker": "", "text": "keep-a"},
|
||||
{"end": 3, "speaker": " ", "text": "keep-b"},
|
||||
{"speaker": "A"},
|
||||
{"speaker": "A", "text": " "},
|
||||
{"start": 9, "end": 4, "speaker": "B", "text": "keep-c"}
|
||||
]`)
|
||||
output := filepath.Join(dir, "normalized.json")
|
||||
reportPath := filepath.Join(dir, "report.json")
|
||||
|
||||
err := executeNormalize(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
"--report-file", reportPath,
|
||||
"--output-schema", config.OutputSchemaIntermediate,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("normalize failed: %v", err)
|
||||
}
|
||||
|
||||
var transcript schema.IntermediateTranscript
|
||||
readJSON(t, output, &transcript)
|
||||
if len(transcript.Segments) != 3 {
|
||||
t.Fatalf("segment count = %d, want 3", len(transcript.Segments))
|
||||
}
|
||||
if transcript.Segments[0].Start != 3 || transcript.Segments[0].End != 3 {
|
||||
t.Fatalf("segment[0] timing = %v..%v, want 3..3", transcript.Segments[0].Start, transcript.Segments[0].End)
|
||||
}
|
||||
if transcript.Segments[1].Start != 4 || transcript.Segments[1].End != 9 {
|
||||
t.Fatalf("segment[1] timing = %v..%v, want 4..9", transcript.Segments[1].Start, transcript.Segments[1].End)
|
||||
}
|
||||
if transcript.Segments[2].Start != 5 || transcript.Segments[2].End != 5 {
|
||||
t.Fatalf("segment[2] timing = %v..%v, want 5..5", transcript.Segments[2].Start, transcript.Segments[2].End)
|
||||
}
|
||||
if transcript.Segments[0].Speaker != "Unknown_Speaker" || transcript.Segments[2].Speaker != "Unknown_Speaker" {
|
||||
t.Fatalf("expected Unknown_Speaker placeholders, got %#v", transcript.Segments)
|
||||
}
|
||||
|
||||
var rpt report.Report
|
||||
readJSON(t, reportPath, &rpt)
|
||||
audit := extractNormalizeAudit(t, rpt)
|
||||
if audit.InputSegmentCount != 5 || audit.OutputSegmentCount != 3 {
|
||||
t.Fatalf("audit counts = in:%d out:%d, want in:5 out:3", audit.InputSegmentCount, audit.OutputSegmentCount)
|
||||
}
|
||||
if audit.TimingFieldsRepaired != 2 {
|
||||
t.Fatalf("timing fields repaired = %d, want 2", audit.TimingFieldsRepaired)
|
||||
}
|
||||
if audit.TimingOrderSwapped != 1 {
|
||||
t.Fatalf("timing order swapped = %d, want 1", audit.TimingOrderSwapped)
|
||||
}
|
||||
if audit.SpeakerFilled != 2 {
|
||||
t.Fatalf("speaker filled = %d, want 2", audit.SpeakerFilled)
|
||||
}
|
||||
if audit.SegmentsDroppedText != 2 {
|
||||
t.Fatalf("segments dropped text = %d, want 2", audit.SegmentsDroppedText)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeSelectedOutputSchemaIsHonored(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeJSONFile(t, dir, "input.json", `{"segments":[{"start":1,"end":2,"speaker":"A","text":"one"}]}`)
|
||||
output := filepath.Join(dir, "normalized.json")
|
||||
|
||||
err := executeNormalize(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
"--output-schema", config.OutputSchemaMinimal,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("normalize failed: %v", err)
|
||||
}
|
||||
|
||||
var transcript schema.MinimalTranscript
|
||||
readJSON(t, output, &transcript)
|
||||
if transcript.Metadata.OutputSchema != config.OutputSchemaMinimal {
|
||||
t.Fatalf("output schema = %q, want %q", transcript.Metadata.OutputSchema, config.OutputSchemaMinimal)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeReportFileWrittenAndContainsObjectInputShape(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeJSONFile(t, dir, "input.json", `{"segments":[{"start":1,"end":2,"speaker":"A","text":"one"}]}`)
|
||||
output := filepath.Join(dir, "normalized.json")
|
||||
reportPath := filepath.Join(dir, "report.json")
|
||||
|
||||
err := executeNormalize(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
"--report-file", reportPath,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("normalize failed: %v", err)
|
||||
}
|
||||
|
||||
var rpt report.Report
|
||||
readJSON(t, reportPath, &rpt)
|
||||
audit := extractNormalizeAudit(t, rpt)
|
||||
if audit.InputShape != "object_with_segments" {
|
||||
t.Fatalf("input shape = %q, want object_with_segments", audit.InputShape)
|
||||
}
|
||||
if audit.InputSegmentCount != 1 {
|
||||
t.Fatalf("input segment count = %d, want 1", audit.InputSegmentCount)
|
||||
}
|
||||
if audit.OutputSchema != config.OutputSchemaIntermediate {
|
||||
t.Fatalf("output schema = %q, want %q", audit.OutputSchema, config.OutputSchemaIntermediate)
|
||||
}
|
||||
if len(audit.OutputModules) != 1 || audit.OutputModules[0] != "json" {
|
||||
t.Fatalf("output modules = %v, want [json]", audit.OutputModules)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeReportIncludesBareArrayShape(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeJSONFile(t, dir, "input.json", `[{"start":1,"end":2,"speaker":"A","text":"one"}]`)
|
||||
output := filepath.Join(dir, "normalized.json")
|
||||
reportPath := filepath.Join(dir, "report.json")
|
||||
|
||||
err := executeNormalize(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
"--report-file", reportPath,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("normalize failed: %v", err)
|
||||
}
|
||||
|
||||
var rpt report.Report
|
||||
readJSON(t, reportPath, &rpt)
|
||||
audit := extractNormalizeAudit(t, rpt)
|
||||
if audit.InputShape != "bare_segments_array" {
|
||||
t.Fatalf("input shape = %q, want bare_segments_array", audit.InputShape)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeReportDoesNotIncludeTranscriptText(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
const segmentText = "normalize-report-secret-text"
|
||||
input := writeJSONFile(t, dir, "input.json", `[{"start":1,"end":2,"speaker":"A","text":"`+segmentText+`"}]`)
|
||||
output := filepath.Join(dir, "normalized.json")
|
||||
reportPath := filepath.Join(dir, "report.json")
|
||||
|
||||
err := executeNormalize(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
"--report-file", reportPath,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("normalize failed: %v", err)
|
||||
}
|
||||
|
||||
var rpt report.Report
|
||||
readJSON(t, reportPath, &rpt)
|
||||
for _, event := range rpt.Events {
|
||||
if strings.Contains(event.Message, segmentText) {
|
||||
t.Fatalf("report unexpectedly contained transcript text in event %#v", event)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeReportEmptyInputEmitsWarning(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeJSONFile(t, dir, "input.json", `{"segments":[]}`)
|
||||
output := filepath.Join(dir, "normalized.json")
|
||||
reportPath := filepath.Join(dir, "report.json")
|
||||
|
||||
err := executeNormalize(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
"--report-file", reportPath,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("normalize failed: %v", err)
|
||||
}
|
||||
|
||||
var rpt report.Report
|
||||
readJSON(t, reportPath, &rpt)
|
||||
found := false
|
||||
for _, event := range rpt.Events {
|
||||
if event.Stage == "normalize" && event.Module == "normalize" && event.Severity == report.SeverityWarning &&
|
||||
strings.Contains(event.Message, "zero segments") {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("expected empty transcript warning event, got %#v", rpt.Events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeReportWriteFailureReturnsClearError(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeJSONFile(t, dir, "input.json", `{"segments":[{"start":1,"end":2,"speaker":"A","text":"one"}]}`)
|
||||
output := filepath.Join(dir, "normalized.json")
|
||||
|
||||
err := executeNormalize(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
"--report-file", dir,
|
||||
)
|
||||
if err == nil {
|
||||
t.Fatal("expected report write failure")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "write --report-file") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func executeNormalize(args ...string) error {
|
||||
cmd := NewRootCommand()
|
||||
cmd.SetArgs(append([]string{"normalize"}, args...))
|
||||
return cmd.Execute()
|
||||
}
|
||||
|
||||
type normalizeAudit struct {
|
||||
Command string `json:"command"`
|
||||
InputFile string `json:"input_file"`
|
||||
OutputFile string `json:"output_file"`
|
||||
InputShape string `json:"input_shape"`
|
||||
InputSegmentCount int `json:"input_segment_count"`
|
||||
OutputSegmentCount int `json:"output_segment_count"`
|
||||
OutputSchema string `json:"output_schema"`
|
||||
OutputModules []string `json:"output_modules"`
|
||||
IDsReassigned bool `json:"ids_reassigned"`
|
||||
SortingChangedInput bool `json:"sorting_changed_input_order"`
|
||||
SegmentsWithCategories int `json:"segments_with_categories"`
|
||||
TimingFieldsRepaired int `json:"timing_fields_repaired"`
|
||||
TimingOrderSwapped int `json:"timing_order_swapped"`
|
||||
SpeakerFilled int `json:"speaker_filled"`
|
||||
SegmentsDroppedText int `json:"segments_dropped_text"`
|
||||
}
|
||||
|
||||
func extractNormalizeAudit(t *testing.T, rpt report.Report) normalizeAudit {
|
||||
t.Helper()
|
||||
for _, event := range rpt.Events {
|
||||
if event.Stage == "normalize" && event.Module == "normalize-audit" {
|
||||
var audit normalizeAudit
|
||||
if err := json.Unmarshal([]byte(event.Message), &audit); err != nil {
|
||||
t.Fatalf("decode normalize audit: %v", err)
|
||||
}
|
||||
return audit
|
||||
}
|
||||
}
|
||||
t.Fatalf("missing normalize-audit event: %#v", rpt.Events)
|
||||
return normalizeAudit{}
|
||||
}
|
||||
@@ -10,12 +10,14 @@ import (
|
||||
func NewRootCommand() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "seriatim",
|
||||
Short: "Merge per-speaker transcripts into a chronological transcript",
|
||||
Short: "Merge, trim, and normalize transcript artifacts",
|
||||
Version: buildinfo.Version,
|
||||
SilenceErrors: true,
|
||||
SilenceUsage: true,
|
||||
}
|
||||
|
||||
cmd.AddCommand(newMergeCommand())
|
||||
cmd.AddCommand(newNormalizeCommand())
|
||||
cmd.AddCommand(newTrimCommand())
|
||||
return cmd
|
||||
}
|
||||
|
||||
191
internal/cli/trim.go
Normal file
191
internal/cli/trim.go
Normal file
@@ -0,0 +1,191 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.maximumdirect.net/eric/seriatim/internal/config"
|
||||
"gitea.maximumdirect.net/eric/seriatim/internal/report"
|
||||
triminternal "gitea.maximumdirect.net/eric/seriatim/internal/trim"
|
||||
)
|
||||
|
||||
type trimAuditReport struct {
|
||||
Operation string `json:"operation"`
|
||||
InputFile string `json:"input_file"`
|
||||
OutputFile string `json:"output_file"`
|
||||
InputSchema string `json:"input_schema"`
|
||||
OutputSchema string `json:"output_schema"`
|
||||
Mode string `json:"mode"`
|
||||
Selector string `json:"selector"`
|
||||
SelectedIDs []int `json:"selected_ids"`
|
||||
AllowEmpty bool `json:"allow_empty"`
|
||||
InputSegmentCount int `json:"input_segment_count"`
|
||||
RetainedSegmentCount int `json:"retained_segment_count"`
|
||||
RemovedSegmentCount int `json:"removed_segment_count"`
|
||||
RemovedInputIDs []int `json:"removed_input_ids"`
|
||||
OldToNewIDMapping []trimIDMapping `json:"old_to_new_id_mapping"`
|
||||
OverlapGroupsRecomputed bool `json:"overlap_groups_recomputed"`
|
||||
}
|
||||
|
||||
type trimIDMapping struct {
|
||||
OldID int `json:"old_id"`
|
||||
NewID int `json:"new_id"`
|
||||
}
|
||||
|
||||
func newTrimCommand() *cobra.Command {
|
||||
var opts config.TrimOptions
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "trim",
|
||||
Short: "Trim an existing seriatim transcript artifact by segment ID",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
trimOpts := opts
|
||||
if !cmd.Flags().Changed("output-schema") {
|
||||
trimOpts.OutputSchema = ""
|
||||
}
|
||||
|
||||
cfg, err := config.NewTrimConfig(trimOpts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
selector, err := triminternal.ParseSelector(cfg.Selector)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid selector %q: %w", cfg.Selector, err)
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(cfg.InputFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read --input-file %q: %w", cfg.InputFile, err)
|
||||
}
|
||||
|
||||
artifact, err := triminternal.ParseArtifactJSON(data)
|
||||
if err != nil {
|
||||
return fmt.Errorf("--input-file %q: %w", cfg.InputFile, err)
|
||||
}
|
||||
inputSegmentCount := artifact.SegmentCount()
|
||||
inputSchema := artifact.Schema
|
||||
|
||||
mode := triminternal.ModeKeep
|
||||
if cfg.Mode == "remove" {
|
||||
mode = triminternal.ModeRemove
|
||||
}
|
||||
|
||||
trimmed, err := triminternal.ApplyArtifact(artifact, triminternal.Options{
|
||||
Mode: mode,
|
||||
Selector: selector,
|
||||
AllowEmpty: cfg.AllowEmpty,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
outputSchema := artifact.Schema
|
||||
if cfg.OutputSchema != "" {
|
||||
outputSchema = cfg.OutputSchema
|
||||
}
|
||||
|
||||
outputArtifact, err := triminternal.ConvertArtifact(trimmed.Artifact, outputSchema)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := triminternal.ValidateArtifact(outputArtifact); err != nil {
|
||||
return fmt.Errorf("validate trimmed output: %w", err)
|
||||
}
|
||||
|
||||
if err := writeOutputJSON(cfg.OutputFile, outputArtifact.Value()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if cfg.ReportFile != "" {
|
||||
audit := trimAuditReport{
|
||||
Operation: "trim",
|
||||
InputFile: cfg.InputFile,
|
||||
OutputFile: cfg.OutputFile,
|
||||
InputSchema: inputSchema,
|
||||
OutputSchema: outputArtifact.Schema,
|
||||
Mode: cfg.Mode,
|
||||
Selector: cfg.Selector,
|
||||
SelectedIDs: selector.IDs(),
|
||||
AllowEmpty: cfg.AllowEmpty,
|
||||
InputSegmentCount: inputSegmentCount,
|
||||
RetainedSegmentCount: len(trimmed.OldToNewID),
|
||||
RemovedSegmentCount: len(trimmed.RemovedIDs),
|
||||
RemovedInputIDs: append([]int(nil), trimmed.RemovedIDs...),
|
||||
OldToNewIDMapping: orderedIDMapping(trimmed.OldToNewID),
|
||||
OverlapGroupsRecomputed: trimmed.OverlapGroupsRecomputed,
|
||||
}
|
||||
auditJSON, err := json.Marshal(audit)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal trim audit report: %w", err)
|
||||
}
|
||||
|
||||
rpt := report.Report{
|
||||
Metadata: report.Metadata{
|
||||
Application: outputArtifact.Application(),
|
||||
Version: outputArtifact.Version(),
|
||||
InputReader: "trim-artifact",
|
||||
InputFiles: []string{cfg.InputFile},
|
||||
OutputModules: []string{"json"},
|
||||
},
|
||||
Events: []report.Event{
|
||||
report.Info("trim", "trim", fmt.Sprintf("trimmed %d input segment(s) into %d output segment(s) with mode=%s", inputSegmentCount, outputArtifact.SegmentCount(), cfg.Mode)),
|
||||
report.Info("trim", "trim-audit", string(auditJSON)),
|
||||
report.Info("trim", "validate-output", fmt.Sprintf("validated %d output segment(s)", outputArtifact.SegmentCount())),
|
||||
report.Info("output", "json", "wrote transcript JSON"),
|
||||
},
|
||||
}
|
||||
if err := report.WriteJSON(cfg.ReportFile, rpt); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
flags := cmd.Flags()
|
||||
flags.StringVar(&opts.InputFile, "input-file", "", "input seriatim transcript artifact JSON file")
|
||||
flags.StringVar(&opts.OutputFile, "output-file", "", "output transcript JSON file")
|
||||
flags.StringVar(&opts.ReportFile, "report-file", "", "optional report JSON file")
|
||||
flags.StringVar(&opts.Keep, "keep", "", "segment ID selector to keep (for example: 1-10,15)")
|
||||
flags.StringVar(&opts.Remove, "remove", "", "segment ID selector to remove (for example: 1-10,15)")
|
||||
flags.StringVar(&opts.OutputSchema, "output-schema", "", "optional output JSON schema override: seriatim-minimal, seriatim-intermediate, or seriatim-full")
|
||||
flags.BoolVar(&opts.AllowEmpty, "allow-empty", false, "allow trimming to an empty transcript")
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
func writeOutputJSON(path string, value any) error {
|
||||
file, err := os.Create(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
enc := json.NewEncoder(file)
|
||||
enc.SetIndent("", " ")
|
||||
return enc.Encode(value)
|
||||
}
|
||||
|
||||
func orderedIDMapping(mapping map[int]int) []trimIDMapping {
|
||||
keys := make([]int, 0, len(mapping))
|
||||
for oldID := range mapping {
|
||||
keys = append(keys, oldID)
|
||||
}
|
||||
sort.Ints(keys)
|
||||
|
||||
pairs := make([]trimIDMapping, 0, len(keys))
|
||||
for _, oldID := range keys {
|
||||
pairs = append(pairs, trimIDMapping{
|
||||
OldID: oldID,
|
||||
NewID: mapping[oldID],
|
||||
})
|
||||
}
|
||||
return pairs
|
||||
}
|
||||
758
internal/cli/trim_test.go
Normal file
758
internal/cli/trim_test.go
Normal file
@@ -0,0 +1,758 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/seriatim/internal/config"
|
||||
"gitea.maximumdirect.net/eric/seriatim/internal/report"
|
||||
"gitea.maximumdirect.net/eric/seriatim/schema"
|
||||
)
|
||||
|
||||
func TestTrimKeepModeEndToEnd(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeTrimFullFixture(t, dir, "input.json")
|
||||
output := filepath.Join(dir, "trimmed.json")
|
||||
|
||||
err := executeTrim(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
"--keep", "2,4",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("trim failed: %v", err)
|
||||
}
|
||||
|
||||
var transcript schema.Transcript
|
||||
readJSON(t, output, &transcript)
|
||||
if len(transcript.Segments) != 2 {
|
||||
t.Fatalf("segment count = %d, want 2", len(transcript.Segments))
|
||||
}
|
||||
if transcript.Segments[0].Text != "two" || transcript.Segments[1].Text != "four" {
|
||||
t.Fatalf("unexpected kept text order: %#v", transcript.Segments)
|
||||
}
|
||||
assertSequentialIDs(t, []int{transcript.Segments[0].ID, transcript.Segments[1].ID})
|
||||
}
|
||||
|
||||
func TestTrimRemoveModeEndToEnd(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeTrimFullFixture(t, dir, "input.json")
|
||||
output := filepath.Join(dir, "trimmed.json")
|
||||
|
||||
err := executeTrim(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
"--remove", "2,4",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("trim failed: %v", err)
|
||||
}
|
||||
|
||||
var transcript schema.Transcript
|
||||
readJSON(t, output, &transcript)
|
||||
if len(transcript.Segments) != 2 {
|
||||
t.Fatalf("segment count = %d, want 2", len(transcript.Segments))
|
||||
}
|
||||
if transcript.Segments[0].Text != "one" || transcript.Segments[1].Text != "three" {
|
||||
t.Fatalf("unexpected remaining text order: %#v", transcript.Segments)
|
||||
}
|
||||
assertSequentialIDs(t, []int{transcript.Segments[0].ID, transcript.Segments[1].ID})
|
||||
}
|
||||
|
||||
func TestTrimMutualExclusionFailure(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeTrimFullFixture(t, dir, "input.json")
|
||||
output := filepath.Join(dir, "trimmed.json")
|
||||
|
||||
err := executeTrim(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
"--keep", "1",
|
||||
"--remove", "2",
|
||||
)
|
||||
if err == nil {
|
||||
t.Fatal("expected mutual exclusion error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "mutually exclusive") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrimMissingSelectionFailure(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeTrimFullFixture(t, dir, "input.json")
|
||||
output := filepath.Join(dir, "trimmed.json")
|
||||
|
||||
err := executeTrim(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
)
|
||||
if err == nil {
|
||||
t.Fatal("expected selection flag error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "exactly one of --keep or --remove is required") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrimInvalidSelectedIDFailure(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeTrimFullFixture(t, dir, "input.json")
|
||||
output := filepath.Join(dir, "trimmed.json")
|
||||
|
||||
err := executeTrim(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
"--keep", "99",
|
||||
)
|
||||
if err == nil {
|
||||
t.Fatal("expected missing selected ID error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "does not exist") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrimOmittedOutputSchemaPreservesInputSchema(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeTrimMinimalFixture(t, dir, "input-minimal.json")
|
||||
output := filepath.Join(dir, "trimmed.json")
|
||||
|
||||
err := executeTrim(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
"--keep", "1",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("trim failed: %v", err)
|
||||
}
|
||||
|
||||
var transcript schema.MinimalTranscript
|
||||
readJSON(t, output, &transcript)
|
||||
if transcript.Metadata.OutputSchema != config.OutputSchemaMinimal {
|
||||
t.Fatalf("output_schema = %q, want %q", transcript.Metadata.OutputSchema, config.OutputSchemaMinimal)
|
||||
}
|
||||
if len(transcript.Segments) != 1 || transcript.Segments[0].ID != 1 {
|
||||
t.Fatalf("unexpected minimal trim output: %#v", transcript.Segments)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrimExplicitOutputSchemaChangesOutputSchema(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeTrimFullFixture(t, dir, "input.json")
|
||||
output := filepath.Join(dir, "trimmed.json")
|
||||
|
||||
err := executeTrim(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
"--keep", "1,3",
|
||||
"--output-schema", config.OutputSchemaMinimal,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("trim failed: %v", err)
|
||||
}
|
||||
|
||||
var transcript schema.MinimalTranscript
|
||||
readJSON(t, output, &transcript)
|
||||
if transcript.Metadata.OutputSchema != config.OutputSchemaMinimal {
|
||||
t.Fatalf("output_schema = %q, want %q", transcript.Metadata.OutputSchema, config.OutputSchemaMinimal)
|
||||
}
|
||||
if len(transcript.Segments) != 2 {
|
||||
t.Fatalf("segment count = %d, want 2", len(transcript.Segments))
|
||||
}
|
||||
assertSequentialIDs(t, []int{transcript.Segments[0].ID, transcript.Segments[1].ID})
|
||||
}
|
||||
|
||||
func TestTrimExplicitOutputSchemaConvertsMinimalToIntermediate(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeTrimMinimalFixture(t, dir, "input-minimal.json")
|
||||
output := filepath.Join(dir, "trimmed.json")
|
||||
|
||||
err := executeTrim(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
"--keep", "1-2",
|
||||
"--output-schema", config.OutputSchemaIntermediate,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("trim failed: %v", err)
|
||||
}
|
||||
|
||||
var transcript schema.IntermediateTranscript
|
||||
readJSON(t, output, &transcript)
|
||||
if transcript.Metadata.OutputSchema != config.OutputSchemaIntermediate {
|
||||
t.Fatalf("output_schema = %q, want %q", transcript.Metadata.OutputSchema, config.OutputSchemaIntermediate)
|
||||
}
|
||||
if len(transcript.Segments) != 2 {
|
||||
t.Fatalf("segment count = %d, want 2", len(transcript.Segments))
|
||||
}
|
||||
assertSequentialIDs(t, []int{transcript.Segments[0].ID, transcript.Segments[1].ID})
|
||||
}
|
||||
|
||||
func TestTrimIntermediateInputPreservesIntermediateOutputAndCategories(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeTrimIntermediateFixture(t, dir, "input-intermediate.json")
|
||||
output := filepath.Join(dir, "trimmed.json")
|
||||
|
||||
err := executeTrim(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
"--keep", "2",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("trim failed: %v", err)
|
||||
}
|
||||
|
||||
var transcript schema.IntermediateTranscript
|
||||
readJSON(t, output, &transcript)
|
||||
if transcript.Metadata.OutputSchema != config.OutputSchemaIntermediate {
|
||||
t.Fatalf("output_schema = %q, want %q", transcript.Metadata.OutputSchema, config.OutputSchemaIntermediate)
|
||||
}
|
||||
if len(transcript.Segments) != 1 {
|
||||
t.Fatalf("segment count = %d, want 1", len(transcript.Segments))
|
||||
}
|
||||
if transcript.Segments[0].ID != 1 {
|
||||
t.Fatalf("segment ID = %d, want 1", transcript.Segments[0].ID)
|
||||
}
|
||||
assertIntSliceEqual(t, []int{len(transcript.Segments[0].Categories)}, []int{2})
|
||||
if transcript.Segments[0].Categories[0] != "filler" || transcript.Segments[0].Categories[1] != "backchannel" {
|
||||
t.Fatalf("categories = %v, want [filler backchannel]", transcript.Segments[0].Categories)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrimFullInputPreservesFullShapeAndRecomputesOverlapGroups(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeTrimFullOverlapFixture(t, dir, "input-full-overlap.json")
|
||||
output := filepath.Join(dir, "trimmed.json")
|
||||
|
||||
err := executeTrim(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
"--keep", "1,2",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("trim failed: %v", err)
|
||||
}
|
||||
|
||||
var transcript schema.Transcript
|
||||
readJSON(t, output, &transcript)
|
||||
if len(transcript.Segments) != 2 {
|
||||
t.Fatalf("segment count = %d, want 2", len(transcript.Segments))
|
||||
}
|
||||
assertSequentialIDs(t, []int{transcript.Segments[0].ID, transcript.Segments[1].ID})
|
||||
if len(transcript.OverlapGroups) != 1 {
|
||||
t.Fatalf("overlap group count = %d, want 1", len(transcript.OverlapGroups))
|
||||
}
|
||||
if transcript.OverlapGroups[0].ID != 1 {
|
||||
t.Fatalf("overlap group id = %d, want 1", transcript.OverlapGroups[0].ID)
|
||||
}
|
||||
if transcript.Segments[0].OverlapGroupID != 1 || transcript.Segments[1].OverlapGroupID != 1 {
|
||||
t.Fatalf("segment overlap IDs = %d,%d, want 1,1", transcript.Segments[0].OverlapGroupID, transcript.Segments[1].OverlapGroupID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrimMalformedSelectorFailsWithClearError(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeTrimFullFixture(t, dir, "input.json")
|
||||
output := filepath.Join(dir, "trimmed.json")
|
||||
|
||||
err := executeTrim(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
"--keep", "1-",
|
||||
)
|
||||
if err == nil {
|
||||
t.Fatal("expected malformed selector error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "invalid selector") || !strings.Contains(err.Error(), "malformed element") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrimMalformedInputArtifactFailsClearly(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeJSONFile(t, dir, "broken.json", `{"metadata":`)
|
||||
output := filepath.Join(dir, "trimmed.json")
|
||||
|
||||
err := executeTrim(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
"--keep", "1",
|
||||
)
|
||||
if err == nil {
|
||||
t.Fatal("expected malformed artifact error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "input JSON is malformed") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrimDuplicateInputSegmentIDsFail(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeTrimMinimalWithIDsFixture(t, dir, "input-dup.json", []int{1, 1})
|
||||
output := filepath.Join(dir, "trimmed.json")
|
||||
|
||||
err := executeTrim(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
"--keep", "1",
|
||||
)
|
||||
if err == nil {
|
||||
t.Fatal("expected duplicate segment ID failure")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "not a valid seriatim output artifact") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrimNonSequentialInputSegmentIDsFail(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeTrimMinimalWithIDsFixture(t, dir, "input-nonseq.json", []int{1, 3})
|
||||
output := filepath.Join(dir, "trimmed.json")
|
||||
|
||||
err := executeTrim(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
"--keep", "1",
|
||||
)
|
||||
if err == nil {
|
||||
t.Fatal("expected non-sequential segment ID failure")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "not a valid seriatim output artifact") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrimKeepSelectorWithOverlappingRanges(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeTrimFullFixture(t, dir, "input.json")
|
||||
output := filepath.Join(dir, "trimmed.json")
|
||||
|
||||
err := executeTrim(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
"--keep", "1-3,2-4",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("trim failed: %v", err)
|
||||
}
|
||||
|
||||
var transcript schema.Transcript
|
||||
readJSON(t, output, &transcript)
|
||||
if len(transcript.Segments) != 4 {
|
||||
t.Fatalf("segment count = %d, want 4", len(transcript.Segments))
|
||||
}
|
||||
assertSequentialIDs(t, []int{
|
||||
transcript.Segments[0].ID,
|
||||
transcript.Segments[1].ID,
|
||||
transcript.Segments[2].ID,
|
||||
transcript.Segments[3].ID,
|
||||
})
|
||||
}
|
||||
|
||||
func TestTrimRemoveSelectorWithOverlappingRanges(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeTrimFullFixture(t, dir, "input.json")
|
||||
output := filepath.Join(dir, "trimmed.json")
|
||||
|
||||
err := executeTrim(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
"--remove", "2-3,3-4",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("trim failed: %v", err)
|
||||
}
|
||||
|
||||
var transcript schema.Transcript
|
||||
readJSON(t, output, &transcript)
|
||||
if len(transcript.Segments) != 1 {
|
||||
t.Fatalf("segment count = %d, want 1", len(transcript.Segments))
|
||||
}
|
||||
if transcript.Segments[0].Text != "one" {
|
||||
t.Fatalf("remaining segment = %#v, want one", transcript.Segments[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrimSelectorOrderDoesNotAffectTranscriptOrder(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeTrimFullFixture(t, dir, "input.json")
|
||||
output := filepath.Join(dir, "trimmed.json")
|
||||
|
||||
err := executeTrim(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
"--keep", "4,1,3",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("trim failed: %v", err)
|
||||
}
|
||||
|
||||
var transcript schema.Transcript
|
||||
readJSON(t, output, &transcript)
|
||||
if len(transcript.Segments) != 3 {
|
||||
t.Fatalf("segment count = %d, want 3", len(transcript.Segments))
|
||||
}
|
||||
got := []string{
|
||||
transcript.Segments[0].Text,
|
||||
transcript.Segments[1].Text,
|
||||
transcript.Segments[2].Text,
|
||||
}
|
||||
want := []string{"one", "three", "four"}
|
||||
if got[0] != want[0] || got[1] != want[1] || got[2] != want[2] {
|
||||
t.Fatalf("segment text order = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrimAllowEmptyBehavior(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeTrimFullFixture(t, dir, "input.json")
|
||||
output := filepath.Join(dir, "trimmed.json")
|
||||
|
||||
err := executeTrim(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
"--remove", "1-4",
|
||||
)
|
||||
if err == nil {
|
||||
t.Fatal("expected empty-output error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "empty transcript") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
err = executeTrim(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
"--remove", "1-4",
|
||||
"--allow-empty",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("trim with --allow-empty failed: %v", err)
|
||||
}
|
||||
|
||||
var transcript schema.Transcript
|
||||
readJSON(t, output, &transcript)
|
||||
if len(transcript.Segments) != 0 {
|
||||
t.Fatalf("segment count = %d, want 0", len(transcript.Segments))
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrimRejectsNonSeriatimInputArtifacts(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeJSONFile(t, dir, "raw-whisperx.json", `{
|
||||
"segments": [
|
||||
{"start": 1, "end": 2, "text": "hello"}
|
||||
]
|
||||
}`)
|
||||
output := filepath.Join(dir, "trimmed.json")
|
||||
|
||||
err := executeTrim(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
"--keep", "1",
|
||||
)
|
||||
if err == nil {
|
||||
t.Fatal("expected invalid artifact error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "not a valid seriatim output artifact") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrimReportFileContainsAuditFields(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeTrimFullFixture(t, dir, "input.json")
|
||||
output := filepath.Join(dir, "trimmed.json")
|
||||
reportPath := filepath.Join(dir, "trim-report.json")
|
||||
|
||||
err := executeTrim(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
"--report-file", reportPath,
|
||||
"--remove", "4,2",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("trim failed: %v", err)
|
||||
}
|
||||
|
||||
var rpt report.Report
|
||||
readJSON(t, reportPath, &rpt)
|
||||
if len(rpt.Events) == 0 {
|
||||
t.Fatal("expected report events")
|
||||
}
|
||||
if !hasReportEvent(rpt, "trim", "trim", "trimmed 4 input segment(s) into 2 output segment(s) with mode=remove") {
|
||||
t.Fatal("expected trim summary event")
|
||||
}
|
||||
if !hasReportEvent(rpt, "trim", "validate-output", "validated 2 output segment(s)") {
|
||||
t.Fatal("expected validation event")
|
||||
}
|
||||
|
||||
audit := extractTrimAuditEvent(t, rpt)
|
||||
if audit.Operation != "trim" {
|
||||
t.Fatalf("operation = %q, want trim", audit.Operation)
|
||||
}
|
||||
if audit.InputFile != input {
|
||||
t.Fatalf("input_file = %q, want %q", audit.InputFile, input)
|
||||
}
|
||||
if audit.OutputFile != output {
|
||||
t.Fatalf("output_file = %q, want %q", audit.OutputFile, output)
|
||||
}
|
||||
if audit.InputSchema != config.OutputSchemaFull || audit.OutputSchema != config.OutputSchemaFull {
|
||||
t.Fatalf("schemas = %q -> %q, want full -> full", audit.InputSchema, audit.OutputSchema)
|
||||
}
|
||||
if audit.Mode != "remove" {
|
||||
t.Fatalf("mode = %q, want remove", audit.Mode)
|
||||
}
|
||||
if audit.Selector != "4,2" {
|
||||
t.Fatalf("selector = %q, want %q", audit.Selector, "4,2")
|
||||
}
|
||||
assertIntSliceEqual(t, audit.SelectedIDs, []int{2, 4})
|
||||
if audit.AllowEmpty {
|
||||
t.Fatal("allow_empty should be false")
|
||||
}
|
||||
if audit.InputSegmentCount != 4 || audit.RetainedSegmentCount != 2 || audit.RemovedSegmentCount != 2 {
|
||||
t.Fatalf("counts = input:%d retained:%d removed:%d, want 4/2/2", audit.InputSegmentCount, audit.RetainedSegmentCount, audit.RemovedSegmentCount)
|
||||
}
|
||||
assertIntSliceEqual(t, audit.RemovedInputIDs, []int{2, 4})
|
||||
if len(audit.OldToNewIDMapping) != 2 {
|
||||
t.Fatalf("mapping length = %d, want 2", len(audit.OldToNewIDMapping))
|
||||
}
|
||||
if audit.OldToNewIDMapping[0].OldID != 1 || audit.OldToNewIDMapping[0].NewID != 1 {
|
||||
t.Fatalf("mapping[0] = %#v, want old_id=1 new_id=1", audit.OldToNewIDMapping[0])
|
||||
}
|
||||
if audit.OldToNewIDMapping[1].OldID != 3 || audit.OldToNewIDMapping[1].NewID != 2 {
|
||||
t.Fatalf("mapping[1] = %#v, want old_id=3 new_id=2", audit.OldToNewIDMapping[1])
|
||||
}
|
||||
if !audit.OverlapGroupsRecomputed {
|
||||
t.Fatal("expected overlap_groups_recomputed=true for full schema trim")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrimReportOldToNewMappingIsDeterministicSorted(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeTrimFullFixture(t, dir, "input.json")
|
||||
output := filepath.Join(dir, "trimmed.json")
|
||||
reportPath := filepath.Join(dir, "trim-report.json")
|
||||
|
||||
err := executeTrim(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
"--report-file", reportPath,
|
||||
"--keep", "4,1,3",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("trim failed: %v", err)
|
||||
}
|
||||
|
||||
var rpt report.Report
|
||||
readJSON(t, reportPath, &rpt)
|
||||
audit := extractTrimAuditEvent(t, rpt)
|
||||
if len(audit.OldToNewIDMapping) != 3 {
|
||||
t.Fatalf("mapping length = %d, want 3", len(audit.OldToNewIDMapping))
|
||||
}
|
||||
for index, expectedOld := range []int{1, 3, 4} {
|
||||
if audit.OldToNewIDMapping[index].OldID != expectedOld {
|
||||
t.Fatalf("mapping[%d].old_id = %d, want %d", index, audit.OldToNewIDMapping[index].OldID, expectedOld)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrimNoReportFileWhenOmitted(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeTrimFullFixture(t, dir, "input.json")
|
||||
output := filepath.Join(dir, "trimmed.json")
|
||||
reportPath := filepath.Join(dir, "trim-report.json")
|
||||
|
||||
err := executeTrim(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
"--keep", "1",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("trim failed: %v", err)
|
||||
}
|
||||
|
||||
_, statErr := os.Stat(reportPath)
|
||||
if !os.IsNotExist(statErr) {
|
||||
t.Fatalf("expected no report file at %q, got err=%v", reportPath, statErr)
|
||||
}
|
||||
}
|
||||
|
||||
func executeTrim(args ...string) error {
|
||||
cmd := NewRootCommand()
|
||||
cmd.SetArgs(append([]string{"trim"}, args...))
|
||||
return cmd.Execute()
|
||||
}
|
||||
|
||||
func writeTrimFullFixture(t *testing.T, dir string, name string) string {
|
||||
t.Helper()
|
||||
|
||||
first := 10
|
||||
second := 20
|
||||
third := 30
|
||||
fourth := 40
|
||||
value := schema.Transcript{
|
||||
Metadata: schema.Metadata{
|
||||
Application: "seriatim",
|
||||
Version: "v-test",
|
||||
InputReader: "json-files",
|
||||
InputFiles: []string{"a.json"},
|
||||
PreprocessingModules: []string{"validate-raw"},
|
||||
PostprocessingModules: []string{"assign-ids"},
|
||||
OutputModules: []string{"json"},
|
||||
},
|
||||
Segments: []schema.Segment{
|
||||
{ID: 1, Source: "a.json", SourceSegmentIndex: &first, SourceRef: "a.json#10", Speaker: "A", Start: 1, End: 2, Text: "one", OverlapGroupID: 9},
|
||||
{ID: 2, Source: "a.json", SourceSegmentIndex: &second, SourceRef: "a.json#20", Speaker: "B", Start: 2, End: 3, Text: "two", OverlapGroupID: 9},
|
||||
{ID: 3, Source: "a.json", SourceSegmentIndex: &third, SourceRef: "a.json#30", Speaker: "C", Start: 4, End: 5, Text: "three", OverlapGroupID: 10},
|
||||
{ID: 4, Source: "a.json", SourceSegmentIndex: &fourth, SourceRef: "a.json#40", Speaker: "D", Start: 5, End: 6, Text: "four", OverlapGroupID: 10},
|
||||
},
|
||||
OverlapGroups: []schema.OverlapGroup{
|
||||
{ID: 9, Start: 1, End: 3, Segments: []string{"a.json#10", "a.json#20"}, Speakers: []string{"A", "B"}, Class: "unknown", Resolution: "unresolved"},
|
||||
},
|
||||
}
|
||||
|
||||
return writeTrimArtifactFile(t, dir, name, value)
|
||||
}
|
||||
|
||||
func writeTrimMinimalFixture(t *testing.T, dir string, name string) string {
|
||||
t.Helper()
|
||||
|
||||
value := schema.MinimalTranscript{
|
||||
Metadata: schema.MinimalMetadata{
|
||||
Application: "seriatim",
|
||||
Version: "v-test",
|
||||
OutputSchema: config.OutputSchemaMinimal,
|
||||
},
|
||||
Segments: []schema.MinimalSegment{
|
||||
{ID: 1, Start: 1, End: 2, Speaker: "A", Text: "one"},
|
||||
{ID: 2, Start: 2, End: 3, Speaker: "B", Text: "two"},
|
||||
},
|
||||
}
|
||||
|
||||
return writeTrimArtifactFile(t, dir, name, value)
|
||||
}
|
||||
|
||||
func writeTrimIntermediateFixture(t *testing.T, dir string, name string) string {
|
||||
t.Helper()
|
||||
|
||||
value := schema.IntermediateTranscript{
|
||||
Metadata: schema.IntermediateMetadata{
|
||||
Application: "seriatim",
|
||||
Version: "v-test",
|
||||
OutputSchema: config.OutputSchemaIntermediate,
|
||||
},
|
||||
Segments: []schema.IntermediateSegment{
|
||||
{ID: 1, Start: 1, End: 2, Speaker: "A", Text: "one", Categories: []string{"word-run"}},
|
||||
{ID: 2, Start: 2, End: 3, Speaker: "B", Text: "two", Categories: []string{"filler", "backchannel"}},
|
||||
},
|
||||
}
|
||||
|
||||
return writeTrimArtifactFile(t, dir, name, value)
|
||||
}
|
||||
|
||||
func writeTrimMinimalWithIDsFixture(t *testing.T, dir string, name string, ids []int) string {
|
||||
t.Helper()
|
||||
|
||||
if len(ids) < 2 {
|
||||
t.Fatalf("need at least two IDs, got %d", len(ids))
|
||||
}
|
||||
value := schema.MinimalTranscript{
|
||||
Metadata: schema.MinimalMetadata{
|
||||
Application: "seriatim",
|
||||
Version: "v-test",
|
||||
OutputSchema: config.OutputSchemaMinimal,
|
||||
},
|
||||
Segments: []schema.MinimalSegment{
|
||||
{ID: ids[0], Start: 1, End: 2, Speaker: "A", Text: "one"},
|
||||
{ID: ids[1], Start: 2, End: 3, Speaker: "B", Text: "two"},
|
||||
},
|
||||
}
|
||||
|
||||
return writeTrimArtifactFile(t, dir, name, value)
|
||||
}
|
||||
|
||||
func writeTrimFullOverlapFixture(t *testing.T, dir string, name string) string {
|
||||
t.Helper()
|
||||
|
||||
first := 10
|
||||
second := 20
|
||||
third := 30
|
||||
value := schema.Transcript{
|
||||
Metadata: schema.Metadata{
|
||||
Application: "seriatim",
|
||||
Version: "v-test",
|
||||
InputReader: "json-files",
|
||||
InputFiles: []string{"a.json"},
|
||||
PreprocessingModules: []string{"validate-raw"},
|
||||
PostprocessingModules: []string{"detect-overlaps", "assign-ids"},
|
||||
OutputModules: []string{"json"},
|
||||
},
|
||||
Segments: []schema.Segment{
|
||||
{ID: 1, Source: "a.json", SourceSegmentIndex: &first, SourceRef: "a.json#10", Speaker: "A", Start: 1, End: 3, Text: "one", OverlapGroupID: 5},
|
||||
{ID: 2, Source: "a.json", SourceSegmentIndex: &second, SourceRef: "a.json#20", Speaker: "B", Start: 2, End: 4, Text: "two", OverlapGroupID: 5},
|
||||
{ID: 3, Source: "a.json", SourceSegmentIndex: &third, SourceRef: "a.json#30", Speaker: "C", Start: 6, End: 7, Text: "three", OverlapGroupID: 6},
|
||||
},
|
||||
OverlapGroups: []schema.OverlapGroup{
|
||||
{ID: 99, Start: 0, End: 100, Segments: []string{"stale"}, Speakers: []string{"stale"}, Class: "unknown", Resolution: "unresolved"},
|
||||
},
|
||||
}
|
||||
|
||||
return writeTrimArtifactFile(t, dir, name, value)
|
||||
}
|
||||
|
||||
func writeTrimArtifactFile(t *testing.T, dir string, name string, value any) string {
|
||||
t.Helper()
|
||||
|
||||
data, err := json.MarshalIndent(value, "", " ")
|
||||
if err != nil {
|
||||
t.Fatalf("marshal fixture: %v", err)
|
||||
}
|
||||
path := filepath.Join(dir, name)
|
||||
if err := os.WriteFile(path, append(data, '\n'), 0o600); err != nil {
|
||||
t.Fatalf("write fixture: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func assertSequentialIDs(t *testing.T, ids []int) {
|
||||
t.Helper()
|
||||
for index, id := range ids {
|
||||
want := index + 1
|
||||
if id != want {
|
||||
t.Fatalf("id at index %d = %d, want %d", index, id, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func extractTrimAuditEvent(t *testing.T, rpt report.Report) trimAuditReport {
|
||||
t.Helper()
|
||||
|
||||
for _, event := range rpt.Events {
|
||||
if event.Stage == "trim" && event.Module == "trim-audit" {
|
||||
var audit trimAuditReport
|
||||
if err := json.Unmarshal([]byte(event.Message), &audit); err != nil {
|
||||
t.Fatalf("decode trim audit event: %v", err)
|
||||
}
|
||||
return audit
|
||||
}
|
||||
}
|
||||
t.Fatal("missing trim-audit event")
|
||||
return trimAuditReport{}
|
||||
}
|
||||
|
||||
func assertIntSliceEqual(t *testing.T, got []int, want []int) {
|
||||
t.Helper()
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("slice length = %d, want %d", len(got), len(want))
|
||||
}
|
||||
for index := range got {
|
||||
if got[index] != want[index] {
|
||||
t.Fatalf("slice[%d] = %d, want %d (full got=%v, want=%v)", index, got[index], want[index], got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -47,6 +47,26 @@ type MergeOptions struct {
|
||||
CoalesceGap string
|
||||
}
|
||||
|
||||
// TrimOptions captures raw CLI option values before validation.
|
||||
type TrimOptions struct {
|
||||
InputFile string
|
||||
OutputFile string
|
||||
ReportFile string
|
||||
Keep string
|
||||
Remove string
|
||||
OutputSchema string
|
||||
AllowEmpty bool
|
||||
}
|
||||
|
||||
// NormalizeOptions captures raw CLI option values before validation.
|
||||
type NormalizeOptions struct {
|
||||
InputFile string
|
||||
OutputFile string
|
||||
ReportFile string
|
||||
OutputSchema string
|
||||
OutputModules string
|
||||
}
|
||||
|
||||
// Config is the validated runtime configuration for a merge invocation.
|
||||
type Config struct {
|
||||
InputFiles []string
|
||||
@@ -66,6 +86,26 @@ type Config struct {
|
||||
FillerMaxDuration float64
|
||||
}
|
||||
|
||||
// TrimConfig is the validated runtime configuration for a trim invocation.
|
||||
type TrimConfig struct {
|
||||
InputFile string
|
||||
OutputFile string
|
||||
ReportFile string
|
||||
Mode string
|
||||
Selector string
|
||||
OutputSchema string
|
||||
AllowEmpty bool
|
||||
}
|
||||
|
||||
// NormalizeConfig is the validated runtime configuration for a normalize invocation.
|
||||
type NormalizeConfig struct {
|
||||
InputFile string
|
||||
OutputFile string
|
||||
ReportFile string
|
||||
OutputSchema string
|
||||
OutputModules []string
|
||||
}
|
||||
|
||||
// NewMergeConfig validates raw merge options and returns normalized config.
|
||||
func NewMergeConfig(opts MergeOptions) (Config, error) {
|
||||
cfg := Config{
|
||||
@@ -168,6 +208,111 @@ func NewMergeConfig(opts MergeOptions) (Config, error) {
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// NewTrimConfig validates raw trim options and returns normalized config.
|
||||
func NewTrimConfig(opts TrimOptions) (TrimConfig, error) {
|
||||
inputFile := filepath.Clean(strings.TrimSpace(opts.InputFile))
|
||||
if strings.TrimSpace(opts.InputFile) == "" {
|
||||
return TrimConfig{}, errors.New("--input-file is required")
|
||||
}
|
||||
if err := requireFile(inputFile, "--input-file"); err != nil {
|
||||
return TrimConfig{}, err
|
||||
}
|
||||
|
||||
outputFile, err := normalizeOutputPath(opts.OutputFile, "--output-file")
|
||||
if err != nil {
|
||||
return TrimConfig{}, err
|
||||
}
|
||||
|
||||
reportFile := ""
|
||||
if strings.TrimSpace(opts.ReportFile) != "" {
|
||||
reportFile, err = normalizeOutputPath(opts.ReportFile, "--report-file")
|
||||
if err != nil {
|
||||
return TrimConfig{}, err
|
||||
}
|
||||
}
|
||||
|
||||
keep := strings.TrimSpace(opts.Keep)
|
||||
remove := strings.TrimSpace(opts.Remove)
|
||||
if keep == "" && remove == "" {
|
||||
return TrimConfig{}, errors.New("exactly one of --keep or --remove is required")
|
||||
}
|
||||
if keep != "" && remove != "" {
|
||||
return TrimConfig{}, errors.New("--keep and --remove are mutually exclusive")
|
||||
}
|
||||
|
||||
mode := "keep"
|
||||
selector := keep
|
||||
if remove != "" {
|
||||
mode = "remove"
|
||||
selector = remove
|
||||
}
|
||||
|
||||
outputSchema := strings.TrimSpace(opts.OutputSchema)
|
||||
if outputSchema != "" {
|
||||
if err := validateOutputSchema(outputSchema); err != nil {
|
||||
return TrimConfig{}, err
|
||||
}
|
||||
}
|
||||
|
||||
return TrimConfig{
|
||||
InputFile: inputFile,
|
||||
OutputFile: outputFile,
|
||||
ReportFile: reportFile,
|
||||
Mode: mode,
|
||||
Selector: selector,
|
||||
OutputSchema: outputSchema,
|
||||
AllowEmpty: opts.AllowEmpty,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// NewNormalizeConfig validates raw normalize options and returns normalized config.
|
||||
func NewNormalizeConfig(opts NormalizeOptions) (NormalizeConfig, error) {
|
||||
inputFile := filepath.Clean(strings.TrimSpace(opts.InputFile))
|
||||
if strings.TrimSpace(opts.InputFile) == "" {
|
||||
return NormalizeConfig{}, errors.New("--input-file is required")
|
||||
}
|
||||
if err := requireFile(inputFile, "--input-file"); err != nil {
|
||||
return NormalizeConfig{}, err
|
||||
}
|
||||
|
||||
outputFile, err := normalizeOutputPath(opts.OutputFile, "--output-file")
|
||||
if err != nil {
|
||||
return NormalizeConfig{}, err
|
||||
}
|
||||
|
||||
reportFile := ""
|
||||
if strings.TrimSpace(opts.ReportFile) != "" {
|
||||
reportFile, err = normalizeOutputPath(opts.ReportFile, "--report-file")
|
||||
if err != nil {
|
||||
return NormalizeConfig{}, err
|
||||
}
|
||||
}
|
||||
|
||||
outputSchema, err := resolveOutputSchema(opts.OutputSchema)
|
||||
if err != nil {
|
||||
return NormalizeConfig{}, err
|
||||
}
|
||||
|
||||
outputModules, err := parseModuleList(opts.OutputModules)
|
||||
if err != nil {
|
||||
return NormalizeConfig{}, fmt.Errorf("--output-modules: %w", err)
|
||||
}
|
||||
if len(outputModules) == 0 {
|
||||
return NormalizeConfig{}, errors.New("--output-modules must include at least one module")
|
||||
}
|
||||
if err := validateNormalizeOutputModules(outputModules); err != nil {
|
||||
return NormalizeConfig{}, err
|
||||
}
|
||||
|
||||
return NormalizeConfig{
|
||||
InputFile: inputFile,
|
||||
OutputFile: outputFile,
|
||||
ReportFile: reportFile,
|
||||
OutputSchema: outputSchema,
|
||||
OutputModules: outputModules,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func parseModuleList(value string) ([]string, error) {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
@@ -321,3 +466,12 @@ func contains(values []string, target string) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func validateNormalizeOutputModules(modules []string) error {
|
||||
for _, module := range modules {
|
||||
if module != "json" {
|
||||
return fmt.Errorf("unknown output module %q", module)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -612,6 +612,206 @@ func TestCoalesceGapRejectsInvalidOverride(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewTrimConfigRequiresInputAndOutput(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeTempFile(t, dir, "input.json")
|
||||
output := filepath.Join(dir, "trimmed.json")
|
||||
|
||||
_, err := NewTrimConfig(TrimOptions{
|
||||
OutputFile: output,
|
||||
Keep: "1",
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "--input-file is required") {
|
||||
t.Fatalf("expected input-file required error, got %v", err)
|
||||
}
|
||||
|
||||
_, err = NewTrimConfig(TrimOptions{
|
||||
InputFile: input,
|
||||
Keep: "1",
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "--output-file is required") {
|
||||
t.Fatalf("expected output-file required error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewTrimConfigRequiresExactlyOneSelectorFlag(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeTempFile(t, dir, "input.json")
|
||||
output := filepath.Join(dir, "trimmed.json")
|
||||
|
||||
_, err := NewTrimConfig(TrimOptions{
|
||||
InputFile: input,
|
||||
OutputFile: output,
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "exactly one of --keep or --remove is required") {
|
||||
t.Fatalf("expected missing selector error, got %v", err)
|
||||
}
|
||||
|
||||
_, err = NewTrimConfig(TrimOptions{
|
||||
InputFile: input,
|
||||
OutputFile: output,
|
||||
Keep: "1",
|
||||
Remove: "2",
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "mutually exclusive") {
|
||||
t.Fatalf("expected mutually exclusive selector error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewTrimConfigAcceptsOutputSchemaOverride(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeTempFile(t, dir, "input.json")
|
||||
output := filepath.Join(dir, "trimmed.json")
|
||||
reportPath := filepath.Join(dir, "report.json")
|
||||
|
||||
cfg, err := NewTrimConfig(TrimOptions{
|
||||
InputFile: input,
|
||||
OutputFile: output,
|
||||
ReportFile: reportPath,
|
||||
Remove: "3-5",
|
||||
OutputSchema: OutputSchemaMinimal,
|
||||
AllowEmpty: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("config failed: %v", err)
|
||||
}
|
||||
if cfg.Mode != "remove" {
|
||||
t.Fatalf("mode = %q, want remove", cfg.Mode)
|
||||
}
|
||||
if cfg.Selector != "3-5" {
|
||||
t.Fatalf("selector = %q, want 3-5", cfg.Selector)
|
||||
}
|
||||
if cfg.OutputSchema != OutputSchemaMinimal {
|
||||
t.Fatalf("output schema = %q, want %q", cfg.OutputSchema, OutputSchemaMinimal)
|
||||
}
|
||||
if !cfg.AllowEmpty {
|
||||
t.Fatal("allow empty should be true")
|
||||
}
|
||||
if cfg.ReportFile != reportPath {
|
||||
t.Fatalf("report file = %q, want %q", cfg.ReportFile, reportPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewTrimConfigRejectsInvalidOutputSchemaOverride(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeTempFile(t, dir, "input.json")
|
||||
output := filepath.Join(dir, "trimmed.json")
|
||||
|
||||
_, err := NewTrimConfig(TrimOptions{
|
||||
InputFile: input,
|
||||
OutputFile: output,
|
||||
Keep: "1",
|
||||
OutputSchema: "compact",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected output schema validation error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "--output-schema must be one of") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewNormalizeConfigRequiresInputFile(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
output := filepath.Join(dir, "normalized.json")
|
||||
|
||||
_, err := NewNormalizeConfig(NormalizeOptions{
|
||||
OutputFile: output,
|
||||
OutputModules: DefaultOutputModules,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected input-file required error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "--input-file is required") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewNormalizeConfigRequiresOutputFile(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeTempFile(t, dir, "input.json")
|
||||
|
||||
_, err := NewNormalizeConfig(NormalizeOptions{
|
||||
InputFile: input,
|
||||
OutputModules: DefaultOutputModules,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected output-file required error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "--output-file is required") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewNormalizeConfigResolvesOutputSchemaDefaultAndEnv(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeTempFile(t, dir, "input.json")
|
||||
output := filepath.Join(dir, "normalized.json")
|
||||
|
||||
t.Setenv(OutputSchemaEnv, "")
|
||||
cfg, err := NewNormalizeConfig(NormalizeOptions{
|
||||
InputFile: input,
|
||||
OutputFile: output,
|
||||
OutputModules: DefaultOutputModules,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("config failed: %v", err)
|
||||
}
|
||||
if cfg.OutputSchema != DefaultOutputSchema {
|
||||
t.Fatalf("output schema = %q, want %q", cfg.OutputSchema, DefaultOutputSchema)
|
||||
}
|
||||
|
||||
t.Setenv(OutputSchemaEnv, OutputSchemaMinimal)
|
||||
cfg, err = NewNormalizeConfig(NormalizeOptions{
|
||||
InputFile: input,
|
||||
OutputFile: output,
|
||||
OutputModules: DefaultOutputModules,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("config failed: %v", err)
|
||||
}
|
||||
if cfg.OutputSchema != OutputSchemaMinimal {
|
||||
t.Fatalf("output schema = %q, want %q", cfg.OutputSchema, OutputSchemaMinimal)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewNormalizeConfigRejectsInvalidOutputSchema(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeTempFile(t, dir, "input.json")
|
||||
output := filepath.Join(dir, "normalized.json")
|
||||
|
||||
_, err := NewNormalizeConfig(NormalizeOptions{
|
||||
InputFile: input,
|
||||
OutputFile: output,
|
||||
OutputSchema: "compact",
|
||||
OutputModules: DefaultOutputModules,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected output schema error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "--output-schema must be one of") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewNormalizeConfigRejectsUnknownOutputModule(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeTempFile(t, dir, "input.json")
|
||||
output := filepath.Join(dir, "normalized.json")
|
||||
|
||||
_, err := NewNormalizeConfig(NormalizeOptions{
|
||||
InputFile: input,
|
||||
OutputFile: output,
|
||||
OutputModules: "json,yaml",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected output module error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "unknown output module") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func assertPositiveFloatEnvValidation(t *testing.T, envName string) {
|
||||
t.Helper()
|
||||
|
||||
|
||||
220
internal/normalize/build.go
Normal file
220
internal/normalize/build.go
Normal file
@@ -0,0 +1,220 @@
|
||||
package normalize
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/seriatim/internal/artifact"
|
||||
"gitea.maximumdirect.net/eric/seriatim/internal/buildinfo"
|
||||
"gitea.maximumdirect.net/eric/seriatim/internal/config"
|
||||
"gitea.maximumdirect.net/eric/seriatim/schema"
|
||||
)
|
||||
|
||||
// BuildResult contains normalize output plus deterministic transformation diagnostics.
|
||||
type BuildResult struct {
|
||||
Output any
|
||||
OutputSegmentCount int
|
||||
SortingChanged bool
|
||||
IDsReassigned bool
|
||||
SegmentsWithCategories int
|
||||
}
|
||||
|
||||
// Build converts parsed normalize input into a selected seriatim output schema.
|
||||
func Build(parsed ParsedTranscript, cfg config.NormalizeConfig) (BuildResult, error) {
|
||||
ordered := sortedSegments(parsed.Segments)
|
||||
sortingChanged := didSortingChangeOrder(ordered)
|
||||
idsReassigned := didReassignIDs(ordered)
|
||||
segmentsWithCategories := countSegmentsWithCategories(ordered)
|
||||
|
||||
switch cfg.OutputSchema {
|
||||
case config.OutputSchemaMinimal:
|
||||
output := buildMinimal(ordered)
|
||||
if err := schema.ValidateMinimalTranscript(output); err != nil {
|
||||
return BuildResult{}, fmt.Errorf("validate normalize output: %w", err)
|
||||
}
|
||||
return BuildResult{
|
||||
Output: output,
|
||||
OutputSegmentCount: len(ordered),
|
||||
SortingChanged: sortingChanged,
|
||||
IDsReassigned: idsReassigned,
|
||||
SegmentsWithCategories: segmentsWithCategories,
|
||||
}, nil
|
||||
case config.OutputSchemaIntermediate:
|
||||
output := buildIntermediate(ordered)
|
||||
if err := schema.ValidateIntermediateTranscript(output); err != nil {
|
||||
return BuildResult{}, fmt.Errorf("validate normalize output: %w", err)
|
||||
}
|
||||
return BuildResult{
|
||||
Output: output,
|
||||
OutputSegmentCount: len(ordered),
|
||||
SortingChanged: sortingChanged,
|
||||
IDsReassigned: idsReassigned,
|
||||
SegmentsWithCategories: segmentsWithCategories,
|
||||
}, nil
|
||||
case config.OutputSchemaFull:
|
||||
output := buildFull(ordered, cfg)
|
||||
if err := schema.ValidateTranscript(output); err != nil {
|
||||
return BuildResult{}, fmt.Errorf("validate normalize output: %w", err)
|
||||
}
|
||||
return BuildResult{
|
||||
Output: output,
|
||||
OutputSegmentCount: len(ordered),
|
||||
SortingChanged: sortingChanged,
|
||||
IDsReassigned: idsReassigned,
|
||||
SegmentsWithCategories: segmentsWithCategories,
|
||||
}, nil
|
||||
default:
|
||||
return BuildResult{}, fmt.Errorf("unsupported output schema %q", cfg.OutputSchema)
|
||||
}
|
||||
}
|
||||
|
||||
func sortedSegments(input []InputSegment) []InputSegment {
|
||||
ordered := make([]InputSegment, len(input))
|
||||
copy(ordered, input)
|
||||
sort.SliceStable(ordered, func(i, j int) bool {
|
||||
left := ordered[i]
|
||||
right := ordered[j]
|
||||
if left.Start != right.Start {
|
||||
return left.Start < right.Start
|
||||
}
|
||||
if left.End != right.End {
|
||||
return left.End < right.End
|
||||
}
|
||||
if left.InputIndex != right.InputIndex {
|
||||
return left.InputIndex < right.InputIndex
|
||||
}
|
||||
return left.Speaker < right.Speaker
|
||||
})
|
||||
return ordered
|
||||
}
|
||||
|
||||
func buildMinimal(segments []InputSegment) schema.MinimalTranscript {
|
||||
outputSegments := make([]schema.MinimalSegment, len(segments))
|
||||
for index, segment := range segments {
|
||||
outputSegments[index] = schema.MinimalSegment{
|
||||
ID: index + 1,
|
||||
Start: segment.Start,
|
||||
End: segment.End,
|
||||
Speaker: segment.Speaker,
|
||||
Text: segment.Text,
|
||||
}
|
||||
}
|
||||
|
||||
return schema.MinimalTranscript{
|
||||
Metadata: schema.MinimalMetadata{
|
||||
Application: artifact.ApplicationName,
|
||||
Version: buildinfo.Version,
|
||||
OutputSchema: config.OutputSchemaMinimal,
|
||||
},
|
||||
Segments: outputSegments,
|
||||
}
|
||||
}
|
||||
|
||||
func buildIntermediate(segments []InputSegment) schema.IntermediateTranscript {
|
||||
outputSegments := make([]schema.IntermediateSegment, len(segments))
|
||||
for index, segment := range segments {
|
||||
outputSegments[index] = schema.IntermediateSegment{
|
||||
ID: index + 1,
|
||||
Start: segment.Start,
|
||||
End: segment.End,
|
||||
Speaker: segment.Speaker,
|
||||
Text: segment.Text,
|
||||
Categories: append([]string(nil), segment.Categories...),
|
||||
}
|
||||
}
|
||||
|
||||
return schema.IntermediateTranscript{
|
||||
Metadata: schema.IntermediateMetadata{
|
||||
Application: artifact.ApplicationName,
|
||||
Version: buildinfo.Version,
|
||||
OutputSchema: config.OutputSchemaIntermediate,
|
||||
},
|
||||
Segments: outputSegments,
|
||||
}
|
||||
}
|
||||
|
||||
func buildFull(segments []InputSegment, cfg config.NormalizeConfig) schema.Transcript {
|
||||
defaultSource := filepath.Base(cfg.InputFile)
|
||||
outputSegments := make([]schema.Segment, len(segments))
|
||||
for index, segment := range segments {
|
||||
source := strings.TrimSpace(segment.Source)
|
||||
if source == "" {
|
||||
source = defaultSource
|
||||
}
|
||||
|
||||
sourceSegmentIndex := copyIntPtr(segment.SourceSegmentIndex)
|
||||
if sourceSegmentIndex == nil {
|
||||
fallback := segment.InputIndex
|
||||
sourceSegmentIndex = &fallback
|
||||
}
|
||||
|
||||
outputSegments[index] = schema.Segment{
|
||||
ID: index + 1,
|
||||
Source: source,
|
||||
SourceSegmentIndex: sourceSegmentIndex,
|
||||
SourceRef: segment.SourceRef,
|
||||
DerivedFrom: append([]string(nil), segment.DerivedFrom...),
|
||||
Speaker: segment.Speaker,
|
||||
Start: segment.Start,
|
||||
End: segment.End,
|
||||
Text: segment.Text,
|
||||
Categories: append([]string(nil), segment.Categories...),
|
||||
}
|
||||
}
|
||||
|
||||
return schema.Transcript{
|
||||
Metadata: schema.Metadata{
|
||||
Application: artifact.ApplicationName,
|
||||
Version: buildinfo.Version,
|
||||
InputReader: "normalize-input",
|
||||
InputFiles: []string{cfg.InputFile},
|
||||
PreprocessingModules: []string{},
|
||||
PostprocessingModules: []string{},
|
||||
OutputModules: append([]string(nil), cfg.OutputModules...),
|
||||
},
|
||||
Segments: outputSegments,
|
||||
OverlapGroups: []schema.OverlapGroup{},
|
||||
}
|
||||
}
|
||||
|
||||
func copyIntPtr(value *int) *int {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
copied := *value
|
||||
return &copied
|
||||
}
|
||||
|
||||
func didSortingChangeOrder(segments []InputSegment) bool {
|
||||
for index, segment := range segments {
|
||||
if segment.InputIndex != index {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func didReassignIDs(segments []InputSegment) bool {
|
||||
if len(segments) == 0 {
|
||||
return false
|
||||
}
|
||||
for index, segment := range segments {
|
||||
newID := index + 1
|
||||
if segment.OriginalID == nil || *segment.OriginalID != newID {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func countSegmentsWithCategories(segments []InputSegment) int {
|
||||
count := 0
|
||||
for _, segment := range segments {
|
||||
if len(segment.Categories) > 0 {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
135
internal/normalize/normalize.go
Normal file
135
internal/normalize/normalize.go
Normal file
@@ -0,0 +1,135 @@
|
||||
package normalize
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/seriatim/internal/artifact"
|
||||
"gitea.maximumdirect.net/eric/seriatim/internal/buildinfo"
|
||||
"gitea.maximumdirect.net/eric/seriatim/internal/config"
|
||||
"gitea.maximumdirect.net/eric/seriatim/internal/report"
|
||||
)
|
||||
|
||||
type normalizeAudit struct {
|
||||
Command string `json:"command"`
|
||||
InputFile string `json:"input_file"`
|
||||
OutputFile string `json:"output_file"`
|
||||
InputShape string `json:"input_shape"`
|
||||
InputSegmentCount int `json:"input_segment_count"`
|
||||
OutputSegmentCount int `json:"output_segment_count"`
|
||||
OutputSchema string `json:"output_schema"`
|
||||
OutputModules []string `json:"output_modules"`
|
||||
IDsReassigned bool `json:"ids_reassigned"`
|
||||
SortingChangedInput bool `json:"sorting_changed_input_order"`
|
||||
SegmentsWithCategories int `json:"segments_with_categories"`
|
||||
TimingFieldsRepaired int `json:"timing_fields_repaired"`
|
||||
TimingOrderSwapped int `json:"timing_order_swapped"`
|
||||
SpeakerFilled int `json:"speaker_filled"`
|
||||
SegmentsDroppedText int `json:"segments_dropped_text"`
|
||||
}
|
||||
|
||||
// Run executes artifact-level normalization.
|
||||
func Run(ctx context.Context, cfg config.NormalizeConfig) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
parsed, err := ParseFile(cfg.InputFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
built, err := Build(parsed, cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := writeOutputJSON(cfg.OutputFile, built.Output); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if cfg.ReportFile != "" {
|
||||
audit := normalizeAudit{
|
||||
Command: "normalize",
|
||||
InputFile: cfg.InputFile,
|
||||
OutputFile: cfg.OutputFile,
|
||||
InputShape: string(parsed.Shape),
|
||||
InputSegmentCount: parsed.InputSegmentCount,
|
||||
OutputSegmentCount: built.OutputSegmentCount,
|
||||
OutputSchema: cfg.OutputSchema,
|
||||
OutputModules: append([]string(nil), cfg.OutputModules...),
|
||||
IDsReassigned: built.IDsReassigned,
|
||||
SortingChangedInput: built.SortingChanged,
|
||||
SegmentsWithCategories: built.SegmentsWithCategories,
|
||||
TimingFieldsRepaired: parsed.Stats.TimingFieldsRepaired,
|
||||
TimingOrderSwapped: parsed.Stats.TimingOrderSwapped,
|
||||
SpeakerFilled: parsed.Stats.SpeakerFilled,
|
||||
SegmentsDroppedText: parsed.Stats.SegmentsDroppedText,
|
||||
}
|
||||
auditJSON, err := json.Marshal(audit)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal normalize audit: %w", err)
|
||||
}
|
||||
|
||||
events := []report.Event{
|
||||
report.Info("normalize", "normalize", "started normalize command"),
|
||||
report.Info("normalize", "normalize", fmt.Sprintf("input file: %s", cfg.InputFile)),
|
||||
report.Info("normalize", "normalize", fmt.Sprintf("detected input shape: %s", parsed.Shape)),
|
||||
report.Info("normalize", "normalize", fmt.Sprintf("input segment count: %d", parsed.InputSegmentCount)),
|
||||
report.Info("normalize", "normalize", fmt.Sprintf("selected output schema: %s", cfg.OutputSchema)),
|
||||
report.Info("normalize", "normalize", fmt.Sprintf("selected output modules: %s", strings.Join(cfg.OutputModules, ","))),
|
||||
report.Info("normalize", "normalize", fmt.Sprintf("output file: %s", cfg.OutputFile)),
|
||||
report.Info("normalize", "normalize", fmt.Sprintf("ids reassigned: %t", built.IDsReassigned)),
|
||||
report.Info("normalize", "normalize", fmt.Sprintf("sorting changed input order: %t", built.SortingChanged)),
|
||||
report.Info("normalize", "normalize", fmt.Sprintf("segments with categories: %d", built.SegmentsWithCategories)),
|
||||
report.Info("normalize", "normalize", fmt.Sprintf("timing fields repaired: %d", parsed.Stats.TimingFieldsRepaired)),
|
||||
report.Info("normalize", "normalize", fmt.Sprintf("timing order swapped: %d", parsed.Stats.TimingOrderSwapped)),
|
||||
report.Info("normalize", "normalize", fmt.Sprintf("speaker placeholders added: %d", parsed.Stats.SpeakerFilled)),
|
||||
report.Info("normalize", "normalize", fmt.Sprintf("segments dropped for empty text: %d", parsed.Stats.SegmentsDroppedText)),
|
||||
report.Info("normalize", "normalize-audit", string(auditJSON)),
|
||||
}
|
||||
if parsed.InputSegmentCount == 0 {
|
||||
events = append(events, report.Warning("normalize", "normalize", "input transcript contains zero segments"))
|
||||
}
|
||||
events = append(events,
|
||||
report.Info("normalize", "validate-output", fmt.Sprintf("validated %d output segment(s)", built.OutputSegmentCount)),
|
||||
report.Info("output", "json", "wrote transcript JSON"),
|
||||
)
|
||||
|
||||
rpt := report.Report{
|
||||
Metadata: report.Metadata{
|
||||
Application: artifact.ApplicationName,
|
||||
Version: buildinfo.Version,
|
||||
InputReader: "normalize-input",
|
||||
InputFiles: []string{cfg.InputFile},
|
||||
PreprocessingModules: []string{},
|
||||
PostprocessingModules: []string{},
|
||||
OutputModules: append([]string(nil), cfg.OutputModules...),
|
||||
},
|
||||
Events: events,
|
||||
}
|
||||
if err := report.WriteJSON(cfg.ReportFile, rpt); err != nil {
|
||||
return fmt.Errorf("write --report-file %q: %w", cfg.ReportFile, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeOutputJSON(path string, value any) error {
|
||||
file, err := os.Create(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
encoder := json.NewEncoder(file)
|
||||
encoder.SetIndent("", " ")
|
||||
if err := encoder.Encode(value); err != nil {
|
||||
return fmt.Errorf("encode normalize output JSON: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
307
internal/normalize/parse.go
Normal file
307
internal/normalize/parse.go
Normal file
@@ -0,0 +1,307 @@
|
||||
package normalize
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// InputShape identifies which top-level input shape was parsed.
|
||||
type InputShape string
|
||||
|
||||
const (
|
||||
ShapeObjectWithSegments InputShape = "object_with_segments"
|
||||
ShapeBareSegmentsArray InputShape = "bare_segments_array"
|
||||
)
|
||||
|
||||
// ParsedTranscript is the validated normalize input model.
|
||||
type ParsedTranscript struct {
|
||||
Shape InputShape
|
||||
InputSegmentCount int
|
||||
Stats NormalizeStats
|
||||
Segments []InputSegment
|
||||
}
|
||||
|
||||
// InputSegment is a validated segment from normalize input.
|
||||
type InputSegment struct {
|
||||
InputIndex int
|
||||
OriginalID *int
|
||||
StartPresent bool
|
||||
EndPresent bool
|
||||
SpeakerPresent bool
|
||||
TextPresent bool
|
||||
Start float64
|
||||
End float64
|
||||
Speaker string
|
||||
Text string
|
||||
Categories []string
|
||||
Source string
|
||||
SourceSegmentIndex *int
|
||||
SourceRef string
|
||||
DerivedFrom []string
|
||||
OverlapGroupID *int
|
||||
}
|
||||
|
||||
// NormalizeStats captures deterministic repair/drop outcomes from normalize input processing.
|
||||
type NormalizeStats struct {
|
||||
TimingFieldsRepaired int
|
||||
TimingOrderSwapped int
|
||||
SpeakerFilled int
|
||||
SegmentsDroppedText int
|
||||
}
|
||||
|
||||
type inputSegmentPayload struct {
|
||||
ID *int `json:"id"`
|
||||
Start *float64 `json:"start"`
|
||||
End *float64 `json:"end"`
|
||||
Speaker *string `json:"speaker"`
|
||||
Text *string `json:"text"`
|
||||
Categories []string `json:"categories"`
|
||||
Source string `json:"source"`
|
||||
SourceSegmentIndex *int `json:"source_segment_index"`
|
||||
SourceRef string `json:"source_ref"`
|
||||
DerivedFrom []string `json:"derived_from"`
|
||||
OverlapGroupID *int `json:"overlap_group_id"`
|
||||
}
|
||||
|
||||
// ParseFile parses normalize input JSON from file path.
|
||||
func ParseFile(path string) (ParsedTranscript, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return ParsedTranscript{}, err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
return ParseReader(file)
|
||||
}
|
||||
|
||||
// ParseReader parses normalize input JSON from a reader.
|
||||
func ParseReader(reader io.Reader) (ParsedTranscript, error) {
|
||||
var raw json.RawMessage
|
||||
decoder := json.NewDecoder(reader)
|
||||
decoder.UseNumber()
|
||||
if err := decoder.Decode(&raw); err != nil {
|
||||
return ParsedTranscript{}, fmt.Errorf("decode normalize input JSON: %w", err)
|
||||
}
|
||||
if err := ensureSingleValue(decoder); err != nil {
|
||||
return ParsedTranscript{}, err
|
||||
}
|
||||
|
||||
trimmed := bytes.TrimSpace(raw)
|
||||
if len(trimmed) == 0 {
|
||||
return ParsedTranscript{}, fmt.Errorf("normalize input is empty")
|
||||
}
|
||||
|
||||
switch trimmed[0] {
|
||||
case '{':
|
||||
return parseObjectShape(trimmed)
|
||||
case '[':
|
||||
segments, stats, inputCount, err := parseSegmentsArray(trimmed)
|
||||
if err != nil {
|
||||
return ParsedTranscript{}, err
|
||||
}
|
||||
return ParsedTranscript{
|
||||
Shape: ShapeBareSegmentsArray,
|
||||
InputSegmentCount: inputCount,
|
||||
Stats: stats,
|
||||
Segments: segments,
|
||||
}, nil
|
||||
default:
|
||||
return ParsedTranscript{}, fmt.Errorf("normalize input must be a top-level object with \"segments\" or a top-level segment array")
|
||||
}
|
||||
}
|
||||
|
||||
func ensureSingleValue(decoder *json.Decoder) error {
|
||||
var extra json.RawMessage
|
||||
err := decoder.Decode(&extra)
|
||||
if err == io.EOF {
|
||||
return nil
|
||||
}
|
||||
if err == nil {
|
||||
return fmt.Errorf("normalize input must contain exactly one top-level JSON value")
|
||||
}
|
||||
return fmt.Errorf("decode normalize input JSON: %w", err)
|
||||
}
|
||||
|
||||
func parseObjectShape(raw []byte) (ParsedTranscript, error) {
|
||||
var object map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raw, &object); err != nil {
|
||||
return ParsedTranscript{}, fmt.Errorf("decode normalize object input: %w", err)
|
||||
}
|
||||
|
||||
segmentsRaw, exists := object["segments"]
|
||||
if !exists {
|
||||
return ParsedTranscript{}, fmt.Errorf("normalize object input must contain a \"segments\" field")
|
||||
}
|
||||
|
||||
segments, stats, inputCount, err := parseSegmentsArray(segmentsRaw)
|
||||
if err != nil {
|
||||
return ParsedTranscript{}, err
|
||||
}
|
||||
|
||||
return ParsedTranscript{
|
||||
Shape: ShapeObjectWithSegments,
|
||||
InputSegmentCount: inputCount,
|
||||
Stats: stats,
|
||||
Segments: segments,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func parseSegmentsArray(raw []byte) ([]InputSegment, NormalizeStats, int, error) {
|
||||
var segmentValues []json.RawMessage
|
||||
if err := json.Unmarshal(raw, &segmentValues); err != nil {
|
||||
return nil, NormalizeStats{}, 0, fmt.Errorf("normalize input \"segments\" must be an array")
|
||||
}
|
||||
|
||||
segments := make([]InputSegment, len(segmentValues))
|
||||
for index, segmentRaw := range segmentValues {
|
||||
segment, err := decodeSegment(index, segmentRaw)
|
||||
if err != nil {
|
||||
return nil, NormalizeStats{}, 0, err
|
||||
}
|
||||
segments[index] = segment
|
||||
}
|
||||
normalized, stats, err := normalizeSegments(segments)
|
||||
return normalized, stats, len(segmentValues), err
|
||||
}
|
||||
|
||||
func decodeSegment(index int, raw []byte) (InputSegment, error) {
|
||||
var payload inputSegmentPayload
|
||||
if err := json.Unmarshal(raw, &payload); err != nil {
|
||||
return InputSegment{}, fmt.Errorf("segment %d: invalid segment object: %w", index, err)
|
||||
}
|
||||
|
||||
var start float64
|
||||
if payload.Start != nil {
|
||||
start = *payload.Start
|
||||
}
|
||||
var end float64
|
||||
if payload.End != nil {
|
||||
end = *payload.End
|
||||
}
|
||||
|
||||
speaker := ""
|
||||
if payload.Speaker != nil {
|
||||
speaker = strings.TrimSpace(*payload.Speaker)
|
||||
}
|
||||
|
||||
text := ""
|
||||
if payload.Text != nil {
|
||||
text = *payload.Text
|
||||
}
|
||||
|
||||
return InputSegment{
|
||||
InputIndex: index,
|
||||
OriginalID: payload.ID,
|
||||
StartPresent: payload.Start != nil,
|
||||
EndPresent: payload.End != nil,
|
||||
SpeakerPresent: payload.Speaker != nil,
|
||||
TextPresent: payload.Text != nil,
|
||||
Start: start,
|
||||
End: end,
|
||||
Speaker: speaker,
|
||||
Text: text,
|
||||
Categories: append([]string(nil), payload.Categories...),
|
||||
Source: payload.Source,
|
||||
SourceSegmentIndex: payload.SourceSegmentIndex,
|
||||
SourceRef: payload.SourceRef,
|
||||
DerivedFrom: append([]string(nil), payload.DerivedFrom...),
|
||||
OverlapGroupID: payload.OverlapGroupID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func normalizeSegments(segments []InputSegment) ([]InputSegment, NormalizeStats, error) {
|
||||
stats := NormalizeStats{}
|
||||
filtered := make([]InputSegment, 0, len(segments))
|
||||
for _, segment := range segments {
|
||||
if strings.TrimSpace(segment.Text) == "" {
|
||||
stats.SegmentsDroppedText++
|
||||
continue
|
||||
}
|
||||
filtered = append(filtered, segment)
|
||||
}
|
||||
|
||||
if len(filtered) == 0 {
|
||||
return filtered, stats, nil
|
||||
}
|
||||
|
||||
hasStart := make([]bool, len(filtered))
|
||||
hasEnd := make([]bool, len(filtered))
|
||||
for index, segment := range filtered {
|
||||
hasStart[index] = segment.StartPresent
|
||||
hasEnd[index] = segment.EndPresent
|
||||
}
|
||||
for index := range filtered {
|
||||
if !hasStart[index] && !hasEnd[index] {
|
||||
midpoint := inferTimestampFromNeighbors(filtered, hasStart, hasEnd, index)
|
||||
filtered[index].Start = midpoint
|
||||
filtered[index].End = midpoint
|
||||
stats.TimingFieldsRepaired += 2
|
||||
hasStart[index] = true
|
||||
hasEnd[index] = true
|
||||
continue
|
||||
}
|
||||
if hasStart[index] && !hasEnd[index] {
|
||||
filtered[index].End = filtered[index].Start
|
||||
stats.TimingFieldsRepaired++
|
||||
hasEnd[index] = true
|
||||
}
|
||||
if !hasStart[index] && hasEnd[index] {
|
||||
filtered[index].Start = filtered[index].End
|
||||
stats.TimingFieldsRepaired++
|
||||
hasStart[index] = true
|
||||
}
|
||||
}
|
||||
|
||||
for index := range filtered {
|
||||
if filtered[index].Start < 0 {
|
||||
return nil, NormalizeStats{}, fmt.Errorf("segment %d has start %v; start must be >= 0", filtered[index].InputIndex, filtered[index].Start)
|
||||
}
|
||||
if filtered[index].End < filtered[index].Start {
|
||||
filtered[index].Start, filtered[index].End = filtered[index].End, filtered[index].Start
|
||||
stats.TimingOrderSwapped++
|
||||
}
|
||||
if strings.TrimSpace(filtered[index].Speaker) == "" {
|
||||
filtered[index].Speaker = "Unknown_Speaker"
|
||||
stats.SpeakerFilled++
|
||||
}
|
||||
}
|
||||
|
||||
return filtered, stats, nil
|
||||
}
|
||||
|
||||
func inferTimestampFromNeighbors(segments []InputSegment, hasStart []bool, hasEnd []bool, index int) float64 {
|
||||
prevEnd, hasPrev := nearestPreviousEnd(segments, hasEnd, index)
|
||||
nextStart, hasNext := nearestNextStart(segments, hasStart, index)
|
||||
if hasPrev && hasNext {
|
||||
return (prevEnd + nextStart) / 2
|
||||
}
|
||||
if hasPrev {
|
||||
return prevEnd
|
||||
}
|
||||
if hasNext {
|
||||
return nextStart
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func nearestPreviousEnd(segments []InputSegment, hasEnd []bool, index int) (float64, bool) {
|
||||
for i := index - 1; i >= 0; i-- {
|
||||
if hasEnd[i] {
|
||||
return segments[i].End, true
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func nearestNextStart(segments []InputSegment, hasStart []bool, index int) (float64, bool) {
|
||||
for i := index + 1; i < len(segments); i++ {
|
||||
if hasStart[i] {
|
||||
return segments[i].Start, true
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
279
internal/normalize/parse_test.go
Normal file
279
internal/normalize/parse_test.go
Normal file
@@ -0,0 +1,279 @@
|
||||
package normalize
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseReaderObjectWithSegmentsParses(t *testing.T) {
|
||||
input := `{
|
||||
"segments": [
|
||||
{"start": 1.0, "end": 2.0, "speaker": " Alice ", "text": "hello", "id": 100}
|
||||
]
|
||||
}`
|
||||
|
||||
parsed, err := ParseReader(strings.NewReader(input))
|
||||
if err != nil {
|
||||
t.Fatalf("parse failed: %v", err)
|
||||
}
|
||||
if parsed.Shape != ShapeObjectWithSegments {
|
||||
t.Fatalf("shape = %q, want %q", parsed.Shape, ShapeObjectWithSegments)
|
||||
}
|
||||
if len(parsed.Segments) != 1 {
|
||||
t.Fatalf("segment count = %d, want 1", len(parsed.Segments))
|
||||
}
|
||||
segment := parsed.Segments[0]
|
||||
if segment.Speaker != "Alice" {
|
||||
t.Fatalf("speaker = %q, want %q", segment.Speaker, "Alice")
|
||||
}
|
||||
if segment.OriginalID == nil || *segment.OriginalID != 100 {
|
||||
t.Fatalf("original id = %v, want 100", segment.OriginalID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseReaderBareSegmentArrayParses(t *testing.T) {
|
||||
input := `[
|
||||
{"start": 1.0, "end": 2.0, "speaker": "Alice", "text": "hello"},
|
||||
{"start": 3.0, "end": 4.0, "speaker": "Bob", "text": "world"}
|
||||
]`
|
||||
|
||||
parsed, err := ParseReader(strings.NewReader(input))
|
||||
if err != nil {
|
||||
t.Fatalf("parse failed: %v", err)
|
||||
}
|
||||
if parsed.Shape != ShapeBareSegmentsArray {
|
||||
t.Fatalf("shape = %q, want %q", parsed.Shape, ShapeBareSegmentsArray)
|
||||
}
|
||||
if len(parsed.Segments) != 2 {
|
||||
t.Fatalf("segment count = %d, want 2", len(parsed.Segments))
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseReaderInvalidJSONFails(t *testing.T) {
|
||||
_, err := ParseReader(strings.NewReader(`{"segments":`))
|
||||
if err == nil {
|
||||
t.Fatal("expected parse error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "decode normalize input JSON") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseReaderObjectMissingSegmentsFails(t *testing.T) {
|
||||
_, err := ParseReader(strings.NewReader(`{"items":[]}`))
|
||||
if err == nil {
|
||||
t.Fatal("expected missing segments error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "must contain a \"segments\" field") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseReaderSegmentsNotArrayFails(t *testing.T) {
|
||||
_, err := ParseReader(strings.NewReader(`{"segments": {}}`))
|
||||
if err == nil {
|
||||
t.Fatal("expected segments not array error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "\"segments\" must be an array") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseReaderTopLevelScalarShapesFail(t *testing.T) {
|
||||
tests := []string{`"text"`, `42`, `null`, `true`}
|
||||
for _, input := range tests {
|
||||
_, err := ParseReader(strings.NewReader(input))
|
||||
if err == nil {
|
||||
t.Fatalf("expected top-level shape error for %s", input)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "top-level object") {
|
||||
t.Fatalf("unexpected error for %s: %v", input, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseReaderMissingStartUsesEndValue(t *testing.T) {
|
||||
parsed, err := ParseReader(strings.NewReader(`[{"end":2,"speaker":"A","text":"t"}]`))
|
||||
if err != nil {
|
||||
t.Fatalf("parse failed: %v", err)
|
||||
}
|
||||
if len(parsed.Segments) != 1 {
|
||||
t.Fatalf("segment count = %d, want 1", len(parsed.Segments))
|
||||
}
|
||||
if parsed.Segments[0].Start != 2 || parsed.Segments[0].End != 2 {
|
||||
t.Fatalf("segment timing = %v..%v, want 2..2", parsed.Segments[0].Start, parsed.Segments[0].End)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseReaderMissingEndUsesStartValue(t *testing.T) {
|
||||
parsed, err := ParseReader(strings.NewReader(`[{"start":1,"speaker":"A","text":"t"}]`))
|
||||
if err != nil {
|
||||
t.Fatalf("parse failed: %v", err)
|
||||
}
|
||||
if len(parsed.Segments) != 1 {
|
||||
t.Fatalf("segment count = %d, want 1", len(parsed.Segments))
|
||||
}
|
||||
if parsed.Segments[0].Start != 1 || parsed.Segments[0].End != 1 {
|
||||
t.Fatalf("segment timing = %v..%v, want 1..1", parsed.Segments[0].Start, parsed.Segments[0].End)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseReaderMissingSpeakerUsesUnknownPlaceholder(t *testing.T) {
|
||||
parsed, err := ParseReader(strings.NewReader(`[{"start":1,"end":2,"text":"t"}]`))
|
||||
if err != nil {
|
||||
t.Fatalf("parse failed: %v", err)
|
||||
}
|
||||
if parsed.Segments[0].Speaker != "Unknown_Speaker" {
|
||||
t.Fatalf("speaker = %q, want Unknown_Speaker", parsed.Segments[0].Speaker)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseReaderEmptySpeakerUsesUnknownPlaceholder(t *testing.T) {
|
||||
parsed, err := ParseReader(strings.NewReader(`[{"start":1,"end":2,"speaker":" ","text":"t"}]`))
|
||||
if err != nil {
|
||||
t.Fatalf("parse failed: %v", err)
|
||||
}
|
||||
if parsed.Segments[0].Speaker != "Unknown_Speaker" {
|
||||
t.Fatalf("speaker = %q, want Unknown_Speaker", parsed.Segments[0].Speaker)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseReaderMissingTextDropsSegment(t *testing.T) {
|
||||
parsed, err := ParseReader(strings.NewReader(`[{"start":1,"end":2,"speaker":"A"}]`))
|
||||
if err != nil {
|
||||
t.Fatalf("parse failed: %v", err)
|
||||
}
|
||||
if len(parsed.Segments) != 0 {
|
||||
t.Fatalf("segment count = %d, want 0", len(parsed.Segments))
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseReaderEndBeforeStartSwapsValues(t *testing.T) {
|
||||
parsed, err := ParseReader(strings.NewReader(`[{"start":3,"end":2,"speaker":"A","text":"t"}]`))
|
||||
if err != nil {
|
||||
t.Fatalf("parse failed: %v", err)
|
||||
}
|
||||
if parsed.Segments[0].Start != 2 || parsed.Segments[0].End != 3 {
|
||||
t.Fatalf("segment timing = %v..%v, want 2..3", parsed.Segments[0].Start, parsed.Segments[0].End)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseReaderNegativeStartFails(t *testing.T) {
|
||||
_, err := ParseReader(strings.NewReader(`[{"start":-1,"end":2,"speaker":"A","text":"t"}]`))
|
||||
assertContains(t, err, "start must be >= 0")
|
||||
}
|
||||
|
||||
func TestParseReaderEmptySegmentsArrayAccepted(t *testing.T) {
|
||||
parsed, err := ParseReader(strings.NewReader(`{"segments":[]}`))
|
||||
if err != nil {
|
||||
t.Fatalf("parse failed: %v", err)
|
||||
}
|
||||
if len(parsed.Segments) != 0 {
|
||||
t.Fatalf("segment count = %d, want 0", len(parsed.Segments))
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseReaderCategoriesPreservedWhenValid(t *testing.T) {
|
||||
parsed, err := ParseReader(strings.NewReader(`[{"start":1,"end":2,"speaker":"A","text":"t","categories":["filler","backchannel"]}]`))
|
||||
if err != nil {
|
||||
t.Fatalf("parse failed: %v", err)
|
||||
}
|
||||
if len(parsed.Segments) != 1 {
|
||||
t.Fatalf("segment count = %d, want 1", len(parsed.Segments))
|
||||
}
|
||||
if len(parsed.Segments[0].Categories) != 2 {
|
||||
t.Fatalf("categories length = %d, want 2", len(parsed.Segments[0].Categories))
|
||||
}
|
||||
if parsed.Segments[0].Categories[0] != "filler" || parsed.Segments[0].Categories[1] != "backchannel" {
|
||||
t.Fatalf("categories = %v", parsed.Segments[0].Categories)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseReaderMissingBothTimesMiddleUsesNeighborMidpoint(t *testing.T) {
|
||||
parsed, err := ParseReader(strings.NewReader(`[
|
||||
{"start":1,"end":2,"speaker":"A","text":"left"},
|
||||
{"speaker":"B","text":"middle"},
|
||||
{"start":6,"end":7,"speaker":"C","text":"right"}
|
||||
]`))
|
||||
if err != nil {
|
||||
t.Fatalf("parse failed: %v", err)
|
||||
}
|
||||
if parsed.Segments[1].Start != 4 || parsed.Segments[1].End != 4 {
|
||||
t.Fatalf("middle timing = %v..%v, want 4..4", parsed.Segments[1].Start, parsed.Segments[1].End)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseReaderMissingBothTimesEdgeFallbacks(t *testing.T) {
|
||||
parsedFirst, err := ParseReader(strings.NewReader(`[
|
||||
{"speaker":"A","text":"first"},
|
||||
{"start":5,"end":6,"speaker":"B","text":"second"}
|
||||
]`))
|
||||
if err != nil {
|
||||
t.Fatalf("parse first failed: %v", err)
|
||||
}
|
||||
if parsedFirst.Segments[0].Start != 5 || parsedFirst.Segments[0].End != 5 {
|
||||
t.Fatalf("first timing = %v..%v, want 5..5", parsedFirst.Segments[0].Start, parsedFirst.Segments[0].End)
|
||||
}
|
||||
|
||||
parsedLast, err := ParseReader(strings.NewReader(`[
|
||||
{"start":1,"end":2,"speaker":"A","text":"first"},
|
||||
{"speaker":"B","text":"last"}
|
||||
]`))
|
||||
if err != nil {
|
||||
t.Fatalf("parse last failed: %v", err)
|
||||
}
|
||||
if parsedLast.Segments[1].Start != 2 || parsedLast.Segments[1].End != 2 {
|
||||
t.Fatalf("last timing = %v..%v, want 2..2", parsedLast.Segments[1].Start, parsedLast.Segments[1].End)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseReaderMissingBothTimesSingleSegmentUsesZero(t *testing.T) {
|
||||
parsed, err := ParseReader(strings.NewReader(`[{"speaker":"A","text":"only"}]`))
|
||||
if err != nil {
|
||||
t.Fatalf("parse failed: %v", err)
|
||||
}
|
||||
if parsed.Segments[0].Start != 0 || parsed.Segments[0].End != 0 {
|
||||
t.Fatalf("timing = %v..%v, want 0..0", parsed.Segments[0].Start, parsed.Segments[0].End)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseReaderEmptyOrWhitespaceTextDropped(t *testing.T) {
|
||||
parsed, err := ParseReader(strings.NewReader(`[
|
||||
{"start":1,"end":2,"speaker":"A","text":"ok"},
|
||||
{"start":2,"end":3,"speaker":"A","text":""},
|
||||
{"start":3,"end":4,"speaker":"A","text":" "}
|
||||
]`))
|
||||
if err != nil {
|
||||
t.Fatalf("parse failed: %v", err)
|
||||
}
|
||||
if len(parsed.Segments) != 1 {
|
||||
t.Fatalf("segment count = %d, want 1", len(parsed.Segments))
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseReaderOriginalInputIndexPreserved(t *testing.T) {
|
||||
input := `[
|
||||
{"start":1,"end":2,"speaker":"A","text":"one"},
|
||||
{"start":2,"end":3,"speaker":"B","text":"two"},
|
||||
{"start":3,"end":4,"speaker":"C","text":"three"}
|
||||
]`
|
||||
parsed, err := ParseReader(strings.NewReader(input))
|
||||
if err != nil {
|
||||
t.Fatalf("parse failed: %v", err)
|
||||
}
|
||||
for index, segment := range parsed.Segments {
|
||||
if segment.InputIndex != index {
|
||||
t.Fatalf("segment %d input index = %d, want %d", index, segment.InputIndex, index)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertContains(t *testing.T, err error, fragment string) {
|
||||
t.Helper()
|
||||
if err == nil {
|
||||
t.Fatalf("expected error containing %q", fragment)
|
||||
}
|
||||
if !strings.Contains(err.Error(), fragment) {
|
||||
t.Fatalf("error = %q, want substring %q", err.Error(), fragment)
|
||||
}
|
||||
}
|
||||
367
internal/trim/apply.go
Normal file
367
internal/trim/apply.go
Normal file
@@ -0,0 +1,367 @@
|
||||
package trim
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/seriatim/internal/model"
|
||||
"gitea.maximumdirect.net/eric/seriatim/internal/overlap"
|
||||
"gitea.maximumdirect.net/eric/seriatim/schema"
|
||||
)
|
||||
|
||||
// Mode controls how selector IDs are applied.
|
||||
type Mode string
|
||||
|
||||
const (
|
||||
ModeKeep Mode = "keep"
|
||||
ModeRemove Mode = "remove"
|
||||
)
|
||||
|
||||
// Options configures transcript trimming.
|
||||
type Options struct {
|
||||
Mode Mode
|
||||
Selector Selector
|
||||
AllowEmpty bool
|
||||
}
|
||||
|
||||
// Result contains trimming output and ID mapping metadata.
|
||||
type Result struct {
|
||||
Transcript schema.Transcript
|
||||
OldToNewID map[int]int
|
||||
RemovedIDs []int
|
||||
}
|
||||
|
||||
// IntermediateResult contains trimming output for intermediate schema artifacts.
|
||||
type IntermediateResult struct {
|
||||
Transcript schema.IntermediateTranscript
|
||||
OldToNewID map[int]int
|
||||
RemovedIDs []int
|
||||
}
|
||||
|
||||
// MinimalResult contains trimming output for minimal schema artifacts.
|
||||
type MinimalResult struct {
|
||||
Transcript schema.MinimalTranscript
|
||||
OldToNewID map[int]int
|
||||
RemovedIDs []int
|
||||
}
|
||||
|
||||
// Apply trims a full seriatim output transcript by segment ID.
|
||||
func Apply(input schema.Transcript, opts Options) (Result, error) {
|
||||
if err := validateMode(opts.Mode); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
|
||||
selected := opts.Selector.IDs()
|
||||
if len(selected) == 0 {
|
||||
return Result{}, fmt.Errorf("selector cannot be empty")
|
||||
}
|
||||
|
||||
inputIDs := make([]int, len(input.Segments))
|
||||
for index, segment := range input.Segments {
|
||||
inputIDs[index] = segment.ID
|
||||
}
|
||||
|
||||
idIndex, err := validateInputIDs(inputIDs)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
|
||||
if err := validateSelectedIDsExist(selected, idIndex); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
|
||||
kept := make([]schema.Segment, 0, len(input.Segments))
|
||||
removed := make([]int, 0, len(input.Segments))
|
||||
oldToNew := make(map[int]int, len(input.Segments))
|
||||
for _, segment := range input.Segments {
|
||||
keep := opts.Mode == ModeKeep && opts.Selector.Contains(segment.ID)
|
||||
if opts.Mode == ModeRemove {
|
||||
keep = !opts.Selector.Contains(segment.ID)
|
||||
}
|
||||
|
||||
if !keep {
|
||||
removed = append(removed, segment.ID)
|
||||
continue
|
||||
}
|
||||
|
||||
rewritten := copySegment(segment)
|
||||
rewritten.ID = len(kept) + 1
|
||||
rewritten.OverlapGroupID = 0
|
||||
kept = append(kept, rewritten)
|
||||
oldToNew[segment.ID] = rewritten.ID
|
||||
}
|
||||
|
||||
if len(kept) == 0 && !opts.AllowEmpty {
|
||||
return Result{}, fmt.Errorf("trim operation produced an empty transcript; set AllowEmpty to true to permit this")
|
||||
}
|
||||
|
||||
kept, groups := recomputeOverlapGroups(kept)
|
||||
if groups == nil {
|
||||
groups = make([]schema.OverlapGroup, 0)
|
||||
}
|
||||
|
||||
out := copyTranscript(input)
|
||||
out.Segments = kept
|
||||
out.OverlapGroups = groups
|
||||
return Result{
|
||||
Transcript: out,
|
||||
OldToNewID: oldToNew,
|
||||
RemovedIDs: removed,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ApplyIntermediate trims an intermediate seriatim output transcript by
|
||||
// segment ID.
|
||||
func ApplyIntermediate(input schema.IntermediateTranscript, opts Options) (IntermediateResult, error) {
|
||||
if err := validateMode(opts.Mode); err != nil {
|
||||
return IntermediateResult{}, err
|
||||
}
|
||||
|
||||
selected := opts.Selector.IDs()
|
||||
if len(selected) == 0 {
|
||||
return IntermediateResult{}, fmt.Errorf("selector cannot be empty")
|
||||
}
|
||||
|
||||
inputIDs := make([]int, len(input.Segments))
|
||||
for index, segment := range input.Segments {
|
||||
inputIDs[index] = segment.ID
|
||||
}
|
||||
idIndex, err := validateInputIDs(inputIDs)
|
||||
if err != nil {
|
||||
return IntermediateResult{}, err
|
||||
}
|
||||
if err := validateSelectedIDsExist(selected, idIndex); err != nil {
|
||||
return IntermediateResult{}, err
|
||||
}
|
||||
|
||||
kept := make([]schema.IntermediateSegment, 0, len(input.Segments))
|
||||
removed := make([]int, 0, len(input.Segments))
|
||||
oldToNew := make(map[int]int, len(input.Segments))
|
||||
for _, segment := range input.Segments {
|
||||
keep := opts.Mode == ModeKeep && opts.Selector.Contains(segment.ID)
|
||||
if opts.Mode == ModeRemove {
|
||||
keep = !opts.Selector.Contains(segment.ID)
|
||||
}
|
||||
if !keep {
|
||||
removed = append(removed, segment.ID)
|
||||
continue
|
||||
}
|
||||
|
||||
rewritten := schema.IntermediateSegment{
|
||||
ID: len(kept) + 1,
|
||||
Start: segment.Start,
|
||||
End: segment.End,
|
||||
Speaker: segment.Speaker,
|
||||
Text: segment.Text,
|
||||
Categories: append([]string(nil), segment.Categories...),
|
||||
}
|
||||
kept = append(kept, rewritten)
|
||||
oldToNew[segment.ID] = rewritten.ID
|
||||
}
|
||||
|
||||
if len(kept) == 0 && !opts.AllowEmpty {
|
||||
return IntermediateResult{}, fmt.Errorf("trim operation produced an empty transcript; set AllowEmpty to true to permit this")
|
||||
}
|
||||
|
||||
return IntermediateResult{
|
||||
Transcript: schema.IntermediateTranscript{
|
||||
Metadata: schema.IntermediateMetadata{
|
||||
Application: input.Metadata.Application,
|
||||
Version: input.Metadata.Version,
|
||||
OutputSchema: input.Metadata.OutputSchema,
|
||||
},
|
||||
Segments: kept,
|
||||
},
|
||||
OldToNewID: oldToNew,
|
||||
RemovedIDs: removed,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ApplyMinimal trims a minimal seriatim output transcript by segment ID.
|
||||
func ApplyMinimal(input schema.MinimalTranscript, opts Options) (MinimalResult, error) {
|
||||
if err := validateMode(opts.Mode); err != nil {
|
||||
return MinimalResult{}, err
|
||||
}
|
||||
|
||||
selected := opts.Selector.IDs()
|
||||
if len(selected) == 0 {
|
||||
return MinimalResult{}, fmt.Errorf("selector cannot be empty")
|
||||
}
|
||||
|
||||
inputIDs := make([]int, len(input.Segments))
|
||||
for index, segment := range input.Segments {
|
||||
inputIDs[index] = segment.ID
|
||||
}
|
||||
idIndex, err := validateInputIDs(inputIDs)
|
||||
if err != nil {
|
||||
return MinimalResult{}, err
|
||||
}
|
||||
if err := validateSelectedIDsExist(selected, idIndex); err != nil {
|
||||
return MinimalResult{}, err
|
||||
}
|
||||
|
||||
kept := make([]schema.MinimalSegment, 0, len(input.Segments))
|
||||
removed := make([]int, 0, len(input.Segments))
|
||||
oldToNew := make(map[int]int, len(input.Segments))
|
||||
for _, segment := range input.Segments {
|
||||
keep := opts.Mode == ModeKeep && opts.Selector.Contains(segment.ID)
|
||||
if opts.Mode == ModeRemove {
|
||||
keep = !opts.Selector.Contains(segment.ID)
|
||||
}
|
||||
if !keep {
|
||||
removed = append(removed, segment.ID)
|
||||
continue
|
||||
}
|
||||
|
||||
rewritten := schema.MinimalSegment{
|
||||
ID: len(kept) + 1,
|
||||
Start: segment.Start,
|
||||
End: segment.End,
|
||||
Speaker: segment.Speaker,
|
||||
Text: segment.Text,
|
||||
}
|
||||
kept = append(kept, rewritten)
|
||||
oldToNew[segment.ID] = rewritten.ID
|
||||
}
|
||||
|
||||
if len(kept) == 0 && !opts.AllowEmpty {
|
||||
return MinimalResult{}, fmt.Errorf("trim operation produced an empty transcript; set AllowEmpty to true to permit this")
|
||||
}
|
||||
|
||||
return MinimalResult{
|
||||
Transcript: schema.MinimalTranscript{
|
||||
Metadata: schema.MinimalMetadata{
|
||||
Application: input.Metadata.Application,
|
||||
Version: input.Metadata.Version,
|
||||
OutputSchema: input.Metadata.OutputSchema,
|
||||
},
|
||||
Segments: kept,
|
||||
},
|
||||
OldToNewID: oldToNew,
|
||||
RemovedIDs: removed,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func validateMode(mode Mode) error {
|
||||
switch mode {
|
||||
case ModeKeep, ModeRemove:
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("invalid trim mode %q", mode)
|
||||
}
|
||||
}
|
||||
|
||||
func validateInputIDs(ids []int) (map[int]int, error) {
|
||||
seen := make(map[int]int, len(ids))
|
||||
for index, id := range ids {
|
||||
if id <= 0 {
|
||||
return nil, fmt.Errorf("input transcript has non-positive segment ID %d at index %d", id, index)
|
||||
}
|
||||
if firstIndex, exists := seen[id]; exists {
|
||||
return nil, fmt.Errorf("input transcript has duplicate segment ID %d at indexes %d and %d", id, firstIndex, index)
|
||||
}
|
||||
seen[id] = index
|
||||
}
|
||||
|
||||
for id := 1; id <= len(ids); id++ {
|
||||
if _, exists := seen[id]; !exists {
|
||||
return nil, fmt.Errorf("input transcript segment IDs must be sequential 1..%d; missing ID %d", len(ids), id)
|
||||
}
|
||||
}
|
||||
return seen, nil
|
||||
}
|
||||
|
||||
func validateSelectedIDsExist(selected []int, idIndex map[int]int) error {
|
||||
for _, id := range selected {
|
||||
if _, exists := idIndex[id]; !exists {
|
||||
return fmt.Errorf("selected segment ID %d does not exist in input transcript", id)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func recomputeOverlapGroups(segments []schema.Segment) ([]schema.Segment, []schema.OverlapGroup) {
|
||||
if len(segments) == 0 {
|
||||
return segments, make([]schema.OverlapGroup, 0)
|
||||
}
|
||||
|
||||
modelSegments := make([]model.Segment, len(segments))
|
||||
for index, segment := range segments {
|
||||
modelSegments[index] = model.Segment{
|
||||
ID: segment.ID,
|
||||
Source: segment.Source,
|
||||
SourceSegmentIndex: copyIntPtr(segment.SourceSegmentIndex),
|
||||
SourceRef: segment.SourceRef,
|
||||
DerivedFrom: append([]string(nil), segment.DerivedFrom...),
|
||||
Speaker: segment.Speaker,
|
||||
Start: segment.Start,
|
||||
End: segment.End,
|
||||
Text: segment.Text,
|
||||
Categories: append([]string(nil), segment.Categories...),
|
||||
OverlapGroupID: segment.OverlapGroupID,
|
||||
}
|
||||
}
|
||||
|
||||
detected := overlap.Detect(model.MergedTranscript{
|
||||
Segments: modelSegments,
|
||||
})
|
||||
rewrittenSegments := make([]schema.Segment, len(segments))
|
||||
for index, segment := range segments {
|
||||
rewritten := copySegment(segment)
|
||||
rewritten.OverlapGroupID = detected.Segments[index].OverlapGroupID
|
||||
rewrittenSegments[index] = rewritten
|
||||
}
|
||||
|
||||
groups := make([]schema.OverlapGroup, len(detected.OverlapGroups))
|
||||
for index, group := range detected.OverlapGroups {
|
||||
groups[index] = schema.OverlapGroup{
|
||||
ID: group.ID,
|
||||
Start: group.Start,
|
||||
End: group.End,
|
||||
Segments: append([]string(nil), group.Segments...),
|
||||
Speakers: append([]string(nil), group.Speakers...),
|
||||
Class: group.Class,
|
||||
Resolution: group.Resolution,
|
||||
}
|
||||
}
|
||||
return rewrittenSegments, groups
|
||||
}
|
||||
|
||||
func copyTranscript(input schema.Transcript) schema.Transcript {
|
||||
return schema.Transcript{
|
||||
Metadata: schema.Metadata{
|
||||
Application: input.Metadata.Application,
|
||||
Version: input.Metadata.Version,
|
||||
InputReader: input.Metadata.InputReader,
|
||||
InputFiles: append([]string(nil), input.Metadata.InputFiles...),
|
||||
PreprocessingModules: append([]string(nil), input.Metadata.PreprocessingModules...),
|
||||
PostprocessingModules: append([]string(nil), input.Metadata.PostprocessingModules...),
|
||||
OutputModules: append([]string(nil), input.Metadata.OutputModules...),
|
||||
},
|
||||
Segments: append([]schema.Segment(nil), input.Segments...),
|
||||
OverlapGroups: append([]schema.OverlapGroup(nil), input.OverlapGroups...),
|
||||
}
|
||||
}
|
||||
|
||||
func copySegment(input schema.Segment) schema.Segment {
|
||||
return schema.Segment{
|
||||
ID: input.ID,
|
||||
Source: input.Source,
|
||||
SourceSegmentIndex: copyIntPtr(input.SourceSegmentIndex),
|
||||
SourceRef: input.SourceRef,
|
||||
DerivedFrom: append([]string(nil), input.DerivedFrom...),
|
||||
Speaker: input.Speaker,
|
||||
Start: input.Start,
|
||||
End: input.End,
|
||||
Text: input.Text,
|
||||
Categories: append([]string(nil), input.Categories...),
|
||||
OverlapGroupID: input.OverlapGroupID,
|
||||
}
|
||||
}
|
||||
|
||||
func copyIntPtr(value *int) *int {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
copied := *value
|
||||
return &copied
|
||||
}
|
||||
668
internal/trim/apply_test.go
Normal file
668
internal/trim/apply_test.go
Normal file
@@ -0,0 +1,668 @@
|
||||
package trim
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/seriatim/schema"
|
||||
)
|
||||
|
||||
func TestApplyKeepModeRenumbersFromOne(t *testing.T) {
|
||||
input := fullTranscriptFixture()
|
||||
selector := mustParseSelector(t, "2,4")
|
||||
|
||||
result, err := Apply(input, Options{
|
||||
Mode: ModeKeep,
|
||||
Selector: selector,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("apply failed: %v", err)
|
||||
}
|
||||
|
||||
if len(result.Transcript.Segments) != 2 {
|
||||
t.Fatalf("segment count = %d, want 2", len(result.Transcript.Segments))
|
||||
}
|
||||
assertSegmentIDs(t, result.Transcript.Segments, []int{1, 2})
|
||||
assertSegmentTexts(t, result.Transcript.Segments, []string{"beta", "delta"})
|
||||
assertIntMap(t, result.OldToNewID, map[int]int{2: 1, 4: 2})
|
||||
assertIntSlice(t, result.RemovedIDs, []int{1, 3})
|
||||
}
|
||||
|
||||
func TestApplyRemoveModeRenumbersFromOne(t *testing.T) {
|
||||
input := fullTranscriptFixture()
|
||||
selector := mustParseSelector(t, "2,4")
|
||||
|
||||
result, err := Apply(input, Options{
|
||||
Mode: ModeRemove,
|
||||
Selector: selector,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("apply failed: %v", err)
|
||||
}
|
||||
|
||||
assertSegmentIDs(t, result.Transcript.Segments, []int{1, 2})
|
||||
assertSegmentTexts(t, result.Transcript.Segments, []string{"alpha", "gamma"})
|
||||
assertIntMap(t, result.OldToNewID, map[int]int{1: 1, 3: 2})
|
||||
assertIntSlice(t, result.RemovedIDs, []int{2, 4})
|
||||
}
|
||||
|
||||
func TestApplySelectorOrderDoesNotChangeTranscriptOrder(t *testing.T) {
|
||||
input := fullTranscriptFixture()
|
||||
selector := mustParseSelector(t, "4,1,3")
|
||||
|
||||
result, err := Apply(input, Options{
|
||||
Mode: ModeKeep,
|
||||
Selector: selector,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("apply failed: %v", err)
|
||||
}
|
||||
|
||||
assertSegmentIDs(t, result.Transcript.Segments, []int{1, 2, 3})
|
||||
assertSegmentTexts(t, result.Transcript.Segments, []string{"alpha", "gamma", "delta"})
|
||||
}
|
||||
|
||||
func TestApplyFailsWhenSelectedIDDoesNotExist(t *testing.T) {
|
||||
input := fullTranscriptFixture()
|
||||
selector := mustParseSelector(t, "2,99")
|
||||
|
||||
_, err := Apply(input, Options{
|
||||
Mode: ModeKeep,
|
||||
Selector: selector,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected missing selected ID error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "does not exist") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFailsOnDuplicateInputIDs(t *testing.T) {
|
||||
input := fullTranscriptFixture()
|
||||
input.Segments[2].ID = 2
|
||||
selector := mustParseSelector(t, "2")
|
||||
|
||||
_, err := Apply(input, Options{
|
||||
Mode: ModeKeep,
|
||||
Selector: selector,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected duplicate input ID error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "duplicate segment ID") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFailsOnMissingOrNonSequentialInputIDs(t *testing.T) {
|
||||
input := fullTranscriptFixture()
|
||||
input.Segments[1].ID = 5
|
||||
selector := mustParseSelector(t, "1")
|
||||
|
||||
_, err := Apply(input, Options{
|
||||
Mode: ModeKeep,
|
||||
Selector: selector,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected non-sequential input ID error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "must be sequential") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFailsOnNonPositiveInputIDs(t *testing.T) {
|
||||
input := fullTranscriptFixture()
|
||||
input.Segments[0].ID = 0
|
||||
selector := mustParseSelector(t, "1")
|
||||
|
||||
_, err := Apply(input, Options{
|
||||
Mode: ModeKeep,
|
||||
Selector: selector,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected non-positive input ID error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "non-positive") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyEmptyOutputFailsUnlessAllowEmpty(t *testing.T) {
|
||||
input := fullTranscriptFixture()
|
||||
selector := mustParseSelector(t, "1-4")
|
||||
|
||||
_, err := Apply(input, Options{
|
||||
Mode: ModeRemove,
|
||||
Selector: selector,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected empty-output error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "empty transcript") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
allowed, err := Apply(input, Options{
|
||||
Mode: ModeRemove,
|
||||
Selector: selector,
|
||||
AllowEmpty: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("apply with AllowEmpty failed: %v", err)
|
||||
}
|
||||
if len(allowed.Transcript.Segments) != 0 {
|
||||
t.Fatalf("segment count = %d, want 0", len(allowed.Transcript.Segments))
|
||||
}
|
||||
assertIntMap(t, allowed.OldToNewID, map[int]int{})
|
||||
assertIntSlice(t, allowed.RemovedIDs, []int{1, 2, 3, 4})
|
||||
}
|
||||
|
||||
func TestApplyPreservesRetainedSegmentFieldsAndClearsOverlapIDs(t *testing.T) {
|
||||
input := fullTranscriptFixture()
|
||||
selector := mustParseSelector(t, "2")
|
||||
|
||||
result, err := Apply(input, Options{
|
||||
Mode: ModeKeep,
|
||||
Selector: selector,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("apply failed: %v", err)
|
||||
}
|
||||
|
||||
if len(result.Transcript.Segments) != 1 {
|
||||
t.Fatalf("segment count = %d, want 1", len(result.Transcript.Segments))
|
||||
}
|
||||
segment := result.Transcript.Segments[0]
|
||||
if segment.ID != 1 {
|
||||
t.Fatalf("segment ID = %d, want 1", segment.ID)
|
||||
}
|
||||
if segment.Source != "b.json" {
|
||||
t.Fatalf("source = %q, want %q", segment.Source, "b.json")
|
||||
}
|
||||
if segment.SourceSegmentIndex == nil || *segment.SourceSegmentIndex != 20 {
|
||||
t.Fatalf("source_segment_index = %v, want 20", segment.SourceSegmentIndex)
|
||||
}
|
||||
if segment.SourceRef != "b.json#20" {
|
||||
t.Fatalf("source_ref = %q, want %q", segment.SourceRef, "b.json#20")
|
||||
}
|
||||
if !equalStringSlices(segment.DerivedFrom, []string{"b.json#19", "b.json#20"}) {
|
||||
t.Fatalf("derived_from = %v, want %v", segment.DerivedFrom, []string{"b.json#19", "b.json#20"})
|
||||
}
|
||||
if !equalStringSlices(segment.Categories, []string{"filler", "backchannel"}) {
|
||||
t.Fatalf("categories = %v, want %v", segment.Categories, []string{"filler", "backchannel"})
|
||||
}
|
||||
if segment.Speaker != "Bob" {
|
||||
t.Fatalf("speaker = %q, want Bob", segment.Speaker)
|
||||
}
|
||||
if segment.Start != 2 || segment.End != 3 {
|
||||
t.Fatalf("times = %.3f-%.3f, want 2.000-3.000", segment.Start, segment.End)
|
||||
}
|
||||
if segment.Text != "beta" {
|
||||
t.Fatalf("text = %q, want beta", segment.Text)
|
||||
}
|
||||
if segment.OverlapGroupID != 0 {
|
||||
t.Fatalf("overlap_group_id = %d, want 0", segment.OverlapGroupID)
|
||||
}
|
||||
if len(result.Transcript.OverlapGroups) != 0 {
|
||||
t.Fatalf("overlap_groups count = %d, want 0", len(result.Transcript.OverlapGroups))
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFullSchemaRemovesStaleOverlapGroups(t *testing.T) {
|
||||
input := overlapTranscriptFixture()
|
||||
selector := mustParseSelector(t, "1,3")
|
||||
|
||||
result, err := Apply(input, Options{
|
||||
Mode: ModeKeep,
|
||||
Selector: selector,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("apply failed: %v", err)
|
||||
}
|
||||
|
||||
if len(result.Transcript.OverlapGroups) != 0 {
|
||||
t.Fatalf("overlap_groups count = %d, want 0", len(result.Transcript.OverlapGroups))
|
||||
}
|
||||
for index, segment := range result.Transcript.Segments {
|
||||
if segment.OverlapGroupID != 0 {
|
||||
t.Fatalf("segment %d overlap_group_id = %d, want 0", index, segment.OverlapGroupID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFullSchemaRecomputesOverlapGroup(t *testing.T) {
|
||||
input := overlapTranscriptFixture()
|
||||
selector := mustParseSelector(t, "1,2")
|
||||
|
||||
result, err := Apply(input, Options{
|
||||
Mode: ModeKeep,
|
||||
Selector: selector,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("apply failed: %v", err)
|
||||
}
|
||||
|
||||
assertSegmentIDs(t, result.Transcript.Segments, []int{1, 2})
|
||||
assertIntSlice(t, []int{
|
||||
result.Transcript.Segments[0].OverlapGroupID,
|
||||
result.Transcript.Segments[1].OverlapGroupID,
|
||||
}, []int{1, 1})
|
||||
if len(result.Transcript.OverlapGroups) != 1 {
|
||||
t.Fatalf("overlap_groups count = %d, want 1", len(result.Transcript.OverlapGroups))
|
||||
}
|
||||
group := result.Transcript.OverlapGroups[0]
|
||||
if group.ID != 1 {
|
||||
t.Fatalf("group ID = %d, want 1", group.ID)
|
||||
}
|
||||
if group.Start != 1 || group.End != 4 {
|
||||
t.Fatalf("group times = %.3f-%.3f, want 1.000-4.000", group.Start, group.End)
|
||||
}
|
||||
if !equalStringSlices(group.Segments, []string{"a.json#10", "b.json#20"}) {
|
||||
t.Fatalf("group segments = %v, want %v", group.Segments, []string{"a.json#10", "b.json#20"})
|
||||
}
|
||||
if !equalStringSlices(group.Speakers, []string{"Alice", "Bob"}) {
|
||||
t.Fatalf("group speakers = %v, want %v", group.Speakers, []string{"Alice", "Bob"})
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFullSchemaDropsGroupWhenFewerThanTwoSpeakersRemain(t *testing.T) {
|
||||
input := overlapTranscriptFixture()
|
||||
selector := mustParseSelector(t, "1")
|
||||
|
||||
result, err := Apply(input, Options{
|
||||
Mode: ModeKeep,
|
||||
Selector: selector,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("apply failed: %v", err)
|
||||
}
|
||||
|
||||
if len(result.Transcript.OverlapGroups) != 0 {
|
||||
t.Fatalf("overlap_groups count = %d, want 0", len(result.Transcript.OverlapGroups))
|
||||
}
|
||||
if len(result.Transcript.Segments) != 1 {
|
||||
t.Fatalf("segment count = %d, want 1", len(result.Transcript.Segments))
|
||||
}
|
||||
if result.Transcript.Segments[0].OverlapGroupID != 0 {
|
||||
t.Fatalf("segment overlap_group_id = %d, want 0", result.Transcript.Segments[0].OverlapGroupID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFullSchemaHandlesTransitiveOverlaps(t *testing.T) {
|
||||
input := transitiveOverlapFixture()
|
||||
selector := mustParseSelector(t, "1-3")
|
||||
|
||||
result, err := Apply(input, Options{
|
||||
Mode: ModeKeep,
|
||||
Selector: selector,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("apply failed: %v", err)
|
||||
}
|
||||
|
||||
if len(result.Transcript.OverlapGroups) != 1 {
|
||||
t.Fatalf("overlap_groups count = %d, want 1", len(result.Transcript.OverlapGroups))
|
||||
}
|
||||
assertIntSlice(t, []int{
|
||||
result.Transcript.Segments[0].OverlapGroupID,
|
||||
result.Transcript.Segments[1].OverlapGroupID,
|
||||
result.Transcript.Segments[2].OverlapGroupID,
|
||||
}, []int{1, 1, 1})
|
||||
group := result.Transcript.OverlapGroups[0]
|
||||
if group.Start != 10 || group.End != 15 {
|
||||
t.Fatalf("group times = %.3f-%.3f, want 10.000-15.000", group.Start, group.End)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFullSchemaBoundaryTouchingNotGrouped(t *testing.T) {
|
||||
input := boundaryFixture()
|
||||
selector := mustParseSelector(t, "1-2")
|
||||
|
||||
result, err := Apply(input, Options{
|
||||
Mode: ModeKeep,
|
||||
Selector: selector,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("apply failed: %v", err)
|
||||
}
|
||||
|
||||
if len(result.Transcript.OverlapGroups) != 0 {
|
||||
t.Fatalf("overlap_groups count = %d, want 0", len(result.Transcript.OverlapGroups))
|
||||
}
|
||||
assertIntSlice(t, []int{
|
||||
result.Transcript.Segments[0].OverlapGroupID,
|
||||
result.Transcript.Segments[1].OverlapGroupID,
|
||||
}, []int{0, 0})
|
||||
}
|
||||
|
||||
func TestApplyIntermediateDoesNotIncludeOverlapGroups(t *testing.T) {
|
||||
input := schema.IntermediateTranscript{
|
||||
Metadata: schema.IntermediateMetadata{
|
||||
Application: "seriatim",
|
||||
Version: "v-test",
|
||||
OutputSchema: "seriatim-intermediate",
|
||||
},
|
||||
Segments: []schema.IntermediateSegment{
|
||||
{ID: 1, Start: 1, End: 3, Speaker: "Alice", Text: "alpha", Categories: []string{"word-run"}},
|
||||
{ID: 2, Start: 2, End: 4, Speaker: "Bob", Text: "beta", Categories: []string{"filler"}},
|
||||
},
|
||||
}
|
||||
selector := mustParseSelector(t, "1")
|
||||
result, err := ApplyIntermediate(input, Options{
|
||||
Mode: ModeKeep,
|
||||
Selector: selector,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("apply intermediate failed: %v", err)
|
||||
}
|
||||
if len(result.Transcript.Segments) != 1 {
|
||||
t.Fatalf("segment count = %d, want 1", len(result.Transcript.Segments))
|
||||
}
|
||||
if result.Transcript.Segments[0].ID != 1 {
|
||||
t.Fatalf("segment id = %d, want 1", result.Transcript.Segments[0].ID)
|
||||
}
|
||||
if err := schema.ValidateIntermediateTranscript(result.Transcript); err != nil {
|
||||
t.Fatalf("intermediate output should remain valid: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyMinimalDoesNotIncludeOverlapGroups(t *testing.T) {
|
||||
input := schema.MinimalTranscript{
|
||||
Metadata: schema.MinimalMetadata{
|
||||
Application: "seriatim",
|
||||
Version: "v-test",
|
||||
OutputSchema: "seriatim-minimal",
|
||||
},
|
||||
Segments: []schema.MinimalSegment{
|
||||
{ID: 1, Start: 1, End: 3, Speaker: "Alice", Text: "alpha"},
|
||||
{ID: 2, Start: 2, End: 4, Speaker: "Bob", Text: "beta"},
|
||||
},
|
||||
}
|
||||
selector := mustParseSelector(t, "2")
|
||||
result, err := ApplyMinimal(input, Options{
|
||||
Mode: ModeKeep,
|
||||
Selector: selector,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("apply minimal failed: %v", err)
|
||||
}
|
||||
if len(result.Transcript.Segments) != 1 {
|
||||
t.Fatalf("segment count = %d, want 1", len(result.Transcript.Segments))
|
||||
}
|
||||
if result.Transcript.Segments[0].ID != 1 {
|
||||
t.Fatalf("segment id = %d, want 1", result.Transcript.Segments[0].ID)
|
||||
}
|
||||
if err := schema.ValidateMinimalTranscript(result.Transcript); err != nil {
|
||||
t.Fatalf("minimal output should remain valid: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyOutputInvariantsValidAfterRenumberAndOverlapRecompute(t *testing.T) {
|
||||
input := overlapTranscriptFixture()
|
||||
selector := mustParseSelector(t, "2,1")
|
||||
|
||||
result, err := Apply(input, Options{
|
||||
Mode: ModeKeep,
|
||||
Selector: selector,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("apply failed: %v", err)
|
||||
}
|
||||
|
||||
if err := schema.ValidateTranscript(result.Transcript); err != nil {
|
||||
t.Fatalf("trim output should remain valid: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustParseSelector(t *testing.T, value string) Selector {
|
||||
t.Helper()
|
||||
selector, err := ParseSelector(value)
|
||||
if err != nil {
|
||||
t.Fatalf("selector parse failed for %q: %v", value, err)
|
||||
}
|
||||
return selector
|
||||
}
|
||||
|
||||
func fullTranscriptFixture() schema.Transcript {
|
||||
firstIndex := 10
|
||||
secondIndex := 20
|
||||
thirdIndex := 30
|
||||
fourthIndex := 40
|
||||
|
||||
return schema.Transcript{
|
||||
Metadata: schema.Metadata{
|
||||
Application: "seriatim",
|
||||
Version: "v-test",
|
||||
InputReader: "json-files",
|
||||
InputFiles: []string{"a.json", "b.json"},
|
||||
PreprocessingModules: []string{"validate-raw"},
|
||||
PostprocessingModules: []string{"detect-overlaps"},
|
||||
OutputModules: []string{"json"},
|
||||
},
|
||||
Segments: []schema.Segment{
|
||||
{
|
||||
ID: 1,
|
||||
Source: "a.json",
|
||||
SourceSegmentIndex: &firstIndex,
|
||||
SourceRef: "a.json#10",
|
||||
DerivedFrom: []string{"a.json#10"},
|
||||
Speaker: "Alice",
|
||||
Start: 1,
|
||||
End: 2,
|
||||
Text: "alpha",
|
||||
Categories: []string{"word-run"},
|
||||
OverlapGroupID: 7,
|
||||
},
|
||||
{
|
||||
ID: 2,
|
||||
Source: "b.json",
|
||||
SourceSegmentIndex: &secondIndex,
|
||||
SourceRef: "b.json#20",
|
||||
DerivedFrom: []string{"b.json#19", "b.json#20"},
|
||||
Speaker: "Bob",
|
||||
Start: 2,
|
||||
End: 3,
|
||||
Text: "beta",
|
||||
Categories: []string{"filler", "backchannel"},
|
||||
OverlapGroupID: 7,
|
||||
},
|
||||
{
|
||||
ID: 3,
|
||||
Source: "c.json",
|
||||
SourceSegmentIndex: &thirdIndex,
|
||||
SourceRef: "c.json#30",
|
||||
DerivedFrom: []string{"c.json#30"},
|
||||
Speaker: "Carol",
|
||||
Start: 3,
|
||||
End: 4,
|
||||
Text: "gamma",
|
||||
Categories: []string{"normal"},
|
||||
OverlapGroupID: 8,
|
||||
},
|
||||
{
|
||||
ID: 4,
|
||||
Source: "d.json",
|
||||
SourceSegmentIndex: &fourthIndex,
|
||||
SourceRef: "d.json#40",
|
||||
DerivedFrom: []string{"d.json#40"},
|
||||
Speaker: "Dan",
|
||||
Start: 4,
|
||||
End: 5,
|
||||
Text: "delta",
|
||||
Categories: []string{"normal"},
|
||||
OverlapGroupID: 9,
|
||||
},
|
||||
},
|
||||
OverlapGroups: []schema.OverlapGroup{
|
||||
{
|
||||
ID: 7,
|
||||
Start: 1.5,
|
||||
End: 3.1,
|
||||
Segments: []string{"a.json#10", "b.json#20"},
|
||||
Speakers: []string{"Alice", "Bob"},
|
||||
Class: "unknown",
|
||||
Resolution: "unresolved",
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func overlapTranscriptFixture() schema.Transcript {
|
||||
first := 10
|
||||
second := 20
|
||||
third := 30
|
||||
|
||||
return schema.Transcript{
|
||||
Metadata: schema.Metadata{
|
||||
Application: "seriatim",
|
||||
Version: "v-test",
|
||||
InputReader: "json-files",
|
||||
InputFiles: []string{"a.json", "b.json", "c.json"},
|
||||
PreprocessingModules: []string{"validate-raw"},
|
||||
PostprocessingModules: []string{"detect-overlaps"},
|
||||
OutputModules: []string{"json"},
|
||||
},
|
||||
Segments: []schema.Segment{
|
||||
{
|
||||
ID: 1,
|
||||
Source: "a.json",
|
||||
SourceSegmentIndex: &first,
|
||||
SourceRef: "a.json#10",
|
||||
Speaker: "Alice",
|
||||
Start: 1,
|
||||
End: 4,
|
||||
Text: "a",
|
||||
OverlapGroupID: 99,
|
||||
},
|
||||
{
|
||||
ID: 2,
|
||||
Source: "b.json",
|
||||
SourceSegmentIndex: &second,
|
||||
SourceRef: "b.json#20",
|
||||
Speaker: "Bob",
|
||||
Start: 2,
|
||||
End: 3,
|
||||
Text: "b",
|
||||
OverlapGroupID: 99,
|
||||
},
|
||||
{
|
||||
ID: 3,
|
||||
Source: "c.json",
|
||||
SourceSegmentIndex: &third,
|
||||
SourceRef: "c.json#30",
|
||||
Speaker: "Carol",
|
||||
Start: 10,
|
||||
End: 11,
|
||||
Text: "c",
|
||||
OverlapGroupID: 100,
|
||||
},
|
||||
},
|
||||
OverlapGroups: []schema.OverlapGroup{
|
||||
{
|
||||
ID: 99,
|
||||
Start: 0,
|
||||
End: 100,
|
||||
Segments: []string{"stale#1", "stale#2"},
|
||||
Speakers: []string{"stale"},
|
||||
Class: "unknown",
|
||||
Resolution: "unresolved",
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func transitiveOverlapFixture() schema.Transcript {
|
||||
one := 1
|
||||
two := 2
|
||||
three := 3
|
||||
return schema.Transcript{
|
||||
Metadata: schema.Metadata{
|
||||
Application: "seriatim",
|
||||
Version: "v-test",
|
||||
},
|
||||
Segments: []schema.Segment{
|
||||
{ID: 1, Source: "a.json", SourceSegmentIndex: &one, Speaker: "Alice", Start: 10, End: 14, Text: "a"},
|
||||
{ID: 2, Source: "b.json", SourceSegmentIndex: &two, Speaker: "Bob", Start: 12, End: 13, Text: "b"},
|
||||
{ID: 3, Source: "c.json", SourceSegmentIndex: &three, Speaker: "Carol", Start: 13.5, End: 15, Text: "c"},
|
||||
},
|
||||
OverlapGroups: []schema.OverlapGroup{{ID: 77}},
|
||||
}
|
||||
}
|
||||
|
||||
func boundaryFixture() schema.Transcript {
|
||||
one := 1
|
||||
two := 2
|
||||
return schema.Transcript{
|
||||
Metadata: schema.Metadata{
|
||||
Application: "seriatim",
|
||||
Version: "v-test",
|
||||
},
|
||||
Segments: []schema.Segment{
|
||||
{ID: 1, Source: "a.json", SourceSegmentIndex: &one, Speaker: "Alice", Start: 1, End: 2, Text: "a", OverlapGroupID: 7},
|
||||
{ID: 2, Source: "b.json", SourceSegmentIndex: &two, Speaker: "Bob", Start: 2, End: 3, Text: "b", OverlapGroupID: 7},
|
||||
},
|
||||
OverlapGroups: []schema.OverlapGroup{{ID: 7, Start: 1, End: 3}},
|
||||
}
|
||||
}
|
||||
|
||||
func assertSegmentIDs(t *testing.T, segments []schema.Segment, want []int) {
|
||||
t.Helper()
|
||||
got := make([]int, len(segments))
|
||||
for index, segment := range segments {
|
||||
got[index] = segment.ID
|
||||
}
|
||||
assertIntSlice(t, got, want)
|
||||
}
|
||||
|
||||
func assertSegmentTexts(t *testing.T, segments []schema.Segment, want []string) {
|
||||
t.Helper()
|
||||
got := make([]string, len(segments))
|
||||
for index, segment := range segments {
|
||||
got[index] = segment.Text
|
||||
}
|
||||
if !equalStringSlices(got, want) {
|
||||
t.Fatalf("segment texts = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func assertIntSlice(t *testing.T, got []int, want []int) {
|
||||
t.Helper()
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("slice length = %d, want %d", len(got), len(want))
|
||||
}
|
||||
for index := range got {
|
||||
if got[index] != want[index] {
|
||||
t.Fatalf("slice[%d] = %d, want %d (full got=%v, want=%v)", index, got[index], want[index], got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertIntMap(t *testing.T, got map[int]int, want map[int]int) {
|
||||
t.Helper()
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("map length = %d, want %d", len(got), len(want))
|
||||
}
|
||||
for key, wantValue := range want {
|
||||
gotValue, exists := got[key]
|
||||
if !exists {
|
||||
t.Fatalf("missing map key %d", key)
|
||||
}
|
||||
if gotValue != wantValue {
|
||||
t.Fatalf("map[%d] = %d, want %d", key, gotValue, wantValue)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func equalStringSlices(got []string, want []string) bool {
|
||||
if len(got) != len(want) {
|
||||
return false
|
||||
}
|
||||
for index := range got {
|
||||
if got[index] != want[index] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
396
internal/trim/artifact.go
Normal file
396
internal/trim/artifact.go
Normal file
@@ -0,0 +1,396 @@
|
||||
package trim
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/seriatim/schema"
|
||||
)
|
||||
|
||||
const (
|
||||
SchemaMinimal = "seriatim-minimal"
|
||||
SchemaIntermediate = "seriatim-intermediate"
|
||||
SchemaFull = "seriatim-full"
|
||||
)
|
||||
|
||||
// Artifact stores a parsed seriatim output artifact of one supported schema.
|
||||
type Artifact struct {
|
||||
Schema string
|
||||
Full *schema.Transcript
|
||||
Intermediate *schema.IntermediateTranscript
|
||||
Minimal *schema.MinimalTranscript
|
||||
}
|
||||
|
||||
// ApplyArtifactResult contains trimmed artifact output and ID mapping metadata.
|
||||
type ApplyArtifactResult struct {
|
||||
Artifact Artifact
|
||||
OldToNewID map[int]int
|
||||
RemovedIDs []int
|
||||
OverlapGroupsRecomputed bool
|
||||
}
|
||||
|
||||
// ParseArtifactJSON parses and validates a serialized seriatim output artifact.
|
||||
func ParseArtifactJSON(data []byte) (Artifact, error) {
|
||||
var decoded any
|
||||
if err := json.Unmarshal(data, &decoded); err != nil {
|
||||
return Artifact{}, fmt.Errorf("input JSON is malformed: %w", err)
|
||||
}
|
||||
|
||||
var full schema.Transcript
|
||||
if err := json.Unmarshal(data, &full); err == nil {
|
||||
if err := schema.ValidateTranscript(full); err == nil {
|
||||
return Artifact{
|
||||
Schema: SchemaFull,
|
||||
Full: &full,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
var intermediate schema.IntermediateTranscript
|
||||
if err := json.Unmarshal(data, &intermediate); err == nil {
|
||||
if err := schema.ValidateIntermediateTranscript(intermediate); err == nil {
|
||||
return Artifact{
|
||||
Schema: SchemaIntermediate,
|
||||
Intermediate: &intermediate,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
var minimal schema.MinimalTranscript
|
||||
if err := json.Unmarshal(data, &minimal); err == nil {
|
||||
if err := schema.ValidateMinimalTranscript(minimal); err == nil {
|
||||
return Artifact{
|
||||
Schema: SchemaMinimal,
|
||||
Minimal: &minimal,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
return Artifact{}, fmt.Errorf("input JSON is not a valid seriatim output artifact")
|
||||
}
|
||||
|
||||
// ValidateArtifact validates an artifact against its declared schema.
|
||||
func ValidateArtifact(artifact Artifact) error {
|
||||
switch artifact.Schema {
|
||||
case SchemaFull:
|
||||
if artifact.Full == nil {
|
||||
return fmt.Errorf("full artifact payload is missing")
|
||||
}
|
||||
return schema.ValidateTranscript(*artifact.Full)
|
||||
case SchemaIntermediate:
|
||||
if artifact.Intermediate == nil {
|
||||
return fmt.Errorf("intermediate artifact payload is missing")
|
||||
}
|
||||
return schema.ValidateIntermediateTranscript(*artifact.Intermediate)
|
||||
case SchemaMinimal:
|
||||
if artifact.Minimal == nil {
|
||||
return fmt.Errorf("minimal artifact payload is missing")
|
||||
}
|
||||
return schema.ValidateMinimalTranscript(*artifact.Minimal)
|
||||
default:
|
||||
return fmt.Errorf("unsupported artifact schema %q", artifact.Schema)
|
||||
}
|
||||
}
|
||||
|
||||
// Value returns the artifact value for JSON serialization.
|
||||
func (artifact Artifact) Value() any {
|
||||
switch artifact.Schema {
|
||||
case SchemaFull:
|
||||
if artifact.Full == nil {
|
||||
return schema.Transcript{}
|
||||
}
|
||||
return *artifact.Full
|
||||
case SchemaIntermediate:
|
||||
if artifact.Intermediate == nil {
|
||||
return schema.IntermediateTranscript{}
|
||||
}
|
||||
return *artifact.Intermediate
|
||||
case SchemaMinimal:
|
||||
if artifact.Minimal == nil {
|
||||
return schema.MinimalTranscript{}
|
||||
}
|
||||
return *artifact.Minimal
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// SegmentCount returns the number of segments in the artifact.
|
||||
func (artifact Artifact) SegmentCount() int {
|
||||
switch artifact.Schema {
|
||||
case SchemaFull:
|
||||
if artifact.Full == nil {
|
||||
return 0
|
||||
}
|
||||
return len(artifact.Full.Segments)
|
||||
case SchemaIntermediate:
|
||||
if artifact.Intermediate == nil {
|
||||
return 0
|
||||
}
|
||||
return len(artifact.Intermediate.Segments)
|
||||
case SchemaMinimal:
|
||||
if artifact.Minimal == nil {
|
||||
return 0
|
||||
}
|
||||
return len(artifact.Minimal.Segments)
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
// Application returns artifact metadata application name.
|
||||
func (artifact Artifact) Application() string {
|
||||
switch artifact.Schema {
|
||||
case SchemaFull:
|
||||
if artifact.Full == nil {
|
||||
return ""
|
||||
}
|
||||
return artifact.Full.Metadata.Application
|
||||
case SchemaIntermediate:
|
||||
if artifact.Intermediate == nil {
|
||||
return ""
|
||||
}
|
||||
return artifact.Intermediate.Metadata.Application
|
||||
case SchemaMinimal:
|
||||
if artifact.Minimal == nil {
|
||||
return ""
|
||||
}
|
||||
return artifact.Minimal.Metadata.Application
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// Version returns artifact metadata version.
|
||||
func (artifact Artifact) Version() string {
|
||||
switch artifact.Schema {
|
||||
case SchemaFull:
|
||||
if artifact.Full == nil {
|
||||
return ""
|
||||
}
|
||||
return artifact.Full.Metadata.Version
|
||||
case SchemaIntermediate:
|
||||
if artifact.Intermediate == nil {
|
||||
return ""
|
||||
}
|
||||
return artifact.Intermediate.Metadata.Version
|
||||
case SchemaMinimal:
|
||||
if artifact.Minimal == nil {
|
||||
return ""
|
||||
}
|
||||
return artifact.Minimal.Metadata.Version
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// ApplyArtifact trims a parsed artifact while preserving its input schema.
|
||||
func ApplyArtifact(input Artifact, opts Options) (ApplyArtifactResult, error) {
|
||||
switch input.Schema {
|
||||
case SchemaFull:
|
||||
if input.Full == nil {
|
||||
return ApplyArtifactResult{}, fmt.Errorf("full artifact payload is missing")
|
||||
}
|
||||
result, err := Apply(*input.Full, opts)
|
||||
if err != nil {
|
||||
return ApplyArtifactResult{}, err
|
||||
}
|
||||
out := result.Transcript
|
||||
return ApplyArtifactResult{
|
||||
Artifact: Artifact{
|
||||
Schema: SchemaFull,
|
||||
Full: &out,
|
||||
},
|
||||
OldToNewID: result.OldToNewID,
|
||||
RemovedIDs: result.RemovedIDs,
|
||||
OverlapGroupsRecomputed: true,
|
||||
}, nil
|
||||
case SchemaIntermediate:
|
||||
if input.Intermediate == nil {
|
||||
return ApplyArtifactResult{}, fmt.Errorf("intermediate artifact payload is missing")
|
||||
}
|
||||
result, err := ApplyIntermediate(*input.Intermediate, opts)
|
||||
if err != nil {
|
||||
return ApplyArtifactResult{}, err
|
||||
}
|
||||
out := result.Transcript
|
||||
return ApplyArtifactResult{
|
||||
Artifact: Artifact{
|
||||
Schema: SchemaIntermediate,
|
||||
Intermediate: &out,
|
||||
},
|
||||
OldToNewID: result.OldToNewID,
|
||||
RemovedIDs: result.RemovedIDs,
|
||||
OverlapGroupsRecomputed: false,
|
||||
}, nil
|
||||
case SchemaMinimal:
|
||||
if input.Minimal == nil {
|
||||
return ApplyArtifactResult{}, fmt.Errorf("minimal artifact payload is missing")
|
||||
}
|
||||
result, err := ApplyMinimal(*input.Minimal, opts)
|
||||
if err != nil {
|
||||
return ApplyArtifactResult{}, err
|
||||
}
|
||||
out := result.Transcript
|
||||
return ApplyArtifactResult{
|
||||
Artifact: Artifact{
|
||||
Schema: SchemaMinimal,
|
||||
Minimal: &out,
|
||||
},
|
||||
OldToNewID: result.OldToNewID,
|
||||
RemovedIDs: result.RemovedIDs,
|
||||
OverlapGroupsRecomputed: false,
|
||||
}, nil
|
||||
default:
|
||||
return ApplyArtifactResult{}, fmt.Errorf("unsupported artifact schema %q", input.Schema)
|
||||
}
|
||||
}
|
||||
|
||||
// ConvertArtifact converts a parsed artifact to another supported output schema.
|
||||
func ConvertArtifact(input Artifact, outputSchema string) (Artifact, error) {
|
||||
if outputSchema == "" || outputSchema == input.Schema {
|
||||
return input, nil
|
||||
}
|
||||
|
||||
switch input.Schema {
|
||||
case SchemaFull:
|
||||
if input.Full == nil {
|
||||
return Artifact{}, fmt.Errorf("full artifact payload is missing")
|
||||
}
|
||||
switch outputSchema {
|
||||
case SchemaIntermediate:
|
||||
out := intermediateFromFull(*input.Full)
|
||||
return Artifact{
|
||||
Schema: SchemaIntermediate,
|
||||
Intermediate: &out,
|
||||
}, nil
|
||||
case SchemaMinimal:
|
||||
out := minimalFromFull(*input.Full)
|
||||
return Artifact{
|
||||
Schema: SchemaMinimal,
|
||||
Minimal: &out,
|
||||
}, nil
|
||||
default:
|
||||
return Artifact{}, fmt.Errorf("unsupported output schema %q", outputSchema)
|
||||
}
|
||||
case SchemaIntermediate:
|
||||
if input.Intermediate == nil {
|
||||
return Artifact{}, fmt.Errorf("intermediate artifact payload is missing")
|
||||
}
|
||||
switch outputSchema {
|
||||
case SchemaMinimal:
|
||||
out := minimalFromIntermediate(*input.Intermediate)
|
||||
return Artifact{
|
||||
Schema: SchemaMinimal,
|
||||
Minimal: &out,
|
||||
}, nil
|
||||
case SchemaFull:
|
||||
return Artifact{}, fmt.Errorf("cannot emit %q from %q input artifact", SchemaFull, SchemaIntermediate)
|
||||
default:
|
||||
return Artifact{}, fmt.Errorf("unsupported output schema %q", outputSchema)
|
||||
}
|
||||
case SchemaMinimal:
|
||||
if input.Minimal == nil {
|
||||
return Artifact{}, fmt.Errorf("minimal artifact payload is missing")
|
||||
}
|
||||
switch outputSchema {
|
||||
case SchemaIntermediate:
|
||||
out := intermediateFromMinimal(*input.Minimal)
|
||||
return Artifact{
|
||||
Schema: SchemaIntermediate,
|
||||
Intermediate: &out,
|
||||
}, nil
|
||||
case SchemaFull:
|
||||
return Artifact{}, fmt.Errorf("cannot emit %q from %q input artifact", SchemaFull, SchemaMinimal)
|
||||
default:
|
||||
return Artifact{}, fmt.Errorf("unsupported output schema %q", outputSchema)
|
||||
}
|
||||
default:
|
||||
return Artifact{}, fmt.Errorf("unsupported input schema %q", input.Schema)
|
||||
}
|
||||
}
|
||||
|
||||
func intermediateFromFull(input schema.Transcript) schema.IntermediateTranscript {
|
||||
segments := make([]schema.IntermediateSegment, len(input.Segments))
|
||||
for index, segment := range input.Segments {
|
||||
segments[index] = schema.IntermediateSegment{
|
||||
ID: segment.ID,
|
||||
Start: segment.Start,
|
||||
End: segment.End,
|
||||
Speaker: segment.Speaker,
|
||||
Text: segment.Text,
|
||||
Categories: append([]string(nil), segment.Categories...),
|
||||
}
|
||||
}
|
||||
return schema.IntermediateTranscript{
|
||||
Metadata: schema.IntermediateMetadata{
|
||||
Application: input.Metadata.Application,
|
||||
Version: input.Metadata.Version,
|
||||
OutputSchema: SchemaIntermediate,
|
||||
},
|
||||
Segments: segments,
|
||||
}
|
||||
}
|
||||
|
||||
func minimalFromFull(input schema.Transcript) schema.MinimalTranscript {
|
||||
segments := make([]schema.MinimalSegment, len(input.Segments))
|
||||
for index, segment := range input.Segments {
|
||||
segments[index] = schema.MinimalSegment{
|
||||
ID: segment.ID,
|
||||
Start: segment.Start,
|
||||
End: segment.End,
|
||||
Speaker: segment.Speaker,
|
||||
Text: segment.Text,
|
||||
}
|
||||
}
|
||||
return schema.MinimalTranscript{
|
||||
Metadata: schema.MinimalMetadata{
|
||||
Application: input.Metadata.Application,
|
||||
Version: input.Metadata.Version,
|
||||
OutputSchema: SchemaMinimal,
|
||||
},
|
||||
Segments: segments,
|
||||
}
|
||||
}
|
||||
|
||||
func minimalFromIntermediate(input schema.IntermediateTranscript) schema.MinimalTranscript {
|
||||
segments := make([]schema.MinimalSegment, len(input.Segments))
|
||||
for index, segment := range input.Segments {
|
||||
segments[index] = schema.MinimalSegment{
|
||||
ID: segment.ID,
|
||||
Start: segment.Start,
|
||||
End: segment.End,
|
||||
Speaker: segment.Speaker,
|
||||
Text: segment.Text,
|
||||
}
|
||||
}
|
||||
return schema.MinimalTranscript{
|
||||
Metadata: schema.MinimalMetadata{
|
||||
Application: input.Metadata.Application,
|
||||
Version: input.Metadata.Version,
|
||||
OutputSchema: SchemaMinimal,
|
||||
},
|
||||
Segments: segments,
|
||||
}
|
||||
}
|
||||
|
||||
func intermediateFromMinimal(input schema.MinimalTranscript) schema.IntermediateTranscript {
|
||||
segments := make([]schema.IntermediateSegment, len(input.Segments))
|
||||
for index, segment := range input.Segments {
|
||||
segments[index] = schema.IntermediateSegment{
|
||||
ID: segment.ID,
|
||||
Start: segment.Start,
|
||||
End: segment.End,
|
||||
Speaker: segment.Speaker,
|
||||
Text: segment.Text,
|
||||
}
|
||||
}
|
||||
return schema.IntermediateTranscript{
|
||||
Metadata: schema.IntermediateMetadata{
|
||||
Application: input.Metadata.Application,
|
||||
Version: input.Metadata.Version,
|
||||
OutputSchema: SchemaIntermediate,
|
||||
},
|
||||
Segments: segments,
|
||||
}
|
||||
}
|
||||
138
internal/trim/artifact_test.go
Normal file
138
internal/trim/artifact_test.go
Normal file
@@ -0,0 +1,138 @@
|
||||
package trim
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/seriatim/schema"
|
||||
)
|
||||
|
||||
func TestParseArtifactJSONRejectsMalformedJSON(t *testing.T) {
|
||||
_, err := ParseArtifactJSON([]byte(`{"metadata":`))
|
||||
if err == nil {
|
||||
t.Fatal("expected malformed JSON error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "input JSON is malformed") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseArtifactJSONRejectsDuplicateSegmentIDs(t *testing.T) {
|
||||
first := 10
|
||||
second := 20
|
||||
value := schema.Transcript{
|
||||
Metadata: schema.Metadata{
|
||||
Application: "seriatim",
|
||||
Version: "v-test",
|
||||
},
|
||||
Segments: []schema.Segment{
|
||||
{ID: 1, Source: "a.json", SourceSegmentIndex: &first, Speaker: "A", Start: 1, End: 2, Text: "one"},
|
||||
{ID: 1, Source: "a.json", SourceSegmentIndex: &second, Speaker: "B", Start: 2, End: 3, Text: "two"},
|
||||
},
|
||||
OverlapGroups: []schema.OverlapGroup{},
|
||||
}
|
||||
data := mustMarshalJSON(t, value)
|
||||
|
||||
_, err := ParseArtifactJSON(data)
|
||||
if err == nil {
|
||||
t.Fatal("expected invalid artifact error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "not a valid seriatim output artifact") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseArtifactJSONRejectsNonSequentialSegmentIDs(t *testing.T) {
|
||||
first := 10
|
||||
second := 20
|
||||
value := schema.Transcript{
|
||||
Metadata: schema.Metadata{
|
||||
Application: "seriatim",
|
||||
Version: "v-test",
|
||||
},
|
||||
Segments: []schema.Segment{
|
||||
{ID: 1, Source: "a.json", SourceSegmentIndex: &first, Speaker: "A", Start: 1, End: 2, Text: "one"},
|
||||
{ID: 3, Source: "a.json", SourceSegmentIndex: &second, Speaker: "B", Start: 2, End: 3, Text: "two"},
|
||||
},
|
||||
OverlapGroups: []schema.OverlapGroup{},
|
||||
}
|
||||
data := mustMarshalJSON(t, value)
|
||||
|
||||
_, err := ParseArtifactJSON(data)
|
||||
if err == nil {
|
||||
t.Fatal("expected invalid artifact error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "not a valid seriatim output artifact") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvertArtifactMinimalToIntermediate(t *testing.T) {
|
||||
value := schema.MinimalTranscript{
|
||||
Metadata: schema.MinimalMetadata{
|
||||
Application: "seriatim",
|
||||
Version: "v-test",
|
||||
OutputSchema: SchemaMinimal,
|
||||
},
|
||||
Segments: []schema.MinimalSegment{
|
||||
{ID: 1, Start: 1, End: 2, Speaker: "A", Text: "one"},
|
||||
{ID: 2, Start: 2, End: 3, Speaker: "B", Text: "two"},
|
||||
},
|
||||
}
|
||||
artifact := Artifact{
|
||||
Schema: SchemaMinimal,
|
||||
Minimal: &value,
|
||||
}
|
||||
|
||||
converted, err := ConvertArtifact(artifact, SchemaIntermediate)
|
||||
if err != nil {
|
||||
t.Fatalf("convert failed: %v", err)
|
||||
}
|
||||
if converted.Schema != SchemaIntermediate {
|
||||
t.Fatalf("schema = %q, want %q", converted.Schema, SchemaIntermediate)
|
||||
}
|
||||
if converted.Intermediate == nil {
|
||||
t.Fatal("expected intermediate artifact")
|
||||
}
|
||||
if len(converted.Intermediate.Segments) != 2 {
|
||||
t.Fatalf("segment count = %d, want 2", len(converted.Intermediate.Segments))
|
||||
}
|
||||
if converted.Intermediate.Segments[0].ID != 1 || converted.Intermediate.Segments[1].ID != 2 {
|
||||
t.Fatalf("unexpected IDs: %#v", converted.Intermediate.Segments)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvertArtifactMinimalToFullFails(t *testing.T) {
|
||||
value := schema.MinimalTranscript{
|
||||
Metadata: schema.MinimalMetadata{
|
||||
Application: "seriatim",
|
||||
Version: "v-test",
|
||||
OutputSchema: SchemaMinimal,
|
||||
},
|
||||
Segments: []schema.MinimalSegment{
|
||||
{ID: 1, Start: 1, End: 2, Speaker: "A", Text: "one"},
|
||||
},
|
||||
}
|
||||
artifact := Artifact{
|
||||
Schema: SchemaMinimal,
|
||||
Minimal: &value,
|
||||
}
|
||||
|
||||
_, err := ConvertArtifact(artifact, SchemaFull)
|
||||
if err == nil {
|
||||
t.Fatal("expected conversion error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "cannot emit") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustMarshalJSON(t *testing.T, value any) []byte {
|
||||
t.Helper()
|
||||
data, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal: %v", err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
156
internal/trim/selector.go
Normal file
156
internal/trim/selector.go
Normal file
@@ -0,0 +1,156 @@
|
||||
package trim
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var selectorElementPattern = regexp.MustCompile(`^([+-]?\d+)(?:\s*-\s*([+-]?\d+))?$`)
|
||||
|
||||
// Selector represents a normalized union of segment IDs.
|
||||
type Selector struct {
|
||||
ranges []idRange
|
||||
}
|
||||
|
||||
type idRange struct {
|
||||
start int
|
||||
end int
|
||||
}
|
||||
|
||||
// ParseSelector parses an inline segment selector expression.
|
||||
func ParseSelector(input string) (Selector, error) {
|
||||
if strings.TrimSpace(input) == "" {
|
||||
return Selector{}, fmt.Errorf("selector cannot be empty")
|
||||
}
|
||||
|
||||
parts := strings.Split(input, ",")
|
||||
ranges := make([]idRange, 0, len(parts))
|
||||
for index, raw := range parts {
|
||||
element := strings.TrimSpace(raw)
|
||||
if element == "" {
|
||||
return Selector{}, fmt.Errorf("selector element %d cannot be empty", index+1)
|
||||
}
|
||||
|
||||
rangeValue, err := parseElement(element)
|
||||
if err != nil {
|
||||
return Selector{}, fmt.Errorf("selector element %d %q: %w", index+1, element, err)
|
||||
}
|
||||
ranges = append(ranges, rangeValue)
|
||||
}
|
||||
|
||||
normalized := normalizeRanges(ranges)
|
||||
if len(normalized) == 0 {
|
||||
return Selector{}, fmt.Errorf("selector cannot be empty")
|
||||
}
|
||||
return Selector{ranges: normalized}, nil
|
||||
}
|
||||
|
||||
// Contains returns true when id is included by this selector.
|
||||
func (s Selector) Contains(id int) bool {
|
||||
if id <= 0 {
|
||||
return false
|
||||
}
|
||||
index := sort.Search(len(s.ranges), func(i int) bool {
|
||||
return s.ranges[i].end >= id
|
||||
})
|
||||
if index == len(s.ranges) {
|
||||
return false
|
||||
}
|
||||
rangeValue := s.ranges[index]
|
||||
return id >= rangeValue.start && id <= rangeValue.end
|
||||
}
|
||||
|
||||
// IDs returns a deterministic ascending list of unique segment IDs.
|
||||
func (s Selector) IDs() []int {
|
||||
total := 0
|
||||
for _, rangeValue := range s.ranges {
|
||||
total += rangeValue.end - rangeValue.start + 1
|
||||
}
|
||||
|
||||
ids := make([]int, 0, total)
|
||||
for _, rangeValue := range s.ranges {
|
||||
for id := rangeValue.start; id <= rangeValue.end; id++ {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
func parseElement(element string) (idRange, error) {
|
||||
matches := selectorElementPattern.FindStringSubmatch(element)
|
||||
if matches == nil {
|
||||
return idRange{}, fmt.Errorf("malformed element")
|
||||
}
|
||||
|
||||
start, err := parseID(matches[1])
|
||||
if err != nil {
|
||||
return idRange{}, err
|
||||
}
|
||||
|
||||
if matches[2] == "" {
|
||||
return idRange{start: start, end: start}, nil
|
||||
}
|
||||
|
||||
end, err := parseID(matches[2])
|
||||
if err != nil {
|
||||
return idRange{}, fmt.Errorf("invalid range end: %w", err)
|
||||
}
|
||||
if start > end {
|
||||
return idRange{}, fmt.Errorf("descending range %d-%d is invalid", start, end)
|
||||
}
|
||||
return idRange{start: start, end: end}, nil
|
||||
}
|
||||
|
||||
func parseID(value string) (int, error) {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return 0, fmt.Errorf("missing segment ID")
|
||||
}
|
||||
|
||||
id, err := strconv.Atoi(value)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("segment ID must be an integer")
|
||||
}
|
||||
if id <= 0 {
|
||||
return 0, fmt.Errorf("segment ID must be positive")
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func normalizeRanges(in []idRange) []idRange {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
sorted := make([]idRange, len(in))
|
||||
copy(sorted, in)
|
||||
sort.Slice(sorted, func(i, j int) bool {
|
||||
if sorted[i].start == sorted[j].start {
|
||||
return sorted[i].end < sorted[j].end
|
||||
}
|
||||
return sorted[i].start < sorted[j].start
|
||||
})
|
||||
|
||||
merged := make([]idRange, 0, len(sorted))
|
||||
for _, next := range sorted {
|
||||
if len(merged) == 0 {
|
||||
merged = append(merged, next)
|
||||
continue
|
||||
}
|
||||
|
||||
last := &merged[len(merged)-1]
|
||||
if next.start <= last.end+1 {
|
||||
if next.end > last.end {
|
||||
last.end = next.end
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
merged = append(merged, next)
|
||||
}
|
||||
|
||||
return merged
|
||||
}
|
||||
127
internal/trim/selector_test.go
Normal file
127
internal/trim/selector_test.go
Normal file
@@ -0,0 +1,127 @@
|
||||
package trim
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseSelectorSingleID(t *testing.T) {
|
||||
selector, err := ParseSelector("1")
|
||||
if err != nil {
|
||||
t.Fatalf("parse failed: %v", err)
|
||||
}
|
||||
assertIDs(t, selector, []int{1})
|
||||
assertContains(t, selector, map[int]bool{1: true, 2: false, 0: false, -1: false})
|
||||
}
|
||||
|
||||
func TestParseSelectorInclusiveRange(t *testing.T) {
|
||||
selector, err := ParseSelector("1-3")
|
||||
if err != nil {
|
||||
t.Fatalf("parse failed: %v", err)
|
||||
}
|
||||
assertIDs(t, selector, []int{1, 2, 3})
|
||||
}
|
||||
|
||||
func TestParseSelectorCommaSeparatedCombination(t *testing.T) {
|
||||
selector, err := ParseSelector("1-3,8,10-12")
|
||||
if err != nil {
|
||||
t.Fatalf("parse failed: %v", err)
|
||||
}
|
||||
assertIDs(t, selector, []int{1, 2, 3, 8, 10, 11, 12})
|
||||
}
|
||||
|
||||
func TestParseSelectorWhitespaceTolerance(t *testing.T) {
|
||||
selector, err := ParseSelector(" 1 - 3 , 8 , 10 - 12 ")
|
||||
if err != nil {
|
||||
t.Fatalf("parse failed: %v", err)
|
||||
}
|
||||
assertIDs(t, selector, []int{1, 2, 3, 8, 10, 11, 12})
|
||||
}
|
||||
|
||||
func TestParseSelectorDuplicatesAndOverlapsNormalizeUnion(t *testing.T) {
|
||||
selector, err := ParseSelector("1-4,2,4,3-6,6")
|
||||
if err != nil {
|
||||
t.Fatalf("parse failed: %v", err)
|
||||
}
|
||||
assertIDs(t, selector, []int{1, 2, 3, 4, 5, 6})
|
||||
assertContains(t, selector, map[int]bool{1: true, 5: true, 6: true, 7: false})
|
||||
}
|
||||
|
||||
func TestParseSelectorDeterministicNormalizedOutput(t *testing.T) {
|
||||
left, err := ParseSelector("8,1-3,2,10-12")
|
||||
if err != nil {
|
||||
t.Fatalf("parse left failed: %v", err)
|
||||
}
|
||||
right, err := ParseSelector("10-12,3,2,1,8")
|
||||
if err != nil {
|
||||
t.Fatalf("parse right failed: %v", err)
|
||||
}
|
||||
|
||||
leftIDs := left.IDs()
|
||||
rightIDs := right.IDs()
|
||||
if !equalInts(leftIDs, rightIDs) {
|
||||
t.Fatalf("normalized IDs mismatch: %v vs %v", leftIDs, rightIDs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSelectorFailures(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
selector string
|
||||
wantError string
|
||||
}{
|
||||
{name: "empty", selector: "", wantError: "cannot be empty"},
|
||||
{name: "whitespace only", selector: " ", wantError: "cannot be empty"},
|
||||
{name: "zero", selector: "0", wantError: "must be positive"},
|
||||
{name: "negative", selector: "-1", wantError: "must be positive"},
|
||||
{name: "range includes zero", selector: "0-2", wantError: "must be positive"},
|
||||
{name: "descending range", selector: "10-1", wantError: "descending range"},
|
||||
{name: "empty element", selector: "1,,2", wantError: "cannot be empty"},
|
||||
{name: "trailing comma", selector: "1,", wantError: "cannot be empty"},
|
||||
{name: "malformed alpha", selector: "abc", wantError: "malformed element"},
|
||||
{name: "malformed range", selector: "1-2-3", wantError: "malformed element"},
|
||||
{name: "missing end", selector: "1-", wantError: "malformed element"},
|
||||
{name: "missing start", selector: "-2", wantError: "must be positive"},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
_, err := ParseSelector(test.selector)
|
||||
if err == nil {
|
||||
t.Fatalf("expected error for %q", test.selector)
|
||||
}
|
||||
if !strings.Contains(err.Error(), test.wantError) {
|
||||
t.Fatalf("error = %q, want substring %q", err.Error(), test.wantError)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func assertIDs(t *testing.T, selector Selector, want []int) {
|
||||
t.Helper()
|
||||
got := selector.IDs()
|
||||
if !equalInts(got, want) {
|
||||
t.Fatalf("IDs = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func assertContains(t *testing.T, selector Selector, checks map[int]bool) {
|
||||
t.Helper()
|
||||
for id, want := range checks {
|
||||
if got := selector.Contains(id); got != want {
|
||||
t.Fatalf("Contains(%d) = %t, want %t", id, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func equalInts(left []int, right []int) bool {
|
||||
if len(left) != len(right) {
|
||||
return false
|
||||
}
|
||||
for index := range left {
|
||||
if left[index] != right[index] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
Reference in New Issue
Block a user