Compare commits
40 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 9202ccddb9 | |||
| f40d4add91 | |||
| 16bb12face | |||
| f18e2428dc | |||
| 3b64e784a1 | |||
| 3744d229a2 | |||
| 9bbe1fb7f1 | |||
| b7a66f6cc4 | |||
| c8efdb53d3 | |||
| ab4b252b08 | |||
| e9028e08a4 | |||
| 332884f887 | |||
| e5173c78fe | |||
| 546be2ab92 | |||
| 7743b397a6 | |||
| d23a95471c | |||
| f8ab117bfc | |||
| 88018c9e76 | |||
| b3e7dc3136 | |||
| 385c62a5b4 | |||
| 7d9bf33d18 | |||
| b7cc5fb980 | |||
| b20438acf0 | |||
| 6dbb7ab17e | |||
| 3591041fa8 | |||
| 5b008e272c | |||
| 6c780f6293 | |||
| c132f3fd5d | |||
| 3679435063 | |||
| e6d3b4a46e | |||
| 54f7717de8 | |||
| c48b02d2ec | |||
| ac3dcf2557 | |||
| 1c0e4438ae | |||
| 52f7729100 | |||
| 2c82f8bf5c | |||
| d865bda4a9 | |||
| f20f06db12 | |||
| c9e98e14b5 | |||
| be6a53bbbd |
4
.gitignore
vendored
4
.gitignore
vendored
@@ -1,3 +1,7 @@
|
|||||||
|
# ---> Codex
|
||||||
|
.codex
|
||||||
|
AGENTS.md
|
||||||
|
|
||||||
# ---> Go
|
# ---> Go
|
||||||
# If you prefer the allow list template instead of the deny list, see community template:
|
# If you prefer the allow list template instead of the deny list, see community template:
|
||||||
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
|
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ steps:
|
|||||||
- |
|
- |
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
version="${CI_COMMIT_TAG}"
|
version="$CI_COMMIT_TAG"
|
||||||
dist="dist"
|
dist="dist"
|
||||||
pkg="gitea.maximumdirect.net/eric/seriatim/cmd/seriatim"
|
pkg="gitea.maximumdirect.net/eric/seriatim/cmd/seriatim"
|
||||||
|
|
||||||
@@ -19,10 +19,10 @@ steps:
|
|||||||
goos="$1"
|
goos="$1"
|
||||||
goarch="$2"
|
goarch="$2"
|
||||||
suffix="$3"
|
suffix="$3"
|
||||||
output="$dist/seriatim_${version}_${goos}_${goarch}${suffix}"
|
output="$dist/seriatim-$version-$goos-$goarch$suffix"
|
||||||
|
|
||||||
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" \
|
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" \
|
||||||
go build -trimpath -ldflags "-s -w -X gitea.maximumdirect.net/eric/seriatim/internal/buildinfo.Version=${version}" \
|
go build -trimpath -ldflags "-s -w -X gitea.maximumdirect.net/eric/seriatim/internal/buildinfo.Version=$version" \
|
||||||
-o "$output" "$pkg"
|
-o "$output" "$pkg"
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -33,11 +33,6 @@ steps:
|
|||||||
build_binary windows amd64 ".exe"
|
build_binary windows amd64 ".exe"
|
||||||
build_binary windows arm64 ".exe"
|
build_binary windows arm64 ".exe"
|
||||||
|
|
||||||
(
|
|
||||||
cd "$dist"
|
|
||||||
sha256sum seriatim_* > SHA256SUMS
|
|
||||||
)
|
|
||||||
|
|
||||||
- name: publish-release
|
- name: publish-release
|
||||||
image: woodpeckerci/plugin-release
|
image: woodpeckerci/plugin-release
|
||||||
depends_on:
|
depends_on:
|
||||||
@@ -46,8 +41,10 @@ steps:
|
|||||||
api_key:
|
api_key:
|
||||||
from_secret: GITEA_RELEASE_TOKEN
|
from_secret: GITEA_RELEASE_TOKEN
|
||||||
files:
|
files:
|
||||||
- dist/seriatim_*
|
- dist/seriatim-*
|
||||||
- dist/SHA256SUMS
|
checksum: sha256
|
||||||
|
checksum-file: SHA256SUMS
|
||||||
|
checksum-flatten: true
|
||||||
file-exists: skip
|
file-exists: skip
|
||||||
overwrite: false
|
overwrite: false
|
||||||
prerelease: false
|
prerelease: false
|
||||||
|
|||||||
2
LICENSE
2
LICENSE
@@ -1,4 +1,4 @@
|
|||||||
Copyright (c) 2026 eric.
|
Copyright (c) 2026 Eric Rakestraw.
|
||||||
|
|
||||||
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
|
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
|
||||||
|
|
||||||
|
|||||||
423
README.md
423
README.md
@@ -1,400 +1,45 @@
|
|||||||
# seriatim
|
# seriatim
|
||||||
|
|
||||||
`seriatim` merges per-speaker WhisperX-style JSON transcripts into a single JSON transcript that preserves speaker identity and chronological order.
|
`seriatim` is a Go CLI for transcript artifact processing.
|
||||||
|
|
||||||
The current implementation supports the `merge` command. It reads one or more input JSON files, optionally maps each input file to a canonical speaker using `speakers.yml`, sorts all segments by timestamp, detects and resolves overlaps when word-level timing is available, assigns consecutive numeric `id` values, and writes a merged JSON artifact.
|
It merges per-speaker WhisperX-style JSON into one deterministic transcript, trims existing seriatim artifacts by segment ID, and normalizes transcript-like JSON into standard seriatim output schemas.
|
||||||
|
|
||||||
## Usage
|
## Quickstart
|
||||||
|
|
||||||
Run from source:
|
Shortest useful merge command:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
go run ./cmd/seriatim merge \
|
go run ./cmd/seriatim merge \
|
||||||
--input-file samples/raw/2026-04-19-Eric_Rakestraw.json \
|
--input-file speaker-a.json \
|
||||||
--input-file samples/raw/2026-04-19-Mike_Brown.json \
|
--input-file speaker-b.json \
|
||||||
--output-file merged.json
|
--output-file merged.json
|
||||||
```
|
```
|
||||||
|
|
||||||
Optional report output:
|
## Commands
|
||||||
|
|
||||||
```sh
|
- `merge`: merge one or more input transcript JSON files.
|
||||||
go run ./cmd/seriatim merge \
|
- `trim`: keep/remove segment IDs from an existing seriatim artifact.
|
||||||
--input-file eric.json \
|
- `normalize`: canonicalize transcript-like JSON into a seriatim artifact.
|
||||||
--input-file mike.json \
|
|
||||||
--output-file merged.json \
|
## Documentation
|
||||||
--report-file report.json
|
|
||||||
```
|
- CLI reference: [docs/cli.md](docs/cli.md)
|
||||||
|
- Configuration reference: [docs/config.md](docs/config.md)
|
||||||
## CLI
|
- Operations guide: [docs/operations.md](docs/operations.md)
|
||||||
|
- Troubleshooting: [docs/troubleshooting.md](docs/troubleshooting.md)
|
||||||
```text
|
- Integration references:
|
||||||
seriatim merge [flags]
|
- [docs/integrations/whisperx-json.md](docs/integrations/whisperx-json.md)
|
||||||
```
|
- [docs/integrations/output-schemas.md](docs/integrations/output-schemas.md)
|
||||||
|
- Development policies:
|
||||||
Global flags:
|
- [docs/policy/architecture.md](docs/policy/architecture.md)
|
||||||
|
- [docs/policy/development.md](docs/policy/development.md)
|
||||||
| Flag | Description |
|
- [docs/policy/documentation.md](docs/policy/documentation.md)
|
||||||
| --- | --- |
|
- Internal implementation references:
|
||||||
| `--help` | Show command help. |
|
- [docs/internal/pipeline.md](docs/internal/pipeline.md)
|
||||||
| `--version` | Show application version. Local builds default to `dev`; release builds inject the release version. |
|
- [docs/internal/artifacts.md](docs/internal/artifacts.md)
|
||||||
|
- [docs/internal/modules.md](docs/internal/modules.md)
|
||||||
`merge` flags:
|
- Public JSON schema files:
|
||||||
|
- [schema/minimal-output.schema.json](schema/minimal-output.schema.json)
|
||||||
| Flag | Required | Default | Description |
|
- [schema/intermediate-output.schema.json](schema/intermediate-output.schema.json)
|
||||||
| --- | --- | --- | --- |
|
- [schema/full-output.schema.json](schema/full-output.schema.json)
|
||||||
| `--input-file` | Yes | none | Input transcript JSON file. Repeat once per speaker/input file. |
|
- Synthetic examples: [examples/README.md](examples/README.md)
|
||||||
| `--output-file` | Yes | none | Merged transcript JSON output path. |
|
|
||||||
| `--report-file` | No | none | Optional report JSON output path. |
|
|
||||||
| `--speakers` | No | none | Speaker map YAML file. When omitted, input file basenames are used as speaker labels. |
|
|
||||||
| `--autocorrect` | No | none | Autocorrect rules YAML file. When omitted, the default `autocorrect` module leaves text unchanged. |
|
|
||||||
| `--input-reader` | No | `json-files` | Input reader module. |
|
|
||||||
| `--output-modules` | No | `json` | Comma-separated output modules. |
|
|
||||||
| `--output-schema` | No | `default` | JSON output contract. Allowed values are `default`, `minimal`, and `seriatim`. |
|
|
||||||
| `--preprocessing-modules` | No | `validate-raw,normalize-speakers,trim-text` | Comma-separated preprocessing modules, evaluated in order. |
|
|
||||||
| `--postprocessing-modules` | No | `detect-overlaps,resolve-overlaps,backchannel,filler,resolve-danglers,coalesce,detect-overlaps,autocorrect,assign-ids,validate-output` | Comma-separated postprocessing modules, evaluated in order. |
|
|
||||||
| `--coalesce-gap` | No | `3.0` | Maximum same-speaker gap in seconds for `coalesce`; also used as the `resolve-overlaps` context window. Must be a non-negative float. |
|
|
||||||
|
|
||||||
Environment variables:
|
|
||||||
|
|
||||||
| Environment Variable | Default | Description |
|
|
||||||
| --- | --- | --- |
|
|
||||||
| `SERIATIM_OVERLAP_WORD_RUN_GAP` | `0.75` | Maximum gap in seconds between adjacent timed words when `resolve-overlaps` builds word-run replacement segments. Must be a positive float. |
|
|
||||||
| `SERIATIM_OVERLAP_WORD_RUN_REORDER_WINDOW` | `1.0` | Near-start window in seconds for ordering replacement word runs shortest-first. Must be a positive float. |
|
|
||||||
| `SERIATIM_BACKCHANNEL_MAX_DURATION` | `2.0` | Maximum duration in seconds for `backchannel` classification. Must be a positive float. |
|
|
||||||
| `SERIATIM_FILLER_MAX_DURATION` | `1.25` | Maximum duration in seconds for `filler` classification. Must be a positive float. |
|
|
||||||
|
|
||||||
## Input JSON Format
|
|
||||||
|
|
||||||
Each input file must be valid JSON with a top-level `segments` array. The current parser accepts the WhisperX segment subset needed for merging:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"segments": [
|
|
||||||
{
|
|
||||||
"start": 1.25,
|
|
||||||
"end": 3.5,
|
|
||||||
"text": "Hello there.",
|
|
||||||
"words": [
|
|
||||||
{"word": "Hello", "start": 1.25, "end": 1.55, "score": 0.98},
|
|
||||||
{"word": "there.", "start": 1.7, "end": 2.0}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Required segment fields:
|
|
||||||
|
|
||||||
- `start`: number, must be `>= 0`.
|
|
||||||
- `end`: number, must be `>= start`.
|
|
||||||
- `text`: string.
|
|
||||||
|
|
||||||
Optional word fields:
|
|
||||||
|
|
||||||
- `words`: array of word timing objects.
|
|
||||||
- `words[].word`: string.
|
|
||||||
- `words[].start`: optional number, must be `>= 0` when present.
|
|
||||||
- `words[].end`: optional number, must be `>= start` when present with `start`.
|
|
||||||
- `words[].score`: optional number.
|
|
||||||
- `words[].speaker`: optional raw speaker label string.
|
|
||||||
|
|
||||||
Word-level timing is preserved internally for overlap resolution. If a word is missing `start` or `end`, seriatim keeps the word text, emits a warning in the optional report, and does not use that word as a timing anchor. Word timing is not emitted in the final JSON artifact.
|
|
||||||
|
|
||||||
## Speaker Map Format
|
|
||||||
|
|
||||||
`speakers.yml` maps input files to canonical speaker names using ordered substring rules:
|
|
||||||
|
|
||||||
This file is optional. If `--speakers` is omitted, `seriatim` uses each input file basename as the segment speaker label.
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
match:
|
|
||||||
- speaker: "Eric Rakestraw"
|
|
||||||
match:
|
|
||||||
- "Eric_Rakestraw"
|
|
||||||
- "Eric"
|
|
||||||
|
|
||||||
- speaker: "Mike Brown"
|
|
||||||
match:
|
|
||||||
- "Mike_Brown"
|
|
||||||
- "mb"
|
|
||||||
```
|
|
||||||
|
|
||||||
For each `--input-file`, `seriatim` takes the file basename and evaluates the rules in order. The first rule with a matching substring wins, and no later rules are evaluated.
|
|
||||||
|
|
||||||
For example, this input:
|
|
||||||
|
|
||||||
```text
|
|
||||||
samples/raw/2026-04-19-Eric_Rakestraw.json
|
|
||||||
```
|
|
||||||
|
|
||||||
matches this rule because the basename contains `Eric_Rakestraw`:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
- speaker: "Eric Rakestraw"
|
|
||||||
match:
|
|
||||||
- "Eric_Rakestraw"
|
|
||||||
```
|
|
||||||
|
|
||||||
Important details:
|
|
||||||
|
|
||||||
- Matching is against the input file basename, not the full path.
|
|
||||||
- Matching is case-insensitive.
|
|
||||||
- Rules are evaluated from first to last.
|
|
||||||
- Each rule must have a non-empty `speaker`.
|
|
||||||
- Each rule must have at least one non-empty `match` string.
|
|
||||||
- Duplicate speaker names are invalid.
|
|
||||||
- Every input file must match at least one rule or the command fails.
|
|
||||||
|
|
||||||
Deprecated old format:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
inputs:
|
|
||||||
eric.json:
|
|
||||||
speaker: "Eric Rakestraw"
|
|
||||||
```
|
|
||||||
|
|
||||||
The old `inputs:` direct mapping format is no longer supported.
|
|
||||||
|
|
||||||
## Output JSON Format
|
|
||||||
|
|
||||||
`--output-modules json` controls the writer. `--output-schema` controls the JSON contract that writer serializes.
|
|
||||||
|
|
||||||
The `default` schema is the default output contract. It stays close to `minimal`, but adds optional `categories` on each segment:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"metadata": {
|
|
||||||
"application": "seriatim",
|
|
||||||
"version": "dev",
|
|
||||||
"output_schema": "default"
|
|
||||||
},
|
|
||||||
"segments": [
|
|
||||||
{
|
|
||||||
"id": 1,
|
|
||||||
"start": 1.25,
|
|
||||||
"end": 3.5,
|
|
||||||
"speaker": "Eric Rakestraw",
|
|
||||||
"text": "Hello there.",
|
|
||||||
"categories": ["backchannel"]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
The explicit `seriatim` schema uses the full seriatim envelope:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"metadata": {
|
|
||||||
"application": "seriatim",
|
|
||||||
"version": "dev",
|
|
||||||
"input_reader": "json-files",
|
|
||||||
"input_files": ["eric.json", "mike.json"],
|
|
||||||
"preprocessing_modules": ["validate-raw", "normalize-speakers", "trim-text"],
|
|
||||||
"postprocessing_modules": ["detect-overlaps", "resolve-overlaps", "backchannel", "filler", "coalesce", "resolve-danglers", "detect-overlaps", "autocorrect", "assign-ids", "validate-output"],
|
|
||||||
"output_modules": ["json"]
|
|
||||||
},
|
|
||||||
"segments": [
|
|
||||||
{
|
|
||||||
"id": 1,
|
|
||||||
"source": "eric.json",
|
|
||||||
"source_segment_index": 0,
|
|
||||||
"speaker": "Eric Rakestraw",
|
|
||||||
"start": 1.25,
|
|
||||||
"end": 3.5,
|
|
||||||
"text": "Hello there.",
|
|
||||||
"overlap_group_id": 1
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": 2,
|
|
||||||
"source": "eric.json",
|
|
||||||
"source_ref": "word-run:1:1:1",
|
|
||||||
"derived_from": ["eric.json#0"],
|
|
||||||
"speaker": "Eric Rakestraw",
|
|
||||||
"start": 2.0,
|
|
||||||
"end": 2.5,
|
|
||||||
"text": "Resolved word run",
|
|
||||||
"categories": ["backchannel"]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"overlap_groups": [
|
|
||||||
{
|
|
||||||
"id": 1,
|
|
||||||
"start": 1.25,
|
|
||||||
"end": 4.0,
|
|
||||||
"segments": ["eric.json#0", "mike.json#0"],
|
|
||||||
"speakers": ["Eric Rakestraw", "Mike Brown"],
|
|
||||||
"class": "unknown",
|
|
||||||
"resolution": "unresolved"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
The `minimal` schema emits minimal metadata and compact ordered segments:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"metadata": {
|
|
||||||
"application": "seriatim",
|
|
||||||
"version": "dev",
|
|
||||||
"output_schema": "minimal"
|
|
||||||
},
|
|
||||||
"segments": [
|
|
||||||
{
|
|
||||||
"id": 1,
|
|
||||||
"start": 1.25,
|
|
||||||
"end": 3.5,
|
|
||||||
"speaker": "Eric Rakestraw",
|
|
||||||
"text": "Hello there."
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Minimal output intentionally omits categories, overlap groups, source/provenance fields, and pipeline configuration metadata.
|
|
||||||
|
|
||||||
Default output intentionally omits overlap groups and source/provenance fields, but keeps optional `categories` and minimal metadata.
|
|
||||||
|
|
||||||
Segments are sorted deterministically by:
|
|
||||||
|
|
||||||
```text
|
|
||||||
(start, end, source, source_segment_index/source_ref, speaker)
|
|
||||||
```
|
|
||||||
|
|
||||||
Final segment IDs are assigned after sorting and start at `1`.
|
|
||||||
|
|
||||||
The public Go output contract is available from:
|
|
||||||
|
|
||||||
```go
|
|
||||||
import "gitea.maximumdirect.net/eric/seriatim/schema"
|
|
||||||
```
|
|
||||||
|
|
||||||
The same package embeds machine-readable JSON Schemas in `schema/output.schema.json`, `schema/default-output.schema.json`, and `schema/minimal-output.schema.json`. The default `validate-output` postprocessor validates the selected output shape and verifies final segment IDs are present, sequential, and start at `1`.
|
|
||||||
|
|
||||||
## Overlap Detection
|
|
||||||
|
|
||||||
The default postprocessing pipeline detects overlapping segment groups.
|
|
||||||
|
|
||||||
Overlap behavior:
|
|
||||||
|
|
||||||
- A strict timing overlap is required: `next.start < current_group_end`.
|
|
||||||
- Segments that only touch at a boundary are not grouped.
|
|
||||||
- Groups require at least two distinct speakers.
|
|
||||||
- Transitive overlaps are grouped together.
|
|
||||||
- Segments in detected groups receive `overlap_group_id`.
|
|
||||||
- `overlap_groups[].segments` contains stable references in `source#source_segment_index` format.
|
|
||||||
- `class` is currently `unknown`.
|
|
||||||
- `resolution` is `unresolved` until `resolve-overlaps` replaces the group.
|
|
||||||
|
|
||||||
## Overlap Resolution
|
|
||||||
|
|
||||||
The default postprocessing pipeline runs `detect-overlaps`, then `resolve-overlaps`, then `backchannel`, then `filler`, then `resolve-danglers`, then `coalesce`, then a second `detect-overlaps` pass.
|
|
||||||
|
|
||||||
For each detected overlap group, `resolve-overlaps` uses preserved WhisperX word timing to build smaller word-run replacement segments:
|
|
||||||
|
|
||||||
- The resolution window expands the detected overlap group by `--coalesce-gap` seconds on both sides.
|
|
||||||
- Nearby same-speaker context segments are included when they intersect the expanded window and their start or end is within `--coalesce-gap` of the original overlap boundary.
|
|
||||||
- Words are included when their interval intersects the expanded resolution window.
|
|
||||||
- Context segments that are part of another detected overlap group are not pulled into the current group.
|
|
||||||
- Untimed words are included in replacement text in original word order when nearby timed words create a replacement run.
|
|
||||||
- Untimed words do not affect replacement segment start/end times or word-run gap splitting.
|
|
||||||
- Words for the same speaker are merged into one run when the gap between adjacent words is no greater than `SERIATIM_OVERLAP_WORD_RUN_GAP`.
|
|
||||||
- The default word-run gap is `0.75` seconds.
|
|
||||||
- Set `SERIATIM_OVERLAP_WORD_RUN_GAP` to a positive number of seconds to override the default.
|
|
||||||
- Near-start replacement word runs are reordered so shorter segments come first when adjacent starts are within `SERIATIM_OVERLAP_WORD_RUN_REORDER_WINDOW`.
|
|
||||||
- The default word-run reorder window is `1.0` seconds.
|
|
||||||
- Set `SERIATIM_OVERLAP_WORD_RUN_REORDER_WINDOW` to a positive number of seconds to override the default.
|
|
||||||
- Replacement segment text is built by joining word text with single spaces.
|
|
||||||
- Replacement segments include `source_ref` and `derived_from`.
|
|
||||||
- Replacement segments omit `source_segment_index` because they are derived from one or more original segments.
|
|
||||||
- Resolved overlap groups are removed before the second detection pass.
|
|
||||||
- Replacement segments are left without `overlap_group_id` until the second detection pass annotates any remaining overlap.
|
|
||||||
- If a speaker has no usable word timing in a group, that speaker's original segment is kept.
|
|
||||||
- If no speakers in a group have usable word timing, the original group and annotations remain unchanged.
|
|
||||||
|
|
||||||
## Backchannels
|
|
||||||
|
|
||||||
The default pipeline runs `backchannel` before `coalesce`. It tags short acknowledgement segments with:
|
|
||||||
|
|
||||||
```json
|
|
||||||
"categories": ["backchannel"]
|
|
||||||
```
|
|
||||||
|
|
||||||
Backchannel matching is case-insensitive, ignores punctuation for matching and word-count purposes, trims surrounding whitespace, and requires a matching acknowledgement phrase, no more than three whitespace-delimited words, and duration no greater than `SERIATIM_BACKCHANNEL_MAX_DURATION` seconds. The default maximum duration is `2.0` seconds.
|
|
||||||
|
|
||||||
## Fillers
|
|
||||||
|
|
||||||
The default pipeline runs `filler` after `backchannel` and before `coalesce`. It tags short filler utterances with:
|
|
||||||
|
|
||||||
```json
|
|
||||||
"categories": ["filler"]
|
|
||||||
```
|
|
||||||
|
|
||||||
Filler matching is case-insensitive, ignores punctuation for matching and word-count purposes, trims surrounding whitespace, and requires only filler tokens such as `um`, `uh`, `er`, `erm`, `ah`, `eh`, `hmm`, `mm`, or repeated combinations of those tokens. Matching segments must contain no more than three whitespace-delimited words and have duration no greater than `SERIATIM_FILLER_MAX_DURATION` seconds. The default maximum duration is `1.25` seconds.
|
|
||||||
|
|
||||||
## Dangler Resolution
|
|
||||||
|
|
||||||
The default pipeline runs `resolve-danglers` before `coalesce` and before the second overlap detection pass. It repairs short derived fragments when they share provenance with a nearby segment:
|
|
||||||
|
|
||||||
- Dangling-end fragments have no more than two words and end in punctuation.
|
|
||||||
- Dangling-start fragments have no more than two words.
|
|
||||||
- Matching uses any shared `derived_from` value.
|
|
||||||
- Merged segments use `source_ref` values such as `resolve-danglers:1`, keep the target segment's transcript position, and union `derived_from`.
|
|
||||||
|
|
||||||
## Coalescing
|
|
||||||
|
|
||||||
The default pipeline runs `coalesce` after `resolve-danglers` and the second overlap detection pass. It merges adjacent same-speaker segments in the transcript's current order when `next.start - current.end <= --coalesce-gap`.
|
|
||||||
|
|
||||||
Coalesced segments use `source_ref` values such as `coalesce:1`, include `derived_from`, and omit `source_segment_index`.
|
|
||||||
|
|
||||||
Different-speaker backchannel and filler segments do not block coalescing of surrounding same-speaker segments. Same-speaker backchannel and filler segments are merged normally when they are within `--coalesce-gap`. When same-speaker segments are coalesced, any `backchannel` or `filler` category from the merged inputs is dropped from the coalesced segment.
|
|
||||||
|
|
||||||
## Autocorrect
|
|
||||||
|
|
||||||
Autocorrect is included in the default postprocessing pipeline. If `--autocorrect` is omitted, the module leaves transcript text unchanged and records a skip event in the optional report.
|
|
||||||
|
|
||||||
Enable corrections by passing `--autocorrect`:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go run ./cmd/seriatim merge \
|
|
||||||
--input-file input.json \
|
|
||||||
--autocorrect autocorrect.yml \
|
|
||||||
--output-file merged.json
|
|
||||||
```
|
|
||||||
|
|
||||||
`autocorrect.yml` format:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
autocorrect:
|
|
||||||
- target: "Hrank"
|
|
||||||
match:
|
|
||||||
- "hrank"
|
|
||||||
- "Frank"
|
|
||||||
|
|
||||||
- target: "Mike Brown"
|
|
||||||
match:
|
|
||||||
- "Mike Pat"
|
|
||||||
```
|
|
||||||
|
|
||||||
Matching behavior:
|
|
||||||
|
|
||||||
- Matching is case-sensitive.
|
|
||||||
- Matches apply only to whole tokens, not substrings inside larger words.
|
|
||||||
- Punctuation and whitespace can surround a match.
|
|
||||||
- Multi-word and hyphenated matches are supported.
|
|
||||||
- Duplicate match strings are invalid, including duplicates across separate rules.
|
|
||||||
|
|
||||||
## Current Limitations
|
|
||||||
|
|
||||||
- Only JSON input is supported.
|
|
||||||
- Overlap resolution depends on WhisperX word timing; groups without usable word timing remain unresolved.
|
|
||||||
- Alternate output formats are not implemented yet.
|
|
||||||
|
|
||||||
## Release Builds
|
|
||||||
|
|
||||||
Local builds record version metadata as `dev`. Release builds should inject the release version with `ldflags`:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go build -ldflags "-X gitea.maximumdirect.net/eric/seriatim/internal/buildinfo.Version=v1.0.0" ./cmd/seriatim
|
|
||||||
```
|
|
||||||
|
|||||||
445
architecture.md
445
architecture.md
@@ -1,445 +0,0 @@
|
|||||||
# seriatim Architecture
|
|
||||||
|
|
||||||
`seriatim` is a deterministic transcript merge utility for combining multiple per-speaker transcript inputs into a single chronologically ordered diarized transcript.
|
|
||||||
|
|
||||||
The initial use case is merging independently transcribed speaker audio tracks from the same recorded session, such as a weekly tabletop RPG session. The architecture should also support meetings, podcasts, interviews, and other multi-speaker events.
|
|
||||||
|
|
||||||
`seriatim` is implemented in Go.
|
|
||||||
|
|
||||||
## Goals
|
|
||||||
|
|
||||||
`seriatim` should:
|
|
||||||
|
|
||||||
1. Validate runtime configuration before performing transcript processing.
|
|
||||||
2. Support multiple input methods and formats through input readers.
|
|
||||||
3. Normalize raw per-speaker transcripts into a canonical internal model.
|
|
||||||
4. Apply deterministic preprocessing modules to canonical per-speaker transcripts.
|
|
||||||
5. Merge all segments into a deterministic global chronological order.
|
|
||||||
6. Apply deterministic postprocessing modules to the merged transcript.
|
|
||||||
7. Preserve word-level timing data when available.
|
|
||||||
8. Detect and annotate overlapping speech regions.
|
|
||||||
9. Emit one or more output artifacts through output writers.
|
|
||||||
10. Produce report data for validation findings, corrections, and transformations.
|
|
||||||
|
|
||||||
## Non-goals
|
|
||||||
|
|
||||||
The 1.0 release does not attempt to:
|
|
||||||
|
|
||||||
- Perform transcription.
|
|
||||||
- Perform audio diarization.
|
|
||||||
- Use an LLM.
|
|
||||||
- Summarize transcript content.
|
|
||||||
- Infer speaker identity from audio or text.
|
|
||||||
- Fully resolve every crosstalk case.
|
|
||||||
- Load arbitrary third-party code as dynamic plugins.
|
|
||||||
|
|
||||||
The application supports runtime composition of built-in modules by canonical module name. Arbitrary external plugin loading can be considered later.
|
|
||||||
|
|
||||||
## Core Assumption
|
|
||||||
|
|
||||||
The merge algorithm assumes that all input transcript timestamps are measured against the same session clock.
|
|
||||||
|
|
||||||
This is expected when each speaker has a separate recording that preserves silence and starts at the same session recording time. If input files have independent local timelines, `seriatim` cannot safely merge them without a separate alignment step.
|
|
||||||
|
|
||||||
## Pipeline Overview
|
|
||||||
|
|
||||||
The internal pipeline is:
|
|
||||||
|
|
||||||
```text
|
|
||||||
configuration check
|
|
||||||
-> input
|
|
||||||
-> preprocessing
|
|
||||||
-> merge
|
|
||||||
-> postprocessing
|
|
||||||
-> output
|
|
||||||
```
|
|
||||||
|
|
||||||
Each stage has an explicit data contract. Input and output stages perform I/O. Processing stages should be deterministic transformations over in-memory models and should record report events for validation findings, corrections, and transformations.
|
|
||||||
|
|
||||||
## Stage Contracts
|
|
||||||
|
|
||||||
### 1. Configuration Check
|
|
||||||
|
|
||||||
The configuration stage validates all CLI flags, environment variables, module names, input paths, output paths, and module-specific options before transcript data is processed.
|
|
||||||
|
|
||||||
Configuration validation should fail fast for:
|
|
||||||
|
|
||||||
- Missing required input.
|
|
||||||
- Unknown module names.
|
|
||||||
- Unknown input or output formats.
|
|
||||||
- Ambiguous speaker mappings.
|
|
||||||
- Invalid correction policies.
|
|
||||||
- Invalid timing thresholds.
|
|
||||||
- Invalid output paths.
|
|
||||||
|
|
||||||
The configuration stage produces an application config value that is passed through the pipeline.
|
|
||||||
|
|
||||||
### 2. Input Stage
|
|
||||||
|
|
||||||
The input stage converts external inputs into raw transcript documents with source metadata.
|
|
||||||
|
|
||||||
The current input method is one or more JSON files passed with repeated `--input-file` flags:
|
|
||||||
|
|
||||||
```text
|
|
||||||
seriatim merge --input-file eric.json --input-file mike.json --output-file merged.json
|
|
||||||
```
|
|
||||||
|
|
||||||
Future input methods may include:
|
|
||||||
|
|
||||||
- A `.tar.gz` bundle.
|
|
||||||
- A URI.
|
|
||||||
- A directory.
|
|
||||||
|
|
||||||
Future input formats may include:
|
|
||||||
|
|
||||||
- JSON.
|
|
||||||
- SRT.
|
|
||||||
- VTT.
|
|
||||||
|
|
||||||
Input readers should be selected from an explicit registry. A reader is responsible for loading external data and returning raw transcript documents, not for canonical normalization.
|
|
||||||
|
|
||||||
### 3. Preprocessing Stage
|
|
||||||
|
|
||||||
The preprocessing stage applies zero or more modules before global merge.
|
|
||||||
|
|
||||||
Preprocessing starts with raw transcript documents from input readers and must end with canonical per-speaker transcripts. Some preprocessing modules operate on raw transcripts, some perform raw-to-canonical normalization, and some operate only on canonical transcripts.
|
|
||||||
|
|
||||||
Preprocessing modules are selected at runtime with a comma-separated list of canonical module names:
|
|
||||||
|
|
||||||
```text
|
|
||||||
--preprocessing-modules validate-raw,normalize-speakers,trim-text
|
|
||||||
```
|
|
||||||
|
|
||||||
Modules run in the exact order provided. Unknown module names are configuration errors.
|
|
||||||
|
|
||||||
Potential preprocessing modules include:
|
|
||||||
|
|
||||||
- Structural raw transcript validation.
|
|
||||||
- Semantic transcript validation.
|
|
||||||
- Raw-to-canonical transcript normalization.
|
|
||||||
- Speaker name normalization based on input filename.
|
|
||||||
- Timing validation and deterministic correction.
|
|
||||||
- Text trimming.
|
|
||||||
|
|
||||||
Preprocessing should not depend on global chronological ordering across speakers. Modules that need the globally merged transcript belong in postprocessing.
|
|
||||||
|
|
||||||
Each preprocessing module must declare the model state it requires and the model state it produces. For example, `validate-raw` requires raw transcripts and produces raw transcripts, while `normalize-speakers` requires raw transcripts and produces canonical transcripts. Configuration validation should reject module orders that cannot type-check.
|
|
||||||
|
|
||||||
### 4. Merge Stage
|
|
||||||
|
|
||||||
The merge stage extracts all canonical segments from the preprocessed per-speaker transcripts and sorts them into a single deterministic chronological sequence.
|
|
||||||
|
|
||||||
The recommended sort key is:
|
|
||||||
|
|
||||||
```text
|
|
||||||
(start, end, source, source_segment_index, speaker)
|
|
||||||
```
|
|
||||||
|
|
||||||
The exact tie-breaker must be documented and stable across runs.
|
|
||||||
|
|
||||||
The merge stage should assign temporary internal references if needed, but it should not assign final output IDs until after all order-affecting postprocessing is complete.
|
|
||||||
|
|
||||||
### 5. Postprocessing Stage
|
|
||||||
|
|
||||||
The postprocessing stage applies zero or more modules to the merged transcript.
|
|
||||||
|
|
||||||
Postprocessing modules are selected at runtime with a comma-separated list of canonical module names:
|
|
||||||
|
|
||||||
```text
|
|
||||||
--postprocessing-modules detect-overlaps,resolve-overlaps,backchannel,filler,resolve-danglers,coalesce,detect-overlaps,autocorrect,assign-ids,validate-output
|
|
||||||
```
|
|
||||||
|
|
||||||
Modules run in the exact order provided. Unknown module names are configuration errors.
|
|
||||||
|
|
||||||
Potential postprocessing modules include:
|
|
||||||
|
|
||||||
- Overlap group detection.
|
|
||||||
- Overlap group refinement.
|
|
||||||
- Same-speaker segment coalescing.
|
|
||||||
- Deterministic grammar cleanup.
|
|
||||||
- Word replacement from `autocorrect.yml`.
|
|
||||||
- Final segment ID assignment.
|
|
||||||
- Output model validation.
|
|
||||||
|
|
||||||
Any module that can reorder, split, merge, drop, or create segments must run before final ID assignment.
|
|
||||||
|
|
||||||
### 6. Output Stage
|
|
||||||
|
|
||||||
The output stage emits one or more artifacts from the final transcript and report model.
|
|
||||||
|
|
||||||
The current output format is JSON, specified with:
|
|
||||||
|
|
||||||
```text
|
|
||||||
--output-file merged.json
|
|
||||||
```
|
|
||||||
|
|
||||||
Future output formats may include:
|
|
||||||
|
|
||||||
- Markdown.
|
|
||||||
- SRT.
|
|
||||||
- VTT.
|
|
||||||
- Validation reports.
|
|
||||||
- Overlap reports.
|
|
||||||
|
|
||||||
Output writers should be selected from an explicit registry and should consume the final transcript model read-only. Multiple output writers may run for a single invocation.
|
|
||||||
|
|
||||||
## Module Classification
|
|
||||||
|
|
||||||
Modules should be classified by their contract and allowed effects.
|
|
||||||
|
|
||||||
| Class | Input | Output | Allowed effects |
|
|
||||||
| --- | --- | --- | --- |
|
|
||||||
| `InputReader` | External source spec | Raw transcript documents | Reads external data |
|
|
||||||
| `Validator` | Raw, canonical, merged, or final model | Same model plus report events | Observes only |
|
|
||||||
| `Normalizer` | Raw model | Canonical model | Converts representation |
|
|
||||||
| `Corrector` | Canonical model | Canonical model plus report events | Deterministic mutation |
|
|
||||||
| `Annotator` | Canonical or merged model | Same model plus annotations | Adds metadata |
|
|
||||||
| `Transformer` | Canonical or merged model | Updated model plus report events | May reorder, split, merge, drop, or create segments |
|
|
||||||
| `OutputWriter` | Final transcript and report | External artifact | Writes output |
|
|
||||||
|
|
||||||
This classification should guide Go interfaces and package boundaries. It should also determine where a module is allowed to run.
|
|
||||||
|
|
||||||
## Runtime Module Composition
|
|
||||||
|
|
||||||
The application supports runtime composition of built-in modules.
|
|
||||||
|
|
||||||
Module names are canonical strings registered at startup. CLI flags refer to those names. The configuration stage resolves names into module instances before the pipeline runs.
|
|
||||||
|
|
||||||
Example:
|
|
||||||
|
|
||||||
```text
|
|
||||||
seriatim merge \
|
|
||||||
--input-file eric.json \
|
|
||||||
--input-file mike.json \
|
|
||||||
--speakers speakers.yml \
|
|
||||||
--autocorrect autocorrect.yml \
|
|
||||||
--preprocessing-modules validate-raw,normalize-speakers,trim-text \
|
|
||||||
--postprocessing-modules detect-overlaps,resolve-overlaps,backchannel,filler,resolve-danglers,coalesce,detect-overlaps,autocorrect,assign-ids,validate-output \
|
|
||||||
--output-modules json \
|
|
||||||
--output-schema default \
|
|
||||||
--output-file merged.json \
|
|
||||||
--report-file report.json
|
|
||||||
```
|
|
||||||
|
|
||||||
Composition rules:
|
|
||||||
|
|
||||||
- Module order is exactly the order specified by the user.
|
|
||||||
- An empty module list is valid when the stage supports zero modules.
|
|
||||||
- Unknown module names are fatal configuration errors.
|
|
||||||
- Module-specific options are read from the validated application config.
|
|
||||||
- A module must declare which pipeline stage and model type it supports.
|
|
||||||
- Modules should be deterministic for the same inputs, config, and application version.
|
|
||||||
- Modules should not perform I/O unless their class explicitly allows it.
|
|
||||||
|
|
||||||
Some modules may be recommended defaults. Defaults should be explicit in documentation and should be equivalent to passing the corresponding module list.
|
|
||||||
|
|
||||||
## Go Interface Sketch
|
|
||||||
|
|
||||||
The exact implementation may evolve, but the core interfaces should resemble:
|
|
||||||
|
|
||||||
```go
|
|
||||||
type InputReader interface {
|
|
||||||
Name() string
|
|
||||||
Read(ctx context.Context, spec InputSpec, cfg Config) ([]RawTranscript, []ReportEvent, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
type Preprocessor interface {
|
|
||||||
Name() string
|
|
||||||
Requires() ModelState
|
|
||||||
Produces() ModelState
|
|
||||||
Process(ctx context.Context, in PreprocessState, cfg Config) (PreprocessState, []ReportEvent, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
type Merger interface {
|
|
||||||
Merge(ctx context.Context, in []CanonicalTranscript, cfg Config) (MergedTranscript, []ReportEvent, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
type Postprocessor interface {
|
|
||||||
Name() string
|
|
||||||
Process(ctx context.Context, in MergedTranscript, cfg Config) (MergedTranscript, []ReportEvent, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
type OutputWriter interface {
|
|
||||||
Name() string
|
|
||||||
Write(ctx context.Context, out any, report Report, cfg Config) ([]ReportEvent, error)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
`PreprocessState` should carry either raw transcripts, canonical transcripts, or both during migration between representations. The pipeline should validate that the ordered preprocessing list transitions from raw input state to canonical output state exactly once before merge.
|
|
||||||
|
|
||||||
The interfaces should favor value returns over hidden mutation. If pointer-based implementations are chosen for performance, mutation boundaries must still be clear and tested.
|
|
||||||
|
|
||||||
## Canonical Internal Model
|
|
||||||
|
|
||||||
The canonical model should be richer than the final output schema.
|
|
||||||
|
|
||||||
Canonical segment fields should include:
|
|
||||||
|
|
||||||
- Temporary internal reference.
|
|
||||||
- Source identifier.
|
|
||||||
- Source segment index.
|
|
||||||
- Canonical speaker.
|
|
||||||
- Start time.
|
|
||||||
- End time.
|
|
||||||
- Text.
|
|
||||||
- Word-level timing data, if available.
|
|
||||||
- Raw diarization labels, if useful for reporting.
|
|
||||||
- Validation and correction metadata, if needed internally.
|
|
||||||
|
|
||||||
The final output model can omit internal-only fields, but the report should retain enough provenance to diagnose corrections and transformations.
|
|
||||||
|
|
||||||
## Validation Strategy
|
|
||||||
|
|
||||||
Validation occurs at multiple boundaries:
|
|
||||||
|
|
||||||
- Configuration validation before processing.
|
|
||||||
- Raw input structural validation after input loading.
|
|
||||||
- Raw input semantic validation before normalization or correction.
|
|
||||||
- Canonical model validation after normalization and preprocessing.
|
|
||||||
- Merged model validation after merge and postprocessing.
|
|
||||||
- Final output schema validation before writing artifacts.
|
|
||||||
|
|
||||||
Structural validation answers whether data has the required shape and types.
|
|
||||||
|
|
||||||
Semantic validation answers whether the data is plausible and internally consistent.
|
|
||||||
|
|
||||||
Correctable issues should be deterministic and reportable. Fatal issues should stop the run with a non-zero exit code.
|
|
||||||
|
|
||||||
Examples of correctable issues:
|
|
||||||
|
|
||||||
- Leading or trailing whitespace.
|
|
||||||
- Segment `end < start`, when configured correction policy allows deterministic repair.
|
|
||||||
- Missing word speaker labels when canonical speaker is known.
|
|
||||||
- Raw diarization labels that should be replaced with the canonical speaker.
|
|
||||||
|
|
||||||
Examples of fatal issues:
|
|
||||||
|
|
||||||
- Input file is not valid JSON.
|
|
||||||
- Required transcript fields are missing.
|
|
||||||
- Speaker map does not identify a canonical speaker for an input.
|
|
||||||
- Unknown module name.
|
|
||||||
- Output fails final schema validation.
|
|
||||||
|
|
||||||
## Overlap Handling
|
|
||||||
|
|
||||||
Overlap detection should create overlap groups rather than only pairwise annotations.
|
|
||||||
|
|
||||||
Two adjacent sorted segments overlap when:
|
|
||||||
|
|
||||||
```text
|
|
||||||
next.start < current_group_end
|
|
||||||
```
|
|
||||||
|
|
||||||
This supports transitive overlap groups:
|
|
||||||
|
|
||||||
```text
|
|
||||||
A: 10.0-14.0
|
|
||||||
B: 12.0-13.0
|
|
||||||
C: 13.5-15.0
|
|
||||||
```
|
|
||||||
|
|
||||||
These belong to one overlap group spanning `10.0-15.0`.
|
|
||||||
|
|
||||||
Overlap groups should record:
|
|
||||||
|
|
||||||
- Overlap group ID.
|
|
||||||
- Group start time.
|
|
||||||
- Group end time.
|
|
||||||
- Segment references.
|
|
||||||
- Speakers involved.
|
|
||||||
- Classification, if known.
|
|
||||||
- Resolution status.
|
|
||||||
|
|
||||||
Initial classifications may include:
|
|
||||||
|
|
||||||
- `unknown`
|
|
||||||
- `minor_overlap`
|
|
||||||
- `handoff`
|
|
||||||
- `backchannel`
|
|
||||||
- `crosstalk`
|
|
||||||
|
|
||||||
The `resolve-overlaps` module uses preserved word-level timing to replace detected overlap-group segments with smaller word-run segments when usable timing is available. Resolution expands each overlap window by the configured coalesce gap so nearby same-speaker context can be absorbed into the replacement runs. Groups without usable word timing remain unresolved for later passes or human review.
|
|
||||||
|
|
||||||
Overlap resolution should be non-destructive. Original segment text, timing, and source metadata must remain recoverable.
|
|
||||||
|
|
||||||
## Final ID Assignment
|
|
||||||
|
|
||||||
Final segment IDs should be assigned by an explicit postprocessing module after every transformation that can affect segment order.
|
|
||||||
|
|
||||||
Final IDs should be sequential integers starting from `1`.
|
|
||||||
|
|
||||||
Final IDs should reflect final chronological order.
|
|
||||||
|
|
||||||
Before final ID assignment, modules should reference segments using stable internal references rather than final output IDs.
|
|
||||||
|
|
||||||
## Output Invariants
|
|
||||||
|
|
||||||
A valid merged transcript should satisfy:
|
|
||||||
|
|
||||||
- Every segment has a unique integer ID.
|
|
||||||
- Segment IDs begin at `1`.
|
|
||||||
- Segment IDs increase in final chronological order.
|
|
||||||
- Every segment has a canonical speaker.
|
|
||||||
- Every segment has a source.
|
|
||||||
- Every segment has `start >= 0`.
|
|
||||||
- Every segment has `end >= start`.
|
|
||||||
- The segments array is sorted deterministically.
|
|
||||||
- Any `overlap_group_id` on a segment refers to an existing overlap group.
|
|
||||||
- Every overlap group references at least two segments.
|
|
||||||
- Every referenced segment exists.
|
|
||||||
- Output validates against the selected output schema.
|
|
||||||
|
|
||||||
## Determinism Requirements
|
|
||||||
|
|
||||||
Given the same inputs, config, and application version, `seriatim` should produce byte-stable JSON output where practical.
|
|
||||||
|
|
||||||
To support this:
|
|
||||||
|
|
||||||
- Sort input specs deterministically unless explicit input order is meaningful.
|
|
||||||
- Use stable sort keys.
|
|
||||||
- Assign final IDs only after final ordering.
|
|
||||||
- Avoid Go map iteration order affecting output.
|
|
||||||
- Emit JSON through structs with stable field ordering.
|
|
||||||
- Record application version in output metadata.
|
|
||||||
- Record enabled module names and module order in output metadata or report data.
|
|
||||||
|
|
||||||
## Go Package Layout
|
|
||||||
|
|
||||||
```text
|
|
||||||
cmd/seriatim/ CLI entrypoint
|
|
||||||
internal/config/ CLI/env/config loading and validation
|
|
||||||
internal/pipeline/ Pipeline orchestration and module registry
|
|
||||||
internal/builtin/ Built-in pipeline modules
|
|
||||||
internal/artifact/ Conversion from internal model to public output schema
|
|
||||||
internal/buildinfo/ Build-time version metadata
|
|
||||||
internal/speaker/ Speaker map parsing and lookup
|
|
||||||
internal/model/ Canonical and merged transcript models
|
|
||||||
internal/overlap/ Overlap detection and refinement helpers
|
|
||||||
internal/autocorrect/ Word replacement rules
|
|
||||||
internal/report/ Report model and event accumulation
|
|
||||||
schema/ Public output contract and JSON Schema validation
|
|
||||||
```
|
|
||||||
|
|
||||||
Package boundaries should follow data ownership. Shared models belong in `internal/model`; stage-specific behavior belongs in the relevant stage package.
|
|
||||||
|
|
||||||
## Default Modules
|
|
||||||
|
|
||||||
The default pipeline is equivalent to explicit module lists.
|
|
||||||
|
|
||||||
Recommended default preprocessing modules:
|
|
||||||
|
|
||||||
```text
|
|
||||||
validate-raw,normalize-speakers,trim-text
|
|
||||||
```
|
|
||||||
|
|
||||||
Recommended default postprocessing modules:
|
|
||||||
|
|
||||||
```text
|
|
||||||
detect-overlaps,resolve-overlaps,backchannel,filler,resolve-danglers,coalesce,detect-overlaps,autocorrect,assign-ids,validate-output
|
|
||||||
```
|
|
||||||
|
|
||||||
The default output module is:
|
|
||||||
|
|
||||||
```text
|
|
||||||
json
|
|
||||||
```
|
|
||||||
181
docs/cli.md
Normal file
181
docs/cli.md
Normal file
@@ -0,0 +1,181 @@
|
|||||||
|
# CLI Reference
|
||||||
|
|
||||||
|
## Shortest useful command
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/seriatim merge \
|
||||||
|
--input-file speaker-a.json \
|
||||||
|
--input-file speaker-b.json \
|
||||||
|
--output-file merged.json
|
||||||
|
```
|
||||||
|
|
||||||
|
## Command overview
|
||||||
|
|
||||||
|
| Command | Purpose |
|
||||||
|
| --- | --- |
|
||||||
|
| `merge` | Merge one or more raw transcript JSON inputs into one seriatim artifact. |
|
||||||
|
| `trim` | Keep or remove segment IDs from an existing seriatim artifact. |
|
||||||
|
| `normalize` | Canonicalize transcript-like JSON into a seriatim artifact. |
|
||||||
|
|
||||||
|
Root usage:
|
||||||
|
|
||||||
|
```text
|
||||||
|
seriatim [command]
|
||||||
|
```
|
||||||
|
|
||||||
|
## Global flags
|
||||||
|
|
||||||
|
| Flag | Description |
|
||||||
|
| --- | --- |
|
||||||
|
| `-h, --help` | Show help. |
|
||||||
|
| `-v, --version` | Show build version. |
|
||||||
|
|
||||||
|
## `merge`
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
|
||||||
|
```text
|
||||||
|
seriatim merge [flags]
|
||||||
|
```
|
||||||
|
|
||||||
|
Flags:
|
||||||
|
|
||||||
|
| Flag | Required | Default | Description |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| `--input-file stringArray` | Yes, repeat at least once | none | Input transcript JSON file(s). |
|
||||||
|
| `--output-file string` | Yes | none | Output transcript JSON file path. |
|
||||||
|
| `--report-file string` | No | none | Optional report JSON path. |
|
||||||
|
| `--speakers string` | No | none | Speaker-map YAML file. |
|
||||||
|
| `--autocorrect string` | No | none | Autocorrect YAML file. |
|
||||||
|
| `--input-reader string` | No | `json-files` | Input reader module name. |
|
||||||
|
| `--output-modules string` | No | `json` | Comma-separated output module names. |
|
||||||
|
| `--output-schema string` | No | `seriatim-intermediate` | Output schema name: `seriatim-minimal`, `seriatim-intermediate`, `seriatim-full`. |
|
||||||
|
| `--preprocessing-modules string` | No | `validate-raw,normalize-speakers,trim-text` | Comma-separated preprocessing module names, run in order. |
|
||||||
|
| `--postprocessing-modules string` | No | `detect-overlaps,resolve-overlaps,backchannel,filler,resolve-danglers,coalesce,detect-overlaps,autocorrect,assign-ids,validate-output` | Comma-separated postprocessing module names, run in order. |
|
||||||
|
| `--coalesce-gap string` | No | `3.0` | Non-negative seconds for coalescing and overlap-resolution context. |
|
||||||
|
|
||||||
|
`merge` behavior and validation:
|
||||||
|
|
||||||
|
- Unknown input reader, preprocessing module, postprocessing module, or output module fails the command.
|
||||||
|
- Preprocessing order must satisfy module state requirements (`raw` -> `canonical`); invalid order fails.
|
||||||
|
- Input files are validated, deduplicated, normalized, then sorted for deterministic processing.
|
||||||
|
- Optional report output is written only when `--report-file` is set.
|
||||||
|
- When `--output-schema` is omitted, schema resolution is: `SERIATIM_OUTPUT_SCHEMA` -> default `seriatim-intermediate`.
|
||||||
|
|
||||||
|
## `trim`
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
|
||||||
|
```text
|
||||||
|
seriatim trim [flags]
|
||||||
|
```
|
||||||
|
|
||||||
|
Flags:
|
||||||
|
|
||||||
|
| Flag | Required | Default | Description |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| `--input-file string` | Yes | none | Input seriatim artifact JSON file. |
|
||||||
|
| `--output-file string` | Yes | none | Output transcript JSON file path. |
|
||||||
|
| `--keep string` | Exactly one of `--keep` / `--remove` | none | Segment ID selector to keep. |
|
||||||
|
| `--remove string` | Exactly one of `--keep` / `--remove` | none | Segment ID selector to remove. |
|
||||||
|
| `--output-schema string` | No | preserve input artifact schema | Output schema override: `seriatim-minimal`, `seriatim-intermediate`, `seriatim-full`. |
|
||||||
|
| `--report-file string` | No | none | Optional report JSON path. |
|
||||||
|
| `--allow-empty` | No | `false` | Allow output with zero segments. |
|
||||||
|
|
||||||
|
Selector rules:
|
||||||
|
|
||||||
|
- IDs must be positive integers.
|
||||||
|
- Single IDs and inclusive ranges are supported: `1`, `1-10`.
|
||||||
|
- Comma-separated selectors are supported: `1-10,15,20-25`.
|
||||||
|
- Whitespace around commas and hyphens is allowed.
|
||||||
|
- Descending ranges (example `10-1`) are invalid.
|
||||||
|
- Duplicates and overlapping ranges are normalized as a union.
|
||||||
|
|
||||||
|
`trim` behavior:
|
||||||
|
|
||||||
|
- Input must already be a valid seriatim artifact (not raw merge input JSON).
|
||||||
|
- Output keeps transcript order from input and renumbers retained segment IDs sequentially.
|
||||||
|
- If `--output-schema` is omitted, the input artifact schema is preserved.
|
||||||
|
- `trim` never runs merge preprocessing/postprocessing modules.
|
||||||
|
|
||||||
|
## `normalize`
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
|
||||||
|
```text
|
||||||
|
seriatim normalize [flags]
|
||||||
|
```
|
||||||
|
|
||||||
|
Flags:
|
||||||
|
|
||||||
|
| Flag | Required | Default | Description |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| `--input-file string` | Yes | none | Input transcript JSON file. |
|
||||||
|
| `--output-file string` | Yes | none | Output transcript JSON file path. |
|
||||||
|
| `--output-schema string` | No | `seriatim-intermediate` | Output schema name: `seriatim-minimal`, `seriatim-intermediate`, `seriatim-full`. |
|
||||||
|
| `--output-modules string` | No | `json` | Comma-separated output module names (`json` only). |
|
||||||
|
| `--report-file string` | No | none | Optional report JSON path. |
|
||||||
|
|
||||||
|
`normalize` input shapes:
|
||||||
|
|
||||||
|
- Object with top-level `segments` array.
|
||||||
|
- Bare top-level segment array.
|
||||||
|
|
||||||
|
`normalize` behavior:
|
||||||
|
|
||||||
|
- Sorts deterministically and reassigns output IDs sequentially from `1`.
|
||||||
|
- Fills missing/blank speakers with `Unknown_Speaker`.
|
||||||
|
- Repairs/sanitizes timing fields deterministically; rejects invalid repaired timing.
|
||||||
|
- Drops segments with missing or blank text.
|
||||||
|
- Does not run merge modules.
|
||||||
|
- When `--output-schema` is omitted, schema resolution is: `SERIATIM_OUTPUT_SCHEMA` -> default `seriatim-intermediate`.
|
||||||
|
|
||||||
|
## Common workflows
|
||||||
|
|
||||||
|
Merge with a speaker map and report output:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/seriatim merge \
|
||||||
|
--input-file examples/minimal-merge/input-alice.json \
|
||||||
|
--input-file examples/minimal-merge/input-bob.json \
|
||||||
|
--speakers examples/minimal-merge/speakers.yml \
|
||||||
|
--output-file /tmp/seriatim-example-merge.json \
|
||||||
|
--report-file /tmp/seriatim-example-merge-report.json
|
||||||
|
```
|
||||||
|
|
||||||
|
Trim to a segment subset:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/seriatim trim \
|
||||||
|
--input-file examples/trim/input-full.json \
|
||||||
|
--output-file /tmp/seriatim-example-trim.json \
|
||||||
|
--keep "1-2"
|
||||||
|
```
|
||||||
|
|
||||||
|
Normalize an external transcript JSON file:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/seriatim normalize \
|
||||||
|
--input-file examples/normalize/object-with-segments.json \
|
||||||
|
--output-file /tmp/seriatim-example-normalize-object.json
|
||||||
|
```
|
||||||
|
|
||||||
|
## Exit and errors
|
||||||
|
|
||||||
|
- Commands return exit code `0` on success.
|
||||||
|
- On error, the CLI prints one error line to stderr and exits with status `1`.
|
||||||
|
- Cobra usage text is silenced on runtime errors; use `--help` for command usage.
|
||||||
|
|
||||||
|
## Related docs
|
||||||
|
|
||||||
|
- Configuration reference: [config.md](config.md)
|
||||||
|
- Operations guide: [operations.md](operations.md)
|
||||||
|
- Troubleshooting: [troubleshooting.md](troubleshooting.md)
|
||||||
|
- Integration notes:
|
||||||
|
- [integrations/whisperx-json.md](integrations/whisperx-json.md)
|
||||||
|
- [integrations/output-schemas.md](integrations/output-schemas.md)
|
||||||
|
- Synthetic examples: [../examples/README.md](../examples/README.md)
|
||||||
|
- Public output schemas:
|
||||||
|
- [../schema/minimal-output.schema.json](../schema/minimal-output.schema.json)
|
||||||
|
- [../schema/intermediate-output.schema.json](../schema/intermediate-output.schema.json)
|
||||||
|
- [../schema/full-output.schema.json](../schema/full-output.schema.json)
|
||||||
167
docs/config.md
Normal file
167
docs/config.md
Normal file
@@ -0,0 +1,167 @@
|
|||||||
|
# Configuration Reference
|
||||||
|
|
||||||
|
## Configuration surfaces
|
||||||
|
|
||||||
|
seriatim has no central JSON/TOML/YAML application config file.
|
||||||
|
|
||||||
|
Runtime configuration comes from:
|
||||||
|
|
||||||
|
1. CLI flags
|
||||||
|
2. Environment variables (`SERIATIM_*`)
|
||||||
|
3. Optional YAML rule files referenced by CLI flags (`--speakers`, `--autocorrect`)
|
||||||
|
|
||||||
|
## Output schema precedence
|
||||||
|
|
||||||
|
For `merge` and `normalize`:
|
||||||
|
|
||||||
|
1. `--output-schema` flag (when explicitly set)
|
||||||
|
2. `SERIATIM_OUTPUT_SCHEMA`
|
||||||
|
3. default `seriatim-intermediate`
|
||||||
|
|
||||||
|
For `trim`:
|
||||||
|
|
||||||
|
- If `--output-schema` is omitted, output preserves the input artifact schema.
|
||||||
|
- If `--output-schema` is set, it must be one of `seriatim-minimal`, `seriatim-intermediate`, `seriatim-full`.
|
||||||
|
|
||||||
|
## Merge module defaults
|
||||||
|
|
||||||
|
Default merge module selections:
|
||||||
|
|
||||||
|
- `--input-reader`: `json-files`
|
||||||
|
- `--preprocessing-modules`: `validate-raw,normalize-speakers,trim-text`
|
||||||
|
- `--postprocessing-modules`: `detect-overlaps,resolve-overlaps,backchannel,filler,resolve-danglers,coalesce,detect-overlaps,autocorrect,assign-ids,validate-output`
|
||||||
|
- `--output-modules`: `json`
|
||||||
|
|
||||||
|
Module-list notes:
|
||||||
|
|
||||||
|
- Lists are comma-separated.
|
||||||
|
- Empty module names are invalid.
|
||||||
|
- Unknown module names fail the command.
|
||||||
|
- Preprocessing order must satisfy state requirements.
|
||||||
|
|
||||||
|
## Environment variables
|
||||||
|
|
||||||
|
| Variable | Default | Used by | Rules |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| `SERIATIM_OUTPUT_SCHEMA` | `seriatim-intermediate` | `merge`, `normalize` | Must be `seriatim-minimal`, `seriatim-intermediate`, or `seriatim-full`. Ignored when `--output-schema` is explicitly set. |
|
||||||
|
| `SERIATIM_OVERLAP_WORD_RUN_GAP` | `1.0` | `merge` | Positive float (`> 0`). |
|
||||||
|
| `SERIATIM_OVERLAP_WORD_RUN_REORDER_WINDOW` | `1.0` | `merge` | Positive float (`> 0`). |
|
||||||
|
| `SERIATIM_BACKCHANNEL_MAX_DURATION` | `2.0` | `merge` | Positive float (`> 0`). |
|
||||||
|
| `SERIATIM_FILLER_MAX_DURATION` | `1.25` | `merge` | Positive float (`> 0`). |
|
||||||
|
|
||||||
|
Additional merge threshold flag:
|
||||||
|
|
||||||
|
- `--coalesce-gap` defaults to `3.0` and must be a non-negative float (`>= 0`).
|
||||||
|
|
||||||
|
## `speakers.yml`
|
||||||
|
|
||||||
|
Purpose:
|
||||||
|
|
||||||
|
- Maps each merge input filename basename to a canonical speaker label.
|
||||||
|
|
||||||
|
Top-level key:
|
||||||
|
|
||||||
|
- `match` (array of ordered rules)
|
||||||
|
|
||||||
|
Rule fields:
|
||||||
|
|
||||||
|
- `speaker` (required, non-empty)
|
||||||
|
- `match` (required, non-empty array of non-empty strings)
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
match:
|
||||||
|
- speaker: "Alice"
|
||||||
|
match:
|
||||||
|
- "alice_track"
|
||||||
|
- "alice"
|
||||||
|
|
||||||
|
- speaker: "Bob"
|
||||||
|
match:
|
||||||
|
- "bob_track"
|
||||||
|
```
|
||||||
|
|
||||||
|
Behavior:
|
||||||
|
|
||||||
|
- Matching is case-insensitive.
|
||||||
|
- Matching is against basename only (not full path).
|
||||||
|
- First matching rule wins.
|
||||||
|
- Duplicate `speaker` values are invalid.
|
||||||
|
- If any input file has no match, merge fails.
|
||||||
|
|
||||||
|
## `autocorrect.yml`
|
||||||
|
|
||||||
|
Purpose:
|
||||||
|
|
||||||
|
- Applies ordered token-level text replacements during merge `autocorrect` postprocessing.
|
||||||
|
|
||||||
|
Top-level key:
|
||||||
|
|
||||||
|
- `autocorrect` (array of rules)
|
||||||
|
|
||||||
|
Rule fields:
|
||||||
|
|
||||||
|
- `target` (required, non-empty)
|
||||||
|
- `match` (required, non-empty array of non-empty strings)
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
autocorrect:
|
||||||
|
- target: "Godfrey"
|
||||||
|
match:
|
||||||
|
- "God-free"
|
||||||
|
|
||||||
|
- target: "Mike Brown"
|
||||||
|
match:
|
||||||
|
- "Mike Pat"
|
||||||
|
```
|
||||||
|
|
||||||
|
Behavior:
|
||||||
|
|
||||||
|
- Match strings are case-sensitive.
|
||||||
|
- Replacements are whole-token only (no substring replacement inside larger tokens).
|
||||||
|
- Duplicate match strings within one rule are invalid.
|
||||||
|
- Duplicate match strings across different rules are invalid.
|
||||||
|
- If `--autocorrect` is not provided, the autocorrect module is skipped.
|
||||||
|
|
||||||
|
## Path and validation rules
|
||||||
|
|
||||||
|
All commands:
|
||||||
|
|
||||||
|
- `--input-file` paths must exist and must be files.
|
||||||
|
- Output/report parent directories must already exist.
|
||||||
|
- Paths are normalized before use.
|
||||||
|
|
||||||
|
`merge`:
|
||||||
|
|
||||||
|
- Requires at least one `--input-file`.
|
||||||
|
- Rejects duplicate `--input-file` paths.
|
||||||
|
- Sorts normalized input file paths for deterministic execution.
|
||||||
|
- `--speakers` and `--autocorrect` are optional, but when set they must point to existing files.
|
||||||
|
|
||||||
|
`trim`:
|
||||||
|
|
||||||
|
- Requires exactly one of `--keep` or `--remove`.
|
||||||
|
- `--keep` and `--remove` are mutually exclusive.
|
||||||
|
- Validates optional `--output-schema` when provided.
|
||||||
|
|
||||||
|
`normalize`:
|
||||||
|
|
||||||
|
- Validates `--output-schema` through the same schema set as `merge`.
|
||||||
|
- Currently accepts only `json` in `--output-modules`.
|
||||||
|
|
||||||
|
## Related docs
|
||||||
|
|
||||||
|
- CLI reference: [cli.md](cli.md)
|
||||||
|
- Operations guide: [operations.md](operations.md)
|
||||||
|
- Troubleshooting: [troubleshooting.md](troubleshooting.md)
|
||||||
|
- YAML example files:
|
||||||
|
- [../examples/speakers.yml](../examples/speakers.yml)
|
||||||
|
- [../examples/autocorrect.yml](../examples/autocorrect.yml)
|
||||||
|
- Synthetic examples: [../examples/README.md](../examples/README.md)
|
||||||
|
- Public output schemas:
|
||||||
|
- [../schema/minimal-output.schema.json](../schema/minimal-output.schema.json)
|
||||||
|
- [../schema/intermediate-output.schema.json](../schema/intermediate-output.schema.json)
|
||||||
|
- [../schema/full-output.schema.json](../schema/full-output.schema.json)
|
||||||
68
docs/integrations/output-schemas.md
Normal file
68
docs/integrations/output-schemas.md
Normal file
@@ -0,0 +1,68 @@
|
|||||||
|
# Output Schemas
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
seriatim emits one of three public JSON output contracts:
|
||||||
|
|
||||||
|
- `seriatim-minimal`
|
||||||
|
- `seriatim-intermediate`
|
||||||
|
- `seriatim-full`
|
||||||
|
|
||||||
|
These are used by `merge`, `trim`, and `normalize`.
|
||||||
|
|
||||||
|
## Schema roles
|
||||||
|
|
||||||
|
`seriatim-minimal`:
|
||||||
|
|
||||||
|
- compact metadata plus ordered transcript segments
|
||||||
|
- no source/provenance fields
|
||||||
|
- no overlap groups
|
||||||
|
|
||||||
|
`seriatim-intermediate`:
|
||||||
|
|
||||||
|
- compact metadata plus ordered segments
|
||||||
|
- includes optional segment `categories`
|
||||||
|
- no source/provenance fields
|
||||||
|
- no overlap groups
|
||||||
|
|
||||||
|
`seriatim-full`:
|
||||||
|
|
||||||
|
- full metadata (`input_reader`, module lists, input files, output modules)
|
||||||
|
- source/provenance fields on segments
|
||||||
|
- overlap-group data
|
||||||
|
- version metadata populated from build info (`internal/buildinfo`)
|
||||||
|
|
||||||
|
## Semantic invariants
|
||||||
|
|
||||||
|
All schema outputs enforce:
|
||||||
|
|
||||||
|
- segment IDs are sequential starting at `1`
|
||||||
|
- segment timing uses `end >= start`
|
||||||
|
|
||||||
|
Full schema also enforces overlap-group timing (`end >= start`).
|
||||||
|
|
||||||
|
## Validation APIs
|
||||||
|
|
||||||
|
Go package: `gitea.maximumdirect.net/eric/seriatim/schema`
|
||||||
|
|
||||||
|
Key validators:
|
||||||
|
|
||||||
|
- `schema.ValidateMinimalTranscript`
|
||||||
|
- `schema.ValidateIntermediateTranscript`
|
||||||
|
- `schema.ValidateTranscript`
|
||||||
|
- `schema.ValidateMinimalJSON`
|
||||||
|
- `schema.ValidateIntermediateJSON`
|
||||||
|
- `schema.ValidateJSON`
|
||||||
|
|
||||||
|
## Machine-readable schema files
|
||||||
|
|
||||||
|
- [../../schema/minimal-output.schema.json](../../schema/minimal-output.schema.json)
|
||||||
|
- [../../schema/intermediate-output.schema.json](../../schema/intermediate-output.schema.json)
|
||||||
|
- [../../schema/full-output.schema.json](../../schema/full-output.schema.json)
|
||||||
|
|
||||||
|
## Related docs and examples
|
||||||
|
|
||||||
|
- CLI reference: [../cli.md](../cli.md)
|
||||||
|
- Artifact internals: [../internal/artifacts.md](../internal/artifacts.md)
|
||||||
|
- Trim example input artifact:
|
||||||
|
- [../../examples/trim/input-full.json](../../examples/trim/input-full.json)
|
||||||
85
docs/integrations/whisperx-json.md
Normal file
85
docs/integrations/whisperx-json.md
Normal file
@@ -0,0 +1,85 @@
|
|||||||
|
# WhisperX-Like JSON Input
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
This document covers the implemented JSON subset consumed by `seriatim merge`.
|
||||||
|
It does not describe full WhisperX output.
|
||||||
|
No explicit WhisperX version is encoded in the repository.
|
||||||
|
|
||||||
|
## Supported top-level shape
|
||||||
|
|
||||||
|
Merge expects a JSON object with top-level `segments` array:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"segments": [
|
||||||
|
{
|
||||||
|
"start": 0.0,
|
||||||
|
"end": 1.2,
|
||||||
|
"text": "hello"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Supported segment fields
|
||||||
|
|
||||||
|
Required per segment:
|
||||||
|
|
||||||
|
- `start` (number, `>= 0`)
|
||||||
|
- `end` (number, `>= start`)
|
||||||
|
- `text` (string)
|
||||||
|
|
||||||
|
Optional per segment:
|
||||||
|
|
||||||
|
- `words` (array)
|
||||||
|
|
||||||
|
## Supported word fields
|
||||||
|
|
||||||
|
Required when a word object is present:
|
||||||
|
|
||||||
|
- `word` (string)
|
||||||
|
|
||||||
|
Optional word timing fields:
|
||||||
|
|
||||||
|
- `start` (number)
|
||||||
|
- `end` (number)
|
||||||
|
|
||||||
|
Timing rules:
|
||||||
|
|
||||||
|
- if both `start` and `end` are present, they must be numeric and `end >= start`
|
||||||
|
- if either timing field is missing, the word is accepted but not used as a
|
||||||
|
timing anchor for overlap resolution
|
||||||
|
|
||||||
|
Additional optional word fields:
|
||||||
|
|
||||||
|
- `score` (number)
|
||||||
|
- `speaker` (string)
|
||||||
|
|
||||||
|
## Validation and failure behavior
|
||||||
|
|
||||||
|
Merge fails for:
|
||||||
|
|
||||||
|
- malformed JSON
|
||||||
|
- missing top-level `segments`
|
||||||
|
- non-array `segments`
|
||||||
|
- missing required segment fields
|
||||||
|
- wrong field types
|
||||||
|
- negative segment/word start times
|
||||||
|
- segment/word end before start
|
||||||
|
|
||||||
|
Word timing missing from a word does not fail merge; it emits a warning event
|
||||||
|
in the optional report.
|
||||||
|
|
||||||
|
## Overlap-resolution impact
|
||||||
|
|
||||||
|
- overlap resolution uses timed words when available
|
||||||
|
- untimed words are kept in replacement text but do not provide timing anchors
|
||||||
|
|
||||||
|
## Related docs and examples
|
||||||
|
|
||||||
|
- CLI reference: [../cli.md](../cli.md)
|
||||||
|
- Configuration reference: [../config.md](../config.md)
|
||||||
|
- Minimal merge example inputs:
|
||||||
|
- [../../examples/minimal-merge/input-alice.json](../../examples/minimal-merge/input-alice.json)
|
||||||
|
- [../../examples/minimal-merge/input-bob.json](../../examples/minimal-merge/input-bob.json)
|
||||||
154
docs/internal/artifacts.md
Normal file
154
docs/internal/artifacts.md
Normal file
@@ -0,0 +1,154 @@
|
|||||||
|
# Artifact Internals
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
Describes public artifact conversion and validation internals for merge output,
|
||||||
|
trim, and normalize.
|
||||||
|
|
||||||
|
## Artifact contracts
|
||||||
|
|
||||||
|
Public contracts live in `schema/`:
|
||||||
|
|
||||||
|
- full: `schema.Transcript`
|
||||||
|
- intermediate: `schema.IntermediateTranscript`
|
||||||
|
- minimal: `schema.MinimalTranscript`
|
||||||
|
|
||||||
|
Machine-readable schemas:
|
||||||
|
|
||||||
|
- `schema/full-output.schema.json`
|
||||||
|
- `schema/intermediate-output.schema.json`
|
||||||
|
- `schema/minimal-output.schema.json`
|
||||||
|
|
||||||
|
## Schema selection
|
||||||
|
|
||||||
|
Merge pipeline conversion uses `internal/artifact.SelectedFromMerged`:
|
||||||
|
|
||||||
|
- `seriatim-full` -> `artifact.FromMerged`
|
||||||
|
- `seriatim-intermediate` -> `artifact.IntermediateFromMerged`
|
||||||
|
- `seriatim-minimal` -> `artifact.MinimalFromMerged`
|
||||||
|
|
||||||
|
Unknown/empty selection falls back to intermediate conversion.
|
||||||
|
|
||||||
|
## Merge conversion behavior
|
||||||
|
|
||||||
|
`internal/artifact` converts `model.MergedTranscript` to public contracts:
|
||||||
|
|
||||||
|
- full schema preserves source/provenance, overlap groups, and metadata module
|
||||||
|
lists.
|
||||||
|
- intermediate schema emits segment timing/text/speaker with optional
|
||||||
|
categories and compact metadata.
|
||||||
|
- minimal schema emits compact segment timing/text/speaker and compact
|
||||||
|
metadata.
|
||||||
|
|
||||||
|
## Validation behavior
|
||||||
|
|
||||||
|
`schema/output.go` validates both structure and semantics:
|
||||||
|
|
||||||
|
- embedded JSON Schema validation via `jsonschema/v6`
|
||||||
|
- semantic checks for sequential segment IDs starting at `1`
|
||||||
|
- semantic checks for non-inverted segment timing (`end >= start`)
|
||||||
|
- full schema overlap-group timing checks (`group.end >= group.start`)
|
||||||
|
|
||||||
|
## Trim internals
|
||||||
|
|
||||||
|
`internal/trim` handles artifact-level projection and does not execute merge
|
||||||
|
pipeline modules.
|
||||||
|
|
||||||
|
Run layer (`run.go`):
|
||||||
|
|
||||||
|
1. Parse selector from validated config.
|
||||||
|
2. Read and parse input artifact JSON.
|
||||||
|
3. Apply trim projection through schema-aware artifact handling.
|
||||||
|
4. Resolve output schema (preserve input schema unless overridden).
|
||||||
|
5. Validate output artifact.
|
||||||
|
6. Write output JSON.
|
||||||
|
7. Optionally write report JSON with `trim-audit`.
|
||||||
|
|
||||||
|
Apply layer (`apply.go`):
|
||||||
|
|
||||||
|
- one shared projection policy for selector mode, input ID validation, selected
|
||||||
|
ID existence checks, keep/remove filtering, removed IDs, and old-to-new ID
|
||||||
|
mappings
|
||||||
|
- schema-specific segment reconstruction for full/intermediate/minimal outputs
|
||||||
|
- overlap-group recomputation only for full-schema outputs
|
||||||
|
|
||||||
|
Artifact layer (`artifact.go`):
|
||||||
|
|
||||||
|
- schema detection for full/intermediate/minimal artifacts
|
||||||
|
- schema-preserving trim application
|
||||||
|
- supported schema conversions:
|
||||||
|
- full -> intermediate/minimal
|
||||||
|
- intermediate -> minimal
|
||||||
|
- minimal -> intermediate
|
||||||
|
- rejected conversion:
|
||||||
|
- intermediate/minimal -> full
|
||||||
|
|
||||||
|
Trim invariants:
|
||||||
|
|
||||||
|
- selected IDs must exist in input.
|
||||||
|
- input IDs must be positive, unique, sequential.
|
||||||
|
- retained segment order follows input transcript order.
|
||||||
|
- output IDs are reassigned to `1..N`.
|
||||||
|
|
||||||
|
## Normalize internals
|
||||||
|
|
||||||
|
`internal/normalize` canonicalizes transcript-like JSON input into a selected
|
||||||
|
public schema.
|
||||||
|
|
||||||
|
Parse layer (`parse.go`):
|
||||||
|
|
||||||
|
- accepts object-with-`segments` or bare segment array
|
||||||
|
- repairs missing timing deterministically
|
||||||
|
- swaps inverted timing
|
||||||
|
- fills missing/blank speaker with `Unknown_Speaker`
|
||||||
|
- drops missing/blank text segments
|
||||||
|
|
||||||
|
Build layer (`build.go`):
|
||||||
|
|
||||||
|
- sorts deterministically by `(start, end, input_index, speaker)`
|
||||||
|
- reassigns output IDs sequentially
|
||||||
|
- builds minimal/intermediate/full output shape
|
||||||
|
- validates selected output schema before write
|
||||||
|
|
||||||
|
Run layer (`normalize.go`):
|
||||||
|
|
||||||
|
- writes output JSON
|
||||||
|
- optionally writes report with `normalize-audit`
|
||||||
|
|
||||||
|
Normalize invariant:
|
||||||
|
|
||||||
|
- report events do not embed transcript text.
|
||||||
|
|
||||||
|
## Boundaries
|
||||||
|
|
||||||
|
- CLI flag semantics belong to `docs/cli.md`.
|
||||||
|
- Runtime config/env surfaces belong to `docs/config.md`.
|
||||||
|
- This doc describes internal conversion/validation behavior only.
|
||||||
|
|
||||||
|
## Failure behavior
|
||||||
|
|
||||||
|
Representative failure classes:
|
||||||
|
|
||||||
|
- malformed or unsupported input JSON shape
|
||||||
|
- schema validation failure for parsed artifact or built output
|
||||||
|
- unsupported schema conversion path (trim)
|
||||||
|
- selector or input-ID consistency errors (trim)
|
||||||
|
- output/report file write failures from command paths
|
||||||
|
|
||||||
|
## Tests to inspect before changes
|
||||||
|
|
||||||
|
- `schema/output_test.go`
|
||||||
|
- `internal/artifact/transcript_test.go`
|
||||||
|
- `internal/trim/selector_test.go`
|
||||||
|
- `internal/trim/artifact_test.go`
|
||||||
|
- `internal/trim/apply_test.go`
|
||||||
|
- `internal/normalize/parse_test.go`
|
||||||
|
- `internal/cli/trim_test.go`
|
||||||
|
- `internal/cli/normalize_test.go`
|
||||||
|
|
||||||
|
## Invariants
|
||||||
|
|
||||||
|
- Public artifacts are validated through `schema` before acceptance.
|
||||||
|
- Segment IDs in emitted artifacts are sequential and deterministic.
|
||||||
|
- Internal-only fields are not emitted in minimal/intermediate contracts.
|
||||||
|
- Trim and normalize stay artifact-level and do not execute merge modules.
|
||||||
120
docs/internal/modules.md
Normal file
120
docs/internal/modules.md
Normal file
@@ -0,0 +1,120 @@
|
|||||||
|
# Built-In Modules
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
Describes implemented built-in module behavior and boundaries in
|
||||||
|
`internal/builtin`.
|
||||||
|
|
||||||
|
## Implemented module set
|
||||||
|
|
||||||
|
Input reader:
|
||||||
|
|
||||||
|
- `json-files`
|
||||||
|
|
||||||
|
Preprocessing:
|
||||||
|
|
||||||
|
- `validate-raw`
|
||||||
|
- `normalize-speakers`
|
||||||
|
- `trim-text`
|
||||||
|
|
||||||
|
Merger:
|
||||||
|
|
||||||
|
- `chronological-merge`
|
||||||
|
|
||||||
|
Postprocessing:
|
||||||
|
|
||||||
|
- `detect-overlaps`
|
||||||
|
- `resolve-overlaps`
|
||||||
|
- `backchannel`
|
||||||
|
- `filler`
|
||||||
|
- `resolve-danglers`
|
||||||
|
- `coalesce`
|
||||||
|
- `autocorrect`
|
||||||
|
- `assign-ids`
|
||||||
|
- `validate-output`
|
||||||
|
|
||||||
|
Output writer:
|
||||||
|
|
||||||
|
- `json`
|
||||||
|
|
||||||
|
## Inputs, outputs, and side effects
|
||||||
|
|
||||||
|
- `json-files`: reads JSON files from `cfg.InputFiles`, parses supported
|
||||||
|
segment/word fields, emits warnings for untimed words.
|
||||||
|
- `validate-raw`: validates raw source/timing invariants.
|
||||||
|
- `normalize-speakers`: converts raw transcripts to canonical segments,
|
||||||
|
optionally resolving speakers from `cfg.SpeakersFile`.
|
||||||
|
- `trim-text`: trims canonical segment text whitespace.
|
||||||
|
- `chronological-merge`: flattens canonical segments and applies deterministic
|
||||||
|
sort (`model.SegmentLess`).
|
||||||
|
- `detect-overlaps`: annotates overlap groups.
|
||||||
|
- `resolve-overlaps`: rewrites overlap groups using timed words and thresholds.
|
||||||
|
- `backchannel`/`filler`: classify short utterances using duration thresholds.
|
||||||
|
- `resolve-danglers`: merges dangling derived fragments.
|
||||||
|
- `coalesce`: merges adjacent same-speaker segments within configured gap.
|
||||||
|
- `autocorrect`: applies YAML replacement rules when configured.
|
||||||
|
- `assign-ids`: assigns final sequential IDs.
|
||||||
|
- `validate-output`: validates selected public artifact shape.
|
||||||
|
- `json`: writes artifact JSON to `cfg.OutputFile` through shared deterministic
|
||||||
|
JSON file writing.
|
||||||
|
|
||||||
|
Filesystem side effects are limited to:
|
||||||
|
|
||||||
|
- reading configured input/YAML files
|
||||||
|
- writing configured output artifact
|
||||||
|
|
||||||
|
## Config fields used
|
||||||
|
|
||||||
|
Primary module inputs from `config.Config`:
|
||||||
|
|
||||||
|
- file paths: `InputFiles`, `SpeakersFile`, `AutocorrectFile`, `OutputFile`
|
||||||
|
- schema/modules: `OutputSchema`, `OutputModules`
|
||||||
|
- overlap/coalesce thresholds: `OverlapWordRunGap`,
|
||||||
|
`WordRunReorderWindow`, `CoalesceGap`
|
||||||
|
- category thresholds: `BackchannelMaxDuration`, `FillerMaxDuration`
|
||||||
|
|
||||||
|
## Ordering constraints
|
||||||
|
|
||||||
|
- Preprocessing must satisfy state contracts from `raw` to `canonical`.
|
||||||
|
- Order-sensitive transforms should run before `assign-ids`.
|
||||||
|
- `validate-output` should run after final ID assignment and output-shape
|
||||||
|
mutations.
|
||||||
|
- Default configuration includes a second `detect-overlaps` pass after
|
||||||
|
transformations.
|
||||||
|
|
||||||
|
## Boundaries
|
||||||
|
|
||||||
|
- Modules implement behavior; CLI/config parsing remains outside modules.
|
||||||
|
- Modules communicate through explicit model contracts and report events.
|
||||||
|
- Output modules operate on final artifacts and do not re-run transform logic.
|
||||||
|
|
||||||
|
## Failure behavior
|
||||||
|
|
||||||
|
Representative failures:
|
||||||
|
|
||||||
|
- invalid input JSON shape or typed field errors (`json-files`)
|
||||||
|
- invalid YAML or unmatched speaker map entries
|
||||||
|
- unknown module names during registry resolution
|
||||||
|
- invalid ordering/state transitions in preprocessing chain
|
||||||
|
- validation failure in `validate-output`
|
||||||
|
- output write failure in `json` writer
|
||||||
|
|
||||||
|
## Tests to inspect before changes
|
||||||
|
|
||||||
|
- `internal/builtin/preprocess_test.go`
|
||||||
|
- `internal/builtin/postprocess_test.go`
|
||||||
|
- `internal/overlap/resolve_test.go`
|
||||||
|
- `internal/overlap/detect_test.go`
|
||||||
|
- `internal/coalesce/coalesce_test.go`
|
||||||
|
- `internal/danglers/danglers_test.go`
|
||||||
|
- `internal/backchannel/backchannel_test.go`
|
||||||
|
- `internal/filler/filler_test.go`
|
||||||
|
- `internal/autocorrect/autocorrect_test.go`
|
||||||
|
- `internal/cli/merge_test.go`
|
||||||
|
|
||||||
|
## Invariants
|
||||||
|
|
||||||
|
- Modules are selected by canonical name through the registry.
|
||||||
|
- Execution is sequential and deterministic for a fixed configuration.
|
||||||
|
- `assign-ids` defines final public segment IDs.
|
||||||
|
- `validate-output` enforces public artifact contracts through `schema`.
|
||||||
102
docs/internal/pipeline.md
Normal file
102
docs/internal/pipeline.md
Normal file
@@ -0,0 +1,102 @@
|
|||||||
|
# Pipeline Internals
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
Describes implemented merge pipeline orchestration in `internal/pipeline`.
|
||||||
|
|
||||||
|
## Inputs and outputs
|
||||||
|
|
||||||
|
Input:
|
||||||
|
|
||||||
|
- `config.Config`
|
||||||
|
- registry-resolved modules from `internal/builtin`
|
||||||
|
|
||||||
|
Output:
|
||||||
|
|
||||||
|
- selected public artifact written by output writer modules
|
||||||
|
- optional report JSON when `cfg.ReportFile` is set
|
||||||
|
|
||||||
|
## Stage contracts
|
||||||
|
|
||||||
|
The runner executes these contracts in order:
|
||||||
|
|
||||||
|
1. `InputReader`: external inputs -> `[]model.RawTranscript`
|
||||||
|
2. `Preprocessor`: `PreprocessState` transformations (`raw` -> `canonical`)
|
||||||
|
3. `Merger`: canonical transcripts -> `model.MergedTranscript`
|
||||||
|
4. `Postprocessor`: merged transcript transformations
|
||||||
|
5. `OutputWriter`: serialized artifact writes
|
||||||
|
|
||||||
|
`PreprocessState` must end in `StateCanonical` before merge.
|
||||||
|
|
||||||
|
## Registry resolution
|
||||||
|
|
||||||
|
`resolvePlan` maps configured names to modules:
|
||||||
|
|
||||||
|
- input reader: `cfg.InputReader`
|
||||||
|
- preprocessors: `cfg.PreprocessingModules`
|
||||||
|
- postprocessors: `cfg.PostprocessingModules`
|
||||||
|
- output writers: `cfg.OutputModules`
|
||||||
|
- merger: single registered merger
|
||||||
|
|
||||||
|
Unknown names fail fast with contextual errors.
|
||||||
|
|
||||||
|
## Execution order and reporting
|
||||||
|
|
||||||
|
- Modules run sequentially in configured order.
|
||||||
|
- Events returned by modules are appended in execution order.
|
||||||
|
- Report metadata includes input reader, input files, and module lists.
|
||||||
|
- Output writer events are appended before optional report write.
|
||||||
|
|
||||||
|
## Config fields used
|
||||||
|
|
||||||
|
Runner-level fields:
|
||||||
|
|
||||||
|
- `InputReader`
|
||||||
|
- `InputFiles`
|
||||||
|
- `PreprocessingModules`
|
||||||
|
- `PostprocessingModules`
|
||||||
|
- `OutputModules`
|
||||||
|
- `OutputSchema` (via `artifact.SelectedFromMerged`)
|
||||||
|
- `ReportFile`
|
||||||
|
|
||||||
|
Module-specific settings are consumed inside builtin modules (for example
|
||||||
|
coalesce gap and overlap thresholds).
|
||||||
|
|
||||||
|
## Adapters used
|
||||||
|
|
||||||
|
- Input adapters: registered `InputReader` implementations (default `json-files`).
|
||||||
|
- Output adapters: registered `OutputWriter` implementations (default `json`).
|
||||||
|
- Report adapter: `report.WriteJSON` when `cfg.ReportFile` is provided.
|
||||||
|
|
||||||
|
## Boundaries
|
||||||
|
|
||||||
|
- Pipeline does not parse CLI flags.
|
||||||
|
- Pipeline does not normalize raw CLI strings.
|
||||||
|
- Pipeline delegates conversion to public output contracts to `internal/artifact`.
|
||||||
|
- Artifact-level commands `trim` and `normalize` are outside this pipeline.
|
||||||
|
|
||||||
|
## Failure behavior
|
||||||
|
|
||||||
|
Pipeline returns errors from:
|
||||||
|
|
||||||
|
- registry resolution (unknown modules, missing merger)
|
||||||
|
- invalid preprocessing state transitions
|
||||||
|
- module read/process/merge/write failures
|
||||||
|
- optional report write failure
|
||||||
|
|
||||||
|
No retry/resume state is stored.
|
||||||
|
|
||||||
|
## Tests to inspect before changes
|
||||||
|
|
||||||
|
- `internal/pipeline/runner_test.go`
|
||||||
|
- `internal/builtin/preprocess_test.go`
|
||||||
|
- `internal/builtin/postprocess_test.go`
|
||||||
|
- `internal/cli/merge_test.go`
|
||||||
|
|
||||||
|
## Invariants
|
||||||
|
|
||||||
|
- Sequential deterministic execution order.
|
||||||
|
- Preprocessing state must type-check from `raw` to `canonical`.
|
||||||
|
- Module selection is explicit by canonical names.
|
||||||
|
- Report event order reflects actual execution order.
|
||||||
|
- Output artifact selection is schema-driven via `internal/artifact`.
|
||||||
136
docs/operations.md
Normal file
136
docs/operations.md
Normal file
@@ -0,0 +1,136 @@
|
|||||||
|
# Operations Guide
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
This document covers runtime operation of the implemented CLI commands:
|
||||||
|
|
||||||
|
- `merge`
|
||||||
|
- `trim`
|
||||||
|
- `normalize`
|
||||||
|
|
||||||
|
## Runtime model
|
||||||
|
|
||||||
|
seriatim is a single-process, filesystem-only CLI.
|
||||||
|
|
||||||
|
- Each invocation reads input files, processes in memory, and writes output files.
|
||||||
|
- There is no daemon, queue, database, resume checkpoint, remote storage, or background worker.
|
||||||
|
- On error, the command exits non-zero; there is no built-in retry/resume flow.
|
||||||
|
|
||||||
|
## Filesystem expectations
|
||||||
|
|
||||||
|
All commands require accessible local files and existing parent directories for outputs.
|
||||||
|
|
||||||
|
- Input paths must exist and must be files.
|
||||||
|
- Output/report parent directories must already exist.
|
||||||
|
- Output and report files are created with `os.Create`, so existing files at those paths are overwritten.
|
||||||
|
|
||||||
|
Command-specific expectations:
|
||||||
|
|
||||||
|
- `merge`: requires at least one `--input-file`; optional `--speakers` and `--autocorrect` paths must exist when provided.
|
||||||
|
- `trim`: input must be an existing valid seriatim artifact JSON file.
|
||||||
|
- `normalize`: input must be a JSON object with `segments` or a top-level segment array.
|
||||||
|
|
||||||
|
## Normal workflow
|
||||||
|
|
||||||
|
### Merge
|
||||||
|
|
||||||
|
1. Provide one or more `--input-file` values.
|
||||||
|
2. Optionally provide `--speakers`, `--autocorrect`, and `--report-file`.
|
||||||
|
3. Provide `--output-file`.
|
||||||
|
4. Run command.
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/seriatim merge \
|
||||||
|
--input-file speaker-a.json \
|
||||||
|
--input-file speaker-b.json \
|
||||||
|
--output-file merged.json \
|
||||||
|
--report-file merge-report.json
|
||||||
|
```
|
||||||
|
|
||||||
|
### Trim
|
||||||
|
|
||||||
|
1. Provide existing artifact with `--input-file`.
|
||||||
|
2. Select segments with exactly one of `--keep` or `--remove`.
|
||||||
|
3. Provide `--output-file`.
|
||||||
|
4. Optionally provide `--output-schema`, `--allow-empty`, and `--report-file`.
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/seriatim trim \
|
||||||
|
--input-file merged.json \
|
||||||
|
--output-file trimmed.json \
|
||||||
|
--keep "1-20,25"
|
||||||
|
```
|
||||||
|
|
||||||
|
### Normalize
|
||||||
|
|
||||||
|
1. Provide `--input-file` containing supported JSON shape.
|
||||||
|
2. Provide `--output-file`.
|
||||||
|
3. Optionally provide `--output-schema`, `--output-modules`, and `--report-file`.
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/seriatim normalize \
|
||||||
|
--input-file external.json \
|
||||||
|
--output-file normalized.json \
|
||||||
|
--report-file normalize-report.json
|
||||||
|
```
|
||||||
|
|
||||||
|
## Output and report artifacts
|
||||||
|
|
||||||
|
Primary output:
|
||||||
|
|
||||||
|
- `--output-file` writes JSON transcript artifact in selected schema.
|
||||||
|
|
||||||
|
Optional report output:
|
||||||
|
|
||||||
|
- `--report-file` writes deterministic JSON report events.
|
||||||
|
- `merge` report metadata records reader/modules and event sequence.
|
||||||
|
- `trim` report includes a `trim-audit` event with mode/selector/counts and old-to-new ID mapping.
|
||||||
|
- `normalize` report includes a `normalize-audit` event with input shape, repair stats, and output selection details.
|
||||||
|
|
||||||
|
## Failure and retry behavior
|
||||||
|
|
||||||
|
Failure behavior:
|
||||||
|
|
||||||
|
- Errors are printed once to stderr by the root command and exit status is `1`.
|
||||||
|
- There is no partial-state recovery mechanism.
|
||||||
|
|
||||||
|
Retry guidance:
|
||||||
|
|
||||||
|
1. Fix the reported input/config/path issue.
|
||||||
|
2. Re-run the same command.
|
||||||
|
3. If a prior run created a partial or unwanted output/report file, remove it and rerun.
|
||||||
|
|
||||||
|
Operational note:
|
||||||
|
|
||||||
|
- With identical inputs/config/version, merge behavior is deterministic and input files are sorted before processing.
|
||||||
|
|
||||||
|
## Cleanup
|
||||||
|
|
||||||
|
seriatim does not manage retention.
|
||||||
|
|
||||||
|
- Remove unneeded output/report artifacts manually.
|
||||||
|
- No cache, state directory, or lock files are maintained by the application.
|
||||||
|
|
||||||
|
## Privacy considerations
|
||||||
|
|
||||||
|
Transcript artifacts and reports are local files and may contain sensitive conversational data.
|
||||||
|
|
||||||
|
- Store outputs in controlled directories with appropriate OS permissions.
|
||||||
|
- Share report files carefully; they include file paths and processing diagnostics.
|
||||||
|
- Normalize report events intentionally avoid embedding transcript text, but output artifacts contain transcript content.
|
||||||
|
|
||||||
|
## Related docs
|
||||||
|
|
||||||
|
- CLI reference: [cli.md](cli.md)
|
||||||
|
- Configuration reference: [config.md](config.md)
|
||||||
|
- Troubleshooting: [troubleshooting.md](troubleshooting.md)
|
||||||
|
- Integration notes:
|
||||||
|
- [integrations/whisperx-json.md](integrations/whisperx-json.md)
|
||||||
|
- [integrations/output-schemas.md](integrations/output-schemas.md)
|
||||||
|
- Synthetic examples: [../examples/README.md](../examples/README.md)
|
||||||
219
docs/policy/architecture.md
Normal file
219
docs/policy/architecture.md
Normal file
@@ -0,0 +1,219 @@
|
|||||||
|
# Architecture Policy
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
This document defines seriatim's development architecture and invariants for
|
||||||
|
maintainers and automated coding agents. It describes how the implemented
|
||||||
|
system is intended to be built and changed. It is not a user manual, CLI
|
||||||
|
reference, config reference, or roadmap.
|
||||||
|
|
||||||
|
Keep this document aligned with [documentation policy](documentation.md). It
|
||||||
|
must describe current behavior only; planned or speculative work belongs under
|
||||||
|
`docs/roadmap/`.
|
||||||
|
|
||||||
|
## Project Shape
|
||||||
|
|
||||||
|
seriatim is a Go CLI for transcript artifact processing. The implemented
|
||||||
|
commands are `merge`, `trim`, and `normalize`.
|
||||||
|
|
||||||
|
`merge` reads one or more JSON transcript files, optionally maps input files to
|
||||||
|
canonical speakers, runs a registry-selected preprocessing chain, merges
|
||||||
|
canonical segments into deterministic chronological order, runs a
|
||||||
|
registry-selected postprocessing chain, validates the selected output schema,
|
||||||
|
and writes JSON output plus an optional JSON report.
|
||||||
|
|
||||||
|
`trim` and `normalize` are artifact-level commands outside the merge pipeline.
|
||||||
|
`trim` reads an existing seriatim output artifact and projects it by segment ID.
|
||||||
|
`normalize` reads transcript-like JSON and emits one of seriatim's supported
|
||||||
|
output schemas. Neither command runs merge preprocessing or postprocessing
|
||||||
|
modules.
|
||||||
|
|
||||||
|
The supported public output schemas are `seriatim-minimal`,
|
||||||
|
`seriatim-intermediate`, and `seriatim-full`. For command and flag details, use
|
||||||
|
[CLI reference](../cli.md) and [configuration reference](../config.md).
|
||||||
|
|
||||||
|
## Core Design Principles
|
||||||
|
|
||||||
|
- Keep a hexagonal architecture boundary. Domain models, stage contracts, and
|
||||||
|
deterministic transformations must stay separate from CLI parsing,
|
||||||
|
filesystem access, config loading, reporting, and other external adapters.
|
||||||
|
- Keep stages and modules composable. Built-in modules are selected by
|
||||||
|
canonical registry names and implement explicit interfaces for their pipeline
|
||||||
|
role.
|
||||||
|
- Preserve deterministic behavior. Given the same inputs, configuration, and
|
||||||
|
version, output ordering, segment IDs, schema validation, and report event
|
||||||
|
ordering should remain stable.
|
||||||
|
- Current command execution is sequential. There is no scheduler, worker pool,
|
||||||
|
or concurrent module execution in the implemented pipeline. Any concurrency
|
||||||
|
added later must be bounded, observable, and must not make output handling
|
||||||
|
nondeterministic.
|
||||||
|
- Prefer the Go standard library. Third-party dependencies should remain narrow
|
||||||
|
and justified, such as Cobra for CLI structure, YAML parsing, and JSON Schema
|
||||||
|
validation.
|
||||||
|
- Document current behavior. Architecture, user, and internal docs must not
|
||||||
|
describe planned features as implemented behavior.
|
||||||
|
|
||||||
|
## Architectural Boundaries
|
||||||
|
|
||||||
|
Core transcript data belongs in `internal/model` and public artifact contracts
|
||||||
|
belong in `schema`. Conversion from internal merged data to public JSON shapes
|
||||||
|
belongs at the artifact boundary, not inside CLI code or transformation
|
||||||
|
packages.
|
||||||
|
|
||||||
|
Pipeline orchestration belongs in `internal/pipeline`. It resolves registered
|
||||||
|
modules, validates preprocessing state transitions, executes stages in order,
|
||||||
|
collects report events, converts the final transcript, and writes optional
|
||||||
|
reports. Built-in adapters and modules are registered from `internal/builtin`.
|
||||||
|
|
||||||
|
CLI code in `internal/cli` should parse flags, build validated config values,
|
||||||
|
and delegate. `merge` delegates to `pipeline.Run`; `trim` and `normalize`
|
||||||
|
perform artifact-level orchestration and delegate deterministic parsing,
|
||||||
|
validation, and transformation work to their internal packages.
|
||||||
|
|
||||||
|
Config loading and validation belongs in `internal/config`. Filesystem reads and
|
||||||
|
writes are adapter concerns and should not spread into pure transformation
|
||||||
|
helpers. Existing built-in modules that load configured YAML files must keep
|
||||||
|
that I/O narrow and explicit.
|
||||||
|
|
||||||
|
Reports belong in `internal/report`. Modules and commands should emit concise
|
||||||
|
events for validation findings, corrections, and transformations without
|
||||||
|
turning report messages into a duplicate output artifact.
|
||||||
|
|
||||||
|
Tests and samples are supporting evidence for behavior. Tests should verify
|
||||||
|
stable contracts and edge cases; samples should remain valid examples, not
|
||||||
|
hidden architecture dependencies.
|
||||||
|
|
||||||
|
## Modules or Stages
|
||||||
|
|
||||||
|
The merge pipeline has these implemented stages:
|
||||||
|
|
||||||
|
- `InputReader`: reads configured external input into raw transcripts.
|
||||||
|
- `Preprocessor`: transforms `PreprocessState` from raw to canonical state.
|
||||||
|
- `Merger`: combines canonical transcripts into one merged transcript.
|
||||||
|
- `Postprocessor`: transforms or annotates the merged transcript.
|
||||||
|
- `OutputWriter`: writes the selected output artifact.
|
||||||
|
|
||||||
|
Modules must keep narrow responsibilities, declare their stage through the
|
||||||
|
interface they implement, and use explicit config values. Preprocessors must
|
||||||
|
declare `Requires()` and `Produces()` states; the runner rejects invalid
|
||||||
|
raw/canonical ordering before processing completes.
|
||||||
|
|
||||||
|
Modules run in the configured order. Order-affecting modules must run before
|
||||||
|
`assign-ids`, and `validate-output` must see final IDs that match the selected
|
||||||
|
schema. Accepted and rejected transformations should be deterministic and, when
|
||||||
|
observable, recorded through report events.
|
||||||
|
|
||||||
|
Transformation helpers should avoid hidden global state. Shared caches, such as
|
||||||
|
compiled JSON schemas, must be protected and must not affect output ordering.
|
||||||
|
|
||||||
|
## State, Inputs, and Outputs
|
||||||
|
|
||||||
|
seriatim is file-based. It reads JSON inputs and optional YAML rule files, then
|
||||||
|
writes JSON transcript artifacts and optional JSON reports.
|
||||||
|
|
||||||
|
The implemented application has no durable database, daemon state, resume
|
||||||
|
state, remote storage, or background job state. Runtime state is held in memory
|
||||||
|
for the current command invocation and serialized only through requested output
|
||||||
|
and report files.
|
||||||
|
|
||||||
|
Input file paths are normalized and validated during config construction.
|
||||||
|
`merge` sorts input file paths before processing, then uses stable segment sort
|
||||||
|
keys. `trim` preserves transcript order while renumbering retained IDs.
|
||||||
|
`normalize` sorts by implemented deterministic keys and assigns fresh IDs.
|
||||||
|
|
||||||
|
## Configuration and CLI Boundaries
|
||||||
|
|
||||||
|
The CLI surface is an adapter over validated config structs. Cobra command code
|
||||||
|
should stay thin: parse flags, account for flag/default precedence, call config
|
||||||
|
constructors, and delegate.
|
||||||
|
|
||||||
|
Config constructors validate required paths, output parent directories, module
|
||||||
|
lists, selected schemas, mutually exclusive trim selector options, and supported
|
||||||
|
environment-derived settings. Module name validation is split between config
|
||||||
|
where command-specific names are fixed and the pipeline registry where module
|
||||||
|
composition is resolved.
|
||||||
|
|
||||||
|
Do not duplicate full CLI or config reference material here. Use
|
||||||
|
[CLI reference](../cli.md) and [configuration reference](../config.md) for
|
||||||
|
canonical user-facing details.
|
||||||
|
|
||||||
|
## Errors, Logging, and Diagnostics
|
||||||
|
|
||||||
|
Commands return errors instead of printing inside deep logic. The root command
|
||||||
|
silences Cobra usage/error output, and `cmd/seriatim/main.go` prints one error
|
||||||
|
to stderr and exits with status `1`.
|
||||||
|
|
||||||
|
Validation failures should fail fast with contextual errors. Correctable
|
||||||
|
conditions should be deterministic and, where reports are requested, reflected
|
||||||
|
as report events. Optional reports contain metadata and ordered events; they are
|
||||||
|
not required for command success unless the report file itself cannot be
|
||||||
|
written.
|
||||||
|
|
||||||
|
The implemented code does not use a logging subsystem. Diagnostics are returned
|
||||||
|
as errors or written to optional report JSON. Normalize report events avoid
|
||||||
|
embedding transcript text; keep that privacy-oriented behavior when changing
|
||||||
|
normalize diagnostics.
|
||||||
|
|
||||||
|
## Testing Expectations
|
||||||
|
|
||||||
|
`go test ./...` is the repository-wide check. There is currently no Makefile,
|
||||||
|
taskfile, linter config, or dedicated documentation check.
|
||||||
|
|
||||||
|
When changing config or CLI behavior, inspect `internal/config` and
|
||||||
|
`internal/cli` tests. When changing pipeline composition or stage contracts,
|
||||||
|
inspect `internal/pipeline` and `internal/builtin` tests. When changing
|
||||||
|
correction or annotation modules, inspect the package tests for overlap,
|
||||||
|
coalesce, danglers, backchannel, filler, and autocorrect behavior.
|
||||||
|
|
||||||
|
When changing artifact-level commands, inspect `internal/trim`,
|
||||||
|
`internal/normalize`, and their CLI tests. When changing public output shape or
|
||||||
|
schema validation, inspect `schema` and `internal/artifact` tests. Report and
|
||||||
|
diagnostic changes should be covered through the command or package tests that
|
||||||
|
emit the affected events.
|
||||||
|
|
||||||
|
## Dependency Policy
|
||||||
|
|
||||||
|
Prefer the Go standard library for parsing, data transformation, concurrency
|
||||||
|
primitives, filesystem work, and testing wherever it is reasonable.
|
||||||
|
|
||||||
|
Third-party dependencies must be narrow, justified, and preferably de facto
|
||||||
|
standard for their purpose. Existing examples include Cobra for CLI structure,
|
||||||
|
`gopkg.in/yaml.v3` for YAML files, and `jsonschema/v6` for validating embedded
|
||||||
|
public JSON schemas. Avoid broad framework dependencies for behavior that is
|
||||||
|
already simple and local.
|
||||||
|
|
||||||
|
## Documentation Expectations
|
||||||
|
|
||||||
|
Architecture docs must stay aligned with [documentation policy](documentation.md).
|
||||||
|
Current-behavior docs must not become aspirational. If code and docs disagree,
|
||||||
|
fix the inaccurate current-behavior doc or put planned work under
|
||||||
|
`docs/roadmap/`.
|
||||||
|
|
||||||
|
Prefer links to canonical docs instead of repeating full CLI, config, schema, or
|
||||||
|
operations reference material. Keep examples real, tested where practical, and
|
||||||
|
free of secrets or private transcript data.
|
||||||
|
|
||||||
|
## Architectural Invariants
|
||||||
|
|
||||||
|
- Keep core/domain logic separate from CLI, config, filesystem, reporting, and
|
||||||
|
other adapter concerns.
|
||||||
|
- Centralize default configuration values as constants defined in internal/config/config.go.
|
||||||
|
- Keep modules narrowly scoped, explicitly configured, and composable by
|
||||||
|
registry name.
|
||||||
|
- Preserve deterministic ordering, final segment ID assignment, and schema
|
||||||
|
validation before output acceptance.
|
||||||
|
- Keep `trim` and `normalize` artifact-level; do not run merge modules from
|
||||||
|
those commands.
|
||||||
|
- Keep public output schemas validated through `schema`.
|
||||||
|
- Keep optional reports ordered, concise, and diagnostic.
|
||||||
|
- Avoid broad dependencies without a concrete maintainability benefit.
|
||||||
|
- Do not document unimplemented behavior outside `docs/roadmap/`.
|
||||||
|
|
||||||
|
## Non-Goals
|
||||||
|
|
||||||
|
The implemented application does not perform transcription, audio diarization,
|
||||||
|
speaker inference from audio or text, summarization, daemon operation, remote
|
||||||
|
storage, dynamic external plugin loading, or concurrent pipeline execution.
|
||||||
|
|
||||||
|
The architecture policy is not a package-by-package reference, CLI manual,
|
||||||
|
config reference, schema reference, or roadmap.
|
||||||
111
docs/policy/development.md
Normal file
111
docs/policy/development.md
Normal file
@@ -0,0 +1,111 @@
|
|||||||
|
# Development Policy
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
This document defines contributor workflow for maintainers and coding agents.
|
||||||
|
It complements [architecture policy](architecture.md) and
|
||||||
|
[documentation policy](documentation.md).
|
||||||
|
|
||||||
|
## Repository layout
|
||||||
|
|
||||||
|
- `cmd/seriatim/`: process entrypoint.
|
||||||
|
- `internal/cli/`: Cobra commands and flag wiring.
|
||||||
|
- `internal/config/`: option normalization and validation.
|
||||||
|
- `internal/pipeline/`: orchestration interfaces, registry, runner.
|
||||||
|
- `internal/builtin/`: implemented input/pre/post/output modules and merger.
|
||||||
|
- `internal/artifact/`: conversion from internal merged model to public shapes.
|
||||||
|
- `internal/trim/`: artifact-level trim logic.
|
||||||
|
- `internal/normalize/`: artifact-level normalize parsing/building.
|
||||||
|
- `internal/*` domain packages: overlap, coalesce, danglers, filler,
|
||||||
|
backchannel, speaker, autocorrect, report, model.
|
||||||
|
- `schema/`: public structs plus embedded JSON Schemas and validation.
|
||||||
|
- `docs/`: policy, user docs, roadmap, and internal docs.
|
||||||
|
|
||||||
|
## Local checks
|
||||||
|
|
||||||
|
Primary repository check:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go test ./...
|
||||||
|
```
|
||||||
|
|
||||||
|
Useful manual checks for CLI-facing changes:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/seriatim --help
|
||||||
|
go run ./cmd/seriatim merge --help
|
||||||
|
go run ./cmd/seriatim trim --help
|
||||||
|
go run ./cmd/seriatim normalize --help
|
||||||
|
```
|
||||||
|
|
||||||
|
Current toolchain note:
|
||||||
|
|
||||||
|
- There is no Makefile.
|
||||||
|
- There is no taskfile.
|
||||||
|
- There is no committed linter configuration.
|
||||||
|
- There is no automated documentation checker.
|
||||||
|
|
||||||
|
## Coding conventions
|
||||||
|
|
||||||
|
- Keep core behavior deterministic for identical inputs/config/version.
|
||||||
|
- Keep CLI command functions thin: parse flags, construct config, delegate.
|
||||||
|
- Keep validation in `internal/config` and package-specific validators.
|
||||||
|
- Return errors from deep logic; do not print inside internal packages.
|
||||||
|
- Preserve clear package boundaries between adapters and domain transforms.
|
||||||
|
- Define configuration defaults as constants in internal/config/config.go.
|
||||||
|
|
||||||
|
## Dependency policy
|
||||||
|
|
||||||
|
Prefer the Go standard library first.
|
||||||
|
|
||||||
|
Third-party dependencies should stay narrow and justified. Current direct
|
||||||
|
runtime dependencies are:
|
||||||
|
|
||||||
|
- `github.com/spf13/cobra` for CLI structure.
|
||||||
|
- `gopkg.in/yaml.v3` for YAML rule files.
|
||||||
|
- `github.com/santhosh-tekuri/jsonschema/v6` for public schema validation.
|
||||||
|
|
||||||
|
## Adding CLI flags
|
||||||
|
|
||||||
|
1. Add the flag in the relevant `internal/cli/*.go` command.
|
||||||
|
2. Thread the raw value through `config.*Options`.
|
||||||
|
3. Add normalization/validation in `internal/config/config.go`.
|
||||||
|
4. Update or add CLI/config tests.
|
||||||
|
5. Update canonical docs (`docs/cli.md`, `docs/config.md`) if user-visible.
|
||||||
|
|
||||||
|
## Adding config fields or environment variables
|
||||||
|
|
||||||
|
1. Add field(s) to the relevant config struct(s).
|
||||||
|
2. Parse and validate in `internal/config/config.go`.
|
||||||
|
3. Add tests in `internal/config/config_test.go`.
|
||||||
|
4. Thread validated values into consuming modules.
|
||||||
|
5. Update `docs/config.md` and related docs.
|
||||||
|
|
||||||
|
## Adding modules or pipeline behavior
|
||||||
|
|
||||||
|
1. Implement the module in the appropriate package (often `internal/builtin`).
|
||||||
|
2. Expose a stable module name via `Name()`.
|
||||||
|
3. Register it in `internal/builtin/registry.go`.
|
||||||
|
4. Ensure preprocessing modules declare correct `Requires()`/`Produces()`
|
||||||
|
states.
|
||||||
|
5. Add/adjust tests in module packages and `internal/cli/merge_test.go`.
|
||||||
|
6. Document internal behavior changes in `docs/internal/`.
|
||||||
|
|
||||||
|
## Schema and artifact changes
|
||||||
|
|
||||||
|
1. Update public structs and validation logic in `schema/`.
|
||||||
|
2. Update embedded JSON Schema files (`schema/*.schema.json`) if contract
|
||||||
|
changes.
|
||||||
|
3. Update conversion behavior in `internal/artifact`, `internal/trim`, and/or
|
||||||
|
`internal/normalize` as needed.
|
||||||
|
4. Add tests in `schema/`, `internal/artifact/`, `internal/trim/`,
|
||||||
|
`internal/normalize/`, and CLI tests.
|
||||||
|
5. Update user and internal docs that reference output contracts.
|
||||||
|
|
||||||
|
## Documentation expectations
|
||||||
|
|
||||||
|
- Outside `docs/roadmap/`, document only implemented behavior.
|
||||||
|
- Keep canonical homes: CLI in `docs/cli.md`, config in `docs/config.md`,
|
||||||
|
operations in `docs/operations.md`, troubleshooting in
|
||||||
|
`docs/troubleshooting.md`, internals in `docs/internal/`.
|
||||||
|
- When behavior changes, update docs in the same change.
|
||||||
356
docs/policy/documentation.md
Normal file
356
docs/policy/documentation.md
Normal file
@@ -0,0 +1,356 @@
|
|||||||
|
# Go Project Documentation Policy
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
Project documentation must help four audiences:
|
||||||
|
|
||||||
|
1. users who need to run the application;
|
||||||
|
2. administrators/operators who need to configure and operate it;
|
||||||
|
3. developers who need to understand and change it safely;
|
||||||
|
4. LLM coding agents that need clear scope, boundaries, and invariants.
|
||||||
|
|
||||||
|
Docs should be accurate, concise, task-oriented, and organized by audience. Prefer links to canonical docs over repetition.
|
||||||
|
|
||||||
|
## Core Rules
|
||||||
|
|
||||||
|
### 1. Keep docs concise
|
||||||
|
|
||||||
|
Each document should cover a defined scope and only the essentials for that scope.
|
||||||
|
|
||||||
|
Avoid:
|
||||||
|
- long background explanations;
|
||||||
|
- repeated reference material;
|
||||||
|
- implementation detail in user-facing docs;
|
||||||
|
- aspirational language outside roadmap docs;
|
||||||
|
- verbose examples where one minimal example is clearer.
|
||||||
|
|
||||||
|
### 2. Document only implemented behavior outside roadmap files
|
||||||
|
|
||||||
|
Unimplemented, planned, aspirational, experimental, or future work may be described only under:
|
||||||
|
|
||||||
|
- `docs/roadmap/`
|
||||||
|
|
||||||
|
No other documentation file, including `README.md`, should describe code, features, modules, stages, commands, config fields, or behaviors that do not currently exist.
|
||||||
|
|
||||||
|
If a feature is partial, non-roadmap docs may describe only the implemented portion and its current boundary.
|
||||||
|
|
||||||
|
### 3. Use canonical homes
|
||||||
|
|
||||||
|
Each type of information should have one canonical location.
|
||||||
|
|
||||||
|
Canonical homes:
|
||||||
|
|
||||||
|
- project purpose and quickstart: `README.md`
|
||||||
|
- development principles: `docs/policy/architecture.md`
|
||||||
|
- configuration reference: `docs/config.md`
|
||||||
|
- CLI reference: `docs/cli.md`
|
||||||
|
- operations and recovery: `docs/operations.md`
|
||||||
|
- troubleshooting: `docs/troubleshooting.md`
|
||||||
|
- implemented internals: `docs/internal/`
|
||||||
|
- future work: `docs/roadmap/`
|
||||||
|
- contributor workflow: `docs/policy/development.md`
|
||||||
|
- copyable examples: `examples/`
|
||||||
|
|
||||||
|
Other files should summarize briefly and link to the canonical source.
|
||||||
|
|
||||||
|
### 4. Keep examples real
|
||||||
|
|
||||||
|
Examples should be valid, maintained, and free of secrets.
|
||||||
|
|
||||||
|
Where practical:
|
||||||
|
- example configs should load successfully;
|
||||||
|
- example commands should match real CLI syntax;
|
||||||
|
- important examples should be covered by tests.
|
||||||
|
|
||||||
|
## Documentation Profiles
|
||||||
|
|
||||||
|
All projects require:
|
||||||
|
|
||||||
|
- `README.md`
|
||||||
|
- `docs/policy/architecture.md`
|
||||||
|
|
||||||
|
Additional docs depend on the project.
|
||||||
|
|
||||||
|
### Small library
|
||||||
|
|
||||||
|
Recommended:
|
||||||
|
- `docs/policy/development.md`, if contributor conventions are non-obvious
|
||||||
|
|
||||||
|
### Simple CLI
|
||||||
|
|
||||||
|
Required:
|
||||||
|
- `docs/cli.md`
|
||||||
|
|
||||||
|
Recommended:
|
||||||
|
- `docs/policy/development.md`
|
||||||
|
|
||||||
|
### Config-driven CLI
|
||||||
|
|
||||||
|
Required:
|
||||||
|
- `docs/cli.md`
|
||||||
|
- `docs/config.md`
|
||||||
|
|
||||||
|
Recommended:
|
||||||
|
- `examples/`
|
||||||
|
- `docs/policy/development.md`
|
||||||
|
|
||||||
|
### Stateful or operator-facing application
|
||||||
|
|
||||||
|
Required:
|
||||||
|
- `docs/cli.md`, if CLI-based
|
||||||
|
- `docs/config.md`, if config-driven
|
||||||
|
- `docs/operations.md`
|
||||||
|
|
||||||
|
Recommended:
|
||||||
|
- `docs/troubleshooting.md`
|
||||||
|
- `examples/`
|
||||||
|
- `docs/policy/development.md`
|
||||||
|
|
||||||
|
### Modular, staged, service-oriented, or orchestration application
|
||||||
|
|
||||||
|
Required:
|
||||||
|
- `docs/cli.md`, if CLI-based
|
||||||
|
- `docs/config.md`, if config-driven
|
||||||
|
- `docs/operations.md`
|
||||||
|
- `docs/internal/`
|
||||||
|
- `docs/policy/development.md`
|
||||||
|
|
||||||
|
Recommended:
|
||||||
|
- `docs/troubleshooting.md`
|
||||||
|
- validated examples under `examples/`
|
||||||
|
|
||||||
|
## Required Documents
|
||||||
|
|
||||||
|
### README.md
|
||||||
|
|
||||||
|
**Audience:** users, administrators, operators
|
||||||
|
|
||||||
|
The README is the outward-facing project orientation page.
|
||||||
|
|
||||||
|
It should include, in order:
|
||||||
|
|
||||||
|
1. concise description;
|
||||||
|
2. elevator pitch;
|
||||||
|
3. shortest useful command or usage example;
|
||||||
|
4. links to targeted docs.
|
||||||
|
|
||||||
|
The README should be short. It is not a manual.
|
||||||
|
|
||||||
|
The “shortest useful command” means the simplest command that performs the project’s core use case. (It does not mean `app --help`.)
|
||||||
|
|
||||||
|
### docs/policy/architecture.md
|
||||||
|
|
||||||
|
**Audience:** developers, LLM coding agents
|
||||||
|
|
||||||
|
`docs/policy/architecture.md` is required for every project.
|
||||||
|
|
||||||
|
It is an inward-facing development policy document. It should describe how the project is intended to be built and changed.
|
||||||
|
|
||||||
|
It should include:
|
||||||
|
|
||||||
|
- project shape;
|
||||||
|
- core design principles;
|
||||||
|
- package and boundary philosophy;
|
||||||
|
- state/persistence philosophy, if applicable;
|
||||||
|
- external integration philosophy, if applicable;
|
||||||
|
- error-handling and logging principles;
|
||||||
|
- testing expectations;
|
||||||
|
- documentation expectations;
|
||||||
|
- architectural invariants;
|
||||||
|
- explicit non-goals, if useful.
|
||||||
|
|
||||||
|
For small projects, this file may be brief. It may simply state that the project is intentionally narrow, monolithic, and dependency-light.
|
||||||
|
|
||||||
|
### docs/policy/development.md
|
||||||
|
|
||||||
|
**Audience:** developers, LLM coding agents
|
||||||
|
|
||||||
|
Required for projects maintained by humans and LLM coding agents.
|
||||||
|
|
||||||
|
It should include:
|
||||||
|
|
||||||
|
- repository layout;
|
||||||
|
- build/test commands;
|
||||||
|
- coding conventions;
|
||||||
|
- dependency policy;
|
||||||
|
- how to add config fields;
|
||||||
|
- how to add CLI flags;
|
||||||
|
- how to add stages/modules/adapters, if applicable;
|
||||||
|
- how to update examples;
|
||||||
|
- documentation update expectations.
|
||||||
|
|
||||||
|
### docs/config.md
|
||||||
|
|
||||||
|
**Audience:** administrators, operators, advanced users
|
||||||
|
|
||||||
|
Required for applications with configuration files.
|
||||||
|
|
||||||
|
It should include, in order:
|
||||||
|
|
||||||
|
1. config file locations and discovery precedence;
|
||||||
|
2. minimal working config;
|
||||||
|
3. production-oriented config;
|
||||||
|
4. full configuration reference;
|
||||||
|
5. secrets handling, if applicable;
|
||||||
|
6. links to maintained examples.
|
||||||
|
|
||||||
|
The full configuration reference should be canonical.
|
||||||
|
|
||||||
|
### docs/cli.md
|
||||||
|
|
||||||
|
**Audience:** users, administrators, operators
|
||||||
|
|
||||||
|
Required for CLI applications.
|
||||||
|
|
||||||
|
It should include, in order:
|
||||||
|
|
||||||
|
1. shortest useful command;
|
||||||
|
2. command overview;
|
||||||
|
3. complete flag reference;
|
||||||
|
4. common workflows;
|
||||||
|
5. diagnostic or recovery commands, if applicable.
|
||||||
|
|
||||||
|
Explain when commands are useful, not just their syntax.
|
||||||
|
|
||||||
|
### docs/operations.md
|
||||||
|
|
||||||
|
**Audience:** administrators, operators
|
||||||
|
|
||||||
|
Required for applications that maintain state, support resume behavior, run multiple stages, write durable artifacts, use remote storage, or require recovery procedures.
|
||||||
|
|
||||||
|
It should cover:
|
||||||
|
|
||||||
|
- normal workflow;
|
||||||
|
- filesystem layout;
|
||||||
|
- remote storage layout, if applicable;
|
||||||
|
- logs and manifests;
|
||||||
|
- resume/retry behavior;
|
||||||
|
- cleanup behavior;
|
||||||
|
- archive/backup behavior;
|
||||||
|
- safe recovery procedures;
|
||||||
|
- operational caveats.
|
||||||
|
|
||||||
|
### docs/troubleshooting.md
|
||||||
|
|
||||||
|
**Audience:** administrators, operators
|
||||||
|
|
||||||
|
Recommended once recurring failure modes exist.
|
||||||
|
|
||||||
|
Each entry should include:
|
||||||
|
|
||||||
|
- symptom;
|
||||||
|
- likely cause;
|
||||||
|
- diagnostic command or inspection step;
|
||||||
|
- safe fix;
|
||||||
|
- relevant links.
|
||||||
|
|
||||||
|
### docs/internal/
|
||||||
|
|
||||||
|
**Audience:** developers, LLM coding agents
|
||||||
|
|
||||||
|
Required for modular, staged, service-oriented, or orchestration projects.
|
||||||
|
|
||||||
|
This directory describes implemented internal components. It is not the roadmap.
|
||||||
|
|
||||||
|
Use one file per major component where useful.
|
||||||
|
|
||||||
|
Each component doc should include:
|
||||||
|
|
||||||
|
1. purpose;
|
||||||
|
2. inputs and outputs;
|
||||||
|
3. boundaries;
|
||||||
|
4. config fields used;
|
||||||
|
5. external adapters used;
|
||||||
|
6. state or manifest behavior, if applicable;
|
||||||
|
7. skip/resume behavior, if applicable;
|
||||||
|
8. failure behavior;
|
||||||
|
9. tests to inspect before changing;
|
||||||
|
10. architectural invariants.
|
||||||
|
|
||||||
|
### docs/roadmap/
|
||||||
|
|
||||||
|
**Audience:** maintainers, developers, LLM coding agents
|
||||||
|
|
||||||
|
This is the only place for planned, future, aspirational, experimental, or unimplemented work.
|
||||||
|
|
||||||
|
Roadmap docs should clearly distinguish:
|
||||||
|
|
||||||
|
- proposed work;
|
||||||
|
- accepted plans;
|
||||||
|
- deferred ideas;
|
||||||
|
- rejected ideas;
|
||||||
|
- implementation prompts or task breakdowns, if useful.
|
||||||
|
|
||||||
|
Roadmap docs should not be confused with current behavior.
|
||||||
|
|
||||||
|
### docs/integrations/
|
||||||
|
|
||||||
|
**Audience:** developers, LLM coding agents
|
||||||
|
|
||||||
|
Required for projects that depend on external CLIs, APIs, services, protocols, or file formats where the integration contract is important to maintain.
|
||||||
|
|
||||||
|
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses.
|
||||||
|
|
||||||
|
Use one file per integration where useful.
|
||||||
|
|
||||||
|
## Examples Directory
|
||||||
|
|
||||||
|
Projects with non-trivial configuration or workflows should include `examples/`.
|
||||||
|
|
||||||
|
Useful examples include:
|
||||||
|
|
||||||
|
- minimal working config;
|
||||||
|
- production-oriented config;
|
||||||
|
- full annotated config;
|
||||||
|
- local development config;
|
||||||
|
- remote/object-storage config;
|
||||||
|
- minimal session/input file.
|
||||||
|
|
||||||
|
Examples should be valid, maintained, tested when practical, and linked from relevant docs.
|
||||||
|
|
||||||
|
## Security and Privacy
|
||||||
|
|
||||||
|
Docs and examples must not include:
|
||||||
|
|
||||||
|
- real API keys;
|
||||||
|
- tokens;
|
||||||
|
- passwords;
|
||||||
|
- private keys;
|
||||||
|
- private environment dumps;
|
||||||
|
- sensitive user data;
|
||||||
|
- raw private transcripts;
|
||||||
|
- private infrastructure details unless intentionally public.
|
||||||
|
|
||||||
|
Document secret-handling mechanisms, not actual secret values.
|
||||||
|
|
||||||
|
## Maintenance Rules
|
||||||
|
|
||||||
|
When docs change, verify the affected behavior.
|
||||||
|
|
||||||
|
Where practical:
|
||||||
|
|
||||||
|
- load example config files in tests;
|
||||||
|
- test CLI examples or command parser behavior;
|
||||||
|
- validate documented flags against real flags;
|
||||||
|
- remove stale references;
|
||||||
|
- update links after renames;
|
||||||
|
- keep roadmap content out of non-roadmap docs.
|
||||||
|
|
||||||
|
If documentation and code disagree, fix the documentation and/or open a roadmap item; do not leave aspirational behavior in current-behavior docs.
|
||||||
|
|
||||||
|
Documentation is complete only when it matches the current code.
|
||||||
|
|
||||||
|
## Documentation Change Checklist
|
||||||
|
|
||||||
|
Before merging documentation changes, verify:
|
||||||
|
|
||||||
|
- README is concise and orientation-focused.
|
||||||
|
- `docs/policy/architecture.md` describes development principles.
|
||||||
|
- Future work appears only under `docs/roadmap/`.
|
||||||
|
- User-facing docs avoid unnecessary internals.
|
||||||
|
- Developer-facing docs preserve boundaries and invariants.
|
||||||
|
- Config examples match the schema.
|
||||||
|
- CLI examples match real commands and flags.
|
||||||
|
- Defaults appear in the canonical config reference.
|
||||||
|
- No secrets or private data are included.
|
||||||
|
- Links are accurate.
|
||||||
102
docs/troubleshooting.md
Normal file
102
docs/troubleshooting.md
Normal file
@@ -0,0 +1,102 @@
|
|||||||
|
# Troubleshooting
|
||||||
|
|
||||||
|
Each entry includes symptom, likely cause, inspection step, and safe fix.
|
||||||
|
|
||||||
|
## Missing required flags
|
||||||
|
|
||||||
|
- Symptom: command fails with messages like `--input-file is required`, `--output-file is required`, or `exactly one of --keep or --remove is required`.
|
||||||
|
- Likely cause: required command flags were omitted.
|
||||||
|
- Inspection: run command help for the failing command:
|
||||||
|
- `go run ./cmd/seriatim merge --help`
|
||||||
|
- `go run ./cmd/seriatim trim --help`
|
||||||
|
- `go run ./cmd/seriatim normalize --help`
|
||||||
|
- Safe fix: provide all required flags; for `trim`, provide exactly one selector mode (`--keep` or `--remove`).
|
||||||
|
|
||||||
|
## Invalid output or report path
|
||||||
|
|
||||||
|
- Symptom: errors like `--output-file parent directory ...` or `--report-file parent directory ...`.
|
||||||
|
- Likely cause: parent directory does not exist, is not a directory, or path points to an unusable target.
|
||||||
|
- Inspection: verify paths:
|
||||||
|
- `dirname <path>`
|
||||||
|
- `ls -ld <parent-dir>`
|
||||||
|
- Safe fix: create/fix the parent directory and rerun; avoid using directory paths directly as output/report file targets.
|
||||||
|
|
||||||
|
## Invalid merge input JSON
|
||||||
|
|
||||||
|
- Symptom: merge fails with messages like `parse input file`, `must contain top-level segments array`, `segment 0 missing numeric start`, or `segment 0 words must be an array`.
|
||||||
|
- Likely cause: malformed JSON or unsupported/missing fields in a merge input file.
|
||||||
|
- Inspection: validate input JSON and required fields (`start`, `end`, `text`):
|
||||||
|
- `jq . <input-file>`
|
||||||
|
- Safe fix: correct the JSON structure and segment/word field types, then rerun `merge`.
|
||||||
|
|
||||||
|
## Invalid normalize input shape
|
||||||
|
|
||||||
|
- Symptom: normalize fails with messages like `must contain a "segments" field`, `"segments" must be an array`, or `top-level object with "segments" or a top-level segment array`.
|
||||||
|
- Likely cause: normalize input is neither supported object-with-segments nor top-level segment array.
|
||||||
|
- Inspection: inspect top-level JSON shape:
|
||||||
|
- `jq 'type' <input-file>`
|
||||||
|
- `jq 'keys' <input-file>` (for object input)
|
||||||
|
- Safe fix: reshape input into one supported form and rerun `normalize`.
|
||||||
|
|
||||||
|
## Invalid speaker map or autocorrect YAML
|
||||||
|
|
||||||
|
- Symptom: merge fails with errors such as `must contain at least one match rule`, `must include speaker`, `must include target`, or duplicate match/speaker validation failures.
|
||||||
|
- Likely cause: YAML rule file structure/content does not match expected schema.
|
||||||
|
- Inspection: check YAML validity and required top-level keys:
|
||||||
|
- `speakers.yml` requires top-level `match` rules.
|
||||||
|
- `autocorrect.yml` requires top-level `autocorrect` rules.
|
||||||
|
- Safe fix: correct YAML structure and rule content, then rerun `merge`.
|
||||||
|
|
||||||
|
## Unknown module names
|
||||||
|
|
||||||
|
- Symptom: errors like `unknown input reader`, `unknown preprocessing module`, `unknown postprocessing module`, or `unknown output module`.
|
||||||
|
- Likely cause: module name typo or unsupported module in flag lists.
|
||||||
|
- Inspection: compare provided module names against defaults in CLI help and config docs.
|
||||||
|
- Safe fix: use implemented module names only or remove unsupported modules from comma-separated lists.
|
||||||
|
|
||||||
|
## Invalid output schema value
|
||||||
|
|
||||||
|
- Symptom: errors like `--output-schema must be one of ...`.
|
||||||
|
- Likely cause: unsupported schema value from flag or `SERIATIM_OUTPUT_SCHEMA`.
|
||||||
|
- Inspection: check effective value:
|
||||||
|
- command flags
|
||||||
|
- `echo "$SERIATIM_OUTPUT_SCHEMA"`
|
||||||
|
- Safe fix: use one of `seriatim-minimal`, `seriatim-intermediate`, or `seriatim-full`.
|
||||||
|
|
||||||
|
## Invalid trim selector
|
||||||
|
|
||||||
|
- Symptom: trim fails with messages like `invalid selector ... malformed element`, `segment ID must be positive`, or descending-range errors.
|
||||||
|
- Likely cause: selector syntax is invalid.
|
||||||
|
- Inspection: verify selector format:
|
||||||
|
- single ID: `7`
|
||||||
|
- range: `1-10`
|
||||||
|
- list: `1-10,15,20-25`
|
||||||
|
- Safe fix: correct selector syntax and rerun `trim`.
|
||||||
|
|
||||||
|
## Schema validation failures
|
||||||
|
|
||||||
|
- Symptom: errors such as `validate-output: ...` in merge or `input JSON is not a valid seriatim output artifact` in trim.
|
||||||
|
- Likely cause:
|
||||||
|
- merge module order/config produced invalid final artifact (for example, validating before IDs are assigned), or
|
||||||
|
- trim input is not a valid seriatim artifact.
|
||||||
|
- Inspection:
|
||||||
|
- for merge: inspect customized module ordering flags.
|
||||||
|
- for trim: verify input artifact against known seriatim schema files in `schema/`.
|
||||||
|
- Safe fix:
|
||||||
|
- restore valid merge postprocessing order ending with assigned IDs before validation, or
|
||||||
|
- provide a valid seriatim artifact as trim input.
|
||||||
|
|
||||||
|
## Report write failure
|
||||||
|
|
||||||
|
- Symptom: errors like `write --report-file ...` or file-create failures when report writing is requested.
|
||||||
|
- Likely cause: report path is not writable or is an invalid target (for example a directory path).
|
||||||
|
- Inspection:
|
||||||
|
- `ls -ld <report-parent-dir>`
|
||||||
|
- verify `--report-file` is a file path, not a directory
|
||||||
|
- Safe fix: choose a writable file path under an existing directory and rerun.
|
||||||
|
|
||||||
|
## Related docs
|
||||||
|
|
||||||
|
- CLI reference: [cli.md](cli.md)
|
||||||
|
- Configuration reference: [config.md](config.md)
|
||||||
|
- Operations guide: [operations.md](operations.md)
|
||||||
61
examples/README.md
Normal file
61
examples/README.md
Normal file
@@ -0,0 +1,61 @@
|
|||||||
|
# Examples
|
||||||
|
|
||||||
|
These are small synthetic, copyable example assets for the implemented CLI
|
||||||
|
commands.
|
||||||
|
This directory is the canonical examples home for documentation.
|
||||||
|
|
||||||
|
## Merge example
|
||||||
|
|
||||||
|
Inputs:
|
||||||
|
|
||||||
|
- `minimal-merge/input-alice.json`
|
||||||
|
- `minimal-merge/input-bob.json`
|
||||||
|
- `minimal-merge/speakers.yml`
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/seriatim merge \
|
||||||
|
--input-file examples/minimal-merge/input-alice.json \
|
||||||
|
--input-file examples/minimal-merge/input-bob.json \
|
||||||
|
--speakers examples/minimal-merge/speakers.yml \
|
||||||
|
--output-file /tmp/seriatim-example-merge.json
|
||||||
|
```
|
||||||
|
|
||||||
|
## Normalize examples
|
||||||
|
|
||||||
|
Object-with-segments input:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/seriatim normalize \
|
||||||
|
--input-file examples/normalize/object-with-segments.json \
|
||||||
|
--output-file /tmp/seriatim-example-normalize-object.json
|
||||||
|
```
|
||||||
|
|
||||||
|
Bare-array input:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/seriatim normalize \
|
||||||
|
--input-file examples/normalize/bare-segments-array.json \
|
||||||
|
--output-file /tmp/seriatim-example-normalize-array.json
|
||||||
|
```
|
||||||
|
|
||||||
|
## Trim example
|
||||||
|
|
||||||
|
Input artifact:
|
||||||
|
|
||||||
|
- `trim/input-full.json`
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/seriatim trim \
|
||||||
|
--input-file examples/trim/input-full.json \
|
||||||
|
--output-file /tmp/seriatim-example-trim.json \
|
||||||
|
--keep "1-2"
|
||||||
|
```
|
||||||
|
|
||||||
|
## YAML rule examples
|
||||||
|
|
||||||
|
- `speakers.yml`
|
||||||
|
- `autocorrect.yml`
|
||||||
8
examples/autocorrect.yml
Normal file
8
examples/autocorrect.yml
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
autocorrect:
|
||||||
|
- target: "General Kenobi"
|
||||||
|
match:
|
||||||
|
- "General Kenobi."
|
||||||
|
|
||||||
|
- target: "Okay"
|
||||||
|
match:
|
||||||
|
- "Okay."
|
||||||
14
examples/minimal-merge/input-alice.json
Normal file
14
examples/minimal-merge/input-alice.json
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
{
|
||||||
|
"segments": [
|
||||||
|
{
|
||||||
|
"start": 0.0,
|
||||||
|
"end": 1.2,
|
||||||
|
"text": " Hello there. "
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"start": 2.6,
|
||||||
|
"end": 3.1,
|
||||||
|
"text": "Okay."
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
9
examples/minimal-merge/input-bob.json
Normal file
9
examples/minimal-merge/input-bob.json
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
{
|
||||||
|
"segments": [
|
||||||
|
{
|
||||||
|
"start": 1.3,
|
||||||
|
"end": 2.4,
|
||||||
|
"text": "General Kenobi."
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
8
examples/minimal-merge/speakers.yml
Normal file
8
examples/minimal-merge/speakers.yml
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
match:
|
||||||
|
- speaker: "Alice Example"
|
||||||
|
match:
|
||||||
|
- "alice"
|
||||||
|
|
||||||
|
- speaker: "Bob Example"
|
||||||
|
match:
|
||||||
|
- "bob"
|
||||||
13
examples/normalize/bare-segments-array.json
Normal file
13
examples/normalize/bare-segments-array.json
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"start": 2.5,
|
||||||
|
"end": 3.0,
|
||||||
|
"speaker": "Bob",
|
||||||
|
"text": "later"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"end": 2.0,
|
||||||
|
"speaker": "",
|
||||||
|
"text": "no start uses end"
|
||||||
|
}
|
||||||
|
]
|
||||||
19
examples/normalize/object-with-segments.json
Normal file
19
examples/normalize/object-with-segments.json
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
{
|
||||||
|
"segments": [
|
||||||
|
{
|
||||||
|
"id": 7,
|
||||||
|
"start": 2.0,
|
||||||
|
"end": 2.5,
|
||||||
|
"speaker": "Bob",
|
||||||
|
"text": "second"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"start": 1.0,
|
||||||
|
"end": 1.3,
|
||||||
|
"speaker": "Alice",
|
||||||
|
"text": "first",
|
||||||
|
"categories": ["backchannel"]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
8
examples/speakers.yml
Normal file
8
examples/speakers.yml
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
match:
|
||||||
|
- speaker: "Alice Example"
|
||||||
|
match:
|
||||||
|
- "alice"
|
||||||
|
|
||||||
|
- speaker: "Bob Example"
|
||||||
|
match:
|
||||||
|
- "bob"
|
||||||
64
examples/trim/input-full.json
Normal file
64
examples/trim/input-full.json
Normal file
@@ -0,0 +1,64 @@
|
|||||||
|
{
|
||||||
|
"metadata": {
|
||||||
|
"application": "seriatim",
|
||||||
|
"version": "dev",
|
||||||
|
"input_reader": "json-files",
|
||||||
|
"input_files": [
|
||||||
|
"examples/minimal-merge/input-alice.json",
|
||||||
|
"examples/minimal-merge/input-bob.json"
|
||||||
|
],
|
||||||
|
"preprocessing_modules": [
|
||||||
|
"validate-raw",
|
||||||
|
"normalize-speakers",
|
||||||
|
"trim-text"
|
||||||
|
],
|
||||||
|
"postprocessing_modules": [
|
||||||
|
"detect-overlaps",
|
||||||
|
"resolve-overlaps",
|
||||||
|
"backchannel",
|
||||||
|
"filler",
|
||||||
|
"resolve-danglers",
|
||||||
|
"coalesce",
|
||||||
|
"detect-overlaps",
|
||||||
|
"autocorrect",
|
||||||
|
"assign-ids",
|
||||||
|
"validate-output"
|
||||||
|
],
|
||||||
|
"output_modules": [
|
||||||
|
"json"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"segments": [
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"source": "examples/minimal-merge/input-alice.json",
|
||||||
|
"source_segment_index": 0,
|
||||||
|
"speaker": "Alice Example",
|
||||||
|
"start": 0,
|
||||||
|
"end": 1.2,
|
||||||
|
"text": "Hello there."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 2,
|
||||||
|
"source": "examples/minimal-merge/input-bob.json",
|
||||||
|
"source_segment_index": 0,
|
||||||
|
"speaker": "Bob Example",
|
||||||
|
"start": 1.3,
|
||||||
|
"end": 2.4,
|
||||||
|
"text": "General Kenobi."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 3,
|
||||||
|
"source": "examples/minimal-merge/input-alice.json",
|
||||||
|
"source_segment_index": 1,
|
||||||
|
"speaker": "Alice Example",
|
||||||
|
"start": 2.6,
|
||||||
|
"end": 3.1,
|
||||||
|
"text": "Okay.",
|
||||||
|
"categories": [
|
||||||
|
"backchannel"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"overlap_groups": []
|
||||||
|
}
|
||||||
@@ -57,12 +57,12 @@ func FromMerged(cfg config.Config, merged model.MergedTranscript) schema.Transcr
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// DefaultFromMerged converts the internal merged transcript model into the
|
// IntermediateFromMerged converts the internal merged transcript model into the
|
||||||
// compact default public serialized output contract.
|
// compact intermediate public serialized output contract.
|
||||||
func DefaultFromMerged(cfg config.Config, merged model.MergedTranscript) schema.DefaultTranscript {
|
func IntermediateFromMerged(cfg config.Config, merged model.MergedTranscript) schema.IntermediateTranscript {
|
||||||
segments := make([]schema.DefaultSegment, len(merged.Segments))
|
segments := make([]schema.IntermediateSegment, len(merged.Segments))
|
||||||
for index, segment := range merged.Segments {
|
for index, segment := range merged.Segments {
|
||||||
segments[index] = schema.DefaultSegment{
|
segments[index] = schema.IntermediateSegment{
|
||||||
ID: segment.ID,
|
ID: segment.ID,
|
||||||
Start: segment.Start,
|
Start: segment.Start,
|
||||||
End: segment.End,
|
End: segment.End,
|
||||||
@@ -72,11 +72,11 @@ func DefaultFromMerged(cfg config.Config, merged model.MergedTranscript) schema.
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return schema.DefaultTranscript{
|
return schema.IntermediateTranscript{
|
||||||
Metadata: schema.DefaultMetadata{
|
Metadata: schema.IntermediateMetadata{
|
||||||
Application: ApplicationName,
|
Application: ApplicationName,
|
||||||
Version: buildinfo.Version,
|
Version: buildinfo.Version,
|
||||||
OutputSchema: config.OutputSchemaDefault,
|
OutputSchema: config.OutputSchemaIntermediate,
|
||||||
},
|
},
|
||||||
Segments: segments,
|
Segments: segments,
|
||||||
}
|
}
|
||||||
@@ -110,14 +110,14 @@ func MinimalFromMerged(cfg config.Config, merged model.MergedTranscript) schema.
|
|||||||
// runtime-selected public output contract.
|
// runtime-selected public output contract.
|
||||||
func SelectedFromMerged(cfg config.Config, merged model.MergedTranscript) any {
|
func SelectedFromMerged(cfg config.Config, merged model.MergedTranscript) any {
|
||||||
switch cfg.OutputSchema {
|
switch cfg.OutputSchema {
|
||||||
case config.OutputSchemaDefault:
|
|
||||||
return DefaultFromMerged(cfg, merged)
|
|
||||||
case config.OutputSchemaMinimal:
|
case config.OutputSchemaMinimal:
|
||||||
return MinimalFromMerged(cfg, merged)
|
return MinimalFromMerged(cfg, merged)
|
||||||
case config.OutputSchemaSeriatim:
|
case config.OutputSchemaFull:
|
||||||
return FromMerged(cfg, merged)
|
return FromMerged(cfg, merged)
|
||||||
|
case config.OutputSchemaIntermediate:
|
||||||
|
return IntermediateFromMerged(cfg, merged)
|
||||||
default:
|
default:
|
||||||
return FromMerged(cfg, merged)
|
return IntermediateFromMerged(cfg, merged)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -23,28 +23,28 @@ func TestFromMergedUsesBuildVersion(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSelectedFromMergedDefaultsToSeriatimTranscript(t *testing.T) {
|
func TestSelectedFromMergedDefaultsToIntermediateTranscript(t *testing.T) {
|
||||||
got := SelectedFromMerged(config.Config{}, model.MergedTranscript{})
|
got := SelectedFromMerged(config.Config{}, model.MergedTranscript{})
|
||||||
|
if _, ok := got.(schema.IntermediateTranscript); !ok {
|
||||||
|
t.Fatalf("selected artifact type = %T, want schema.IntermediateTranscript", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSelectedFromMergedUsesIntermediateWhenConfigured(t *testing.T) {
|
||||||
|
got := SelectedFromMerged(config.Config{OutputSchema: config.OutputSchemaIntermediate}, model.MergedTranscript{})
|
||||||
|
if _, ok := got.(schema.IntermediateTranscript); !ok {
|
||||||
|
t.Fatalf("selected artifact type = %T, want schema.IntermediateTranscript", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSelectedFromMergedUsesFullWhenConfigured(t *testing.T) {
|
||||||
|
got := SelectedFromMerged(config.Config{OutputSchema: config.OutputSchemaFull}, model.MergedTranscript{})
|
||||||
if _, ok := got.(schema.Transcript); !ok {
|
if _, ok := got.(schema.Transcript); !ok {
|
||||||
t.Fatalf("selected artifact type = %T, want schema.Transcript", got)
|
t.Fatalf("selected artifact type = %T, want schema.Transcript", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSelectedFromMergedUsesDefaultWhenConfigured(t *testing.T) {
|
func TestIntermediateFromMergedEmitsOnlyIntermediateShape(t *testing.T) {
|
||||||
got := SelectedFromMerged(config.Config{OutputSchema: config.OutputSchemaDefault}, model.MergedTranscript{})
|
|
||||||
if _, ok := got.(schema.DefaultTranscript); !ok {
|
|
||||||
t.Fatalf("selected artifact type = %T, want schema.DefaultTranscript", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSelectedFromMergedUsesSeriatimWhenConfigured(t *testing.T) {
|
|
||||||
got := SelectedFromMerged(config.Config{OutputSchema: config.OutputSchemaSeriatim}, model.MergedTranscript{})
|
|
||||||
if _, ok := got.(schema.Transcript); !ok {
|
|
||||||
t.Fatalf("selected artifact type = %T, want schema.Transcript", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDefaultFromMergedEmitsOnlyDefaultShape(t *testing.T) {
|
|
||||||
merged := model.MergedTranscript{
|
merged := model.MergedTranscript{
|
||||||
Segments: []model.Segment{
|
Segments: []model.Segment{
|
||||||
{
|
{
|
||||||
@@ -65,19 +65,19 @@ func TestDefaultFromMergedEmitsOnlyDefaultShape(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
got := DefaultFromMerged(config.Config{OutputSchema: config.OutputSchemaDefault}, merged)
|
got := IntermediateFromMerged(config.Config{OutputSchema: config.OutputSchemaIntermediate}, merged)
|
||||||
want := schema.DefaultTranscript{
|
want := schema.IntermediateTranscript{
|
||||||
Metadata: schema.DefaultMetadata{
|
Metadata: schema.IntermediateMetadata{
|
||||||
Application: ApplicationName,
|
Application: ApplicationName,
|
||||||
Version: buildinfo.Version,
|
Version: buildinfo.Version,
|
||||||
OutputSchema: config.OutputSchemaDefault,
|
OutputSchema: config.OutputSchemaIntermediate,
|
||||||
},
|
},
|
||||||
Segments: []schema.DefaultSegment{
|
Segments: []schema.IntermediateSegment{
|
||||||
{ID: 1, Start: 1, End: 2, Speaker: "Alice", Text: "hello", Categories: []string{"backchannel"}},
|
{ID: 1, Start: 1, End: 2, Speaker: "Alice", Text: "hello", Categories: []string{"backchannel"}},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
if !reflect.DeepEqual(got, want) {
|
if !reflect.DeepEqual(got, want) {
|
||||||
t.Fatalf("default transcript = %#v, want %#v", got, want)
|
t.Fatalf("intermediate transcript = %#v, want %#v", got, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2,10 +2,9 @@ package builtin
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
|
||||||
"os"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/seriatim/internal/config"
|
"gitea.maximumdirect.net/eric/seriatim/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/internal/jsonfile"
|
||||||
"gitea.maximumdirect.net/eric/seriatim/internal/report"
|
"gitea.maximumdirect.net/eric/seriatim/internal/report"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -20,15 +19,7 @@ func (jsonOutputWriter) Write(ctx context.Context, out any, rpt report.Report, c
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
file, err := os.Create(cfg.OutputFile)
|
if err := jsonfile.Write(cfg.OutputFile, out); err != nil {
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer file.Close()
|
|
||||||
|
|
||||||
enc := json.NewEncoder(file)
|
|
||||||
enc.SetIndent("", " ")
|
|
||||||
if err := enc.Encode(out); err != nil {
|
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -51,8 +51,8 @@ func (validateOutput) Process(ctx context.Context, in model.MergedTranscript, cf
|
|||||||
selected := artifact.SelectedFromMerged(cfg, in)
|
selected := artifact.SelectedFromMerged(cfg, in)
|
||||||
var err error
|
var err error
|
||||||
switch transcript := selected.(type) {
|
switch transcript := selected.(type) {
|
||||||
case schema.DefaultTranscript:
|
case schema.IntermediateTranscript:
|
||||||
err = schema.ValidateDefaultTranscript(transcript)
|
err = schema.ValidateIntermediateTranscript(transcript)
|
||||||
case schema.MinimalTranscript:
|
case schema.MinimalTranscript:
|
||||||
err = schema.ValidateMinimalTranscript(transcript)
|
err = schema.ValidateMinimalTranscript(transcript)
|
||||||
case schema.Transcript:
|
case schema.Transcript:
|
||||||
|
|||||||
@@ -81,7 +81,7 @@ func TestValidateOutputUsesMinimalSchemaWhenConfigured(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestValidateOutputUsesSeriatimSchemaWhenConfigured(t *testing.T) {
|
func TestValidateOutputUsesFullSchemaWhenConfigured(t *testing.T) {
|
||||||
merged := model.MergedTranscript{
|
merged := model.MergedTranscript{
|
||||||
Segments: []model.Segment{
|
Segments: []model.Segment{
|
||||||
{
|
{
|
||||||
@@ -100,7 +100,7 @@ func TestValidateOutputUsesSeriatimSchemaWhenConfigured(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
cfg := testConfig()
|
cfg := testConfig()
|
||||||
cfg.OutputSchema = config.OutputSchemaSeriatim
|
cfg.OutputSchema = config.OutputSchemaFull
|
||||||
got, events, err := validateOutput{}.Process(context.Background(), merged, cfg)
|
got, events, err := validateOutput{}.Process(context.Background(), merged, cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("validate output: %v", err)
|
t.Fatalf("validate output: %v", err)
|
||||||
|
|||||||
31
internal/cli/flags.go
Normal file
31
internal/cli/flags.go
Normal file
@@ -0,0 +1,31 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func addOutputFileFlag(cmd *cobra.Command, target *string) {
|
||||||
|
cmd.Flags().StringVar(target, "output-file", "", "output transcript JSON file")
|
||||||
|
}
|
||||||
|
|
||||||
|
func addReportFileFlag(cmd *cobra.Command, target *string) {
|
||||||
|
cmd.Flags().StringVar(target, "report-file", "", "optional report JSON file")
|
||||||
|
}
|
||||||
|
|
||||||
|
func addOutputModulesFlag(cmd *cobra.Command, target *string) {
|
||||||
|
cmd.Flags().StringVar(target, "output-modules", config.DefaultOutputModules, "comma-separated output modules")
|
||||||
|
}
|
||||||
|
|
||||||
|
func addMergeOutputSchemaFlag(cmd *cobra.Command, target *string) {
|
||||||
|
cmd.Flags().StringVar(target, "output-schema", config.DefaultOutputSchema, "output JSON schema: seriatim-minimal, seriatim-intermediate (default), or seriatim-full")
|
||||||
|
}
|
||||||
|
|
||||||
|
func addNormalizeOutputSchemaFlag(cmd *cobra.Command, target *string) {
|
||||||
|
cmd.Flags().StringVar(target, "output-schema", config.DefaultOutputSchema, "output JSON schema: seriatim-minimal, seriatim-intermediate, or seriatim-full")
|
||||||
|
}
|
||||||
|
|
||||||
|
func addTrimOutputSchemaFlag(cmd *cobra.Command, target *string) {
|
||||||
|
cmd.Flags().StringVar(target, "output-schema", "", "optional output JSON schema override: seriatim-minimal, seriatim-intermediate, or seriatim-full")
|
||||||
|
}
|
||||||
@@ -15,7 +15,12 @@ func newMergeCommand() *cobra.Command {
|
|||||||
Use: "merge",
|
Use: "merge",
|
||||||
Short: "Run the transcript merge pipeline",
|
Short: "Run the transcript merge pipeline",
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
cfg, err := config.NewMergeConfig(opts)
|
mergeOpts := opts
|
||||||
|
if !cmd.Flags().Changed("output-schema") {
|
||||||
|
mergeOpts.OutputSchema = ""
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg, err := config.NewMergeConfig(mergeOpts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -26,13 +31,13 @@ func newMergeCommand() *cobra.Command {
|
|||||||
|
|
||||||
flags := cmd.Flags()
|
flags := cmd.Flags()
|
||||||
flags.StringArrayVar(&opts.InputFiles, "input-file", nil, "input transcript file; may be repeated")
|
flags.StringArrayVar(&opts.InputFiles, "input-file", nil, "input transcript file; may be repeated")
|
||||||
flags.StringVar(&opts.OutputFile, "output-file", "", "output transcript JSON file")
|
addOutputFileFlag(cmd, &opts.OutputFile)
|
||||||
flags.StringVar(&opts.ReportFile, "report-file", "", "optional report JSON file")
|
addReportFileFlag(cmd, &opts.ReportFile)
|
||||||
flags.StringVar(&opts.SpeakersFile, "speakers", "", "speaker map file")
|
flags.StringVar(&opts.SpeakersFile, "speakers", "", "speaker map file")
|
||||||
flags.StringVar(&opts.AutocorrectFile, "autocorrect", "", "autocorrect rules file")
|
flags.StringVar(&opts.AutocorrectFile, "autocorrect", "", "autocorrect rules file")
|
||||||
flags.StringVar(&opts.InputReader, "input-reader", config.DefaultInputReader, "input reader module")
|
flags.StringVar(&opts.InputReader, "input-reader", config.DefaultInputReader, "input reader module")
|
||||||
flags.StringVar(&opts.OutputModules, "output-modules", config.DefaultOutputModules, "comma-separated output modules")
|
addOutputModulesFlag(cmd, &opts.OutputModules)
|
||||||
flags.StringVar(&opts.OutputSchema, "output-schema", config.DefaultOutputSchema, "output JSON schema: default, minimal, or seriatim")
|
addMergeOutputSchemaFlag(cmd, &opts.OutputSchema)
|
||||||
flags.StringVar(&opts.PreprocessingModules, "preprocessing-modules", config.DefaultPreprocessingModules, "comma-separated preprocessing modules")
|
flags.StringVar(&opts.PreprocessingModules, "preprocessing-modules", config.DefaultPreprocessingModules, "comma-separated preprocessing modules")
|
||||||
flags.StringVar(&opts.PostprocessingModules, "postprocessing-modules", config.DefaultPostprocessingModules, "comma-separated postprocessing modules")
|
flags.StringVar(&opts.PostprocessingModules, "postprocessing-modules", config.DefaultPostprocessingModules, "comma-separated postprocessing modules")
|
||||||
flags.StringVar(&opts.CoalesceGap, "coalesce-gap", config.DefaultCoalesceGapValue, "maximum same-speaker gap in seconds for coalesce")
|
flags.StringVar(&opts.CoalesceGap, "coalesce-gap", config.DefaultCoalesceGapValue, "maximum same-speaker gap in seconds for coalesce")
|
||||||
|
|||||||
@@ -114,7 +114,8 @@ func TestMergeWritesMergedOutputAndReport(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMergeWritesDefaultOutputSchema(t *testing.T) {
|
func TestMergeWritesIntermediateOutputSchema(t *testing.T) {
|
||||||
|
t.Setenv(config.OutputSchemaEnv, "")
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
input := writeJSONFile(t, dir, "input.json", `{
|
input := writeJSONFile(t, dir, "input.json", `{
|
||||||
"segments": [
|
"segments": [
|
||||||
@@ -131,10 +132,10 @@ func TestMergeWritesDefaultOutputSchema(t *testing.T) {
|
|||||||
t.Fatalf("merge failed: %v", err)
|
t.Fatalf("merge failed: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
var transcript schema.DefaultTranscript
|
var transcript schema.IntermediateTranscript
|
||||||
readJSON(t, output, &transcript)
|
readJSON(t, output, &transcript)
|
||||||
if transcript.Metadata.OutputSchema != config.OutputSchemaDefault {
|
if transcript.Metadata.OutputSchema != config.OutputSchemaIntermediate {
|
||||||
t.Fatalf("output_schema = %q, want default", transcript.Metadata.OutputSchema)
|
t.Fatalf("output_schema = %q, want %q", transcript.Metadata.OutputSchema, config.OutputSchemaIntermediate)
|
||||||
}
|
}
|
||||||
if len(transcript.Segments) != 1 {
|
if len(transcript.Segments) != 1 {
|
||||||
t.Fatalf("segment count = %d, want 1", len(transcript.Segments))
|
t.Fatalf("segment count = %d, want 1", len(transcript.Segments))
|
||||||
@@ -149,12 +150,12 @@ func TestMergeWritesDefaultOutputSchema(t *testing.T) {
|
|||||||
}
|
}
|
||||||
for _, forbidden := range []string{"overlap_groups", "source", "derived_from", "words"} {
|
for _, forbidden := range []string{"overlap_groups", "source", "derived_from", "words"} {
|
||||||
if strings.Contains(string(outputBytes), forbidden) {
|
if strings.Contains(string(outputBytes), forbidden) {
|
||||||
t.Fatalf("default output contains %q:\n%s", forbidden, outputBytes)
|
t.Fatalf("intermediate output contains %q:\n%s", forbidden, outputBytes)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMergeWritesSeriatimOutputSchema(t *testing.T) {
|
func TestMergeWritesFullOutputSchema(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
input := writeJSONFile(t, dir, "input.json", `{
|
input := writeJSONFile(t, dir, "input.json", `{
|
||||||
"segments": [
|
"segments": [
|
||||||
@@ -166,7 +167,7 @@ func TestMergeWritesSeriatimOutputSchema(t *testing.T) {
|
|||||||
err := executeMergeRaw(
|
err := executeMergeRaw(
|
||||||
"--input-file", input,
|
"--input-file", input,
|
||||||
"--output-file", output,
|
"--output-file", output,
|
||||||
"--output-schema", "seriatim",
|
"--output-schema", config.OutputSchemaFull,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("merge failed: %v", err)
|
t.Fatalf("merge failed: %v", err)
|
||||||
@@ -203,7 +204,7 @@ func TestMergeWritesMinimalOutputSchema(t *testing.T) {
|
|||||||
err := executeMerge(
|
err := executeMerge(
|
||||||
"--input-file", input,
|
"--input-file", input,
|
||||||
"--output-file", output,
|
"--output-file", output,
|
||||||
"--output-schema", "minimal",
|
"--output-schema", config.OutputSchemaMinimal,
|
||||||
"--report-file", reportPath,
|
"--report-file", reportPath,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -215,8 +216,8 @@ func TestMergeWritesMinimalOutputSchema(t *testing.T) {
|
|||||||
if transcript.Metadata.Application != "seriatim" {
|
if transcript.Metadata.Application != "seriatim" {
|
||||||
t.Fatalf("application = %q, want seriatim", transcript.Metadata.Application)
|
t.Fatalf("application = %q, want seriatim", transcript.Metadata.Application)
|
||||||
}
|
}
|
||||||
if transcript.Metadata.OutputSchema != "minimal" {
|
if transcript.Metadata.OutputSchema != config.OutputSchemaMinimal {
|
||||||
t.Fatalf("output_schema = %q, want minimal", transcript.Metadata.OutputSchema)
|
t.Fatalf("output_schema = %q, want %q", transcript.Metadata.OutputSchema, config.OutputSchemaMinimal)
|
||||||
}
|
}
|
||||||
if got, want := len(transcript.Segments), 2; got != want {
|
if got, want := len(transcript.Segments), 2; got != want {
|
||||||
t.Fatalf("segment count = %d, want %d", got, want)
|
t.Fatalf("segment count = %d, want %d", got, want)
|
||||||
@@ -330,7 +331,7 @@ func TestMergeValidateMinimalOutputBeforeAssignIDsFails(t *testing.T) {
|
|||||||
err := executeMerge(
|
err := executeMerge(
|
||||||
"--input-file", input,
|
"--input-file", input,
|
||||||
"--output-file", output,
|
"--output-file", output,
|
||||||
"--output-schema", "minimal",
|
"--output-schema", config.OutputSchemaMinimal,
|
||||||
"--postprocessing-modules", "validate-output,assign-ids",
|
"--postprocessing-modules", "validate-output,assign-ids",
|
||||||
)
|
)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
@@ -472,16 +473,16 @@ func TestMergeResolvesOverlapGroupsWithWordRuns(t *testing.T) {
|
|||||||
|
|
||||||
var transcript model.FinalTranscript
|
var transcript model.FinalTranscript
|
||||||
readJSON(t, output, &transcript)
|
readJSON(t, output, &transcript)
|
||||||
if len(transcript.OverlapGroups) != 0 {
|
if got, want := len(transcript.OverlapGroups), 1; got != want {
|
||||||
t.Fatalf("overlap groups = %#v, want none", transcript.OverlapGroups)
|
t.Fatalf("overlap group count = %d, want %d (%#v)", got, want, transcript.OverlapGroups)
|
||||||
}
|
}
|
||||||
if got, want := len(transcript.Segments), 3; got != want {
|
if got, want := len(transcript.Segments), 2; got != want {
|
||||||
t.Fatalf("segment count = %d, want %d", got, want)
|
t.Fatalf("segment count = %d, want %d", got, want)
|
||||||
}
|
}
|
||||||
|
|
||||||
wantTexts := []string{"outside hello there", "bob reply", "later"}
|
wantTexts := []string{"outside hello there later", "bob reply"}
|
||||||
wantSpeakers := []string{"Alice", "Bob", "Alice"}
|
wantSpeakers := []string{"Alice", "Bob"}
|
||||||
wantRefs := []string{"word-run:1:1:1", "word-run:1:2:1", "word-run:1:1:2"}
|
wantRefs := []string{"word-run:1:1:1", "word-run:1:2:1"}
|
||||||
for index, segment := range transcript.Segments {
|
for index, segment := range transcript.Segments {
|
||||||
if segment.ID != index+1 {
|
if segment.ID != index+1 {
|
||||||
t.Fatalf("segment %d id = %d, want %d", index, segment.ID, index+1)
|
t.Fatalf("segment %d id = %d, want %d", index, segment.ID, index+1)
|
||||||
@@ -498,8 +499,8 @@ func TestMergeResolvesOverlapGroupsWithWordRuns(t *testing.T) {
|
|||||||
if segment.SourceSegmentIndex != nil {
|
if segment.SourceSegmentIndex != nil {
|
||||||
t.Fatalf("segment %d source_segment_index = %d, want nil", index, *segment.SourceSegmentIndex)
|
t.Fatalf("segment %d source_segment_index = %d, want nil", index, *segment.SourceSegmentIndex)
|
||||||
}
|
}
|
||||||
if segment.OverlapGroupID != 0 {
|
if segment.OverlapGroupID != 1 {
|
||||||
t.Fatalf("segment %d overlap_group_id = %d, want 0", index, segment.OverlapGroupID)
|
t.Fatalf("segment %d overlap_group_id = %d, want 1", index, segment.OverlapGroupID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !equalStrings(transcript.Segments[0].DerivedFrom, []string{inputA + "#0"}) {
|
if !equalStrings(transcript.Segments[0].DerivedFrom, []string{inputA + "#0"}) {
|
||||||
@@ -516,7 +517,7 @@ func TestMergeResolvesOverlapGroupsWithWordRuns(t *testing.T) {
|
|||||||
|
|
||||||
var rpt report.Report
|
var rpt report.Report
|
||||||
readJSON(t, reportPath, &rpt)
|
readJSON(t, reportPath, &rpt)
|
||||||
if !hasReportEvent(rpt, "postprocessing", "resolve-overlaps", "processed 1 overlap group(s); changed 1; removed 2 original segment(s); created 3 replacement segment(s)") {
|
if !hasReportEvent(rpt, "postprocessing", "resolve-overlaps", "processed 1 overlap group(s); changed 1; removed 2 original segment(s); created 2 replacement segment(s)") {
|
||||||
t.Fatal("expected resolve-overlaps summary report event")
|
t.Fatal("expected resolve-overlaps summary report event")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1927,7 +1928,7 @@ func TestMergeResolveOverlapsPreservesAbsorbedContextPrefix(t *testing.T) {
|
|||||||
"--input-file", inputA,
|
"--input-file", inputA,
|
||||||
"--input-file", inputB,
|
"--input-file", inputB,
|
||||||
"--speakers", speakers,
|
"--speakers", speakers,
|
||||||
"--output-schema", "minimal",
|
"--output-schema", config.OutputSchemaMinimal,
|
||||||
"--output-file", output,
|
"--output-file", output,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1994,7 +1995,7 @@ func executeMerge(args ...string) error {
|
|||||||
if !hasOutputSchemaFlag(args) {
|
if !hasOutputSchemaFlag(args) {
|
||||||
// Most integration tests were written against the full envelope; keep
|
// Most integration tests were written against the full envelope; keep
|
||||||
// that behavior unless the caller explicitly asks for another schema.
|
// that behavior unless the caller explicitly asks for another schema.
|
||||||
args = append(args, "--output-schema", config.OutputSchemaSeriatim)
|
args = append(args, "--output-schema", config.OutputSchemaFull)
|
||||||
}
|
}
|
||||||
return executeMergeRaw(args...)
|
return executeMergeRaw(args...)
|
||||||
}
|
}
|
||||||
|
|||||||
39
internal/cli/normalize.go
Normal file
39
internal/cli/normalize.go
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/internal/normalize"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newNormalizeCommand() *cobra.Command {
|
||||||
|
var opts config.NormalizeOptions
|
||||||
|
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "normalize",
|
||||||
|
Short: "Normalize a transcript artifact into a standard seriatim output shape",
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
normalizeOpts := opts
|
||||||
|
if !cmd.Flags().Changed("output-schema") {
|
||||||
|
normalizeOpts.OutputSchema = ""
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg, err := config.NewNormalizeConfig(normalizeOpts)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return normalize.Run(cmd.Context(), cfg)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
flags := cmd.Flags()
|
||||||
|
flags.StringVar(&opts.InputFile, "input-file", "", "input transcript JSON file")
|
||||||
|
addOutputFileFlag(cmd, &opts.OutputFile)
|
||||||
|
addReportFileFlag(cmd, &opts.ReportFile)
|
||||||
|
addNormalizeOutputSchemaFlag(cmd, &opts.OutputSchema)
|
||||||
|
addOutputModulesFlag(cmd, &opts.OutputModules)
|
||||||
|
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
522
internal/cli/normalize_test.go
Normal file
522
internal/cli/normalize_test.go
Normal file
@@ -0,0 +1,522 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/internal/report"
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/schema"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNormalizeCommandIsRecognized(t *testing.T) {
|
||||||
|
cmd := NewRootCommand()
|
||||||
|
cmd.SetArgs([]string{"normalize", "--help"})
|
||||||
|
if err := cmd.Execute(); err != nil {
|
||||||
|
t.Fatalf("normalize command should be recognized: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeMissingInputFileFails(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
output := filepath.Join(dir, "normalized.json")
|
||||||
|
|
||||||
|
err := executeNormalize(
|
||||||
|
"--output-file", output,
|
||||||
|
)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected missing input-file error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "--input-file is required") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeMissingOutputFileFails(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeJSONFile(t, dir, "input.json", `{"segments":[]}`)
|
||||||
|
|
||||||
|
err := executeNormalize(
|
||||||
|
"--input-file", input,
|
||||||
|
)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected missing output-file error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "--output-file is required") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeInvalidOutputSchemaFails(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeJSONFile(t, dir, "input.json", `{"segments":[]}`)
|
||||||
|
output := filepath.Join(dir, "normalized.json")
|
||||||
|
|
||||||
|
err := executeNormalize(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
"--output-schema", "compact",
|
||||||
|
)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected invalid output schema error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "--output-schema must be one of") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeInvalidOutputModuleFails(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeJSONFile(t, dir, "input.json", `{"segments":[]}`)
|
||||||
|
output := filepath.Join(dir, "normalized.json")
|
||||||
|
|
||||||
|
err := executeNormalize(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
"--output-modules", "yaml",
|
||||||
|
)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected invalid output module error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "unknown output module") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeDefaultOutputSchemaIsIntermediate(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeJSONFile(t, dir, "input.json", `{
|
||||||
|
"segments": [
|
||||||
|
{"id": 99, "start": 5, "end": 6, "speaker": "Bob", "text": "second", "categories": ["filler"]},
|
||||||
|
{"id": 10, "start": 1, "end": 2, "speaker": "Alice", "text": "first", "categories": ["backchannel"]}
|
||||||
|
]
|
||||||
|
}`)
|
||||||
|
output := filepath.Join(dir, "normalized.json")
|
||||||
|
|
||||||
|
err := executeNormalize(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("normalize failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var transcript schema.IntermediateTranscript
|
||||||
|
readJSON(t, output, &transcript)
|
||||||
|
if transcript.Metadata.OutputSchema != config.OutputSchemaIntermediate {
|
||||||
|
t.Fatalf("output schema = %q, want %q", transcript.Metadata.OutputSchema, config.OutputSchemaIntermediate)
|
||||||
|
}
|
||||||
|
if len(transcript.Segments) != 2 {
|
||||||
|
t.Fatalf("segment count = %d, want 2", len(transcript.Segments))
|
||||||
|
}
|
||||||
|
if transcript.Segments[0].ID != 1 || transcript.Segments[1].ID != 2 {
|
||||||
|
t.Fatalf("segment IDs = %d,%d, want 1,2", transcript.Segments[0].ID, transcript.Segments[1].ID)
|
||||||
|
}
|
||||||
|
if transcript.Segments[0].Text != "first" || transcript.Segments[1].Text != "second" {
|
||||||
|
t.Fatalf("unexpected sort order: %#v", transcript.Segments)
|
||||||
|
}
|
||||||
|
if len(transcript.Segments[0].Categories) != 1 || transcript.Segments[0].Categories[0] != "backchannel" {
|
||||||
|
t.Fatalf("expected categories preserved on first segment, got %#v", transcript.Segments[0].Categories)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeBareArrayInputToIntermediateOutput(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeJSONFile(t, dir, "input.json", `[
|
||||||
|
{"start": 2, "end": 3, "speaker": "Bob", "text": "second"},
|
||||||
|
{"start": 1, "end": 2, "speaker": "Alice", "text": "first"}
|
||||||
|
]`)
|
||||||
|
output := filepath.Join(dir, "normalized.json")
|
||||||
|
|
||||||
|
err := executeNormalize(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
"--output-schema", config.OutputSchemaIntermediate,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("normalize failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var transcript schema.IntermediateTranscript
|
||||||
|
readJSON(t, output, &transcript)
|
||||||
|
if len(transcript.Segments) != 2 {
|
||||||
|
t.Fatalf("segment count = %d, want 2", len(transcript.Segments))
|
||||||
|
}
|
||||||
|
if transcript.Segments[0].Speaker != "Alice" || transcript.Segments[1].Speaker != "Bob" {
|
||||||
|
t.Fatalf("unexpected sorted speakers: %#v", transcript.Segments)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeInputIndexTieBreakerIsDeterministic(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeJSONFile(t, dir, "input.json", `[
|
||||||
|
{"start": 1, "end": 2, "speaker": "Zulu", "text": "first in"},
|
||||||
|
{"start": 1, "end": 2, "speaker": "Alpha", "text": "second in"}
|
||||||
|
]`)
|
||||||
|
output := filepath.Join(dir, "normalized.json")
|
||||||
|
|
||||||
|
err := executeNormalize(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("normalize failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var transcript schema.IntermediateTranscript
|
||||||
|
readJSON(t, output, &transcript)
|
||||||
|
if transcript.Segments[0].Speaker != "Zulu" || transcript.Segments[1].Speaker != "Alpha" {
|
||||||
|
t.Fatalf("tie-break order mismatch: %#v", transcript.Segments)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeMinimalSchemaOmitsCategories(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeJSONFile(t, dir, "input.json", `{
|
||||||
|
"segments": [
|
||||||
|
{"start": 1, "end": 2, "speaker": "Alice", "text": "first", "categories": ["filler"]}
|
||||||
|
]
|
||||||
|
}`)
|
||||||
|
output := filepath.Join(dir, "normalized.json")
|
||||||
|
|
||||||
|
err := executeNormalize(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
"--output-schema", config.OutputSchemaMinimal,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("normalize failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var transcript schema.MinimalTranscript
|
||||||
|
readJSON(t, output, &transcript)
|
||||||
|
if transcript.Metadata.OutputSchema != config.OutputSchemaMinimal {
|
||||||
|
t.Fatalf("output schema = %q, want %q", transcript.Metadata.OutputSchema, config.OutputSchemaMinimal)
|
||||||
|
}
|
||||||
|
if len(transcript.Segments) != 1 || transcript.Segments[0].ID != 1 {
|
||||||
|
t.Fatalf("unexpected minimal output: %#v", transcript.Segments)
|
||||||
|
}
|
||||||
|
bytes, readErr := os.ReadFile(output)
|
||||||
|
if readErr != nil {
|
||||||
|
t.Fatalf("read output: %v", readErr)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(bytes), "categories") {
|
||||||
|
t.Fatalf("minimal output unexpectedly contains categories:\n%s", string(bytes))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeFullSchemaOutputValidatesAndHasProvenanceFallback(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeJSONFile(t, dir, "input.json", `[
|
||||||
|
{"start": 1, "end": 2, "speaker": "Alice", "text": "first"},
|
||||||
|
{"start": 3, "end": 4, "speaker": "Bob", "text": "second", "source":"custom.json", "source_segment_index": 7}
|
||||||
|
]`)
|
||||||
|
output := filepath.Join(dir, "normalized.json")
|
||||||
|
|
||||||
|
err := executeNormalize(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
"--output-schema", config.OutputSchemaFull,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("normalize failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var transcript schema.Transcript
|
||||||
|
readJSON(t, output, &transcript)
|
||||||
|
if err := schema.ValidateTranscript(transcript); err != nil {
|
||||||
|
t.Fatalf("full output should validate: %v", err)
|
||||||
|
}
|
||||||
|
if len(transcript.Segments) != 2 {
|
||||||
|
t.Fatalf("segment count = %d, want 2", len(transcript.Segments))
|
||||||
|
}
|
||||||
|
if transcript.Segments[0].Source != filepath.Base(input) {
|
||||||
|
t.Fatalf("source fallback = %q, want %q", transcript.Segments[0].Source, filepath.Base(input))
|
||||||
|
}
|
||||||
|
if transcript.Segments[0].SourceSegmentIndex == nil || *transcript.Segments[0].SourceSegmentIndex != 0 {
|
||||||
|
t.Fatalf("source_segment_index fallback = %v, want 0", transcript.Segments[0].SourceSegmentIndex)
|
||||||
|
}
|
||||||
|
if transcript.Segments[1].Source != "custom.json" {
|
||||||
|
t.Fatalf("explicit source preserved = %q, want custom.json", transcript.Segments[1].Source)
|
||||||
|
}
|
||||||
|
if transcript.Segments[1].SourceSegmentIndex == nil || *transcript.Segments[1].SourceSegmentIndex != 7 {
|
||||||
|
t.Fatalf("explicit source_segment_index preserved = %v, want 7", transcript.Segments[1].SourceSegmentIndex)
|
||||||
|
}
|
||||||
|
if transcript.OverlapGroups == nil || len(transcript.OverlapGroups) != 0 {
|
||||||
|
t.Fatalf("overlap_groups = %#v, want empty array", transcript.OverlapGroups)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeEmptySegmentsArrayProducesValidOutput(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeJSONFile(t, dir, "input.json", `{"segments":[]}`)
|
||||||
|
output := filepath.Join(dir, "normalized.json")
|
||||||
|
|
||||||
|
err := executeNormalize(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("normalize failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var transcript schema.IntermediateTranscript
|
||||||
|
readJSON(t, output, &transcript)
|
||||||
|
if len(transcript.Segments) != 0 {
|
||||||
|
t.Fatalf("segment count = %d, want 0", len(transcript.Segments))
|
||||||
|
}
|
||||||
|
if err := schema.ValidateIntermediateTranscript(transcript); err != nil {
|
||||||
|
t.Fatalf("intermediate output should validate: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeRepairsAndDropsDefectiveSegments(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeJSONFile(t, dir, "input.json", `[
|
||||||
|
{"start": 5, "speaker": "", "text": "keep-a"},
|
||||||
|
{"end": 3, "speaker": " ", "text": "keep-b"},
|
||||||
|
{"speaker": "A"},
|
||||||
|
{"speaker": "A", "text": " "},
|
||||||
|
{"start": 9, "end": 4, "speaker": "B", "text": "keep-c"}
|
||||||
|
]`)
|
||||||
|
output := filepath.Join(dir, "normalized.json")
|
||||||
|
reportPath := filepath.Join(dir, "report.json")
|
||||||
|
|
||||||
|
err := executeNormalize(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
"--report-file", reportPath,
|
||||||
|
"--output-schema", config.OutputSchemaIntermediate,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("normalize failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var transcript schema.IntermediateTranscript
|
||||||
|
readJSON(t, output, &transcript)
|
||||||
|
if len(transcript.Segments) != 3 {
|
||||||
|
t.Fatalf("segment count = %d, want 3", len(transcript.Segments))
|
||||||
|
}
|
||||||
|
if transcript.Segments[0].Start != 3 || transcript.Segments[0].End != 3 {
|
||||||
|
t.Fatalf("segment[0] timing = %v..%v, want 3..3", transcript.Segments[0].Start, transcript.Segments[0].End)
|
||||||
|
}
|
||||||
|
if transcript.Segments[1].Start != 4 || transcript.Segments[1].End != 9 {
|
||||||
|
t.Fatalf("segment[1] timing = %v..%v, want 4..9", transcript.Segments[1].Start, transcript.Segments[1].End)
|
||||||
|
}
|
||||||
|
if transcript.Segments[2].Start != 5 || transcript.Segments[2].End != 5 {
|
||||||
|
t.Fatalf("segment[2] timing = %v..%v, want 5..5", transcript.Segments[2].Start, transcript.Segments[2].End)
|
||||||
|
}
|
||||||
|
if transcript.Segments[0].Speaker != "Unknown_Speaker" || transcript.Segments[2].Speaker != "Unknown_Speaker" {
|
||||||
|
t.Fatalf("expected Unknown_Speaker placeholders, got %#v", transcript.Segments)
|
||||||
|
}
|
||||||
|
|
||||||
|
var rpt report.Report
|
||||||
|
readJSON(t, reportPath, &rpt)
|
||||||
|
audit := extractNormalizeAudit(t, rpt)
|
||||||
|
if audit.InputSegmentCount != 5 || audit.OutputSegmentCount != 3 {
|
||||||
|
t.Fatalf("audit counts = in:%d out:%d, want in:5 out:3", audit.InputSegmentCount, audit.OutputSegmentCount)
|
||||||
|
}
|
||||||
|
if audit.TimingFieldsRepaired != 2 {
|
||||||
|
t.Fatalf("timing fields repaired = %d, want 2", audit.TimingFieldsRepaired)
|
||||||
|
}
|
||||||
|
if audit.TimingOrderSwapped != 1 {
|
||||||
|
t.Fatalf("timing order swapped = %d, want 1", audit.TimingOrderSwapped)
|
||||||
|
}
|
||||||
|
if audit.SpeakerFilled != 2 {
|
||||||
|
t.Fatalf("speaker filled = %d, want 2", audit.SpeakerFilled)
|
||||||
|
}
|
||||||
|
if audit.SegmentsDroppedText != 2 {
|
||||||
|
t.Fatalf("segments dropped text = %d, want 2", audit.SegmentsDroppedText)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeSelectedOutputSchemaIsHonored(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeJSONFile(t, dir, "input.json", `{"segments":[{"start":1,"end":2,"speaker":"A","text":"one"}]}`)
|
||||||
|
output := filepath.Join(dir, "normalized.json")
|
||||||
|
|
||||||
|
err := executeNormalize(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
"--output-schema", config.OutputSchemaMinimal,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("normalize failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var transcript schema.MinimalTranscript
|
||||||
|
readJSON(t, output, &transcript)
|
||||||
|
if transcript.Metadata.OutputSchema != config.OutputSchemaMinimal {
|
||||||
|
t.Fatalf("output schema = %q, want %q", transcript.Metadata.OutputSchema, config.OutputSchemaMinimal)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeReportFileWrittenAndContainsObjectInputShape(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeJSONFile(t, dir, "input.json", `{"segments":[{"start":1,"end":2,"speaker":"A","text":"one"}]}`)
|
||||||
|
output := filepath.Join(dir, "normalized.json")
|
||||||
|
reportPath := filepath.Join(dir, "report.json")
|
||||||
|
|
||||||
|
err := executeNormalize(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
"--report-file", reportPath,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("normalize failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var rpt report.Report
|
||||||
|
readJSON(t, reportPath, &rpt)
|
||||||
|
audit := extractNormalizeAudit(t, rpt)
|
||||||
|
if audit.InputShape != "object_with_segments" {
|
||||||
|
t.Fatalf("input shape = %q, want object_with_segments", audit.InputShape)
|
||||||
|
}
|
||||||
|
if audit.InputSegmentCount != 1 {
|
||||||
|
t.Fatalf("input segment count = %d, want 1", audit.InputSegmentCount)
|
||||||
|
}
|
||||||
|
if audit.OutputSchema != config.OutputSchemaIntermediate {
|
||||||
|
t.Fatalf("output schema = %q, want %q", audit.OutputSchema, config.OutputSchemaIntermediate)
|
||||||
|
}
|
||||||
|
if len(audit.OutputModules) != 1 || audit.OutputModules[0] != "json" {
|
||||||
|
t.Fatalf("output modules = %v, want [json]", audit.OutputModules)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeReportIncludesBareArrayShape(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeJSONFile(t, dir, "input.json", `[{"start":1,"end":2,"speaker":"A","text":"one"}]`)
|
||||||
|
output := filepath.Join(dir, "normalized.json")
|
||||||
|
reportPath := filepath.Join(dir, "report.json")
|
||||||
|
|
||||||
|
err := executeNormalize(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
"--report-file", reportPath,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("normalize failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var rpt report.Report
|
||||||
|
readJSON(t, reportPath, &rpt)
|
||||||
|
audit := extractNormalizeAudit(t, rpt)
|
||||||
|
if audit.InputShape != "bare_segments_array" {
|
||||||
|
t.Fatalf("input shape = %q, want bare_segments_array", audit.InputShape)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeReportDoesNotIncludeTranscriptText(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
const segmentText = "normalize-report-secret-text"
|
||||||
|
input := writeJSONFile(t, dir, "input.json", `[{"start":1,"end":2,"speaker":"A","text":"`+segmentText+`"}]`)
|
||||||
|
output := filepath.Join(dir, "normalized.json")
|
||||||
|
reportPath := filepath.Join(dir, "report.json")
|
||||||
|
|
||||||
|
err := executeNormalize(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
"--report-file", reportPath,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("normalize failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var rpt report.Report
|
||||||
|
readJSON(t, reportPath, &rpt)
|
||||||
|
for _, event := range rpt.Events {
|
||||||
|
if strings.Contains(event.Message, segmentText) {
|
||||||
|
t.Fatalf("report unexpectedly contained transcript text in event %#v", event)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeReportEmptyInputEmitsWarning(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeJSONFile(t, dir, "input.json", `{"segments":[]}`)
|
||||||
|
output := filepath.Join(dir, "normalized.json")
|
||||||
|
reportPath := filepath.Join(dir, "report.json")
|
||||||
|
|
||||||
|
err := executeNormalize(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
"--report-file", reportPath,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("normalize failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var rpt report.Report
|
||||||
|
readJSON(t, reportPath, &rpt)
|
||||||
|
found := false
|
||||||
|
for _, event := range rpt.Events {
|
||||||
|
if event.Stage == "normalize" && event.Module == "normalize" && event.Severity == report.SeverityWarning &&
|
||||||
|
strings.Contains(event.Message, "zero segments") {
|
||||||
|
found = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Fatalf("expected empty transcript warning event, got %#v", rpt.Events)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeReportWriteFailureReturnsClearError(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeJSONFile(t, dir, "input.json", `{"segments":[{"start":1,"end":2,"speaker":"A","text":"one"}]}`)
|
||||||
|
output := filepath.Join(dir, "normalized.json")
|
||||||
|
|
||||||
|
err := executeNormalize(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
"--report-file", dir,
|
||||||
|
)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected report write failure")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "write --report-file") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func executeNormalize(args ...string) error {
|
||||||
|
cmd := NewRootCommand()
|
||||||
|
cmd.SetArgs(append([]string{"normalize"}, args...))
|
||||||
|
return cmd.Execute()
|
||||||
|
}
|
||||||
|
|
||||||
|
type normalizeAudit struct {
|
||||||
|
Command string `json:"command"`
|
||||||
|
InputFile string `json:"input_file"`
|
||||||
|
OutputFile string `json:"output_file"`
|
||||||
|
InputShape string `json:"input_shape"`
|
||||||
|
InputSegmentCount int `json:"input_segment_count"`
|
||||||
|
OutputSegmentCount int `json:"output_segment_count"`
|
||||||
|
OutputSchema string `json:"output_schema"`
|
||||||
|
OutputModules []string `json:"output_modules"`
|
||||||
|
IDsReassigned bool `json:"ids_reassigned"`
|
||||||
|
SortingChangedInput bool `json:"sorting_changed_input_order"`
|
||||||
|
SegmentsWithCategories int `json:"segments_with_categories"`
|
||||||
|
TimingFieldsRepaired int `json:"timing_fields_repaired"`
|
||||||
|
TimingOrderSwapped int `json:"timing_order_swapped"`
|
||||||
|
SpeakerFilled int `json:"speaker_filled"`
|
||||||
|
SegmentsDroppedText int `json:"segments_dropped_text"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractNormalizeAudit(t *testing.T, rpt report.Report) normalizeAudit {
|
||||||
|
t.Helper()
|
||||||
|
for _, event := range rpt.Events {
|
||||||
|
if event.Stage == "normalize" && event.Module == "normalize-audit" {
|
||||||
|
var audit normalizeAudit
|
||||||
|
if err := json.Unmarshal([]byte(event.Message), &audit); err != nil {
|
||||||
|
t.Fatalf("decode normalize audit: %v", err)
|
||||||
|
}
|
||||||
|
return audit
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("missing normalize-audit event: %#v", rpt.Events)
|
||||||
|
return normalizeAudit{}
|
||||||
|
}
|
||||||
@@ -10,12 +10,14 @@ import (
|
|||||||
func NewRootCommand() *cobra.Command {
|
func NewRootCommand() *cobra.Command {
|
||||||
cmd := &cobra.Command{
|
cmd := &cobra.Command{
|
||||||
Use: "seriatim",
|
Use: "seriatim",
|
||||||
Short: "Merge per-speaker transcripts into a chronological transcript",
|
Short: "Merge, trim, and normalize transcript artifacts",
|
||||||
Version: buildinfo.Version,
|
Version: buildinfo.Version,
|
||||||
SilenceErrors: true,
|
SilenceErrors: true,
|
||||||
SilenceUsage: true,
|
SilenceUsage: true,
|
||||||
}
|
}
|
||||||
|
|
||||||
cmd.AddCommand(newMergeCommand())
|
cmd.AddCommand(newMergeCommand())
|
||||||
|
cmd.AddCommand(newNormalizeCommand())
|
||||||
|
cmd.AddCommand(newTrimCommand())
|
||||||
return cmd
|
return cmd
|
||||||
}
|
}
|
||||||
|
|||||||
41
internal/cli/trim.go
Normal file
41
internal/cli/trim.go
Normal file
@@ -0,0 +1,41 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/internal/trim"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newTrimCommand() *cobra.Command {
|
||||||
|
var opts config.TrimOptions
|
||||||
|
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "trim",
|
||||||
|
Short: "Trim an existing seriatim transcript artifact by segment ID",
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
trimOpts := opts
|
||||||
|
if !cmd.Flags().Changed("output-schema") {
|
||||||
|
trimOpts.OutputSchema = ""
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg, err := config.NewTrimConfig(trimOpts)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return trim.Run(cmd.Context(), cfg)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
flags := cmd.Flags()
|
||||||
|
flags.StringVar(&opts.InputFile, "input-file", "", "input seriatim transcript artifact JSON file")
|
||||||
|
addOutputFileFlag(cmd, &opts.OutputFile)
|
||||||
|
addReportFileFlag(cmd, &opts.ReportFile)
|
||||||
|
flags.StringVar(&opts.Keep, "keep", "", "segment ID selector to keep (for example: 1-10,15)")
|
||||||
|
flags.StringVar(&opts.Remove, "remove", "", "segment ID selector to remove (for example: 1-10,15)")
|
||||||
|
addTrimOutputSchemaFlag(cmd, &opts.OutputSchema)
|
||||||
|
flags.BoolVar(&opts.AllowEmpty, "allow-empty", false, "allow trimming to an empty transcript")
|
||||||
|
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
781
internal/cli/trim_test.go
Normal file
781
internal/cli/trim_test.go
Normal file
@@ -0,0 +1,781 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/internal/report"
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/schema"
|
||||||
|
)
|
||||||
|
|
||||||
|
type trimAuditReport struct {
|
||||||
|
Operation string `json:"operation"`
|
||||||
|
InputFile string `json:"input_file"`
|
||||||
|
OutputFile string `json:"output_file"`
|
||||||
|
InputSchema string `json:"input_schema"`
|
||||||
|
OutputSchema string `json:"output_schema"`
|
||||||
|
Mode string `json:"mode"`
|
||||||
|
Selector string `json:"selector"`
|
||||||
|
SelectedIDs []int `json:"selected_ids"`
|
||||||
|
AllowEmpty bool `json:"allow_empty"`
|
||||||
|
InputSegmentCount int `json:"input_segment_count"`
|
||||||
|
RetainedSegmentCount int `json:"retained_segment_count"`
|
||||||
|
RemovedSegmentCount int `json:"removed_segment_count"`
|
||||||
|
RemovedInputIDs []int `json:"removed_input_ids"`
|
||||||
|
OldToNewIDMapping []trimIDMapping `json:"old_to_new_id_mapping"`
|
||||||
|
OverlapGroupsRecomputed bool `json:"overlap_groups_recomputed"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type trimIDMapping struct {
|
||||||
|
OldID int `json:"old_id"`
|
||||||
|
NewID int `json:"new_id"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTrimKeepModeEndToEnd(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTrimFullFixture(t, dir, "input.json")
|
||||||
|
output := filepath.Join(dir, "trimmed.json")
|
||||||
|
|
||||||
|
err := executeTrim(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
"--keep", "2,4",
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("trim failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var transcript schema.Transcript
|
||||||
|
readJSON(t, output, &transcript)
|
||||||
|
if len(transcript.Segments) != 2 {
|
||||||
|
t.Fatalf("segment count = %d, want 2", len(transcript.Segments))
|
||||||
|
}
|
||||||
|
if transcript.Segments[0].Text != "two" || transcript.Segments[1].Text != "four" {
|
||||||
|
t.Fatalf("unexpected kept text order: %#v", transcript.Segments)
|
||||||
|
}
|
||||||
|
assertSequentialIDs(t, []int{transcript.Segments[0].ID, transcript.Segments[1].ID})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTrimRemoveModeEndToEnd(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTrimFullFixture(t, dir, "input.json")
|
||||||
|
output := filepath.Join(dir, "trimmed.json")
|
||||||
|
|
||||||
|
err := executeTrim(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
"--remove", "2,4",
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("trim failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var transcript schema.Transcript
|
||||||
|
readJSON(t, output, &transcript)
|
||||||
|
if len(transcript.Segments) != 2 {
|
||||||
|
t.Fatalf("segment count = %d, want 2", len(transcript.Segments))
|
||||||
|
}
|
||||||
|
if transcript.Segments[0].Text != "one" || transcript.Segments[1].Text != "three" {
|
||||||
|
t.Fatalf("unexpected remaining text order: %#v", transcript.Segments)
|
||||||
|
}
|
||||||
|
assertSequentialIDs(t, []int{transcript.Segments[0].ID, transcript.Segments[1].ID})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTrimMutualExclusionFailure(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTrimFullFixture(t, dir, "input.json")
|
||||||
|
output := filepath.Join(dir, "trimmed.json")
|
||||||
|
|
||||||
|
err := executeTrim(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
"--keep", "1",
|
||||||
|
"--remove", "2",
|
||||||
|
)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected mutual exclusion error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "mutually exclusive") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTrimMissingSelectionFailure(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTrimFullFixture(t, dir, "input.json")
|
||||||
|
output := filepath.Join(dir, "trimmed.json")
|
||||||
|
|
||||||
|
err := executeTrim(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected selection flag error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "exactly one of --keep or --remove is required") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTrimInvalidSelectedIDFailure(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTrimFullFixture(t, dir, "input.json")
|
||||||
|
output := filepath.Join(dir, "trimmed.json")
|
||||||
|
|
||||||
|
err := executeTrim(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
"--keep", "99",
|
||||||
|
)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected missing selected ID error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "does not exist") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTrimOmittedOutputSchemaPreservesInputSchema(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTrimMinimalFixture(t, dir, "input-minimal.json")
|
||||||
|
output := filepath.Join(dir, "trimmed.json")
|
||||||
|
|
||||||
|
err := executeTrim(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
"--keep", "1",
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("trim failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var transcript schema.MinimalTranscript
|
||||||
|
readJSON(t, output, &transcript)
|
||||||
|
if transcript.Metadata.OutputSchema != config.OutputSchemaMinimal {
|
||||||
|
t.Fatalf("output_schema = %q, want %q", transcript.Metadata.OutputSchema, config.OutputSchemaMinimal)
|
||||||
|
}
|
||||||
|
if len(transcript.Segments) != 1 || transcript.Segments[0].ID != 1 {
|
||||||
|
t.Fatalf("unexpected minimal trim output: %#v", transcript.Segments)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTrimExplicitOutputSchemaChangesOutputSchema(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTrimFullFixture(t, dir, "input.json")
|
||||||
|
output := filepath.Join(dir, "trimmed.json")
|
||||||
|
|
||||||
|
err := executeTrim(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
"--keep", "1,3",
|
||||||
|
"--output-schema", config.OutputSchemaMinimal,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("trim failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var transcript schema.MinimalTranscript
|
||||||
|
readJSON(t, output, &transcript)
|
||||||
|
if transcript.Metadata.OutputSchema != config.OutputSchemaMinimal {
|
||||||
|
t.Fatalf("output_schema = %q, want %q", transcript.Metadata.OutputSchema, config.OutputSchemaMinimal)
|
||||||
|
}
|
||||||
|
if len(transcript.Segments) != 2 {
|
||||||
|
t.Fatalf("segment count = %d, want 2", len(transcript.Segments))
|
||||||
|
}
|
||||||
|
assertSequentialIDs(t, []int{transcript.Segments[0].ID, transcript.Segments[1].ID})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTrimExplicitOutputSchemaConvertsMinimalToIntermediate(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTrimMinimalFixture(t, dir, "input-minimal.json")
|
||||||
|
output := filepath.Join(dir, "trimmed.json")
|
||||||
|
|
||||||
|
err := executeTrim(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
"--keep", "1-2",
|
||||||
|
"--output-schema", config.OutputSchemaIntermediate,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("trim failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var transcript schema.IntermediateTranscript
|
||||||
|
readJSON(t, output, &transcript)
|
||||||
|
if transcript.Metadata.OutputSchema != config.OutputSchemaIntermediate {
|
||||||
|
t.Fatalf("output_schema = %q, want %q", transcript.Metadata.OutputSchema, config.OutputSchemaIntermediate)
|
||||||
|
}
|
||||||
|
if len(transcript.Segments) != 2 {
|
||||||
|
t.Fatalf("segment count = %d, want 2", len(transcript.Segments))
|
||||||
|
}
|
||||||
|
assertSequentialIDs(t, []int{transcript.Segments[0].ID, transcript.Segments[1].ID})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTrimIntermediateInputPreservesIntermediateOutputAndCategories(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTrimIntermediateFixture(t, dir, "input-intermediate.json")
|
||||||
|
output := filepath.Join(dir, "trimmed.json")
|
||||||
|
|
||||||
|
err := executeTrim(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
"--keep", "2",
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("trim failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var transcript schema.IntermediateTranscript
|
||||||
|
readJSON(t, output, &transcript)
|
||||||
|
if transcript.Metadata.OutputSchema != config.OutputSchemaIntermediate {
|
||||||
|
t.Fatalf("output_schema = %q, want %q", transcript.Metadata.OutputSchema, config.OutputSchemaIntermediate)
|
||||||
|
}
|
||||||
|
if len(transcript.Segments) != 1 {
|
||||||
|
t.Fatalf("segment count = %d, want 1", len(transcript.Segments))
|
||||||
|
}
|
||||||
|
if transcript.Segments[0].ID != 1 {
|
||||||
|
t.Fatalf("segment ID = %d, want 1", transcript.Segments[0].ID)
|
||||||
|
}
|
||||||
|
assertIntSliceEqual(t, []int{len(transcript.Segments[0].Categories)}, []int{2})
|
||||||
|
if transcript.Segments[0].Categories[0] != "filler" || transcript.Segments[0].Categories[1] != "backchannel" {
|
||||||
|
t.Fatalf("categories = %v, want [filler backchannel]", transcript.Segments[0].Categories)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTrimFullInputPreservesFullShapeAndRecomputesOverlapGroups(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTrimFullOverlapFixture(t, dir, "input-full-overlap.json")
|
||||||
|
output := filepath.Join(dir, "trimmed.json")
|
||||||
|
|
||||||
|
err := executeTrim(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
"--keep", "1,2",
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("trim failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var transcript schema.Transcript
|
||||||
|
readJSON(t, output, &transcript)
|
||||||
|
if len(transcript.Segments) != 2 {
|
||||||
|
t.Fatalf("segment count = %d, want 2", len(transcript.Segments))
|
||||||
|
}
|
||||||
|
assertSequentialIDs(t, []int{transcript.Segments[0].ID, transcript.Segments[1].ID})
|
||||||
|
if len(transcript.OverlapGroups) != 1 {
|
||||||
|
t.Fatalf("overlap group count = %d, want 1", len(transcript.OverlapGroups))
|
||||||
|
}
|
||||||
|
if transcript.OverlapGroups[0].ID != 1 {
|
||||||
|
t.Fatalf("overlap group id = %d, want 1", transcript.OverlapGroups[0].ID)
|
||||||
|
}
|
||||||
|
if transcript.Segments[0].OverlapGroupID != 1 || transcript.Segments[1].OverlapGroupID != 1 {
|
||||||
|
t.Fatalf("segment overlap IDs = %d,%d, want 1,1", transcript.Segments[0].OverlapGroupID, transcript.Segments[1].OverlapGroupID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTrimMalformedSelectorFailsWithClearError(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTrimFullFixture(t, dir, "input.json")
|
||||||
|
output := filepath.Join(dir, "trimmed.json")
|
||||||
|
|
||||||
|
err := executeTrim(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
"--keep", "1-",
|
||||||
|
)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected malformed selector error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "invalid selector") || !strings.Contains(err.Error(), "malformed element") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTrimMalformedInputArtifactFailsClearly(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeJSONFile(t, dir, "broken.json", `{"metadata":`)
|
||||||
|
output := filepath.Join(dir, "trimmed.json")
|
||||||
|
|
||||||
|
err := executeTrim(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
"--keep", "1",
|
||||||
|
)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected malformed artifact error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "input JSON is malformed") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTrimDuplicateInputSegmentIDsFail(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTrimMinimalWithIDsFixture(t, dir, "input-dup.json", []int{1, 1})
|
||||||
|
output := filepath.Join(dir, "trimmed.json")
|
||||||
|
|
||||||
|
err := executeTrim(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
"--keep", "1",
|
||||||
|
)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected duplicate segment ID failure")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "not a valid seriatim output artifact") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTrimNonSequentialInputSegmentIDsFail(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTrimMinimalWithIDsFixture(t, dir, "input-nonseq.json", []int{1, 3})
|
||||||
|
output := filepath.Join(dir, "trimmed.json")
|
||||||
|
|
||||||
|
err := executeTrim(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
"--keep", "1",
|
||||||
|
)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected non-sequential segment ID failure")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "not a valid seriatim output artifact") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTrimKeepSelectorWithOverlappingRanges(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTrimFullFixture(t, dir, "input.json")
|
||||||
|
output := filepath.Join(dir, "trimmed.json")
|
||||||
|
|
||||||
|
err := executeTrim(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
"--keep", "1-3,2-4",
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("trim failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var transcript schema.Transcript
|
||||||
|
readJSON(t, output, &transcript)
|
||||||
|
if len(transcript.Segments) != 4 {
|
||||||
|
t.Fatalf("segment count = %d, want 4", len(transcript.Segments))
|
||||||
|
}
|
||||||
|
assertSequentialIDs(t, []int{
|
||||||
|
transcript.Segments[0].ID,
|
||||||
|
transcript.Segments[1].ID,
|
||||||
|
transcript.Segments[2].ID,
|
||||||
|
transcript.Segments[3].ID,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTrimRemoveSelectorWithOverlappingRanges(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTrimFullFixture(t, dir, "input.json")
|
||||||
|
output := filepath.Join(dir, "trimmed.json")
|
||||||
|
|
||||||
|
err := executeTrim(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
"--remove", "2-3,3-4",
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("trim failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var transcript schema.Transcript
|
||||||
|
readJSON(t, output, &transcript)
|
||||||
|
if len(transcript.Segments) != 1 {
|
||||||
|
t.Fatalf("segment count = %d, want 1", len(transcript.Segments))
|
||||||
|
}
|
||||||
|
if transcript.Segments[0].Text != "one" {
|
||||||
|
t.Fatalf("remaining segment = %#v, want one", transcript.Segments[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTrimSelectorOrderDoesNotAffectTranscriptOrder(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTrimFullFixture(t, dir, "input.json")
|
||||||
|
output := filepath.Join(dir, "trimmed.json")
|
||||||
|
|
||||||
|
err := executeTrim(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
"--keep", "4,1,3",
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("trim failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var transcript schema.Transcript
|
||||||
|
readJSON(t, output, &transcript)
|
||||||
|
if len(transcript.Segments) != 3 {
|
||||||
|
t.Fatalf("segment count = %d, want 3", len(transcript.Segments))
|
||||||
|
}
|
||||||
|
got := []string{
|
||||||
|
transcript.Segments[0].Text,
|
||||||
|
transcript.Segments[1].Text,
|
||||||
|
transcript.Segments[2].Text,
|
||||||
|
}
|
||||||
|
want := []string{"one", "three", "four"}
|
||||||
|
if got[0] != want[0] || got[1] != want[1] || got[2] != want[2] {
|
||||||
|
t.Fatalf("segment text order = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTrimAllowEmptyBehavior(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTrimFullFixture(t, dir, "input.json")
|
||||||
|
output := filepath.Join(dir, "trimmed.json")
|
||||||
|
|
||||||
|
err := executeTrim(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
"--remove", "1-4",
|
||||||
|
)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected empty-output error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "empty transcript") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = executeTrim(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
"--remove", "1-4",
|
||||||
|
"--allow-empty",
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("trim with --allow-empty failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var transcript schema.Transcript
|
||||||
|
readJSON(t, output, &transcript)
|
||||||
|
if len(transcript.Segments) != 0 {
|
||||||
|
t.Fatalf("segment count = %d, want 0", len(transcript.Segments))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTrimRejectsNonSeriatimInputArtifacts(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeJSONFile(t, dir, "raw-whisperx.json", `{
|
||||||
|
"segments": [
|
||||||
|
{"start": 1, "end": 2, "text": "hello"}
|
||||||
|
]
|
||||||
|
}`)
|
||||||
|
output := filepath.Join(dir, "trimmed.json")
|
||||||
|
|
||||||
|
err := executeTrim(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
"--keep", "1",
|
||||||
|
)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected invalid artifact error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "not a valid seriatim output artifact") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTrimReportFileContainsAuditFields(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTrimFullFixture(t, dir, "input.json")
|
||||||
|
output := filepath.Join(dir, "trimmed.json")
|
||||||
|
reportPath := filepath.Join(dir, "trim-report.json")
|
||||||
|
|
||||||
|
err := executeTrim(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
"--report-file", reportPath,
|
||||||
|
"--remove", "4,2",
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("trim failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var rpt report.Report
|
||||||
|
readJSON(t, reportPath, &rpt)
|
||||||
|
if len(rpt.Events) == 0 {
|
||||||
|
t.Fatal("expected report events")
|
||||||
|
}
|
||||||
|
if !hasReportEvent(rpt, "trim", "trim", "trimmed 4 input segment(s) into 2 output segment(s) with mode=remove") {
|
||||||
|
t.Fatal("expected trim summary event")
|
||||||
|
}
|
||||||
|
if !hasReportEvent(rpt, "trim", "validate-output", "validated 2 output segment(s)") {
|
||||||
|
t.Fatal("expected validation event")
|
||||||
|
}
|
||||||
|
|
||||||
|
audit := extractTrimAuditEvent(t, rpt)
|
||||||
|
if audit.Operation != "trim" {
|
||||||
|
t.Fatalf("operation = %q, want trim", audit.Operation)
|
||||||
|
}
|
||||||
|
if audit.InputFile != input {
|
||||||
|
t.Fatalf("input_file = %q, want %q", audit.InputFile, input)
|
||||||
|
}
|
||||||
|
if audit.OutputFile != output {
|
||||||
|
t.Fatalf("output_file = %q, want %q", audit.OutputFile, output)
|
||||||
|
}
|
||||||
|
if audit.InputSchema != config.OutputSchemaFull || audit.OutputSchema != config.OutputSchemaFull {
|
||||||
|
t.Fatalf("schemas = %q -> %q, want full -> full", audit.InputSchema, audit.OutputSchema)
|
||||||
|
}
|
||||||
|
if audit.Mode != "remove" {
|
||||||
|
t.Fatalf("mode = %q, want remove", audit.Mode)
|
||||||
|
}
|
||||||
|
if audit.Selector != "4,2" {
|
||||||
|
t.Fatalf("selector = %q, want %q", audit.Selector, "4,2")
|
||||||
|
}
|
||||||
|
assertIntSliceEqual(t, audit.SelectedIDs, []int{2, 4})
|
||||||
|
if audit.AllowEmpty {
|
||||||
|
t.Fatal("allow_empty should be false")
|
||||||
|
}
|
||||||
|
if audit.InputSegmentCount != 4 || audit.RetainedSegmentCount != 2 || audit.RemovedSegmentCount != 2 {
|
||||||
|
t.Fatalf("counts = input:%d retained:%d removed:%d, want 4/2/2", audit.InputSegmentCount, audit.RetainedSegmentCount, audit.RemovedSegmentCount)
|
||||||
|
}
|
||||||
|
assertIntSliceEqual(t, audit.RemovedInputIDs, []int{2, 4})
|
||||||
|
if len(audit.OldToNewIDMapping) != 2 {
|
||||||
|
t.Fatalf("mapping length = %d, want 2", len(audit.OldToNewIDMapping))
|
||||||
|
}
|
||||||
|
if audit.OldToNewIDMapping[0].OldID != 1 || audit.OldToNewIDMapping[0].NewID != 1 {
|
||||||
|
t.Fatalf("mapping[0] = %#v, want old_id=1 new_id=1", audit.OldToNewIDMapping[0])
|
||||||
|
}
|
||||||
|
if audit.OldToNewIDMapping[1].OldID != 3 || audit.OldToNewIDMapping[1].NewID != 2 {
|
||||||
|
t.Fatalf("mapping[1] = %#v, want old_id=3 new_id=2", audit.OldToNewIDMapping[1])
|
||||||
|
}
|
||||||
|
if !audit.OverlapGroupsRecomputed {
|
||||||
|
t.Fatal("expected overlap_groups_recomputed=true for full schema trim")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTrimReportOldToNewMappingIsDeterministicSorted(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTrimFullFixture(t, dir, "input.json")
|
||||||
|
output := filepath.Join(dir, "trimmed.json")
|
||||||
|
reportPath := filepath.Join(dir, "trim-report.json")
|
||||||
|
|
||||||
|
err := executeTrim(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
"--report-file", reportPath,
|
||||||
|
"--keep", "4,1,3",
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("trim failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var rpt report.Report
|
||||||
|
readJSON(t, reportPath, &rpt)
|
||||||
|
audit := extractTrimAuditEvent(t, rpt)
|
||||||
|
if len(audit.OldToNewIDMapping) != 3 {
|
||||||
|
t.Fatalf("mapping length = %d, want 3", len(audit.OldToNewIDMapping))
|
||||||
|
}
|
||||||
|
for index, expectedOld := range []int{1, 3, 4} {
|
||||||
|
if audit.OldToNewIDMapping[index].OldID != expectedOld {
|
||||||
|
t.Fatalf("mapping[%d].old_id = %d, want %d", index, audit.OldToNewIDMapping[index].OldID, expectedOld)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTrimNoReportFileWhenOmitted(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTrimFullFixture(t, dir, "input.json")
|
||||||
|
output := filepath.Join(dir, "trimmed.json")
|
||||||
|
reportPath := filepath.Join(dir, "trim-report.json")
|
||||||
|
|
||||||
|
err := executeTrim(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
"--keep", "1",
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("trim failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, statErr := os.Stat(reportPath)
|
||||||
|
if !os.IsNotExist(statErr) {
|
||||||
|
t.Fatalf("expected no report file at %q, got err=%v", reportPath, statErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func executeTrim(args ...string) error {
|
||||||
|
cmd := NewRootCommand()
|
||||||
|
cmd.SetArgs(append([]string{"trim"}, args...))
|
||||||
|
return cmd.Execute()
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeTrimFullFixture(t *testing.T, dir string, name string) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
first := 10
|
||||||
|
second := 20
|
||||||
|
third := 30
|
||||||
|
fourth := 40
|
||||||
|
value := schema.Transcript{
|
||||||
|
Metadata: schema.Metadata{
|
||||||
|
Application: "seriatim",
|
||||||
|
Version: "v-test",
|
||||||
|
InputReader: "json-files",
|
||||||
|
InputFiles: []string{"a.json"},
|
||||||
|
PreprocessingModules: []string{"validate-raw"},
|
||||||
|
PostprocessingModules: []string{"assign-ids"},
|
||||||
|
OutputModules: []string{"json"},
|
||||||
|
},
|
||||||
|
Segments: []schema.Segment{
|
||||||
|
{ID: 1, Source: "a.json", SourceSegmentIndex: &first, SourceRef: "a.json#10", Speaker: "A", Start: 1, End: 2, Text: "one", OverlapGroupID: 9},
|
||||||
|
{ID: 2, Source: "a.json", SourceSegmentIndex: &second, SourceRef: "a.json#20", Speaker: "B", Start: 2, End: 3, Text: "two", OverlapGroupID: 9},
|
||||||
|
{ID: 3, Source: "a.json", SourceSegmentIndex: &third, SourceRef: "a.json#30", Speaker: "C", Start: 4, End: 5, Text: "three", OverlapGroupID: 10},
|
||||||
|
{ID: 4, Source: "a.json", SourceSegmentIndex: &fourth, SourceRef: "a.json#40", Speaker: "D", Start: 5, End: 6, Text: "four", OverlapGroupID: 10},
|
||||||
|
},
|
||||||
|
OverlapGroups: []schema.OverlapGroup{
|
||||||
|
{ID: 9, Start: 1, End: 3, Segments: []string{"a.json#10", "a.json#20"}, Speakers: []string{"A", "B"}, Class: "unknown", Resolution: "unresolved"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
return writeTrimArtifactFile(t, dir, name, value)
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeTrimMinimalFixture(t *testing.T, dir string, name string) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
value := schema.MinimalTranscript{
|
||||||
|
Metadata: schema.MinimalMetadata{
|
||||||
|
Application: "seriatim",
|
||||||
|
Version: "v-test",
|
||||||
|
OutputSchema: config.OutputSchemaMinimal,
|
||||||
|
},
|
||||||
|
Segments: []schema.MinimalSegment{
|
||||||
|
{ID: 1, Start: 1, End: 2, Speaker: "A", Text: "one"},
|
||||||
|
{ID: 2, Start: 2, End: 3, Speaker: "B", Text: "two"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
return writeTrimArtifactFile(t, dir, name, value)
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeTrimIntermediateFixture(t *testing.T, dir string, name string) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
value := schema.IntermediateTranscript{
|
||||||
|
Metadata: schema.IntermediateMetadata{
|
||||||
|
Application: "seriatim",
|
||||||
|
Version: "v-test",
|
||||||
|
OutputSchema: config.OutputSchemaIntermediate,
|
||||||
|
},
|
||||||
|
Segments: []schema.IntermediateSegment{
|
||||||
|
{ID: 1, Start: 1, End: 2, Speaker: "A", Text: "one", Categories: []string{"word-run"}},
|
||||||
|
{ID: 2, Start: 2, End: 3, Speaker: "B", Text: "two", Categories: []string{"filler", "backchannel"}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
return writeTrimArtifactFile(t, dir, name, value)
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeTrimMinimalWithIDsFixture(t *testing.T, dir string, name string, ids []int) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
if len(ids) < 2 {
|
||||||
|
t.Fatalf("need at least two IDs, got %d", len(ids))
|
||||||
|
}
|
||||||
|
value := schema.MinimalTranscript{
|
||||||
|
Metadata: schema.MinimalMetadata{
|
||||||
|
Application: "seriatim",
|
||||||
|
Version: "v-test",
|
||||||
|
OutputSchema: config.OutputSchemaMinimal,
|
||||||
|
},
|
||||||
|
Segments: []schema.MinimalSegment{
|
||||||
|
{ID: ids[0], Start: 1, End: 2, Speaker: "A", Text: "one"},
|
||||||
|
{ID: ids[1], Start: 2, End: 3, Speaker: "B", Text: "two"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
return writeTrimArtifactFile(t, dir, name, value)
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeTrimFullOverlapFixture(t *testing.T, dir string, name string) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
first := 10
|
||||||
|
second := 20
|
||||||
|
third := 30
|
||||||
|
value := schema.Transcript{
|
||||||
|
Metadata: schema.Metadata{
|
||||||
|
Application: "seriatim",
|
||||||
|
Version: "v-test",
|
||||||
|
InputReader: "json-files",
|
||||||
|
InputFiles: []string{"a.json"},
|
||||||
|
PreprocessingModules: []string{"validate-raw"},
|
||||||
|
PostprocessingModules: []string{"detect-overlaps", "assign-ids"},
|
||||||
|
OutputModules: []string{"json"},
|
||||||
|
},
|
||||||
|
Segments: []schema.Segment{
|
||||||
|
{ID: 1, Source: "a.json", SourceSegmentIndex: &first, SourceRef: "a.json#10", Speaker: "A", Start: 1, End: 3, Text: "one", OverlapGroupID: 5},
|
||||||
|
{ID: 2, Source: "a.json", SourceSegmentIndex: &second, SourceRef: "a.json#20", Speaker: "B", Start: 2, End: 4, Text: "two", OverlapGroupID: 5},
|
||||||
|
{ID: 3, Source: "a.json", SourceSegmentIndex: &third, SourceRef: "a.json#30", Speaker: "C", Start: 6, End: 7, Text: "three", OverlapGroupID: 6},
|
||||||
|
},
|
||||||
|
OverlapGroups: []schema.OverlapGroup{
|
||||||
|
{ID: 99, Start: 0, End: 100, Segments: []string{"stale"}, Speakers: []string{"stale"}, Class: "unknown", Resolution: "unresolved"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
return writeTrimArtifactFile(t, dir, name, value)
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeTrimArtifactFile(t *testing.T, dir string, name string, value any) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
data, err := json.MarshalIndent(value, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal fixture: %v", err)
|
||||||
|
}
|
||||||
|
path := filepath.Join(dir, name)
|
||||||
|
if err := os.WriteFile(path, append(data, '\n'), 0o600); err != nil {
|
||||||
|
t.Fatalf("write fixture: %v", err)
|
||||||
|
}
|
||||||
|
return path
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertSequentialIDs(t *testing.T, ids []int) {
|
||||||
|
t.Helper()
|
||||||
|
for index, id := range ids {
|
||||||
|
want := index + 1
|
||||||
|
if id != want {
|
||||||
|
t.Fatalf("id at index %d = %d, want %d", index, id, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractTrimAuditEvent(t *testing.T, rpt report.Report) trimAuditReport {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
for _, event := range rpt.Events {
|
||||||
|
if event.Stage == "trim" && event.Module == "trim-audit" {
|
||||||
|
var audit trimAuditReport
|
||||||
|
if err := json.Unmarshal([]byte(event.Message), &audit); err != nil {
|
||||||
|
t.Fatalf("decode trim audit event: %v", err)
|
||||||
|
}
|
||||||
|
return audit
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatal("missing trim-audit event")
|
||||||
|
return trimAuditReport{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertIntSliceEqual(t *testing.T, got []int, want []int) {
|
||||||
|
t.Helper()
|
||||||
|
if len(got) != len(want) {
|
||||||
|
t.Fatalf("slice length = %d, want %d", len(got), len(want))
|
||||||
|
}
|
||||||
|
for index := range got {
|
||||||
|
if got[index] != want[index] {
|
||||||
|
t.Fatalf("slice[%d] = %d, want %d (full got=%v, want=%v)", index, got[index], want[index], got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -160,13 +160,7 @@ func (r run) coalescedSegment(id int) model.Segment {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func segmentRef(segment model.Segment) string {
|
func segmentRef(segment model.Segment) string {
|
||||||
if segment.SourceSegmentIndex != nil {
|
return model.SegmentReference(segment)
|
||||||
return fmt.Sprintf("%s#%d", segment.Source, *segment.SourceSegmentIndex)
|
|
||||||
}
|
|
||||||
if segment.SourceRef != "" {
|
|
||||||
return segment.SourceRef
|
|
||||||
}
|
|
||||||
return segment.Source
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func isSkippableInterjection(segment model.Segment) bool {
|
func isSkippableInterjection(segment model.Segment) bool {
|
||||||
|
|||||||
@@ -8,27 +8,30 @@ import (
|
|||||||
"sort"
|
"sort"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/schema"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
DefaultInputReader = "json-files"
|
DefaultInputReader = "json-files"
|
||||||
DefaultOutputModules = "json"
|
DefaultOutputModules = "json"
|
||||||
DefaultOutputSchema = OutputSchemaDefault
|
DefaultOutputSchema = OutputSchemaIntermediate
|
||||||
DefaultPreprocessingModules = "validate-raw,normalize-speakers,trim-text"
|
DefaultPreprocessingModules = "validate-raw,normalize-speakers,trim-text"
|
||||||
DefaultPostprocessingModules = "detect-overlaps,resolve-overlaps,backchannel,filler,resolve-danglers,coalesce,detect-overlaps,autocorrect,assign-ids,validate-output"
|
DefaultPostprocessingModules = "detect-overlaps,resolve-overlaps,backchannel,filler,resolve-danglers,coalesce,detect-overlaps,autocorrect,assign-ids,validate-output"
|
||||||
DefaultOverlapWordRunGap = 0.75
|
DefaultOverlapWordRunGap = 1.0
|
||||||
DefaultWordRunReorderWindow = 1.0
|
DefaultWordRunReorderWindow = 1.0
|
||||||
DefaultCoalesceGap = 3.0
|
DefaultCoalesceGap = 3.0
|
||||||
DefaultCoalesceGapValue = "3.0"
|
DefaultCoalesceGapValue = "3.0"
|
||||||
DefaultBackchannelMaxDuration = 2.0
|
DefaultBackchannelMaxDuration = 2.0
|
||||||
DefaultFillerMaxDuration = 1.25
|
DefaultFillerMaxDuration = 1.25
|
||||||
|
OutputSchemaEnv = "SERIATIM_OUTPUT_SCHEMA"
|
||||||
OverlapWordRunGapEnv = "SERIATIM_OVERLAP_WORD_RUN_GAP"
|
OverlapWordRunGapEnv = "SERIATIM_OVERLAP_WORD_RUN_GAP"
|
||||||
WordRunReorderWindowEnv = "SERIATIM_OVERLAP_WORD_RUN_REORDER_WINDOW"
|
WordRunReorderWindowEnv = "SERIATIM_OVERLAP_WORD_RUN_REORDER_WINDOW"
|
||||||
BackchannelMaxDurationEnv = "SERIATIM_BACKCHANNEL_MAX_DURATION"
|
BackchannelMaxDurationEnv = "SERIATIM_BACKCHANNEL_MAX_DURATION"
|
||||||
FillerMaxDurationEnv = "SERIATIM_FILLER_MAX_DURATION"
|
FillerMaxDurationEnv = "SERIATIM_FILLER_MAX_DURATION"
|
||||||
OutputSchemaDefault = "default"
|
OutputSchemaMinimal = schema.OutputSchemaMinimal
|
||||||
OutputSchemaSeriatim = "seriatim"
|
OutputSchemaIntermediate = schema.OutputSchemaIntermediate
|
||||||
OutputSchemaMinimal = "minimal"
|
OutputSchemaFull = schema.OutputSchemaFull
|
||||||
)
|
)
|
||||||
|
|
||||||
// MergeOptions captures raw CLI option values before validation.
|
// MergeOptions captures raw CLI option values before validation.
|
||||||
@@ -46,6 +49,26 @@ type MergeOptions struct {
|
|||||||
CoalesceGap string
|
CoalesceGap string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TrimOptions captures raw CLI option values before validation.
|
||||||
|
type TrimOptions struct {
|
||||||
|
InputFile string
|
||||||
|
OutputFile string
|
||||||
|
ReportFile string
|
||||||
|
Keep string
|
||||||
|
Remove string
|
||||||
|
OutputSchema string
|
||||||
|
AllowEmpty bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// NormalizeOptions captures raw CLI option values before validation.
|
||||||
|
type NormalizeOptions struct {
|
||||||
|
InputFile string
|
||||||
|
OutputFile string
|
||||||
|
ReportFile string
|
||||||
|
OutputSchema string
|
||||||
|
OutputModules string
|
||||||
|
}
|
||||||
|
|
||||||
// Config is the validated runtime configuration for a merge invocation.
|
// Config is the validated runtime configuration for a merge invocation.
|
||||||
type Config struct {
|
type Config struct {
|
||||||
InputFiles []string
|
InputFiles []string
|
||||||
@@ -65,12 +88,32 @@ type Config struct {
|
|||||||
FillerMaxDuration float64
|
FillerMaxDuration float64
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TrimConfig is the validated runtime configuration for a trim invocation.
|
||||||
|
type TrimConfig struct {
|
||||||
|
InputFile string
|
||||||
|
OutputFile string
|
||||||
|
ReportFile string
|
||||||
|
Mode string
|
||||||
|
Selector string
|
||||||
|
OutputSchema string
|
||||||
|
AllowEmpty bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// NormalizeConfig is the validated runtime configuration for a normalize invocation.
|
||||||
|
type NormalizeConfig struct {
|
||||||
|
InputFile string
|
||||||
|
OutputFile string
|
||||||
|
ReportFile string
|
||||||
|
OutputSchema string
|
||||||
|
OutputModules []string
|
||||||
|
}
|
||||||
|
|
||||||
// NewMergeConfig validates raw merge options and returns normalized config.
|
// NewMergeConfig validates raw merge options and returns normalized config.
|
||||||
func NewMergeConfig(opts MergeOptions) (Config, error) {
|
func NewMergeConfig(opts MergeOptions) (Config, error) {
|
||||||
cfg := Config{
|
cfg := Config{
|
||||||
InputReader: strings.TrimSpace(opts.InputReader),
|
InputReader: strings.TrimSpace(opts.InputReader),
|
||||||
OutputModules: nil,
|
OutputModules: nil,
|
||||||
OutputSchema: strings.TrimSpace(opts.OutputSchema),
|
OutputSchema: "",
|
||||||
PreprocessingModules: nil,
|
PreprocessingModules: nil,
|
||||||
PostprocessingModules: nil,
|
PostprocessingModules: nil,
|
||||||
OverlapWordRunGap: DefaultOverlapWordRunGap,
|
OverlapWordRunGap: DefaultOverlapWordRunGap,
|
||||||
@@ -83,14 +126,12 @@ func NewMergeConfig(opts MergeOptions) (Config, error) {
|
|||||||
if cfg.InputReader == "" {
|
if cfg.InputReader == "" {
|
||||||
return Config{}, errors.New("--input-reader is required")
|
return Config{}, errors.New("--input-reader is required")
|
||||||
}
|
}
|
||||||
if cfg.OutputSchema == "" {
|
var err error
|
||||||
cfg.OutputSchema = DefaultOutputSchema
|
cfg.OutputSchema, err = resolveOutputSchema(opts.OutputSchema)
|
||||||
}
|
if err != nil {
|
||||||
if err := validateOutputSchema(cfg.OutputSchema); err != nil {
|
|
||||||
return Config{}, err
|
return Config{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
var err error
|
|
||||||
cfg.OutputModules, err = parseModuleList(opts.OutputModules)
|
cfg.OutputModules, err = parseModuleList(opts.OutputModules)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Config{}, fmt.Errorf("--output-modules: %w", err)
|
return Config{}, fmt.Errorf("--output-modules: %w", err)
|
||||||
@@ -169,6 +210,99 @@ func NewMergeConfig(opts MergeOptions) (Config, error) {
|
|||||||
return cfg, nil
|
return cfg, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NewTrimConfig validates raw trim options and returns normalized config.
|
||||||
|
func NewTrimConfig(opts TrimOptions) (TrimConfig, error) {
|
||||||
|
inputFile, err := normalizeSingleInputFile(opts.InputFile, "--input-file")
|
||||||
|
if err != nil {
|
||||||
|
return TrimConfig{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
outputFile, err := normalizeOutputPath(opts.OutputFile, "--output-file")
|
||||||
|
if err != nil {
|
||||||
|
return TrimConfig{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
reportFile, err := normalizeOptionalOutputPath(opts.ReportFile, "--report-file")
|
||||||
|
if err != nil {
|
||||||
|
return TrimConfig{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
keep := strings.TrimSpace(opts.Keep)
|
||||||
|
remove := strings.TrimSpace(opts.Remove)
|
||||||
|
if keep == "" && remove == "" {
|
||||||
|
return TrimConfig{}, errors.New("exactly one of --keep or --remove is required")
|
||||||
|
}
|
||||||
|
if keep != "" && remove != "" {
|
||||||
|
return TrimConfig{}, errors.New("--keep and --remove are mutually exclusive")
|
||||||
|
}
|
||||||
|
|
||||||
|
mode := "keep"
|
||||||
|
selector := keep
|
||||||
|
if remove != "" {
|
||||||
|
mode = "remove"
|
||||||
|
selector = remove
|
||||||
|
}
|
||||||
|
|
||||||
|
outputSchema := strings.TrimSpace(opts.OutputSchema)
|
||||||
|
if outputSchema != "" {
|
||||||
|
if err := validateOutputSchema(outputSchema); err != nil {
|
||||||
|
return TrimConfig{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return TrimConfig{
|
||||||
|
InputFile: inputFile,
|
||||||
|
OutputFile: outputFile,
|
||||||
|
ReportFile: reportFile,
|
||||||
|
Mode: mode,
|
||||||
|
Selector: selector,
|
||||||
|
OutputSchema: outputSchema,
|
||||||
|
AllowEmpty: opts.AllowEmpty,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewNormalizeConfig validates raw normalize options and returns normalized config.
|
||||||
|
func NewNormalizeConfig(opts NormalizeOptions) (NormalizeConfig, error) {
|
||||||
|
inputFile, err := normalizeSingleInputFile(opts.InputFile, "--input-file")
|
||||||
|
if err != nil {
|
||||||
|
return NormalizeConfig{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
outputFile, err := normalizeOutputPath(opts.OutputFile, "--output-file")
|
||||||
|
if err != nil {
|
||||||
|
return NormalizeConfig{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
reportFile, err := normalizeOptionalOutputPath(opts.ReportFile, "--report-file")
|
||||||
|
if err != nil {
|
||||||
|
return NormalizeConfig{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
outputSchema, err := resolveOutputSchema(opts.OutputSchema)
|
||||||
|
if err != nil {
|
||||||
|
return NormalizeConfig{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
outputModules, err := parseModuleList(opts.OutputModules)
|
||||||
|
if err != nil {
|
||||||
|
return NormalizeConfig{}, fmt.Errorf("--output-modules: %w", err)
|
||||||
|
}
|
||||||
|
if len(outputModules) == 0 {
|
||||||
|
return NormalizeConfig{}, errors.New("--output-modules must include at least one module")
|
||||||
|
}
|
||||||
|
if err := validateNormalizeOutputModules(outputModules); err != nil {
|
||||||
|
return NormalizeConfig{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return NormalizeConfig{
|
||||||
|
InputFile: inputFile,
|
||||||
|
OutputFile: outputFile,
|
||||||
|
ReportFile: reportFile,
|
||||||
|
OutputSchema: outputSchema,
|
||||||
|
OutputModules: outputModules,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
func parseModuleList(value string) ([]string, error) {
|
func parseModuleList(value string) ([]string, error) {
|
||||||
value = strings.TrimSpace(value)
|
value = strings.TrimSpace(value)
|
||||||
if value == "" {
|
if value == "" {
|
||||||
@@ -188,12 +322,26 @@ func parseModuleList(value string) ([]string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func validateOutputSchema(value string) error {
|
func validateOutputSchema(value string) error {
|
||||||
switch value {
|
if schema.ValidOutputSchemaName(value) {
|
||||||
case OutputSchemaDefault, OutputSchemaSeriatim, OutputSchemaMinimal:
|
|
||||||
return nil
|
return nil
|
||||||
default:
|
|
||||||
return fmt.Errorf("--output-schema must be one of %q, %q, or %q", OutputSchemaDefault, OutputSchemaMinimal, OutputSchemaSeriatim)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
names := schema.OutputSchemaNames()
|
||||||
|
return fmt.Errorf("--output-schema must be one of %q, %q, or %q", names[0], names[1], names[2])
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveOutputSchema(value string) (string, error) {
|
||||||
|
value = strings.TrimSpace(value)
|
||||||
|
if value == "" {
|
||||||
|
value = strings.TrimSpace(os.Getenv(OutputSchemaEnv))
|
||||||
|
}
|
||||||
|
if value == "" {
|
||||||
|
value = DefaultOutputSchema
|
||||||
|
}
|
||||||
|
if err := validateOutputSchema(value); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return value, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func normalizeInputFiles(paths []string) ([]string, error) {
|
func normalizeInputFiles(paths []string) ([]string, error) {
|
||||||
@@ -223,6 +371,26 @@ func normalizeInputFiles(paths []string) ([]string, error) {
|
|||||||
return normalized, nil
|
return normalized, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func normalizeSingleInputFile(path string, flag string) (string, error) {
|
||||||
|
path = strings.TrimSpace(path)
|
||||||
|
if path == "" {
|
||||||
|
return "", fmt.Errorf("%s is required", flag)
|
||||||
|
}
|
||||||
|
|
||||||
|
clean := filepath.Clean(path)
|
||||||
|
if err := requireFile(clean, flag); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return clean, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeOptionalOutputPath(path string, flag string) (string, error) {
|
||||||
|
if strings.TrimSpace(path) == "" {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
return normalizeOutputPath(path, flag)
|
||||||
|
}
|
||||||
|
|
||||||
func normalizeOutputPath(path string, flag string) (string, error) {
|
func normalizeOutputPath(path string, flag string) (string, error) {
|
||||||
path = strings.TrimSpace(path)
|
path = strings.TrimSpace(path)
|
||||||
if path == "" {
|
if path == "" {
|
||||||
@@ -308,3 +476,12 @@ func contains(values []string, target string) bool {
|
|||||||
}
|
}
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validateNormalizeOutputModules(modules []string) error {
|
||||||
|
for _, module := range modules {
|
||||||
|
if module != "json" {
|
||||||
|
return fmt.Errorf("unknown output module %q", module)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -46,7 +46,8 @@ func TestDuplicateInputFilesFailValidation(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestOutputSchemaDefaultsToDefault(t *testing.T) {
|
func TestOutputSchemaDefaultsToIntermediate(t *testing.T) {
|
||||||
|
t.Setenv(OutputSchemaEnv, "")
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
input := writeTempFile(t, dir, "input.json")
|
input := writeTempFile(t, dir, "input.json")
|
||||||
output := filepath.Join(dir, "merged.json")
|
output := filepath.Join(dir, "merged.json")
|
||||||
@@ -67,7 +68,8 @@ func TestOutputSchemaDefaultsToDefault(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestOutputSchemaAcceptsDefault(t *testing.T) {
|
func TestOutputSchemaAcceptsIntermediate(t *testing.T) {
|
||||||
|
t.Setenv(OutputSchemaEnv, "")
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
input := writeTempFile(t, dir, "input.json")
|
input := writeTempFile(t, dir, "input.json")
|
||||||
output := filepath.Join(dir, "merged.json")
|
output := filepath.Join(dir, "merged.json")
|
||||||
@@ -77,19 +79,20 @@ func TestOutputSchemaAcceptsDefault(t *testing.T) {
|
|||||||
OutputFile: output,
|
OutputFile: output,
|
||||||
InputReader: DefaultInputReader,
|
InputReader: DefaultInputReader,
|
||||||
OutputModules: DefaultOutputModules,
|
OutputModules: DefaultOutputModules,
|
||||||
OutputSchema: OutputSchemaDefault,
|
OutputSchema: OutputSchemaIntermediate,
|
||||||
PreprocessingModules: DefaultPreprocessingModules,
|
PreprocessingModules: DefaultPreprocessingModules,
|
||||||
PostprocessingModules: DefaultPostprocessingModules,
|
PostprocessingModules: DefaultPostprocessingModules,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("config failed: %v", err)
|
t.Fatalf("config failed: %v", err)
|
||||||
}
|
}
|
||||||
if cfg.OutputSchema != OutputSchemaDefault {
|
if cfg.OutputSchema != OutputSchemaIntermediate {
|
||||||
t.Fatalf("output schema = %q, want %q", cfg.OutputSchema, OutputSchemaDefault)
|
t.Fatalf("output schema = %q, want %q", cfg.OutputSchema, OutputSchemaIntermediate)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestOutputSchemaAcceptsMinimal(t *testing.T) {
|
func TestOutputSchemaAcceptsMinimal(t *testing.T) {
|
||||||
|
t.Setenv(OutputSchemaEnv, "")
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
input := writeTempFile(t, dir, "input.json")
|
input := writeTempFile(t, dir, "input.json")
|
||||||
output := filepath.Join(dir, "merged.json")
|
output := filepath.Join(dir, "merged.json")
|
||||||
@@ -111,7 +114,8 @@ func TestOutputSchemaAcceptsMinimal(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestOutputSchemaAcceptsSeriatim(t *testing.T) {
|
func TestOutputSchemaAcceptsFull(t *testing.T) {
|
||||||
|
t.Setenv(OutputSchemaEnv, "")
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
input := writeTempFile(t, dir, "input.json")
|
input := writeTempFile(t, dir, "input.json")
|
||||||
output := filepath.Join(dir, "merged.json")
|
output := filepath.Join(dir, "merged.json")
|
||||||
@@ -121,19 +125,87 @@ func TestOutputSchemaAcceptsSeriatim(t *testing.T) {
|
|||||||
OutputFile: output,
|
OutputFile: output,
|
||||||
InputReader: DefaultInputReader,
|
InputReader: DefaultInputReader,
|
||||||
OutputModules: DefaultOutputModules,
|
OutputModules: DefaultOutputModules,
|
||||||
OutputSchema: OutputSchemaSeriatim,
|
OutputSchema: OutputSchemaFull,
|
||||||
PreprocessingModules: DefaultPreprocessingModules,
|
PreprocessingModules: DefaultPreprocessingModules,
|
||||||
PostprocessingModules: DefaultPostprocessingModules,
|
PostprocessingModules: DefaultPostprocessingModules,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("config failed: %v", err)
|
t.Fatalf("config failed: %v", err)
|
||||||
}
|
}
|
||||||
if cfg.OutputSchema != OutputSchemaSeriatim {
|
if cfg.OutputSchema != OutputSchemaFull {
|
||||||
t.Fatalf("output schema = %q, want %q", cfg.OutputSchema, OutputSchemaSeriatim)
|
t.Fatalf("output schema = %q, want %q", cfg.OutputSchema, OutputSchemaFull)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOutputSchemaUsesEnvWhenFlagOmitted(t *testing.T) {
|
||||||
|
t.Setenv(OutputSchemaEnv, OutputSchemaFull)
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTempFile(t, dir, "input.json")
|
||||||
|
output := filepath.Join(dir, "merged.json")
|
||||||
|
|
||||||
|
cfg, err := NewMergeConfig(MergeOptions{
|
||||||
|
InputFiles: []string{input},
|
||||||
|
OutputFile: output,
|
||||||
|
InputReader: DefaultInputReader,
|
||||||
|
OutputModules: DefaultOutputModules,
|
||||||
|
PreprocessingModules: DefaultPreprocessingModules,
|
||||||
|
PostprocessingModules: DefaultPostprocessingModules,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config failed: %v", err)
|
||||||
|
}
|
||||||
|
if cfg.OutputSchema != OutputSchemaFull {
|
||||||
|
t.Fatalf("output schema = %q, want %q", cfg.OutputSchema, OutputSchemaFull)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOutputSchemaFlagOverridesEnv(t *testing.T) {
|
||||||
|
t.Setenv(OutputSchemaEnv, OutputSchemaFull)
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTempFile(t, dir, "input.json")
|
||||||
|
output := filepath.Join(dir, "merged.json")
|
||||||
|
|
||||||
|
cfg, err := NewMergeConfig(MergeOptions{
|
||||||
|
InputFiles: []string{input},
|
||||||
|
OutputFile: output,
|
||||||
|
InputReader: DefaultInputReader,
|
||||||
|
OutputModules: DefaultOutputModules,
|
||||||
|
OutputSchema: OutputSchemaMinimal,
|
||||||
|
PreprocessingModules: DefaultPreprocessingModules,
|
||||||
|
PostprocessingModules: DefaultPostprocessingModules,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config failed: %v", err)
|
||||||
|
}
|
||||||
|
if cfg.OutputSchema != OutputSchemaMinimal {
|
||||||
|
t.Fatalf("output schema = %q, want %q", cfg.OutputSchema, OutputSchemaMinimal)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOutputSchemaRejectsInvalidEnvValue(t *testing.T) {
|
||||||
|
t.Setenv(OutputSchemaEnv, "compact")
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTempFile(t, dir, "input.json")
|
||||||
|
output := filepath.Join(dir, "merged.json")
|
||||||
|
|
||||||
|
_, err := NewMergeConfig(MergeOptions{
|
||||||
|
InputFiles: []string{input},
|
||||||
|
OutputFile: output,
|
||||||
|
InputReader: DefaultInputReader,
|
||||||
|
OutputModules: DefaultOutputModules,
|
||||||
|
PreprocessingModules: DefaultPreprocessingModules,
|
||||||
|
PostprocessingModules: DefaultPostprocessingModules,
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected output schema error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "--output-schema must be one of") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestOutputSchemaRejectsUnknownValue(t *testing.T) {
|
func TestOutputSchemaRejectsUnknownValue(t *testing.T) {
|
||||||
|
t.Setenv(OutputSchemaEnv, "")
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
input := writeTempFile(t, dir, "input.json")
|
input := writeTempFile(t, dir, "input.json")
|
||||||
output := filepath.Join(dir, "merged.json")
|
output := filepath.Join(dir, "merged.json")
|
||||||
@@ -155,7 +227,36 @@ func TestOutputSchemaRejectsUnknownValue(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestOverlapWordRunGapDefaultsTo075(t *testing.T) {
|
func TestOutputSchemaRejectsLegacyValues(t *testing.T) {
|
||||||
|
tests := []string{"default", "minimal", "seriatim"}
|
||||||
|
|
||||||
|
for _, legacy := range tests {
|
||||||
|
t.Run(legacy, func(t *testing.T) {
|
||||||
|
t.Setenv(OutputSchemaEnv, "")
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTempFile(t, dir, "input.json")
|
||||||
|
output := filepath.Join(dir, "merged.json")
|
||||||
|
|
||||||
|
_, err := NewMergeConfig(MergeOptions{
|
||||||
|
InputFiles: []string{input},
|
||||||
|
OutputFile: output,
|
||||||
|
InputReader: DefaultInputReader,
|
||||||
|
OutputModules: DefaultOutputModules,
|
||||||
|
OutputSchema: legacy,
|
||||||
|
PreprocessingModules: DefaultPreprocessingModules,
|
||||||
|
PostprocessingModules: DefaultPostprocessingModules,
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected output schema error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "--output-schema must be one of") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOverlapWordRunGapDefaultsTo1(t *testing.T) {
|
||||||
t.Setenv(OverlapWordRunGapEnv, "")
|
t.Setenv(OverlapWordRunGapEnv, "")
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
input := writeTempFile(t, dir, "input.json")
|
input := writeTempFile(t, dir, "input.json")
|
||||||
@@ -437,15 +538,9 @@ func TestCoalesceGapUsesValidOverride(t *testing.T) {
|
|||||||
input := writeTempFile(t, dir, "input.json")
|
input := writeTempFile(t, dir, "input.json")
|
||||||
output := filepath.Join(dir, "merged.json")
|
output := filepath.Join(dir, "merged.json")
|
||||||
|
|
||||||
cfg, err := NewMergeConfig(MergeOptions{
|
opts := validMergeOptions(input, output)
|
||||||
InputFiles: []string{input},
|
opts.CoalesceGap = "1.5"
|
||||||
OutputFile: output,
|
cfg, err := NewMergeConfig(opts)
|
||||||
InputReader: DefaultInputReader,
|
|
||||||
OutputModules: DefaultOutputModules,
|
|
||||||
PreprocessingModules: DefaultPreprocessingModules,
|
|
||||||
PostprocessingModules: DefaultPostprocessingModules,
|
|
||||||
CoalesceGap: "1.5",
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("config failed: %v", err)
|
t.Fatalf("config failed: %v", err)
|
||||||
}
|
}
|
||||||
@@ -459,15 +554,9 @@ func TestCoalesceGapAllowsZero(t *testing.T) {
|
|||||||
input := writeTempFile(t, dir, "input.json")
|
input := writeTempFile(t, dir, "input.json")
|
||||||
output := filepath.Join(dir, "merged.json")
|
output := filepath.Join(dir, "merged.json")
|
||||||
|
|
||||||
cfg, err := NewMergeConfig(MergeOptions{
|
opts := validMergeOptions(input, output)
|
||||||
InputFiles: []string{input},
|
opts.CoalesceGap = "0"
|
||||||
OutputFile: output,
|
cfg, err := NewMergeConfig(opts)
|
||||||
InputReader: DefaultInputReader,
|
|
||||||
OutputModules: DefaultOutputModules,
|
|
||||||
PreprocessingModules: DefaultPreprocessingModules,
|
|
||||||
PostprocessingModules: DefaultPostprocessingModules,
|
|
||||||
CoalesceGap: "0",
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("config failed: %v", err)
|
t.Fatalf("config failed: %v", err)
|
||||||
}
|
}
|
||||||
@@ -492,15 +581,9 @@ func TestCoalesceGapRejectsInvalidOverride(t *testing.T) {
|
|||||||
input := writeTempFile(t, dir, "input.json")
|
input := writeTempFile(t, dir, "input.json")
|
||||||
output := filepath.Join(dir, "merged.json")
|
output := filepath.Join(dir, "merged.json")
|
||||||
|
|
||||||
_, err := NewMergeConfig(MergeOptions{
|
opts := validMergeOptions(input, output)
|
||||||
InputFiles: []string{input},
|
opts.CoalesceGap = test.value
|
||||||
OutputFile: output,
|
_, err := NewMergeConfig(opts)
|
||||||
InputReader: DefaultInputReader,
|
|
||||||
OutputModules: DefaultOutputModules,
|
|
||||||
PreprocessingModules: DefaultPreprocessingModules,
|
|
||||||
PostprocessingModules: DefaultPostprocessingModules,
|
|
||||||
CoalesceGap: test.value,
|
|
||||||
})
|
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected error")
|
t.Fatal("expected error")
|
||||||
}
|
}
|
||||||
@@ -511,6 +594,216 @@ func TestCoalesceGapRejectsInvalidOverride(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestNewTrimConfigRequiresInputAndOutput(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTempFile(t, dir, "input.json")
|
||||||
|
output := filepath.Join(dir, "trimmed.json")
|
||||||
|
|
||||||
|
_, err := NewTrimConfig(TrimOptions{
|
||||||
|
OutputFile: output,
|
||||||
|
Keep: "1",
|
||||||
|
})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "--input-file is required") {
|
||||||
|
t.Fatalf("expected input-file required error, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = NewTrimConfig(TrimOptions{
|
||||||
|
InputFile: input,
|
||||||
|
Keep: "1",
|
||||||
|
})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "--output-file is required") {
|
||||||
|
t.Fatalf("expected output-file required error, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewTrimConfigRequiresExactlyOneSelectorFlag(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTempFile(t, dir, "input.json")
|
||||||
|
output := filepath.Join(dir, "trimmed.json")
|
||||||
|
|
||||||
|
opts := validTrimOptions(input, output)
|
||||||
|
opts.Keep = ""
|
||||||
|
_, err := NewTrimConfig(opts)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "exactly one of --keep or --remove is required") {
|
||||||
|
t.Fatalf("expected missing selector error, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
opts = validTrimOptions(input, output)
|
||||||
|
opts.Remove = "2"
|
||||||
|
_, err = NewTrimConfig(opts)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "mutually exclusive") {
|
||||||
|
t.Fatalf("expected mutually exclusive selector error, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewTrimConfigAcceptsOutputSchemaOverride(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTempFile(t, dir, "input.json")
|
||||||
|
output := filepath.Join(dir, "trimmed.json")
|
||||||
|
reportPath := filepath.Join(dir, "report.json")
|
||||||
|
|
||||||
|
opts := validTrimOptions(input, output)
|
||||||
|
opts.Keep = ""
|
||||||
|
opts.Remove = "3-5"
|
||||||
|
opts.ReportFile = reportPath
|
||||||
|
opts.OutputSchema = OutputSchemaMinimal
|
||||||
|
opts.AllowEmpty = true
|
||||||
|
cfg, err := NewTrimConfig(opts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config failed: %v", err)
|
||||||
|
}
|
||||||
|
if cfg.Mode != "remove" {
|
||||||
|
t.Fatalf("mode = %q, want remove", cfg.Mode)
|
||||||
|
}
|
||||||
|
if cfg.Selector != "3-5" {
|
||||||
|
t.Fatalf("selector = %q, want 3-5", cfg.Selector)
|
||||||
|
}
|
||||||
|
if cfg.OutputSchema != OutputSchemaMinimal {
|
||||||
|
t.Fatalf("output schema = %q, want %q", cfg.OutputSchema, OutputSchemaMinimal)
|
||||||
|
}
|
||||||
|
if !cfg.AllowEmpty {
|
||||||
|
t.Fatal("allow empty should be true")
|
||||||
|
}
|
||||||
|
if cfg.ReportFile != reportPath {
|
||||||
|
t.Fatalf("report file = %q, want %q", cfg.ReportFile, reportPath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewTrimConfigTreatsWhitespaceReportFileAsOmitted(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTempFile(t, dir, "input.json")
|
||||||
|
output := filepath.Join(dir, "trimmed.json")
|
||||||
|
|
||||||
|
opts := validTrimOptions(input, output)
|
||||||
|
opts.ReportFile = " \t "
|
||||||
|
cfg, err := NewTrimConfig(opts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config failed: %v", err)
|
||||||
|
}
|
||||||
|
if cfg.ReportFile != "" {
|
||||||
|
t.Fatalf("report file = %q, want empty", cfg.ReportFile)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewTrimConfigRejectsInvalidOutputSchemaOverride(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTempFile(t, dir, "input.json")
|
||||||
|
output := filepath.Join(dir, "trimmed.json")
|
||||||
|
|
||||||
|
opts := validTrimOptions(input, output)
|
||||||
|
opts.OutputSchema = "compact"
|
||||||
|
_, err := NewTrimConfig(opts)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected output schema validation error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "--output-schema must be one of") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewNormalizeConfigRequiresInputFile(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
output := filepath.Join(dir, "normalized.json")
|
||||||
|
|
||||||
|
_, err := NewNormalizeConfig(NormalizeOptions{
|
||||||
|
OutputFile: output,
|
||||||
|
OutputModules: DefaultOutputModules,
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected input-file required error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "--input-file is required") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewNormalizeConfigRequiresOutputFile(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTempFile(t, dir, "input.json")
|
||||||
|
|
||||||
|
opts := validNormalizeOptions(input, "")
|
||||||
|
_, err := NewNormalizeConfig(opts)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected output-file required error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "--output-file is required") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewNormalizeConfigResolvesOutputSchemaDefaultAndEnv(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTempFile(t, dir, "input.json")
|
||||||
|
output := filepath.Join(dir, "normalized.json")
|
||||||
|
|
||||||
|
t.Setenv(OutputSchemaEnv, "")
|
||||||
|
opts := validNormalizeOptions(input, output)
|
||||||
|
cfg, err := NewNormalizeConfig(opts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config failed: %v", err)
|
||||||
|
}
|
||||||
|
if cfg.OutputSchema != DefaultOutputSchema {
|
||||||
|
t.Fatalf("output schema = %q, want %q", cfg.OutputSchema, DefaultOutputSchema)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Setenv(OutputSchemaEnv, OutputSchemaMinimal)
|
||||||
|
cfg, err = NewNormalizeConfig(opts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config failed: %v", err)
|
||||||
|
}
|
||||||
|
if cfg.OutputSchema != OutputSchemaMinimal {
|
||||||
|
t.Fatalf("output schema = %q, want %q", cfg.OutputSchema, OutputSchemaMinimal)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewNormalizeConfigRejectsInvalidOutputSchema(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTempFile(t, dir, "input.json")
|
||||||
|
output := filepath.Join(dir, "normalized.json")
|
||||||
|
|
||||||
|
opts := validNormalizeOptions(input, output)
|
||||||
|
opts.OutputSchema = "compact"
|
||||||
|
_, err := NewNormalizeConfig(opts)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected output schema error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "--output-schema must be one of") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewNormalizeConfigRejectsUnknownOutputModule(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTempFile(t, dir, "input.json")
|
||||||
|
output := filepath.Join(dir, "normalized.json")
|
||||||
|
|
||||||
|
opts := validNormalizeOptions(input, output)
|
||||||
|
opts.OutputModules = "json,yaml"
|
||||||
|
_, err := NewNormalizeConfig(opts)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected output module error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "unknown output module") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewNormalizeConfigTreatsWhitespaceReportFileAsOmitted(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTempFile(t, dir, "input.json")
|
||||||
|
output := filepath.Join(dir, "normalized.json")
|
||||||
|
|
||||||
|
opts := validNormalizeOptions(input, output)
|
||||||
|
opts.ReportFile = "\n\t "
|
||||||
|
cfg, err := NewNormalizeConfig(opts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config failed: %v", err)
|
||||||
|
}
|
||||||
|
if cfg.ReportFile != "" {
|
||||||
|
t.Fatalf("report file = %q, want empty", cfg.ReportFile)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func assertPositiveFloatEnvValidation(t *testing.T, envName string) {
|
func assertPositiveFloatEnvValidation(t *testing.T, envName string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -531,14 +824,7 @@ func assertPositiveFloatEnvValidation(t *testing.T, envName string) {
|
|||||||
input := writeTempFile(t, dir, "input.json")
|
input := writeTempFile(t, dir, "input.json")
|
||||||
output := filepath.Join(dir, "merged.json")
|
output := filepath.Join(dir, "merged.json")
|
||||||
|
|
||||||
_, err := NewMergeConfig(MergeOptions{
|
_, err := NewMergeConfig(validMergeOptions(input, output))
|
||||||
InputFiles: []string{input},
|
|
||||||
OutputFile: output,
|
|
||||||
InputReader: DefaultInputReader,
|
|
||||||
OutputModules: DefaultOutputModules,
|
|
||||||
PreprocessingModules: DefaultPreprocessingModules,
|
|
||||||
PostprocessingModules: DefaultPostprocessingModules,
|
|
||||||
})
|
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected error")
|
t.Fatal("expected error")
|
||||||
}
|
}
|
||||||
@@ -549,6 +835,33 @@ func assertPositiveFloatEnvValidation(t *testing.T, envName string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validMergeOptions(inputFile string, outputFile string) MergeOptions {
|
||||||
|
return MergeOptions{
|
||||||
|
InputFiles: []string{inputFile},
|
||||||
|
OutputFile: outputFile,
|
||||||
|
InputReader: DefaultInputReader,
|
||||||
|
OutputModules: DefaultOutputModules,
|
||||||
|
PreprocessingModules: DefaultPreprocessingModules,
|
||||||
|
PostprocessingModules: DefaultPostprocessingModules,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func validTrimOptions(inputFile string, outputFile string) TrimOptions {
|
||||||
|
return TrimOptions{
|
||||||
|
InputFile: inputFile,
|
||||||
|
OutputFile: outputFile,
|
||||||
|
Keep: "1",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func validNormalizeOptions(inputFile string, outputFile string) NormalizeOptions {
|
||||||
|
return NormalizeOptions{
|
||||||
|
InputFile: inputFile,
|
||||||
|
OutputFile: outputFile,
|
||||||
|
OutputModules: DefaultOutputModules,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func writeTempFile(t *testing.T, dir string, name string) string {
|
func writeTempFile(t *testing.T, dir string, name string) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
|
|||||||
@@ -184,7 +184,7 @@ func nearestPriorMatch(segments []model.Segment, consumed []bool, index int) int
|
|||||||
if consumed[candidate] {
|
if consumed[candidate] {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if sharesDerivedFrom(segments[index], segments[candidate]) {
|
if segments[index].Speaker == segments[candidate].Speaker && sharesDerivedFrom(segments[index], segments[candidate]) {
|
||||||
return candidate
|
return candidate
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -196,7 +196,7 @@ func nearestSubsequentMatch(segments []model.Segment, consumed []bool, index int
|
|||||||
if consumed[candidate] {
|
if consumed[candidate] {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if sharesDerivedFrom(segments[index], segments[candidate]) {
|
if segments[index].Speaker == segments[candidate].Speaker && sharesDerivedFrom(segments[index], segments[candidate]) {
|
||||||
return candidate
|
return candidate
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -51,6 +51,30 @@ func TestApplyUsesAnyDerivedFromIntersection(t *testing.T) {
|
|||||||
assertSegment(t, got.Segments[0], "resolve-danglers:1", "target end.", 1, 4, []string{"source#1", "source#2", "source#3"})
|
assertSegment(t, got.Segments[0], "resolve-danglers:1", "target end.", 1, 4, []string{"source#1", "source#2", "source#3"})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestApplyDoesNotMergeDanglersAcrossSpeakersBackward(t *testing.T) {
|
||||||
|
in := transcript(
|
||||||
|
segment("a", "Alice", 1, 2, "target", []string{"source#1"}),
|
||||||
|
segment("b", "Bob", 3, 4, "end.", []string{"source#1"}),
|
||||||
|
)
|
||||||
|
|
||||||
|
got, summary := Apply(in)
|
||||||
|
if summary.DanglersMerged != 0 || !reflect.DeepEqual(got, in) {
|
||||||
|
t.Fatalf("unexpected merge:\ngot %#v\nwant %#v", got, in)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyDoesNotMergeDanglersAcrossSpeakersForward(t *testing.T) {
|
||||||
|
in := transcript(
|
||||||
|
segment("a", "Alice", 1, 2, "start", []string{"source#1"}),
|
||||||
|
segment("b", "Bob", 3, 4, "target", []string{"source#1"}),
|
||||||
|
)
|
||||||
|
|
||||||
|
got, summary := Apply(in)
|
||||||
|
if summary.DanglersMerged != 0 || !reflect.DeepEqual(got, in) {
|
||||||
|
t.Fatalf("unexpected merge:\ngot %#v\nwant %#v", got, in)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestApplyDoesNotMergeWithoutSharedProvenance(t *testing.T) {
|
func TestApplyDoesNotMergeWithoutSharedProvenance(t *testing.T) {
|
||||||
in := transcript(
|
in := transcript(
|
||||||
segment("a", "Alice", 1, 2, "target", []string{"source#1"}),
|
segment("a", "Alice", 1, 2, "target", []string{"source#1"}),
|
||||||
|
|||||||
28
internal/jsonfile/jsonfile.go
Normal file
28
internal/jsonfile/jsonfile.go
Normal file
@@ -0,0 +1,28 @@
|
|||||||
|
package jsonfile
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Write creates or truncates path and writes deterministic indented JSON.
|
||||||
|
func Write(path string, value any) (err error) {
|
||||||
|
file, err := os.Create(path)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("create %q: %w", path, err)
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
closeErr := file.Close()
|
||||||
|
if err == nil && closeErr != nil {
|
||||||
|
err = fmt.Errorf("close %q: %w", path, closeErr)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
encoder := json.NewEncoder(file)
|
||||||
|
encoder.SetIndent("", " ")
|
||||||
|
if err := encoder.Encode(value); err != nil {
|
||||||
|
return fmt.Errorf("encode %q: %w", path, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
69
internal/jsonfile/jsonfile_test.go
Normal file
69
internal/jsonfile/jsonfile_test.go
Normal file
@@ -0,0 +1,69 @@
|
|||||||
|
package jsonfile
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestWriteFormatsWithTwoSpaceIndentAndTrailingNewline(t *testing.T) {
|
||||||
|
type payload struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Items []int `json:"items"`
|
||||||
|
}
|
||||||
|
|
||||||
|
path := filepath.Join(t.TempDir(), "out.json")
|
||||||
|
value := payload{
|
||||||
|
Name: "alpha",
|
||||||
|
Items: []int{1, 2},
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := Write(path, value); err != nil {
|
||||||
|
t.Fatalf("write failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read output: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got := string(data)
|
||||||
|
want := "{\n \"name\": \"alpha\",\n \"items\": [\n 1,\n 2\n ]\n}\n"
|
||||||
|
if got != want {
|
||||||
|
t.Fatalf("formatted JSON mismatch\nwant:\n%s\ngot:\n%s", want, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWriteProducesValidJSON(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "out.json")
|
||||||
|
|
||||||
|
value := map[string]any{
|
||||||
|
"application": "seriatim",
|
||||||
|
"segments": []map[string]any{
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"speaker": "A",
|
||||||
|
"text": "hello",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := Write(path, value); err != nil {
|
||||||
|
t.Fatalf("write failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read output: %v", err)
|
||||||
|
}
|
||||||
|
if !strings.HasSuffix(string(data), "\n") {
|
||||||
|
t.Fatalf("output missing trailing newline: %q", string(data))
|
||||||
|
}
|
||||||
|
|
||||||
|
var decoded map[string]any
|
||||||
|
if err := json.Unmarshal(data, &decoded); err != nil {
|
||||||
|
t.Fatalf("output is not valid JSON: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,5 +1,7 @@
|
|||||||
package model
|
package model
|
||||||
|
|
||||||
|
import "fmt"
|
||||||
|
|
||||||
// RawTranscript is a loaded input document before canonical normalization.
|
// RawTranscript is a loaded input document before canonical normalization.
|
||||||
type RawTranscript struct {
|
type RawTranscript struct {
|
||||||
Source string `json:"source"`
|
Source string `json:"source"`
|
||||||
@@ -61,6 +63,17 @@ type Segment struct {
|
|||||||
OverlapGroupID int `json:"overlap_group_id,omitempty"`
|
OverlapGroupID int `json:"overlap_group_id,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SegmentReference returns the best available external reference for a segment.
|
||||||
|
func SegmentReference(segment Segment) string {
|
||||||
|
if segment.Source != "" && segment.SourceSegmentIndex != nil {
|
||||||
|
return fmt.Sprintf("%s#%d", segment.Source, *segment.SourceSegmentIndex)
|
||||||
|
}
|
||||||
|
if segment.SourceRef != "" {
|
||||||
|
return segment.SourceRef
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
// Word preserves optional word-level timing data.
|
// Word preserves optional word-level timing data.
|
||||||
type Word struct {
|
type Word struct {
|
||||||
Text string `json:"text"`
|
Text string `json:"text"`
|
||||||
|
|||||||
41
internal/model/model_test.go
Normal file
41
internal/model/model_test.go
Normal file
@@ -0,0 +1,41 @@
|
|||||||
|
package model
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestSegmentReferenceUsesSourceAndIndexWhenAvailable(t *testing.T) {
|
||||||
|
index := 3
|
||||||
|
segment := Segment{
|
||||||
|
Source: "input.json",
|
||||||
|
SourceSegmentIndex: &index,
|
||||||
|
SourceRef: "word-run:1:2:3",
|
||||||
|
}
|
||||||
|
|
||||||
|
got := SegmentReference(segment)
|
||||||
|
want := "input.json#3"
|
||||||
|
if got != want {
|
||||||
|
t.Fatalf("reference = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSegmentReferenceFallsBackToSourceRef(t *testing.T) {
|
||||||
|
segment := Segment{
|
||||||
|
Source: "input.json",
|
||||||
|
SourceRef: "coalesce:2",
|
||||||
|
}
|
||||||
|
|
||||||
|
got := SegmentReference(segment)
|
||||||
|
want := "coalesce:2"
|
||||||
|
if got != want {
|
||||||
|
t.Fatalf("reference = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSegmentReferenceReturnsEmptyWhenNoReferenceFieldsPresent(t *testing.T) {
|
||||||
|
segment := Segment{
|
||||||
|
Source: "input.json",
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := SegmentReference(segment); got != "" {
|
||||||
|
t.Fatalf("reference = %q, want empty", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
220
internal/normalize/build.go
Normal file
220
internal/normalize/build.go
Normal file
@@ -0,0 +1,220 @@
|
|||||||
|
package normalize
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/internal/artifact"
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/internal/buildinfo"
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/schema"
|
||||||
|
)
|
||||||
|
|
||||||
|
// BuildResult contains normalize output plus deterministic transformation diagnostics.
|
||||||
|
type BuildResult struct {
|
||||||
|
Output any
|
||||||
|
OutputSegmentCount int
|
||||||
|
SortingChanged bool
|
||||||
|
IDsReassigned bool
|
||||||
|
SegmentsWithCategories int
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build converts parsed normalize input into a selected seriatim output schema.
|
||||||
|
func Build(parsed ParsedTranscript, cfg config.NormalizeConfig) (BuildResult, error) {
|
||||||
|
ordered := sortedSegments(parsed.Segments)
|
||||||
|
sortingChanged := didSortingChangeOrder(ordered)
|
||||||
|
idsReassigned := didReassignIDs(ordered)
|
||||||
|
segmentsWithCategories := countSegmentsWithCategories(ordered)
|
||||||
|
|
||||||
|
switch cfg.OutputSchema {
|
||||||
|
case config.OutputSchemaMinimal:
|
||||||
|
output := buildMinimal(ordered)
|
||||||
|
if err := schema.ValidateMinimalTranscript(output); err != nil {
|
||||||
|
return BuildResult{}, fmt.Errorf("validate normalize output: %w", err)
|
||||||
|
}
|
||||||
|
return BuildResult{
|
||||||
|
Output: output,
|
||||||
|
OutputSegmentCount: len(ordered),
|
||||||
|
SortingChanged: sortingChanged,
|
||||||
|
IDsReassigned: idsReassigned,
|
||||||
|
SegmentsWithCategories: segmentsWithCategories,
|
||||||
|
}, nil
|
||||||
|
case config.OutputSchemaIntermediate:
|
||||||
|
output := buildIntermediate(ordered)
|
||||||
|
if err := schema.ValidateIntermediateTranscript(output); err != nil {
|
||||||
|
return BuildResult{}, fmt.Errorf("validate normalize output: %w", err)
|
||||||
|
}
|
||||||
|
return BuildResult{
|
||||||
|
Output: output,
|
||||||
|
OutputSegmentCount: len(ordered),
|
||||||
|
SortingChanged: sortingChanged,
|
||||||
|
IDsReassigned: idsReassigned,
|
||||||
|
SegmentsWithCategories: segmentsWithCategories,
|
||||||
|
}, nil
|
||||||
|
case config.OutputSchemaFull:
|
||||||
|
output := buildFull(ordered, cfg)
|
||||||
|
if err := schema.ValidateTranscript(output); err != nil {
|
||||||
|
return BuildResult{}, fmt.Errorf("validate normalize output: %w", err)
|
||||||
|
}
|
||||||
|
return BuildResult{
|
||||||
|
Output: output,
|
||||||
|
OutputSegmentCount: len(ordered),
|
||||||
|
SortingChanged: sortingChanged,
|
||||||
|
IDsReassigned: idsReassigned,
|
||||||
|
SegmentsWithCategories: segmentsWithCategories,
|
||||||
|
}, nil
|
||||||
|
default:
|
||||||
|
return BuildResult{}, fmt.Errorf("unsupported output schema %q", cfg.OutputSchema)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func sortedSegments(input []InputSegment) []InputSegment {
|
||||||
|
ordered := make([]InputSegment, len(input))
|
||||||
|
copy(ordered, input)
|
||||||
|
sort.SliceStable(ordered, func(i, j int) bool {
|
||||||
|
left := ordered[i]
|
||||||
|
right := ordered[j]
|
||||||
|
if left.Start != right.Start {
|
||||||
|
return left.Start < right.Start
|
||||||
|
}
|
||||||
|
if left.End != right.End {
|
||||||
|
return left.End < right.End
|
||||||
|
}
|
||||||
|
if left.InputIndex != right.InputIndex {
|
||||||
|
return left.InputIndex < right.InputIndex
|
||||||
|
}
|
||||||
|
return left.Speaker < right.Speaker
|
||||||
|
})
|
||||||
|
return ordered
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildMinimal(segments []InputSegment) schema.MinimalTranscript {
|
||||||
|
outputSegments := make([]schema.MinimalSegment, len(segments))
|
||||||
|
for index, segment := range segments {
|
||||||
|
outputSegments[index] = schema.MinimalSegment{
|
||||||
|
ID: index + 1,
|
||||||
|
Start: segment.Start,
|
||||||
|
End: segment.End,
|
||||||
|
Speaker: segment.Speaker,
|
||||||
|
Text: segment.Text,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return schema.MinimalTranscript{
|
||||||
|
Metadata: schema.MinimalMetadata{
|
||||||
|
Application: artifact.ApplicationName,
|
||||||
|
Version: buildinfo.Version,
|
||||||
|
OutputSchema: config.OutputSchemaMinimal,
|
||||||
|
},
|
||||||
|
Segments: outputSegments,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildIntermediate(segments []InputSegment) schema.IntermediateTranscript {
|
||||||
|
outputSegments := make([]schema.IntermediateSegment, len(segments))
|
||||||
|
for index, segment := range segments {
|
||||||
|
outputSegments[index] = schema.IntermediateSegment{
|
||||||
|
ID: index + 1,
|
||||||
|
Start: segment.Start,
|
||||||
|
End: segment.End,
|
||||||
|
Speaker: segment.Speaker,
|
||||||
|
Text: segment.Text,
|
||||||
|
Categories: append([]string(nil), segment.Categories...),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return schema.IntermediateTranscript{
|
||||||
|
Metadata: schema.IntermediateMetadata{
|
||||||
|
Application: artifact.ApplicationName,
|
||||||
|
Version: buildinfo.Version,
|
||||||
|
OutputSchema: config.OutputSchemaIntermediate,
|
||||||
|
},
|
||||||
|
Segments: outputSegments,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildFull(segments []InputSegment, cfg config.NormalizeConfig) schema.Transcript {
|
||||||
|
defaultSource := filepath.Base(cfg.InputFile)
|
||||||
|
outputSegments := make([]schema.Segment, len(segments))
|
||||||
|
for index, segment := range segments {
|
||||||
|
source := strings.TrimSpace(segment.Source)
|
||||||
|
if source == "" {
|
||||||
|
source = defaultSource
|
||||||
|
}
|
||||||
|
|
||||||
|
sourceSegmentIndex := copyIntPtr(segment.SourceSegmentIndex)
|
||||||
|
if sourceSegmentIndex == nil {
|
||||||
|
fallback := segment.InputIndex
|
||||||
|
sourceSegmentIndex = &fallback
|
||||||
|
}
|
||||||
|
|
||||||
|
outputSegments[index] = schema.Segment{
|
||||||
|
ID: index + 1,
|
||||||
|
Source: source,
|
||||||
|
SourceSegmentIndex: sourceSegmentIndex,
|
||||||
|
SourceRef: segment.SourceRef,
|
||||||
|
DerivedFrom: append([]string(nil), segment.DerivedFrom...),
|
||||||
|
Speaker: segment.Speaker,
|
||||||
|
Start: segment.Start,
|
||||||
|
End: segment.End,
|
||||||
|
Text: segment.Text,
|
||||||
|
Categories: append([]string(nil), segment.Categories...),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return schema.Transcript{
|
||||||
|
Metadata: schema.Metadata{
|
||||||
|
Application: artifact.ApplicationName,
|
||||||
|
Version: buildinfo.Version,
|
||||||
|
InputReader: "normalize-input",
|
||||||
|
InputFiles: []string{cfg.InputFile},
|
||||||
|
PreprocessingModules: []string{},
|
||||||
|
PostprocessingModules: []string{},
|
||||||
|
OutputModules: append([]string(nil), cfg.OutputModules...),
|
||||||
|
},
|
||||||
|
Segments: outputSegments,
|
||||||
|
OverlapGroups: []schema.OverlapGroup{},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyIntPtr(value *int) *int {
|
||||||
|
if value == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
copied := *value
|
||||||
|
return &copied
|
||||||
|
}
|
||||||
|
|
||||||
|
func didSortingChangeOrder(segments []InputSegment) bool {
|
||||||
|
for index, segment := range segments {
|
||||||
|
if segment.InputIndex != index {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func didReassignIDs(segments []InputSegment) bool {
|
||||||
|
if len(segments) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for index, segment := range segments {
|
||||||
|
newID := index + 1
|
||||||
|
if segment.OriginalID == nil || *segment.OriginalID != newID {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func countSegmentsWithCategories(segments []InputSegment) int {
|
||||||
|
count := 0
|
||||||
|
for _, segment := range segments {
|
||||||
|
if len(segment.Categories) > 0 {
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return count
|
||||||
|
}
|
||||||
120
internal/normalize/normalize.go
Normal file
120
internal/normalize/normalize.go
Normal file
@@ -0,0 +1,120 @@
|
|||||||
|
package normalize
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/internal/artifact"
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/internal/buildinfo"
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/internal/jsonfile"
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/internal/report"
|
||||||
|
)
|
||||||
|
|
||||||
|
type normalizeAudit struct {
|
||||||
|
Command string `json:"command"`
|
||||||
|
InputFile string `json:"input_file"`
|
||||||
|
OutputFile string `json:"output_file"`
|
||||||
|
InputShape string `json:"input_shape"`
|
||||||
|
InputSegmentCount int `json:"input_segment_count"`
|
||||||
|
OutputSegmentCount int `json:"output_segment_count"`
|
||||||
|
OutputSchema string `json:"output_schema"`
|
||||||
|
OutputModules []string `json:"output_modules"`
|
||||||
|
IDsReassigned bool `json:"ids_reassigned"`
|
||||||
|
SortingChangedInput bool `json:"sorting_changed_input_order"`
|
||||||
|
SegmentsWithCategories int `json:"segments_with_categories"`
|
||||||
|
TimingFieldsRepaired int `json:"timing_fields_repaired"`
|
||||||
|
TimingOrderSwapped int `json:"timing_order_swapped"`
|
||||||
|
SpeakerFilled int `json:"speaker_filled"`
|
||||||
|
SegmentsDroppedText int `json:"segments_dropped_text"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run executes artifact-level normalization.
|
||||||
|
func Run(ctx context.Context, cfg config.NormalizeConfig) error {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
parsed, err := ParseFile(cfg.InputFile)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
built, err := Build(parsed, cfg)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := jsonfile.Write(cfg.OutputFile, built.Output); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if cfg.ReportFile != "" {
|
||||||
|
audit := normalizeAudit{
|
||||||
|
Command: "normalize",
|
||||||
|
InputFile: cfg.InputFile,
|
||||||
|
OutputFile: cfg.OutputFile,
|
||||||
|
InputShape: string(parsed.Shape),
|
||||||
|
InputSegmentCount: parsed.InputSegmentCount,
|
||||||
|
OutputSegmentCount: built.OutputSegmentCount,
|
||||||
|
OutputSchema: cfg.OutputSchema,
|
||||||
|
OutputModules: append([]string(nil), cfg.OutputModules...),
|
||||||
|
IDsReassigned: built.IDsReassigned,
|
||||||
|
SortingChangedInput: built.SortingChanged,
|
||||||
|
SegmentsWithCategories: built.SegmentsWithCategories,
|
||||||
|
TimingFieldsRepaired: parsed.Stats.TimingFieldsRepaired,
|
||||||
|
TimingOrderSwapped: parsed.Stats.TimingOrderSwapped,
|
||||||
|
SpeakerFilled: parsed.Stats.SpeakerFilled,
|
||||||
|
SegmentsDroppedText: parsed.Stats.SegmentsDroppedText,
|
||||||
|
}
|
||||||
|
auditJSON, err := json.Marshal(audit)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("marshal normalize audit: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
events := []report.Event{
|
||||||
|
report.Info("normalize", "normalize", "started normalize command"),
|
||||||
|
report.Info("normalize", "normalize", fmt.Sprintf("input file: %s", cfg.InputFile)),
|
||||||
|
report.Info("normalize", "normalize", fmt.Sprintf("detected input shape: %s", parsed.Shape)),
|
||||||
|
report.Info("normalize", "normalize", fmt.Sprintf("input segment count: %d", parsed.InputSegmentCount)),
|
||||||
|
report.Info("normalize", "normalize", fmt.Sprintf("selected output schema: %s", cfg.OutputSchema)),
|
||||||
|
report.Info("normalize", "normalize", fmt.Sprintf("selected output modules: %s", strings.Join(cfg.OutputModules, ","))),
|
||||||
|
report.Info("normalize", "normalize", fmt.Sprintf("output file: %s", cfg.OutputFile)),
|
||||||
|
report.Info("normalize", "normalize", fmt.Sprintf("ids reassigned: %t", built.IDsReassigned)),
|
||||||
|
report.Info("normalize", "normalize", fmt.Sprintf("sorting changed input order: %t", built.SortingChanged)),
|
||||||
|
report.Info("normalize", "normalize", fmt.Sprintf("segments with categories: %d", built.SegmentsWithCategories)),
|
||||||
|
report.Info("normalize", "normalize", fmt.Sprintf("timing fields repaired: %d", parsed.Stats.TimingFieldsRepaired)),
|
||||||
|
report.Info("normalize", "normalize", fmt.Sprintf("timing order swapped: %d", parsed.Stats.TimingOrderSwapped)),
|
||||||
|
report.Info("normalize", "normalize", fmt.Sprintf("speaker placeholders added: %d", parsed.Stats.SpeakerFilled)),
|
||||||
|
report.Info("normalize", "normalize", fmt.Sprintf("segments dropped for empty text: %d", parsed.Stats.SegmentsDroppedText)),
|
||||||
|
report.Info("normalize", "normalize-audit", string(auditJSON)),
|
||||||
|
}
|
||||||
|
if parsed.InputSegmentCount == 0 {
|
||||||
|
events = append(events, report.Warning("normalize", "normalize", "input transcript contains zero segments"))
|
||||||
|
}
|
||||||
|
events = append(events,
|
||||||
|
report.Info("normalize", "validate-output", fmt.Sprintf("validated %d output segment(s)", built.OutputSegmentCount)),
|
||||||
|
report.Info("output", "json", "wrote transcript JSON"),
|
||||||
|
)
|
||||||
|
|
||||||
|
rpt := report.Report{
|
||||||
|
Metadata: report.Metadata{
|
||||||
|
Application: artifact.ApplicationName,
|
||||||
|
Version: buildinfo.Version,
|
||||||
|
InputReader: "normalize-input",
|
||||||
|
InputFiles: []string{cfg.InputFile},
|
||||||
|
PreprocessingModules: []string{},
|
||||||
|
PostprocessingModules: []string{},
|
||||||
|
OutputModules: append([]string(nil), cfg.OutputModules...),
|
||||||
|
},
|
||||||
|
Events: events,
|
||||||
|
}
|
||||||
|
if err := report.WriteJSON(cfg.ReportFile, rpt); err != nil {
|
||||||
|
return fmt.Errorf("write --report-file %q: %w", cfg.ReportFile, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
307
internal/normalize/parse.go
Normal file
307
internal/normalize/parse.go
Normal file
@@ -0,0 +1,307 @@
|
|||||||
|
package normalize
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// InputShape identifies which top-level input shape was parsed.
|
||||||
|
type InputShape string
|
||||||
|
|
||||||
|
const (
|
||||||
|
ShapeObjectWithSegments InputShape = "object_with_segments"
|
||||||
|
ShapeBareSegmentsArray InputShape = "bare_segments_array"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ParsedTranscript is the validated normalize input model.
|
||||||
|
type ParsedTranscript struct {
|
||||||
|
Shape InputShape
|
||||||
|
InputSegmentCount int
|
||||||
|
Stats NormalizeStats
|
||||||
|
Segments []InputSegment
|
||||||
|
}
|
||||||
|
|
||||||
|
// InputSegment is a validated segment from normalize input.
|
||||||
|
type InputSegment struct {
|
||||||
|
InputIndex int
|
||||||
|
OriginalID *int
|
||||||
|
StartPresent bool
|
||||||
|
EndPresent bool
|
||||||
|
SpeakerPresent bool
|
||||||
|
TextPresent bool
|
||||||
|
Start float64
|
||||||
|
End float64
|
||||||
|
Speaker string
|
||||||
|
Text string
|
||||||
|
Categories []string
|
||||||
|
Source string
|
||||||
|
SourceSegmentIndex *int
|
||||||
|
SourceRef string
|
||||||
|
DerivedFrom []string
|
||||||
|
OverlapGroupID *int
|
||||||
|
}
|
||||||
|
|
||||||
|
// NormalizeStats captures deterministic repair/drop outcomes from normalize input processing.
|
||||||
|
type NormalizeStats struct {
|
||||||
|
TimingFieldsRepaired int
|
||||||
|
TimingOrderSwapped int
|
||||||
|
SpeakerFilled int
|
||||||
|
SegmentsDroppedText int
|
||||||
|
}
|
||||||
|
|
||||||
|
type inputSegmentPayload struct {
|
||||||
|
ID *int `json:"id"`
|
||||||
|
Start *float64 `json:"start"`
|
||||||
|
End *float64 `json:"end"`
|
||||||
|
Speaker *string `json:"speaker"`
|
||||||
|
Text *string `json:"text"`
|
||||||
|
Categories []string `json:"categories"`
|
||||||
|
Source string `json:"source"`
|
||||||
|
SourceSegmentIndex *int `json:"source_segment_index"`
|
||||||
|
SourceRef string `json:"source_ref"`
|
||||||
|
DerivedFrom []string `json:"derived_from"`
|
||||||
|
OverlapGroupID *int `json:"overlap_group_id"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseFile parses normalize input JSON from file path.
|
||||||
|
func ParseFile(path string) (ParsedTranscript, error) {
|
||||||
|
file, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return ParsedTranscript{}, err
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
return ParseReader(file)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseReader parses normalize input JSON from a reader.
|
||||||
|
func ParseReader(reader io.Reader) (ParsedTranscript, error) {
|
||||||
|
var raw json.RawMessage
|
||||||
|
decoder := json.NewDecoder(reader)
|
||||||
|
decoder.UseNumber()
|
||||||
|
if err := decoder.Decode(&raw); err != nil {
|
||||||
|
return ParsedTranscript{}, fmt.Errorf("decode normalize input JSON: %w", err)
|
||||||
|
}
|
||||||
|
if err := ensureSingleValue(decoder); err != nil {
|
||||||
|
return ParsedTranscript{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
trimmed := bytes.TrimSpace(raw)
|
||||||
|
if len(trimmed) == 0 {
|
||||||
|
return ParsedTranscript{}, fmt.Errorf("normalize input is empty")
|
||||||
|
}
|
||||||
|
|
||||||
|
switch trimmed[0] {
|
||||||
|
case '{':
|
||||||
|
return parseObjectShape(trimmed)
|
||||||
|
case '[':
|
||||||
|
segments, stats, inputCount, err := parseSegmentsArray(trimmed)
|
||||||
|
if err != nil {
|
||||||
|
return ParsedTranscript{}, err
|
||||||
|
}
|
||||||
|
return ParsedTranscript{
|
||||||
|
Shape: ShapeBareSegmentsArray,
|
||||||
|
InputSegmentCount: inputCount,
|
||||||
|
Stats: stats,
|
||||||
|
Segments: segments,
|
||||||
|
}, nil
|
||||||
|
default:
|
||||||
|
return ParsedTranscript{}, fmt.Errorf("normalize input must be a top-level object with \"segments\" or a top-level segment array")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func ensureSingleValue(decoder *json.Decoder) error {
|
||||||
|
var extra json.RawMessage
|
||||||
|
err := decoder.Decode(&extra)
|
||||||
|
if err == io.EOF {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err == nil {
|
||||||
|
return fmt.Errorf("normalize input must contain exactly one top-level JSON value")
|
||||||
|
}
|
||||||
|
return fmt.Errorf("decode normalize input JSON: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseObjectShape(raw []byte) (ParsedTranscript, error) {
|
||||||
|
var object map[string]json.RawMessage
|
||||||
|
if err := json.Unmarshal(raw, &object); err != nil {
|
||||||
|
return ParsedTranscript{}, fmt.Errorf("decode normalize object input: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
segmentsRaw, exists := object["segments"]
|
||||||
|
if !exists {
|
||||||
|
return ParsedTranscript{}, fmt.Errorf("normalize object input must contain a \"segments\" field")
|
||||||
|
}
|
||||||
|
|
||||||
|
segments, stats, inputCount, err := parseSegmentsArray(segmentsRaw)
|
||||||
|
if err != nil {
|
||||||
|
return ParsedTranscript{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return ParsedTranscript{
|
||||||
|
Shape: ShapeObjectWithSegments,
|
||||||
|
InputSegmentCount: inputCount,
|
||||||
|
Stats: stats,
|
||||||
|
Segments: segments,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseSegmentsArray(raw []byte) ([]InputSegment, NormalizeStats, int, error) {
|
||||||
|
var segmentValues []json.RawMessage
|
||||||
|
if err := json.Unmarshal(raw, &segmentValues); err != nil {
|
||||||
|
return nil, NormalizeStats{}, 0, fmt.Errorf("normalize input \"segments\" must be an array")
|
||||||
|
}
|
||||||
|
|
||||||
|
segments := make([]InputSegment, len(segmentValues))
|
||||||
|
for index, segmentRaw := range segmentValues {
|
||||||
|
segment, err := decodeSegment(index, segmentRaw)
|
||||||
|
if err != nil {
|
||||||
|
return nil, NormalizeStats{}, 0, err
|
||||||
|
}
|
||||||
|
segments[index] = segment
|
||||||
|
}
|
||||||
|
normalized, stats, err := normalizeSegments(segments)
|
||||||
|
return normalized, stats, len(segmentValues), err
|
||||||
|
}
|
||||||
|
|
||||||
|
func decodeSegment(index int, raw []byte) (InputSegment, error) {
|
||||||
|
var payload inputSegmentPayload
|
||||||
|
if err := json.Unmarshal(raw, &payload); err != nil {
|
||||||
|
return InputSegment{}, fmt.Errorf("segment %d: invalid segment object: %w", index, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var start float64
|
||||||
|
if payload.Start != nil {
|
||||||
|
start = *payload.Start
|
||||||
|
}
|
||||||
|
var end float64
|
||||||
|
if payload.End != nil {
|
||||||
|
end = *payload.End
|
||||||
|
}
|
||||||
|
|
||||||
|
speaker := ""
|
||||||
|
if payload.Speaker != nil {
|
||||||
|
speaker = strings.TrimSpace(*payload.Speaker)
|
||||||
|
}
|
||||||
|
|
||||||
|
text := ""
|
||||||
|
if payload.Text != nil {
|
||||||
|
text = *payload.Text
|
||||||
|
}
|
||||||
|
|
||||||
|
return InputSegment{
|
||||||
|
InputIndex: index,
|
||||||
|
OriginalID: payload.ID,
|
||||||
|
StartPresent: payload.Start != nil,
|
||||||
|
EndPresent: payload.End != nil,
|
||||||
|
SpeakerPresent: payload.Speaker != nil,
|
||||||
|
TextPresent: payload.Text != nil,
|
||||||
|
Start: start,
|
||||||
|
End: end,
|
||||||
|
Speaker: speaker,
|
||||||
|
Text: text,
|
||||||
|
Categories: append([]string(nil), payload.Categories...),
|
||||||
|
Source: payload.Source,
|
||||||
|
SourceSegmentIndex: payload.SourceSegmentIndex,
|
||||||
|
SourceRef: payload.SourceRef,
|
||||||
|
DerivedFrom: append([]string(nil), payload.DerivedFrom...),
|
||||||
|
OverlapGroupID: payload.OverlapGroupID,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeSegments(segments []InputSegment) ([]InputSegment, NormalizeStats, error) {
|
||||||
|
stats := NormalizeStats{}
|
||||||
|
filtered := make([]InputSegment, 0, len(segments))
|
||||||
|
for _, segment := range segments {
|
||||||
|
if strings.TrimSpace(segment.Text) == "" {
|
||||||
|
stats.SegmentsDroppedText++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
filtered = append(filtered, segment)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(filtered) == 0 {
|
||||||
|
return filtered, stats, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
hasStart := make([]bool, len(filtered))
|
||||||
|
hasEnd := make([]bool, len(filtered))
|
||||||
|
for index, segment := range filtered {
|
||||||
|
hasStart[index] = segment.StartPresent
|
||||||
|
hasEnd[index] = segment.EndPresent
|
||||||
|
}
|
||||||
|
for index := range filtered {
|
||||||
|
if !hasStart[index] && !hasEnd[index] {
|
||||||
|
midpoint := inferTimestampFromNeighbors(filtered, hasStart, hasEnd, index)
|
||||||
|
filtered[index].Start = midpoint
|
||||||
|
filtered[index].End = midpoint
|
||||||
|
stats.TimingFieldsRepaired += 2
|
||||||
|
hasStart[index] = true
|
||||||
|
hasEnd[index] = true
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if hasStart[index] && !hasEnd[index] {
|
||||||
|
filtered[index].End = filtered[index].Start
|
||||||
|
stats.TimingFieldsRepaired++
|
||||||
|
hasEnd[index] = true
|
||||||
|
}
|
||||||
|
if !hasStart[index] && hasEnd[index] {
|
||||||
|
filtered[index].Start = filtered[index].End
|
||||||
|
stats.TimingFieldsRepaired++
|
||||||
|
hasStart[index] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for index := range filtered {
|
||||||
|
if filtered[index].Start < 0 {
|
||||||
|
return nil, NormalizeStats{}, fmt.Errorf("segment %d has start %v; start must be >= 0", filtered[index].InputIndex, filtered[index].Start)
|
||||||
|
}
|
||||||
|
if filtered[index].End < filtered[index].Start {
|
||||||
|
filtered[index].Start, filtered[index].End = filtered[index].End, filtered[index].Start
|
||||||
|
stats.TimingOrderSwapped++
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(filtered[index].Speaker) == "" {
|
||||||
|
filtered[index].Speaker = "Unknown_Speaker"
|
||||||
|
stats.SpeakerFilled++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return filtered, stats, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func inferTimestampFromNeighbors(segments []InputSegment, hasStart []bool, hasEnd []bool, index int) float64 {
|
||||||
|
prevEnd, hasPrev := nearestPreviousEnd(segments, hasEnd, index)
|
||||||
|
nextStart, hasNext := nearestNextStart(segments, hasStart, index)
|
||||||
|
if hasPrev && hasNext {
|
||||||
|
return (prevEnd + nextStart) / 2
|
||||||
|
}
|
||||||
|
if hasPrev {
|
||||||
|
return prevEnd
|
||||||
|
}
|
||||||
|
if hasNext {
|
||||||
|
return nextStart
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func nearestPreviousEnd(segments []InputSegment, hasEnd []bool, index int) (float64, bool) {
|
||||||
|
for i := index - 1; i >= 0; i-- {
|
||||||
|
if hasEnd[i] {
|
||||||
|
return segments[i].End, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
|
||||||
|
func nearestNextStart(segments []InputSegment, hasStart []bool, index int) (float64, bool) {
|
||||||
|
for i := index + 1; i < len(segments); i++ {
|
||||||
|
if hasStart[i] {
|
||||||
|
return segments[i].Start, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
279
internal/normalize/parse_test.go
Normal file
279
internal/normalize/parse_test.go
Normal file
@@ -0,0 +1,279 @@
|
|||||||
|
package normalize
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestParseReaderObjectWithSegmentsParses(t *testing.T) {
|
||||||
|
input := `{
|
||||||
|
"segments": [
|
||||||
|
{"start": 1.0, "end": 2.0, "speaker": " Alice ", "text": "hello", "id": 100}
|
||||||
|
]
|
||||||
|
}`
|
||||||
|
|
||||||
|
parsed, err := ParseReader(strings.NewReader(input))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse failed: %v", err)
|
||||||
|
}
|
||||||
|
if parsed.Shape != ShapeObjectWithSegments {
|
||||||
|
t.Fatalf("shape = %q, want %q", parsed.Shape, ShapeObjectWithSegments)
|
||||||
|
}
|
||||||
|
if len(parsed.Segments) != 1 {
|
||||||
|
t.Fatalf("segment count = %d, want 1", len(parsed.Segments))
|
||||||
|
}
|
||||||
|
segment := parsed.Segments[0]
|
||||||
|
if segment.Speaker != "Alice" {
|
||||||
|
t.Fatalf("speaker = %q, want %q", segment.Speaker, "Alice")
|
||||||
|
}
|
||||||
|
if segment.OriginalID == nil || *segment.OriginalID != 100 {
|
||||||
|
t.Fatalf("original id = %v, want 100", segment.OriginalID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseReaderBareSegmentArrayParses(t *testing.T) {
|
||||||
|
input := `[
|
||||||
|
{"start": 1.0, "end": 2.0, "speaker": "Alice", "text": "hello"},
|
||||||
|
{"start": 3.0, "end": 4.0, "speaker": "Bob", "text": "world"}
|
||||||
|
]`
|
||||||
|
|
||||||
|
parsed, err := ParseReader(strings.NewReader(input))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse failed: %v", err)
|
||||||
|
}
|
||||||
|
if parsed.Shape != ShapeBareSegmentsArray {
|
||||||
|
t.Fatalf("shape = %q, want %q", parsed.Shape, ShapeBareSegmentsArray)
|
||||||
|
}
|
||||||
|
if len(parsed.Segments) != 2 {
|
||||||
|
t.Fatalf("segment count = %d, want 2", len(parsed.Segments))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseReaderInvalidJSONFails(t *testing.T) {
|
||||||
|
_, err := ParseReader(strings.NewReader(`{"segments":`))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected parse error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "decode normalize input JSON") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseReaderObjectMissingSegmentsFails(t *testing.T) {
|
||||||
|
_, err := ParseReader(strings.NewReader(`{"items":[]}`))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected missing segments error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "must contain a \"segments\" field") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseReaderSegmentsNotArrayFails(t *testing.T) {
|
||||||
|
_, err := ParseReader(strings.NewReader(`{"segments": {}}`))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected segments not array error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "\"segments\" must be an array") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseReaderTopLevelScalarShapesFail(t *testing.T) {
|
||||||
|
tests := []string{`"text"`, `42`, `null`, `true`}
|
||||||
|
for _, input := range tests {
|
||||||
|
_, err := ParseReader(strings.NewReader(input))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("expected top-level shape error for %s", input)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "top-level object") {
|
||||||
|
t.Fatalf("unexpected error for %s: %v", input, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseReaderMissingStartUsesEndValue(t *testing.T) {
|
||||||
|
parsed, err := ParseReader(strings.NewReader(`[{"end":2,"speaker":"A","text":"t"}]`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse failed: %v", err)
|
||||||
|
}
|
||||||
|
if len(parsed.Segments) != 1 {
|
||||||
|
t.Fatalf("segment count = %d, want 1", len(parsed.Segments))
|
||||||
|
}
|
||||||
|
if parsed.Segments[0].Start != 2 || parsed.Segments[0].End != 2 {
|
||||||
|
t.Fatalf("segment timing = %v..%v, want 2..2", parsed.Segments[0].Start, parsed.Segments[0].End)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseReaderMissingEndUsesStartValue(t *testing.T) {
|
||||||
|
parsed, err := ParseReader(strings.NewReader(`[{"start":1,"speaker":"A","text":"t"}]`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse failed: %v", err)
|
||||||
|
}
|
||||||
|
if len(parsed.Segments) != 1 {
|
||||||
|
t.Fatalf("segment count = %d, want 1", len(parsed.Segments))
|
||||||
|
}
|
||||||
|
if parsed.Segments[0].Start != 1 || parsed.Segments[0].End != 1 {
|
||||||
|
t.Fatalf("segment timing = %v..%v, want 1..1", parsed.Segments[0].Start, parsed.Segments[0].End)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseReaderMissingSpeakerUsesUnknownPlaceholder(t *testing.T) {
|
||||||
|
parsed, err := ParseReader(strings.NewReader(`[{"start":1,"end":2,"text":"t"}]`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse failed: %v", err)
|
||||||
|
}
|
||||||
|
if parsed.Segments[0].Speaker != "Unknown_Speaker" {
|
||||||
|
t.Fatalf("speaker = %q, want Unknown_Speaker", parsed.Segments[0].Speaker)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseReaderEmptySpeakerUsesUnknownPlaceholder(t *testing.T) {
|
||||||
|
parsed, err := ParseReader(strings.NewReader(`[{"start":1,"end":2,"speaker":" ","text":"t"}]`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse failed: %v", err)
|
||||||
|
}
|
||||||
|
if parsed.Segments[0].Speaker != "Unknown_Speaker" {
|
||||||
|
t.Fatalf("speaker = %q, want Unknown_Speaker", parsed.Segments[0].Speaker)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseReaderMissingTextDropsSegment(t *testing.T) {
|
||||||
|
parsed, err := ParseReader(strings.NewReader(`[{"start":1,"end":2,"speaker":"A"}]`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse failed: %v", err)
|
||||||
|
}
|
||||||
|
if len(parsed.Segments) != 0 {
|
||||||
|
t.Fatalf("segment count = %d, want 0", len(parsed.Segments))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseReaderEndBeforeStartSwapsValues(t *testing.T) {
|
||||||
|
parsed, err := ParseReader(strings.NewReader(`[{"start":3,"end":2,"speaker":"A","text":"t"}]`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse failed: %v", err)
|
||||||
|
}
|
||||||
|
if parsed.Segments[0].Start != 2 || parsed.Segments[0].End != 3 {
|
||||||
|
t.Fatalf("segment timing = %v..%v, want 2..3", parsed.Segments[0].Start, parsed.Segments[0].End)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseReaderNegativeStartFails(t *testing.T) {
|
||||||
|
_, err := ParseReader(strings.NewReader(`[{"start":-1,"end":2,"speaker":"A","text":"t"}]`))
|
||||||
|
assertContains(t, err, "start must be >= 0")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseReaderEmptySegmentsArrayAccepted(t *testing.T) {
|
||||||
|
parsed, err := ParseReader(strings.NewReader(`{"segments":[]}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse failed: %v", err)
|
||||||
|
}
|
||||||
|
if len(parsed.Segments) != 0 {
|
||||||
|
t.Fatalf("segment count = %d, want 0", len(parsed.Segments))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseReaderCategoriesPreservedWhenValid(t *testing.T) {
|
||||||
|
parsed, err := ParseReader(strings.NewReader(`[{"start":1,"end":2,"speaker":"A","text":"t","categories":["filler","backchannel"]}]`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse failed: %v", err)
|
||||||
|
}
|
||||||
|
if len(parsed.Segments) != 1 {
|
||||||
|
t.Fatalf("segment count = %d, want 1", len(parsed.Segments))
|
||||||
|
}
|
||||||
|
if len(parsed.Segments[0].Categories) != 2 {
|
||||||
|
t.Fatalf("categories length = %d, want 2", len(parsed.Segments[0].Categories))
|
||||||
|
}
|
||||||
|
if parsed.Segments[0].Categories[0] != "filler" || parsed.Segments[0].Categories[1] != "backchannel" {
|
||||||
|
t.Fatalf("categories = %v", parsed.Segments[0].Categories)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseReaderMissingBothTimesMiddleUsesNeighborMidpoint(t *testing.T) {
|
||||||
|
parsed, err := ParseReader(strings.NewReader(`[
|
||||||
|
{"start":1,"end":2,"speaker":"A","text":"left"},
|
||||||
|
{"speaker":"B","text":"middle"},
|
||||||
|
{"start":6,"end":7,"speaker":"C","text":"right"}
|
||||||
|
]`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse failed: %v", err)
|
||||||
|
}
|
||||||
|
if parsed.Segments[1].Start != 4 || parsed.Segments[1].End != 4 {
|
||||||
|
t.Fatalf("middle timing = %v..%v, want 4..4", parsed.Segments[1].Start, parsed.Segments[1].End)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseReaderMissingBothTimesEdgeFallbacks(t *testing.T) {
|
||||||
|
parsedFirst, err := ParseReader(strings.NewReader(`[
|
||||||
|
{"speaker":"A","text":"first"},
|
||||||
|
{"start":5,"end":6,"speaker":"B","text":"second"}
|
||||||
|
]`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse first failed: %v", err)
|
||||||
|
}
|
||||||
|
if parsedFirst.Segments[0].Start != 5 || parsedFirst.Segments[0].End != 5 {
|
||||||
|
t.Fatalf("first timing = %v..%v, want 5..5", parsedFirst.Segments[0].Start, parsedFirst.Segments[0].End)
|
||||||
|
}
|
||||||
|
|
||||||
|
parsedLast, err := ParseReader(strings.NewReader(`[
|
||||||
|
{"start":1,"end":2,"speaker":"A","text":"first"},
|
||||||
|
{"speaker":"B","text":"last"}
|
||||||
|
]`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse last failed: %v", err)
|
||||||
|
}
|
||||||
|
if parsedLast.Segments[1].Start != 2 || parsedLast.Segments[1].End != 2 {
|
||||||
|
t.Fatalf("last timing = %v..%v, want 2..2", parsedLast.Segments[1].Start, parsedLast.Segments[1].End)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseReaderMissingBothTimesSingleSegmentUsesZero(t *testing.T) {
|
||||||
|
parsed, err := ParseReader(strings.NewReader(`[{"speaker":"A","text":"only"}]`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse failed: %v", err)
|
||||||
|
}
|
||||||
|
if parsed.Segments[0].Start != 0 || parsed.Segments[0].End != 0 {
|
||||||
|
t.Fatalf("timing = %v..%v, want 0..0", parsed.Segments[0].Start, parsed.Segments[0].End)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseReaderEmptyOrWhitespaceTextDropped(t *testing.T) {
|
||||||
|
parsed, err := ParseReader(strings.NewReader(`[
|
||||||
|
{"start":1,"end":2,"speaker":"A","text":"ok"},
|
||||||
|
{"start":2,"end":3,"speaker":"A","text":""},
|
||||||
|
{"start":3,"end":4,"speaker":"A","text":" "}
|
||||||
|
]`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse failed: %v", err)
|
||||||
|
}
|
||||||
|
if len(parsed.Segments) != 1 {
|
||||||
|
t.Fatalf("segment count = %d, want 1", len(parsed.Segments))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseReaderOriginalInputIndexPreserved(t *testing.T) {
|
||||||
|
input := `[
|
||||||
|
{"start":1,"end":2,"speaker":"A","text":"one"},
|
||||||
|
{"start":2,"end":3,"speaker":"B","text":"two"},
|
||||||
|
{"start":3,"end":4,"speaker":"C","text":"three"}
|
||||||
|
]`
|
||||||
|
parsed, err := ParseReader(strings.NewReader(input))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse failed: %v", err)
|
||||||
|
}
|
||||||
|
for index, segment := range parsed.Segments {
|
||||||
|
if segment.InputIndex != index {
|
||||||
|
t.Fatalf("segment %d input index = %d, want %d", index, segment.InputIndex, index)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertContains(t *testing.T, err error, fragment string) {
|
||||||
|
t.Helper()
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("expected error containing %q", fragment)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), fragment) {
|
||||||
|
t.Fatalf("error = %q, want substring %q", err.Error(), fragment)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,7 +1,6 @@
|
|||||||
package overlap
|
package overlap
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
|
||||||
"sort"
|
"sort"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/seriatim/internal/model"
|
"gitea.maximumdirect.net/eric/seriatim/internal/model"
|
||||||
@@ -121,13 +120,7 @@ func distinctSpeakers(segments []model.Segment, indices []int) []string {
|
|||||||
|
|
||||||
// SegmentRef returns the stable overlap reference for a segment.
|
// SegmentRef returns the stable overlap reference for a segment.
|
||||||
func SegmentRef(segment model.Segment) string {
|
func SegmentRef(segment model.Segment) string {
|
||||||
if segment.SourceSegmentIndex != nil {
|
return model.SegmentReference(segment)
|
||||||
return fmt.Sprintf("%s#%d", segment.Source, *segment.SourceSegmentIndex)
|
|
||||||
}
|
|
||||||
if segment.SourceRef != "" {
|
|
||||||
return segment.SourceRef
|
|
||||||
}
|
|
||||||
return segment.Source
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func clearExisting(in *model.MergedTranscript) {
|
func clearExisting(in *model.MergedTranscript) {
|
||||||
|
|||||||
@@ -1,9 +1,6 @@
|
|||||||
package report
|
package report
|
||||||
|
|
||||||
import (
|
import "gitea.maximumdirect.net/eric/seriatim/internal/jsonfile"
|
||||||
"encoding/json"
|
|
||||||
"os"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Severity classifies report events.
|
// Severity classifies report events.
|
||||||
type Severity string
|
type Severity string
|
||||||
@@ -62,13 +59,5 @@ func Warning(stage string, module string, message string) Event {
|
|||||||
|
|
||||||
// WriteJSON writes a deterministic JSON report.
|
// WriteJSON writes a deterministic JSON report.
|
||||||
func WriteJSON(path string, rpt Report) error {
|
func WriteJSON(path string, rpt Report) error {
|
||||||
file, err := os.Create(path)
|
return jsonfile.Write(path, rpt)
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer file.Close()
|
|
||||||
|
|
||||||
enc := json.NewEncoder(file)
|
|
||||||
enc.SetIndent("", " ")
|
|
||||||
return enc.Encode(rpt)
|
|
||||||
}
|
}
|
||||||
|
|||||||
330
internal/trim/apply.go
Normal file
330
internal/trim/apply.go
Normal file
@@ -0,0 +1,330 @@
|
|||||||
|
package trim
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/internal/model"
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/internal/overlap"
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/schema"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Mode controls how selector IDs are applied.
|
||||||
|
type Mode string
|
||||||
|
|
||||||
|
const (
|
||||||
|
ModeKeep Mode = "keep"
|
||||||
|
ModeRemove Mode = "remove"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Options configures transcript trimming.
|
||||||
|
type Options struct {
|
||||||
|
Mode Mode
|
||||||
|
Selector Selector
|
||||||
|
AllowEmpty bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// Result contains trimming output and ID mapping metadata.
|
||||||
|
type Result struct {
|
||||||
|
Transcript schema.Transcript
|
||||||
|
OldToNewID map[int]int
|
||||||
|
RemovedIDs []int
|
||||||
|
}
|
||||||
|
|
||||||
|
// IntermediateResult contains trimming output for intermediate schema artifacts.
|
||||||
|
type IntermediateResult struct {
|
||||||
|
Transcript schema.IntermediateTranscript
|
||||||
|
OldToNewID map[int]int
|
||||||
|
RemovedIDs []int
|
||||||
|
}
|
||||||
|
|
||||||
|
// MinimalResult contains trimming output for minimal schema artifacts.
|
||||||
|
type MinimalResult struct {
|
||||||
|
Transcript schema.MinimalTranscript
|
||||||
|
OldToNewID map[int]int
|
||||||
|
RemovedIDs []int
|
||||||
|
}
|
||||||
|
|
||||||
|
type projection struct {
|
||||||
|
retainedIndexes []int
|
||||||
|
oldToNewID map[int]int
|
||||||
|
removedIDs []int
|
||||||
|
}
|
||||||
|
|
||||||
|
// Apply trims a full seriatim output transcript by segment ID.
|
||||||
|
func Apply(input schema.Transcript, opts Options) (Result, error) {
|
||||||
|
inputIDs := make([]int, len(input.Segments))
|
||||||
|
for index, segment := range input.Segments {
|
||||||
|
inputIDs[index] = segment.ID
|
||||||
|
}
|
||||||
|
proj, err := projectSegmentIDs(inputIDs, opts)
|
||||||
|
if err != nil {
|
||||||
|
return Result{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
kept := make([]schema.Segment, len(proj.retainedIndexes))
|
||||||
|
for outputIndex, inputIndex := range proj.retainedIndexes {
|
||||||
|
rewritten := copySegment(input.Segments[inputIndex])
|
||||||
|
rewritten.ID = outputIndex + 1
|
||||||
|
rewritten.OverlapGroupID = 0
|
||||||
|
kept[outputIndex] = rewritten
|
||||||
|
}
|
||||||
|
|
||||||
|
kept, groups := recomputeOverlapGroups(kept)
|
||||||
|
if groups == nil {
|
||||||
|
groups = make([]schema.OverlapGroup, 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
out := copyTranscript(input)
|
||||||
|
out.Segments = kept
|
||||||
|
out.OverlapGroups = groups
|
||||||
|
return Result{
|
||||||
|
Transcript: out,
|
||||||
|
OldToNewID: proj.oldToNewID,
|
||||||
|
RemovedIDs: proj.removedIDs,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ApplyIntermediate trims an intermediate seriatim output transcript by
|
||||||
|
// segment ID.
|
||||||
|
func ApplyIntermediate(input schema.IntermediateTranscript, opts Options) (IntermediateResult, error) {
|
||||||
|
inputIDs := make([]int, len(input.Segments))
|
||||||
|
for index, segment := range input.Segments {
|
||||||
|
inputIDs[index] = segment.ID
|
||||||
|
}
|
||||||
|
proj, err := projectSegmentIDs(inputIDs, opts)
|
||||||
|
if err != nil {
|
||||||
|
return IntermediateResult{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
kept := make([]schema.IntermediateSegment, len(proj.retainedIndexes))
|
||||||
|
for outputIndex, inputIndex := range proj.retainedIndexes {
|
||||||
|
segment := input.Segments[inputIndex]
|
||||||
|
rewritten := schema.IntermediateSegment{
|
||||||
|
ID: outputIndex + 1,
|
||||||
|
Start: segment.Start,
|
||||||
|
End: segment.End,
|
||||||
|
Speaker: segment.Speaker,
|
||||||
|
Text: segment.Text,
|
||||||
|
Categories: append([]string(nil), segment.Categories...),
|
||||||
|
}
|
||||||
|
kept[outputIndex] = rewritten
|
||||||
|
}
|
||||||
|
|
||||||
|
return IntermediateResult{
|
||||||
|
Transcript: schema.IntermediateTranscript{
|
||||||
|
Metadata: schema.IntermediateMetadata{
|
||||||
|
Application: input.Metadata.Application,
|
||||||
|
Version: input.Metadata.Version,
|
||||||
|
OutputSchema: input.Metadata.OutputSchema,
|
||||||
|
},
|
||||||
|
Segments: kept,
|
||||||
|
},
|
||||||
|
OldToNewID: proj.oldToNewID,
|
||||||
|
RemovedIDs: proj.removedIDs,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ApplyMinimal trims a minimal seriatim output transcript by segment ID.
|
||||||
|
func ApplyMinimal(input schema.MinimalTranscript, opts Options) (MinimalResult, error) {
|
||||||
|
inputIDs := make([]int, len(input.Segments))
|
||||||
|
for index, segment := range input.Segments {
|
||||||
|
inputIDs[index] = segment.ID
|
||||||
|
}
|
||||||
|
proj, err := projectSegmentIDs(inputIDs, opts)
|
||||||
|
if err != nil {
|
||||||
|
return MinimalResult{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
kept := make([]schema.MinimalSegment, len(proj.retainedIndexes))
|
||||||
|
for outputIndex, inputIndex := range proj.retainedIndexes {
|
||||||
|
segment := input.Segments[inputIndex]
|
||||||
|
rewritten := schema.MinimalSegment{
|
||||||
|
ID: outputIndex + 1,
|
||||||
|
Start: segment.Start,
|
||||||
|
End: segment.End,
|
||||||
|
Speaker: segment.Speaker,
|
||||||
|
Text: segment.Text,
|
||||||
|
}
|
||||||
|
kept[outputIndex] = rewritten
|
||||||
|
}
|
||||||
|
|
||||||
|
return MinimalResult{
|
||||||
|
Transcript: schema.MinimalTranscript{
|
||||||
|
Metadata: schema.MinimalMetadata{
|
||||||
|
Application: input.Metadata.Application,
|
||||||
|
Version: input.Metadata.Version,
|
||||||
|
OutputSchema: input.Metadata.OutputSchema,
|
||||||
|
},
|
||||||
|
Segments: kept,
|
||||||
|
},
|
||||||
|
OldToNewID: proj.oldToNewID,
|
||||||
|
RemovedIDs: proj.removedIDs,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func projectSegmentIDs(ids []int, opts Options) (projection, error) {
|
||||||
|
if err := validateMode(opts.Mode); err != nil {
|
||||||
|
return projection{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
selected := opts.Selector.IDs()
|
||||||
|
if len(selected) == 0 {
|
||||||
|
return projection{}, fmt.Errorf("selector cannot be empty")
|
||||||
|
}
|
||||||
|
|
||||||
|
idIndex, err := validateInputIDs(ids)
|
||||||
|
if err != nil {
|
||||||
|
return projection{}, err
|
||||||
|
}
|
||||||
|
if err := validateSelectedIDsExist(selected, idIndex); err != nil {
|
||||||
|
return projection{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
result := projection{
|
||||||
|
retainedIndexes: make([]int, 0, len(ids)),
|
||||||
|
oldToNewID: make(map[int]int, len(ids)),
|
||||||
|
removedIDs: make([]int, 0, len(ids)),
|
||||||
|
}
|
||||||
|
for index, id := range ids {
|
||||||
|
keep := opts.Mode == ModeKeep && opts.Selector.Contains(id)
|
||||||
|
if opts.Mode == ModeRemove {
|
||||||
|
keep = !opts.Selector.Contains(id)
|
||||||
|
}
|
||||||
|
if !keep {
|
||||||
|
result.removedIDs = append(result.removedIDs, id)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
result.retainedIndexes = append(result.retainedIndexes, index)
|
||||||
|
result.oldToNewID[id] = len(result.retainedIndexes)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(result.retainedIndexes) == 0 && !opts.AllowEmpty {
|
||||||
|
return projection{}, fmt.Errorf("trim operation produced an empty transcript; set AllowEmpty to true to permit this")
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateMode(mode Mode) error {
|
||||||
|
switch mode {
|
||||||
|
case ModeKeep, ModeRemove:
|
||||||
|
return nil
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("invalid trim mode %q", mode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateInputIDs(ids []int) (map[int]int, error) {
|
||||||
|
seen := make(map[int]int, len(ids))
|
||||||
|
for index, id := range ids {
|
||||||
|
if id <= 0 {
|
||||||
|
return nil, fmt.Errorf("input transcript has non-positive segment ID %d at index %d", id, index)
|
||||||
|
}
|
||||||
|
if firstIndex, exists := seen[id]; exists {
|
||||||
|
return nil, fmt.Errorf("input transcript has duplicate segment ID %d at indexes %d and %d", id, firstIndex, index)
|
||||||
|
}
|
||||||
|
seen[id] = index
|
||||||
|
}
|
||||||
|
|
||||||
|
for id := 1; id <= len(ids); id++ {
|
||||||
|
if _, exists := seen[id]; !exists {
|
||||||
|
return nil, fmt.Errorf("input transcript segment IDs must be sequential 1..%d; missing ID %d", len(ids), id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return seen, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateSelectedIDsExist(selected []int, idIndex map[int]int) error {
|
||||||
|
for _, id := range selected {
|
||||||
|
if _, exists := idIndex[id]; !exists {
|
||||||
|
return fmt.Errorf("selected segment ID %d does not exist in input transcript", id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func recomputeOverlapGroups(segments []schema.Segment) ([]schema.Segment, []schema.OverlapGroup) {
|
||||||
|
if len(segments) == 0 {
|
||||||
|
return segments, make([]schema.OverlapGroup, 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
modelSegments := make([]model.Segment, len(segments))
|
||||||
|
for index, segment := range segments {
|
||||||
|
modelSegments[index] = model.Segment{
|
||||||
|
ID: segment.ID,
|
||||||
|
Source: segment.Source,
|
||||||
|
SourceSegmentIndex: copyIntPtr(segment.SourceSegmentIndex),
|
||||||
|
SourceRef: segment.SourceRef,
|
||||||
|
DerivedFrom: append([]string(nil), segment.DerivedFrom...),
|
||||||
|
Speaker: segment.Speaker,
|
||||||
|
Start: segment.Start,
|
||||||
|
End: segment.End,
|
||||||
|
Text: segment.Text,
|
||||||
|
Categories: append([]string(nil), segment.Categories...),
|
||||||
|
OverlapGroupID: segment.OverlapGroupID,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
detected := overlap.Detect(model.MergedTranscript{
|
||||||
|
Segments: modelSegments,
|
||||||
|
})
|
||||||
|
rewrittenSegments := make([]schema.Segment, len(segments))
|
||||||
|
for index, segment := range segments {
|
||||||
|
rewritten := copySegment(segment)
|
||||||
|
rewritten.OverlapGroupID = detected.Segments[index].OverlapGroupID
|
||||||
|
rewrittenSegments[index] = rewritten
|
||||||
|
}
|
||||||
|
|
||||||
|
groups := make([]schema.OverlapGroup, len(detected.OverlapGroups))
|
||||||
|
for index, group := range detected.OverlapGroups {
|
||||||
|
groups[index] = schema.OverlapGroup{
|
||||||
|
ID: group.ID,
|
||||||
|
Start: group.Start,
|
||||||
|
End: group.End,
|
||||||
|
Segments: append([]string(nil), group.Segments...),
|
||||||
|
Speakers: append([]string(nil), group.Speakers...),
|
||||||
|
Class: group.Class,
|
||||||
|
Resolution: group.Resolution,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return rewrittenSegments, groups
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyTranscript(input schema.Transcript) schema.Transcript {
|
||||||
|
return schema.Transcript{
|
||||||
|
Metadata: schema.Metadata{
|
||||||
|
Application: input.Metadata.Application,
|
||||||
|
Version: input.Metadata.Version,
|
||||||
|
InputReader: input.Metadata.InputReader,
|
||||||
|
InputFiles: append([]string(nil), input.Metadata.InputFiles...),
|
||||||
|
PreprocessingModules: append([]string(nil), input.Metadata.PreprocessingModules...),
|
||||||
|
PostprocessingModules: append([]string(nil), input.Metadata.PostprocessingModules...),
|
||||||
|
OutputModules: append([]string(nil), input.Metadata.OutputModules...),
|
||||||
|
},
|
||||||
|
Segments: append([]schema.Segment(nil), input.Segments...),
|
||||||
|
OverlapGroups: append([]schema.OverlapGroup(nil), input.OverlapGroups...),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func copySegment(input schema.Segment) schema.Segment {
|
||||||
|
return schema.Segment{
|
||||||
|
ID: input.ID,
|
||||||
|
Source: input.Source,
|
||||||
|
SourceSegmentIndex: copyIntPtr(input.SourceSegmentIndex),
|
||||||
|
SourceRef: input.SourceRef,
|
||||||
|
DerivedFrom: append([]string(nil), input.DerivedFrom...),
|
||||||
|
Speaker: input.Speaker,
|
||||||
|
Start: input.Start,
|
||||||
|
End: input.End,
|
||||||
|
Text: input.Text,
|
||||||
|
Categories: append([]string(nil), input.Categories...),
|
||||||
|
OverlapGroupID: input.OverlapGroupID,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyIntPtr(value *int) *int {
|
||||||
|
if value == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
copied := *value
|
||||||
|
return &copied
|
||||||
|
}
|
||||||
873
internal/trim/apply_test.go
Normal file
873
internal/trim/apply_test.go
Normal file
@@ -0,0 +1,873 @@
|
|||||||
|
package trim
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/schema"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestApplyKeepModeRenumbersFromOne(t *testing.T) {
|
||||||
|
input := fullTranscriptFixture()
|
||||||
|
selector := mustParseSelector(t, "2,4")
|
||||||
|
|
||||||
|
result, err := Apply(input, Options{
|
||||||
|
Mode: ModeKeep,
|
||||||
|
Selector: selector,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("apply failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(result.Transcript.Segments) != 2 {
|
||||||
|
t.Fatalf("segment count = %d, want 2", len(result.Transcript.Segments))
|
||||||
|
}
|
||||||
|
assertSegmentIDs(t, result.Transcript.Segments, []int{1, 2})
|
||||||
|
assertSegmentTexts(t, result.Transcript.Segments, []string{"beta", "delta"})
|
||||||
|
assertIntMap(t, result.OldToNewID, map[int]int{2: 1, 4: 2})
|
||||||
|
assertIntSlice(t, result.RemovedIDs, []int{1, 3})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyRemoveModeRenumbersFromOne(t *testing.T) {
|
||||||
|
input := fullTranscriptFixture()
|
||||||
|
selector := mustParseSelector(t, "2,4")
|
||||||
|
|
||||||
|
result, err := Apply(input, Options{
|
||||||
|
Mode: ModeRemove,
|
||||||
|
Selector: selector,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("apply failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertSegmentIDs(t, result.Transcript.Segments, []int{1, 2})
|
||||||
|
assertSegmentTexts(t, result.Transcript.Segments, []string{"alpha", "gamma"})
|
||||||
|
assertIntMap(t, result.OldToNewID, map[int]int{1: 1, 3: 2})
|
||||||
|
assertIntSlice(t, result.RemovedIDs, []int{2, 4})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplySelectorOrderDoesNotChangeTranscriptOrder(t *testing.T) {
|
||||||
|
input := fullTranscriptFixture()
|
||||||
|
selector := mustParseSelector(t, "4,1,3")
|
||||||
|
|
||||||
|
result, err := Apply(input, Options{
|
||||||
|
Mode: ModeKeep,
|
||||||
|
Selector: selector,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("apply failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertSegmentIDs(t, result.Transcript.Segments, []int{1, 2, 3})
|
||||||
|
assertSegmentTexts(t, result.Transcript.Segments, []string{"alpha", "gamma", "delta"})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyFailsWhenSelectedIDDoesNotExist(t *testing.T) {
|
||||||
|
input := fullTranscriptFixture()
|
||||||
|
selector := mustParseSelector(t, "2,99")
|
||||||
|
|
||||||
|
_, err := Apply(input, Options{
|
||||||
|
Mode: ModeKeep,
|
||||||
|
Selector: selector,
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected missing selected ID error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "does not exist") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyFailsOnDuplicateInputIDs(t *testing.T) {
|
||||||
|
input := fullTranscriptFixture()
|
||||||
|
input.Segments[2].ID = 2
|
||||||
|
selector := mustParseSelector(t, "2")
|
||||||
|
|
||||||
|
_, err := Apply(input, Options{
|
||||||
|
Mode: ModeKeep,
|
||||||
|
Selector: selector,
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected duplicate input ID error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "duplicate segment ID") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyFailsOnMissingOrNonSequentialInputIDs(t *testing.T) {
|
||||||
|
input := fullTranscriptFixture()
|
||||||
|
input.Segments[1].ID = 5
|
||||||
|
selector := mustParseSelector(t, "1")
|
||||||
|
|
||||||
|
_, err := Apply(input, Options{
|
||||||
|
Mode: ModeKeep,
|
||||||
|
Selector: selector,
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected non-sequential input ID error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "must be sequential") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyFailsOnNonPositiveInputIDs(t *testing.T) {
|
||||||
|
input := fullTranscriptFixture()
|
||||||
|
input.Segments[0].ID = 0
|
||||||
|
selector := mustParseSelector(t, "1")
|
||||||
|
|
||||||
|
_, err := Apply(input, Options{
|
||||||
|
Mode: ModeKeep,
|
||||||
|
Selector: selector,
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected non-positive input ID error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "non-positive") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyEmptyOutputFailsUnlessAllowEmpty(t *testing.T) {
|
||||||
|
input := fullTranscriptFixture()
|
||||||
|
selector := mustParseSelector(t, "1-4")
|
||||||
|
|
||||||
|
_, err := Apply(input, Options{
|
||||||
|
Mode: ModeRemove,
|
||||||
|
Selector: selector,
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected empty-output error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "empty transcript") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
allowed, err := Apply(input, Options{
|
||||||
|
Mode: ModeRemove,
|
||||||
|
Selector: selector,
|
||||||
|
AllowEmpty: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("apply with AllowEmpty failed: %v", err)
|
||||||
|
}
|
||||||
|
if len(allowed.Transcript.Segments) != 0 {
|
||||||
|
t.Fatalf("segment count = %d, want 0", len(allowed.Transcript.Segments))
|
||||||
|
}
|
||||||
|
assertIntMap(t, allowed.OldToNewID, map[int]int{})
|
||||||
|
assertIntSlice(t, allowed.RemovedIDs, []int{1, 2, 3, 4})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyPreservesRetainedSegmentFieldsAndClearsOverlapIDs(t *testing.T) {
|
||||||
|
input := fullTranscriptFixture()
|
||||||
|
selector := mustParseSelector(t, "2")
|
||||||
|
|
||||||
|
result, err := Apply(input, Options{
|
||||||
|
Mode: ModeKeep,
|
||||||
|
Selector: selector,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("apply failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(result.Transcript.Segments) != 1 {
|
||||||
|
t.Fatalf("segment count = %d, want 1", len(result.Transcript.Segments))
|
||||||
|
}
|
||||||
|
segment := result.Transcript.Segments[0]
|
||||||
|
if segment.ID != 1 {
|
||||||
|
t.Fatalf("segment ID = %d, want 1", segment.ID)
|
||||||
|
}
|
||||||
|
if segment.Source != "b.json" {
|
||||||
|
t.Fatalf("source = %q, want %q", segment.Source, "b.json")
|
||||||
|
}
|
||||||
|
if segment.SourceSegmentIndex == nil || *segment.SourceSegmentIndex != 20 {
|
||||||
|
t.Fatalf("source_segment_index = %v, want 20", segment.SourceSegmentIndex)
|
||||||
|
}
|
||||||
|
if segment.SourceRef != "b.json#20" {
|
||||||
|
t.Fatalf("source_ref = %q, want %q", segment.SourceRef, "b.json#20")
|
||||||
|
}
|
||||||
|
if !equalStringSlices(segment.DerivedFrom, []string{"b.json#19", "b.json#20"}) {
|
||||||
|
t.Fatalf("derived_from = %v, want %v", segment.DerivedFrom, []string{"b.json#19", "b.json#20"})
|
||||||
|
}
|
||||||
|
if !equalStringSlices(segment.Categories, []string{"filler", "backchannel"}) {
|
||||||
|
t.Fatalf("categories = %v, want %v", segment.Categories, []string{"filler", "backchannel"})
|
||||||
|
}
|
||||||
|
if segment.Speaker != "Bob" {
|
||||||
|
t.Fatalf("speaker = %q, want Bob", segment.Speaker)
|
||||||
|
}
|
||||||
|
if segment.Start != 2 || segment.End != 3 {
|
||||||
|
t.Fatalf("times = %.3f-%.3f, want 2.000-3.000", segment.Start, segment.End)
|
||||||
|
}
|
||||||
|
if segment.Text != "beta" {
|
||||||
|
t.Fatalf("text = %q, want beta", segment.Text)
|
||||||
|
}
|
||||||
|
if segment.OverlapGroupID != 0 {
|
||||||
|
t.Fatalf("overlap_group_id = %d, want 0", segment.OverlapGroupID)
|
||||||
|
}
|
||||||
|
if len(result.Transcript.OverlapGroups) != 0 {
|
||||||
|
t.Fatalf("overlap_groups count = %d, want 0", len(result.Transcript.OverlapGroups))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyFullSchemaRemovesStaleOverlapGroups(t *testing.T) {
|
||||||
|
input := overlapTranscriptFixture()
|
||||||
|
selector := mustParseSelector(t, "1,3")
|
||||||
|
|
||||||
|
result, err := Apply(input, Options{
|
||||||
|
Mode: ModeKeep,
|
||||||
|
Selector: selector,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("apply failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(result.Transcript.OverlapGroups) != 0 {
|
||||||
|
t.Fatalf("overlap_groups count = %d, want 0", len(result.Transcript.OverlapGroups))
|
||||||
|
}
|
||||||
|
for index, segment := range result.Transcript.Segments {
|
||||||
|
if segment.OverlapGroupID != 0 {
|
||||||
|
t.Fatalf("segment %d overlap_group_id = %d, want 0", index, segment.OverlapGroupID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyFullSchemaRecomputesOverlapGroup(t *testing.T) {
|
||||||
|
input := overlapTranscriptFixture()
|
||||||
|
selector := mustParseSelector(t, "1,2")
|
||||||
|
|
||||||
|
result, err := Apply(input, Options{
|
||||||
|
Mode: ModeKeep,
|
||||||
|
Selector: selector,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("apply failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertSegmentIDs(t, result.Transcript.Segments, []int{1, 2})
|
||||||
|
assertIntSlice(t, []int{
|
||||||
|
result.Transcript.Segments[0].OverlapGroupID,
|
||||||
|
result.Transcript.Segments[1].OverlapGroupID,
|
||||||
|
}, []int{1, 1})
|
||||||
|
if len(result.Transcript.OverlapGroups) != 1 {
|
||||||
|
t.Fatalf("overlap_groups count = %d, want 1", len(result.Transcript.OverlapGroups))
|
||||||
|
}
|
||||||
|
group := result.Transcript.OverlapGroups[0]
|
||||||
|
if group.ID != 1 {
|
||||||
|
t.Fatalf("group ID = %d, want 1", group.ID)
|
||||||
|
}
|
||||||
|
if group.Start != 1 || group.End != 4 {
|
||||||
|
t.Fatalf("group times = %.3f-%.3f, want 1.000-4.000", group.Start, group.End)
|
||||||
|
}
|
||||||
|
if !equalStringSlices(group.Segments, []string{"a.json#10", "b.json#20"}) {
|
||||||
|
t.Fatalf("group segments = %v, want %v", group.Segments, []string{"a.json#10", "b.json#20"})
|
||||||
|
}
|
||||||
|
if !equalStringSlices(group.Speakers, []string{"Alice", "Bob"}) {
|
||||||
|
t.Fatalf("group speakers = %v, want %v", group.Speakers, []string{"Alice", "Bob"})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyFullSchemaDropsGroupWhenFewerThanTwoSpeakersRemain(t *testing.T) {
|
||||||
|
input := overlapTranscriptFixture()
|
||||||
|
selector := mustParseSelector(t, "1")
|
||||||
|
|
||||||
|
result, err := Apply(input, Options{
|
||||||
|
Mode: ModeKeep,
|
||||||
|
Selector: selector,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("apply failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(result.Transcript.OverlapGroups) != 0 {
|
||||||
|
t.Fatalf("overlap_groups count = %d, want 0", len(result.Transcript.OverlapGroups))
|
||||||
|
}
|
||||||
|
if len(result.Transcript.Segments) != 1 {
|
||||||
|
t.Fatalf("segment count = %d, want 1", len(result.Transcript.Segments))
|
||||||
|
}
|
||||||
|
if result.Transcript.Segments[0].OverlapGroupID != 0 {
|
||||||
|
t.Fatalf("segment overlap_group_id = %d, want 0", result.Transcript.Segments[0].OverlapGroupID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyFullSchemaHandlesTransitiveOverlaps(t *testing.T) {
|
||||||
|
input := transitiveOverlapFixture()
|
||||||
|
selector := mustParseSelector(t, "1-3")
|
||||||
|
|
||||||
|
result, err := Apply(input, Options{
|
||||||
|
Mode: ModeKeep,
|
||||||
|
Selector: selector,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("apply failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(result.Transcript.OverlapGroups) != 1 {
|
||||||
|
t.Fatalf("overlap_groups count = %d, want 1", len(result.Transcript.OverlapGroups))
|
||||||
|
}
|
||||||
|
assertIntSlice(t, []int{
|
||||||
|
result.Transcript.Segments[0].OverlapGroupID,
|
||||||
|
result.Transcript.Segments[1].OverlapGroupID,
|
||||||
|
result.Transcript.Segments[2].OverlapGroupID,
|
||||||
|
}, []int{1, 1, 1})
|
||||||
|
group := result.Transcript.OverlapGroups[0]
|
||||||
|
if group.Start != 10 || group.End != 15 {
|
||||||
|
t.Fatalf("group times = %.3f-%.3f, want 10.000-15.000", group.Start, group.End)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyFullSchemaBoundaryTouchingNotGrouped(t *testing.T) {
|
||||||
|
input := boundaryFixture()
|
||||||
|
selector := mustParseSelector(t, "1-2")
|
||||||
|
|
||||||
|
result, err := Apply(input, Options{
|
||||||
|
Mode: ModeKeep,
|
||||||
|
Selector: selector,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("apply failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(result.Transcript.OverlapGroups) != 0 {
|
||||||
|
t.Fatalf("overlap_groups count = %d, want 0", len(result.Transcript.OverlapGroups))
|
||||||
|
}
|
||||||
|
assertIntSlice(t, []int{
|
||||||
|
result.Transcript.Segments[0].OverlapGroupID,
|
||||||
|
result.Transcript.Segments[1].OverlapGroupID,
|
||||||
|
}, []int{0, 0})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyIntermediateDoesNotIncludeOverlapGroups(t *testing.T) {
|
||||||
|
input := schema.IntermediateTranscript{
|
||||||
|
Metadata: schema.IntermediateMetadata{
|
||||||
|
Application: "seriatim",
|
||||||
|
Version: "v-test",
|
||||||
|
OutputSchema: "seriatim-intermediate",
|
||||||
|
},
|
||||||
|
Segments: []schema.IntermediateSegment{
|
||||||
|
{ID: 1, Start: 1, End: 3, Speaker: "Alice", Text: "alpha", Categories: []string{"word-run"}},
|
||||||
|
{ID: 2, Start: 2, End: 4, Speaker: "Bob", Text: "beta", Categories: []string{"filler"}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
selector := mustParseSelector(t, "1")
|
||||||
|
result, err := ApplyIntermediate(input, Options{
|
||||||
|
Mode: ModeKeep,
|
||||||
|
Selector: selector,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("apply intermediate failed: %v", err)
|
||||||
|
}
|
||||||
|
if len(result.Transcript.Segments) != 1 {
|
||||||
|
t.Fatalf("segment count = %d, want 1", len(result.Transcript.Segments))
|
||||||
|
}
|
||||||
|
if result.Transcript.Segments[0].ID != 1 {
|
||||||
|
t.Fatalf("segment id = %d, want 1", result.Transcript.Segments[0].ID)
|
||||||
|
}
|
||||||
|
if err := schema.ValidateIntermediateTranscript(result.Transcript); err != nil {
|
||||||
|
t.Fatalf("intermediate output should remain valid: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyMinimalDoesNotIncludeOverlapGroups(t *testing.T) {
|
||||||
|
input := schema.MinimalTranscript{
|
||||||
|
Metadata: schema.MinimalMetadata{
|
||||||
|
Application: "seriatim",
|
||||||
|
Version: "v-test",
|
||||||
|
OutputSchema: "seriatim-minimal",
|
||||||
|
},
|
||||||
|
Segments: []schema.MinimalSegment{
|
||||||
|
{ID: 1, Start: 1, End: 3, Speaker: "Alice", Text: "alpha"},
|
||||||
|
{ID: 2, Start: 2, End: 4, Speaker: "Bob", Text: "beta"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
selector := mustParseSelector(t, "2")
|
||||||
|
result, err := ApplyMinimal(input, Options{
|
||||||
|
Mode: ModeKeep,
|
||||||
|
Selector: selector,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("apply minimal failed: %v", err)
|
||||||
|
}
|
||||||
|
if len(result.Transcript.Segments) != 1 {
|
||||||
|
t.Fatalf("segment count = %d, want 1", len(result.Transcript.Segments))
|
||||||
|
}
|
||||||
|
if result.Transcript.Segments[0].ID != 1 {
|
||||||
|
t.Fatalf("segment id = %d, want 1", result.Transcript.Segments[0].ID)
|
||||||
|
}
|
||||||
|
if err := schema.ValidateMinimalTranscript(result.Transcript); err != nil {
|
||||||
|
t.Fatalf("minimal output should remain valid: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplySelectorPolicyIsSharedAcrossSchemas(t *testing.T) {
|
||||||
|
type testCase struct {
|
||||||
|
name string
|
||||||
|
opts Options
|
||||||
|
wantTexts []string
|
||||||
|
wantOldToNew map[int]int
|
||||||
|
wantRemoved []int
|
||||||
|
wantSegmentCount int
|
||||||
|
wantErrorSubstring string
|
||||||
|
}
|
||||||
|
|
||||||
|
cases := []testCase{
|
||||||
|
{
|
||||||
|
name: "keep preserves input order regardless of selector order",
|
||||||
|
opts: Options{Mode: ModeKeep, Selector: mustParseSelector(t, "4,1,3")},
|
||||||
|
wantTexts: []string{"alpha", "gamma", "delta"},
|
||||||
|
wantOldToNew: map[int]int{1: 1, 3: 2, 4: 3},
|
||||||
|
wantRemoved: []int{2},
|
||||||
|
wantSegmentCount: 3,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "remove reports deterministic renumbering metadata",
|
||||||
|
opts: Options{Mode: ModeRemove, Selector: mustParseSelector(t, "2,4")},
|
||||||
|
wantTexts: []string{"alpha", "gamma"},
|
||||||
|
wantOldToNew: map[int]int{1: 1, 3: 2},
|
||||||
|
wantRemoved: []int{2, 4},
|
||||||
|
wantSegmentCount: 2,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "missing selected id returns error",
|
||||||
|
opts: Options{Mode: ModeKeep, Selector: mustParseSelector(t, "9")},
|
||||||
|
wantErrorSubstring: "does not exist",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty selector returns error",
|
||||||
|
opts: Options{Mode: ModeKeep, Selector: Selector{}},
|
||||||
|
wantErrorSubstring: "selector cannot be empty",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid mode returns error",
|
||||||
|
opts: Options{Mode: Mode("bad"), Selector: mustParseSelector(t, "1")},
|
||||||
|
wantErrorSubstring: `invalid trim mode "bad"`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty output blocked when allow empty is false",
|
||||||
|
opts: Options{Mode: ModeRemove, Selector: mustParseSelector(t, "1-4")},
|
||||||
|
wantErrorSubstring: "empty transcript",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty output allowed when allow empty is true",
|
||||||
|
opts: Options{Mode: ModeRemove, Selector: mustParseSelector(t, "1-4"), AllowEmpty: true},
|
||||||
|
wantTexts: []string{},
|
||||||
|
wantOldToNew: map[int]int{},
|
||||||
|
wantRemoved: []int{1, 2, 3, 4},
|
||||||
|
wantSegmentCount: 0,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range cases {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
fullInput := fullTranscriptFixture()
|
||||||
|
intermediateInput := intermediateFixture()
|
||||||
|
minimalInput := minimalFixture()
|
||||||
|
|
||||||
|
fullResult, fullErr := Apply(fullInput, test.opts)
|
||||||
|
intermediateResult, intermediateErr := ApplyIntermediate(intermediateInput, test.opts)
|
||||||
|
minimalResult, minimalErr := ApplyMinimal(minimalInput, test.opts)
|
||||||
|
|
||||||
|
if test.wantErrorSubstring != "" {
|
||||||
|
assertErrorContains(t, fullErr, test.wantErrorSubstring)
|
||||||
|
assertErrorContains(t, intermediateErr, test.wantErrorSubstring)
|
||||||
|
assertErrorContains(t, minimalErr, test.wantErrorSubstring)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if fullErr != nil {
|
||||||
|
t.Fatalf("apply full failed: %v", fullErr)
|
||||||
|
}
|
||||||
|
if intermediateErr != nil {
|
||||||
|
t.Fatalf("apply intermediate failed: %v", intermediateErr)
|
||||||
|
}
|
||||||
|
if minimalErr != nil {
|
||||||
|
t.Fatalf("apply minimal failed: %v", minimalErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertIntSlice(t, extractFullIDs(fullResult.Transcript.Segments), extractSequentialIDs(test.wantSegmentCount))
|
||||||
|
assertIntSlice(t, extractIntermediateIDs(intermediateResult.Transcript.Segments), extractSequentialIDs(test.wantSegmentCount))
|
||||||
|
assertIntSlice(t, extractMinimalIDs(minimalResult.Transcript.Segments), extractSequentialIDs(test.wantSegmentCount))
|
||||||
|
assertStringSlice(t, extractFullTexts(fullResult.Transcript.Segments), test.wantTexts)
|
||||||
|
assertStringSlice(t, extractIntermediateTexts(intermediateResult.Transcript.Segments), test.wantTexts)
|
||||||
|
assertStringSlice(t, extractMinimalTexts(minimalResult.Transcript.Segments), test.wantTexts)
|
||||||
|
assertIntMap(t, fullResult.OldToNewID, test.wantOldToNew)
|
||||||
|
assertIntMap(t, intermediateResult.OldToNewID, test.wantOldToNew)
|
||||||
|
assertIntMap(t, minimalResult.OldToNewID, test.wantOldToNew)
|
||||||
|
assertIntSlice(t, fullResult.RemovedIDs, test.wantRemoved)
|
||||||
|
assertIntSlice(t, intermediateResult.RemovedIDs, test.wantRemoved)
|
||||||
|
assertIntSlice(t, minimalResult.RemovedIDs, test.wantRemoved)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyOutputInvariantsValidAfterRenumberAndOverlapRecompute(t *testing.T) {
|
||||||
|
input := overlapTranscriptFixture()
|
||||||
|
selector := mustParseSelector(t, "2,1")
|
||||||
|
|
||||||
|
result, err := Apply(input, Options{
|
||||||
|
Mode: ModeKeep,
|
||||||
|
Selector: selector,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("apply failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := schema.ValidateTranscript(result.Transcript); err != nil {
|
||||||
|
t.Fatalf("trim output should remain valid: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func mustParseSelector(t *testing.T, value string) Selector {
|
||||||
|
t.Helper()
|
||||||
|
selector, err := ParseSelector(value)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("selector parse failed for %q: %v", value, err)
|
||||||
|
}
|
||||||
|
return selector
|
||||||
|
}
|
||||||
|
|
||||||
|
func fullTranscriptFixture() schema.Transcript {
|
||||||
|
firstIndex := 10
|
||||||
|
secondIndex := 20
|
||||||
|
thirdIndex := 30
|
||||||
|
fourthIndex := 40
|
||||||
|
|
||||||
|
return schema.Transcript{
|
||||||
|
Metadata: schema.Metadata{
|
||||||
|
Application: "seriatim",
|
||||||
|
Version: "v-test",
|
||||||
|
InputReader: "json-files",
|
||||||
|
InputFiles: []string{"a.json", "b.json"},
|
||||||
|
PreprocessingModules: []string{"validate-raw"},
|
||||||
|
PostprocessingModules: []string{"detect-overlaps"},
|
||||||
|
OutputModules: []string{"json"},
|
||||||
|
},
|
||||||
|
Segments: []schema.Segment{
|
||||||
|
{
|
||||||
|
ID: 1,
|
||||||
|
Source: "a.json",
|
||||||
|
SourceSegmentIndex: &firstIndex,
|
||||||
|
SourceRef: "a.json#10",
|
||||||
|
DerivedFrom: []string{"a.json#10"},
|
||||||
|
Speaker: "Alice",
|
||||||
|
Start: 1,
|
||||||
|
End: 2,
|
||||||
|
Text: "alpha",
|
||||||
|
Categories: []string{"word-run"},
|
||||||
|
OverlapGroupID: 7,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ID: 2,
|
||||||
|
Source: "b.json",
|
||||||
|
SourceSegmentIndex: &secondIndex,
|
||||||
|
SourceRef: "b.json#20",
|
||||||
|
DerivedFrom: []string{"b.json#19", "b.json#20"},
|
||||||
|
Speaker: "Bob",
|
||||||
|
Start: 2,
|
||||||
|
End: 3,
|
||||||
|
Text: "beta",
|
||||||
|
Categories: []string{"filler", "backchannel"},
|
||||||
|
OverlapGroupID: 7,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ID: 3,
|
||||||
|
Source: "c.json",
|
||||||
|
SourceSegmentIndex: &thirdIndex,
|
||||||
|
SourceRef: "c.json#30",
|
||||||
|
DerivedFrom: []string{"c.json#30"},
|
||||||
|
Speaker: "Carol",
|
||||||
|
Start: 3,
|
||||||
|
End: 4,
|
||||||
|
Text: "gamma",
|
||||||
|
Categories: []string{"normal"},
|
||||||
|
OverlapGroupID: 8,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ID: 4,
|
||||||
|
Source: "d.json",
|
||||||
|
SourceSegmentIndex: &fourthIndex,
|
||||||
|
SourceRef: "d.json#40",
|
||||||
|
DerivedFrom: []string{"d.json#40"},
|
||||||
|
Speaker: "Dan",
|
||||||
|
Start: 4,
|
||||||
|
End: 5,
|
||||||
|
Text: "delta",
|
||||||
|
Categories: []string{"normal"},
|
||||||
|
OverlapGroupID: 9,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
OverlapGroups: []schema.OverlapGroup{
|
||||||
|
{
|
||||||
|
ID: 7,
|
||||||
|
Start: 1.5,
|
||||||
|
End: 3.1,
|
||||||
|
Segments: []string{"a.json#10", "b.json#20"},
|
||||||
|
Speakers: []string{"Alice", "Bob"},
|
||||||
|
Class: "unknown",
|
||||||
|
Resolution: "unresolved",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func overlapTranscriptFixture() schema.Transcript {
|
||||||
|
first := 10
|
||||||
|
second := 20
|
||||||
|
third := 30
|
||||||
|
|
||||||
|
return schema.Transcript{
|
||||||
|
Metadata: schema.Metadata{
|
||||||
|
Application: "seriatim",
|
||||||
|
Version: "v-test",
|
||||||
|
InputReader: "json-files",
|
||||||
|
InputFiles: []string{"a.json", "b.json", "c.json"},
|
||||||
|
PreprocessingModules: []string{"validate-raw"},
|
||||||
|
PostprocessingModules: []string{"detect-overlaps"},
|
||||||
|
OutputModules: []string{"json"},
|
||||||
|
},
|
||||||
|
Segments: []schema.Segment{
|
||||||
|
{
|
||||||
|
ID: 1,
|
||||||
|
Source: "a.json",
|
||||||
|
SourceSegmentIndex: &first,
|
||||||
|
SourceRef: "a.json#10",
|
||||||
|
Speaker: "Alice",
|
||||||
|
Start: 1,
|
||||||
|
End: 4,
|
||||||
|
Text: "a",
|
||||||
|
OverlapGroupID: 99,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ID: 2,
|
||||||
|
Source: "b.json",
|
||||||
|
SourceSegmentIndex: &second,
|
||||||
|
SourceRef: "b.json#20",
|
||||||
|
Speaker: "Bob",
|
||||||
|
Start: 2,
|
||||||
|
End: 3,
|
||||||
|
Text: "b",
|
||||||
|
OverlapGroupID: 99,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ID: 3,
|
||||||
|
Source: "c.json",
|
||||||
|
SourceSegmentIndex: &third,
|
||||||
|
SourceRef: "c.json#30",
|
||||||
|
Speaker: "Carol",
|
||||||
|
Start: 10,
|
||||||
|
End: 11,
|
||||||
|
Text: "c",
|
||||||
|
OverlapGroupID: 100,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
OverlapGroups: []schema.OverlapGroup{
|
||||||
|
{
|
||||||
|
ID: 99,
|
||||||
|
Start: 0,
|
||||||
|
End: 100,
|
||||||
|
Segments: []string{"stale#1", "stale#2"},
|
||||||
|
Speakers: []string{"stale"},
|
||||||
|
Class: "unknown",
|
||||||
|
Resolution: "unresolved",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func transitiveOverlapFixture() schema.Transcript {
|
||||||
|
one := 1
|
||||||
|
two := 2
|
||||||
|
three := 3
|
||||||
|
return schema.Transcript{
|
||||||
|
Metadata: schema.Metadata{
|
||||||
|
Application: "seriatim",
|
||||||
|
Version: "v-test",
|
||||||
|
},
|
||||||
|
Segments: []schema.Segment{
|
||||||
|
{ID: 1, Source: "a.json", SourceSegmentIndex: &one, Speaker: "Alice", Start: 10, End: 14, Text: "a"},
|
||||||
|
{ID: 2, Source: "b.json", SourceSegmentIndex: &two, Speaker: "Bob", Start: 12, End: 13, Text: "b"},
|
||||||
|
{ID: 3, Source: "c.json", SourceSegmentIndex: &three, Speaker: "Carol", Start: 13.5, End: 15, Text: "c"},
|
||||||
|
},
|
||||||
|
OverlapGroups: []schema.OverlapGroup{{ID: 77}},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func boundaryFixture() schema.Transcript {
|
||||||
|
one := 1
|
||||||
|
two := 2
|
||||||
|
return schema.Transcript{
|
||||||
|
Metadata: schema.Metadata{
|
||||||
|
Application: "seriatim",
|
||||||
|
Version: "v-test",
|
||||||
|
},
|
||||||
|
Segments: []schema.Segment{
|
||||||
|
{ID: 1, Source: "a.json", SourceSegmentIndex: &one, Speaker: "Alice", Start: 1, End: 2, Text: "a", OverlapGroupID: 7},
|
||||||
|
{ID: 2, Source: "b.json", SourceSegmentIndex: &two, Speaker: "Bob", Start: 2, End: 3, Text: "b", OverlapGroupID: 7},
|
||||||
|
},
|
||||||
|
OverlapGroups: []schema.OverlapGroup{{ID: 7, Start: 1, End: 3}},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertSegmentIDs(t *testing.T, segments []schema.Segment, want []int) {
|
||||||
|
t.Helper()
|
||||||
|
got := make([]int, len(segments))
|
||||||
|
for index, segment := range segments {
|
||||||
|
got[index] = segment.ID
|
||||||
|
}
|
||||||
|
assertIntSlice(t, got, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertSegmentTexts(t *testing.T, segments []schema.Segment, want []string) {
|
||||||
|
t.Helper()
|
||||||
|
got := make([]string, len(segments))
|
||||||
|
for index, segment := range segments {
|
||||||
|
got[index] = segment.Text
|
||||||
|
}
|
||||||
|
if !equalStringSlices(got, want) {
|
||||||
|
t.Fatalf("segment texts = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertIntSlice(t *testing.T, got []int, want []int) {
|
||||||
|
t.Helper()
|
||||||
|
if len(got) != len(want) {
|
||||||
|
t.Fatalf("slice length = %d, want %d", len(got), len(want))
|
||||||
|
}
|
||||||
|
for index := range got {
|
||||||
|
if got[index] != want[index] {
|
||||||
|
t.Fatalf("slice[%d] = %d, want %d (full got=%v, want=%v)", index, got[index], want[index], got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertIntMap(t *testing.T, got map[int]int, want map[int]int) {
|
||||||
|
t.Helper()
|
||||||
|
if len(got) != len(want) {
|
||||||
|
t.Fatalf("map length = %d, want %d", len(got), len(want))
|
||||||
|
}
|
||||||
|
for key, wantValue := range want {
|
||||||
|
gotValue, exists := got[key]
|
||||||
|
if !exists {
|
||||||
|
t.Fatalf("missing map key %d", key)
|
||||||
|
}
|
||||||
|
if gotValue != wantValue {
|
||||||
|
t.Fatalf("map[%d] = %d, want %d", key, gotValue, wantValue)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func equalStringSlices(got []string, want []string) bool {
|
||||||
|
if len(got) != len(want) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for index := range got {
|
||||||
|
if got[index] != want[index] {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertErrorContains(t *testing.T, err error, substring string) {
|
||||||
|
t.Helper()
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("expected error containing %q", substring)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), substring) {
|
||||||
|
t.Fatalf("error %q does not contain %q", err.Error(), substring)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertStringSlice(t *testing.T, got []string, want []string) {
|
||||||
|
t.Helper()
|
||||||
|
if !equalStringSlices(got, want) {
|
||||||
|
t.Fatalf("slice = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractSequentialIDs(count int) []int {
|
||||||
|
ids := make([]int, count)
|
||||||
|
for index := range ids {
|
||||||
|
ids[index] = index + 1
|
||||||
|
}
|
||||||
|
return ids
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractFullIDs(segments []schema.Segment) []int {
|
||||||
|
ids := make([]int, len(segments))
|
||||||
|
for index, segment := range segments {
|
||||||
|
ids[index] = segment.ID
|
||||||
|
}
|
||||||
|
return ids
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractIntermediateIDs(segments []schema.IntermediateSegment) []int {
|
||||||
|
ids := make([]int, len(segments))
|
||||||
|
for index, segment := range segments {
|
||||||
|
ids[index] = segment.ID
|
||||||
|
}
|
||||||
|
return ids
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractMinimalIDs(segments []schema.MinimalSegment) []int {
|
||||||
|
ids := make([]int, len(segments))
|
||||||
|
for index, segment := range segments {
|
||||||
|
ids[index] = segment.ID
|
||||||
|
}
|
||||||
|
return ids
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractFullTexts(segments []schema.Segment) []string {
|
||||||
|
texts := make([]string, len(segments))
|
||||||
|
for index, segment := range segments {
|
||||||
|
texts[index] = segment.Text
|
||||||
|
}
|
||||||
|
return texts
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractIntermediateTexts(segments []schema.IntermediateSegment) []string {
|
||||||
|
texts := make([]string, len(segments))
|
||||||
|
for index, segment := range segments {
|
||||||
|
texts[index] = segment.Text
|
||||||
|
}
|
||||||
|
return texts
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractMinimalTexts(segments []schema.MinimalSegment) []string {
|
||||||
|
texts := make([]string, len(segments))
|
||||||
|
for index, segment := range segments {
|
||||||
|
texts[index] = segment.Text
|
||||||
|
}
|
||||||
|
return texts
|
||||||
|
}
|
||||||
|
|
||||||
|
func intermediateFixture() schema.IntermediateTranscript {
|
||||||
|
return schema.IntermediateTranscript{
|
||||||
|
Metadata: schema.IntermediateMetadata{
|
||||||
|
Application: "seriatim",
|
||||||
|
Version: "v-test",
|
||||||
|
OutputSchema: schema.OutputSchemaIntermediate,
|
||||||
|
},
|
||||||
|
Segments: []schema.IntermediateSegment{
|
||||||
|
{ID: 1, Start: 1, End: 2, Speaker: "Alice", Text: "alpha", Categories: []string{"word-run"}},
|
||||||
|
{ID: 2, Start: 2, End: 3, Speaker: "Bob", Text: "beta", Categories: []string{"filler", "backchannel"}},
|
||||||
|
{ID: 3, Start: 3, End: 4, Speaker: "Carol", Text: "gamma", Categories: []string{"normal"}},
|
||||||
|
{ID: 4, Start: 4, End: 5, Speaker: "Dan", Text: "delta", Categories: []string{"normal"}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func minimalFixture() schema.MinimalTranscript {
|
||||||
|
return schema.MinimalTranscript{
|
||||||
|
Metadata: schema.MinimalMetadata{
|
||||||
|
Application: "seriatim",
|
||||||
|
Version: "v-test",
|
||||||
|
OutputSchema: schema.OutputSchemaMinimal,
|
||||||
|
},
|
||||||
|
Segments: []schema.MinimalSegment{
|
||||||
|
{ID: 1, Start: 1, End: 2, Speaker: "Alice", Text: "alpha"},
|
||||||
|
{ID: 2, Start: 2, End: 3, Speaker: "Bob", Text: "beta"},
|
||||||
|
{ID: 3, Start: 3, End: 4, Speaker: "Carol", Text: "gamma"},
|
||||||
|
{ID: 4, Start: 4, End: 5, Speaker: "Dan", Text: "delta"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
426
internal/trim/artifact.go
Normal file
426
internal/trim/artifact.go
Normal file
@@ -0,0 +1,426 @@
|
|||||||
|
package trim
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/schema"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
SchemaMinimal = schema.OutputSchemaMinimal
|
||||||
|
SchemaIntermediate = schema.OutputSchemaIntermediate
|
||||||
|
SchemaFull = schema.OutputSchemaFull
|
||||||
|
)
|
||||||
|
|
||||||
|
// Artifact stores a parsed seriatim output artifact of one supported schema.
|
||||||
|
type Artifact struct {
|
||||||
|
Schema string
|
||||||
|
Full *schema.Transcript
|
||||||
|
Intermediate *schema.IntermediateTranscript
|
||||||
|
Minimal *schema.MinimalTranscript
|
||||||
|
}
|
||||||
|
|
||||||
|
// ApplyArtifactResult contains trimmed artifact output and ID mapping metadata.
|
||||||
|
type ApplyArtifactResult struct {
|
||||||
|
Artifact Artifact
|
||||||
|
OldToNewID map[int]int
|
||||||
|
RemovedIDs []int
|
||||||
|
OverlapGroupsRecomputed bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseArtifactJSON parses and validates a serialized seriatim output artifact.
|
||||||
|
func ParseArtifactJSON(data []byte) (Artifact, error) {
|
||||||
|
var decoded any
|
||||||
|
if err := json.Unmarshal(data, &decoded); err != nil {
|
||||||
|
return Artifact{}, fmt.Errorf("input JSON is malformed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var full schema.Transcript
|
||||||
|
if err := json.Unmarshal(data, &full); err == nil {
|
||||||
|
if err := schema.ValidateTranscript(full); err == nil {
|
||||||
|
return Artifact{
|
||||||
|
Schema: SchemaFull,
|
||||||
|
Full: &full,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var intermediate schema.IntermediateTranscript
|
||||||
|
if err := json.Unmarshal(data, &intermediate); err == nil {
|
||||||
|
if err := schema.ValidateIntermediateTranscript(intermediate); err == nil {
|
||||||
|
return Artifact{
|
||||||
|
Schema: SchemaIntermediate,
|
||||||
|
Intermediate: &intermediate,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var minimal schema.MinimalTranscript
|
||||||
|
if err := json.Unmarshal(data, &minimal); err == nil {
|
||||||
|
if err := schema.ValidateMinimalTranscript(minimal); err == nil {
|
||||||
|
return Artifact{
|
||||||
|
Schema: SchemaMinimal,
|
||||||
|
Minimal: &minimal,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return Artifact{}, fmt.Errorf("input JSON is not a valid seriatim output artifact")
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidateArtifact validates an artifact against its declared schema.
|
||||||
|
func ValidateArtifact(artifact Artifact) error {
|
||||||
|
switch artifact.Schema {
|
||||||
|
case SchemaFull:
|
||||||
|
payload, err := artifact.fullPayload()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return schema.ValidateTranscript(*payload)
|
||||||
|
case SchemaIntermediate:
|
||||||
|
payload, err := artifact.intermediatePayload()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return schema.ValidateIntermediateTranscript(*payload)
|
||||||
|
case SchemaMinimal:
|
||||||
|
payload, err := artifact.minimalPayload()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return schema.ValidateMinimalTranscript(*payload)
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("unsupported artifact schema %q", artifact.Schema)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Value returns the artifact value for JSON serialization.
|
||||||
|
func (artifact Artifact) Value() any {
|
||||||
|
switch artifact.Schema {
|
||||||
|
case SchemaFull:
|
||||||
|
if artifact.Full == nil {
|
||||||
|
return schema.Transcript{}
|
||||||
|
}
|
||||||
|
return *artifact.Full
|
||||||
|
case SchemaIntermediate:
|
||||||
|
if artifact.Intermediate == nil {
|
||||||
|
return schema.IntermediateTranscript{}
|
||||||
|
}
|
||||||
|
return *artifact.Intermediate
|
||||||
|
case SchemaMinimal:
|
||||||
|
if artifact.Minimal == nil {
|
||||||
|
return schema.MinimalTranscript{}
|
||||||
|
}
|
||||||
|
return *artifact.Minimal
|
||||||
|
default:
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SegmentCount returns the number of segments in the artifact.
|
||||||
|
func (artifact Artifact) SegmentCount() int {
|
||||||
|
switch artifact.Schema {
|
||||||
|
case SchemaFull:
|
||||||
|
if artifact.Full == nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return len(artifact.Full.Segments)
|
||||||
|
case SchemaIntermediate:
|
||||||
|
if artifact.Intermediate == nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return len(artifact.Intermediate.Segments)
|
||||||
|
case SchemaMinimal:
|
||||||
|
if artifact.Minimal == nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return len(artifact.Minimal.Segments)
|
||||||
|
default:
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Application returns artifact metadata application name.
|
||||||
|
func (artifact Artifact) Application() string {
|
||||||
|
switch artifact.Schema {
|
||||||
|
case SchemaFull:
|
||||||
|
if artifact.Full == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return artifact.Full.Metadata.Application
|
||||||
|
case SchemaIntermediate:
|
||||||
|
if artifact.Intermediate == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return artifact.Intermediate.Metadata.Application
|
||||||
|
case SchemaMinimal:
|
||||||
|
if artifact.Minimal == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return artifact.Minimal.Metadata.Application
|
||||||
|
default:
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Version returns artifact metadata version.
|
||||||
|
func (artifact Artifact) Version() string {
|
||||||
|
switch artifact.Schema {
|
||||||
|
case SchemaFull:
|
||||||
|
if artifact.Full == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return artifact.Full.Metadata.Version
|
||||||
|
case SchemaIntermediate:
|
||||||
|
if artifact.Intermediate == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return artifact.Intermediate.Metadata.Version
|
||||||
|
case SchemaMinimal:
|
||||||
|
if artifact.Minimal == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return artifact.Minimal.Metadata.Version
|
||||||
|
default:
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ApplyArtifact trims a parsed artifact while preserving its input schema.
|
||||||
|
func ApplyArtifact(input Artifact, opts Options) (ApplyArtifactResult, error) {
|
||||||
|
switch input.Schema {
|
||||||
|
case SchemaFull:
|
||||||
|
payload, err := input.fullPayload()
|
||||||
|
if err != nil {
|
||||||
|
return ApplyArtifactResult{}, err
|
||||||
|
}
|
||||||
|
result, err := Apply(*payload, opts)
|
||||||
|
if err != nil {
|
||||||
|
return ApplyArtifactResult{}, err
|
||||||
|
}
|
||||||
|
out := result.Transcript
|
||||||
|
return ApplyArtifactResult{
|
||||||
|
Artifact: Artifact{
|
||||||
|
Schema: SchemaFull,
|
||||||
|
Full: &out,
|
||||||
|
},
|
||||||
|
OldToNewID: result.OldToNewID,
|
||||||
|
RemovedIDs: result.RemovedIDs,
|
||||||
|
OverlapGroupsRecomputed: true,
|
||||||
|
}, nil
|
||||||
|
case SchemaIntermediate:
|
||||||
|
payload, err := input.intermediatePayload()
|
||||||
|
if err != nil {
|
||||||
|
return ApplyArtifactResult{}, err
|
||||||
|
}
|
||||||
|
result, err := ApplyIntermediate(*payload, opts)
|
||||||
|
if err != nil {
|
||||||
|
return ApplyArtifactResult{}, err
|
||||||
|
}
|
||||||
|
out := result.Transcript
|
||||||
|
return ApplyArtifactResult{
|
||||||
|
Artifact: Artifact{
|
||||||
|
Schema: SchemaIntermediate,
|
||||||
|
Intermediate: &out,
|
||||||
|
},
|
||||||
|
OldToNewID: result.OldToNewID,
|
||||||
|
RemovedIDs: result.RemovedIDs,
|
||||||
|
OverlapGroupsRecomputed: false,
|
||||||
|
}, nil
|
||||||
|
case SchemaMinimal:
|
||||||
|
payload, err := input.minimalPayload()
|
||||||
|
if err != nil {
|
||||||
|
return ApplyArtifactResult{}, err
|
||||||
|
}
|
||||||
|
result, err := ApplyMinimal(*payload, opts)
|
||||||
|
if err != nil {
|
||||||
|
return ApplyArtifactResult{}, err
|
||||||
|
}
|
||||||
|
out := result.Transcript
|
||||||
|
return ApplyArtifactResult{
|
||||||
|
Artifact: Artifact{
|
||||||
|
Schema: SchemaMinimal,
|
||||||
|
Minimal: &out,
|
||||||
|
},
|
||||||
|
OldToNewID: result.OldToNewID,
|
||||||
|
RemovedIDs: result.RemovedIDs,
|
||||||
|
OverlapGroupsRecomputed: false,
|
||||||
|
}, nil
|
||||||
|
default:
|
||||||
|
return ApplyArtifactResult{}, fmt.Errorf("unsupported artifact schema %q", input.Schema)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ConvertArtifact converts a parsed artifact to another supported output schema.
|
||||||
|
func ConvertArtifact(input Artifact, outputSchema string) (Artifact, error) {
|
||||||
|
if outputSchema == "" || outputSchema == input.Schema {
|
||||||
|
return input, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
switch input.Schema {
|
||||||
|
case SchemaFull:
|
||||||
|
payload, err := input.fullPayload()
|
||||||
|
if err != nil {
|
||||||
|
return Artifact{}, err
|
||||||
|
}
|
||||||
|
switch outputSchema {
|
||||||
|
case SchemaIntermediate:
|
||||||
|
out := intermediateFromFull(*payload)
|
||||||
|
return Artifact{
|
||||||
|
Schema: SchemaIntermediate,
|
||||||
|
Intermediate: &out,
|
||||||
|
}, nil
|
||||||
|
case SchemaMinimal:
|
||||||
|
out := minimalFromFull(*payload)
|
||||||
|
return Artifact{
|
||||||
|
Schema: SchemaMinimal,
|
||||||
|
Minimal: &out,
|
||||||
|
}, nil
|
||||||
|
default:
|
||||||
|
return Artifact{}, fmt.Errorf("unsupported output schema %q", outputSchema)
|
||||||
|
}
|
||||||
|
case SchemaIntermediate:
|
||||||
|
payload, err := input.intermediatePayload()
|
||||||
|
if err != nil {
|
||||||
|
return Artifact{}, err
|
||||||
|
}
|
||||||
|
switch outputSchema {
|
||||||
|
case SchemaMinimal:
|
||||||
|
out := minimalFromIntermediate(*payload)
|
||||||
|
return Artifact{
|
||||||
|
Schema: SchemaMinimal,
|
||||||
|
Minimal: &out,
|
||||||
|
}, nil
|
||||||
|
case SchemaFull:
|
||||||
|
return Artifact{}, fmt.Errorf("cannot emit %q from %q input artifact", SchemaFull, SchemaIntermediate)
|
||||||
|
default:
|
||||||
|
return Artifact{}, fmt.Errorf("unsupported output schema %q", outputSchema)
|
||||||
|
}
|
||||||
|
case SchemaMinimal:
|
||||||
|
payload, err := input.minimalPayload()
|
||||||
|
if err != nil {
|
||||||
|
return Artifact{}, err
|
||||||
|
}
|
||||||
|
switch outputSchema {
|
||||||
|
case SchemaIntermediate:
|
||||||
|
out := intermediateFromMinimal(*payload)
|
||||||
|
return Artifact{
|
||||||
|
Schema: SchemaIntermediate,
|
||||||
|
Intermediate: &out,
|
||||||
|
}, nil
|
||||||
|
case SchemaFull:
|
||||||
|
return Artifact{}, fmt.Errorf("cannot emit %q from %q input artifact", SchemaFull, SchemaMinimal)
|
||||||
|
default:
|
||||||
|
return Artifact{}, fmt.Errorf("unsupported output schema %q", outputSchema)
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
return Artifact{}, fmt.Errorf("unsupported input schema %q", input.Schema)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (artifact Artifact) fullPayload() (*schema.Transcript, error) {
|
||||||
|
if artifact.Full == nil {
|
||||||
|
return nil, fmt.Errorf("full artifact payload is missing")
|
||||||
|
}
|
||||||
|
return artifact.Full, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (artifact Artifact) intermediatePayload() (*schema.IntermediateTranscript, error) {
|
||||||
|
if artifact.Intermediate == nil {
|
||||||
|
return nil, fmt.Errorf("intermediate artifact payload is missing")
|
||||||
|
}
|
||||||
|
return artifact.Intermediate, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (artifact Artifact) minimalPayload() (*schema.MinimalTranscript, error) {
|
||||||
|
if artifact.Minimal == nil {
|
||||||
|
return nil, fmt.Errorf("minimal artifact payload is missing")
|
||||||
|
}
|
||||||
|
return artifact.Minimal, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func intermediateFromFull(input schema.Transcript) schema.IntermediateTranscript {
|
||||||
|
segments := make([]schema.IntermediateSegment, len(input.Segments))
|
||||||
|
for index, segment := range input.Segments {
|
||||||
|
segments[index] = schema.IntermediateSegment{
|
||||||
|
ID: segment.ID,
|
||||||
|
Start: segment.Start,
|
||||||
|
End: segment.End,
|
||||||
|
Speaker: segment.Speaker,
|
||||||
|
Text: segment.Text,
|
||||||
|
Categories: append([]string(nil), segment.Categories...),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return schema.IntermediateTranscript{
|
||||||
|
Metadata: schema.IntermediateMetadata{
|
||||||
|
Application: input.Metadata.Application,
|
||||||
|
Version: input.Metadata.Version,
|
||||||
|
OutputSchema: SchemaIntermediate,
|
||||||
|
},
|
||||||
|
Segments: segments,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func minimalFromFull(input schema.Transcript) schema.MinimalTranscript {
|
||||||
|
segments := make([]schema.MinimalSegment, len(input.Segments))
|
||||||
|
for index, segment := range input.Segments {
|
||||||
|
segments[index] = schema.MinimalSegment{
|
||||||
|
ID: segment.ID,
|
||||||
|
Start: segment.Start,
|
||||||
|
End: segment.End,
|
||||||
|
Speaker: segment.Speaker,
|
||||||
|
Text: segment.Text,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return schema.MinimalTranscript{
|
||||||
|
Metadata: schema.MinimalMetadata{
|
||||||
|
Application: input.Metadata.Application,
|
||||||
|
Version: input.Metadata.Version,
|
||||||
|
OutputSchema: SchemaMinimal,
|
||||||
|
},
|
||||||
|
Segments: segments,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func minimalFromIntermediate(input schema.IntermediateTranscript) schema.MinimalTranscript {
|
||||||
|
segments := make([]schema.MinimalSegment, len(input.Segments))
|
||||||
|
for index, segment := range input.Segments {
|
||||||
|
segments[index] = schema.MinimalSegment{
|
||||||
|
ID: segment.ID,
|
||||||
|
Start: segment.Start,
|
||||||
|
End: segment.End,
|
||||||
|
Speaker: segment.Speaker,
|
||||||
|
Text: segment.Text,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return schema.MinimalTranscript{
|
||||||
|
Metadata: schema.MinimalMetadata{
|
||||||
|
Application: input.Metadata.Application,
|
||||||
|
Version: input.Metadata.Version,
|
||||||
|
OutputSchema: SchemaMinimal,
|
||||||
|
},
|
||||||
|
Segments: segments,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func intermediateFromMinimal(input schema.MinimalTranscript) schema.IntermediateTranscript {
|
||||||
|
segments := make([]schema.IntermediateSegment, len(input.Segments))
|
||||||
|
for index, segment := range input.Segments {
|
||||||
|
segments[index] = schema.IntermediateSegment{
|
||||||
|
ID: segment.ID,
|
||||||
|
Start: segment.Start,
|
||||||
|
End: segment.End,
|
||||||
|
Speaker: segment.Speaker,
|
||||||
|
Text: segment.Text,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return schema.IntermediateTranscript{
|
||||||
|
Metadata: schema.IntermediateMetadata{
|
||||||
|
Application: input.Metadata.Application,
|
||||||
|
Version: input.Metadata.Version,
|
||||||
|
OutputSchema: SchemaIntermediate,
|
||||||
|
},
|
||||||
|
Segments: segments,
|
||||||
|
}
|
||||||
|
}
|
||||||
193
internal/trim/artifact_test.go
Normal file
193
internal/trim/artifact_test.go
Normal file
@@ -0,0 +1,193 @@
|
|||||||
|
package trim
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/schema"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestParseArtifactJSONRejectsMalformedJSON(t *testing.T) {
|
||||||
|
_, err := ParseArtifactJSON([]byte(`{"metadata":`))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected malformed JSON error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "input JSON is malformed") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseArtifactJSONRejectsDuplicateSegmentIDs(t *testing.T) {
|
||||||
|
first := 10
|
||||||
|
second := 20
|
||||||
|
value := schema.Transcript{
|
||||||
|
Metadata: schema.Metadata{
|
||||||
|
Application: "seriatim",
|
||||||
|
Version: "v-test",
|
||||||
|
},
|
||||||
|
Segments: []schema.Segment{
|
||||||
|
{ID: 1, Source: "a.json", SourceSegmentIndex: &first, Speaker: "A", Start: 1, End: 2, Text: "one"},
|
||||||
|
{ID: 1, Source: "a.json", SourceSegmentIndex: &second, Speaker: "B", Start: 2, End: 3, Text: "two"},
|
||||||
|
},
|
||||||
|
OverlapGroups: []schema.OverlapGroup{},
|
||||||
|
}
|
||||||
|
data := mustMarshalJSON(t, value)
|
||||||
|
|
||||||
|
_, err := ParseArtifactJSON(data)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected invalid artifact error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "not a valid seriatim output artifact") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseArtifactJSONRejectsNonSequentialSegmentIDs(t *testing.T) {
|
||||||
|
first := 10
|
||||||
|
second := 20
|
||||||
|
value := schema.Transcript{
|
||||||
|
Metadata: schema.Metadata{
|
||||||
|
Application: "seriatim",
|
||||||
|
Version: "v-test",
|
||||||
|
},
|
||||||
|
Segments: []schema.Segment{
|
||||||
|
{ID: 1, Source: "a.json", SourceSegmentIndex: &first, Speaker: "A", Start: 1, End: 2, Text: "one"},
|
||||||
|
{ID: 3, Source: "a.json", SourceSegmentIndex: &second, Speaker: "B", Start: 2, End: 3, Text: "two"},
|
||||||
|
},
|
||||||
|
OverlapGroups: []schema.OverlapGroup{},
|
||||||
|
}
|
||||||
|
data := mustMarshalJSON(t, value)
|
||||||
|
|
||||||
|
_, err := ParseArtifactJSON(data)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected invalid artifact error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "not a valid seriatim output artifact") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvertArtifactMinimalToIntermediate(t *testing.T) {
|
||||||
|
value := schema.MinimalTranscript{
|
||||||
|
Metadata: schema.MinimalMetadata{
|
||||||
|
Application: "seriatim",
|
||||||
|
Version: "v-test",
|
||||||
|
OutputSchema: SchemaMinimal,
|
||||||
|
},
|
||||||
|
Segments: []schema.MinimalSegment{
|
||||||
|
{ID: 1, Start: 1, End: 2, Speaker: "A", Text: "one"},
|
||||||
|
{ID: 2, Start: 2, End: 3, Speaker: "B", Text: "two"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
artifact := Artifact{
|
||||||
|
Schema: SchemaMinimal,
|
||||||
|
Minimal: &value,
|
||||||
|
}
|
||||||
|
|
||||||
|
converted, err := ConvertArtifact(artifact, SchemaIntermediate)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("convert failed: %v", err)
|
||||||
|
}
|
||||||
|
if converted.Schema != SchemaIntermediate {
|
||||||
|
t.Fatalf("schema = %q, want %q", converted.Schema, SchemaIntermediate)
|
||||||
|
}
|
||||||
|
if converted.Intermediate == nil {
|
||||||
|
t.Fatal("expected intermediate artifact")
|
||||||
|
}
|
||||||
|
if len(converted.Intermediate.Segments) != 2 {
|
||||||
|
t.Fatalf("segment count = %d, want 2", len(converted.Intermediate.Segments))
|
||||||
|
}
|
||||||
|
if converted.Intermediate.Segments[0].ID != 1 || converted.Intermediate.Segments[1].ID != 2 {
|
||||||
|
t.Fatalf("unexpected IDs: %#v", converted.Intermediate.Segments)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvertArtifactMinimalToFullFails(t *testing.T) {
|
||||||
|
value := schema.MinimalTranscript{
|
||||||
|
Metadata: schema.MinimalMetadata{
|
||||||
|
Application: "seriatim",
|
||||||
|
Version: "v-test",
|
||||||
|
OutputSchema: SchemaMinimal,
|
||||||
|
},
|
||||||
|
Segments: []schema.MinimalSegment{
|
||||||
|
{ID: 1, Start: 1, End: 2, Speaker: "A", Text: "one"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
artifact := Artifact{
|
||||||
|
Schema: SchemaMinimal,
|
||||||
|
Minimal: &value,
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := ConvertArtifact(artifact, SchemaFull)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected conversion error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "cannot emit") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateArtifactRejectsMissingPayloads(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
artifact Artifact
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "full",
|
||||||
|
artifact: Artifact{Schema: SchemaFull},
|
||||||
|
want: "full artifact payload is missing",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "intermediate",
|
||||||
|
artifact: Artifact{Schema: SchemaIntermediate},
|
||||||
|
want: "intermediate artifact payload is missing",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "minimal",
|
||||||
|
artifact: Artifact{Schema: SchemaMinimal},
|
||||||
|
want: "minimal artifact payload is missing",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
err := ValidateArtifact(test.artifact)
|
||||||
|
assertErrorContains(t, err, test.want)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyArtifactRejectsMissingPayload(t *testing.T) {
|
||||||
|
_, err := ApplyArtifact(Artifact{Schema: SchemaFull}, Options{})
|
||||||
|
assertErrorContains(t, err, "full artifact payload is missing")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvertArtifactRejectsMissingPayloadWhenConversionRequested(t *testing.T) {
|
||||||
|
_, err := ConvertArtifact(Artifact{Schema: SchemaFull}, SchemaMinimal)
|
||||||
|
assertErrorContains(t, err, "full artifact payload is missing")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvertArtifactSameSchemaDoesNotRequirePayload(t *testing.T) {
|
||||||
|
artifact := Artifact{Schema: SchemaFull}
|
||||||
|
converted, err := ConvertArtifact(artifact, SchemaFull)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("convert failed: %v", err)
|
||||||
|
}
|
||||||
|
if converted.Schema != SchemaFull {
|
||||||
|
t.Fatalf("schema = %q, want %q", converted.Schema, SchemaFull)
|
||||||
|
}
|
||||||
|
if converted.Full != nil {
|
||||||
|
t.Fatalf("full payload = %#v, want nil", converted.Full)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func mustMarshalJSON(t *testing.T, value any) []byte {
|
||||||
|
t.Helper()
|
||||||
|
data, err := json.Marshal(value)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal: %v", err)
|
||||||
|
}
|
||||||
|
return data
|
||||||
|
}
|
||||||
156
internal/trim/run.go
Normal file
156
internal/trim/run.go
Normal file
@@ -0,0 +1,156 @@
|
|||||||
|
package trim
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"sort"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/internal/jsonfile"
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/internal/report"
|
||||||
|
)
|
||||||
|
|
||||||
|
type auditReport struct {
|
||||||
|
Operation string `json:"operation"`
|
||||||
|
InputFile string `json:"input_file"`
|
||||||
|
OutputFile string `json:"output_file"`
|
||||||
|
InputSchema string `json:"input_schema"`
|
||||||
|
OutputSchema string `json:"output_schema"`
|
||||||
|
Mode string `json:"mode"`
|
||||||
|
Selector string `json:"selector"`
|
||||||
|
SelectedIDs []int `json:"selected_ids"`
|
||||||
|
AllowEmpty bool `json:"allow_empty"`
|
||||||
|
InputSegmentCount int `json:"input_segment_count"`
|
||||||
|
RetainedSegmentCount int `json:"retained_segment_count"`
|
||||||
|
RemovedSegmentCount int `json:"removed_segment_count"`
|
||||||
|
RemovedInputIDs []int `json:"removed_input_ids"`
|
||||||
|
OldToNewIDMapping []idMapping `json:"old_to_new_id_mapping"`
|
||||||
|
OverlapGroupsRecomputed bool `json:"overlap_groups_recomputed"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type idMapping struct {
|
||||||
|
OldID int `json:"old_id"`
|
||||||
|
NewID int `json:"new_id"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run executes artifact-level trim orchestration.
|
||||||
|
func Run(ctx context.Context, cfg config.TrimConfig) error {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
selector, err := ParseSelector(cfg.Selector)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("invalid selector %q: %w", cfg.Selector, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := os.ReadFile(cfg.InputFile)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("read --input-file %q: %w", cfg.InputFile, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
artifact, err := ParseArtifactJSON(data)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("--input-file %q: %w", cfg.InputFile, err)
|
||||||
|
}
|
||||||
|
inputSegmentCount := artifact.SegmentCount()
|
||||||
|
inputSchema := artifact.Schema
|
||||||
|
|
||||||
|
mode := ModeKeep
|
||||||
|
if cfg.Mode == "remove" {
|
||||||
|
mode = ModeRemove
|
||||||
|
}
|
||||||
|
|
||||||
|
trimmed, err := ApplyArtifact(artifact, Options{
|
||||||
|
Mode: mode,
|
||||||
|
Selector: selector,
|
||||||
|
AllowEmpty: cfg.AllowEmpty,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
outputSchema := artifact.Schema
|
||||||
|
if cfg.OutputSchema != "" {
|
||||||
|
outputSchema = cfg.OutputSchema
|
||||||
|
}
|
||||||
|
|
||||||
|
outputArtifact, err := ConvertArtifact(trimmed.Artifact, outputSchema)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := ValidateArtifact(outputArtifact); err != nil {
|
||||||
|
return fmt.Errorf("validate trimmed output: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := jsonfile.Write(cfg.OutputFile, outputArtifact.Value()); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if cfg.ReportFile == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
audit := auditReport{
|
||||||
|
Operation: "trim",
|
||||||
|
InputFile: cfg.InputFile,
|
||||||
|
OutputFile: cfg.OutputFile,
|
||||||
|
InputSchema: inputSchema,
|
||||||
|
OutputSchema: outputArtifact.Schema,
|
||||||
|
Mode: cfg.Mode,
|
||||||
|
Selector: cfg.Selector,
|
||||||
|
SelectedIDs: selector.IDs(),
|
||||||
|
AllowEmpty: cfg.AllowEmpty,
|
||||||
|
InputSegmentCount: inputSegmentCount,
|
||||||
|
RetainedSegmentCount: len(trimmed.OldToNewID),
|
||||||
|
RemovedSegmentCount: len(trimmed.RemovedIDs),
|
||||||
|
RemovedInputIDs: append([]int(nil), trimmed.RemovedIDs...),
|
||||||
|
OldToNewIDMapping: orderedIDMapping(trimmed.OldToNewID),
|
||||||
|
OverlapGroupsRecomputed: trimmed.OverlapGroupsRecomputed,
|
||||||
|
}
|
||||||
|
auditJSON, err := json.Marshal(audit)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("marshal trim audit report: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
rpt := report.Report{
|
||||||
|
Metadata: report.Metadata{
|
||||||
|
Application: outputArtifact.Application(),
|
||||||
|
Version: outputArtifact.Version(),
|
||||||
|
InputReader: "trim-artifact",
|
||||||
|
InputFiles: []string{cfg.InputFile},
|
||||||
|
OutputModules: []string{"json"},
|
||||||
|
},
|
||||||
|
Events: []report.Event{
|
||||||
|
report.Info("trim", "trim", fmt.Sprintf("trimmed %d input segment(s) into %d output segment(s) with mode=%s", inputSegmentCount, outputArtifact.SegmentCount(), cfg.Mode)),
|
||||||
|
report.Info("trim", "trim-audit", string(auditJSON)),
|
||||||
|
report.Info("trim", "validate-output", fmt.Sprintf("validated %d output segment(s)", outputArtifact.SegmentCount())),
|
||||||
|
report.Info("output", "json", "wrote transcript JSON"),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if err := report.WriteJSON(cfg.ReportFile, rpt); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func orderedIDMapping(mapping map[int]int) []idMapping {
|
||||||
|
keys := make([]int, 0, len(mapping))
|
||||||
|
for oldID := range mapping {
|
||||||
|
keys = append(keys, oldID)
|
||||||
|
}
|
||||||
|
sort.Ints(keys)
|
||||||
|
|
||||||
|
pairs := make([]idMapping, 0, len(keys))
|
||||||
|
for _, oldID := range keys {
|
||||||
|
pairs = append(pairs, idMapping{
|
||||||
|
OldID: oldID,
|
||||||
|
NewID: mapping[oldID],
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return pairs
|
||||||
|
}
|
||||||
28
internal/trim/run_test.go
Normal file
28
internal/trim/run_test.go
Normal file
@@ -0,0 +1,28 @@
|
|||||||
|
package trim
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestRunReturnsContextErrorBeforeWork(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
err := Run(ctx, config.TrimConfig{
|
||||||
|
InputFile: filepath.Join(dir, "input.json"),
|
||||||
|
OutputFile: filepath.Join(dir, "output.json"),
|
||||||
|
Mode: "keep",
|
||||||
|
Selector: "1",
|
||||||
|
OutputSchema: "",
|
||||||
|
AllowEmpty: false,
|
||||||
|
})
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("error = %v, want context.Canceled", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
156
internal/trim/selector.go
Normal file
156
internal/trim/selector.go
Normal file
@@ -0,0 +1,156 @@
|
|||||||
|
package trim
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"regexp"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
var selectorElementPattern = regexp.MustCompile(`^([+-]?\d+)(?:\s*-\s*([+-]?\d+))?$`)
|
||||||
|
|
||||||
|
// Selector represents a normalized union of segment IDs.
|
||||||
|
type Selector struct {
|
||||||
|
ranges []idRange
|
||||||
|
}
|
||||||
|
|
||||||
|
type idRange struct {
|
||||||
|
start int
|
||||||
|
end int
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseSelector parses an inline segment selector expression.
|
||||||
|
func ParseSelector(input string) (Selector, error) {
|
||||||
|
if strings.TrimSpace(input) == "" {
|
||||||
|
return Selector{}, fmt.Errorf("selector cannot be empty")
|
||||||
|
}
|
||||||
|
|
||||||
|
parts := strings.Split(input, ",")
|
||||||
|
ranges := make([]idRange, 0, len(parts))
|
||||||
|
for index, raw := range parts {
|
||||||
|
element := strings.TrimSpace(raw)
|
||||||
|
if element == "" {
|
||||||
|
return Selector{}, fmt.Errorf("selector element %d cannot be empty", index+1)
|
||||||
|
}
|
||||||
|
|
||||||
|
rangeValue, err := parseElement(element)
|
||||||
|
if err != nil {
|
||||||
|
return Selector{}, fmt.Errorf("selector element %d %q: %w", index+1, element, err)
|
||||||
|
}
|
||||||
|
ranges = append(ranges, rangeValue)
|
||||||
|
}
|
||||||
|
|
||||||
|
normalized := normalizeRanges(ranges)
|
||||||
|
if len(normalized) == 0 {
|
||||||
|
return Selector{}, fmt.Errorf("selector cannot be empty")
|
||||||
|
}
|
||||||
|
return Selector{ranges: normalized}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Contains returns true when id is included by this selector.
|
||||||
|
func (s Selector) Contains(id int) bool {
|
||||||
|
if id <= 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
index := sort.Search(len(s.ranges), func(i int) bool {
|
||||||
|
return s.ranges[i].end >= id
|
||||||
|
})
|
||||||
|
if index == len(s.ranges) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
rangeValue := s.ranges[index]
|
||||||
|
return id >= rangeValue.start && id <= rangeValue.end
|
||||||
|
}
|
||||||
|
|
||||||
|
// IDs returns a deterministic ascending list of unique segment IDs.
|
||||||
|
func (s Selector) IDs() []int {
|
||||||
|
total := 0
|
||||||
|
for _, rangeValue := range s.ranges {
|
||||||
|
total += rangeValue.end - rangeValue.start + 1
|
||||||
|
}
|
||||||
|
|
||||||
|
ids := make([]int, 0, total)
|
||||||
|
for _, rangeValue := range s.ranges {
|
||||||
|
for id := rangeValue.start; id <= rangeValue.end; id++ {
|
||||||
|
ids = append(ids, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ids
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseElement(element string) (idRange, error) {
|
||||||
|
matches := selectorElementPattern.FindStringSubmatch(element)
|
||||||
|
if matches == nil {
|
||||||
|
return idRange{}, fmt.Errorf("malformed element")
|
||||||
|
}
|
||||||
|
|
||||||
|
start, err := parseID(matches[1])
|
||||||
|
if err != nil {
|
||||||
|
return idRange{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if matches[2] == "" {
|
||||||
|
return idRange{start: start, end: start}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
end, err := parseID(matches[2])
|
||||||
|
if err != nil {
|
||||||
|
return idRange{}, fmt.Errorf("invalid range end: %w", err)
|
||||||
|
}
|
||||||
|
if start > end {
|
||||||
|
return idRange{}, fmt.Errorf("descending range %d-%d is invalid", start, end)
|
||||||
|
}
|
||||||
|
return idRange{start: start, end: end}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseID(value string) (int, error) {
|
||||||
|
value = strings.TrimSpace(value)
|
||||||
|
if value == "" {
|
||||||
|
return 0, fmt.Errorf("missing segment ID")
|
||||||
|
}
|
||||||
|
|
||||||
|
id, err := strconv.Atoi(value)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("segment ID must be an integer")
|
||||||
|
}
|
||||||
|
if id <= 0 {
|
||||||
|
return 0, fmt.Errorf("segment ID must be positive")
|
||||||
|
}
|
||||||
|
return id, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeRanges(in []idRange) []idRange {
|
||||||
|
if len(in) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
sorted := make([]idRange, len(in))
|
||||||
|
copy(sorted, in)
|
||||||
|
sort.Slice(sorted, func(i, j int) bool {
|
||||||
|
if sorted[i].start == sorted[j].start {
|
||||||
|
return sorted[i].end < sorted[j].end
|
||||||
|
}
|
||||||
|
return sorted[i].start < sorted[j].start
|
||||||
|
})
|
||||||
|
|
||||||
|
merged := make([]idRange, 0, len(sorted))
|
||||||
|
for _, next := range sorted {
|
||||||
|
if len(merged) == 0 {
|
||||||
|
merged = append(merged, next)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
last := &merged[len(merged)-1]
|
||||||
|
if next.start <= last.end+1 {
|
||||||
|
if next.end > last.end {
|
||||||
|
last.end = next.end
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
merged = append(merged, next)
|
||||||
|
}
|
||||||
|
|
||||||
|
return merged
|
||||||
|
}
|
||||||
127
internal/trim/selector_test.go
Normal file
127
internal/trim/selector_test.go
Normal file
@@ -0,0 +1,127 @@
|
|||||||
|
package trim
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestParseSelectorSingleID(t *testing.T) {
|
||||||
|
selector, err := ParseSelector("1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse failed: %v", err)
|
||||||
|
}
|
||||||
|
assertIDs(t, selector, []int{1})
|
||||||
|
assertContains(t, selector, map[int]bool{1: true, 2: false, 0: false, -1: false})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseSelectorInclusiveRange(t *testing.T) {
|
||||||
|
selector, err := ParseSelector("1-3")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse failed: %v", err)
|
||||||
|
}
|
||||||
|
assertIDs(t, selector, []int{1, 2, 3})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseSelectorCommaSeparatedCombination(t *testing.T) {
|
||||||
|
selector, err := ParseSelector("1-3,8,10-12")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse failed: %v", err)
|
||||||
|
}
|
||||||
|
assertIDs(t, selector, []int{1, 2, 3, 8, 10, 11, 12})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseSelectorWhitespaceTolerance(t *testing.T) {
|
||||||
|
selector, err := ParseSelector(" 1 - 3 , 8 , 10 - 12 ")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse failed: %v", err)
|
||||||
|
}
|
||||||
|
assertIDs(t, selector, []int{1, 2, 3, 8, 10, 11, 12})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseSelectorDuplicatesAndOverlapsNormalizeUnion(t *testing.T) {
|
||||||
|
selector, err := ParseSelector("1-4,2,4,3-6,6")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse failed: %v", err)
|
||||||
|
}
|
||||||
|
assertIDs(t, selector, []int{1, 2, 3, 4, 5, 6})
|
||||||
|
assertContains(t, selector, map[int]bool{1: true, 5: true, 6: true, 7: false})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseSelectorDeterministicNormalizedOutput(t *testing.T) {
|
||||||
|
left, err := ParseSelector("8,1-3,2,10-12")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse left failed: %v", err)
|
||||||
|
}
|
||||||
|
right, err := ParseSelector("10-12,3,2,1,8")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse right failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
leftIDs := left.IDs()
|
||||||
|
rightIDs := right.IDs()
|
||||||
|
if !equalInts(leftIDs, rightIDs) {
|
||||||
|
t.Fatalf("normalized IDs mismatch: %v vs %v", leftIDs, rightIDs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseSelectorFailures(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
selector string
|
||||||
|
wantError string
|
||||||
|
}{
|
||||||
|
{name: "empty", selector: "", wantError: "cannot be empty"},
|
||||||
|
{name: "whitespace only", selector: " ", wantError: "cannot be empty"},
|
||||||
|
{name: "zero", selector: "0", wantError: "must be positive"},
|
||||||
|
{name: "negative", selector: "-1", wantError: "must be positive"},
|
||||||
|
{name: "range includes zero", selector: "0-2", wantError: "must be positive"},
|
||||||
|
{name: "descending range", selector: "10-1", wantError: "descending range"},
|
||||||
|
{name: "empty element", selector: "1,,2", wantError: "cannot be empty"},
|
||||||
|
{name: "trailing comma", selector: "1,", wantError: "cannot be empty"},
|
||||||
|
{name: "malformed alpha", selector: "abc", wantError: "malformed element"},
|
||||||
|
{name: "malformed range", selector: "1-2-3", wantError: "malformed element"},
|
||||||
|
{name: "missing end", selector: "1-", wantError: "malformed element"},
|
||||||
|
{name: "missing start", selector: "-2", wantError: "must be positive"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
_, err := ParseSelector(test.selector)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("expected error for %q", test.selector)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), test.wantError) {
|
||||||
|
t.Fatalf("error = %q, want substring %q", err.Error(), test.wantError)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertIDs(t *testing.T, selector Selector, want []int) {
|
||||||
|
t.Helper()
|
||||||
|
got := selector.IDs()
|
||||||
|
if !equalInts(got, want) {
|
||||||
|
t.Fatalf("IDs = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertContains(t *testing.T, selector Selector, checks map[int]bool) {
|
||||||
|
t.Helper()
|
||||||
|
for id, want := range checks {
|
||||||
|
if got := selector.Contains(id); got != want {
|
||||||
|
t.Fatalf("Contains(%d) = %t, want %t", id, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func equalInts(left []int, right []int) bool {
|
||||||
|
if len(left) != len(right) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for index := range left {
|
||||||
|
if left[index] != right[index] {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
"$id": "https://gitea.maximumdirect.net/eric/seriatim/schema/output.schema.json",
|
"$id": "https://gitea.maximumdirect.net/eric/seriatim/schema/full-output.schema.json",
|
||||||
"title": "seriatim full output transcript",
|
"title": "seriatim full output transcript",
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"additionalProperties": false,
|
"additionalProperties": false,
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
"$id": "https://gitea.maximumdirect.net/eric/seriatim/schema/default-output.schema.json",
|
"$id": "https://gitea.maximumdirect.net/eric/seriatim/schema/intermediate-output.schema.json",
|
||||||
"title": "seriatim default output transcript",
|
"title": "seriatim intermediate output transcript",
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"additionalProperties": false,
|
"additionalProperties": false,
|
||||||
"required": ["metadata", "segments"],
|
"required": ["metadata", "segments"],
|
||||||
@@ -13,7 +13,7 @@
|
|||||||
"properties": {
|
"properties": {
|
||||||
"application": { "type": "string" },
|
"application": { "type": "string" },
|
||||||
"version": { "type": "string" },
|
"version": { "type": "string" },
|
||||||
"output_schema": { "type": "string", "const": "default" }
|
"output_schema": { "type": "string", "const": "seriatim-intermediate" }
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"segments": {
|
"segments": {
|
||||||
@@ -13,7 +13,7 @@
|
|||||||
"properties": {
|
"properties": {
|
||||||
"application": { "type": "string" },
|
"application": { "type": "string" },
|
||||||
"version": { "type": "string" },
|
"version": { "type": "string" },
|
||||||
"output_schema": { "type": "string", "const": "minimal" }
|
"output_schema": { "type": "string", "const": "seriatim-minimal" }
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"segments": {
|
"segments": {
|
||||||
|
|||||||
126
schema/output.go
126
schema/output.go
@@ -14,9 +14,13 @@ import (
|
|||||||
var schemaFS embed.FS
|
var schemaFS embed.FS
|
||||||
|
|
||||||
const (
|
const (
|
||||||
outputSchemaPath = "output.schema.json"
|
OutputSchemaMinimal = "seriatim-minimal"
|
||||||
defaultOutputSchemaPath = "default-output.schema.json"
|
OutputSchemaIntermediate = "seriatim-intermediate"
|
||||||
minimalOutputSchemaPath = "minimal-output.schema.json"
|
OutputSchemaFull = "seriatim-full"
|
||||||
|
|
||||||
|
fullOutputSchemaPath = "full-output.schema.json"
|
||||||
|
intermediateOutputSchemaPath = "intermediate-output.schema.json"
|
||||||
|
minimalOutputSchemaPath = "minimal-output.schema.json"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -32,10 +36,10 @@ type Transcript struct {
|
|||||||
OverlapGroups []OverlapGroup `json:"overlap_groups"`
|
OverlapGroups []OverlapGroup `json:"overlap_groups"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// DefaultTranscript is seriatim's default public JSON output contract.
|
// IntermediateTranscript is seriatim's intermediate public JSON output contract.
|
||||||
type DefaultTranscript struct {
|
type IntermediateTranscript struct {
|
||||||
Metadata DefaultMetadata `json:"metadata"`
|
Metadata IntermediateMetadata `json:"metadata"`
|
||||||
Segments []DefaultSegment `json:"segments"`
|
Segments []IntermediateSegment `json:"segments"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// MinimalTranscript is seriatim's compact public JSON output contract.
|
// MinimalTranscript is seriatim's compact public JSON output contract.
|
||||||
@@ -55,8 +59,8 @@ type Metadata struct {
|
|||||||
OutputModules []string `json:"output_modules"`
|
OutputModules []string `json:"output_modules"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// DefaultMetadata records default artifact identity.
|
// IntermediateMetadata records intermediate artifact identity.
|
||||||
type DefaultMetadata struct {
|
type IntermediateMetadata struct {
|
||||||
Application string `json:"application"`
|
Application string `json:"application"`
|
||||||
Version string `json:"version"`
|
Version string `json:"version"`
|
||||||
OutputSchema string `json:"output_schema"`
|
OutputSchema string `json:"output_schema"`
|
||||||
@@ -84,9 +88,9 @@ type Segment struct {
|
|||||||
OverlapGroupID int `json:"overlap_group_id,omitempty"`
|
OverlapGroupID int `json:"overlap_group_id,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// DefaultSegment is the compact public transcript segment shape with
|
// IntermediateSegment is the compact public transcript segment shape with
|
||||||
// categories.
|
// categories.
|
||||||
type DefaultSegment struct {
|
type IntermediateSegment struct {
|
||||||
ID int `json:"id"`
|
ID int `json:"id"`
|
||||||
Start float64 `json:"start"`
|
Start float64 `json:"start"`
|
||||||
End float64 `json:"end"`
|
End float64 `json:"end"`
|
||||||
@@ -115,7 +119,26 @@ type OverlapGroup struct {
|
|||||||
Resolution string `json:"resolution"`
|
Resolution string `json:"resolution"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ValidateTranscript validates a typed transcript against the public JSON
|
// ValidOutputSchemaName reports whether value is a supported output schema name.
|
||||||
|
func ValidOutputSchemaName(value string) bool {
|
||||||
|
switch value {
|
||||||
|
case OutputSchemaMinimal, OutputSchemaIntermediate, OutputSchemaFull:
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// OutputSchemaNames returns supported output schema names in validation order.
|
||||||
|
func OutputSchemaNames() []string {
|
||||||
|
return []string{
|
||||||
|
OutputSchemaMinimal,
|
||||||
|
OutputSchemaIntermediate,
|
||||||
|
OutputSchemaFull,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidateTranscript validates a full transcript against the public JSON
|
||||||
// schema and seriatim-specific semantic rules.
|
// schema and seriatim-specific semantic rules.
|
||||||
func ValidateTranscript(transcript Transcript) error {
|
func ValidateTranscript(transcript Transcript) error {
|
||||||
if err := validateSemantics(transcript); err != nil {
|
if err := validateSemantics(transcript); err != nil {
|
||||||
@@ -129,18 +152,18 @@ func ValidateTranscript(transcript Transcript) error {
|
|||||||
return ValidateJSON(data)
|
return ValidateJSON(data)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ValidateDefaultTranscript validates the default transcript against the
|
// ValidateIntermediateTranscript validates the intermediate transcript against
|
||||||
// default JSON schema and seriatim-specific semantic rules.
|
// the intermediate JSON schema and seriatim-specific semantic rules.
|
||||||
func ValidateDefaultTranscript(transcript DefaultTranscript) error {
|
func ValidateIntermediateTranscript(transcript IntermediateTranscript) error {
|
||||||
if err := validateDefaultSemantics(transcript); err != nil {
|
if err := validateIntermediateSemantics(transcript); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
data, err := json.Marshal(transcript)
|
data, err := json.Marshal(transcript)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("marshal default transcript for schema validation: %w", err)
|
return fmt.Errorf("marshal intermediate transcript for schema validation: %w", err)
|
||||||
}
|
}
|
||||||
return ValidateDefaultJSON(data)
|
return ValidateIntermediateJSON(data)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ValidateMinimalTranscript validates a minimal transcript against the minimal
|
// ValidateMinimalTranscript validates a minimal transcript against the minimal
|
||||||
@@ -159,13 +182,13 @@ func ValidateMinimalTranscript(transcript MinimalTranscript) error {
|
|||||||
|
|
||||||
// ValidateJSON validates serialized output JSON against the public schema.
|
// ValidateJSON validates serialized output JSON against the public schema.
|
||||||
func ValidateJSON(data []byte) error {
|
func ValidateJSON(data []byte) error {
|
||||||
return validateJSONWithSchema(data, outputSchemaPath)
|
return validateJSONWithSchema(data, fullOutputSchemaPath)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ValidateDefaultJSON validates serialized default output JSON against the
|
// ValidateIntermediateJSON validates serialized intermediate output JSON
|
||||||
// default public schema.
|
// against the intermediate public schema.
|
||||||
func ValidateDefaultJSON(data []byte) error {
|
func ValidateIntermediateJSON(data []byte) error {
|
||||||
return validateJSONWithSchema(data, defaultOutputSchemaPath)
|
return validateJSONWithSchema(data, intermediateOutputSchemaPath)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ValidateMinimalJSON validates serialized minimal output JSON against the
|
// ValidateMinimalJSON validates serialized minimal output JSON against the
|
||||||
@@ -228,15 +251,17 @@ func outputSchema(schemaPath string) (*jsonschema.Schema, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func validateSemantics(transcript Transcript) error {
|
func validateSemantics(transcript Transcript) error {
|
||||||
|
segments := make([]segmentSemantics, len(transcript.Segments))
|
||||||
for index, segment := range transcript.Segments {
|
for index, segment := range transcript.Segments {
|
||||||
wantID := index + 1
|
segments[index] = segmentSemantics{
|
||||||
if segment.ID != wantID {
|
id: segment.ID,
|
||||||
return fmt.Errorf("segment %d has id %d; want %d", index, segment.ID, wantID)
|
start: segment.Start,
|
||||||
}
|
end: segment.End,
|
||||||
if segment.End < segment.Start {
|
|
||||||
return fmt.Errorf("segment %d has end %.3f before start %.3f", index, segment.End, segment.Start)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if err := validateSegmentSemantics(segments); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
for index, group := range transcript.OverlapGroups {
|
for index, group := range transcript.OverlapGroups {
|
||||||
if group.End < group.Start {
|
if group.End < group.Start {
|
||||||
return fmt.Errorf("overlap_group %d has end %.3f before start %.3f", index, group.End, group.Start)
|
return fmt.Errorf("overlap_group %d has end %.3f before start %.3f", index, group.End, group.Start)
|
||||||
@@ -245,27 +270,44 @@ func validateSemantics(transcript Transcript) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateDefaultSemantics(transcript DefaultTranscript) error {
|
func validateIntermediateSemantics(transcript IntermediateTranscript) error {
|
||||||
|
segments := make([]segmentSemantics, len(transcript.Segments))
|
||||||
for index, segment := range transcript.Segments {
|
for index, segment := range transcript.Segments {
|
||||||
wantID := index + 1
|
segments[index] = segmentSemantics{
|
||||||
if segment.ID != wantID {
|
id: segment.ID,
|
||||||
return fmt.Errorf("segment %d has id %d; want %d", index, segment.ID, wantID)
|
start: segment.Start,
|
||||||
}
|
end: segment.End,
|
||||||
if segment.End < segment.Start {
|
|
||||||
return fmt.Errorf("segment %d has end %.3f before start %.3f", index, segment.End, segment.Start)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return validateSegmentSemantics(segments)
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateMinimalSemantics(transcript MinimalTranscript) error {
|
func validateMinimalSemantics(transcript MinimalTranscript) error {
|
||||||
|
segments := make([]segmentSemantics, len(transcript.Segments))
|
||||||
for index, segment := range transcript.Segments {
|
for index, segment := range transcript.Segments {
|
||||||
wantID := index + 1
|
segments[index] = segmentSemantics{
|
||||||
if segment.ID != wantID {
|
id: segment.ID,
|
||||||
return fmt.Errorf("segment %d has id %d; want %d", index, segment.ID, wantID)
|
start: segment.Start,
|
||||||
|
end: segment.End,
|
||||||
}
|
}
|
||||||
if segment.End < segment.Start {
|
}
|
||||||
return fmt.Errorf("segment %d has end %.3f before start %.3f", index, segment.End, segment.Start)
|
return validateSegmentSemantics(segments)
|
||||||
|
}
|
||||||
|
|
||||||
|
type segmentSemantics struct {
|
||||||
|
id int
|
||||||
|
start float64
|
||||||
|
end float64
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateSegmentSemantics(segments []segmentSemantics) error {
|
||||||
|
for index, segment := range segments {
|
||||||
|
wantID := index + 1
|
||||||
|
if segment.id != wantID {
|
||||||
|
return fmt.Errorf("segment %d has id %d; want %d", index, segment.id, wantID)
|
||||||
|
}
|
||||||
|
if segment.end < segment.start {
|
||||||
|
return fmt.Errorf("segment %d has end %.3f before start %.3f", index, segment.end, segment.start)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -5,6 +5,43 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func TestValidOutputSchemaName(t *testing.T) {
|
||||||
|
valid := []string{
|
||||||
|
OutputSchemaMinimal,
|
||||||
|
OutputSchemaIntermediate,
|
||||||
|
OutputSchemaFull,
|
||||||
|
}
|
||||||
|
for _, name := range valid {
|
||||||
|
if !ValidOutputSchemaName(name) {
|
||||||
|
t.Fatalf("expected %q to be valid", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
invalid := []string{"", "compact", "minimal", "seriatim"}
|
||||||
|
for _, name := range invalid {
|
||||||
|
if ValidOutputSchemaName(name) {
|
||||||
|
t.Fatalf("expected %q to be invalid", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOutputSchemaNames(t *testing.T) {
|
||||||
|
names := OutputSchemaNames()
|
||||||
|
want := []string{
|
||||||
|
OutputSchemaMinimal,
|
||||||
|
OutputSchemaIntermediate,
|
||||||
|
OutputSchemaFull,
|
||||||
|
}
|
||||||
|
if len(names) != len(want) {
|
||||||
|
t.Fatalf("len(names) = %d, want %d", len(names), len(want))
|
||||||
|
}
|
||||||
|
for index := range want {
|
||||||
|
if names[index] != want[index] {
|
||||||
|
t.Fatalf("names[%d] = %q, want %q", index, names[index], want[index])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestValidateTranscriptAcceptsValidTranscript(t *testing.T) {
|
func TestValidateTranscriptAcceptsValidTranscript(t *testing.T) {
|
||||||
transcript := validTranscript()
|
transcript := validTranscript()
|
||||||
|
|
||||||
@@ -21,11 +58,11 @@ func TestValidateMinimalTranscriptAcceptsValidTranscript(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestValidateDefaultTranscriptAcceptsValidTranscript(t *testing.T) {
|
func TestValidateIntermediateTranscriptAcceptsValidTranscript(t *testing.T) {
|
||||||
transcript := validDefaultTranscript()
|
transcript := validIntermediateTranscript()
|
||||||
|
|
||||||
if err := ValidateDefaultTranscript(transcript); err != nil {
|
if err := ValidateIntermediateTranscript(transcript); err != nil {
|
||||||
t.Fatalf("validate default transcript: %v", err)
|
t.Fatalf("validate intermediate transcript: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -34,7 +71,7 @@ func TestValidateMinimalJSONRejectsMissingRequiredField(t *testing.T) {
|
|||||||
"metadata": {
|
"metadata": {
|
||||||
"application": "seriatim",
|
"application": "seriatim",
|
||||||
"version": "dev",
|
"version": "dev",
|
||||||
"output_schema": "minimal"
|
"output_schema": "seriatim-minimal"
|
||||||
}
|
}
|
||||||
}`))
|
}`))
|
||||||
assertErrorContains(t, err, "segments")
|
assertErrorContains(t, err, "segments")
|
||||||
@@ -45,7 +82,7 @@ func TestValidateMinimalJSONRejectsWrongFieldType(t *testing.T) {
|
|||||||
"metadata": {
|
"metadata": {
|
||||||
"application": "seriatim",
|
"application": "seriatim",
|
||||||
"version": "dev",
|
"version": "dev",
|
||||||
"output_schema": "minimal"
|
"output_schema": "seriatim-minimal"
|
||||||
},
|
},
|
||||||
"segments": [
|
"segments": [
|
||||||
{
|
{
|
||||||
@@ -68,7 +105,7 @@ func TestValidateMinimalJSONRejectsUnexpectedFields(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "top-level overlap groups",
|
name: "top-level overlap groups",
|
||||||
json: `{
|
json: `{
|
||||||
"metadata": {"application": "seriatim", "version": "dev", "output_schema": "minimal"},
|
"metadata": {"application": "seriatim", "version": "dev", "output_schema": "seriatim-minimal"},
|
||||||
"segments": [],
|
"segments": [],
|
||||||
"overlap_groups": []
|
"overlap_groups": []
|
||||||
}`,
|
}`,
|
||||||
@@ -76,28 +113,28 @@ func TestValidateMinimalJSONRejectsUnexpectedFields(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "segment source",
|
name: "segment source",
|
||||||
json: `{
|
json: `{
|
||||||
"metadata": {"application": "seriatim", "version": "dev", "output_schema": "minimal"},
|
"metadata": {"application": "seriatim", "version": "dev", "output_schema": "seriatim-minimal"},
|
||||||
"segments": [{"id": 1, "source": "input.json", "start": 1, "end": 2, "speaker": "Alice", "text": "hello"}]
|
"segments": [{"id": 1, "source": "input.json", "start": 1, "end": 2, "speaker": "Alice", "text": "hello"}]
|
||||||
}`,
|
}`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "segment categories",
|
name: "segment categories",
|
||||||
json: `{
|
json: `{
|
||||||
"metadata": {"application": "seriatim", "version": "dev", "output_schema": "minimal"},
|
"metadata": {"application": "seriatim", "version": "dev", "output_schema": "seriatim-minimal"},
|
||||||
"segments": [{"id": 1, "start": 1, "end": 2, "speaker": "Alice", "text": "hello", "categories": ["backchannel"]}]
|
"segments": [{"id": 1, "start": 1, "end": 2, "speaker": "Alice", "text": "hello", "categories": ["backchannel"]}]
|
||||||
}`,
|
}`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "segment derived from",
|
name: "segment derived from",
|
||||||
json: `{
|
json: `{
|
||||||
"metadata": {"application": "seriatim", "version": "dev", "output_schema": "minimal"},
|
"metadata": {"application": "seriatim", "version": "dev", "output_schema": "seriatim-minimal"},
|
||||||
"segments": [{"id": 1, "start": 1, "end": 2, "speaker": "Alice", "text": "hello", "derived_from": ["input.json#0"]}]
|
"segments": [{"id": 1, "start": 1, "end": 2, "speaker": "Alice", "text": "hello", "derived_from": ["input.json#0"]}]
|
||||||
}`,
|
}`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "segment words",
|
name: "segment words",
|
||||||
json: `{
|
json: `{
|
||||||
"metadata": {"application": "seriatim", "version": "dev", "output_schema": "minimal"},
|
"metadata": {"application": "seriatim", "version": "dev", "output_schema": "seriatim-minimal"},
|
||||||
"segments": [{"id": 1, "start": 1, "end": 2, "speaker": "Alice", "text": "hello", "words": []}]
|
"segments": [{"id": 1, "start": 1, "end": 2, "speaker": "Alice", "text": "hello", "words": []}]
|
||||||
}`,
|
}`,
|
||||||
},
|
},
|
||||||
@@ -111,24 +148,24 @@ func TestValidateMinimalJSONRejectsUnexpectedFields(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestValidateDefaultJSONRejectsMissingRequiredField(t *testing.T) {
|
func TestValidateIntermediateJSONRejectsMissingRequiredField(t *testing.T) {
|
||||||
err := ValidateDefaultJSON([]byte(`{
|
err := ValidateIntermediateJSON([]byte(`{
|
||||||
"metadata": {
|
"metadata": {
|
||||||
"application": "seriatim",
|
"application": "seriatim",
|
||||||
"version": "dev",
|
"version": "dev",
|
||||||
"output_schema": "default"
|
"output_schema": "seriatim-intermediate"
|
||||||
}
|
}
|
||||||
}`))
|
}`))
|
||||||
assertErrorContains(t, err, "segments")
|
assertErrorContains(t, err, "segments")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestValidateDefaultJSONRejectsWrongFieldType(t *testing.T) {
|
func TestValidateIntermediateJSONRejectsWrongFieldType(t *testing.T) {
|
||||||
err := ValidateDefaultJSON([]byte(`{
|
err := ValidateIntermediateJSON([]byte(`{
|
||||||
"metadata": {
|
"metadata": {
|
||||||
"application": "seriatim",
|
"application": "seriatim",
|
||||||
"version": "dev",
|
"version": "dev",
|
||||||
"output_schema": "default"
|
"output_schema": "seriatim-intermediate"
|
||||||
},
|
},
|
||||||
"segments": [
|
"segments": [
|
||||||
{
|
{
|
||||||
"id": "1",
|
"id": "1",
|
||||||
@@ -142,7 +179,7 @@ func TestValidateDefaultJSONRejectsWrongFieldType(t *testing.T) {
|
|||||||
assertErrorContains(t, err, "id")
|
assertErrorContains(t, err, "id")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestValidateDefaultJSONRejectsUnexpectedFields(t *testing.T) {
|
func TestValidateIntermediateJSONRejectsUnexpectedFields(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
json string
|
json string
|
||||||
@@ -150,7 +187,7 @@ func TestValidateDefaultJSONRejectsUnexpectedFields(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "top-level overlap groups",
|
name: "top-level overlap groups",
|
||||||
json: `{
|
json: `{
|
||||||
"metadata": {"application": "seriatim", "version": "dev", "output_schema": "default"},
|
"metadata": {"application": "seriatim", "version": "dev", "output_schema": "seriatim-intermediate"},
|
||||||
"segments": [],
|
"segments": [],
|
||||||
"overlap_groups": []
|
"overlap_groups": []
|
||||||
}`,
|
}`,
|
||||||
@@ -158,21 +195,21 @@ func TestValidateDefaultJSONRejectsUnexpectedFields(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "segment source",
|
name: "segment source",
|
||||||
json: `{
|
json: `{
|
||||||
"metadata": {"application": "seriatim", "version": "dev", "output_schema": "default"},
|
"metadata": {"application": "seriatim", "version": "dev", "output_schema": "seriatim-intermediate"},
|
||||||
"segments": [{"id": 1, "source": "input.json", "start": 1, "end": 2, "speaker": "Alice", "text": "hello"}]
|
"segments": [{"id": 1, "source": "input.json", "start": 1, "end": 2, "speaker": "Alice", "text": "hello"}]
|
||||||
}`,
|
}`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "segment derived from",
|
name: "segment derived from",
|
||||||
json: `{
|
json: `{
|
||||||
"metadata": {"application": "seriatim", "version": "dev", "output_schema": "default"},
|
"metadata": {"application": "seriatim", "version": "dev", "output_schema": "seriatim-intermediate"},
|
||||||
"segments": [{"id": 1, "start": 1, "end": 2, "speaker": "Alice", "text": "hello", "derived_from": ["input.json#0"]}]
|
"segments": [{"id": 1, "start": 1, "end": 2, "speaker": "Alice", "text": "hello", "derived_from": ["input.json#0"]}]
|
||||||
}`,
|
}`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "segment words",
|
name: "segment words",
|
||||||
json: `{
|
json: `{
|
||||||
"metadata": {"application": "seriatim", "version": "dev", "output_schema": "default"},
|
"metadata": {"application": "seriatim", "version": "dev", "output_schema": "seriatim-intermediate"},
|
||||||
"segments": [{"id": 1, "start": 1, "end": 2, "speaker": "Alice", "text": "hello", "words": []}]
|
"segments": [{"id": 1, "start": 1, "end": 2, "speaker": "Alice", "text": "hello", "words": []}]
|
||||||
}`,
|
}`,
|
||||||
},
|
},
|
||||||
@@ -180,7 +217,7 @@ func TestValidateDefaultJSONRejectsUnexpectedFields(t *testing.T) {
|
|||||||
|
|
||||||
for _, test := range tests {
|
for _, test := range tests {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
err := ValidateDefaultJSON([]byte(test.json))
|
err := ValidateIntermediateJSON([]byte(test.json))
|
||||||
assertErrorContains(t, err, "additional properties")
|
assertErrorContains(t, err, "additional properties")
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -267,7 +304,7 @@ func TestValidateJSONRejectsUnexpectedInternalFields(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestValidateDefaultTranscriptRejectsMissingOrNonSequentialIDs(t *testing.T) {
|
func TestValidateIntermediateTranscriptRejectsMissingOrNonSequentialIDs(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
ids []int
|
ids []int
|
||||||
@@ -280,10 +317,10 @@ func TestValidateDefaultTranscriptRejectsMissingOrNonSequentialIDs(t *testing.T)
|
|||||||
|
|
||||||
for _, test := range tests {
|
for _, test := range tests {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
transcript := validDefaultTranscript()
|
transcript := validIntermediateTranscript()
|
||||||
transcript.Segments = transcript.Segments[:0]
|
transcript.Segments = transcript.Segments[:0]
|
||||||
for index, id := range test.ids {
|
for index, id := range test.ids {
|
||||||
transcript.Segments = append(transcript.Segments, DefaultSegment{
|
transcript.Segments = append(transcript.Segments, IntermediateSegment{
|
||||||
ID: id,
|
ID: id,
|
||||||
Start: float64(index),
|
Start: float64(index),
|
||||||
End: float64(index) + 1,
|
End: float64(index) + 1,
|
||||||
@@ -292,18 +329,18 @@ func TestValidateDefaultTranscriptRejectsMissingOrNonSequentialIDs(t *testing.T)
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
err := ValidateDefaultTranscript(transcript)
|
err := ValidateIntermediateTranscript(transcript)
|
||||||
assertErrorContains(t, err, test.want)
|
assertErrorContains(t, err, test.want)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestValidateDefaultTranscriptRejectsInvalidTiming(t *testing.T) {
|
func TestValidateIntermediateTranscriptRejectsInvalidTiming(t *testing.T) {
|
||||||
transcript := validDefaultTranscript()
|
transcript := validIntermediateTranscript()
|
||||||
transcript.Segments[0].Start = 2
|
transcript.Segments[0].Start = 2
|
||||||
transcript.Segments[0].End = 1
|
transcript.Segments[0].End = 1
|
||||||
|
|
||||||
err := ValidateDefaultTranscript(transcript)
|
err := ValidateIntermediateTranscript(transcript)
|
||||||
assertErrorContains(t, err, "segment 0 has end")
|
assertErrorContains(t, err, "segment 0 has end")
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -411,7 +448,7 @@ func validMinimalTranscript() MinimalTranscript {
|
|||||||
Metadata: MinimalMetadata{
|
Metadata: MinimalMetadata{
|
||||||
Application: "seriatim",
|
Application: "seriatim",
|
||||||
Version: "dev",
|
Version: "dev",
|
||||||
OutputSchema: "minimal",
|
OutputSchema: "seriatim-minimal",
|
||||||
},
|
},
|
||||||
Segments: []MinimalSegment{
|
Segments: []MinimalSegment{
|
||||||
{
|
{
|
||||||
@@ -425,14 +462,14 @@ func validMinimalTranscript() MinimalTranscript {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func validDefaultTranscript() DefaultTranscript {
|
func validIntermediateTranscript() IntermediateTranscript {
|
||||||
return DefaultTranscript{
|
return IntermediateTranscript{
|
||||||
Metadata: DefaultMetadata{
|
Metadata: IntermediateMetadata{
|
||||||
Application: "seriatim",
|
Application: "seriatim",
|
||||||
Version: "dev",
|
Version: "dev",
|
||||||
OutputSchema: "default",
|
OutputSchema: "seriatim-intermediate",
|
||||||
},
|
},
|
||||||
Segments: []DefaultSegment{
|
Segments: []IntermediateSegment{
|
||||||
{
|
{
|
||||||
ID: 1,
|
ID: 1,
|
||||||
Start: 1,
|
Start: 1,
|
||||||
|
|||||||
Reference in New Issue
Block a user