Compare commits
3 Commits
v0.9.1
...
d865bda4a9
| Author | SHA1 | Date | |
|---|---|---|---|
| d865bda4a9 | |||
| f20f06db12 | |||
| c9e98e14b5 |
4
.gitignore
vendored
4
.gitignore
vendored
@@ -1,3 +1,7 @@
|
|||||||
|
# ---> Codex
|
||||||
|
.codex
|
||||||
|
AGENTS.md
|
||||||
|
|
||||||
# ---> Go
|
# ---> Go
|
||||||
# If you prefer the allow list template instead of the deny list, see community template:
|
# If you prefer the allow list template instead of the deny list, see community template:
|
||||||
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
|
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
|
||||||
|
|||||||
@@ -42,8 +42,9 @@ steps:
|
|||||||
from_secret: GITEA_RELEASE_TOKEN
|
from_secret: GITEA_RELEASE_TOKEN
|
||||||
files:
|
files:
|
||||||
- dist/seriatim-*
|
- dist/seriatim-*
|
||||||
checksum: dist/seriatim-*
|
checksum: sha256
|
||||||
checksum-file: SHA256SUMS
|
checksum-file: SHA256SUMS
|
||||||
|
checksum-flatten: true
|
||||||
file-exists: skip
|
file-exists: skip
|
||||||
overwrite: false
|
overwrite: false
|
||||||
prerelease: false
|
prerelease: false
|
||||||
|
|||||||
31
README.md
31
README.md
@@ -49,7 +49,7 @@ Global flags:
|
|||||||
| `--autocorrect` | No | none | Autocorrect rules YAML file. When omitted, the default `autocorrect` module leaves text unchanged. |
|
| `--autocorrect` | No | none | Autocorrect rules YAML file. When omitted, the default `autocorrect` module leaves text unchanged. |
|
||||||
| `--input-reader` | No | `json-files` | Input reader module. |
|
| `--input-reader` | No | `json-files` | Input reader module. |
|
||||||
| `--output-modules` | No | `json` | Comma-separated output modules. |
|
| `--output-modules` | No | `json` | Comma-separated output modules. |
|
||||||
| `--output-schema` | No | `default` | JSON output contract. Allowed values are `default`, `minimal`, and `seriatim`. |
|
| `--output-schema` | No | `seriatim-intermediate` | JSON output contract. Allowed values are `seriatim-minimal`, `seriatim-intermediate`, and `seriatim-full`. If omitted, the runtime default is used; consumers that depend on a specific shape should set this explicitly. |
|
||||||
| `--preprocessing-modules` | No | `validate-raw,normalize-speakers,trim-text` | Comma-separated preprocessing modules, evaluated in order. |
|
| `--preprocessing-modules` | No | `validate-raw,normalize-speakers,trim-text` | Comma-separated preprocessing modules, evaluated in order. |
|
||||||
| `--postprocessing-modules` | No | `detect-overlaps,resolve-overlaps,backchannel,filler,resolve-danglers,coalesce,detect-overlaps,autocorrect,assign-ids,validate-output` | Comma-separated postprocessing modules, evaluated in order. |
|
| `--postprocessing-modules` | No | `detect-overlaps,resolve-overlaps,backchannel,filler,resolve-danglers,coalesce,detect-overlaps,autocorrect,assign-ids,validate-output` | Comma-separated postprocessing modules, evaluated in order. |
|
||||||
| `--coalesce-gap` | No | `3.0` | Maximum same-speaker gap in seconds for `coalesce`; also used as the `resolve-overlaps` context window. Must be a non-negative float. |
|
| `--coalesce-gap` | No | `3.0` | Maximum same-speaker gap in seconds for `coalesce`; also used as the `resolve-overlaps` context window. Must be a non-negative float. |
|
||||||
@@ -58,7 +58,8 @@ Environment variables:
|
|||||||
|
|
||||||
| Environment Variable | Default | Description |
|
| Environment Variable | Default | Description |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| `SERIATIM_OVERLAP_WORD_RUN_GAP` | `0.75` | Maximum gap in seconds between adjacent timed words when `resolve-overlaps` builds word-run replacement segments. Must be a positive float. |
|
| `SERIATIM_OUTPUT_SCHEMA` | `seriatim-intermediate` | Output schema used when `--output-schema` is not explicitly provided. Allowed values are `seriatim-minimal`, `seriatim-intermediate`, and `seriatim-full`. The CLI flag takes precedence. |
|
||||||
|
| `SERIATIM_OVERLAP_WORD_RUN_GAP` | `1.0` | Maximum gap in seconds between adjacent timed words when `resolve-overlaps` builds word-run replacement segments. Must be a positive float. |
|
||||||
| `SERIATIM_OVERLAP_WORD_RUN_REORDER_WINDOW` | `1.0` | Near-start window in seconds for ordering replacement word runs shortest-first. Must be a positive float. |
|
| `SERIATIM_OVERLAP_WORD_RUN_REORDER_WINDOW` | `1.0` | Near-start window in seconds for ordering replacement word runs shortest-first. Must be a positive float. |
|
||||||
| `SERIATIM_BACKCHANNEL_MAX_DURATION` | `2.0` | Maximum duration in seconds for `backchannel` classification. Must be a positive float. |
|
| `SERIATIM_BACKCHANNEL_MAX_DURATION` | `2.0` | Maximum duration in seconds for `backchannel` classification. Must be a positive float. |
|
||||||
| `SERIATIM_FILLER_MAX_DURATION` | `1.25` | Maximum duration in seconds for `filler` classification. Must be a positive float. |
|
| `SERIATIM_FILLER_MAX_DURATION` | `1.25` | Maximum duration in seconds for `filler` classification. Must be a positive float. |
|
||||||
@@ -159,14 +160,16 @@ The old `inputs:` direct mapping format is no longer supported.
|
|||||||
|
|
||||||
`--output-modules json` controls the writer. `--output-schema` controls the JSON contract that writer serializes.
|
`--output-modules json` controls the writer. `--output-schema` controls the JSON contract that writer serializes.
|
||||||
|
|
||||||
The `default` schema is the default output contract. It stays close to `minimal`, but adds optional `categories` on each segment:
|
The named schemas are stable public contracts. If a consumer depends on a specific shape, it should request that schema explicitly at runtime. The runtime default selection may change in a future release.
|
||||||
|
|
||||||
|
The `seriatim-intermediate` schema is the current default selection when neither `--output-schema` nor `SERIATIM_OUTPUT_SCHEMA` is set. It stays close to the minimal schema, but adds optional `categories` on each segment:
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
"metadata": {
|
"metadata": {
|
||||||
"application": "seriatim",
|
"application": "seriatim",
|
||||||
"version": "dev",
|
"version": "dev",
|
||||||
"output_schema": "default"
|
"output_schema": "seriatim-intermediate"
|
||||||
},
|
},
|
||||||
"segments": [
|
"segments": [
|
||||||
{
|
{
|
||||||
@@ -181,7 +184,7 @@ The `default` schema is the default output contract. It stays close to `minimal`
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
The explicit `seriatim` schema uses the full seriatim envelope:
|
The `seriatim-full` schema uses the full seriatim envelope:
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
@@ -191,7 +194,7 @@ The explicit `seriatim` schema uses the full seriatim envelope:
|
|||||||
"input_reader": "json-files",
|
"input_reader": "json-files",
|
||||||
"input_files": ["eric.json", "mike.json"],
|
"input_files": ["eric.json", "mike.json"],
|
||||||
"preprocessing_modules": ["validate-raw", "normalize-speakers", "trim-text"],
|
"preprocessing_modules": ["validate-raw", "normalize-speakers", "trim-text"],
|
||||||
"postprocessing_modules": ["detect-overlaps", "resolve-overlaps", "backchannel", "filler", "coalesce", "resolve-danglers", "detect-overlaps", "autocorrect", "assign-ids", "validate-output"],
|
"postprocessing_modules": ["detect-overlaps", "resolve-overlaps", "backchannel", "filler", "resolve-danglers", "coalesce", "detect-overlaps", "autocorrect", "assign-ids", "validate-output"],
|
||||||
"output_modules": ["json"]
|
"output_modules": ["json"]
|
||||||
},
|
},
|
||||||
"segments": [
|
"segments": [
|
||||||
@@ -231,14 +234,14 @@ The explicit `seriatim` schema uses the full seriatim envelope:
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
The `minimal` schema emits minimal metadata and compact ordered segments:
|
The `seriatim-minimal` schema emits minimal metadata and compact ordered segments:
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
"metadata": {
|
"metadata": {
|
||||||
"application": "seriatim",
|
"application": "seriatim",
|
||||||
"version": "dev",
|
"version": "dev",
|
||||||
"output_schema": "minimal"
|
"output_schema": "seriatim-minimal"
|
||||||
},
|
},
|
||||||
"segments": [
|
"segments": [
|
||||||
{
|
{
|
||||||
@@ -254,7 +257,7 @@ The `minimal` schema emits minimal metadata and compact ordered segments:
|
|||||||
|
|
||||||
Minimal output intentionally omits categories, overlap groups, source/provenance fields, and pipeline configuration metadata.
|
Minimal output intentionally omits categories, overlap groups, source/provenance fields, and pipeline configuration metadata.
|
||||||
|
|
||||||
Default output intentionally omits overlap groups and source/provenance fields, but keeps optional `categories` and minimal metadata.
|
Intermediate output intentionally omits overlap groups and source/provenance fields, but keeps optional `categories` and minimal metadata.
|
||||||
|
|
||||||
Segments are sorted deterministically by:
|
Segments are sorted deterministically by:
|
||||||
|
|
||||||
@@ -270,7 +273,7 @@ The public Go output contract is available from:
|
|||||||
import "gitea.maximumdirect.net/eric/seriatim/schema"
|
import "gitea.maximumdirect.net/eric/seriatim/schema"
|
||||||
```
|
```
|
||||||
|
|
||||||
The same package embeds machine-readable JSON Schemas in `schema/output.schema.json`, `schema/default-output.schema.json`, and `schema/minimal-output.schema.json`. The default `validate-output` postprocessor validates the selected output shape and verifies final segment IDs are present, sequential, and start at `1`.
|
The same package embeds machine-readable JSON Schemas in `schema/full-output.schema.json`, `schema/intermediate-output.schema.json`, and `schema/minimal-output.schema.json`. The default `validate-output` postprocessor validates the selected output shape and verifies final segment IDs are present, sequential, and start at `1`.
|
||||||
|
|
||||||
## Overlap Detection
|
## Overlap Detection
|
||||||
|
|
||||||
@@ -295,12 +298,12 @@ For each detected overlap group, `resolve-overlaps` uses preserved WhisperX word
|
|||||||
|
|
||||||
- The resolution window expands the detected overlap group by `--coalesce-gap` seconds on both sides.
|
- The resolution window expands the detected overlap group by `--coalesce-gap` seconds on both sides.
|
||||||
- Nearby same-speaker context segments are included when they intersect the expanded window and their start or end is within `--coalesce-gap` of the original overlap boundary.
|
- Nearby same-speaker context segments are included when they intersect the expanded window and their start or end is within `--coalesce-gap` of the original overlap boundary.
|
||||||
- Words are included when their interval intersects the expanded resolution window.
|
- Once a segment is selected for replacement, all timed words from that segment participate in word-run construction; the window controls segment selection, not per-word clipping.
|
||||||
- Context segments that are part of another detected overlap group are not pulled into the current group.
|
- Context segments that are part of another detected overlap group are not pulled into the current group.
|
||||||
- Untimed words are included in replacement text in original word order when nearby timed words create a replacement run.
|
- Untimed words are included in replacement text in original word order when nearby timed words create a replacement run.
|
||||||
- Untimed words do not affect replacement segment start/end times or word-run gap splitting.
|
- Untimed words do not affect replacement segment start/end times or word-run gap splitting.
|
||||||
- Words for the same speaker are merged into one run when the gap between adjacent words is no greater than `SERIATIM_OVERLAP_WORD_RUN_GAP`.
|
- Words for the same speaker are merged into one run when the gap between adjacent words is no greater than `SERIATIM_OVERLAP_WORD_RUN_GAP`.
|
||||||
- The default word-run gap is `0.75` seconds.
|
- The default word-run gap is `1.0` seconds.
|
||||||
- Set `SERIATIM_OVERLAP_WORD_RUN_GAP` to a positive number of seconds to override the default.
|
- Set `SERIATIM_OVERLAP_WORD_RUN_GAP` to a positive number of seconds to override the default.
|
||||||
- Near-start replacement word runs are reordered so shorter segments come first when adjacent starts are within `SERIATIM_OVERLAP_WORD_RUN_REORDER_WINDOW`.
|
- Near-start replacement word runs are reordered so shorter segments come first when adjacent starts are within `SERIATIM_OVERLAP_WORD_RUN_REORDER_WINDOW`.
|
||||||
- The default word-run reorder window is `1.0` seconds.
|
- The default word-run reorder window is `1.0` seconds.
|
||||||
@@ -339,12 +342,12 @@ The default pipeline runs `resolve-danglers` before `coalesce` and before the se
|
|||||||
|
|
||||||
- Dangling-end fragments have no more than two words and end in punctuation.
|
- Dangling-end fragments have no more than two words and end in punctuation.
|
||||||
- Dangling-start fragments have no more than two words.
|
- Dangling-start fragments have no more than two words.
|
||||||
- Matching uses any shared `derived_from` value.
|
- Matching uses same-speaker segments with any shared `derived_from` value.
|
||||||
- Merged segments use `source_ref` values such as `resolve-danglers:1`, keep the target segment's transcript position, and union `derived_from`.
|
- Merged segments use `source_ref` values such as `resolve-danglers:1`, keep the target segment's transcript position, and union `derived_from`.
|
||||||
|
|
||||||
## Coalescing
|
## Coalescing
|
||||||
|
|
||||||
The default pipeline runs `coalesce` after `resolve-danglers` and the second overlap detection pass. It merges adjacent same-speaker segments in the transcript's current order when `next.start - current.end <= --coalesce-gap`.
|
The default pipeline runs `coalesce` after `resolve-danglers` and before the second overlap detection pass. It merges adjacent same-speaker segments in the transcript's current order when `next.start - current.end <= --coalesce-gap`.
|
||||||
|
|
||||||
Coalesced segments use `source_ref` values such as `coalesce:1`, include `derived_from`, and omit `source_segment_index`.
|
Coalesced segments use `source_ref` values such as `coalesce:1`, include `derived_from`, and omit `source_segment_index`.
|
||||||
|
|
||||||
|
|||||||
@@ -173,6 +173,14 @@ The current output format is JSON, specified with:
|
|||||||
--output-file merged.json
|
--output-file merged.json
|
||||||
```
|
```
|
||||||
|
|
||||||
|
The current named JSON schemas are:
|
||||||
|
|
||||||
|
- `seriatim-minimal`
|
||||||
|
- `seriatim-intermediate`
|
||||||
|
- `seriatim-full`
|
||||||
|
|
||||||
|
The current runtime default selection is `seriatim-intermediate`, but default selection may change over time. Consumers that depend on a specific schema should request it explicitly.
|
||||||
|
|
||||||
Future output formats may include:
|
Future output formats may include:
|
||||||
|
|
||||||
- Markdown.
|
- Markdown.
|
||||||
@@ -216,7 +224,7 @@ seriatim merge \
|
|||||||
--preprocessing-modules validate-raw,normalize-speakers,trim-text \
|
--preprocessing-modules validate-raw,normalize-speakers,trim-text \
|
||||||
--postprocessing-modules detect-overlaps,resolve-overlaps,backchannel,filler,resolve-danglers,coalesce,detect-overlaps,autocorrect,assign-ids,validate-output \
|
--postprocessing-modules detect-overlaps,resolve-overlaps,backchannel,filler,resolve-danglers,coalesce,detect-overlaps,autocorrect,assign-ids,validate-output \
|
||||||
--output-modules json \
|
--output-modules json \
|
||||||
--output-schema default \
|
--output-schema seriatim-intermediate \
|
||||||
--output-file merged.json \
|
--output-file merged.json \
|
||||||
--report-file report.json
|
--report-file report.json
|
||||||
```
|
```
|
||||||
@@ -358,7 +366,7 @@ Initial classifications may include:
|
|||||||
- `backchannel`
|
- `backchannel`
|
||||||
- `crosstalk`
|
- `crosstalk`
|
||||||
|
|
||||||
The `resolve-overlaps` module uses preserved word-level timing to replace detected overlap-group segments with smaller word-run segments when usable timing is available. Resolution expands each overlap window by the configured coalesce gap so nearby same-speaker context can be absorbed into the replacement runs. Groups without usable word timing remain unresolved for later passes or human review.
|
The `resolve-overlaps` module uses preserved word-level timing to replace detected overlap-group segments with smaller word-run segments when usable timing is available. Resolution expands each overlap window by the configured coalesce gap so nearby same-speaker context can be absorbed into the replacement runs. Once a segment is selected for replacement, all timed words from that segment participate in word-run construction so text is not clipped at the window boundary. Groups without usable word timing remain unresolved for later passes or human review.
|
||||||
|
|
||||||
Overlap resolution should be non-destructive. Original segment text, timing, and source metadata must remain recoverable.
|
Overlap resolution should be non-destructive. Original segment text, timing, and source metadata must remain recoverable.
|
||||||
|
|
||||||
|
|||||||
@@ -57,12 +57,12 @@ func FromMerged(cfg config.Config, merged model.MergedTranscript) schema.Transcr
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// DefaultFromMerged converts the internal merged transcript model into the
|
// IntermediateFromMerged converts the internal merged transcript model into the
|
||||||
// compact default public serialized output contract.
|
// compact intermediate public serialized output contract.
|
||||||
func DefaultFromMerged(cfg config.Config, merged model.MergedTranscript) schema.DefaultTranscript {
|
func IntermediateFromMerged(cfg config.Config, merged model.MergedTranscript) schema.IntermediateTranscript {
|
||||||
segments := make([]schema.DefaultSegment, len(merged.Segments))
|
segments := make([]schema.IntermediateSegment, len(merged.Segments))
|
||||||
for index, segment := range merged.Segments {
|
for index, segment := range merged.Segments {
|
||||||
segments[index] = schema.DefaultSegment{
|
segments[index] = schema.IntermediateSegment{
|
||||||
ID: segment.ID,
|
ID: segment.ID,
|
||||||
Start: segment.Start,
|
Start: segment.Start,
|
||||||
End: segment.End,
|
End: segment.End,
|
||||||
@@ -72,11 +72,11 @@ func DefaultFromMerged(cfg config.Config, merged model.MergedTranscript) schema.
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return schema.DefaultTranscript{
|
return schema.IntermediateTranscript{
|
||||||
Metadata: schema.DefaultMetadata{
|
Metadata: schema.IntermediateMetadata{
|
||||||
Application: ApplicationName,
|
Application: ApplicationName,
|
||||||
Version: buildinfo.Version,
|
Version: buildinfo.Version,
|
||||||
OutputSchema: config.OutputSchemaDefault,
|
OutputSchema: config.OutputSchemaIntermediate,
|
||||||
},
|
},
|
||||||
Segments: segments,
|
Segments: segments,
|
||||||
}
|
}
|
||||||
@@ -110,14 +110,14 @@ func MinimalFromMerged(cfg config.Config, merged model.MergedTranscript) schema.
|
|||||||
// runtime-selected public output contract.
|
// runtime-selected public output contract.
|
||||||
func SelectedFromMerged(cfg config.Config, merged model.MergedTranscript) any {
|
func SelectedFromMerged(cfg config.Config, merged model.MergedTranscript) any {
|
||||||
switch cfg.OutputSchema {
|
switch cfg.OutputSchema {
|
||||||
case config.OutputSchemaDefault:
|
|
||||||
return DefaultFromMerged(cfg, merged)
|
|
||||||
case config.OutputSchemaMinimal:
|
case config.OutputSchemaMinimal:
|
||||||
return MinimalFromMerged(cfg, merged)
|
return MinimalFromMerged(cfg, merged)
|
||||||
case config.OutputSchemaSeriatim:
|
case config.OutputSchemaFull:
|
||||||
return FromMerged(cfg, merged)
|
return FromMerged(cfg, merged)
|
||||||
|
case config.OutputSchemaIntermediate:
|
||||||
|
return IntermediateFromMerged(cfg, merged)
|
||||||
default:
|
default:
|
||||||
return FromMerged(cfg, merged)
|
return IntermediateFromMerged(cfg, merged)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -23,28 +23,28 @@ func TestFromMergedUsesBuildVersion(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSelectedFromMergedDefaultsToSeriatimTranscript(t *testing.T) {
|
func TestSelectedFromMergedDefaultsToIntermediateTranscript(t *testing.T) {
|
||||||
got := SelectedFromMerged(config.Config{}, model.MergedTranscript{})
|
got := SelectedFromMerged(config.Config{}, model.MergedTranscript{})
|
||||||
|
if _, ok := got.(schema.IntermediateTranscript); !ok {
|
||||||
|
t.Fatalf("selected artifact type = %T, want schema.IntermediateTranscript", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSelectedFromMergedUsesIntermediateWhenConfigured(t *testing.T) {
|
||||||
|
got := SelectedFromMerged(config.Config{OutputSchema: config.OutputSchemaIntermediate}, model.MergedTranscript{})
|
||||||
|
if _, ok := got.(schema.IntermediateTranscript); !ok {
|
||||||
|
t.Fatalf("selected artifact type = %T, want schema.IntermediateTranscript", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSelectedFromMergedUsesFullWhenConfigured(t *testing.T) {
|
||||||
|
got := SelectedFromMerged(config.Config{OutputSchema: config.OutputSchemaFull}, model.MergedTranscript{})
|
||||||
if _, ok := got.(schema.Transcript); !ok {
|
if _, ok := got.(schema.Transcript); !ok {
|
||||||
t.Fatalf("selected artifact type = %T, want schema.Transcript", got)
|
t.Fatalf("selected artifact type = %T, want schema.Transcript", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSelectedFromMergedUsesDefaultWhenConfigured(t *testing.T) {
|
func TestIntermediateFromMergedEmitsOnlyIntermediateShape(t *testing.T) {
|
||||||
got := SelectedFromMerged(config.Config{OutputSchema: config.OutputSchemaDefault}, model.MergedTranscript{})
|
|
||||||
if _, ok := got.(schema.DefaultTranscript); !ok {
|
|
||||||
t.Fatalf("selected artifact type = %T, want schema.DefaultTranscript", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSelectedFromMergedUsesSeriatimWhenConfigured(t *testing.T) {
|
|
||||||
got := SelectedFromMerged(config.Config{OutputSchema: config.OutputSchemaSeriatim}, model.MergedTranscript{})
|
|
||||||
if _, ok := got.(schema.Transcript); !ok {
|
|
||||||
t.Fatalf("selected artifact type = %T, want schema.Transcript", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDefaultFromMergedEmitsOnlyDefaultShape(t *testing.T) {
|
|
||||||
merged := model.MergedTranscript{
|
merged := model.MergedTranscript{
|
||||||
Segments: []model.Segment{
|
Segments: []model.Segment{
|
||||||
{
|
{
|
||||||
@@ -65,19 +65,19 @@ func TestDefaultFromMergedEmitsOnlyDefaultShape(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
got := DefaultFromMerged(config.Config{OutputSchema: config.OutputSchemaDefault}, merged)
|
got := IntermediateFromMerged(config.Config{OutputSchema: config.OutputSchemaIntermediate}, merged)
|
||||||
want := schema.DefaultTranscript{
|
want := schema.IntermediateTranscript{
|
||||||
Metadata: schema.DefaultMetadata{
|
Metadata: schema.IntermediateMetadata{
|
||||||
Application: ApplicationName,
|
Application: ApplicationName,
|
||||||
Version: buildinfo.Version,
|
Version: buildinfo.Version,
|
||||||
OutputSchema: config.OutputSchemaDefault,
|
OutputSchema: config.OutputSchemaIntermediate,
|
||||||
},
|
},
|
||||||
Segments: []schema.DefaultSegment{
|
Segments: []schema.IntermediateSegment{
|
||||||
{ID: 1, Start: 1, End: 2, Speaker: "Alice", Text: "hello", Categories: []string{"backchannel"}},
|
{ID: 1, Start: 1, End: 2, Speaker: "Alice", Text: "hello", Categories: []string{"backchannel"}},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
if !reflect.DeepEqual(got, want) {
|
if !reflect.DeepEqual(got, want) {
|
||||||
t.Fatalf("default transcript = %#v, want %#v", got, want)
|
t.Fatalf("intermediate transcript = %#v, want %#v", got, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -51,8 +51,8 @@ func (validateOutput) Process(ctx context.Context, in model.MergedTranscript, cf
|
|||||||
selected := artifact.SelectedFromMerged(cfg, in)
|
selected := artifact.SelectedFromMerged(cfg, in)
|
||||||
var err error
|
var err error
|
||||||
switch transcript := selected.(type) {
|
switch transcript := selected.(type) {
|
||||||
case schema.DefaultTranscript:
|
case schema.IntermediateTranscript:
|
||||||
err = schema.ValidateDefaultTranscript(transcript)
|
err = schema.ValidateIntermediateTranscript(transcript)
|
||||||
case schema.MinimalTranscript:
|
case schema.MinimalTranscript:
|
||||||
err = schema.ValidateMinimalTranscript(transcript)
|
err = schema.ValidateMinimalTranscript(transcript)
|
||||||
case schema.Transcript:
|
case schema.Transcript:
|
||||||
|
|||||||
@@ -81,7 +81,7 @@ func TestValidateOutputUsesMinimalSchemaWhenConfigured(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestValidateOutputUsesSeriatimSchemaWhenConfigured(t *testing.T) {
|
func TestValidateOutputUsesFullSchemaWhenConfigured(t *testing.T) {
|
||||||
merged := model.MergedTranscript{
|
merged := model.MergedTranscript{
|
||||||
Segments: []model.Segment{
|
Segments: []model.Segment{
|
||||||
{
|
{
|
||||||
@@ -100,7 +100,7 @@ func TestValidateOutputUsesSeriatimSchemaWhenConfigured(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
cfg := testConfig()
|
cfg := testConfig()
|
||||||
cfg.OutputSchema = config.OutputSchemaSeriatim
|
cfg.OutputSchema = config.OutputSchemaFull
|
||||||
got, events, err := validateOutput{}.Process(context.Background(), merged, cfg)
|
got, events, err := validateOutput{}.Process(context.Background(), merged, cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("validate output: %v", err)
|
t.Fatalf("validate output: %v", err)
|
||||||
|
|||||||
@@ -15,7 +15,12 @@ func newMergeCommand() *cobra.Command {
|
|||||||
Use: "merge",
|
Use: "merge",
|
||||||
Short: "Run the transcript merge pipeline",
|
Short: "Run the transcript merge pipeline",
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
cfg, err := config.NewMergeConfig(opts)
|
mergeOpts := opts
|
||||||
|
if !cmd.Flags().Changed("output-schema") {
|
||||||
|
mergeOpts.OutputSchema = ""
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg, err := config.NewMergeConfig(mergeOpts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -32,7 +37,7 @@ func newMergeCommand() *cobra.Command {
|
|||||||
flags.StringVar(&opts.AutocorrectFile, "autocorrect", "", "autocorrect rules file")
|
flags.StringVar(&opts.AutocorrectFile, "autocorrect", "", "autocorrect rules file")
|
||||||
flags.StringVar(&opts.InputReader, "input-reader", config.DefaultInputReader, "input reader module")
|
flags.StringVar(&opts.InputReader, "input-reader", config.DefaultInputReader, "input reader module")
|
||||||
flags.StringVar(&opts.OutputModules, "output-modules", config.DefaultOutputModules, "comma-separated output modules")
|
flags.StringVar(&opts.OutputModules, "output-modules", config.DefaultOutputModules, "comma-separated output modules")
|
||||||
flags.StringVar(&opts.OutputSchema, "output-schema", config.DefaultOutputSchema, "output JSON schema: default, minimal, or seriatim")
|
flags.StringVar(&opts.OutputSchema, "output-schema", config.DefaultOutputSchema, "output JSON schema: seriatim-minimal, seriatim-intermediate (default), or seriatim-full")
|
||||||
flags.StringVar(&opts.PreprocessingModules, "preprocessing-modules", config.DefaultPreprocessingModules, "comma-separated preprocessing modules")
|
flags.StringVar(&opts.PreprocessingModules, "preprocessing-modules", config.DefaultPreprocessingModules, "comma-separated preprocessing modules")
|
||||||
flags.StringVar(&opts.PostprocessingModules, "postprocessing-modules", config.DefaultPostprocessingModules, "comma-separated postprocessing modules")
|
flags.StringVar(&opts.PostprocessingModules, "postprocessing-modules", config.DefaultPostprocessingModules, "comma-separated postprocessing modules")
|
||||||
flags.StringVar(&opts.CoalesceGap, "coalesce-gap", config.DefaultCoalesceGapValue, "maximum same-speaker gap in seconds for coalesce")
|
flags.StringVar(&opts.CoalesceGap, "coalesce-gap", config.DefaultCoalesceGapValue, "maximum same-speaker gap in seconds for coalesce")
|
||||||
|
|||||||
@@ -114,7 +114,8 @@ func TestMergeWritesMergedOutputAndReport(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMergeWritesDefaultOutputSchema(t *testing.T) {
|
func TestMergeWritesIntermediateOutputSchema(t *testing.T) {
|
||||||
|
t.Setenv(config.OutputSchemaEnv, "")
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
input := writeJSONFile(t, dir, "input.json", `{
|
input := writeJSONFile(t, dir, "input.json", `{
|
||||||
"segments": [
|
"segments": [
|
||||||
@@ -131,10 +132,10 @@ func TestMergeWritesDefaultOutputSchema(t *testing.T) {
|
|||||||
t.Fatalf("merge failed: %v", err)
|
t.Fatalf("merge failed: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
var transcript schema.DefaultTranscript
|
var transcript schema.IntermediateTranscript
|
||||||
readJSON(t, output, &transcript)
|
readJSON(t, output, &transcript)
|
||||||
if transcript.Metadata.OutputSchema != config.OutputSchemaDefault {
|
if transcript.Metadata.OutputSchema != config.OutputSchemaIntermediate {
|
||||||
t.Fatalf("output_schema = %q, want default", transcript.Metadata.OutputSchema)
|
t.Fatalf("output_schema = %q, want %q", transcript.Metadata.OutputSchema, config.OutputSchemaIntermediate)
|
||||||
}
|
}
|
||||||
if len(transcript.Segments) != 1 {
|
if len(transcript.Segments) != 1 {
|
||||||
t.Fatalf("segment count = %d, want 1", len(transcript.Segments))
|
t.Fatalf("segment count = %d, want 1", len(transcript.Segments))
|
||||||
@@ -149,12 +150,12 @@ func TestMergeWritesDefaultOutputSchema(t *testing.T) {
|
|||||||
}
|
}
|
||||||
for _, forbidden := range []string{"overlap_groups", "source", "derived_from", "words"} {
|
for _, forbidden := range []string{"overlap_groups", "source", "derived_from", "words"} {
|
||||||
if strings.Contains(string(outputBytes), forbidden) {
|
if strings.Contains(string(outputBytes), forbidden) {
|
||||||
t.Fatalf("default output contains %q:\n%s", forbidden, outputBytes)
|
t.Fatalf("intermediate output contains %q:\n%s", forbidden, outputBytes)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMergeWritesSeriatimOutputSchema(t *testing.T) {
|
func TestMergeWritesFullOutputSchema(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
input := writeJSONFile(t, dir, "input.json", `{
|
input := writeJSONFile(t, dir, "input.json", `{
|
||||||
"segments": [
|
"segments": [
|
||||||
@@ -166,7 +167,7 @@ func TestMergeWritesSeriatimOutputSchema(t *testing.T) {
|
|||||||
err := executeMergeRaw(
|
err := executeMergeRaw(
|
||||||
"--input-file", input,
|
"--input-file", input,
|
||||||
"--output-file", output,
|
"--output-file", output,
|
||||||
"--output-schema", "seriatim",
|
"--output-schema", config.OutputSchemaFull,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("merge failed: %v", err)
|
t.Fatalf("merge failed: %v", err)
|
||||||
@@ -203,7 +204,7 @@ func TestMergeWritesMinimalOutputSchema(t *testing.T) {
|
|||||||
err := executeMerge(
|
err := executeMerge(
|
||||||
"--input-file", input,
|
"--input-file", input,
|
||||||
"--output-file", output,
|
"--output-file", output,
|
||||||
"--output-schema", "minimal",
|
"--output-schema", config.OutputSchemaMinimal,
|
||||||
"--report-file", reportPath,
|
"--report-file", reportPath,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -215,8 +216,8 @@ func TestMergeWritesMinimalOutputSchema(t *testing.T) {
|
|||||||
if transcript.Metadata.Application != "seriatim" {
|
if transcript.Metadata.Application != "seriatim" {
|
||||||
t.Fatalf("application = %q, want seriatim", transcript.Metadata.Application)
|
t.Fatalf("application = %q, want seriatim", transcript.Metadata.Application)
|
||||||
}
|
}
|
||||||
if transcript.Metadata.OutputSchema != "minimal" {
|
if transcript.Metadata.OutputSchema != config.OutputSchemaMinimal {
|
||||||
t.Fatalf("output_schema = %q, want minimal", transcript.Metadata.OutputSchema)
|
t.Fatalf("output_schema = %q, want %q", transcript.Metadata.OutputSchema, config.OutputSchemaMinimal)
|
||||||
}
|
}
|
||||||
if got, want := len(transcript.Segments), 2; got != want {
|
if got, want := len(transcript.Segments), 2; got != want {
|
||||||
t.Fatalf("segment count = %d, want %d", got, want)
|
t.Fatalf("segment count = %d, want %d", got, want)
|
||||||
@@ -330,7 +331,7 @@ func TestMergeValidateMinimalOutputBeforeAssignIDsFails(t *testing.T) {
|
|||||||
err := executeMerge(
|
err := executeMerge(
|
||||||
"--input-file", input,
|
"--input-file", input,
|
||||||
"--output-file", output,
|
"--output-file", output,
|
||||||
"--output-schema", "minimal",
|
"--output-schema", config.OutputSchemaMinimal,
|
||||||
"--postprocessing-modules", "validate-output,assign-ids",
|
"--postprocessing-modules", "validate-output,assign-ids",
|
||||||
)
|
)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
@@ -472,16 +473,16 @@ func TestMergeResolvesOverlapGroupsWithWordRuns(t *testing.T) {
|
|||||||
|
|
||||||
var transcript model.FinalTranscript
|
var transcript model.FinalTranscript
|
||||||
readJSON(t, output, &transcript)
|
readJSON(t, output, &transcript)
|
||||||
if len(transcript.OverlapGroups) != 0 {
|
if got, want := len(transcript.OverlapGroups), 1; got != want {
|
||||||
t.Fatalf("overlap groups = %#v, want none", transcript.OverlapGroups)
|
t.Fatalf("overlap group count = %d, want %d (%#v)", got, want, transcript.OverlapGroups)
|
||||||
}
|
}
|
||||||
if got, want := len(transcript.Segments), 3; got != want {
|
if got, want := len(transcript.Segments), 2; got != want {
|
||||||
t.Fatalf("segment count = %d, want %d", got, want)
|
t.Fatalf("segment count = %d, want %d", got, want)
|
||||||
}
|
}
|
||||||
|
|
||||||
wantTexts := []string{"outside hello there", "bob reply", "later"}
|
wantTexts := []string{"outside hello there later", "bob reply"}
|
||||||
wantSpeakers := []string{"Alice", "Bob", "Alice"}
|
wantSpeakers := []string{"Alice", "Bob"}
|
||||||
wantRefs := []string{"word-run:1:1:1", "word-run:1:2:1", "word-run:1:1:2"}
|
wantRefs := []string{"word-run:1:1:1", "word-run:1:2:1"}
|
||||||
for index, segment := range transcript.Segments {
|
for index, segment := range transcript.Segments {
|
||||||
if segment.ID != index+1 {
|
if segment.ID != index+1 {
|
||||||
t.Fatalf("segment %d id = %d, want %d", index, segment.ID, index+1)
|
t.Fatalf("segment %d id = %d, want %d", index, segment.ID, index+1)
|
||||||
@@ -498,8 +499,8 @@ func TestMergeResolvesOverlapGroupsWithWordRuns(t *testing.T) {
|
|||||||
if segment.SourceSegmentIndex != nil {
|
if segment.SourceSegmentIndex != nil {
|
||||||
t.Fatalf("segment %d source_segment_index = %d, want nil", index, *segment.SourceSegmentIndex)
|
t.Fatalf("segment %d source_segment_index = %d, want nil", index, *segment.SourceSegmentIndex)
|
||||||
}
|
}
|
||||||
if segment.OverlapGroupID != 0 {
|
if segment.OverlapGroupID != 1 {
|
||||||
t.Fatalf("segment %d overlap_group_id = %d, want 0", index, segment.OverlapGroupID)
|
t.Fatalf("segment %d overlap_group_id = %d, want 1", index, segment.OverlapGroupID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !equalStrings(transcript.Segments[0].DerivedFrom, []string{inputA + "#0"}) {
|
if !equalStrings(transcript.Segments[0].DerivedFrom, []string{inputA + "#0"}) {
|
||||||
@@ -516,7 +517,7 @@ func TestMergeResolvesOverlapGroupsWithWordRuns(t *testing.T) {
|
|||||||
|
|
||||||
var rpt report.Report
|
var rpt report.Report
|
||||||
readJSON(t, reportPath, &rpt)
|
readJSON(t, reportPath, &rpt)
|
||||||
if !hasReportEvent(rpt, "postprocessing", "resolve-overlaps", "processed 1 overlap group(s); changed 1; removed 2 original segment(s); created 3 replacement segment(s)") {
|
if !hasReportEvent(rpt, "postprocessing", "resolve-overlaps", "processed 1 overlap group(s); changed 1; removed 2 original segment(s); created 2 replacement segment(s)") {
|
||||||
t.Fatal("expected resolve-overlaps summary report event")
|
t.Fatal("expected resolve-overlaps summary report event")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1927,7 +1928,7 @@ func TestMergeResolveOverlapsPreservesAbsorbedContextPrefix(t *testing.T) {
|
|||||||
"--input-file", inputA,
|
"--input-file", inputA,
|
||||||
"--input-file", inputB,
|
"--input-file", inputB,
|
||||||
"--speakers", speakers,
|
"--speakers", speakers,
|
||||||
"--output-schema", "minimal",
|
"--output-schema", config.OutputSchemaMinimal,
|
||||||
"--output-file", output,
|
"--output-file", output,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1994,7 +1995,7 @@ func executeMerge(args ...string) error {
|
|||||||
if !hasOutputSchemaFlag(args) {
|
if !hasOutputSchemaFlag(args) {
|
||||||
// Most integration tests were written against the full envelope; keep
|
// Most integration tests were written against the full envelope; keep
|
||||||
// that behavior unless the caller explicitly asks for another schema.
|
// that behavior unless the caller explicitly asks for another schema.
|
||||||
args = append(args, "--output-schema", config.OutputSchemaSeriatim)
|
args = append(args, "--output-schema", config.OutputSchemaFull)
|
||||||
}
|
}
|
||||||
return executeMergeRaw(args...)
|
return executeMergeRaw(args...)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,22 +13,23 @@ import (
|
|||||||
const (
|
const (
|
||||||
DefaultInputReader = "json-files"
|
DefaultInputReader = "json-files"
|
||||||
DefaultOutputModules = "json"
|
DefaultOutputModules = "json"
|
||||||
DefaultOutputSchema = OutputSchemaDefault
|
DefaultOutputSchema = OutputSchemaIntermediate
|
||||||
DefaultPreprocessingModules = "validate-raw,normalize-speakers,trim-text"
|
DefaultPreprocessingModules = "validate-raw,normalize-speakers,trim-text"
|
||||||
DefaultPostprocessingModules = "detect-overlaps,resolve-overlaps,backchannel,filler,resolve-danglers,coalesce,detect-overlaps,autocorrect,assign-ids,validate-output"
|
DefaultPostprocessingModules = "detect-overlaps,resolve-overlaps,backchannel,filler,resolve-danglers,coalesce,detect-overlaps,autocorrect,assign-ids,validate-output"
|
||||||
DefaultOverlapWordRunGap = 0.75
|
DefaultOverlapWordRunGap = 1.0
|
||||||
DefaultWordRunReorderWindow = 1.0
|
DefaultWordRunReorderWindow = 1.0
|
||||||
DefaultCoalesceGap = 3.0
|
DefaultCoalesceGap = 3.0
|
||||||
DefaultCoalesceGapValue = "3.0"
|
DefaultCoalesceGapValue = "3.0"
|
||||||
DefaultBackchannelMaxDuration = 2.0
|
DefaultBackchannelMaxDuration = 2.0
|
||||||
DefaultFillerMaxDuration = 1.25
|
DefaultFillerMaxDuration = 1.25
|
||||||
|
OutputSchemaEnv = "SERIATIM_OUTPUT_SCHEMA"
|
||||||
OverlapWordRunGapEnv = "SERIATIM_OVERLAP_WORD_RUN_GAP"
|
OverlapWordRunGapEnv = "SERIATIM_OVERLAP_WORD_RUN_GAP"
|
||||||
WordRunReorderWindowEnv = "SERIATIM_OVERLAP_WORD_RUN_REORDER_WINDOW"
|
WordRunReorderWindowEnv = "SERIATIM_OVERLAP_WORD_RUN_REORDER_WINDOW"
|
||||||
BackchannelMaxDurationEnv = "SERIATIM_BACKCHANNEL_MAX_DURATION"
|
BackchannelMaxDurationEnv = "SERIATIM_BACKCHANNEL_MAX_DURATION"
|
||||||
FillerMaxDurationEnv = "SERIATIM_FILLER_MAX_DURATION"
|
FillerMaxDurationEnv = "SERIATIM_FILLER_MAX_DURATION"
|
||||||
OutputSchemaDefault = "default"
|
OutputSchemaMinimal = "seriatim-minimal"
|
||||||
OutputSchemaSeriatim = "seriatim"
|
OutputSchemaIntermediate = "seriatim-intermediate"
|
||||||
OutputSchemaMinimal = "minimal"
|
OutputSchemaFull = "seriatim-full"
|
||||||
)
|
)
|
||||||
|
|
||||||
// MergeOptions captures raw CLI option values before validation.
|
// MergeOptions captures raw CLI option values before validation.
|
||||||
@@ -70,7 +71,7 @@ func NewMergeConfig(opts MergeOptions) (Config, error) {
|
|||||||
cfg := Config{
|
cfg := Config{
|
||||||
InputReader: strings.TrimSpace(opts.InputReader),
|
InputReader: strings.TrimSpace(opts.InputReader),
|
||||||
OutputModules: nil,
|
OutputModules: nil,
|
||||||
OutputSchema: strings.TrimSpace(opts.OutputSchema),
|
OutputSchema: "",
|
||||||
PreprocessingModules: nil,
|
PreprocessingModules: nil,
|
||||||
PostprocessingModules: nil,
|
PostprocessingModules: nil,
|
||||||
OverlapWordRunGap: DefaultOverlapWordRunGap,
|
OverlapWordRunGap: DefaultOverlapWordRunGap,
|
||||||
@@ -83,14 +84,12 @@ func NewMergeConfig(opts MergeOptions) (Config, error) {
|
|||||||
if cfg.InputReader == "" {
|
if cfg.InputReader == "" {
|
||||||
return Config{}, errors.New("--input-reader is required")
|
return Config{}, errors.New("--input-reader is required")
|
||||||
}
|
}
|
||||||
if cfg.OutputSchema == "" {
|
var err error
|
||||||
cfg.OutputSchema = DefaultOutputSchema
|
cfg.OutputSchema, err = resolveOutputSchema(opts.OutputSchema)
|
||||||
}
|
if err != nil {
|
||||||
if err := validateOutputSchema(cfg.OutputSchema); err != nil {
|
|
||||||
return Config{}, err
|
return Config{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
var err error
|
|
||||||
cfg.OutputModules, err = parseModuleList(opts.OutputModules)
|
cfg.OutputModules, err = parseModuleList(opts.OutputModules)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Config{}, fmt.Errorf("--output-modules: %w", err)
|
return Config{}, fmt.Errorf("--output-modules: %w", err)
|
||||||
@@ -189,13 +188,27 @@ func parseModuleList(value string) ([]string, error) {
|
|||||||
|
|
||||||
func validateOutputSchema(value string) error {
|
func validateOutputSchema(value string) error {
|
||||||
switch value {
|
switch value {
|
||||||
case OutputSchemaDefault, OutputSchemaSeriatim, OutputSchemaMinimal:
|
case OutputSchemaMinimal, OutputSchemaIntermediate, OutputSchemaFull:
|
||||||
return nil
|
return nil
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("--output-schema must be one of %q, %q, or %q", OutputSchemaDefault, OutputSchemaMinimal, OutputSchemaSeriatim)
|
return fmt.Errorf("--output-schema must be one of %q, %q, or %q", OutputSchemaMinimal, OutputSchemaIntermediate, OutputSchemaFull)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func resolveOutputSchema(value string) (string, error) {
|
||||||
|
value = strings.TrimSpace(value)
|
||||||
|
if value == "" {
|
||||||
|
value = strings.TrimSpace(os.Getenv(OutputSchemaEnv))
|
||||||
|
}
|
||||||
|
if value == "" {
|
||||||
|
value = DefaultOutputSchema
|
||||||
|
}
|
||||||
|
if err := validateOutputSchema(value); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return value, nil
|
||||||
|
}
|
||||||
|
|
||||||
func normalizeInputFiles(paths []string) ([]string, error) {
|
func normalizeInputFiles(paths []string) ([]string, error) {
|
||||||
if len(paths) == 0 {
|
if len(paths) == 0 {
|
||||||
return nil, errors.New("at least one --input-file is required")
|
return nil, errors.New("at least one --input-file is required")
|
||||||
|
|||||||
@@ -46,7 +46,8 @@ func TestDuplicateInputFilesFailValidation(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestOutputSchemaDefaultsToDefault(t *testing.T) {
|
func TestOutputSchemaDefaultsToIntermediate(t *testing.T) {
|
||||||
|
t.Setenv(OutputSchemaEnv, "")
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
input := writeTempFile(t, dir, "input.json")
|
input := writeTempFile(t, dir, "input.json")
|
||||||
output := filepath.Join(dir, "merged.json")
|
output := filepath.Join(dir, "merged.json")
|
||||||
@@ -67,7 +68,8 @@ func TestOutputSchemaDefaultsToDefault(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestOutputSchemaAcceptsDefault(t *testing.T) {
|
func TestOutputSchemaAcceptsIntermediate(t *testing.T) {
|
||||||
|
t.Setenv(OutputSchemaEnv, "")
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
input := writeTempFile(t, dir, "input.json")
|
input := writeTempFile(t, dir, "input.json")
|
||||||
output := filepath.Join(dir, "merged.json")
|
output := filepath.Join(dir, "merged.json")
|
||||||
@@ -77,19 +79,20 @@ func TestOutputSchemaAcceptsDefault(t *testing.T) {
|
|||||||
OutputFile: output,
|
OutputFile: output,
|
||||||
InputReader: DefaultInputReader,
|
InputReader: DefaultInputReader,
|
||||||
OutputModules: DefaultOutputModules,
|
OutputModules: DefaultOutputModules,
|
||||||
OutputSchema: OutputSchemaDefault,
|
OutputSchema: OutputSchemaIntermediate,
|
||||||
PreprocessingModules: DefaultPreprocessingModules,
|
PreprocessingModules: DefaultPreprocessingModules,
|
||||||
PostprocessingModules: DefaultPostprocessingModules,
|
PostprocessingModules: DefaultPostprocessingModules,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("config failed: %v", err)
|
t.Fatalf("config failed: %v", err)
|
||||||
}
|
}
|
||||||
if cfg.OutputSchema != OutputSchemaDefault {
|
if cfg.OutputSchema != OutputSchemaIntermediate {
|
||||||
t.Fatalf("output schema = %q, want %q", cfg.OutputSchema, OutputSchemaDefault)
|
t.Fatalf("output schema = %q, want %q", cfg.OutputSchema, OutputSchemaIntermediate)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestOutputSchemaAcceptsMinimal(t *testing.T) {
|
func TestOutputSchemaAcceptsMinimal(t *testing.T) {
|
||||||
|
t.Setenv(OutputSchemaEnv, "")
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
input := writeTempFile(t, dir, "input.json")
|
input := writeTempFile(t, dir, "input.json")
|
||||||
output := filepath.Join(dir, "merged.json")
|
output := filepath.Join(dir, "merged.json")
|
||||||
@@ -111,7 +114,8 @@ func TestOutputSchemaAcceptsMinimal(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestOutputSchemaAcceptsSeriatim(t *testing.T) {
|
func TestOutputSchemaAcceptsFull(t *testing.T) {
|
||||||
|
t.Setenv(OutputSchemaEnv, "")
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
input := writeTempFile(t, dir, "input.json")
|
input := writeTempFile(t, dir, "input.json")
|
||||||
output := filepath.Join(dir, "merged.json")
|
output := filepath.Join(dir, "merged.json")
|
||||||
@@ -121,19 +125,87 @@ func TestOutputSchemaAcceptsSeriatim(t *testing.T) {
|
|||||||
OutputFile: output,
|
OutputFile: output,
|
||||||
InputReader: DefaultInputReader,
|
InputReader: DefaultInputReader,
|
||||||
OutputModules: DefaultOutputModules,
|
OutputModules: DefaultOutputModules,
|
||||||
OutputSchema: OutputSchemaSeriatim,
|
OutputSchema: OutputSchemaFull,
|
||||||
PreprocessingModules: DefaultPreprocessingModules,
|
PreprocessingModules: DefaultPreprocessingModules,
|
||||||
PostprocessingModules: DefaultPostprocessingModules,
|
PostprocessingModules: DefaultPostprocessingModules,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("config failed: %v", err)
|
t.Fatalf("config failed: %v", err)
|
||||||
}
|
}
|
||||||
if cfg.OutputSchema != OutputSchemaSeriatim {
|
if cfg.OutputSchema != OutputSchemaFull {
|
||||||
t.Fatalf("output schema = %q, want %q", cfg.OutputSchema, OutputSchemaSeriatim)
|
t.Fatalf("output schema = %q, want %q", cfg.OutputSchema, OutputSchemaFull)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOutputSchemaUsesEnvWhenFlagOmitted(t *testing.T) {
|
||||||
|
t.Setenv(OutputSchemaEnv, OutputSchemaFull)
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTempFile(t, dir, "input.json")
|
||||||
|
output := filepath.Join(dir, "merged.json")
|
||||||
|
|
||||||
|
cfg, err := NewMergeConfig(MergeOptions{
|
||||||
|
InputFiles: []string{input},
|
||||||
|
OutputFile: output,
|
||||||
|
InputReader: DefaultInputReader,
|
||||||
|
OutputModules: DefaultOutputModules,
|
||||||
|
PreprocessingModules: DefaultPreprocessingModules,
|
||||||
|
PostprocessingModules: DefaultPostprocessingModules,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config failed: %v", err)
|
||||||
|
}
|
||||||
|
if cfg.OutputSchema != OutputSchemaFull {
|
||||||
|
t.Fatalf("output schema = %q, want %q", cfg.OutputSchema, OutputSchemaFull)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOutputSchemaFlagOverridesEnv(t *testing.T) {
|
||||||
|
t.Setenv(OutputSchemaEnv, OutputSchemaFull)
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTempFile(t, dir, "input.json")
|
||||||
|
output := filepath.Join(dir, "merged.json")
|
||||||
|
|
||||||
|
cfg, err := NewMergeConfig(MergeOptions{
|
||||||
|
InputFiles: []string{input},
|
||||||
|
OutputFile: output,
|
||||||
|
InputReader: DefaultInputReader,
|
||||||
|
OutputModules: DefaultOutputModules,
|
||||||
|
OutputSchema: OutputSchemaMinimal,
|
||||||
|
PreprocessingModules: DefaultPreprocessingModules,
|
||||||
|
PostprocessingModules: DefaultPostprocessingModules,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config failed: %v", err)
|
||||||
|
}
|
||||||
|
if cfg.OutputSchema != OutputSchemaMinimal {
|
||||||
|
t.Fatalf("output schema = %q, want %q", cfg.OutputSchema, OutputSchemaMinimal)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOutputSchemaRejectsInvalidEnvValue(t *testing.T) {
|
||||||
|
t.Setenv(OutputSchemaEnv, "compact")
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTempFile(t, dir, "input.json")
|
||||||
|
output := filepath.Join(dir, "merged.json")
|
||||||
|
|
||||||
|
_, err := NewMergeConfig(MergeOptions{
|
||||||
|
InputFiles: []string{input},
|
||||||
|
OutputFile: output,
|
||||||
|
InputReader: DefaultInputReader,
|
||||||
|
OutputModules: DefaultOutputModules,
|
||||||
|
PreprocessingModules: DefaultPreprocessingModules,
|
||||||
|
PostprocessingModules: DefaultPostprocessingModules,
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected output schema error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "--output-schema must be one of") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestOutputSchemaRejectsUnknownValue(t *testing.T) {
|
func TestOutputSchemaRejectsUnknownValue(t *testing.T) {
|
||||||
|
t.Setenv(OutputSchemaEnv, "")
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
input := writeTempFile(t, dir, "input.json")
|
input := writeTempFile(t, dir, "input.json")
|
||||||
output := filepath.Join(dir, "merged.json")
|
output := filepath.Join(dir, "merged.json")
|
||||||
@@ -155,7 +227,36 @@ func TestOutputSchemaRejectsUnknownValue(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestOverlapWordRunGapDefaultsTo075(t *testing.T) {
|
func TestOutputSchemaRejectsLegacyValues(t *testing.T) {
|
||||||
|
tests := []string{"default", "minimal", "seriatim"}
|
||||||
|
|
||||||
|
for _, legacy := range tests {
|
||||||
|
t.Run(legacy, func(t *testing.T) {
|
||||||
|
t.Setenv(OutputSchemaEnv, "")
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTempFile(t, dir, "input.json")
|
||||||
|
output := filepath.Join(dir, "merged.json")
|
||||||
|
|
||||||
|
_, err := NewMergeConfig(MergeOptions{
|
||||||
|
InputFiles: []string{input},
|
||||||
|
OutputFile: output,
|
||||||
|
InputReader: DefaultInputReader,
|
||||||
|
OutputModules: DefaultOutputModules,
|
||||||
|
OutputSchema: legacy,
|
||||||
|
PreprocessingModules: DefaultPreprocessingModules,
|
||||||
|
PostprocessingModules: DefaultPostprocessingModules,
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected output schema error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "--output-schema must be one of") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOverlapWordRunGapDefaultsTo1(t *testing.T) {
|
||||||
t.Setenv(OverlapWordRunGapEnv, "")
|
t.Setenv(OverlapWordRunGapEnv, "")
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
input := writeTempFile(t, dir, "input.json")
|
input := writeTempFile(t, dir, "input.json")
|
||||||
|
|||||||
@@ -184,7 +184,7 @@ func nearestPriorMatch(segments []model.Segment, consumed []bool, index int) int
|
|||||||
if consumed[candidate] {
|
if consumed[candidate] {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if sharesDerivedFrom(segments[index], segments[candidate]) {
|
if segments[index].Speaker == segments[candidate].Speaker && sharesDerivedFrom(segments[index], segments[candidate]) {
|
||||||
return candidate
|
return candidate
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -196,7 +196,7 @@ func nearestSubsequentMatch(segments []model.Segment, consumed []bool, index int
|
|||||||
if consumed[candidate] {
|
if consumed[candidate] {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if sharesDerivedFrom(segments[index], segments[candidate]) {
|
if segments[index].Speaker == segments[candidate].Speaker && sharesDerivedFrom(segments[index], segments[candidate]) {
|
||||||
return candidate
|
return candidate
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -51,6 +51,30 @@ func TestApplyUsesAnyDerivedFromIntersection(t *testing.T) {
|
|||||||
assertSegment(t, got.Segments[0], "resolve-danglers:1", "target end.", 1, 4, []string{"source#1", "source#2", "source#3"})
|
assertSegment(t, got.Segments[0], "resolve-danglers:1", "target end.", 1, 4, []string{"source#1", "source#2", "source#3"})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestApplyDoesNotMergeDanglersAcrossSpeakersBackward(t *testing.T) {
|
||||||
|
in := transcript(
|
||||||
|
segment("a", "Alice", 1, 2, "target", []string{"source#1"}),
|
||||||
|
segment("b", "Bob", 3, 4, "end.", []string{"source#1"}),
|
||||||
|
)
|
||||||
|
|
||||||
|
got, summary := Apply(in)
|
||||||
|
if summary.DanglersMerged != 0 || !reflect.DeepEqual(got, in) {
|
||||||
|
t.Fatalf("unexpected merge:\ngot %#v\nwant %#v", got, in)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyDoesNotMergeDanglersAcrossSpeakersForward(t *testing.T) {
|
||||||
|
in := transcript(
|
||||||
|
segment("a", "Alice", 1, 2, "start", []string{"source#1"}),
|
||||||
|
segment("b", "Bob", 3, 4, "target", []string{"source#1"}),
|
||||||
|
)
|
||||||
|
|
||||||
|
got, summary := Apply(in)
|
||||||
|
if summary.DanglersMerged != 0 || !reflect.DeepEqual(got, in) {
|
||||||
|
t.Fatalf("unexpected merge:\ngot %#v\nwant %#v", got, in)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestApplyDoesNotMergeWithoutSharedProvenance(t *testing.T) {
|
func TestApplyDoesNotMergeWithoutSharedProvenance(t *testing.T) {
|
||||||
in := transcript(
|
in := transcript(
|
||||||
segment("a", "Alice", 1, 2, "target", []string{"source#1"}),
|
segment("a", "Alice", 1, 2, "target", []string{"source#1"}),
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
"$id": "https://gitea.maximumdirect.net/eric/seriatim/schema/output.schema.json",
|
"$id": "https://gitea.maximumdirect.net/eric/seriatim/schema/full-output.schema.json",
|
||||||
"title": "seriatim full output transcript",
|
"title": "seriatim full output transcript",
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"additionalProperties": false,
|
"additionalProperties": false,
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
"$id": "https://gitea.maximumdirect.net/eric/seriatim/schema/default-output.schema.json",
|
"$id": "https://gitea.maximumdirect.net/eric/seriatim/schema/intermediate-output.schema.json",
|
||||||
"title": "seriatim default output transcript",
|
"title": "seriatim intermediate output transcript",
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"additionalProperties": false,
|
"additionalProperties": false,
|
||||||
"required": ["metadata", "segments"],
|
"required": ["metadata", "segments"],
|
||||||
@@ -13,7 +13,7 @@
|
|||||||
"properties": {
|
"properties": {
|
||||||
"application": { "type": "string" },
|
"application": { "type": "string" },
|
||||||
"version": { "type": "string" },
|
"version": { "type": "string" },
|
||||||
"output_schema": { "type": "string", "const": "default" }
|
"output_schema": { "type": "string", "const": "seriatim-intermediate" }
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"segments": {
|
"segments": {
|
||||||
@@ -13,7 +13,7 @@
|
|||||||
"properties": {
|
"properties": {
|
||||||
"application": { "type": "string" },
|
"application": { "type": "string" },
|
||||||
"version": { "type": "string" },
|
"version": { "type": "string" },
|
||||||
"output_schema": { "type": "string", "const": "minimal" }
|
"output_schema": { "type": "string", "const": "seriatim-minimal" }
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"segments": {
|
"segments": {
|
||||||
|
|||||||
@@ -14,9 +14,9 @@ import (
|
|||||||
var schemaFS embed.FS
|
var schemaFS embed.FS
|
||||||
|
|
||||||
const (
|
const (
|
||||||
outputSchemaPath = "output.schema.json"
|
fullOutputSchemaPath = "full-output.schema.json"
|
||||||
defaultOutputSchemaPath = "default-output.schema.json"
|
intermediateOutputSchemaPath = "intermediate-output.schema.json"
|
||||||
minimalOutputSchemaPath = "minimal-output.schema.json"
|
minimalOutputSchemaPath = "minimal-output.schema.json"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -32,10 +32,10 @@ type Transcript struct {
|
|||||||
OverlapGroups []OverlapGroup `json:"overlap_groups"`
|
OverlapGroups []OverlapGroup `json:"overlap_groups"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// DefaultTranscript is seriatim's default public JSON output contract.
|
// IntermediateTranscript is seriatim's intermediate public JSON output contract.
|
||||||
type DefaultTranscript struct {
|
type IntermediateTranscript struct {
|
||||||
Metadata DefaultMetadata `json:"metadata"`
|
Metadata IntermediateMetadata `json:"metadata"`
|
||||||
Segments []DefaultSegment `json:"segments"`
|
Segments []IntermediateSegment `json:"segments"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// MinimalTranscript is seriatim's compact public JSON output contract.
|
// MinimalTranscript is seriatim's compact public JSON output contract.
|
||||||
@@ -55,8 +55,8 @@ type Metadata struct {
|
|||||||
OutputModules []string `json:"output_modules"`
|
OutputModules []string `json:"output_modules"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// DefaultMetadata records default artifact identity.
|
// IntermediateMetadata records intermediate artifact identity.
|
||||||
type DefaultMetadata struct {
|
type IntermediateMetadata struct {
|
||||||
Application string `json:"application"`
|
Application string `json:"application"`
|
||||||
Version string `json:"version"`
|
Version string `json:"version"`
|
||||||
OutputSchema string `json:"output_schema"`
|
OutputSchema string `json:"output_schema"`
|
||||||
@@ -84,9 +84,9 @@ type Segment struct {
|
|||||||
OverlapGroupID int `json:"overlap_group_id,omitempty"`
|
OverlapGroupID int `json:"overlap_group_id,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// DefaultSegment is the compact public transcript segment shape with
|
// IntermediateSegment is the compact public transcript segment shape with
|
||||||
// categories.
|
// categories.
|
||||||
type DefaultSegment struct {
|
type IntermediateSegment struct {
|
||||||
ID int `json:"id"`
|
ID int `json:"id"`
|
||||||
Start float64 `json:"start"`
|
Start float64 `json:"start"`
|
||||||
End float64 `json:"end"`
|
End float64 `json:"end"`
|
||||||
@@ -115,7 +115,7 @@ type OverlapGroup struct {
|
|||||||
Resolution string `json:"resolution"`
|
Resolution string `json:"resolution"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ValidateTranscript validates a typed transcript against the public JSON
|
// ValidateTranscript validates a full transcript against the public JSON
|
||||||
// schema and seriatim-specific semantic rules.
|
// schema and seriatim-specific semantic rules.
|
||||||
func ValidateTranscript(transcript Transcript) error {
|
func ValidateTranscript(transcript Transcript) error {
|
||||||
if err := validateSemantics(transcript); err != nil {
|
if err := validateSemantics(transcript); err != nil {
|
||||||
@@ -129,18 +129,18 @@ func ValidateTranscript(transcript Transcript) error {
|
|||||||
return ValidateJSON(data)
|
return ValidateJSON(data)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ValidateDefaultTranscript validates the default transcript against the
|
// ValidateIntermediateTranscript validates the intermediate transcript against
|
||||||
// default JSON schema and seriatim-specific semantic rules.
|
// the intermediate JSON schema and seriatim-specific semantic rules.
|
||||||
func ValidateDefaultTranscript(transcript DefaultTranscript) error {
|
func ValidateIntermediateTranscript(transcript IntermediateTranscript) error {
|
||||||
if err := validateDefaultSemantics(transcript); err != nil {
|
if err := validateIntermediateSemantics(transcript); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
data, err := json.Marshal(transcript)
|
data, err := json.Marshal(transcript)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("marshal default transcript for schema validation: %w", err)
|
return fmt.Errorf("marshal intermediate transcript for schema validation: %w", err)
|
||||||
}
|
}
|
||||||
return ValidateDefaultJSON(data)
|
return ValidateIntermediateJSON(data)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ValidateMinimalTranscript validates a minimal transcript against the minimal
|
// ValidateMinimalTranscript validates a minimal transcript against the minimal
|
||||||
@@ -159,13 +159,13 @@ func ValidateMinimalTranscript(transcript MinimalTranscript) error {
|
|||||||
|
|
||||||
// ValidateJSON validates serialized output JSON against the public schema.
|
// ValidateJSON validates serialized output JSON against the public schema.
|
||||||
func ValidateJSON(data []byte) error {
|
func ValidateJSON(data []byte) error {
|
||||||
return validateJSONWithSchema(data, outputSchemaPath)
|
return validateJSONWithSchema(data, fullOutputSchemaPath)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ValidateDefaultJSON validates serialized default output JSON against the
|
// ValidateIntermediateJSON validates serialized intermediate output JSON
|
||||||
// default public schema.
|
// against the intermediate public schema.
|
||||||
func ValidateDefaultJSON(data []byte) error {
|
func ValidateIntermediateJSON(data []byte) error {
|
||||||
return validateJSONWithSchema(data, defaultOutputSchemaPath)
|
return validateJSONWithSchema(data, intermediateOutputSchemaPath)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ValidateMinimalJSON validates serialized minimal output JSON against the
|
// ValidateMinimalJSON validates serialized minimal output JSON against the
|
||||||
@@ -245,7 +245,7 @@ func validateSemantics(transcript Transcript) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateDefaultSemantics(transcript DefaultTranscript) error {
|
func validateIntermediateSemantics(transcript IntermediateTranscript) error {
|
||||||
for index, segment := range transcript.Segments {
|
for index, segment := range transcript.Segments {
|
||||||
wantID := index + 1
|
wantID := index + 1
|
||||||
if segment.ID != wantID {
|
if segment.ID != wantID {
|
||||||
|
|||||||
@@ -21,11 +21,11 @@ func TestValidateMinimalTranscriptAcceptsValidTranscript(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestValidateDefaultTranscriptAcceptsValidTranscript(t *testing.T) {
|
func TestValidateIntermediateTranscriptAcceptsValidTranscript(t *testing.T) {
|
||||||
transcript := validDefaultTranscript()
|
transcript := validIntermediateTranscript()
|
||||||
|
|
||||||
if err := ValidateDefaultTranscript(transcript); err != nil {
|
if err := ValidateIntermediateTranscript(transcript); err != nil {
|
||||||
t.Fatalf("validate default transcript: %v", err)
|
t.Fatalf("validate intermediate transcript: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -34,7 +34,7 @@ func TestValidateMinimalJSONRejectsMissingRequiredField(t *testing.T) {
|
|||||||
"metadata": {
|
"metadata": {
|
||||||
"application": "seriatim",
|
"application": "seriatim",
|
||||||
"version": "dev",
|
"version": "dev",
|
||||||
"output_schema": "minimal"
|
"output_schema": "seriatim-minimal"
|
||||||
}
|
}
|
||||||
}`))
|
}`))
|
||||||
assertErrorContains(t, err, "segments")
|
assertErrorContains(t, err, "segments")
|
||||||
@@ -45,7 +45,7 @@ func TestValidateMinimalJSONRejectsWrongFieldType(t *testing.T) {
|
|||||||
"metadata": {
|
"metadata": {
|
||||||
"application": "seriatim",
|
"application": "seriatim",
|
||||||
"version": "dev",
|
"version": "dev",
|
||||||
"output_schema": "minimal"
|
"output_schema": "seriatim-minimal"
|
||||||
},
|
},
|
||||||
"segments": [
|
"segments": [
|
||||||
{
|
{
|
||||||
@@ -68,7 +68,7 @@ func TestValidateMinimalJSONRejectsUnexpectedFields(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "top-level overlap groups",
|
name: "top-level overlap groups",
|
||||||
json: `{
|
json: `{
|
||||||
"metadata": {"application": "seriatim", "version": "dev", "output_schema": "minimal"},
|
"metadata": {"application": "seriatim", "version": "dev", "output_schema": "seriatim-minimal"},
|
||||||
"segments": [],
|
"segments": [],
|
||||||
"overlap_groups": []
|
"overlap_groups": []
|
||||||
}`,
|
}`,
|
||||||
@@ -76,28 +76,28 @@ func TestValidateMinimalJSONRejectsUnexpectedFields(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "segment source",
|
name: "segment source",
|
||||||
json: `{
|
json: `{
|
||||||
"metadata": {"application": "seriatim", "version": "dev", "output_schema": "minimal"},
|
"metadata": {"application": "seriatim", "version": "dev", "output_schema": "seriatim-minimal"},
|
||||||
"segments": [{"id": 1, "source": "input.json", "start": 1, "end": 2, "speaker": "Alice", "text": "hello"}]
|
"segments": [{"id": 1, "source": "input.json", "start": 1, "end": 2, "speaker": "Alice", "text": "hello"}]
|
||||||
}`,
|
}`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "segment categories",
|
name: "segment categories",
|
||||||
json: `{
|
json: `{
|
||||||
"metadata": {"application": "seriatim", "version": "dev", "output_schema": "minimal"},
|
"metadata": {"application": "seriatim", "version": "dev", "output_schema": "seriatim-minimal"},
|
||||||
"segments": [{"id": 1, "start": 1, "end": 2, "speaker": "Alice", "text": "hello", "categories": ["backchannel"]}]
|
"segments": [{"id": 1, "start": 1, "end": 2, "speaker": "Alice", "text": "hello", "categories": ["backchannel"]}]
|
||||||
}`,
|
}`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "segment derived from",
|
name: "segment derived from",
|
||||||
json: `{
|
json: `{
|
||||||
"metadata": {"application": "seriatim", "version": "dev", "output_schema": "minimal"},
|
"metadata": {"application": "seriatim", "version": "dev", "output_schema": "seriatim-minimal"},
|
||||||
"segments": [{"id": 1, "start": 1, "end": 2, "speaker": "Alice", "text": "hello", "derived_from": ["input.json#0"]}]
|
"segments": [{"id": 1, "start": 1, "end": 2, "speaker": "Alice", "text": "hello", "derived_from": ["input.json#0"]}]
|
||||||
}`,
|
}`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "segment words",
|
name: "segment words",
|
||||||
json: `{
|
json: `{
|
||||||
"metadata": {"application": "seriatim", "version": "dev", "output_schema": "minimal"},
|
"metadata": {"application": "seriatim", "version": "dev", "output_schema": "seriatim-minimal"},
|
||||||
"segments": [{"id": 1, "start": 1, "end": 2, "speaker": "Alice", "text": "hello", "words": []}]
|
"segments": [{"id": 1, "start": 1, "end": 2, "speaker": "Alice", "text": "hello", "words": []}]
|
||||||
}`,
|
}`,
|
||||||
},
|
},
|
||||||
@@ -111,24 +111,24 @@ func TestValidateMinimalJSONRejectsUnexpectedFields(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestValidateDefaultJSONRejectsMissingRequiredField(t *testing.T) {
|
func TestValidateIntermediateJSONRejectsMissingRequiredField(t *testing.T) {
|
||||||
err := ValidateDefaultJSON([]byte(`{
|
err := ValidateIntermediateJSON([]byte(`{
|
||||||
"metadata": {
|
"metadata": {
|
||||||
"application": "seriatim",
|
"application": "seriatim",
|
||||||
"version": "dev",
|
"version": "dev",
|
||||||
"output_schema": "default"
|
"output_schema": "seriatim-intermediate"
|
||||||
}
|
}
|
||||||
}`))
|
}`))
|
||||||
assertErrorContains(t, err, "segments")
|
assertErrorContains(t, err, "segments")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestValidateDefaultJSONRejectsWrongFieldType(t *testing.T) {
|
func TestValidateIntermediateJSONRejectsWrongFieldType(t *testing.T) {
|
||||||
err := ValidateDefaultJSON([]byte(`{
|
err := ValidateIntermediateJSON([]byte(`{
|
||||||
"metadata": {
|
"metadata": {
|
||||||
"application": "seriatim",
|
"application": "seriatim",
|
||||||
"version": "dev",
|
"version": "dev",
|
||||||
"output_schema": "default"
|
"output_schema": "seriatim-intermediate"
|
||||||
},
|
},
|
||||||
"segments": [
|
"segments": [
|
||||||
{
|
{
|
||||||
"id": "1",
|
"id": "1",
|
||||||
@@ -142,7 +142,7 @@ func TestValidateDefaultJSONRejectsWrongFieldType(t *testing.T) {
|
|||||||
assertErrorContains(t, err, "id")
|
assertErrorContains(t, err, "id")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestValidateDefaultJSONRejectsUnexpectedFields(t *testing.T) {
|
func TestValidateIntermediateJSONRejectsUnexpectedFields(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
json string
|
json string
|
||||||
@@ -150,7 +150,7 @@ func TestValidateDefaultJSONRejectsUnexpectedFields(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "top-level overlap groups",
|
name: "top-level overlap groups",
|
||||||
json: `{
|
json: `{
|
||||||
"metadata": {"application": "seriatim", "version": "dev", "output_schema": "default"},
|
"metadata": {"application": "seriatim", "version": "dev", "output_schema": "seriatim-intermediate"},
|
||||||
"segments": [],
|
"segments": [],
|
||||||
"overlap_groups": []
|
"overlap_groups": []
|
||||||
}`,
|
}`,
|
||||||
@@ -158,21 +158,21 @@ func TestValidateDefaultJSONRejectsUnexpectedFields(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "segment source",
|
name: "segment source",
|
||||||
json: `{
|
json: `{
|
||||||
"metadata": {"application": "seriatim", "version": "dev", "output_schema": "default"},
|
"metadata": {"application": "seriatim", "version": "dev", "output_schema": "seriatim-intermediate"},
|
||||||
"segments": [{"id": 1, "source": "input.json", "start": 1, "end": 2, "speaker": "Alice", "text": "hello"}]
|
"segments": [{"id": 1, "source": "input.json", "start": 1, "end": 2, "speaker": "Alice", "text": "hello"}]
|
||||||
}`,
|
}`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "segment derived from",
|
name: "segment derived from",
|
||||||
json: `{
|
json: `{
|
||||||
"metadata": {"application": "seriatim", "version": "dev", "output_schema": "default"},
|
"metadata": {"application": "seriatim", "version": "dev", "output_schema": "seriatim-intermediate"},
|
||||||
"segments": [{"id": 1, "start": 1, "end": 2, "speaker": "Alice", "text": "hello", "derived_from": ["input.json#0"]}]
|
"segments": [{"id": 1, "start": 1, "end": 2, "speaker": "Alice", "text": "hello", "derived_from": ["input.json#0"]}]
|
||||||
}`,
|
}`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "segment words",
|
name: "segment words",
|
||||||
json: `{
|
json: `{
|
||||||
"metadata": {"application": "seriatim", "version": "dev", "output_schema": "default"},
|
"metadata": {"application": "seriatim", "version": "dev", "output_schema": "seriatim-intermediate"},
|
||||||
"segments": [{"id": 1, "start": 1, "end": 2, "speaker": "Alice", "text": "hello", "words": []}]
|
"segments": [{"id": 1, "start": 1, "end": 2, "speaker": "Alice", "text": "hello", "words": []}]
|
||||||
}`,
|
}`,
|
||||||
},
|
},
|
||||||
@@ -180,7 +180,7 @@ func TestValidateDefaultJSONRejectsUnexpectedFields(t *testing.T) {
|
|||||||
|
|
||||||
for _, test := range tests {
|
for _, test := range tests {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
err := ValidateDefaultJSON([]byte(test.json))
|
err := ValidateIntermediateJSON([]byte(test.json))
|
||||||
assertErrorContains(t, err, "additional properties")
|
assertErrorContains(t, err, "additional properties")
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -267,7 +267,7 @@ func TestValidateJSONRejectsUnexpectedInternalFields(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestValidateDefaultTranscriptRejectsMissingOrNonSequentialIDs(t *testing.T) {
|
func TestValidateIntermediateTranscriptRejectsMissingOrNonSequentialIDs(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
ids []int
|
ids []int
|
||||||
@@ -280,10 +280,10 @@ func TestValidateDefaultTranscriptRejectsMissingOrNonSequentialIDs(t *testing.T)
|
|||||||
|
|
||||||
for _, test := range tests {
|
for _, test := range tests {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
transcript := validDefaultTranscript()
|
transcript := validIntermediateTranscript()
|
||||||
transcript.Segments = transcript.Segments[:0]
|
transcript.Segments = transcript.Segments[:0]
|
||||||
for index, id := range test.ids {
|
for index, id := range test.ids {
|
||||||
transcript.Segments = append(transcript.Segments, DefaultSegment{
|
transcript.Segments = append(transcript.Segments, IntermediateSegment{
|
||||||
ID: id,
|
ID: id,
|
||||||
Start: float64(index),
|
Start: float64(index),
|
||||||
End: float64(index) + 1,
|
End: float64(index) + 1,
|
||||||
@@ -292,18 +292,18 @@ func TestValidateDefaultTranscriptRejectsMissingOrNonSequentialIDs(t *testing.T)
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
err := ValidateDefaultTranscript(transcript)
|
err := ValidateIntermediateTranscript(transcript)
|
||||||
assertErrorContains(t, err, test.want)
|
assertErrorContains(t, err, test.want)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestValidateDefaultTranscriptRejectsInvalidTiming(t *testing.T) {
|
func TestValidateIntermediateTranscriptRejectsInvalidTiming(t *testing.T) {
|
||||||
transcript := validDefaultTranscript()
|
transcript := validIntermediateTranscript()
|
||||||
transcript.Segments[0].Start = 2
|
transcript.Segments[0].Start = 2
|
||||||
transcript.Segments[0].End = 1
|
transcript.Segments[0].End = 1
|
||||||
|
|
||||||
err := ValidateDefaultTranscript(transcript)
|
err := ValidateIntermediateTranscript(transcript)
|
||||||
assertErrorContains(t, err, "segment 0 has end")
|
assertErrorContains(t, err, "segment 0 has end")
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -411,7 +411,7 @@ func validMinimalTranscript() MinimalTranscript {
|
|||||||
Metadata: MinimalMetadata{
|
Metadata: MinimalMetadata{
|
||||||
Application: "seriatim",
|
Application: "seriatim",
|
||||||
Version: "dev",
|
Version: "dev",
|
||||||
OutputSchema: "minimal",
|
OutputSchema: "seriatim-minimal",
|
||||||
},
|
},
|
||||||
Segments: []MinimalSegment{
|
Segments: []MinimalSegment{
|
||||||
{
|
{
|
||||||
@@ -425,14 +425,14 @@ func validMinimalTranscript() MinimalTranscript {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func validDefaultTranscript() DefaultTranscript {
|
func validIntermediateTranscript() IntermediateTranscript {
|
||||||
return DefaultTranscript{
|
return IntermediateTranscript{
|
||||||
Metadata: DefaultMetadata{
|
Metadata: IntermediateMetadata{
|
||||||
Application: "seriatim",
|
Application: "seriatim",
|
||||||
Version: "dev",
|
Version: "dev",
|
||||||
OutputSchema: "default",
|
OutputSchema: "seriatim-intermediate",
|
||||||
},
|
},
|
||||||
Segments: []DefaultSegment{
|
Segments: []IntermediateSegment{
|
||||||
{
|
{
|
||||||
ID: 1,
|
ID: 1,
|
||||||
Start: 1,
|
Start: 1,
|
||||||
|
|||||||
Reference in New Issue
Block a user