Compare commits
15 Commits
d23a95471c
...
v1.4.0
| Author | SHA1 | Date | |
|---|---|---|---|
| 9202ccddb9 | |||
| f40d4add91 | |||
| 16bb12face | |||
| f18e2428dc | |||
| 3b64e784a1 | |||
| 3744d229a2 | |||
| 9bbe1fb7f1 | |||
| b7a66f6cc4 | |||
| c8efdb53d3 | |||
| ab4b252b08 | |||
| e9028e08a4 | |||
| 332884f887 | |||
| e5173c78fe | |||
| 546be2ab92 | |||
| 7743b397a6 |
14
README.md
14
README.md
@@ -27,19 +27,19 @@ go run ./cmd/seriatim merge \
|
|||||||
- Configuration reference: [docs/config.md](docs/config.md)
|
- Configuration reference: [docs/config.md](docs/config.md)
|
||||||
- Operations guide: [docs/operations.md](docs/operations.md)
|
- Operations guide: [docs/operations.md](docs/operations.md)
|
||||||
- Troubleshooting: [docs/troubleshooting.md](docs/troubleshooting.md)
|
- Troubleshooting: [docs/troubleshooting.md](docs/troubleshooting.md)
|
||||||
- Integrations:
|
- Integration references:
|
||||||
- [docs/integrations/whisperx-json.md](docs/integrations/whisperx-json.md)
|
- [docs/integrations/whisperx-json.md](docs/integrations/whisperx-json.md)
|
||||||
- [docs/integrations/output-schemas.md](docs/integrations/output-schemas.md)
|
- [docs/integrations/output-schemas.md](docs/integrations/output-schemas.md)
|
||||||
- Development architecture policy: [docs/policy/architecture.md](docs/policy/architecture.md)
|
- Development policies:
|
||||||
- Contributor workflow: [docs/policy/development.md](docs/policy/development.md)
|
- [docs/policy/architecture.md](docs/policy/architecture.md)
|
||||||
- Documentation policy: [docs/policy/documentation.md](docs/policy/documentation.md)
|
- [docs/policy/development.md](docs/policy/development.md)
|
||||||
- Internal implementation docs:
|
- [docs/policy/documentation.md](docs/policy/documentation.md)
|
||||||
|
- Internal implementation references:
|
||||||
- [docs/internal/pipeline.md](docs/internal/pipeline.md)
|
- [docs/internal/pipeline.md](docs/internal/pipeline.md)
|
||||||
- [docs/internal/artifacts.md](docs/internal/artifacts.md)
|
- [docs/internal/artifacts.md](docs/internal/artifacts.md)
|
||||||
- [docs/internal/modules.md](docs/internal/modules.md)
|
- [docs/internal/modules.md](docs/internal/modules.md)
|
||||||
- Public JSON schemas:
|
- Public JSON schema files:
|
||||||
- [schema/minimal-output.schema.json](schema/minimal-output.schema.json)
|
- [schema/minimal-output.schema.json](schema/minimal-output.schema.json)
|
||||||
- [schema/intermediate-output.schema.json](schema/intermediate-output.schema.json)
|
- [schema/intermediate-output.schema.json](schema/intermediate-output.schema.json)
|
||||||
- [schema/full-output.schema.json](schema/full-output.schema.json)
|
- [schema/full-output.schema.json](schema/full-output.schema.json)
|
||||||
- Synthetic examples: [examples/README.md](examples/README.md)
|
- Synthetic examples: [examples/README.md](examples/README.md)
|
||||||
- Documentation roadmap: [docs/roadmap/documentation.md](docs/roadmap/documentation.md)
|
|
||||||
|
|||||||
@@ -179,4 +179,3 @@ go run ./cmd/seriatim normalize \
|
|||||||
- [../schema/minimal-output.schema.json](../schema/minimal-output.schema.json)
|
- [../schema/minimal-output.schema.json](../schema/minimal-output.schema.json)
|
||||||
- [../schema/intermediate-output.schema.json](../schema/intermediate-output.schema.json)
|
- [../schema/intermediate-output.schema.json](../schema/intermediate-output.schema.json)
|
||||||
- [../schema/full-output.schema.json](../schema/full-output.schema.json)
|
- [../schema/full-output.schema.json](../schema/full-output.schema.json)
|
||||||
- Documentation roadmap: [roadmap/documentation.md](roadmap/documentation.md)
|
|
||||||
|
|||||||
@@ -165,4 +165,3 @@ All commands:
|
|||||||
- [../schema/minimal-output.schema.json](../schema/minimal-output.schema.json)
|
- [../schema/minimal-output.schema.json](../schema/minimal-output.schema.json)
|
||||||
- [../schema/intermediate-output.schema.json](../schema/intermediate-output.schema.json)
|
- [../schema/intermediate-output.schema.json](../schema/intermediate-output.schema.json)
|
||||||
- [../schema/full-output.schema.json](../schema/full-output.schema.json)
|
- [../schema/full-output.schema.json](../schema/full-output.schema.json)
|
||||||
- Documentation roadmap: [roadmap/documentation.md](roadmap/documentation.md)
|
|
||||||
|
|||||||
@@ -51,29 +51,43 @@ Unknown/empty selection falls back to intermediate conversion.
|
|||||||
|
|
||||||
## Trim internals
|
## Trim internals
|
||||||
|
|
||||||
`internal/trim` is artifact-level projection, not merge reprocessing.
|
`internal/trim` handles artifact-level projection and does not execute merge
|
||||||
|
pipeline modules.
|
||||||
|
|
||||||
Core flow:
|
Run layer (`run.go`):
|
||||||
|
|
||||||
1. Parse selector (`internal/trim/selector.go`).
|
1. Parse selector from validated config.
|
||||||
2. Parse input artifact and detect schema (`ParseArtifactJSON`).
|
2. Read and parse input artifact JSON.
|
||||||
3. Apply keep/remove projection with sequential ID renumbering.
|
3. Apply trim projection through schema-aware artifact handling.
|
||||||
4. Recompute overlap groups only for full-schema artifacts.
|
4. Resolve output schema (preserve input schema unless overridden).
|
||||||
5. Optionally convert output schema when supported.
|
5. Validate output artifact.
|
||||||
6. Validate output artifact before write.
|
6. Write output JSON.
|
||||||
|
7. Optionally write report JSON with `trim-audit`.
|
||||||
|
|
||||||
Schema-conversion limits:
|
Apply layer (`apply.go`):
|
||||||
|
|
||||||
- full -> intermediate/minimal supported.
|
- one shared projection policy for selector mode, input ID validation, selected
|
||||||
- intermediate -> minimal supported.
|
ID existence checks, keep/remove filtering, removed IDs, and old-to-new ID
|
||||||
- minimal -> intermediate supported.
|
mappings
|
||||||
- intermediate/minimal -> full is rejected.
|
- schema-specific segment reconstruction for full/intermediate/minimal outputs
|
||||||
|
- overlap-group recomputation only for full-schema outputs
|
||||||
|
|
||||||
|
Artifact layer (`artifact.go`):
|
||||||
|
|
||||||
|
- schema detection for full/intermediate/minimal artifacts
|
||||||
|
- schema-preserving trim application
|
||||||
|
- supported schema conversions:
|
||||||
|
- full -> intermediate/minimal
|
||||||
|
- intermediate -> minimal
|
||||||
|
- minimal -> intermediate
|
||||||
|
- rejected conversion:
|
||||||
|
- intermediate/minimal -> full
|
||||||
|
|
||||||
Trim invariants:
|
Trim invariants:
|
||||||
|
|
||||||
- selected IDs must exist in input.
|
- selected IDs must exist in input.
|
||||||
- input IDs must be positive, unique, sequential.
|
- input IDs must be positive, unique, sequential.
|
||||||
- retained order follows input transcript order.
|
- retained segment order follows input transcript order.
|
||||||
- output IDs are reassigned to `1..N`.
|
- output IDs are reassigned to `1..N`.
|
||||||
|
|
||||||
## Normalize internals
|
## Normalize internals
|
||||||
|
|||||||
@@ -55,7 +55,8 @@ Output writer:
|
|||||||
- `autocorrect`: applies YAML replacement rules when configured.
|
- `autocorrect`: applies YAML replacement rules when configured.
|
||||||
- `assign-ids`: assigns final sequential IDs.
|
- `assign-ids`: assigns final sequential IDs.
|
||||||
- `validate-output`: validates selected public artifact shape.
|
- `validate-output`: validates selected public artifact shape.
|
||||||
- `json`: writes artifact JSON to `cfg.OutputFile`.
|
- `json`: writes artifact JSON to `cfg.OutputFile` through shared deterministic
|
||||||
|
JSON file writing.
|
||||||
|
|
||||||
Filesystem side effects are limited to:
|
Filesystem side effects are limited to:
|
||||||
|
|
||||||
|
|||||||
@@ -197,6 +197,7 @@ free of secrets or private transcript data.
|
|||||||
|
|
||||||
- Keep core/domain logic separate from CLI, config, filesystem, reporting, and
|
- Keep core/domain logic separate from CLI, config, filesystem, reporting, and
|
||||||
other adapter concerns.
|
other adapter concerns.
|
||||||
|
- Centralize default configuration values as constants defined in internal/config/config.go.
|
||||||
- Keep modules narrowly scoped, explicitly configured, and composable by
|
- Keep modules narrowly scoped, explicitly configured, and composable by
|
||||||
registry name.
|
registry name.
|
||||||
- Preserve deterministic ordering, final segment ID assignment, and schema
|
- Preserve deterministic ordering, final segment ID assignment, and schema
|
||||||
|
|||||||
@@ -52,6 +52,7 @@ Current toolchain note:
|
|||||||
- Keep validation in `internal/config` and package-specific validators.
|
- Keep validation in `internal/config` and package-specific validators.
|
||||||
- Return errors from deep logic; do not print inside internal packages.
|
- Return errors from deep logic; do not print inside internal packages.
|
||||||
- Preserve clear package boundaries between adapters and domain transforms.
|
- Preserve clear package boundaries between adapters and domain transforms.
|
||||||
|
- Define configuration defaults as constants in internal/config/config.go.
|
||||||
|
|
||||||
## Dependency policy
|
## Dependency policy
|
||||||
|
|
||||||
|
|||||||
@@ -1,586 +0,0 @@
|
|||||||
# Documentation Roadmap
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
This roadmap defines the work required to bring seriatim's documentation into
|
|
||||||
compliance with `docs/policy/documentation.md` and the current implementation.
|
|
||||||
It is grounded in the repository as it exists now: the Go CLI, config loading,
|
|
||||||
pipeline modules, artifact commands, schemas, reports, samples, and tests.
|
|
||||||
|
|
||||||
Outside `docs/roadmap/`, documentation must describe only implemented
|
|
||||||
behavior. Planned, future, deprecated, experimental, or unimplemented work must
|
|
||||||
remain in roadmap documents until the code exists.
|
|
||||||
|
|
||||||
## Repository Documentation Inventory
|
|
||||||
|
|
||||||
- `README.md` - keep and rewrite. It currently mixes project orientation,
|
|
||||||
quickstart, full CLI reference, config/env reference, file formats, module
|
|
||||||
internals, limitations, and release build notes. Policy says README should be
|
|
||||||
concise and link to canonical docs.
|
|
||||||
- `docs/policy/documentation.md` - keep and lightly update only if the policy
|
|
||||||
itself changes. It is the controlling documentation layout and maintenance
|
|
||||||
policy.
|
|
||||||
- `docs/policy/architecture.md` - keep and lightly update as implementation
|
|
||||||
changes. It is the canonical development architecture policy.
|
|
||||||
- Root `architecture.md` - delete after salvage, or move only truly roadmap
|
|
||||||
material into `docs/roadmap/`. It is in the wrong canonical home and contains
|
|
||||||
future-oriented and aspirational claims.
|
|
||||||
- `docs/roadmap/documentation.md` - create new. This file is the planning
|
|
||||||
artifact for the documentation migration.
|
|
||||||
- `samples/` - split or move after audit. It contains sample raw transcripts,
|
|
||||||
merged artifacts, reports, `speakers.yml`, and `autocorrect.yml`, but
|
|
||||||
copyable examples belong under `examples/`. The raw sample data is large and
|
|
||||||
should be reviewed for privacy and maintainability before linking from docs.
|
|
||||||
- `schema/*.schema.json` - keep. These are public output contracts and should
|
|
||||||
be linked from documentation instead of duplicated in full.
|
|
||||||
- Missing canonical docs - create `docs/cli.md`, `docs/config.md`,
|
|
||||||
`docs/operations.md`, `docs/policy/development.md`, `docs/internal/`, and
|
|
||||||
likely `docs/troubleshooting.md`, `docs/integrations/`, and `examples/`.
|
|
||||||
|
|
||||||
## Policy Compliance Assessment
|
|
||||||
|
|
||||||
Required documents missing for seriatim's current shape as a modular, staged,
|
|
||||||
CLI/config-driven project:
|
|
||||||
|
|
||||||
- `docs/cli.md`
|
|
||||||
- `docs/config.md`
|
|
||||||
- `docs/operations.md`
|
|
||||||
- `docs/internal/`
|
|
||||||
- `docs/policy/development.md`
|
|
||||||
|
|
||||||
Recommended documents and directories missing:
|
|
||||||
|
|
||||||
- `docs/troubleshooting.md`
|
|
||||||
- maintained copyable examples under `examples/`
|
|
||||||
- concise integration notes under `docs/integrations/`
|
|
||||||
|
|
||||||
Existing compliance issues:
|
|
||||||
|
|
||||||
- `README.md` is too broad for its canonical scope. It should keep project
|
|
||||||
purpose, quickstart, and links, then delegate CLI, config, operations,
|
|
||||||
internals, and schema details.
|
|
||||||
- Root `architecture.md` is stale and in the wrong home. It includes future
|
|
||||||
input methods and formats, future output formats, dynamic plugin speculation,
|
|
||||||
an LLM non-goal, interface sketches that diverge from code, and other
|
|
||||||
development-policy content now covered by `docs/policy/architecture.md`.
|
|
||||||
- Non-roadmap docs should not carry forward claims about future defaults,
|
|
||||||
future formats, unimplemented plugin systems, or unimplemented alternate
|
|
||||||
input/output methods.
|
|
||||||
- Historical or deprecated wording, such as the old speaker map format, should
|
|
||||||
move out of the README unless it is still needed in troubleshooting or a
|
|
||||||
narrow migration note.
|
|
||||||
- There is no `examples/` directory. `samples/` exists but is not the canonical
|
|
||||||
examples home and should not be treated as copyable public examples without a
|
|
||||||
privacy and size audit.
|
|
||||||
- Links need verification after migration: README should link to all new
|
|
||||||
canonical docs, docs should link to schema files and maintained examples, and
|
|
||||||
no doc should link to the deleted root `architecture.md`.
|
|
||||||
|
|
||||||
## Target Documentation Set
|
|
||||||
|
|
||||||
### `README.md`
|
|
||||||
|
|
||||||
- Audience: users, administrators, and operators.
|
|
||||||
- Purpose: project orientation and shortest useful quickstart.
|
|
||||||
- Canonical scope: concise project purpose, elevator pitch, one minimal command,
|
|
||||||
and links to targeted docs.
|
|
||||||
- Recommended outline: project description; shortest merge command; command
|
|
||||||
summary; links to CLI, config, operations, architecture, development, schemas,
|
|
||||||
examples, and troubleshooting.
|
|
||||||
- Source of truth: current `README.md`, `internal/cli`, `internal/config`,
|
|
||||||
`cmd/seriatim/main.go`, and CLI tests.
|
|
||||||
- Acceptance criteria: no full flag tables, no full config reference, no module
|
|
||||||
manual, no future-feature claims, and all links resolve.
|
|
||||||
|
|
||||||
### `docs/cli.md`
|
|
||||||
|
|
||||||
- Audience: users, administrators, and operators.
|
|
||||||
- Purpose: canonical CLI reference and workflows.
|
|
||||||
- Canonical scope: shortest useful command, command overview, complete flag
|
|
||||||
reference, common workflows, diagnostics and report flags.
|
|
||||||
- Recommended outline: shortest useful command; global flags; `merge`; `trim`;
|
|
||||||
`normalize`; common workflows; exit/error behavior; links to config,
|
|
||||||
operations, examples, and schemas.
|
|
||||||
- Source of truth: `internal/cli/root.go`, `internal/cli/merge.go`,
|
|
||||||
`internal/cli/trim.go`, `internal/cli/normalize.go`, `internal/config`, and
|
|
||||||
`internal/cli/*_test.go`.
|
|
||||||
- Acceptance criteria: every documented flag, default, and required/mutually
|
|
||||||
exclusive rule matches code; package internals are linked rather than
|
|
||||||
explained in depth.
|
|
||||||
|
|
||||||
### `docs/config.md`
|
|
||||||
|
|
||||||
- Audience: administrators, operators, and advanced users.
|
|
||||||
- Purpose: canonical runtime configuration reference.
|
|
||||||
- Canonical scope: environment variables, default module lists, output schema
|
|
||||||
selection, `speakers.yml`, `autocorrect.yml`, path validation, and precedence.
|
|
||||||
- Recommended outline: config surfaces; output schema precedence; merge module
|
|
||||||
defaults; environment variables; speaker map YAML; autocorrect YAML; path and
|
|
||||||
validation rules; links to examples.
|
|
||||||
- Source of truth: `internal/config/config.go`, `internal/speaker/map.go`,
|
|
||||||
`internal/autocorrect/autocorrect.go`, `internal/config/config_test.go`,
|
|
||||||
`internal/speaker/map_test.go`, and `internal/autocorrect/autocorrect_test.go`.
|
|
||||||
- Acceptance criteria: all config fields and `SERIATIM_*` env vars match code;
|
|
||||||
unsupported config files or unimplemented formats are not described.
|
|
||||||
|
|
||||||
### `docs/operations.md`
|
|
||||||
|
|
||||||
- Audience: administrators and operators.
|
|
||||||
- Purpose: operational behavior for running commands safely.
|
|
||||||
- Canonical scope: file workflow, filesystem layout expectations, output and
|
|
||||||
report files, retry behavior, cleanup, validation failures, and operational
|
|
||||||
caveats.
|
|
||||||
- Recommended outline: normal workflow; input/output/report files; no durable
|
|
||||||
state; failure and retry behavior; reports and diagnostics; cleanup; privacy
|
|
||||||
considerations for transcript artifacts.
|
|
||||||
- Source of truth: `cmd/seriatim/main.go`, `internal/cli`, `internal/config`,
|
|
||||||
`internal/report`, `internal/builtin/output.go`, `internal/normalize`, and
|
|
||||||
trim/merge/normalize CLI tests.
|
|
||||||
- Acceptance criteria: clearly states there is no daemon, database, resume
|
|
||||||
state, remote storage, or background job state; does not invent recovery
|
|
||||||
workflows.
|
|
||||||
|
|
||||||
### `docs/policy/development.md`
|
|
||||||
|
|
||||||
- Audience: developers and coding agents.
|
|
||||||
- Purpose: contributor workflow and change guidance.
|
|
||||||
- Canonical scope: repository layout, build/test commands, coding conventions,
|
|
||||||
dependency policy, adding flags/config fields/modules/docs/examples.
|
|
||||||
- Recommended outline: repo layout; local checks; coding conventions; adding
|
|
||||||
CLI flags; adding config/env vars; adding modules/stages; schema changes;
|
|
||||||
examples and documentation updates.
|
|
||||||
- Source of truth: `docs/policy/documentation.md`,
|
|
||||||
`docs/policy/architecture.md`, `go.mod`, package layout, and test layout.
|
|
||||||
- Acceptance criteria: includes `go test ./...`; states there is no current
|
|
||||||
Makefile, taskfile, linter config, or automated doc checker; aligns with the
|
|
||||||
architecture policy.
|
|
||||||
|
|
||||||
### `docs/internal/pipeline.md`
|
|
||||||
|
|
||||||
- Audience: developers and coding agents.
|
|
||||||
- Purpose: implemented merge pipeline internals.
|
|
||||||
- Canonical scope: registry, stage interfaces, preprocessing state transitions,
|
|
||||||
module order, report event accumulation, final output/report writing.
|
|
||||||
- Recommended outline: purpose; inputs and outputs; stage contracts; registry
|
|
||||||
resolution; execution order; config fields used; adapters; failure behavior;
|
|
||||||
tests; invariants.
|
|
||||||
- Source of truth: `internal/pipeline`, `internal/builtin`, `internal/model`,
|
|
||||||
`internal/report`, `internal/pipeline/runner_test.go`,
|
|
||||||
`internal/builtin/*_test.go`, and `internal/cli/merge_test.go`.
|
|
||||||
- Acceptance criteria: describes only implemented sequential execution; does
|
|
||||||
not document concurrency, plugins, or future formats.
|
|
||||||
|
|
||||||
### `docs/internal/artifacts.md`
|
|
||||||
|
|
||||||
- Audience: developers and coding agents.
|
|
||||||
- Purpose: public artifact conversion and validation internals.
|
|
||||||
- Canonical scope: schema structs, embedded JSON Schemas, conversion from merged
|
|
||||||
model, trim/normalize artifact handling, and output validation.
|
|
||||||
- Recommended outline: artifact contracts; schema selection; conversion;
|
|
||||||
validation; trim projection; normalize canonicalization; tests; invariants.
|
|
||||||
- Source of truth: `schema`, `internal/artifact`, `internal/trim`,
|
|
||||||
`internal/normalize`, and related tests.
|
|
||||||
- Acceptance criteria: links to `schema/*.schema.json`; does not duplicate full
|
|
||||||
schemas or describe unavailable output formats.
|
|
||||||
|
|
||||||
### `docs/internal/modules.md`
|
|
||||||
|
|
||||||
- Audience: developers and coding agents.
|
|
||||||
- Purpose: implemented built-in module behavior and boundaries.
|
|
||||||
- Canonical scope: `json-files`, preprocessing modules, chronological merge,
|
|
||||||
postprocessing modules, and JSON output writer.
|
|
||||||
- Recommended outline: module list; inputs/outputs; config fields used; allowed
|
|
||||||
side effects; ordering constraints; failure behavior; tests; invariants.
|
|
||||||
- Source of truth: `internal/builtin`, `internal/overlap`, `internal/coalesce`,
|
|
||||||
`internal/danglers`, `internal/backchannel`, `internal/filler`,
|
|
||||||
`internal/autocorrect`, and package tests.
|
|
||||||
- Acceptance criteria: avoids full CLI/config duplication; identifies
|
|
||||||
order-sensitive transforms that must run before `assign-ids`.
|
|
||||||
|
|
||||||
### `docs/troubleshooting.md`
|
|
||||||
|
|
||||||
- Audience: users, administrators, and operators.
|
|
||||||
- Purpose: common failure symptoms and safe fixes.
|
|
||||||
- Canonical scope: implemented validation and runtime failures observed in
|
|
||||||
error paths and tests.
|
|
||||||
- Recommended outline: invalid JSON/input shape; missing required flags; invalid
|
|
||||||
output parent directory; invalid speaker/autocorrect YAML; unknown module;
|
|
||||||
invalid output schema; invalid trim selector; schema validation failure;
|
|
||||||
report write failure.
|
|
||||||
- Source of truth: `internal/config`, `internal/cli/*_test.go`,
|
|
||||||
`internal/trim/*_test.go`, `internal/normalize/*_test.go`,
|
|
||||||
`internal/speaker/*_test.go`, and `internal/autocorrect/*_test.go`.
|
|
||||||
- Acceptance criteria: each entry has symptom, likely cause, inspection step,
|
|
||||||
safe fix, and link; no speculative failure modes.
|
|
||||||
|
|
||||||
### `docs/integrations/whisperx-json.md`
|
|
||||||
|
|
||||||
- Audience: developers and coding agents.
|
|
||||||
- Purpose: external input JSON contract used by `merge`.
|
|
||||||
- Canonical scope: the supported WhisperX-like subset only.
|
|
||||||
- Recommended outline: top-level shape; required segment fields; optional word
|
|
||||||
timing fields; validation/failure behavior; how word timing affects overlap
|
|
||||||
resolution; links to CLI and examples.
|
|
||||||
- Source of truth: `internal/builtin/input.go`, merge CLI tests, and README
|
|
||||||
input-format material.
|
|
||||||
- Acceptance criteria: does not attempt to document full WhisperX behavior or
|
|
||||||
unsupported input formats.
|
|
||||||
|
|
||||||
### `docs/integrations/output-schemas.md`
|
|
||||||
|
|
||||||
- Audience: developers, coding agents, and artifact consumers.
|
|
||||||
- Purpose: orientation to public JSON output contracts.
|
|
||||||
- Canonical scope: minimal/intermediate/full schema roles and links to schema
|
|
||||||
files.
|
|
||||||
- Recommended outline: schema selection; minimal; intermediate; full; semantic
|
|
||||||
invariants; validation APIs; links to `schema/*.schema.json`.
|
|
||||||
- Source of truth: `schema/output.go`, `schema/*.schema.json`,
|
|
||||||
`schema/output_test.go`, and `internal/artifact`.
|
|
||||||
- Acceptance criteria: links to machine-readable schemas instead of copying
|
|
||||||
them in full.
|
|
||||||
|
|
||||||
### `examples/`
|
|
||||||
|
|
||||||
- Audience: users, administrators, operators, developers, and coding agents.
|
|
||||||
- Purpose: maintained copyable examples.
|
|
||||||
- Canonical scope: small synthetic inputs and config files for implemented
|
|
||||||
commands only.
|
|
||||||
- Source of truth: examples created during the documentation migration and
|
|
||||||
validated through actual command invocations.
|
|
||||||
- Acceptance criteria: examples are valid, free of secrets/private transcript
|
|
||||||
data, and linked from README, CLI, config, and operations docs.
|
|
||||||
|
|
||||||
## File-by-File Rewrite Guidance
|
|
||||||
|
|
||||||
### README
|
|
||||||
|
|
||||||
Cover what seriatim is, the shortest useful `merge` command, a brief command
|
|
||||||
summary, and links to canonical docs. Avoid full flag tables, config/env
|
|
||||||
reference, module internals, schema examples, troubleshooting details, future
|
|
||||||
formats, or release-history narrative. Inspect `internal/cli`, `internal/config`,
|
|
||||||
and CLI tests before updating commands.
|
|
||||||
|
|
||||||
### CLI Reference
|
|
||||||
|
|
||||||
Document actual `merge`, `trim`, and `normalize` flags from `internal/cli`.
|
|
||||||
Include required flags, defaults, mutually exclusive selector rules, schema
|
|
||||||
selection, report flags, and common workflows. Link to `docs/config.md` for
|
|
||||||
environment variables and YAML formats. Avoid internal package explanations.
|
|
||||||
Inspect `internal/cli/*_test.go` for edge cases and examples.
|
|
||||||
|
|
||||||
### Config Reference
|
|
||||||
|
|
||||||
Document all implemented config surfaces: flags that become config values,
|
|
||||||
`SERIATIM_OUTPUT_SCHEMA`, `SERIATIM_OVERLAP_WORD_RUN_GAP`,
|
|
||||||
`SERIATIM_OVERLAP_WORD_RUN_REORDER_WINDOW`,
|
|
||||||
`SERIATIM_BACKCHANNEL_MAX_DURATION`, `SERIATIM_FILLER_MAX_DURATION`, module
|
|
||||||
lists, output schemas, `speakers.yml`, and `autocorrect.yml`. Avoid command
|
|
||||||
tutorials and unimplemented config files. Inspect `internal/config`,
|
|
||||||
`internal/speaker`, `internal/autocorrect`, and tests.
|
|
||||||
|
|
||||||
### Operations
|
|
||||||
|
|
||||||
Document filesystem-only command execution, output/report artifacts, validation
|
|
||||||
failures, retry behavior, and cleanup. Explicitly say there is no daemon,
|
|
||||||
database, remote storage, resume state, or background job state. Avoid
|
|
||||||
unimplemented recovery procedures.
|
|
||||||
|
|
||||||
### Development Policy
|
|
||||||
|
|
||||||
Document repository layout, `go test ./...`, package conventions,
|
|
||||||
standard-library-first dependency guidance, how to add flags/config/modules,
|
|
||||||
and documentation update expectations. State that no Makefile, taskfile,
|
|
||||||
linter config, or automated documentation checker currently exists.
|
|
||||||
|
|
||||||
### Internal Docs
|
|
||||||
|
|
||||||
Keep internal docs behavior-level and concise. Describe implemented inputs,
|
|
||||||
outputs, boundaries, config fields used, adapters, failure behavior, tests, and
|
|
||||||
invariants. Avoid future plugins, future input/output formats, concurrency, or
|
|
||||||
duplicating CLI/config reference material.
|
|
||||||
|
|
||||||
### Root `architecture.md`
|
|
||||||
|
|
||||||
Do not carry forward future input methods, future formats, future output
|
|
||||||
formats, LLM text, dynamic plugin speculation, or interface sketches that
|
|
||||||
diverge from code. Salvage only current-behavior details that are not already
|
|
||||||
covered in `docs/policy/architecture.md` and move any legitimate future ideas
|
|
||||||
under `docs/roadmap/`.
|
|
||||||
|
|
||||||
## Examples Plan
|
|
||||||
|
|
||||||
Create small synthetic examples under `examples/` rather than relying on the
|
|
||||||
current large `samples/raw` data.
|
|
||||||
|
|
||||||
- `examples/minimal-merge/`
|
|
||||||
- Purpose: shortest complete merge workflow with two small raw JSON files and
|
|
||||||
optional `speakers.yml`.
|
|
||||||
- Expected validity check: run `go run ./cmd/seriatim merge` with the example
|
|
||||||
files and validate JSON output is produced.
|
|
||||||
- Docs to link: README, `docs/cli.md`, `docs/config.md`,
|
|
||||||
`docs/operations.md`.
|
|
||||||
- `examples/normalize/`
|
|
||||||
- Purpose: normalize object-with-`segments` and bare segment array inputs.
|
|
||||||
- Expected validity check: run `go run ./cmd/seriatim normalize` for both
|
|
||||||
shapes.
|
|
||||||
- Docs to link: `docs/cli.md`, `docs/operations.md`, and any Audita/bare
|
|
||||||
array integration note if created.
|
|
||||||
- `examples/trim/`
|
|
||||||
- Purpose: trim a small existing seriatim artifact by `--keep` and/or
|
|
||||||
`--remove`.
|
|
||||||
- Expected validity check: run `go run ./cmd/seriatim trim` and validate
|
|
||||||
sequential retained IDs.
|
|
||||||
- Docs to link: `docs/cli.md`, `docs/operations.md`.
|
|
||||||
- `examples/speakers.yml` and `examples/autocorrect.yml`
|
|
||||||
- Purpose: copyable YAML rule examples if linked from `docs/config.md`.
|
|
||||||
- Expected validity check: load through merge command or package tests.
|
|
||||||
- Docs to link: `docs/config.md`, `docs/cli.md`.
|
|
||||||
|
|
||||||
Do not invent examples for unimplemented input methods, output formats,
|
|
||||||
services, or plugin systems. Do not reuse `samples/raw` as public examples
|
|
||||||
without privacy and size review.
|
|
||||||
|
|
||||||
## Internal Documentation Plan
|
|
||||||
|
|
||||||
### Pipeline
|
|
||||||
|
|
||||||
- Path: `docs/internal/pipeline.md`
|
|
||||||
- Purpose: document implemented merge pipeline orchestration.
|
|
||||||
- Inputs and outputs: `config.Config`, raw transcripts, canonical transcripts,
|
|
||||||
merged transcript, selected public artifact, optional report.
|
|
||||||
- Boundaries: registry and runner orchestration; no CLI flag parsing; no schema
|
|
||||||
details beyond output selection.
|
|
||||||
- Config fields used: input reader, module lists, output modules, output schema,
|
|
||||||
input/output/report files, timing thresholds passed through modules.
|
|
||||||
- Adapters used: input reader, output writer, report writer.
|
|
||||||
- Failure behavior: unknown modules, invalid preprocessing state, stage errors,
|
|
||||||
output/report write failures.
|
|
||||||
- Tests to inspect: `internal/pipeline/runner_test.go`,
|
|
||||||
`internal/builtin/*_test.go`, `internal/cli/merge_test.go`.
|
|
||||||
- Architectural invariants: deterministic sequential stage order, explicit
|
|
||||||
raw-to-canonical preprocessing state, output validation before acceptance.
|
|
||||||
|
|
||||||
### Artifacts and Schemas
|
|
||||||
|
|
||||||
- Path: `docs/internal/artifacts.md`
|
|
||||||
- Purpose: document public artifact conversion and validation internals.
|
|
||||||
- Inputs and outputs: merged model, schema structs, serialized JSON artifacts,
|
|
||||||
parsed trim/normalize artifacts.
|
|
||||||
- Boundaries: conversion and validation only; CLI docs own user-facing flags.
|
|
||||||
- Config fields used: output schema, output modules, input files for metadata.
|
|
||||||
- Adapters used: embedded JSON Schema files and JSON encoders/decoders.
|
|
||||||
- Failure behavior: schema validation errors, unsupported artifact/schema
|
|
||||||
conversion, invalid IDs/timing.
|
|
||||||
- Tests to inspect: `schema/output_test.go`,
|
|
||||||
`internal/artifact/transcript_test.go`, `internal/trim/*_test.go`,
|
|
||||||
`internal/normalize/*_test.go`.
|
|
||||||
- Architectural invariants: sequential IDs, selected schema validation, no
|
|
||||||
internal-only fields in public schemas.
|
|
||||||
|
|
||||||
### Built-In Modules
|
|
||||||
|
|
||||||
- Path: `docs/internal/modules.md`
|
|
||||||
- Purpose: document implemented module responsibilities and ordering
|
|
||||||
constraints.
|
|
||||||
- Inputs and outputs: raw transcripts, preprocess state, merged transcript,
|
|
||||||
report events, selected JSON output.
|
|
||||||
- Boundaries: module behavior only; no full CLI/config reference.
|
|
||||||
- Config fields used: speaker file, autocorrect file, coalesce gap, overlap word
|
|
||||||
gap, word run reorder window, backchannel/filler max durations.
|
|
||||||
- Adapters used: JSON input/output, speaker YAML, autocorrect YAML, report
|
|
||||||
events.
|
|
||||||
- Failure behavior: input validation errors, invalid YAML, unknown module names,
|
|
||||||
invalid output schema before write.
|
|
||||||
- Tests to inspect: `internal/builtin`, `internal/overlap`,
|
|
||||||
`internal/coalesce`, `internal/danglers`, `internal/backchannel`,
|
|
||||||
`internal/filler`, `internal/autocorrect`, and CLI merge tests.
|
|
||||||
- Architectural invariants: order-sensitive transforms run before `assign-ids`;
|
|
||||||
modules stay narrow and explicitly configured.
|
|
||||||
|
|
||||||
### Trim
|
|
||||||
|
|
||||||
- Path: include in `docs/internal/artifacts.md` or create
|
|
||||||
`docs/internal/trim.md` if artifacts doc grows too large.
|
|
||||||
- Purpose: document artifact-level segment projection.
|
|
||||||
- Inputs and outputs: existing seriatim artifact, selector, selected output
|
|
||||||
schema, optional report.
|
|
||||||
- Boundaries: no merge postprocessors; no raw WhisperX input.
|
|
||||||
- Config fields used: input/output/report files, keep/remove selector,
|
|
||||||
optional output schema, allow-empty.
|
|
||||||
- Adapters used: file I/O in CLI, artifact parsing/validation, report writer.
|
|
||||||
- Failure behavior: malformed selector, invalid artifact, missing selected IDs,
|
|
||||||
non-sequential input IDs, empty output unless allowed, unsupported schema
|
|
||||||
up-conversion.
|
|
||||||
- Tests to inspect: `internal/trim/*_test.go`, `internal/cli/trim_test.go`.
|
|
||||||
- Architectural invariants: preserve transcript order, renumber retained IDs,
|
|
||||||
recompute full-schema overlap groups, never run merge modules.
|
|
||||||
|
|
||||||
### Normalize
|
|
||||||
|
|
||||||
- Path: include in `docs/internal/artifacts.md` or create
|
|
||||||
`docs/internal/normalize.md` if artifacts doc grows too large.
|
|
||||||
- Purpose: document artifact-level transcript canonicalization.
|
|
||||||
- Inputs and outputs: transcript-like JSON object or bare array, selected
|
|
||||||
seriatim output schema, optional report.
|
|
||||||
- Boundaries: no merge preprocessing or postprocessing modules.
|
|
||||||
- Config fields used: input/output/report files, output schema, output modules.
|
|
||||||
- Adapters used: file I/O, JSON parsing, schema validation, report writer.
|
|
||||||
- Failure behavior: invalid JSON, unsupported top-level shape, invalid timing
|
|
||||||
after repair, unsupported output module/schema, report write failure.
|
|
||||||
- Tests to inspect: `internal/normalize/*_test.go`,
|
|
||||||
`internal/cli/normalize_test.go`.
|
|
||||||
- Architectural invariants: deterministic repair/sort/ID assignment, no
|
|
||||||
transcript text in normalize report events, no merge modules.
|
|
||||||
|
|
||||||
## Integration Documentation Plan
|
|
||||||
|
|
||||||
- `docs/integrations/whisperx-json.md`
|
|
||||||
- External system or contract: WhisperX-like JSON transcript subset.
|
|
||||||
- Current usage: `merge` reads a top-level `segments` array with required
|
|
||||||
segment timing/text and optional word timing.
|
|
||||||
- Version or compatibility notes: no explicit WhisperX version is encoded in
|
|
||||||
the repository; document only the accepted subset.
|
|
||||||
- Document: supported fields, validation, word timing behavior, errors.
|
|
||||||
- Do not document: full WhisperX schema, audio diarization, non-JSON formats.
|
|
||||||
- `docs/integrations/output-schemas.md`
|
|
||||||
- External system or contract: seriatim public JSON output contracts.
|
|
||||||
- Current usage: `merge`, `trim`, and `normalize` emit
|
|
||||||
`seriatim-minimal`, `seriatim-intermediate`, or `seriatim-full`.
|
|
||||||
- Version or compatibility notes: schemas are embedded from `schema/`; release
|
|
||||||
version metadata is injected through build info.
|
|
||||||
- Document: schema roles, semantic invariants, validation APIs, links to
|
|
||||||
schema files.
|
|
||||||
- Do not document: unimplemented output formats or full schema copies.
|
|
||||||
- YAML rule files
|
|
||||||
- Prefer documenting speaker and autocorrect YAML contracts in
|
|
||||||
`docs/config.md`. Create `docs/integrations/yaml-rule-files.md` only if the
|
|
||||||
config reference becomes too large.
|
|
||||||
- Audita-style bare arrays
|
|
||||||
- Cover under `docs/cli.md` normalize behavior unless maintainers need a
|
|
||||||
separate integration note. Do not generalize beyond implemented bare segment
|
|
||||||
arrays.
|
|
||||||
- No external CLI/API/service docs are needed now. The repository implements no
|
|
||||||
external CLI, network API, daemon, remote storage, or service integration.
|
|
||||||
|
|
||||||
## Recommended Implementation Sequence
|
|
||||||
|
|
||||||
### Stage 1: Write Documentation Roadmap
|
|
||||||
|
|
||||||
- Goal: review and finalize this roadmap as the implementation plan for the
|
|
||||||
documentation migration.
|
|
||||||
- Files: `docs/roadmap/documentation.md` only.
|
|
||||||
- Repository areas inspected: documentation policy, architecture policy,
|
|
||||||
`README.md`, root `architecture.md`, and CLI/config/pipeline/schema/report
|
|
||||||
code and tests.
|
|
||||||
- Completion status: complete (2026-05-24).
|
|
||||||
- Completion evidence:
|
|
||||||
- `go test ./...` passed.
|
|
||||||
- `git status --short` confirmed no unrelated working-tree changes before
|
|
||||||
roadmap-only edits.
|
|
||||||
- Acceptance criteria: roadmap is present, action-oriented, and constrained to
|
|
||||||
implemented behavior outside `docs/roadmap/`.
|
|
||||||
- Suggested validation commands: `go test ./...`; `git status --short`.
|
|
||||||
- Prompt size: one implementation prompt.
|
|
||||||
|
|
||||||
### Stage 2: User-Facing Canonical Docs and Slim README
|
|
||||||
|
|
||||||
- Goal: move user reference material out of README into canonical docs.
|
|
||||||
- Files: update `README.md`; create `docs/cli.md` and `docs/config.md`.
|
|
||||||
- Repository areas to inspect: `internal/cli`, `internal/config`,
|
|
||||||
`internal/speaker`, `internal/autocorrect`, CLI/config tests.
|
|
||||||
- Acceptance criteria: README is concise; CLI/config docs match flags, defaults,
|
|
||||||
env vars, YAML formats, and validation; no roadmap-only content appears.
|
|
||||||
- Suggested validation commands: `go test ./...`;
|
|
||||||
`go run ./cmd/seriatim --help`;
|
|
||||||
`go run ./cmd/seriatim merge --help`;
|
|
||||||
`go run ./cmd/seriatim trim --help`;
|
|
||||||
`go run ./cmd/seriatim normalize --help`;
|
|
||||||
stale-term grep from the validation plan.
|
|
||||||
- Prompt size: one prompt if concise; split if README rewrite or config
|
|
||||||
reference grows too large.
|
|
||||||
|
|
||||||
### Stage 3: Operations and Troubleshooting
|
|
||||||
|
|
||||||
- Goal: document runtime operation, reports, failure behavior, and common fixes.
|
|
||||||
- Files: create `docs/operations.md` and `docs/troubleshooting.md`.
|
|
||||||
- Repository areas to inspect: `cmd/seriatim/main.go`, `internal/cli`,
|
|
||||||
`internal/config`, `internal/report`, output writer, normalize/trim/merge
|
|
||||||
tests.
|
|
||||||
- Acceptance criteria: docs describe filesystem-only operation and current
|
|
||||||
failure modes; no daemon, resume, remote storage, or recovery behavior is
|
|
||||||
invented.
|
|
||||||
- Suggested validation commands: `go test ./...`; manual link review.
|
|
||||||
- Prompt size: one prompt.
|
|
||||||
|
|
||||||
### Stage 4: Developer and Internal Docs
|
|
||||||
|
|
||||||
- Goal: create developer workflow and implemented internal component docs.
|
|
||||||
- Files: create `docs/policy/development.md`,
|
|
||||||
`docs/internal/pipeline.md`, `docs/internal/artifacts.md`, and
|
|
||||||
`docs/internal/modules.md`.
|
|
||||||
- Repository areas to inspect: architecture policy, pipeline, modules, schema,
|
|
||||||
artifact conversion, trim/normalize packages, tests.
|
|
||||||
- Acceptance criteria: docs preserve boundaries, avoid CLI/config duplication,
|
|
||||||
and identify tests/invariants for future changes.
|
|
||||||
- Suggested validation commands: `go test ./...`; grep for unimplemented
|
|
||||||
future-format/plugin/concurrency claims outside roadmap.
|
|
||||||
- Prompt size: split into development policy and internal docs if needed.
|
|
||||||
|
|
||||||
### Stage 5: Integrations and Examples
|
|
||||||
|
|
||||||
- Goal: add concise integration notes and maintained synthetic examples.
|
|
||||||
- Files: create `docs/integrations/whisperx-json.md`,
|
|
||||||
`docs/integrations/output-schemas.md`, and `examples/*`; decide whether
|
|
||||||
`samples/` should remain separate.
|
|
||||||
- Repository areas to inspect: `internal/builtin/input.go`, `schema`,
|
|
||||||
`internal/artifact`, CLI tests, existing `samples/`.
|
|
||||||
- Acceptance criteria: examples are small, synthetic, valid, and linked from
|
|
||||||
relevant docs; integration docs document only implemented contracts.
|
|
||||||
- Suggested validation commands: `go test ./...`; run documented example
|
|
||||||
`go run` commands; validate example YAML through command paths.
|
|
||||||
- Prompt size: split if examples need tests or sample cleanup decisions.
|
|
||||||
|
|
||||||
### Stage 6: Stale Documentation Cleanup
|
|
||||||
|
|
||||||
- Goal: remove wrong-home and stale documentation after canonical replacements
|
|
||||||
exist.
|
|
||||||
- Files: delete or relocate root `architecture.md`; remove stale material from
|
|
||||||
README; update links across docs.
|
|
||||||
- Repository areas to inspect: all docs, README, roadmap, root files.
|
|
||||||
- Acceptance criteria: no links to deleted root `architecture.md`; no
|
|
||||||
unimplemented behavior outside `docs/roadmap/`; canonical homes are respected.
|
|
||||||
- Suggested validation commands: `go test ./...`; stale-term grep; manual link
|
|
||||||
check; `git status --short`.
|
|
||||||
- Prompt size: one prompt.
|
|
||||||
|
|
||||||
## Validation Plan
|
|
||||||
|
|
||||||
Use these checks during or after documentation migration:
|
|
||||||
|
|
||||||
- Run `go test ./...`.
|
|
||||||
- Run `go run ./cmd/seriatim --help`.
|
|
||||||
- Run `go run ./cmd/seriatim merge --help`.
|
|
||||||
- Run `go run ./cmd/seriatim trim --help`.
|
|
||||||
- Run `go run ./cmd/seriatim normalize --help`.
|
|
||||||
- Once examples exist, run each documented example command and verify output is
|
|
||||||
produced in a temporary path.
|
|
||||||
- Load example YAML through the merge command or package tests.
|
|
||||||
- Validate example JSON through existing CLI/schema paths where practical.
|
|
||||||
- Grep outside `docs/roadmap/` for stale or roadmap-only terms:
|
|
||||||
`Future input`, `Future output`, `LLM`, `plugin`, `SRT`, `VTT`, `.tar.gz`,
|
|
||||||
`URI`, `old format`, `not implemented yet`, and
|
|
||||||
`runtime default may change`.
|
|
||||||
- Manually check links unless a link checker is added. No automated
|
|
||||||
documentation checker currently exists.
|
|
||||||
- Verify docs and examples contain no secrets, private transcript data, API
|
|
||||||
keys, tokens, passwords, or private infrastructure details.
|
|
||||||
|
|
||||||
## Open Questions
|
|
||||||
|
|
||||||
- Should `samples/` be removed, kept as non-doc sample data, or replaced by
|
|
||||||
small synthetic `examples/`? Recommendation: create small synthetic examples
|
|
||||||
first, then audit `samples/` for privacy, size, and ongoing maintenance before
|
|
||||||
deleting or linking it.
|
|
||||||
- Should Audita-style bare-array normalization have a separate integration doc?
|
|
||||||
Recommendation: cover it in `docs/cli.md` normalize behavior unless a
|
|
||||||
stronger external-contract requirement emerges.
|
|
||||||
@@ -2,10 +2,9 @@ package builtin
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
|
||||||
"os"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/seriatim/internal/config"
|
"gitea.maximumdirect.net/eric/seriatim/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/internal/jsonfile"
|
||||||
"gitea.maximumdirect.net/eric/seriatim/internal/report"
|
"gitea.maximumdirect.net/eric/seriatim/internal/report"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -20,15 +19,7 @@ func (jsonOutputWriter) Write(ctx context.Context, out any, rpt report.Report, c
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
file, err := os.Create(cfg.OutputFile)
|
if err := jsonfile.Write(cfg.OutputFile, out); err != nil {
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer file.Close()
|
|
||||||
|
|
||||||
enc := json.NewEncoder(file)
|
|
||||||
enc.SetIndent("", " ")
|
|
||||||
if err := enc.Encode(out); err != nil {
|
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
31
internal/cli/flags.go
Normal file
31
internal/cli/flags.go
Normal file
@@ -0,0 +1,31 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func addOutputFileFlag(cmd *cobra.Command, target *string) {
|
||||||
|
cmd.Flags().StringVar(target, "output-file", "", "output transcript JSON file")
|
||||||
|
}
|
||||||
|
|
||||||
|
func addReportFileFlag(cmd *cobra.Command, target *string) {
|
||||||
|
cmd.Flags().StringVar(target, "report-file", "", "optional report JSON file")
|
||||||
|
}
|
||||||
|
|
||||||
|
func addOutputModulesFlag(cmd *cobra.Command, target *string) {
|
||||||
|
cmd.Flags().StringVar(target, "output-modules", config.DefaultOutputModules, "comma-separated output modules")
|
||||||
|
}
|
||||||
|
|
||||||
|
func addMergeOutputSchemaFlag(cmd *cobra.Command, target *string) {
|
||||||
|
cmd.Flags().StringVar(target, "output-schema", config.DefaultOutputSchema, "output JSON schema: seriatim-minimal, seriatim-intermediate (default), or seriatim-full")
|
||||||
|
}
|
||||||
|
|
||||||
|
func addNormalizeOutputSchemaFlag(cmd *cobra.Command, target *string) {
|
||||||
|
cmd.Flags().StringVar(target, "output-schema", config.DefaultOutputSchema, "output JSON schema: seriatim-minimal, seriatim-intermediate, or seriatim-full")
|
||||||
|
}
|
||||||
|
|
||||||
|
func addTrimOutputSchemaFlag(cmd *cobra.Command, target *string) {
|
||||||
|
cmd.Flags().StringVar(target, "output-schema", "", "optional output JSON schema override: seriatim-minimal, seriatim-intermediate, or seriatim-full")
|
||||||
|
}
|
||||||
@@ -31,13 +31,13 @@ func newMergeCommand() *cobra.Command {
|
|||||||
|
|
||||||
flags := cmd.Flags()
|
flags := cmd.Flags()
|
||||||
flags.StringArrayVar(&opts.InputFiles, "input-file", nil, "input transcript file; may be repeated")
|
flags.StringArrayVar(&opts.InputFiles, "input-file", nil, "input transcript file; may be repeated")
|
||||||
flags.StringVar(&opts.OutputFile, "output-file", "", "output transcript JSON file")
|
addOutputFileFlag(cmd, &opts.OutputFile)
|
||||||
flags.StringVar(&opts.ReportFile, "report-file", "", "optional report JSON file")
|
addReportFileFlag(cmd, &opts.ReportFile)
|
||||||
flags.StringVar(&opts.SpeakersFile, "speakers", "", "speaker map file")
|
flags.StringVar(&opts.SpeakersFile, "speakers", "", "speaker map file")
|
||||||
flags.StringVar(&opts.AutocorrectFile, "autocorrect", "", "autocorrect rules file")
|
flags.StringVar(&opts.AutocorrectFile, "autocorrect", "", "autocorrect rules file")
|
||||||
flags.StringVar(&opts.InputReader, "input-reader", config.DefaultInputReader, "input reader module")
|
flags.StringVar(&opts.InputReader, "input-reader", config.DefaultInputReader, "input reader module")
|
||||||
flags.StringVar(&opts.OutputModules, "output-modules", config.DefaultOutputModules, "comma-separated output modules")
|
addOutputModulesFlag(cmd, &opts.OutputModules)
|
||||||
flags.StringVar(&opts.OutputSchema, "output-schema", config.DefaultOutputSchema, "output JSON schema: seriatim-minimal, seriatim-intermediate (default), or seriatim-full")
|
addMergeOutputSchemaFlag(cmd, &opts.OutputSchema)
|
||||||
flags.StringVar(&opts.PreprocessingModules, "preprocessing-modules", config.DefaultPreprocessingModules, "comma-separated preprocessing modules")
|
flags.StringVar(&opts.PreprocessingModules, "preprocessing-modules", config.DefaultPreprocessingModules, "comma-separated preprocessing modules")
|
||||||
flags.StringVar(&opts.PostprocessingModules, "postprocessing-modules", config.DefaultPostprocessingModules, "comma-separated postprocessing modules")
|
flags.StringVar(&opts.PostprocessingModules, "postprocessing-modules", config.DefaultPostprocessingModules, "comma-separated postprocessing modules")
|
||||||
flags.StringVar(&opts.CoalesceGap, "coalesce-gap", config.DefaultCoalesceGapValue, "maximum same-speaker gap in seconds for coalesce")
|
flags.StringVar(&opts.CoalesceGap, "coalesce-gap", config.DefaultCoalesceGapValue, "maximum same-speaker gap in seconds for coalesce")
|
||||||
|
|||||||
@@ -30,10 +30,10 @@ func newNormalizeCommand() *cobra.Command {
|
|||||||
|
|
||||||
flags := cmd.Flags()
|
flags := cmd.Flags()
|
||||||
flags.StringVar(&opts.InputFile, "input-file", "", "input transcript JSON file")
|
flags.StringVar(&opts.InputFile, "input-file", "", "input transcript JSON file")
|
||||||
flags.StringVar(&opts.OutputFile, "output-file", "", "output transcript JSON file")
|
addOutputFileFlag(cmd, &opts.OutputFile)
|
||||||
flags.StringVar(&opts.ReportFile, "report-file", "", "optional report JSON file")
|
addReportFileFlag(cmd, &opts.ReportFile)
|
||||||
flags.StringVar(&opts.OutputSchema, "output-schema", config.DefaultOutputSchema, "output JSON schema: seriatim-minimal, seriatim-intermediate, or seriatim-full")
|
addNormalizeOutputSchemaFlag(cmd, &opts.OutputSchema)
|
||||||
flags.StringVar(&opts.OutputModules, "output-modules", config.DefaultOutputModules, "comma-separated output modules")
|
addOutputModulesFlag(cmd, &opts.OutputModules)
|
||||||
|
|
||||||
return cmd
|
return cmd
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,41 +1,12 @@
|
|||||||
package cli
|
package cli
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"sort"
|
|
||||||
|
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/seriatim/internal/config"
|
"gitea.maximumdirect.net/eric/seriatim/internal/config"
|
||||||
"gitea.maximumdirect.net/eric/seriatim/internal/report"
|
"gitea.maximumdirect.net/eric/seriatim/internal/trim"
|
||||||
triminternal "gitea.maximumdirect.net/eric/seriatim/internal/trim"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type trimAuditReport struct {
|
|
||||||
Operation string `json:"operation"`
|
|
||||||
InputFile string `json:"input_file"`
|
|
||||||
OutputFile string `json:"output_file"`
|
|
||||||
InputSchema string `json:"input_schema"`
|
|
||||||
OutputSchema string `json:"output_schema"`
|
|
||||||
Mode string `json:"mode"`
|
|
||||||
Selector string `json:"selector"`
|
|
||||||
SelectedIDs []int `json:"selected_ids"`
|
|
||||||
AllowEmpty bool `json:"allow_empty"`
|
|
||||||
InputSegmentCount int `json:"input_segment_count"`
|
|
||||||
RetainedSegmentCount int `json:"retained_segment_count"`
|
|
||||||
RemovedSegmentCount int `json:"removed_segment_count"`
|
|
||||||
RemovedInputIDs []int `json:"removed_input_ids"`
|
|
||||||
OldToNewIDMapping []trimIDMapping `json:"old_to_new_id_mapping"`
|
|
||||||
OverlapGroupsRecomputed bool `json:"overlap_groups_recomputed"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type trimIDMapping struct {
|
|
||||||
OldID int `json:"old_id"`
|
|
||||||
NewID int `json:"new_id"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func newTrimCommand() *cobra.Command {
|
func newTrimCommand() *cobra.Command {
|
||||||
var opts config.TrimOptions
|
var opts config.TrimOptions
|
||||||
|
|
||||||
@@ -53,139 +24,18 @@ func newTrimCommand() *cobra.Command {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
selector, err := triminternal.ParseSelector(cfg.Selector)
|
return trim.Run(cmd.Context(), cfg)
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("invalid selector %q: %w", cfg.Selector, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
data, err := os.ReadFile(cfg.InputFile)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("read --input-file %q: %w", cfg.InputFile, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
artifact, err := triminternal.ParseArtifactJSON(data)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("--input-file %q: %w", cfg.InputFile, err)
|
|
||||||
}
|
|
||||||
inputSegmentCount := artifact.SegmentCount()
|
|
||||||
inputSchema := artifact.Schema
|
|
||||||
|
|
||||||
mode := triminternal.ModeKeep
|
|
||||||
if cfg.Mode == "remove" {
|
|
||||||
mode = triminternal.ModeRemove
|
|
||||||
}
|
|
||||||
|
|
||||||
trimmed, err := triminternal.ApplyArtifact(artifact, triminternal.Options{
|
|
||||||
Mode: mode,
|
|
||||||
Selector: selector,
|
|
||||||
AllowEmpty: cfg.AllowEmpty,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
outputSchema := artifact.Schema
|
|
||||||
if cfg.OutputSchema != "" {
|
|
||||||
outputSchema = cfg.OutputSchema
|
|
||||||
}
|
|
||||||
|
|
||||||
outputArtifact, err := triminternal.ConvertArtifact(trimmed.Artifact, outputSchema)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := triminternal.ValidateArtifact(outputArtifact); err != nil {
|
|
||||||
return fmt.Errorf("validate trimmed output: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := writeOutputJSON(cfg.OutputFile, outputArtifact.Value()); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if cfg.ReportFile != "" {
|
|
||||||
audit := trimAuditReport{
|
|
||||||
Operation: "trim",
|
|
||||||
InputFile: cfg.InputFile,
|
|
||||||
OutputFile: cfg.OutputFile,
|
|
||||||
InputSchema: inputSchema,
|
|
||||||
OutputSchema: outputArtifact.Schema,
|
|
||||||
Mode: cfg.Mode,
|
|
||||||
Selector: cfg.Selector,
|
|
||||||
SelectedIDs: selector.IDs(),
|
|
||||||
AllowEmpty: cfg.AllowEmpty,
|
|
||||||
InputSegmentCount: inputSegmentCount,
|
|
||||||
RetainedSegmentCount: len(trimmed.OldToNewID),
|
|
||||||
RemovedSegmentCount: len(trimmed.RemovedIDs),
|
|
||||||
RemovedInputIDs: append([]int(nil), trimmed.RemovedIDs...),
|
|
||||||
OldToNewIDMapping: orderedIDMapping(trimmed.OldToNewID),
|
|
||||||
OverlapGroupsRecomputed: trimmed.OverlapGroupsRecomputed,
|
|
||||||
}
|
|
||||||
auditJSON, err := json.Marshal(audit)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("marshal trim audit report: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
rpt := report.Report{
|
|
||||||
Metadata: report.Metadata{
|
|
||||||
Application: outputArtifact.Application(),
|
|
||||||
Version: outputArtifact.Version(),
|
|
||||||
InputReader: "trim-artifact",
|
|
||||||
InputFiles: []string{cfg.InputFile},
|
|
||||||
OutputModules: []string{"json"},
|
|
||||||
},
|
|
||||||
Events: []report.Event{
|
|
||||||
report.Info("trim", "trim", fmt.Sprintf("trimmed %d input segment(s) into %d output segment(s) with mode=%s", inputSegmentCount, outputArtifact.SegmentCount(), cfg.Mode)),
|
|
||||||
report.Info("trim", "trim-audit", string(auditJSON)),
|
|
||||||
report.Info("trim", "validate-output", fmt.Sprintf("validated %d output segment(s)", outputArtifact.SegmentCount())),
|
|
||||||
report.Info("output", "json", "wrote transcript JSON"),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
if err := report.WriteJSON(cfg.ReportFile, rpt); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
flags := cmd.Flags()
|
flags := cmd.Flags()
|
||||||
flags.StringVar(&opts.InputFile, "input-file", "", "input seriatim transcript artifact JSON file")
|
flags.StringVar(&opts.InputFile, "input-file", "", "input seriatim transcript artifact JSON file")
|
||||||
flags.StringVar(&opts.OutputFile, "output-file", "", "output transcript JSON file")
|
addOutputFileFlag(cmd, &opts.OutputFile)
|
||||||
flags.StringVar(&opts.ReportFile, "report-file", "", "optional report JSON file")
|
addReportFileFlag(cmd, &opts.ReportFile)
|
||||||
flags.StringVar(&opts.Keep, "keep", "", "segment ID selector to keep (for example: 1-10,15)")
|
flags.StringVar(&opts.Keep, "keep", "", "segment ID selector to keep (for example: 1-10,15)")
|
||||||
flags.StringVar(&opts.Remove, "remove", "", "segment ID selector to remove (for example: 1-10,15)")
|
flags.StringVar(&opts.Remove, "remove", "", "segment ID selector to remove (for example: 1-10,15)")
|
||||||
flags.StringVar(&opts.OutputSchema, "output-schema", "", "optional output JSON schema override: seriatim-minimal, seriatim-intermediate, or seriatim-full")
|
addTrimOutputSchemaFlag(cmd, &opts.OutputSchema)
|
||||||
flags.BoolVar(&opts.AllowEmpty, "allow-empty", false, "allow trimming to an empty transcript")
|
flags.BoolVar(&opts.AllowEmpty, "allow-empty", false, "allow trimming to an empty transcript")
|
||||||
|
|
||||||
return cmd
|
return cmd
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeOutputJSON(path string, value any) error {
|
|
||||||
file, err := os.Create(path)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer file.Close()
|
|
||||||
|
|
||||||
enc := json.NewEncoder(file)
|
|
||||||
enc.SetIndent("", " ")
|
|
||||||
return enc.Encode(value)
|
|
||||||
}
|
|
||||||
|
|
||||||
func orderedIDMapping(mapping map[int]int) []trimIDMapping {
|
|
||||||
keys := make([]int, 0, len(mapping))
|
|
||||||
for oldID := range mapping {
|
|
||||||
keys = append(keys, oldID)
|
|
||||||
}
|
|
||||||
sort.Ints(keys)
|
|
||||||
|
|
||||||
pairs := make([]trimIDMapping, 0, len(keys))
|
|
||||||
for _, oldID := range keys {
|
|
||||||
pairs = append(pairs, trimIDMapping{
|
|
||||||
OldID: oldID,
|
|
||||||
NewID: mapping[oldID],
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return pairs
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -12,6 +12,29 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/seriatim/schema"
|
"gitea.maximumdirect.net/eric/seriatim/schema"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type trimAuditReport struct {
|
||||||
|
Operation string `json:"operation"`
|
||||||
|
InputFile string `json:"input_file"`
|
||||||
|
OutputFile string `json:"output_file"`
|
||||||
|
InputSchema string `json:"input_schema"`
|
||||||
|
OutputSchema string `json:"output_schema"`
|
||||||
|
Mode string `json:"mode"`
|
||||||
|
Selector string `json:"selector"`
|
||||||
|
SelectedIDs []int `json:"selected_ids"`
|
||||||
|
AllowEmpty bool `json:"allow_empty"`
|
||||||
|
InputSegmentCount int `json:"input_segment_count"`
|
||||||
|
RetainedSegmentCount int `json:"retained_segment_count"`
|
||||||
|
RemovedSegmentCount int `json:"removed_segment_count"`
|
||||||
|
RemovedInputIDs []int `json:"removed_input_ids"`
|
||||||
|
OldToNewIDMapping []trimIDMapping `json:"old_to_new_id_mapping"`
|
||||||
|
OverlapGroupsRecomputed bool `json:"overlap_groups_recomputed"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type trimIDMapping struct {
|
||||||
|
OldID int `json:"old_id"`
|
||||||
|
NewID int `json:"new_id"`
|
||||||
|
}
|
||||||
|
|
||||||
func TestTrimKeepModeEndToEnd(t *testing.T) {
|
func TestTrimKeepModeEndToEnd(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
input := writeTrimFullFixture(t, dir, "input.json")
|
input := writeTrimFullFixture(t, dir, "input.json")
|
||||||
|
|||||||
@@ -160,13 +160,7 @@ func (r run) coalescedSegment(id int) model.Segment {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func segmentRef(segment model.Segment) string {
|
func segmentRef(segment model.Segment) string {
|
||||||
if segment.SourceSegmentIndex != nil {
|
return model.SegmentReference(segment)
|
||||||
return fmt.Sprintf("%s#%d", segment.Source, *segment.SourceSegmentIndex)
|
|
||||||
}
|
|
||||||
if segment.SourceRef != "" {
|
|
||||||
return segment.SourceRef
|
|
||||||
}
|
|
||||||
return segment.Source
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func isSkippableInterjection(segment model.Segment) bool {
|
func isSkippableInterjection(segment model.Segment) bool {
|
||||||
|
|||||||
@@ -8,6 +8,8 @@ import (
|
|||||||
"sort"
|
"sort"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/schema"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -27,9 +29,9 @@ const (
|
|||||||
WordRunReorderWindowEnv = "SERIATIM_OVERLAP_WORD_RUN_REORDER_WINDOW"
|
WordRunReorderWindowEnv = "SERIATIM_OVERLAP_WORD_RUN_REORDER_WINDOW"
|
||||||
BackchannelMaxDurationEnv = "SERIATIM_BACKCHANNEL_MAX_DURATION"
|
BackchannelMaxDurationEnv = "SERIATIM_BACKCHANNEL_MAX_DURATION"
|
||||||
FillerMaxDurationEnv = "SERIATIM_FILLER_MAX_DURATION"
|
FillerMaxDurationEnv = "SERIATIM_FILLER_MAX_DURATION"
|
||||||
OutputSchemaMinimal = "seriatim-minimal"
|
OutputSchemaMinimal = schema.OutputSchemaMinimal
|
||||||
OutputSchemaIntermediate = "seriatim-intermediate"
|
OutputSchemaIntermediate = schema.OutputSchemaIntermediate
|
||||||
OutputSchemaFull = "seriatim-full"
|
OutputSchemaFull = schema.OutputSchemaFull
|
||||||
)
|
)
|
||||||
|
|
||||||
// MergeOptions captures raw CLI option values before validation.
|
// MergeOptions captures raw CLI option values before validation.
|
||||||
@@ -210,11 +212,8 @@ func NewMergeConfig(opts MergeOptions) (Config, error) {
|
|||||||
|
|
||||||
// NewTrimConfig validates raw trim options and returns normalized config.
|
// NewTrimConfig validates raw trim options and returns normalized config.
|
||||||
func NewTrimConfig(opts TrimOptions) (TrimConfig, error) {
|
func NewTrimConfig(opts TrimOptions) (TrimConfig, error) {
|
||||||
inputFile := filepath.Clean(strings.TrimSpace(opts.InputFile))
|
inputFile, err := normalizeSingleInputFile(opts.InputFile, "--input-file")
|
||||||
if strings.TrimSpace(opts.InputFile) == "" {
|
if err != nil {
|
||||||
return TrimConfig{}, errors.New("--input-file is required")
|
|
||||||
}
|
|
||||||
if err := requireFile(inputFile, "--input-file"); err != nil {
|
|
||||||
return TrimConfig{}, err
|
return TrimConfig{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -223,13 +222,10 @@ func NewTrimConfig(opts TrimOptions) (TrimConfig, error) {
|
|||||||
return TrimConfig{}, err
|
return TrimConfig{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
reportFile := ""
|
reportFile, err := normalizeOptionalOutputPath(opts.ReportFile, "--report-file")
|
||||||
if strings.TrimSpace(opts.ReportFile) != "" {
|
|
||||||
reportFile, err = normalizeOutputPath(opts.ReportFile, "--report-file")
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return TrimConfig{}, err
|
return TrimConfig{}, err
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
keep := strings.TrimSpace(opts.Keep)
|
keep := strings.TrimSpace(opts.Keep)
|
||||||
remove := strings.TrimSpace(opts.Remove)
|
remove := strings.TrimSpace(opts.Remove)
|
||||||
@@ -267,11 +263,8 @@ func NewTrimConfig(opts TrimOptions) (TrimConfig, error) {
|
|||||||
|
|
||||||
// NewNormalizeConfig validates raw normalize options and returns normalized config.
|
// NewNormalizeConfig validates raw normalize options and returns normalized config.
|
||||||
func NewNormalizeConfig(opts NormalizeOptions) (NormalizeConfig, error) {
|
func NewNormalizeConfig(opts NormalizeOptions) (NormalizeConfig, error) {
|
||||||
inputFile := filepath.Clean(strings.TrimSpace(opts.InputFile))
|
inputFile, err := normalizeSingleInputFile(opts.InputFile, "--input-file")
|
||||||
if strings.TrimSpace(opts.InputFile) == "" {
|
if err != nil {
|
||||||
return NormalizeConfig{}, errors.New("--input-file is required")
|
|
||||||
}
|
|
||||||
if err := requireFile(inputFile, "--input-file"); err != nil {
|
|
||||||
return NormalizeConfig{}, err
|
return NormalizeConfig{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -280,13 +273,10 @@ func NewNormalizeConfig(opts NormalizeOptions) (NormalizeConfig, error) {
|
|||||||
return NormalizeConfig{}, err
|
return NormalizeConfig{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
reportFile := ""
|
reportFile, err := normalizeOptionalOutputPath(opts.ReportFile, "--report-file")
|
||||||
if strings.TrimSpace(opts.ReportFile) != "" {
|
|
||||||
reportFile, err = normalizeOutputPath(opts.ReportFile, "--report-file")
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return NormalizeConfig{}, err
|
return NormalizeConfig{}, err
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
outputSchema, err := resolveOutputSchema(opts.OutputSchema)
|
outputSchema, err := resolveOutputSchema(opts.OutputSchema)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -332,12 +322,12 @@ func parseModuleList(value string) ([]string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func validateOutputSchema(value string) error {
|
func validateOutputSchema(value string) error {
|
||||||
switch value {
|
if schema.ValidOutputSchemaName(value) {
|
||||||
case OutputSchemaMinimal, OutputSchemaIntermediate, OutputSchemaFull:
|
|
||||||
return nil
|
return nil
|
||||||
default:
|
|
||||||
return fmt.Errorf("--output-schema must be one of %q, %q, or %q", OutputSchemaMinimal, OutputSchemaIntermediate, OutputSchemaFull)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
names := schema.OutputSchemaNames()
|
||||||
|
return fmt.Errorf("--output-schema must be one of %q, %q, or %q", names[0], names[1], names[2])
|
||||||
}
|
}
|
||||||
|
|
||||||
func resolveOutputSchema(value string) (string, error) {
|
func resolveOutputSchema(value string) (string, error) {
|
||||||
@@ -381,6 +371,26 @@ func normalizeInputFiles(paths []string) ([]string, error) {
|
|||||||
return normalized, nil
|
return normalized, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func normalizeSingleInputFile(path string, flag string) (string, error) {
|
||||||
|
path = strings.TrimSpace(path)
|
||||||
|
if path == "" {
|
||||||
|
return "", fmt.Errorf("%s is required", flag)
|
||||||
|
}
|
||||||
|
|
||||||
|
clean := filepath.Clean(path)
|
||||||
|
if err := requireFile(clean, flag); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return clean, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeOptionalOutputPath(path string, flag string) (string, error) {
|
||||||
|
if strings.TrimSpace(path) == "" {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
return normalizeOutputPath(path, flag)
|
||||||
|
}
|
||||||
|
|
||||||
func normalizeOutputPath(path string, flag string) (string, error) {
|
func normalizeOutputPath(path string, flag string) (string, error) {
|
||||||
path = strings.TrimSpace(path)
|
path = strings.TrimSpace(path)
|
||||||
if path == "" {
|
if path == "" {
|
||||||
|
|||||||
@@ -538,15 +538,9 @@ func TestCoalesceGapUsesValidOverride(t *testing.T) {
|
|||||||
input := writeTempFile(t, dir, "input.json")
|
input := writeTempFile(t, dir, "input.json")
|
||||||
output := filepath.Join(dir, "merged.json")
|
output := filepath.Join(dir, "merged.json")
|
||||||
|
|
||||||
cfg, err := NewMergeConfig(MergeOptions{
|
opts := validMergeOptions(input, output)
|
||||||
InputFiles: []string{input},
|
opts.CoalesceGap = "1.5"
|
||||||
OutputFile: output,
|
cfg, err := NewMergeConfig(opts)
|
||||||
InputReader: DefaultInputReader,
|
|
||||||
OutputModules: DefaultOutputModules,
|
|
||||||
PreprocessingModules: DefaultPreprocessingModules,
|
|
||||||
PostprocessingModules: DefaultPostprocessingModules,
|
|
||||||
CoalesceGap: "1.5",
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("config failed: %v", err)
|
t.Fatalf("config failed: %v", err)
|
||||||
}
|
}
|
||||||
@@ -560,15 +554,9 @@ func TestCoalesceGapAllowsZero(t *testing.T) {
|
|||||||
input := writeTempFile(t, dir, "input.json")
|
input := writeTempFile(t, dir, "input.json")
|
||||||
output := filepath.Join(dir, "merged.json")
|
output := filepath.Join(dir, "merged.json")
|
||||||
|
|
||||||
cfg, err := NewMergeConfig(MergeOptions{
|
opts := validMergeOptions(input, output)
|
||||||
InputFiles: []string{input},
|
opts.CoalesceGap = "0"
|
||||||
OutputFile: output,
|
cfg, err := NewMergeConfig(opts)
|
||||||
InputReader: DefaultInputReader,
|
|
||||||
OutputModules: DefaultOutputModules,
|
|
||||||
PreprocessingModules: DefaultPreprocessingModules,
|
|
||||||
PostprocessingModules: DefaultPostprocessingModules,
|
|
||||||
CoalesceGap: "0",
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("config failed: %v", err)
|
t.Fatalf("config failed: %v", err)
|
||||||
}
|
}
|
||||||
@@ -593,15 +581,9 @@ func TestCoalesceGapRejectsInvalidOverride(t *testing.T) {
|
|||||||
input := writeTempFile(t, dir, "input.json")
|
input := writeTempFile(t, dir, "input.json")
|
||||||
output := filepath.Join(dir, "merged.json")
|
output := filepath.Join(dir, "merged.json")
|
||||||
|
|
||||||
_, err := NewMergeConfig(MergeOptions{
|
opts := validMergeOptions(input, output)
|
||||||
InputFiles: []string{input},
|
opts.CoalesceGap = test.value
|
||||||
OutputFile: output,
|
_, err := NewMergeConfig(opts)
|
||||||
InputReader: DefaultInputReader,
|
|
||||||
OutputModules: DefaultOutputModules,
|
|
||||||
PreprocessingModules: DefaultPreprocessingModules,
|
|
||||||
PostprocessingModules: DefaultPostprocessingModules,
|
|
||||||
CoalesceGap: test.value,
|
|
||||||
})
|
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected error")
|
t.Fatal("expected error")
|
||||||
}
|
}
|
||||||
@@ -639,20 +621,16 @@ func TestNewTrimConfigRequiresExactlyOneSelectorFlag(t *testing.T) {
|
|||||||
input := writeTempFile(t, dir, "input.json")
|
input := writeTempFile(t, dir, "input.json")
|
||||||
output := filepath.Join(dir, "trimmed.json")
|
output := filepath.Join(dir, "trimmed.json")
|
||||||
|
|
||||||
_, err := NewTrimConfig(TrimOptions{
|
opts := validTrimOptions(input, output)
|
||||||
InputFile: input,
|
opts.Keep = ""
|
||||||
OutputFile: output,
|
_, err := NewTrimConfig(opts)
|
||||||
})
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "exactly one of --keep or --remove is required") {
|
if err == nil || !strings.Contains(err.Error(), "exactly one of --keep or --remove is required") {
|
||||||
t.Fatalf("expected missing selector error, got %v", err)
|
t.Fatalf("expected missing selector error, got %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = NewTrimConfig(TrimOptions{
|
opts = validTrimOptions(input, output)
|
||||||
InputFile: input,
|
opts.Remove = "2"
|
||||||
OutputFile: output,
|
_, err = NewTrimConfig(opts)
|
||||||
Keep: "1",
|
|
||||||
Remove: "2",
|
|
||||||
})
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "mutually exclusive") {
|
if err == nil || !strings.Contains(err.Error(), "mutually exclusive") {
|
||||||
t.Fatalf("expected mutually exclusive selector error, got %v", err)
|
t.Fatalf("expected mutually exclusive selector error, got %v", err)
|
||||||
}
|
}
|
||||||
@@ -664,14 +642,13 @@ func TestNewTrimConfigAcceptsOutputSchemaOverride(t *testing.T) {
|
|||||||
output := filepath.Join(dir, "trimmed.json")
|
output := filepath.Join(dir, "trimmed.json")
|
||||||
reportPath := filepath.Join(dir, "report.json")
|
reportPath := filepath.Join(dir, "report.json")
|
||||||
|
|
||||||
cfg, err := NewTrimConfig(TrimOptions{
|
opts := validTrimOptions(input, output)
|
||||||
InputFile: input,
|
opts.Keep = ""
|
||||||
OutputFile: output,
|
opts.Remove = "3-5"
|
||||||
ReportFile: reportPath,
|
opts.ReportFile = reportPath
|
||||||
Remove: "3-5",
|
opts.OutputSchema = OutputSchemaMinimal
|
||||||
OutputSchema: OutputSchemaMinimal,
|
opts.AllowEmpty = true
|
||||||
AllowEmpty: true,
|
cfg, err := NewTrimConfig(opts)
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("config failed: %v", err)
|
t.Fatalf("config failed: %v", err)
|
||||||
}
|
}
|
||||||
@@ -692,17 +669,30 @@ func TestNewTrimConfigAcceptsOutputSchemaOverride(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestNewTrimConfigTreatsWhitespaceReportFileAsOmitted(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTempFile(t, dir, "input.json")
|
||||||
|
output := filepath.Join(dir, "trimmed.json")
|
||||||
|
|
||||||
|
opts := validTrimOptions(input, output)
|
||||||
|
opts.ReportFile = " \t "
|
||||||
|
cfg, err := NewTrimConfig(opts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config failed: %v", err)
|
||||||
|
}
|
||||||
|
if cfg.ReportFile != "" {
|
||||||
|
t.Fatalf("report file = %q, want empty", cfg.ReportFile)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestNewTrimConfigRejectsInvalidOutputSchemaOverride(t *testing.T) {
|
func TestNewTrimConfigRejectsInvalidOutputSchemaOverride(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
input := writeTempFile(t, dir, "input.json")
|
input := writeTempFile(t, dir, "input.json")
|
||||||
output := filepath.Join(dir, "trimmed.json")
|
output := filepath.Join(dir, "trimmed.json")
|
||||||
|
|
||||||
_, err := NewTrimConfig(TrimOptions{
|
opts := validTrimOptions(input, output)
|
||||||
InputFile: input,
|
opts.OutputSchema = "compact"
|
||||||
OutputFile: output,
|
_, err := NewTrimConfig(opts)
|
||||||
Keep: "1",
|
|
||||||
OutputSchema: "compact",
|
|
||||||
})
|
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected output schema validation error")
|
t.Fatal("expected output schema validation error")
|
||||||
}
|
}
|
||||||
@@ -731,10 +721,8 @@ func TestNewNormalizeConfigRequiresOutputFile(t *testing.T) {
|
|||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
input := writeTempFile(t, dir, "input.json")
|
input := writeTempFile(t, dir, "input.json")
|
||||||
|
|
||||||
_, err := NewNormalizeConfig(NormalizeOptions{
|
opts := validNormalizeOptions(input, "")
|
||||||
InputFile: input,
|
_, err := NewNormalizeConfig(opts)
|
||||||
OutputModules: DefaultOutputModules,
|
|
||||||
})
|
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected output-file required error")
|
t.Fatal("expected output-file required error")
|
||||||
}
|
}
|
||||||
@@ -749,11 +737,8 @@ func TestNewNormalizeConfigResolvesOutputSchemaDefaultAndEnv(t *testing.T) {
|
|||||||
output := filepath.Join(dir, "normalized.json")
|
output := filepath.Join(dir, "normalized.json")
|
||||||
|
|
||||||
t.Setenv(OutputSchemaEnv, "")
|
t.Setenv(OutputSchemaEnv, "")
|
||||||
cfg, err := NewNormalizeConfig(NormalizeOptions{
|
opts := validNormalizeOptions(input, output)
|
||||||
InputFile: input,
|
cfg, err := NewNormalizeConfig(opts)
|
||||||
OutputFile: output,
|
|
||||||
OutputModules: DefaultOutputModules,
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("config failed: %v", err)
|
t.Fatalf("config failed: %v", err)
|
||||||
}
|
}
|
||||||
@@ -762,11 +747,7 @@ func TestNewNormalizeConfigResolvesOutputSchemaDefaultAndEnv(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
t.Setenv(OutputSchemaEnv, OutputSchemaMinimal)
|
t.Setenv(OutputSchemaEnv, OutputSchemaMinimal)
|
||||||
cfg, err = NewNormalizeConfig(NormalizeOptions{
|
cfg, err = NewNormalizeConfig(opts)
|
||||||
InputFile: input,
|
|
||||||
OutputFile: output,
|
|
||||||
OutputModules: DefaultOutputModules,
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("config failed: %v", err)
|
t.Fatalf("config failed: %v", err)
|
||||||
}
|
}
|
||||||
@@ -780,12 +761,9 @@ func TestNewNormalizeConfigRejectsInvalidOutputSchema(t *testing.T) {
|
|||||||
input := writeTempFile(t, dir, "input.json")
|
input := writeTempFile(t, dir, "input.json")
|
||||||
output := filepath.Join(dir, "normalized.json")
|
output := filepath.Join(dir, "normalized.json")
|
||||||
|
|
||||||
_, err := NewNormalizeConfig(NormalizeOptions{
|
opts := validNormalizeOptions(input, output)
|
||||||
InputFile: input,
|
opts.OutputSchema = "compact"
|
||||||
OutputFile: output,
|
_, err := NewNormalizeConfig(opts)
|
||||||
OutputSchema: "compact",
|
|
||||||
OutputModules: DefaultOutputModules,
|
|
||||||
})
|
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected output schema error")
|
t.Fatal("expected output schema error")
|
||||||
}
|
}
|
||||||
@@ -799,11 +777,9 @@ func TestNewNormalizeConfigRejectsUnknownOutputModule(t *testing.T) {
|
|||||||
input := writeTempFile(t, dir, "input.json")
|
input := writeTempFile(t, dir, "input.json")
|
||||||
output := filepath.Join(dir, "normalized.json")
|
output := filepath.Join(dir, "normalized.json")
|
||||||
|
|
||||||
_, err := NewNormalizeConfig(NormalizeOptions{
|
opts := validNormalizeOptions(input, output)
|
||||||
InputFile: input,
|
opts.OutputModules = "json,yaml"
|
||||||
OutputFile: output,
|
_, err := NewNormalizeConfig(opts)
|
||||||
OutputModules: "json,yaml",
|
|
||||||
})
|
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected output module error")
|
t.Fatal("expected output module error")
|
||||||
}
|
}
|
||||||
@@ -812,6 +788,22 @@ func TestNewNormalizeConfigRejectsUnknownOutputModule(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestNewNormalizeConfigTreatsWhitespaceReportFileAsOmitted(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeTempFile(t, dir, "input.json")
|
||||||
|
output := filepath.Join(dir, "normalized.json")
|
||||||
|
|
||||||
|
opts := validNormalizeOptions(input, output)
|
||||||
|
opts.ReportFile = "\n\t "
|
||||||
|
cfg, err := NewNormalizeConfig(opts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("config failed: %v", err)
|
||||||
|
}
|
||||||
|
if cfg.ReportFile != "" {
|
||||||
|
t.Fatalf("report file = %q, want empty", cfg.ReportFile)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func assertPositiveFloatEnvValidation(t *testing.T, envName string) {
|
func assertPositiveFloatEnvValidation(t *testing.T, envName string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -832,14 +824,7 @@ func assertPositiveFloatEnvValidation(t *testing.T, envName string) {
|
|||||||
input := writeTempFile(t, dir, "input.json")
|
input := writeTempFile(t, dir, "input.json")
|
||||||
output := filepath.Join(dir, "merged.json")
|
output := filepath.Join(dir, "merged.json")
|
||||||
|
|
||||||
_, err := NewMergeConfig(MergeOptions{
|
_, err := NewMergeConfig(validMergeOptions(input, output))
|
||||||
InputFiles: []string{input},
|
|
||||||
OutputFile: output,
|
|
||||||
InputReader: DefaultInputReader,
|
|
||||||
OutputModules: DefaultOutputModules,
|
|
||||||
PreprocessingModules: DefaultPreprocessingModules,
|
|
||||||
PostprocessingModules: DefaultPostprocessingModules,
|
|
||||||
})
|
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected error")
|
t.Fatal("expected error")
|
||||||
}
|
}
|
||||||
@@ -850,6 +835,33 @@ func assertPositiveFloatEnvValidation(t *testing.T, envName string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validMergeOptions(inputFile string, outputFile string) MergeOptions {
|
||||||
|
return MergeOptions{
|
||||||
|
InputFiles: []string{inputFile},
|
||||||
|
OutputFile: outputFile,
|
||||||
|
InputReader: DefaultInputReader,
|
||||||
|
OutputModules: DefaultOutputModules,
|
||||||
|
PreprocessingModules: DefaultPreprocessingModules,
|
||||||
|
PostprocessingModules: DefaultPostprocessingModules,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func validTrimOptions(inputFile string, outputFile string) TrimOptions {
|
||||||
|
return TrimOptions{
|
||||||
|
InputFile: inputFile,
|
||||||
|
OutputFile: outputFile,
|
||||||
|
Keep: "1",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func validNormalizeOptions(inputFile string, outputFile string) NormalizeOptions {
|
||||||
|
return NormalizeOptions{
|
||||||
|
InputFile: inputFile,
|
||||||
|
OutputFile: outputFile,
|
||||||
|
OutputModules: DefaultOutputModules,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func writeTempFile(t *testing.T, dir string, name string) string {
|
func writeTempFile(t *testing.T, dir string, name string) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
|
|||||||
28
internal/jsonfile/jsonfile.go
Normal file
28
internal/jsonfile/jsonfile.go
Normal file
@@ -0,0 +1,28 @@
|
|||||||
|
package jsonfile
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Write creates or truncates path and writes deterministic indented JSON.
|
||||||
|
func Write(path string, value any) (err error) {
|
||||||
|
file, err := os.Create(path)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("create %q: %w", path, err)
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
closeErr := file.Close()
|
||||||
|
if err == nil && closeErr != nil {
|
||||||
|
err = fmt.Errorf("close %q: %w", path, closeErr)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
encoder := json.NewEncoder(file)
|
||||||
|
encoder.SetIndent("", " ")
|
||||||
|
if err := encoder.Encode(value); err != nil {
|
||||||
|
return fmt.Errorf("encode %q: %w", path, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
69
internal/jsonfile/jsonfile_test.go
Normal file
69
internal/jsonfile/jsonfile_test.go
Normal file
@@ -0,0 +1,69 @@
|
|||||||
|
package jsonfile
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestWriteFormatsWithTwoSpaceIndentAndTrailingNewline(t *testing.T) {
|
||||||
|
type payload struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Items []int `json:"items"`
|
||||||
|
}
|
||||||
|
|
||||||
|
path := filepath.Join(t.TempDir(), "out.json")
|
||||||
|
value := payload{
|
||||||
|
Name: "alpha",
|
||||||
|
Items: []int{1, 2},
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := Write(path, value); err != nil {
|
||||||
|
t.Fatalf("write failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read output: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got := string(data)
|
||||||
|
want := "{\n \"name\": \"alpha\",\n \"items\": [\n 1,\n 2\n ]\n}\n"
|
||||||
|
if got != want {
|
||||||
|
t.Fatalf("formatted JSON mismatch\nwant:\n%s\ngot:\n%s", want, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWriteProducesValidJSON(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "out.json")
|
||||||
|
|
||||||
|
value := map[string]any{
|
||||||
|
"application": "seriatim",
|
||||||
|
"segments": []map[string]any{
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"speaker": "A",
|
||||||
|
"text": "hello",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := Write(path, value); err != nil {
|
||||||
|
t.Fatalf("write failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read output: %v", err)
|
||||||
|
}
|
||||||
|
if !strings.HasSuffix(string(data), "\n") {
|
||||||
|
t.Fatalf("output missing trailing newline: %q", string(data))
|
||||||
|
}
|
||||||
|
|
||||||
|
var decoded map[string]any
|
||||||
|
if err := json.Unmarshal(data, &decoded); err != nil {
|
||||||
|
t.Fatalf("output is not valid JSON: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,5 +1,7 @@
|
|||||||
package model
|
package model
|
||||||
|
|
||||||
|
import "fmt"
|
||||||
|
|
||||||
// RawTranscript is a loaded input document before canonical normalization.
|
// RawTranscript is a loaded input document before canonical normalization.
|
||||||
type RawTranscript struct {
|
type RawTranscript struct {
|
||||||
Source string `json:"source"`
|
Source string `json:"source"`
|
||||||
@@ -61,6 +63,17 @@ type Segment struct {
|
|||||||
OverlapGroupID int `json:"overlap_group_id,omitempty"`
|
OverlapGroupID int `json:"overlap_group_id,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SegmentReference returns the best available external reference for a segment.
|
||||||
|
func SegmentReference(segment Segment) string {
|
||||||
|
if segment.Source != "" && segment.SourceSegmentIndex != nil {
|
||||||
|
return fmt.Sprintf("%s#%d", segment.Source, *segment.SourceSegmentIndex)
|
||||||
|
}
|
||||||
|
if segment.SourceRef != "" {
|
||||||
|
return segment.SourceRef
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
// Word preserves optional word-level timing data.
|
// Word preserves optional word-level timing data.
|
||||||
type Word struct {
|
type Word struct {
|
||||||
Text string `json:"text"`
|
Text string `json:"text"`
|
||||||
|
|||||||
41
internal/model/model_test.go
Normal file
41
internal/model/model_test.go
Normal file
@@ -0,0 +1,41 @@
|
|||||||
|
package model
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestSegmentReferenceUsesSourceAndIndexWhenAvailable(t *testing.T) {
|
||||||
|
index := 3
|
||||||
|
segment := Segment{
|
||||||
|
Source: "input.json",
|
||||||
|
SourceSegmentIndex: &index,
|
||||||
|
SourceRef: "word-run:1:2:3",
|
||||||
|
}
|
||||||
|
|
||||||
|
got := SegmentReference(segment)
|
||||||
|
want := "input.json#3"
|
||||||
|
if got != want {
|
||||||
|
t.Fatalf("reference = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSegmentReferenceFallsBackToSourceRef(t *testing.T) {
|
||||||
|
segment := Segment{
|
||||||
|
Source: "input.json",
|
||||||
|
SourceRef: "coalesce:2",
|
||||||
|
}
|
||||||
|
|
||||||
|
got := SegmentReference(segment)
|
||||||
|
want := "coalesce:2"
|
||||||
|
if got != want {
|
||||||
|
t.Fatalf("reference = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSegmentReferenceReturnsEmptyWhenNoReferenceFieldsPresent(t *testing.T) {
|
||||||
|
segment := Segment{
|
||||||
|
Source: "input.json",
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := SegmentReference(segment); got != "" {
|
||||||
|
t.Fatalf("reference = %q, want empty", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -4,12 +4,12 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/seriatim/internal/artifact"
|
"gitea.maximumdirect.net/eric/seriatim/internal/artifact"
|
||||||
"gitea.maximumdirect.net/eric/seriatim/internal/buildinfo"
|
"gitea.maximumdirect.net/eric/seriatim/internal/buildinfo"
|
||||||
"gitea.maximumdirect.net/eric/seriatim/internal/config"
|
"gitea.maximumdirect.net/eric/seriatim/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/internal/jsonfile"
|
||||||
"gitea.maximumdirect.net/eric/seriatim/internal/report"
|
"gitea.maximumdirect.net/eric/seriatim/internal/report"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -47,7 +47,7 @@ func Run(ctx context.Context, cfg config.NormalizeConfig) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := writeOutputJSON(cfg.OutputFile, built.Output); err != nil {
|
if err := jsonfile.Write(cfg.OutputFile, built.Output); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -118,18 +118,3 @@ func Run(ctx context.Context, cfg config.NormalizeConfig) error {
|
|||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeOutputJSON(path string, value any) error {
|
|
||||||
file, err := os.Create(path)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer file.Close()
|
|
||||||
|
|
||||||
encoder := json.NewEncoder(file)
|
|
||||||
encoder.SetIndent("", " ")
|
|
||||||
if err := encoder.Encode(value); err != nil {
|
|
||||||
return fmt.Errorf("encode normalize output JSON: %w", err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
package overlap
|
package overlap
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
|
||||||
"sort"
|
"sort"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/seriatim/internal/model"
|
"gitea.maximumdirect.net/eric/seriatim/internal/model"
|
||||||
@@ -121,13 +120,7 @@ func distinctSpeakers(segments []model.Segment, indices []int) []string {
|
|||||||
|
|
||||||
// SegmentRef returns the stable overlap reference for a segment.
|
// SegmentRef returns the stable overlap reference for a segment.
|
||||||
func SegmentRef(segment model.Segment) string {
|
func SegmentRef(segment model.Segment) string {
|
||||||
if segment.SourceSegmentIndex != nil {
|
return model.SegmentReference(segment)
|
||||||
return fmt.Sprintf("%s#%d", segment.Source, *segment.SourceSegmentIndex)
|
|
||||||
}
|
|
||||||
if segment.SourceRef != "" {
|
|
||||||
return segment.SourceRef
|
|
||||||
}
|
|
||||||
return segment.Source
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func clearExisting(in *model.MergedTranscript) {
|
func clearExisting(in *model.MergedTranscript) {
|
||||||
|
|||||||
@@ -1,9 +1,6 @@
|
|||||||
package report
|
package report
|
||||||
|
|
||||||
import (
|
import "gitea.maximumdirect.net/eric/seriatim/internal/jsonfile"
|
||||||
"encoding/json"
|
|
||||||
"os"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Severity classifies report events.
|
// Severity classifies report events.
|
||||||
type Severity string
|
type Severity string
|
||||||
@@ -62,13 +59,5 @@ func Warning(stage string, module string, message string) Event {
|
|||||||
|
|
||||||
// WriteJSON writes a deterministic JSON report.
|
// WriteJSON writes a deterministic JSON report.
|
||||||
func WriteJSON(path string, rpt Report) error {
|
func WriteJSON(path string, rpt Report) error {
|
||||||
file, err := os.Create(path)
|
return jsonfile.Write(path, rpt)
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer file.Close()
|
|
||||||
|
|
||||||
enc := json.NewEncoder(file)
|
|
||||||
enc.SetIndent("", " ")
|
|
||||||
return enc.Encode(rpt)
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -44,54 +44,29 @@ type MinimalResult struct {
|
|||||||
RemovedIDs []int
|
RemovedIDs []int
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type projection struct {
|
||||||
|
retainedIndexes []int
|
||||||
|
oldToNewID map[int]int
|
||||||
|
removedIDs []int
|
||||||
|
}
|
||||||
|
|
||||||
// Apply trims a full seriatim output transcript by segment ID.
|
// Apply trims a full seriatim output transcript by segment ID.
|
||||||
func Apply(input schema.Transcript, opts Options) (Result, error) {
|
func Apply(input schema.Transcript, opts Options) (Result, error) {
|
||||||
if err := validateMode(opts.Mode); err != nil {
|
|
||||||
return Result{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
selected := opts.Selector.IDs()
|
|
||||||
if len(selected) == 0 {
|
|
||||||
return Result{}, fmt.Errorf("selector cannot be empty")
|
|
||||||
}
|
|
||||||
|
|
||||||
inputIDs := make([]int, len(input.Segments))
|
inputIDs := make([]int, len(input.Segments))
|
||||||
for index, segment := range input.Segments {
|
for index, segment := range input.Segments {
|
||||||
inputIDs[index] = segment.ID
|
inputIDs[index] = segment.ID
|
||||||
}
|
}
|
||||||
|
proj, err := projectSegmentIDs(inputIDs, opts)
|
||||||
idIndex, err := validateInputIDs(inputIDs)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Result{}, err
|
return Result{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := validateSelectedIDsExist(selected, idIndex); err != nil {
|
kept := make([]schema.Segment, len(proj.retainedIndexes))
|
||||||
return Result{}, err
|
for outputIndex, inputIndex := range proj.retainedIndexes {
|
||||||
}
|
rewritten := copySegment(input.Segments[inputIndex])
|
||||||
|
rewritten.ID = outputIndex + 1
|
||||||
kept := make([]schema.Segment, 0, len(input.Segments))
|
|
||||||
removed := make([]int, 0, len(input.Segments))
|
|
||||||
oldToNew := make(map[int]int, len(input.Segments))
|
|
||||||
for _, segment := range input.Segments {
|
|
||||||
keep := opts.Mode == ModeKeep && opts.Selector.Contains(segment.ID)
|
|
||||||
if opts.Mode == ModeRemove {
|
|
||||||
keep = !opts.Selector.Contains(segment.ID)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !keep {
|
|
||||||
removed = append(removed, segment.ID)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
rewritten := copySegment(segment)
|
|
||||||
rewritten.ID = len(kept) + 1
|
|
||||||
rewritten.OverlapGroupID = 0
|
rewritten.OverlapGroupID = 0
|
||||||
kept = append(kept, rewritten)
|
kept[outputIndex] = rewritten
|
||||||
oldToNew[segment.ID] = rewritten.ID
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(kept) == 0 && !opts.AllowEmpty {
|
|
||||||
return Result{}, fmt.Errorf("trim operation produced an empty transcript; set AllowEmpty to true to permit this")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
kept, groups := recomputeOverlapGroups(kept)
|
kept, groups := recomputeOverlapGroups(kept)
|
||||||
@@ -104,62 +79,35 @@ func Apply(input schema.Transcript, opts Options) (Result, error) {
|
|||||||
out.OverlapGroups = groups
|
out.OverlapGroups = groups
|
||||||
return Result{
|
return Result{
|
||||||
Transcript: out,
|
Transcript: out,
|
||||||
OldToNewID: oldToNew,
|
OldToNewID: proj.oldToNewID,
|
||||||
RemovedIDs: removed,
|
RemovedIDs: proj.removedIDs,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ApplyIntermediate trims an intermediate seriatim output transcript by
|
// ApplyIntermediate trims an intermediate seriatim output transcript by
|
||||||
// segment ID.
|
// segment ID.
|
||||||
func ApplyIntermediate(input schema.IntermediateTranscript, opts Options) (IntermediateResult, error) {
|
func ApplyIntermediate(input schema.IntermediateTranscript, opts Options) (IntermediateResult, error) {
|
||||||
if err := validateMode(opts.Mode); err != nil {
|
|
||||||
return IntermediateResult{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
selected := opts.Selector.IDs()
|
|
||||||
if len(selected) == 0 {
|
|
||||||
return IntermediateResult{}, fmt.Errorf("selector cannot be empty")
|
|
||||||
}
|
|
||||||
|
|
||||||
inputIDs := make([]int, len(input.Segments))
|
inputIDs := make([]int, len(input.Segments))
|
||||||
for index, segment := range input.Segments {
|
for index, segment := range input.Segments {
|
||||||
inputIDs[index] = segment.ID
|
inputIDs[index] = segment.ID
|
||||||
}
|
}
|
||||||
idIndex, err := validateInputIDs(inputIDs)
|
proj, err := projectSegmentIDs(inputIDs, opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return IntermediateResult{}, err
|
return IntermediateResult{}, err
|
||||||
}
|
}
|
||||||
if err := validateSelectedIDsExist(selected, idIndex); err != nil {
|
|
||||||
return IntermediateResult{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
kept := make([]schema.IntermediateSegment, 0, len(input.Segments))
|
|
||||||
removed := make([]int, 0, len(input.Segments))
|
|
||||||
oldToNew := make(map[int]int, len(input.Segments))
|
|
||||||
for _, segment := range input.Segments {
|
|
||||||
keep := opts.Mode == ModeKeep && opts.Selector.Contains(segment.ID)
|
|
||||||
if opts.Mode == ModeRemove {
|
|
||||||
keep = !opts.Selector.Contains(segment.ID)
|
|
||||||
}
|
|
||||||
if !keep {
|
|
||||||
removed = append(removed, segment.ID)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
|
kept := make([]schema.IntermediateSegment, len(proj.retainedIndexes))
|
||||||
|
for outputIndex, inputIndex := range proj.retainedIndexes {
|
||||||
|
segment := input.Segments[inputIndex]
|
||||||
rewritten := schema.IntermediateSegment{
|
rewritten := schema.IntermediateSegment{
|
||||||
ID: len(kept) + 1,
|
ID: outputIndex + 1,
|
||||||
Start: segment.Start,
|
Start: segment.Start,
|
||||||
End: segment.End,
|
End: segment.End,
|
||||||
Speaker: segment.Speaker,
|
Speaker: segment.Speaker,
|
||||||
Text: segment.Text,
|
Text: segment.Text,
|
||||||
Categories: append([]string(nil), segment.Categories...),
|
Categories: append([]string(nil), segment.Categories...),
|
||||||
}
|
}
|
||||||
kept = append(kept, rewritten)
|
kept[outputIndex] = rewritten
|
||||||
oldToNew[segment.ID] = rewritten.ID
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(kept) == 0 && !opts.AllowEmpty {
|
|
||||||
return IntermediateResult{}, fmt.Errorf("trim operation produced an empty transcript; set AllowEmpty to true to permit this")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return IntermediateResult{
|
return IntermediateResult{
|
||||||
@@ -171,60 +119,33 @@ func ApplyIntermediate(input schema.IntermediateTranscript, opts Options) (Inter
|
|||||||
},
|
},
|
||||||
Segments: kept,
|
Segments: kept,
|
||||||
},
|
},
|
||||||
OldToNewID: oldToNew,
|
OldToNewID: proj.oldToNewID,
|
||||||
RemovedIDs: removed,
|
RemovedIDs: proj.removedIDs,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ApplyMinimal trims a minimal seriatim output transcript by segment ID.
|
// ApplyMinimal trims a minimal seriatim output transcript by segment ID.
|
||||||
func ApplyMinimal(input schema.MinimalTranscript, opts Options) (MinimalResult, error) {
|
func ApplyMinimal(input schema.MinimalTranscript, opts Options) (MinimalResult, error) {
|
||||||
if err := validateMode(opts.Mode); err != nil {
|
|
||||||
return MinimalResult{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
selected := opts.Selector.IDs()
|
|
||||||
if len(selected) == 0 {
|
|
||||||
return MinimalResult{}, fmt.Errorf("selector cannot be empty")
|
|
||||||
}
|
|
||||||
|
|
||||||
inputIDs := make([]int, len(input.Segments))
|
inputIDs := make([]int, len(input.Segments))
|
||||||
for index, segment := range input.Segments {
|
for index, segment := range input.Segments {
|
||||||
inputIDs[index] = segment.ID
|
inputIDs[index] = segment.ID
|
||||||
}
|
}
|
||||||
idIndex, err := validateInputIDs(inputIDs)
|
proj, err := projectSegmentIDs(inputIDs, opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return MinimalResult{}, err
|
return MinimalResult{}, err
|
||||||
}
|
}
|
||||||
if err := validateSelectedIDsExist(selected, idIndex); err != nil {
|
|
||||||
return MinimalResult{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
kept := make([]schema.MinimalSegment, 0, len(input.Segments))
|
|
||||||
removed := make([]int, 0, len(input.Segments))
|
|
||||||
oldToNew := make(map[int]int, len(input.Segments))
|
|
||||||
for _, segment := range input.Segments {
|
|
||||||
keep := opts.Mode == ModeKeep && opts.Selector.Contains(segment.ID)
|
|
||||||
if opts.Mode == ModeRemove {
|
|
||||||
keep = !opts.Selector.Contains(segment.ID)
|
|
||||||
}
|
|
||||||
if !keep {
|
|
||||||
removed = append(removed, segment.ID)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
|
kept := make([]schema.MinimalSegment, len(proj.retainedIndexes))
|
||||||
|
for outputIndex, inputIndex := range proj.retainedIndexes {
|
||||||
|
segment := input.Segments[inputIndex]
|
||||||
rewritten := schema.MinimalSegment{
|
rewritten := schema.MinimalSegment{
|
||||||
ID: len(kept) + 1,
|
ID: outputIndex + 1,
|
||||||
Start: segment.Start,
|
Start: segment.Start,
|
||||||
End: segment.End,
|
End: segment.End,
|
||||||
Speaker: segment.Speaker,
|
Speaker: segment.Speaker,
|
||||||
Text: segment.Text,
|
Text: segment.Text,
|
||||||
}
|
}
|
||||||
kept = append(kept, rewritten)
|
kept[outputIndex] = rewritten
|
||||||
oldToNew[segment.ID] = rewritten.ID
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(kept) == 0 && !opts.AllowEmpty {
|
|
||||||
return MinimalResult{}, fmt.Errorf("trim operation produced an empty transcript; set AllowEmpty to true to permit this")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return MinimalResult{
|
return MinimalResult{
|
||||||
@@ -236,11 +157,53 @@ func ApplyMinimal(input schema.MinimalTranscript, opts Options) (MinimalResult,
|
|||||||
},
|
},
|
||||||
Segments: kept,
|
Segments: kept,
|
||||||
},
|
},
|
||||||
OldToNewID: oldToNew,
|
OldToNewID: proj.oldToNewID,
|
||||||
RemovedIDs: removed,
|
RemovedIDs: proj.removedIDs,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func projectSegmentIDs(ids []int, opts Options) (projection, error) {
|
||||||
|
if err := validateMode(opts.Mode); err != nil {
|
||||||
|
return projection{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
selected := opts.Selector.IDs()
|
||||||
|
if len(selected) == 0 {
|
||||||
|
return projection{}, fmt.Errorf("selector cannot be empty")
|
||||||
|
}
|
||||||
|
|
||||||
|
idIndex, err := validateInputIDs(ids)
|
||||||
|
if err != nil {
|
||||||
|
return projection{}, err
|
||||||
|
}
|
||||||
|
if err := validateSelectedIDsExist(selected, idIndex); err != nil {
|
||||||
|
return projection{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
result := projection{
|
||||||
|
retainedIndexes: make([]int, 0, len(ids)),
|
||||||
|
oldToNewID: make(map[int]int, len(ids)),
|
||||||
|
removedIDs: make([]int, 0, len(ids)),
|
||||||
|
}
|
||||||
|
for index, id := range ids {
|
||||||
|
keep := opts.Mode == ModeKeep && opts.Selector.Contains(id)
|
||||||
|
if opts.Mode == ModeRemove {
|
||||||
|
keep = !opts.Selector.Contains(id)
|
||||||
|
}
|
||||||
|
if !keep {
|
||||||
|
result.removedIDs = append(result.removedIDs, id)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
result.retainedIndexes = append(result.retainedIndexes, index)
|
||||||
|
result.oldToNewID[id] = len(result.retainedIndexes)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(result.retainedIndexes) == 0 && !opts.AllowEmpty {
|
||||||
|
return projection{}, fmt.Errorf("trim operation produced an empty transcript; set AllowEmpty to true to permit this")
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
func validateMode(mode Mode) error {
|
func validateMode(mode Mode) error {
|
||||||
switch mode {
|
switch mode {
|
||||||
case ModeKeep, ModeRemove:
|
case ModeKeep, ModeRemove:
|
||||||
|
|||||||
@@ -399,6 +399,106 @@ func TestApplyMinimalDoesNotIncludeOverlapGroups(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestApplySelectorPolicyIsSharedAcrossSchemas(t *testing.T) {
|
||||||
|
type testCase struct {
|
||||||
|
name string
|
||||||
|
opts Options
|
||||||
|
wantTexts []string
|
||||||
|
wantOldToNew map[int]int
|
||||||
|
wantRemoved []int
|
||||||
|
wantSegmentCount int
|
||||||
|
wantErrorSubstring string
|
||||||
|
}
|
||||||
|
|
||||||
|
cases := []testCase{
|
||||||
|
{
|
||||||
|
name: "keep preserves input order regardless of selector order",
|
||||||
|
opts: Options{Mode: ModeKeep, Selector: mustParseSelector(t, "4,1,3")},
|
||||||
|
wantTexts: []string{"alpha", "gamma", "delta"},
|
||||||
|
wantOldToNew: map[int]int{1: 1, 3: 2, 4: 3},
|
||||||
|
wantRemoved: []int{2},
|
||||||
|
wantSegmentCount: 3,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "remove reports deterministic renumbering metadata",
|
||||||
|
opts: Options{Mode: ModeRemove, Selector: mustParseSelector(t, "2,4")},
|
||||||
|
wantTexts: []string{"alpha", "gamma"},
|
||||||
|
wantOldToNew: map[int]int{1: 1, 3: 2},
|
||||||
|
wantRemoved: []int{2, 4},
|
||||||
|
wantSegmentCount: 2,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "missing selected id returns error",
|
||||||
|
opts: Options{Mode: ModeKeep, Selector: mustParseSelector(t, "9")},
|
||||||
|
wantErrorSubstring: "does not exist",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty selector returns error",
|
||||||
|
opts: Options{Mode: ModeKeep, Selector: Selector{}},
|
||||||
|
wantErrorSubstring: "selector cannot be empty",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid mode returns error",
|
||||||
|
opts: Options{Mode: Mode("bad"), Selector: mustParseSelector(t, "1")},
|
||||||
|
wantErrorSubstring: `invalid trim mode "bad"`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty output blocked when allow empty is false",
|
||||||
|
opts: Options{Mode: ModeRemove, Selector: mustParseSelector(t, "1-4")},
|
||||||
|
wantErrorSubstring: "empty transcript",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty output allowed when allow empty is true",
|
||||||
|
opts: Options{Mode: ModeRemove, Selector: mustParseSelector(t, "1-4"), AllowEmpty: true},
|
||||||
|
wantTexts: []string{},
|
||||||
|
wantOldToNew: map[int]int{},
|
||||||
|
wantRemoved: []int{1, 2, 3, 4},
|
||||||
|
wantSegmentCount: 0,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range cases {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
fullInput := fullTranscriptFixture()
|
||||||
|
intermediateInput := intermediateFixture()
|
||||||
|
minimalInput := minimalFixture()
|
||||||
|
|
||||||
|
fullResult, fullErr := Apply(fullInput, test.opts)
|
||||||
|
intermediateResult, intermediateErr := ApplyIntermediate(intermediateInput, test.opts)
|
||||||
|
minimalResult, minimalErr := ApplyMinimal(minimalInput, test.opts)
|
||||||
|
|
||||||
|
if test.wantErrorSubstring != "" {
|
||||||
|
assertErrorContains(t, fullErr, test.wantErrorSubstring)
|
||||||
|
assertErrorContains(t, intermediateErr, test.wantErrorSubstring)
|
||||||
|
assertErrorContains(t, minimalErr, test.wantErrorSubstring)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if fullErr != nil {
|
||||||
|
t.Fatalf("apply full failed: %v", fullErr)
|
||||||
|
}
|
||||||
|
if intermediateErr != nil {
|
||||||
|
t.Fatalf("apply intermediate failed: %v", intermediateErr)
|
||||||
|
}
|
||||||
|
if minimalErr != nil {
|
||||||
|
t.Fatalf("apply minimal failed: %v", minimalErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertIntSlice(t, extractFullIDs(fullResult.Transcript.Segments), extractSequentialIDs(test.wantSegmentCount))
|
||||||
|
assertIntSlice(t, extractIntermediateIDs(intermediateResult.Transcript.Segments), extractSequentialIDs(test.wantSegmentCount))
|
||||||
|
assertIntSlice(t, extractMinimalIDs(minimalResult.Transcript.Segments), extractSequentialIDs(test.wantSegmentCount))
|
||||||
|
assertStringSlice(t, extractFullTexts(fullResult.Transcript.Segments), test.wantTexts)
|
||||||
|
assertStringSlice(t, extractIntermediateTexts(intermediateResult.Transcript.Segments), test.wantTexts)
|
||||||
|
assertStringSlice(t, extractMinimalTexts(minimalResult.Transcript.Segments), test.wantTexts)
|
||||||
|
assertIntMap(t, fullResult.OldToNewID, test.wantOldToNew)
|
||||||
|
assertIntMap(t, intermediateResult.OldToNewID, test.wantOldToNew)
|
||||||
|
assertIntMap(t, minimalResult.OldToNewID, test.wantOldToNew)
|
||||||
|
assertIntSlice(t, fullResult.RemovedIDs, test.wantRemoved)
|
||||||
|
assertIntSlice(t, intermediateResult.RemovedIDs, test.wantRemoved)
|
||||||
|
assertIntSlice(t, minimalResult.RemovedIDs, test.wantRemoved)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestApplyOutputInvariantsValidAfterRenumberAndOverlapRecompute(t *testing.T) {
|
func TestApplyOutputInvariantsValidAfterRenumberAndOverlapRecompute(t *testing.T) {
|
||||||
input := overlapTranscriptFixture()
|
input := overlapTranscriptFixture()
|
||||||
selector := mustParseSelector(t, "2,1")
|
selector := mustParseSelector(t, "2,1")
|
||||||
@@ -666,3 +766,108 @@ func equalStringSlices(got []string, want []string) bool {
|
|||||||
}
|
}
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func assertErrorContains(t *testing.T, err error, substring string) {
|
||||||
|
t.Helper()
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("expected error containing %q", substring)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), substring) {
|
||||||
|
t.Fatalf("error %q does not contain %q", err.Error(), substring)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertStringSlice(t *testing.T, got []string, want []string) {
|
||||||
|
t.Helper()
|
||||||
|
if !equalStringSlices(got, want) {
|
||||||
|
t.Fatalf("slice = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractSequentialIDs(count int) []int {
|
||||||
|
ids := make([]int, count)
|
||||||
|
for index := range ids {
|
||||||
|
ids[index] = index + 1
|
||||||
|
}
|
||||||
|
return ids
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractFullIDs(segments []schema.Segment) []int {
|
||||||
|
ids := make([]int, len(segments))
|
||||||
|
for index, segment := range segments {
|
||||||
|
ids[index] = segment.ID
|
||||||
|
}
|
||||||
|
return ids
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractIntermediateIDs(segments []schema.IntermediateSegment) []int {
|
||||||
|
ids := make([]int, len(segments))
|
||||||
|
for index, segment := range segments {
|
||||||
|
ids[index] = segment.ID
|
||||||
|
}
|
||||||
|
return ids
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractMinimalIDs(segments []schema.MinimalSegment) []int {
|
||||||
|
ids := make([]int, len(segments))
|
||||||
|
for index, segment := range segments {
|
||||||
|
ids[index] = segment.ID
|
||||||
|
}
|
||||||
|
return ids
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractFullTexts(segments []schema.Segment) []string {
|
||||||
|
texts := make([]string, len(segments))
|
||||||
|
for index, segment := range segments {
|
||||||
|
texts[index] = segment.Text
|
||||||
|
}
|
||||||
|
return texts
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractIntermediateTexts(segments []schema.IntermediateSegment) []string {
|
||||||
|
texts := make([]string, len(segments))
|
||||||
|
for index, segment := range segments {
|
||||||
|
texts[index] = segment.Text
|
||||||
|
}
|
||||||
|
return texts
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractMinimalTexts(segments []schema.MinimalSegment) []string {
|
||||||
|
texts := make([]string, len(segments))
|
||||||
|
for index, segment := range segments {
|
||||||
|
texts[index] = segment.Text
|
||||||
|
}
|
||||||
|
return texts
|
||||||
|
}
|
||||||
|
|
||||||
|
func intermediateFixture() schema.IntermediateTranscript {
|
||||||
|
return schema.IntermediateTranscript{
|
||||||
|
Metadata: schema.IntermediateMetadata{
|
||||||
|
Application: "seriatim",
|
||||||
|
Version: "v-test",
|
||||||
|
OutputSchema: schema.OutputSchemaIntermediate,
|
||||||
|
},
|
||||||
|
Segments: []schema.IntermediateSegment{
|
||||||
|
{ID: 1, Start: 1, End: 2, Speaker: "Alice", Text: "alpha", Categories: []string{"word-run"}},
|
||||||
|
{ID: 2, Start: 2, End: 3, Speaker: "Bob", Text: "beta", Categories: []string{"filler", "backchannel"}},
|
||||||
|
{ID: 3, Start: 3, End: 4, Speaker: "Carol", Text: "gamma", Categories: []string{"normal"}},
|
||||||
|
{ID: 4, Start: 4, End: 5, Speaker: "Dan", Text: "delta", Categories: []string{"normal"}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func minimalFixture() schema.MinimalTranscript {
|
||||||
|
return schema.MinimalTranscript{
|
||||||
|
Metadata: schema.MinimalMetadata{
|
||||||
|
Application: "seriatim",
|
||||||
|
Version: "v-test",
|
||||||
|
OutputSchema: schema.OutputSchemaMinimal,
|
||||||
|
},
|
||||||
|
Segments: []schema.MinimalSegment{
|
||||||
|
{ID: 1, Start: 1, End: 2, Speaker: "Alice", Text: "alpha"},
|
||||||
|
{ID: 2, Start: 2, End: 3, Speaker: "Bob", Text: "beta"},
|
||||||
|
{ID: 3, Start: 3, End: 4, Speaker: "Carol", Text: "gamma"},
|
||||||
|
{ID: 4, Start: 4, End: 5, Speaker: "Dan", Text: "delta"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -8,9 +8,9 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
SchemaMinimal = "seriatim-minimal"
|
SchemaMinimal = schema.OutputSchemaMinimal
|
||||||
SchemaIntermediate = "seriatim-intermediate"
|
SchemaIntermediate = schema.OutputSchemaIntermediate
|
||||||
SchemaFull = "seriatim-full"
|
SchemaFull = schema.OutputSchemaFull
|
||||||
)
|
)
|
||||||
|
|
||||||
// Artifact stores a parsed seriatim output artifact of one supported schema.
|
// Artifact stores a parsed seriatim output artifact of one supported schema.
|
||||||
@@ -73,20 +73,23 @@ func ParseArtifactJSON(data []byte) (Artifact, error) {
|
|||||||
func ValidateArtifact(artifact Artifact) error {
|
func ValidateArtifact(artifact Artifact) error {
|
||||||
switch artifact.Schema {
|
switch artifact.Schema {
|
||||||
case SchemaFull:
|
case SchemaFull:
|
||||||
if artifact.Full == nil {
|
payload, err := artifact.fullPayload()
|
||||||
return fmt.Errorf("full artifact payload is missing")
|
if err != nil {
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
return schema.ValidateTranscript(*artifact.Full)
|
return schema.ValidateTranscript(*payload)
|
||||||
case SchemaIntermediate:
|
case SchemaIntermediate:
|
||||||
if artifact.Intermediate == nil {
|
payload, err := artifact.intermediatePayload()
|
||||||
return fmt.Errorf("intermediate artifact payload is missing")
|
if err != nil {
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
return schema.ValidateIntermediateTranscript(*artifact.Intermediate)
|
return schema.ValidateIntermediateTranscript(*payload)
|
||||||
case SchemaMinimal:
|
case SchemaMinimal:
|
||||||
if artifact.Minimal == nil {
|
payload, err := artifact.minimalPayload()
|
||||||
return fmt.Errorf("minimal artifact payload is missing")
|
if err != nil {
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
return schema.ValidateMinimalTranscript(*artifact.Minimal)
|
return schema.ValidateMinimalTranscript(*payload)
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("unsupported artifact schema %q", artifact.Schema)
|
return fmt.Errorf("unsupported artifact schema %q", artifact.Schema)
|
||||||
}
|
}
|
||||||
@@ -188,10 +191,11 @@ func (artifact Artifact) Version() string {
|
|||||||
func ApplyArtifact(input Artifact, opts Options) (ApplyArtifactResult, error) {
|
func ApplyArtifact(input Artifact, opts Options) (ApplyArtifactResult, error) {
|
||||||
switch input.Schema {
|
switch input.Schema {
|
||||||
case SchemaFull:
|
case SchemaFull:
|
||||||
if input.Full == nil {
|
payload, err := input.fullPayload()
|
||||||
return ApplyArtifactResult{}, fmt.Errorf("full artifact payload is missing")
|
if err != nil {
|
||||||
|
return ApplyArtifactResult{}, err
|
||||||
}
|
}
|
||||||
result, err := Apply(*input.Full, opts)
|
result, err := Apply(*payload, opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ApplyArtifactResult{}, err
|
return ApplyArtifactResult{}, err
|
||||||
}
|
}
|
||||||
@@ -206,10 +210,11 @@ func ApplyArtifact(input Artifact, opts Options) (ApplyArtifactResult, error) {
|
|||||||
OverlapGroupsRecomputed: true,
|
OverlapGroupsRecomputed: true,
|
||||||
}, nil
|
}, nil
|
||||||
case SchemaIntermediate:
|
case SchemaIntermediate:
|
||||||
if input.Intermediate == nil {
|
payload, err := input.intermediatePayload()
|
||||||
return ApplyArtifactResult{}, fmt.Errorf("intermediate artifact payload is missing")
|
if err != nil {
|
||||||
|
return ApplyArtifactResult{}, err
|
||||||
}
|
}
|
||||||
result, err := ApplyIntermediate(*input.Intermediate, opts)
|
result, err := ApplyIntermediate(*payload, opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ApplyArtifactResult{}, err
|
return ApplyArtifactResult{}, err
|
||||||
}
|
}
|
||||||
@@ -224,10 +229,11 @@ func ApplyArtifact(input Artifact, opts Options) (ApplyArtifactResult, error) {
|
|||||||
OverlapGroupsRecomputed: false,
|
OverlapGroupsRecomputed: false,
|
||||||
}, nil
|
}, nil
|
||||||
case SchemaMinimal:
|
case SchemaMinimal:
|
||||||
if input.Minimal == nil {
|
payload, err := input.minimalPayload()
|
||||||
return ApplyArtifactResult{}, fmt.Errorf("minimal artifact payload is missing")
|
if err != nil {
|
||||||
|
return ApplyArtifactResult{}, err
|
||||||
}
|
}
|
||||||
result, err := ApplyMinimal(*input.Minimal, opts)
|
result, err := ApplyMinimal(*payload, opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ApplyArtifactResult{}, err
|
return ApplyArtifactResult{}, err
|
||||||
}
|
}
|
||||||
@@ -254,18 +260,19 @@ func ConvertArtifact(input Artifact, outputSchema string) (Artifact, error) {
|
|||||||
|
|
||||||
switch input.Schema {
|
switch input.Schema {
|
||||||
case SchemaFull:
|
case SchemaFull:
|
||||||
if input.Full == nil {
|
payload, err := input.fullPayload()
|
||||||
return Artifact{}, fmt.Errorf("full artifact payload is missing")
|
if err != nil {
|
||||||
|
return Artifact{}, err
|
||||||
}
|
}
|
||||||
switch outputSchema {
|
switch outputSchema {
|
||||||
case SchemaIntermediate:
|
case SchemaIntermediate:
|
||||||
out := intermediateFromFull(*input.Full)
|
out := intermediateFromFull(*payload)
|
||||||
return Artifact{
|
return Artifact{
|
||||||
Schema: SchemaIntermediate,
|
Schema: SchemaIntermediate,
|
||||||
Intermediate: &out,
|
Intermediate: &out,
|
||||||
}, nil
|
}, nil
|
||||||
case SchemaMinimal:
|
case SchemaMinimal:
|
||||||
out := minimalFromFull(*input.Full)
|
out := minimalFromFull(*payload)
|
||||||
return Artifact{
|
return Artifact{
|
||||||
Schema: SchemaMinimal,
|
Schema: SchemaMinimal,
|
||||||
Minimal: &out,
|
Minimal: &out,
|
||||||
@@ -274,12 +281,13 @@ func ConvertArtifact(input Artifact, outputSchema string) (Artifact, error) {
|
|||||||
return Artifact{}, fmt.Errorf("unsupported output schema %q", outputSchema)
|
return Artifact{}, fmt.Errorf("unsupported output schema %q", outputSchema)
|
||||||
}
|
}
|
||||||
case SchemaIntermediate:
|
case SchemaIntermediate:
|
||||||
if input.Intermediate == nil {
|
payload, err := input.intermediatePayload()
|
||||||
return Artifact{}, fmt.Errorf("intermediate artifact payload is missing")
|
if err != nil {
|
||||||
|
return Artifact{}, err
|
||||||
}
|
}
|
||||||
switch outputSchema {
|
switch outputSchema {
|
||||||
case SchemaMinimal:
|
case SchemaMinimal:
|
||||||
out := minimalFromIntermediate(*input.Intermediate)
|
out := minimalFromIntermediate(*payload)
|
||||||
return Artifact{
|
return Artifact{
|
||||||
Schema: SchemaMinimal,
|
Schema: SchemaMinimal,
|
||||||
Minimal: &out,
|
Minimal: &out,
|
||||||
@@ -290,12 +298,13 @@ func ConvertArtifact(input Artifact, outputSchema string) (Artifact, error) {
|
|||||||
return Artifact{}, fmt.Errorf("unsupported output schema %q", outputSchema)
|
return Artifact{}, fmt.Errorf("unsupported output schema %q", outputSchema)
|
||||||
}
|
}
|
||||||
case SchemaMinimal:
|
case SchemaMinimal:
|
||||||
if input.Minimal == nil {
|
payload, err := input.minimalPayload()
|
||||||
return Artifact{}, fmt.Errorf("minimal artifact payload is missing")
|
if err != nil {
|
||||||
|
return Artifact{}, err
|
||||||
}
|
}
|
||||||
switch outputSchema {
|
switch outputSchema {
|
||||||
case SchemaIntermediate:
|
case SchemaIntermediate:
|
||||||
out := intermediateFromMinimal(*input.Minimal)
|
out := intermediateFromMinimal(*payload)
|
||||||
return Artifact{
|
return Artifact{
|
||||||
Schema: SchemaIntermediate,
|
Schema: SchemaIntermediate,
|
||||||
Intermediate: &out,
|
Intermediate: &out,
|
||||||
@@ -310,6 +319,27 @@ func ConvertArtifact(input Artifact, outputSchema string) (Artifact, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (artifact Artifact) fullPayload() (*schema.Transcript, error) {
|
||||||
|
if artifact.Full == nil {
|
||||||
|
return nil, fmt.Errorf("full artifact payload is missing")
|
||||||
|
}
|
||||||
|
return artifact.Full, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (artifact Artifact) intermediatePayload() (*schema.IntermediateTranscript, error) {
|
||||||
|
if artifact.Intermediate == nil {
|
||||||
|
return nil, fmt.Errorf("intermediate artifact payload is missing")
|
||||||
|
}
|
||||||
|
return artifact.Intermediate, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (artifact Artifact) minimalPayload() (*schema.MinimalTranscript, error) {
|
||||||
|
if artifact.Minimal == nil {
|
||||||
|
return nil, fmt.Errorf("minimal artifact payload is missing")
|
||||||
|
}
|
||||||
|
return artifact.Minimal, nil
|
||||||
|
}
|
||||||
|
|
||||||
func intermediateFromFull(input schema.Transcript) schema.IntermediateTranscript {
|
func intermediateFromFull(input schema.Transcript) schema.IntermediateTranscript {
|
||||||
segments := make([]schema.IntermediateSegment, len(input.Segments))
|
segments := make([]schema.IntermediateSegment, len(input.Segments))
|
||||||
for index, segment := range input.Segments {
|
for index, segment := range input.Segments {
|
||||||
|
|||||||
@@ -128,6 +128,61 @@ func TestConvertArtifactMinimalToFullFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestValidateArtifactRejectsMissingPayloads(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
artifact Artifact
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "full",
|
||||||
|
artifact: Artifact{Schema: SchemaFull},
|
||||||
|
want: "full artifact payload is missing",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "intermediate",
|
||||||
|
artifact: Artifact{Schema: SchemaIntermediate},
|
||||||
|
want: "intermediate artifact payload is missing",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "minimal",
|
||||||
|
artifact: Artifact{Schema: SchemaMinimal},
|
||||||
|
want: "minimal artifact payload is missing",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
err := ValidateArtifact(test.artifact)
|
||||||
|
assertErrorContains(t, err, test.want)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyArtifactRejectsMissingPayload(t *testing.T) {
|
||||||
|
_, err := ApplyArtifact(Artifact{Schema: SchemaFull}, Options{})
|
||||||
|
assertErrorContains(t, err, "full artifact payload is missing")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvertArtifactRejectsMissingPayloadWhenConversionRequested(t *testing.T) {
|
||||||
|
_, err := ConvertArtifact(Artifact{Schema: SchemaFull}, SchemaMinimal)
|
||||||
|
assertErrorContains(t, err, "full artifact payload is missing")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvertArtifactSameSchemaDoesNotRequirePayload(t *testing.T) {
|
||||||
|
artifact := Artifact{Schema: SchemaFull}
|
||||||
|
converted, err := ConvertArtifact(artifact, SchemaFull)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("convert failed: %v", err)
|
||||||
|
}
|
||||||
|
if converted.Schema != SchemaFull {
|
||||||
|
t.Fatalf("schema = %q, want %q", converted.Schema, SchemaFull)
|
||||||
|
}
|
||||||
|
if converted.Full != nil {
|
||||||
|
t.Fatalf("full payload = %#v, want nil", converted.Full)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func mustMarshalJSON(t *testing.T, value any) []byte {
|
func mustMarshalJSON(t *testing.T, value any) []byte {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
data, err := json.Marshal(value)
|
data, err := json.Marshal(value)
|
||||||
|
|||||||
156
internal/trim/run.go
Normal file
156
internal/trim/run.go
Normal file
@@ -0,0 +1,156 @@
|
|||||||
|
package trim
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"sort"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/internal/jsonfile"
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/internal/report"
|
||||||
|
)
|
||||||
|
|
||||||
|
type auditReport struct {
|
||||||
|
Operation string `json:"operation"`
|
||||||
|
InputFile string `json:"input_file"`
|
||||||
|
OutputFile string `json:"output_file"`
|
||||||
|
InputSchema string `json:"input_schema"`
|
||||||
|
OutputSchema string `json:"output_schema"`
|
||||||
|
Mode string `json:"mode"`
|
||||||
|
Selector string `json:"selector"`
|
||||||
|
SelectedIDs []int `json:"selected_ids"`
|
||||||
|
AllowEmpty bool `json:"allow_empty"`
|
||||||
|
InputSegmentCount int `json:"input_segment_count"`
|
||||||
|
RetainedSegmentCount int `json:"retained_segment_count"`
|
||||||
|
RemovedSegmentCount int `json:"removed_segment_count"`
|
||||||
|
RemovedInputIDs []int `json:"removed_input_ids"`
|
||||||
|
OldToNewIDMapping []idMapping `json:"old_to_new_id_mapping"`
|
||||||
|
OverlapGroupsRecomputed bool `json:"overlap_groups_recomputed"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type idMapping struct {
|
||||||
|
OldID int `json:"old_id"`
|
||||||
|
NewID int `json:"new_id"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run executes artifact-level trim orchestration.
|
||||||
|
func Run(ctx context.Context, cfg config.TrimConfig) error {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
selector, err := ParseSelector(cfg.Selector)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("invalid selector %q: %w", cfg.Selector, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := os.ReadFile(cfg.InputFile)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("read --input-file %q: %w", cfg.InputFile, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
artifact, err := ParseArtifactJSON(data)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("--input-file %q: %w", cfg.InputFile, err)
|
||||||
|
}
|
||||||
|
inputSegmentCount := artifact.SegmentCount()
|
||||||
|
inputSchema := artifact.Schema
|
||||||
|
|
||||||
|
mode := ModeKeep
|
||||||
|
if cfg.Mode == "remove" {
|
||||||
|
mode = ModeRemove
|
||||||
|
}
|
||||||
|
|
||||||
|
trimmed, err := ApplyArtifact(artifact, Options{
|
||||||
|
Mode: mode,
|
||||||
|
Selector: selector,
|
||||||
|
AllowEmpty: cfg.AllowEmpty,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
outputSchema := artifact.Schema
|
||||||
|
if cfg.OutputSchema != "" {
|
||||||
|
outputSchema = cfg.OutputSchema
|
||||||
|
}
|
||||||
|
|
||||||
|
outputArtifact, err := ConvertArtifact(trimmed.Artifact, outputSchema)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := ValidateArtifact(outputArtifact); err != nil {
|
||||||
|
return fmt.Errorf("validate trimmed output: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := jsonfile.Write(cfg.OutputFile, outputArtifact.Value()); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if cfg.ReportFile == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
audit := auditReport{
|
||||||
|
Operation: "trim",
|
||||||
|
InputFile: cfg.InputFile,
|
||||||
|
OutputFile: cfg.OutputFile,
|
||||||
|
InputSchema: inputSchema,
|
||||||
|
OutputSchema: outputArtifact.Schema,
|
||||||
|
Mode: cfg.Mode,
|
||||||
|
Selector: cfg.Selector,
|
||||||
|
SelectedIDs: selector.IDs(),
|
||||||
|
AllowEmpty: cfg.AllowEmpty,
|
||||||
|
InputSegmentCount: inputSegmentCount,
|
||||||
|
RetainedSegmentCount: len(trimmed.OldToNewID),
|
||||||
|
RemovedSegmentCount: len(trimmed.RemovedIDs),
|
||||||
|
RemovedInputIDs: append([]int(nil), trimmed.RemovedIDs...),
|
||||||
|
OldToNewIDMapping: orderedIDMapping(trimmed.OldToNewID),
|
||||||
|
OverlapGroupsRecomputed: trimmed.OverlapGroupsRecomputed,
|
||||||
|
}
|
||||||
|
auditJSON, err := json.Marshal(audit)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("marshal trim audit report: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
rpt := report.Report{
|
||||||
|
Metadata: report.Metadata{
|
||||||
|
Application: outputArtifact.Application(),
|
||||||
|
Version: outputArtifact.Version(),
|
||||||
|
InputReader: "trim-artifact",
|
||||||
|
InputFiles: []string{cfg.InputFile},
|
||||||
|
OutputModules: []string{"json"},
|
||||||
|
},
|
||||||
|
Events: []report.Event{
|
||||||
|
report.Info("trim", "trim", fmt.Sprintf("trimmed %d input segment(s) into %d output segment(s) with mode=%s", inputSegmentCount, outputArtifact.SegmentCount(), cfg.Mode)),
|
||||||
|
report.Info("trim", "trim-audit", string(auditJSON)),
|
||||||
|
report.Info("trim", "validate-output", fmt.Sprintf("validated %d output segment(s)", outputArtifact.SegmentCount())),
|
||||||
|
report.Info("output", "json", "wrote transcript JSON"),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if err := report.WriteJSON(cfg.ReportFile, rpt); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func orderedIDMapping(mapping map[int]int) []idMapping {
|
||||||
|
keys := make([]int, 0, len(mapping))
|
||||||
|
for oldID := range mapping {
|
||||||
|
keys = append(keys, oldID)
|
||||||
|
}
|
||||||
|
sort.Ints(keys)
|
||||||
|
|
||||||
|
pairs := make([]idMapping, 0, len(keys))
|
||||||
|
for _, oldID := range keys {
|
||||||
|
pairs = append(pairs, idMapping{
|
||||||
|
OldID: oldID,
|
||||||
|
NewID: mapping[oldID],
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return pairs
|
||||||
|
}
|
||||||
28
internal/trim/run_test.go
Normal file
28
internal/trim/run_test.go
Normal file
@@ -0,0 +1,28 @@
|
|||||||
|
package trim
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/seriatim/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestRunReturnsContextErrorBeforeWork(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
err := Run(ctx, config.TrimConfig{
|
||||||
|
InputFile: filepath.Join(dir, "input.json"),
|
||||||
|
OutputFile: filepath.Join(dir, "output.json"),
|
||||||
|
Mode: "keep",
|
||||||
|
Selector: "1",
|
||||||
|
OutputSchema: "",
|
||||||
|
AllowEmpty: false,
|
||||||
|
})
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("error = %v, want context.Canceled", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -14,6 +14,10 @@ import (
|
|||||||
var schemaFS embed.FS
|
var schemaFS embed.FS
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
OutputSchemaMinimal = "seriatim-minimal"
|
||||||
|
OutputSchemaIntermediate = "seriatim-intermediate"
|
||||||
|
OutputSchemaFull = "seriatim-full"
|
||||||
|
|
||||||
fullOutputSchemaPath = "full-output.schema.json"
|
fullOutputSchemaPath = "full-output.schema.json"
|
||||||
intermediateOutputSchemaPath = "intermediate-output.schema.json"
|
intermediateOutputSchemaPath = "intermediate-output.schema.json"
|
||||||
minimalOutputSchemaPath = "minimal-output.schema.json"
|
minimalOutputSchemaPath = "minimal-output.schema.json"
|
||||||
@@ -115,6 +119,25 @@ type OverlapGroup struct {
|
|||||||
Resolution string `json:"resolution"`
|
Resolution string `json:"resolution"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ValidOutputSchemaName reports whether value is a supported output schema name.
|
||||||
|
func ValidOutputSchemaName(value string) bool {
|
||||||
|
switch value {
|
||||||
|
case OutputSchemaMinimal, OutputSchemaIntermediate, OutputSchemaFull:
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// OutputSchemaNames returns supported output schema names in validation order.
|
||||||
|
func OutputSchemaNames() []string {
|
||||||
|
return []string{
|
||||||
|
OutputSchemaMinimal,
|
||||||
|
OutputSchemaIntermediate,
|
||||||
|
OutputSchemaFull,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ValidateTranscript validates a full transcript against the public JSON
|
// ValidateTranscript validates a full transcript against the public JSON
|
||||||
// schema and seriatim-specific semantic rules.
|
// schema and seriatim-specific semantic rules.
|
||||||
func ValidateTranscript(transcript Transcript) error {
|
func ValidateTranscript(transcript Transcript) error {
|
||||||
@@ -228,14 +251,16 @@ func outputSchema(schemaPath string) (*jsonschema.Schema, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func validateSemantics(transcript Transcript) error {
|
func validateSemantics(transcript Transcript) error {
|
||||||
|
segments := make([]segmentSemantics, len(transcript.Segments))
|
||||||
for index, segment := range transcript.Segments {
|
for index, segment := range transcript.Segments {
|
||||||
wantID := index + 1
|
segments[index] = segmentSemantics{
|
||||||
if segment.ID != wantID {
|
id: segment.ID,
|
||||||
return fmt.Errorf("segment %d has id %d; want %d", index, segment.ID, wantID)
|
start: segment.Start,
|
||||||
|
end: segment.End,
|
||||||
}
|
}
|
||||||
if segment.End < segment.Start {
|
|
||||||
return fmt.Errorf("segment %d has end %.3f before start %.3f", index, segment.End, segment.Start)
|
|
||||||
}
|
}
|
||||||
|
if err := validateSegmentSemantics(segments); err != nil {
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
for index, group := range transcript.OverlapGroups {
|
for index, group := range transcript.OverlapGroups {
|
||||||
if group.End < group.Start {
|
if group.End < group.Start {
|
||||||
@@ -246,26 +271,43 @@ func validateSemantics(transcript Transcript) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func validateIntermediateSemantics(transcript IntermediateTranscript) error {
|
func validateIntermediateSemantics(transcript IntermediateTranscript) error {
|
||||||
|
segments := make([]segmentSemantics, len(transcript.Segments))
|
||||||
for index, segment := range transcript.Segments {
|
for index, segment := range transcript.Segments {
|
||||||
wantID := index + 1
|
segments[index] = segmentSemantics{
|
||||||
if segment.ID != wantID {
|
id: segment.ID,
|
||||||
return fmt.Errorf("segment %d has id %d; want %d", index, segment.ID, wantID)
|
start: segment.Start,
|
||||||
}
|
end: segment.End,
|
||||||
if segment.End < segment.Start {
|
|
||||||
return fmt.Errorf("segment %d has end %.3f before start %.3f", index, segment.End, segment.Start)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return validateSegmentSemantics(segments)
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateMinimalSemantics(transcript MinimalTranscript) error {
|
func validateMinimalSemantics(transcript MinimalTranscript) error {
|
||||||
|
segments := make([]segmentSemantics, len(transcript.Segments))
|
||||||
for index, segment := range transcript.Segments {
|
for index, segment := range transcript.Segments {
|
||||||
wantID := index + 1
|
segments[index] = segmentSemantics{
|
||||||
if segment.ID != wantID {
|
id: segment.ID,
|
||||||
return fmt.Errorf("segment %d has id %d; want %d", index, segment.ID, wantID)
|
start: segment.Start,
|
||||||
|
end: segment.End,
|
||||||
}
|
}
|
||||||
if segment.End < segment.Start {
|
}
|
||||||
return fmt.Errorf("segment %d has end %.3f before start %.3f", index, segment.End, segment.Start)
|
return validateSegmentSemantics(segments)
|
||||||
|
}
|
||||||
|
|
||||||
|
type segmentSemantics struct {
|
||||||
|
id int
|
||||||
|
start float64
|
||||||
|
end float64
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateSegmentSemantics(segments []segmentSemantics) error {
|
||||||
|
for index, segment := range segments {
|
||||||
|
wantID := index + 1
|
||||||
|
if segment.id != wantID {
|
||||||
|
return fmt.Errorf("segment %d has id %d; want %d", index, segment.id, wantID)
|
||||||
|
}
|
||||||
|
if segment.end < segment.start {
|
||||||
|
return fmt.Errorf("segment %d has end %.3f before start %.3f", index, segment.end, segment.start)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -5,6 +5,43 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func TestValidOutputSchemaName(t *testing.T) {
|
||||||
|
valid := []string{
|
||||||
|
OutputSchemaMinimal,
|
||||||
|
OutputSchemaIntermediate,
|
||||||
|
OutputSchemaFull,
|
||||||
|
}
|
||||||
|
for _, name := range valid {
|
||||||
|
if !ValidOutputSchemaName(name) {
|
||||||
|
t.Fatalf("expected %q to be valid", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
invalid := []string{"", "compact", "minimal", "seriatim"}
|
||||||
|
for _, name := range invalid {
|
||||||
|
if ValidOutputSchemaName(name) {
|
||||||
|
t.Fatalf("expected %q to be invalid", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOutputSchemaNames(t *testing.T) {
|
||||||
|
names := OutputSchemaNames()
|
||||||
|
want := []string{
|
||||||
|
OutputSchemaMinimal,
|
||||||
|
OutputSchemaIntermediate,
|
||||||
|
OutputSchemaFull,
|
||||||
|
}
|
||||||
|
if len(names) != len(want) {
|
||||||
|
t.Fatalf("len(names) = %d, want %d", len(names), len(want))
|
||||||
|
}
|
||||||
|
for index := range want {
|
||||||
|
if names[index] != want[index] {
|
||||||
|
t.Fatalf("names[%d] = %q, want %q", index, names[index], want[index])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestValidateTranscriptAcceptsValidTranscript(t *testing.T) {
|
func TestValidateTranscriptAcceptsValidTranscript(t *testing.T) {
|
||||||
transcript := validTranscript()
|
transcript := validTranscript()
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user