eric/seriatim
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 4 Wiki Activity

Added support for a minimal JSON output schema

Browse Source
This commit is contained in:
Eric Rakestraw
2026-04-28 14:39:00 -05:00
parent a3ca6665a9
commit 9cca88280f
16 changed files with 658 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
internal/artifact/transcript.go
View File

@@ -57,6 +57,41 @@ func FromMerged(cfg config.Config, merged model.MergedTranscript) schema.Transcr
}
}
// MinimalFromMerged converts the internal merged transcript model into the
// compact public serialized output contract.
func MinimalFromMerged(cfg config.Config, merged model.MergedTranscript) schema.MinimalTranscript {
segments := make([]schema.MinimalSegment, len(merged.Segments))
for index, segment := range merged.Segments {
segments[index] = schema.MinimalSegment{
ID: segment.ID,
Start: segment.Start,
End: segment.End,
Speaker: segment.Speaker,
Text: segment.Text,
}
}
return schema.MinimalTranscript{
Metadata: schema.MinimalMetadata{
Application: ApplicationName,
Version: buildinfo.Version,
OutputSchema: config.OutputSchemaMinimal,
},
Segments: segments,
}
}
// SelectedFromMerged converts the internal merged transcript model into the
// runtime-selected public output contract.
func SelectedFromMerged(cfg config.Config, merged model.MergedTranscript) any {
switch cfg.OutputSchema {
case config.OutputSchemaMinimal:
return MinimalFromMerged(cfg, merged)
default:
return FromMerged(cfg, merged)
}
}
func copyIntPtr(value *int) *int {
if value == nil {
return nil

53
internal/artifact/transcript_test.go
View File

@@ -1,11 +1,13 @@
package artifact
import (
"reflect"
"testing"
"gitea.maximumdirect.net/eric/seriatim/internal/buildinfo"
"gitea.maximumdirect.net/eric/seriatim/internal/config"
"gitea.maximumdirect.net/eric/seriatim/internal/model"
"gitea.maximumdirect.net/eric/seriatim/schema"
)
func TestFromMergedUsesBuildVersion(t *testing.T) {
@@ -20,3 +22,54 @@ func TestFromMergedUsesBuildVersion(t *testing.T) {
t.Fatalf("version = %q, want v1.0.0-test", transcript.Metadata.Version)
}
}
func TestSelectedFromMergedDefaultsToSeriatimTranscript(t *testing.T) {
got := SelectedFromMerged(config.Config{}, model.MergedTranscript{})
if _, ok := got.(schema.Transcript); !ok {
t.Fatalf("selected artifact type = %T, want schema.Transcript", got)
}
}
func TestMinimalFromMergedEmitsOnlyMinimalShape(t *testing.T) {
merged := model.MergedTranscript{
Segments: []model.Segment{
{
ID: 1,
Source: "input.json",
SourceRef: "word-run:1:1:1",
DerivedFrom: []string{"input.json#0"},
Speaker: "Alice",
Start: 1,
End: 2,
Text: "hello",
Categories: []string{"backchannel"},
OverlapGroupID: 1,
},
},
OverlapGroups: []model.OverlapGroup{
{ID: 1, Start: 1, End: 2, Segments: []string{"input.json#0"}, Speakers: []string{"Alice"}, Class: "unknown", Resolution: "unresolved"},
},
}
got := MinimalFromMerged(config.Config{OutputSchema: config.OutputSchemaMinimal}, merged)
want := schema.MinimalTranscript{
Metadata: schema.MinimalMetadata{
Application: ApplicationName,
Version: buildinfo.Version,
OutputSchema: config.OutputSchemaMinimal,
},
Segments: []schema.MinimalSegment{
{ID: 1, Start: 1, End: 2, Speaker: "Alice", Text: "hello"},
},
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("minimal transcript = %#v, want %#v", got, want)
}
}
func TestSelectedFromMergedUsesMinimalWhenConfigured(t *testing.T) {
got := SelectedFromMerged(config.Config{OutputSchema: config.OutputSchemaMinimal}, model.MergedTranscript{})
if _, ok := got.(schema.MinimalTranscript); !ok {
t.Fatalf("selected artifact type = %T, want schema.MinimalTranscript", got)
}
}

3
internal/builtin/output.go
View File

@@ -7,7 +7,6 @@ import (
"gitea.maximumdirect.net/eric/seriatim/internal/config"
"gitea.maximumdirect.net/eric/seriatim/internal/report"
"gitea.maximumdirect.net/eric/seriatim/schema"
)
type jsonOutputWriter struct{}
@@ -16,7 +15,7 @@ func (jsonOutputWriter) Name() string {
return "json"
}
func (jsonOutputWriter) Write(ctx context.Context, out schema.Transcript, rpt report.Report, cfg config.Config) ([]report.Event, error) {
func (jsonOutputWriter) Write(ctx context.Context, out any, rpt report.Report, cfg config.Config) ([]report.Event, error) {
if err := ctx.Err(); err != nil {
return nil, err
}

13
internal/builtin/postprocess.go
View File

@@ -47,8 +47,17 @@ func (validateOutput) Process(ctx context.Context, in model.MergedTranscript, cf
return model.MergedTranscript{}, nil, err
}
transcript := artifact.FromMerged(cfg, in)
if err := schema.ValidateTranscript(transcript); err != nil {
selected := artifact.SelectedFromMerged(cfg, in)
var err error
switch transcript := selected.(type) {
case schema.MinimalTranscript:
err = schema.ValidateMinimalTranscript(transcript)
case schema.Transcript:
err = schema.ValidateTranscript(transcript)
default:
err = fmt.Errorf("unsupported output artifact type %T", selected)
}
if err != nil {
return model.MergedTranscript{}, nil, fmt.Errorf("validate-output: %w", err)
}

50
internal/builtin/postprocess_test.go
View File

@@ -49,6 +49,56 @@ func TestValidateOutputFailsBeforeAssignIDs(t *testing.T) {
}
}
func TestValidateOutputUsesMinimalSchemaWhenConfigured(t *testing.T) {
merged := model.MergedTranscript{
Segments: []model.Segment{
{
ID: 1,
Source: "input.json",
SourceRef: "word-run:1:1:1",
DerivedFrom: []string{"input.json#0"},
Speaker: "Alice",
Start: 1,
End: 2,
Text: "hello",
Categories: []string{"backchannel"},
OverlapGroupID: 1,
},
},
}
cfg := testConfig()
cfg.OutputSchema = config.OutputSchemaMinimal
got, events, err := validateOutput{}.Process(context.Background(), merged, cfg)
if err != nil {
t.Fatalf("validate output: %v", err)
}
if len(got.Segments) != 1 {
t.Fatalf("segment count = %d, want 1", len(got.Segments))
}
if len(events) != 1 || !strings.Contains(events[0].Message, "validated 1 output segment(s)") {
t.Fatalf("events = %#v", events)
}
}
func TestValidateOutputMinimalFailsBeforeAssignIDs(t *testing.T) {
merged := model.MergedTranscript{
Segments: []model.Segment{
{Source: "input.json", Speaker: "Alice", Start: 1, End: 2, Text: "hello"},
},
}
cfg := testConfig()
cfg.OutputSchema = config.OutputSchemaMinimal
_, _, err := validateOutput{}.Process(context.Background(), merged, cfg)
if err == nil {
t.Fatal("expected validation error")
}
if !strings.Contains(err.Error(), "segment 0 has id 0; want 1") {
t.Fatalf("unexpected error: %v", err)
}
}
func testConfig() config.Config {
return config.Config{
InputReader: config.DefaultInputReader,

1
internal/cli/merge.go
View File

@@ -32,6 +32,7 @@ func newMergeCommand() *cobra.Command {
flags.StringVar(&opts.AutocorrectFile, "autocorrect", "", "autocorrect rules file")
flags.StringVar(&opts.InputReader, "input-reader", config.DefaultInputReader, "input reader module")
flags.StringVar(&opts.OutputModules, "output-modules", config.DefaultOutputModules, "comma-separated output modules")
flags.StringVar(&opts.OutputSchema, "output-schema", config.DefaultOutputSchema, "output JSON schema: seriatim or minimal")
flags.StringVar(&opts.PreprocessingModules, "preprocessing-modules", config.DefaultPreprocessingModules, "comma-separated preprocessing modules")
flags.StringVar(&opts.PostprocessingModules, "postprocessing-modules", config.DefaultPostprocessingModules, "comma-separated postprocessing modules")
flags.StringVar(&opts.CoalesceGap, "coalesce-gap", config.DefaultCoalesceGapValue, "maximum same-speaker gap in seconds for coalesce")

100
internal/cli/merge_test.go
View File

@@ -10,6 +10,7 @@ import (
"gitea.maximumdirect.net/eric/seriatim/internal/model"
"gitea.maximumdirect.net/eric/seriatim/internal/report"
"gitea.maximumdirect.net/eric/seriatim/schema"
)
func TestMergeWritesMergedOutputAndReport(t *testing.T) {
@@ -111,6 +112,64 @@ func TestMergeWritesMergedOutputAndReport(t *testing.T) {
}
}
func TestMergeWritesMinimalOutputSchema(t *testing.T) {
dir := t.TempDir()
input := writeJSONFile(t, dir, "input.json", `{
"segments": [
{"start": 1, "end": 2, "text": " Yeah. "},
{"start": 8, "end": 9, "text": " next "}
]
}`)
output := filepath.Join(dir, "merged.json")
reportPath := filepath.Join(dir, "report.json")
err := executeMerge(
"--input-file", input,
"--output-file", output,
"--output-schema", "minimal",
"--report-file", reportPath,
)
if err != nil {
t.Fatalf("merge failed: %v", err)
}
var transcript schema.MinimalTranscript
readJSON(t, output, &transcript)
if transcript.Metadata.Application != "seriatim" {
t.Fatalf("application = %q, want seriatim", transcript.Metadata.Application)
}
if transcript.Metadata.OutputSchema != "minimal" {
t.Fatalf("output_schema = %q, want minimal", transcript.Metadata.OutputSchema)
}
if got, want := len(transcript.Segments), 2; got != want {
t.Fatalf("segment count = %d, want %d", got, want)
}
for index, segment := range transcript.Segments {
if segment.ID != index+1 {
t.Fatalf("segment %d id = %d, want %d", index, segment.ID, index+1)
}
}
if transcript.Segments[0].Speaker != "input.json" || transcript.Segments[0].Text != "Yeah." {
t.Fatalf("first segment = %#v", transcript.Segments[0])
}
outputBytes, err := os.ReadFile(output)
if err != nil {
t.Fatalf("read output: %v", err)
}
for _, forbidden := range []string{"overlap_groups", "categories", "source", "derived_from"} {
if strings.Contains(string(outputBytes), forbidden) {
t.Fatalf("minimal output contains %q:\n%s", forbidden, outputBytes)
}
}
var rpt report.Report
readJSON(t, reportPath, &rpt)
if !hasReportEvent(rpt, "postprocessing", "validate-output", "validated 2 output segment(s)") {
t.Fatal("expected validate-output report event")
}
}
func TestMergeTieBreakOrder(t *testing.T) {
dir := t.TempDir()
inputA := writeJSONFile(t, dir, "a.json", `{
@@ -182,6 +241,29 @@ func TestMergeValidateOutputBeforeAssignIDsFails(t *testing.T) {
}
}
func TestMergeValidateMinimalOutputBeforeAssignIDsFails(t *testing.T) {
dir := t.TempDir()
input := writeJSONFile(t, dir, "input.json", `{
"segments": [
{"start": 1, "end": 2, "text": "hello"}
]
}`)
output := filepath.Join(dir, "merged.json")
err := executeMerge(
"--input-file", input,
"--output-file", output,
"--output-schema", "minimal",
"--postprocessing-modules", "validate-output,assign-ids",
)
if err == nil {
t.Fatal("expected validation error")
}
if !strings.Contains(err.Error(), "validate-output: segment 0 has id 0; want 1") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestMergeDetectsOverlapGroups(t *testing.T) {
dir := t.TempDir()
inputA := writeJSONFile(t, dir, "a.json", `{
@@ -963,6 +1045,24 @@ func TestUnknownModulesFailDuringValidation(t *testing.T) {
}
}
func TestUnknownOutputSchemaFailsDuringValidation(t *testing.T) {
dir := t.TempDir()
input := writeJSONFile(t, dir, "input.json", `{"segments":[]}`)
output := filepath.Join(dir, "merged.json")
err := executeMerge(
"--input-file", input,
"--output-file", output,
"--output-schema", "compact",
)
if err == nil {
t.Fatal("expected output schema error")
}
if !strings.Contains(err.Error(), "--output-schema must be one of") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestInvalidPreprocessingOrderFails(t *testing.T) {
dir := t.TempDir()
input := writeJSONFile(t, dir, "input.json", `{"segments":[]}`)

21
internal/config/config.go
View File

@@ -13,6 +13,7 @@ import (
const (
DefaultInputReader = "json-files"
DefaultOutputModules = "json"
DefaultOutputSchema = OutputSchemaSeriatim
DefaultPreprocessingModules = "validate-raw,normalize-speakers,trim-text"
DefaultPostprocessingModules = "detect-overlaps,resolve-overlaps,backchannel,filler,coalesce,detect-overlaps,autocorrect,assign-ids,validate-output"
DefaultOverlapWordRunGap = 0.75
@@ -25,6 +26,8 @@ const (
WordRunReorderWindowEnv = "SERIATIM_OVERLAP_WORD_RUN_REORDER_WINDOW"
BackchannelMaxDurationEnv = "SERIATIM_BACKCHANNEL_MAX_DURATION"
FillerMaxDurationEnv = "SERIATIM_FILLER_MAX_DURATION"
OutputSchemaSeriatim = "seriatim"
OutputSchemaMinimal = "minimal"
)
// MergeOptions captures raw CLI option values before validation.
@@ -36,6 +39,7 @@ type MergeOptions struct {
AutocorrectFile string
InputReader string
OutputModules string
OutputSchema string
PreprocessingModules string
PostprocessingModules string
CoalesceGap string
@@ -50,6 +54,7 @@ type Config struct {
AutocorrectFile string
InputReader string
OutputModules []string
OutputSchema string
PreprocessingModules []string
PostprocessingModules []string
OverlapWordRunGap float64
@@ -64,6 +69,7 @@ func NewMergeConfig(opts MergeOptions) (Config, error) {
cfg := Config{
InputReader: strings.TrimSpace(opts.InputReader),
OutputModules: nil,
OutputSchema: strings.TrimSpace(opts.OutputSchema),
PreprocessingModules: nil,
PostprocessingModules: nil,
OverlapWordRunGap: DefaultOverlapWordRunGap,
@@ -76,6 +82,12 @@ func NewMergeConfig(opts MergeOptions) (Config, error) {
if cfg.InputReader == "" {
return Config{}, errors.New("--input-reader is required")
}
if cfg.OutputSchema == "" {
cfg.OutputSchema = DefaultOutputSchema
}
if err := validateOutputSchema(cfg.OutputSchema); err != nil {
return Config{}, err
}
var err error
cfg.OutputModules, err = parseModuleList(opts.OutputModules)
@@ -174,6 +186,15 @@ func parseModuleList(value string) ([]string, error) {
return names, nil
}
func validateOutputSchema(value string) error {
switch value {
case OutputSchemaSeriatim, OutputSchemaMinimal:
return nil
default:
return fmt.Errorf("--output-schema must be one of %q or %q", OutputSchemaSeriatim, OutputSchemaMinimal)
}
}
func normalizeInputFiles(paths []string) ([]string, error) {
if len(paths) == 0 {
return nil, errors.New("at least one --input-file is required")

65
internal/config/config_test.go
View File

@@ -46,6 +46,71 @@ func TestDuplicateInputFilesFailValidation(t *testing.T) {
}
}
func TestOutputSchemaDefaultsToSeriatim(t *testing.T) {
dir := t.TempDir()
input := writeTempFile(t, dir, "input.json")
output := filepath.Join(dir, "merged.json")
cfg, err := NewMergeConfig(MergeOptions{
InputFiles: []string{input},
OutputFile: output,
InputReader: DefaultInputReader,
OutputModules: DefaultOutputModules,
PreprocessingModules: DefaultPreprocessingModules,
PostprocessingModules: DefaultPostprocessingModules,
})
if err != nil {
t.Fatalf("config failed: %v", err)
}
if cfg.OutputSchema != DefaultOutputSchema {
t.Fatalf("output schema = %q, want %q", cfg.OutputSchema, DefaultOutputSchema)
}
}
func TestOutputSchemaAcceptsMinimal(t *testing.T) {
dir := t.TempDir()
input := writeTempFile(t, dir, "input.json")
output := filepath.Join(dir, "merged.json")
cfg, err := NewMergeConfig(MergeOptions{
InputFiles: []string{input},
OutputFile: output,
InputReader: DefaultInputReader,
OutputModules: DefaultOutputModules,
OutputSchema: OutputSchemaMinimal,
PreprocessingModules: DefaultPreprocessingModules,
PostprocessingModules: DefaultPostprocessingModules,
})
if err != nil {
t.Fatalf("config failed: %v", err)
}
if cfg.OutputSchema != OutputSchemaMinimal {
t.Fatalf("output schema = %q, want %q", cfg.OutputSchema, OutputSchemaMinimal)
}
}
func TestOutputSchemaRejectsUnknownValue(t *testing.T) {
dir := t.TempDir()
input := writeTempFile(t, dir, "input.json")
output := filepath.Join(dir, "merged.json")
_, err := NewMergeConfig(MergeOptions{
InputFiles: []string{input},
OutputFile: output,
InputReader: DefaultInputReader,
OutputModules: DefaultOutputModules,
OutputSchema: "compact",
PreprocessingModules: DefaultPreprocessingModules,
PostprocessingModules: DefaultPostprocessingModules,
})
if err == nil {
t.Fatal("expected output schema error")
}
if !strings.Contains(err.Error(), "--output-schema must be one of") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestOverlapWordRunGapDefaultsTo075(t *testing.T) {
t.Setenv(OverlapWordRunGapEnv, "")
dir := t.TempDir()

3
internal/pipeline/interfaces.go
View File

@@ -6,7 +6,6 @@ import (
"gitea.maximumdirect.net/eric/seriatim/internal/config"
"gitea.maximumdirect.net/eric/seriatim/internal/model"
"gitea.maximumdirect.net/eric/seriatim/internal/report"
"gitea.maximumdirect.net/eric/seriatim/schema"
)
// ModelState identifies which representation a preprocessing module consumes.
@@ -53,5 +52,5 @@ type Postprocessor interface {
// OutputWriter emits final artifacts.
type OutputWriter interface {
Name() string
Write(ctx context.Context, out schema.Transcript, rpt report.Report, cfg config.Config) ([]report.Event, error)
Write(ctx context.Context, out any, rpt report.Report, cfg config.Config) ([]report.Event, error)
}

5
internal/pipeline/runner.go
View File

@@ -9,7 +9,6 @@ import (
"gitea.maximumdirect.net/eric/seriatim/internal/config"
"gitea.maximumdirect.net/eric/seriatim/internal/model"
"gitea.maximumdirect.net/eric/seriatim/internal/report"
"gitea.maximumdirect.net/eric/seriatim/schema"
)
const (
@@ -141,8 +140,8 @@ func validatePreprocessors(modules []Preprocessor) error {
return nil
}
func finalizeTranscript(cfg config.Config, merged model.MergedTranscript) schema.Transcript {
return artifact.FromMerged(cfg, merged)
func finalizeTranscript(cfg config.Config, merged model.MergedTranscript) any {
return artifact.SelectedFromMerged(cfg, merged)
}
func finalizeReport(cfg config.Config, events []report.Event) report.Report {