From 23e5787dc1c4adbf93e51cff7bb54dc3b2c27fc3 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 8 May 2026 17:13:56 +0000 Subject: [PATCH] Use trimmed transcript for session recap --- README.md | 5 ++- architecture.md | 7 +-- docs/integrations/scriptorium.md | 2 +- examples/pipeline.minimal.yml | 2 +- internal/stage/analyze.go | 77 +++++++++++++++++++++++++++++--- internal/stage/analyze_test.go | 71 ++++++++++++++++++++--------- 6 files changed, 128 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index d326832..6210d66 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ scriptorium: inputs: transcript: - source: "processed_transcript" + source: "trimmed_transcript" required: true previous_recap: @@ -133,7 +133,8 @@ For the initial implementation, only `session_recap` generation is supported. Analyze-stage session recap behavior: -- uses processed transcript input (`transcripts/processed.json`) +- defaults to trimmed transcript input (`transcripts/trimmed.json`) when configured with `source: trimmed_transcript` +- still supports full polished transcript input (`transcripts/processed.json`) when configured with `source: processed_transcript` - optionally includes `previous_recap` when configured and resolvable - omits optional previous recap when unavailable - fails if required inputs are missing diff --git a/architecture.md b/architecture.md index 083f725..8821565 100644 --- a/architecture.md +++ b/architecture.md @@ -18,12 +18,12 @@ Implemented: - real Seriatim subprocess adapter - real Audita subprocess adapter - real Scriptorium subprocess adapter +- real `trim` stage producing `transcripts/trimmed.json` - real `analyze` stage for initial `session_recap` generation - optional Scriptorium render diagnostics (`render_debug`) before production run Still placeholder/future: -- `trim` stage behavior - `archive` stage behavior - `notify` stage behavior - additional Scriptorium artifact types beyond `session_recap` @@ -162,8 +162,9 @@ Behavior: - if `pipeline.scriptorium` is missing, analyze returns a skipped result with metadata - if no Scriptorium artifacts are enabled, analyze returns a skipped result with metadata - if enabled artifacts exist but `session_recap` is not enabled, analyze fails clearly -- processed transcript input is resolved from manifest (`polish` output kind `transcript_processed`) when available, otherwise fallback path `work//transcripts/processed.json` -- processed transcript is validated as JSON with top-level `segments` array +- `trimmed_transcript` input is resolved from manifest (`trim` output kind `transcript_trimmed`) when available, otherwise fallback path `work//transcripts/trimmed.json` +- `processed_transcript` input is resolved from manifest (`polish` output kind `transcript_processed`) when available, otherwise fallback path `work//transcripts/processed.json` +- transcript inputs are validated as JSON with top-level `segments` array - configured inputs are resolved by source - optional `previous_recap` is omitted when unavailable - required `previous_recap` fails before invocation when unavailable diff --git a/docs/integrations/scriptorium.md b/docs/integrations/scriptorium.md index dc922eb..8bd75b3 100644 --- a/docs/integrations/scriptorium.md +++ b/docs/integrations/scriptorium.md @@ -235,7 +235,7 @@ scriptorium: render_debug: false # optional artifact override inputs: transcript: - source: processed_transcript + source: trimmed_transcript required: true previous_recap: source: previous_session_artifact diff --git a/examples/pipeline.minimal.yml b/examples/pipeline.minimal.yml index d25b9fb..cc279ef 100644 --- a/examples/pipeline.minimal.yml +++ b/examples/pipeline.minimal.yml @@ -75,7 +75,7 @@ scriptorium: # render_debug: true inputs: transcript: - source: "processed_transcript" + source: "trimmed_transcript" required: true previous_recap: source: "previous_session_artifact" diff --git a/internal/stage/analyze.go b/internal/stage/analyze.go index 59bf305..f9b4462 100644 --- a/internal/stage/analyze.go +++ b/internal/stage/analyze.go @@ -24,6 +24,7 @@ func (analyzeStage) Declares() IODecl { return IODecl{ Inputs: []artifacts.Ref{ {Kind: "transcript_processed", Category: "transcripts", RelativePath: "transcripts/processed.json"}, + {Kind: "transcript_trimmed", Category: "transcripts", RelativePath: "transcripts/trimmed.json"}, }, Outputs: []artifacts.Ref{ {Kind: "session_recap", Category: "artifacts", RelativePath: "artifacts/session_recap.md"}, @@ -85,11 +86,16 @@ func (analyzeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S if err != nil { return nil, fmt.Errorf("analyze: resolve processed transcript: %w", err) } - if processedTranscriptPath == "" { - return nil, fmt.Errorf("analyze: processed transcript input is required") + trimmedTranscriptPath, trimmedSource, err := discoverTrimmedTranscript(m, paths) + if err != nil { + return nil, fmt.Errorf("analyze: resolve trimmed transcript: %w", err) } - if err := validateProcessedTranscriptOutput(processedTranscriptPath); err != nil { - return nil, fmt.Errorf("analyze: processed transcript %q invalid: %w", processedTranscriptPath, err) + + transcriptInputs := analyzeTranscriptInputs{ + ProcessedPath: processedTranscriptPath, + ProcessedSource: processedSource, + TrimmedPath: trimmedTranscriptPath, + TrimmedSource: trimmedSource, } inputPaths := map[string]string{} @@ -98,7 +104,7 @@ func (analyzeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S inputNames := sortedScriptoriumInputNames(artifactCfg.Inputs) for _, inputName := range inputNames { inputCfg := artifactCfg.Inputs[inputName] - resolvedPath, resolved, resolveErr := resolveScriptoriumInput(inputName, inputCfg, processedTranscriptPath, paths, sessionDir) + resolvedPath, resolved, resolveErr := resolveScriptoriumInput(inputName, inputCfg, transcriptInputs, paths, sessionDir) if resolveErr != nil { return nil, fmt.Errorf("analyze: resolve input %q: %w", inputName, resolveErr) } @@ -146,6 +152,8 @@ func (analyzeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S "timeout": timeout.String(), "processed_transcript_path": processedTranscriptPath, "processed_transcript_source": processedSource, + "trimmed_transcript_path": trimmedTranscriptPath, + "trimmed_transcript_source": trimmedSource, "render_debug_enabled": resolveRenderDebugEnabled(env.Config.Pipeline.Scriptorium.RenderDebug, artifactCfg.RenderDebug), } @@ -333,16 +341,71 @@ func discoverProcessedTranscript(m *manifest.Manifest, paths artifacts.SessionPa return "", "", nil } +func discoverTrimmedTranscript(m *manifest.Manifest, paths artifacts.SessionPaths) (string, string, error) { + candidates := []string{} + if m != nil && m.Stages != nil { + if sr := m.Stages["trim"]; sr != nil { + for _, out := range sr.Outputs { + if out.Kind != "transcript_trimmed" { + continue + } + p := strings.TrimSpace(out.LocalPath) + if p == "" { + continue + } + resolved := artifacts.ResolveSessionLocalPathForRead(paths, p) + candidates = append(candidates, filepath.Clean(resolved)) + } + } + } + deduped := dedupeAndSortPaths(candidates) + for _, p := range deduped { + if info, err := os.Stat(p); err == nil && !info.IsDir() { + return p, "manifest.trim.outputs", nil + } + } + + fallback := filepath.Join(paths.TranscriptsDir, "trimmed.json") + if info, err := os.Stat(fallback); err == nil && !info.IsDir() { + return filepath.Clean(fallback), "fallback.transcripts_dir", nil + } + if len(deduped) > 0 { + return deduped[0], "manifest.trim.outputs", nil + } + return "", "", nil +} + +type analyzeTranscriptInputs struct { + ProcessedPath string + ProcessedSource string + TrimmedPath string + TrimmedSource string +} + func resolveScriptoriumInput( inputName string, inputCfg config.ScriptoriumInputConfig, - processedTranscriptPath string, + transcriptInputs analyzeTranscriptInputs, paths artifacts.SessionPaths, sessionDir string, ) (string, bool, error) { switch strings.TrimSpace(inputCfg.Source) { case "processed_transcript": - return processedTranscriptPath, true, nil + if strings.TrimSpace(transcriptInputs.ProcessedPath) == "" { + return "", false, nil + } + if err := validateProcessedTranscriptOutput(transcriptInputs.ProcessedPath); err != nil { + return "", false, fmt.Errorf("processed transcript %q invalid: %w", transcriptInputs.ProcessedPath, err) + } + return transcriptInputs.ProcessedPath, true, nil + case "trimmed_transcript": + if strings.TrimSpace(transcriptInputs.TrimmedPath) == "" { + return "", false, fmt.Errorf("trimmed transcript input is unavailable; run trim stage first") + } + if err := validateProcessedTranscriptOutput(transcriptInputs.TrimmedPath); err != nil { + return "", false, fmt.Errorf("trimmed transcript %q invalid: %w", transcriptInputs.TrimmedPath, err) + } + return transcriptInputs.TrimmedPath, true, nil case "previous_session_artifact": if strings.TrimSpace(inputCfg.Path) == "" { return "", false, nil diff --git a/internal/stage/analyze_test.go b/internal/stage/analyze_test.go index 1c4213f..fd89b62 100644 --- a/internal/stage/analyze_test.go +++ b/internal/stage/analyze_test.go @@ -15,10 +15,10 @@ import ( "gitea.maximumdirect.net/eric/narratio/internal/manifest" ) -func TestAnalyzeGeneratesSessionRecapFromProcessedTranscript(t *testing.T) { +func TestAnalyzeGeneratesSessionRecapFromTrimmedTranscript(t *testing.T) { env, m, fake := setupAnalyzeEnv(t) paths := env.ArtifactStore.SessionPaths(m.SessionID) - writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[]}`) + writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`) result, err := (analyzeStage{}).Run(context.Background(), env, m) if err != nil { @@ -35,8 +35,8 @@ func TestAnalyzeGeneratesSessionRecapFromProcessedTranscript(t *testing.T) { if req.ProfileID != "local-quality" { t.Fatalf("profile id = %q, want local-quality", req.ProfileID) } - if req.InputPaths["transcript"] != filepath.Join(paths.TranscriptsDir, "processed.json") { - t.Fatalf("transcript input = %q, want processed transcript path", req.InputPaths["transcript"]) + if req.InputPaths["transcript"] != filepath.Join(paths.TranscriptsDir, "trimmed.json") { + t.Fatalf("transcript input = %q, want trimmed transcript path", req.InputPaths["transcript"]) } if req.OutputPath != filepath.Join(paths.ArtifactsDir, "session_recap.md") { t.Fatalf("output path = %q, want %q", req.OutputPath, filepath.Join(paths.ArtifactsDir, "session_recap.md")) @@ -71,7 +71,7 @@ func TestAnalyzeGeneratesSessionRecapFromProcessedTranscript(t *testing.T) { func TestAnalyzeRenderDebugFalseDoesNotCallRenderArtifact(t *testing.T) { env, m, fake := setupAnalyzeEnv(t) paths := env.ArtifactStore.SessionPaths(m.SessionID) - writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[]}`) + writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`) env.Config.Pipeline.Scriptorium.RenderDebug = false @@ -87,7 +87,7 @@ func TestAnalyzeRenderDebugFalseDoesNotCallRenderArtifact(t *testing.T) { func TestAnalyzeRenderDebugArtifactOverrideFalseWinsOverGlobalTrue(t *testing.T) { env, m, fake := setupAnalyzeEnv(t) paths := env.ArtifactStore.SessionPaths(m.SessionID) - writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[]}`) + writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`) env.Config.Pipeline.Scriptorium.RenderDebug = true artifact := env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] @@ -110,7 +110,7 @@ func TestAnalyzeRenderDebugArtifactOverrideFalseWinsOverGlobalTrue(t *testing.T) func TestAnalyzeRenderDebugTrueCallsRenderBeforeRun(t *testing.T) { env, m, _ := setupAnalyzeEnv(t) paths := env.ArtifactStore.SessionPaths(m.SessionID) - writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[]}`) + writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`) env.Config.Pipeline.Scriptorium.RenderDebug = true runner := &orderedScriptoriumRunner{ @@ -131,7 +131,7 @@ func TestAnalyzeRenderDebugTrueCallsRenderBeforeRun(t *testing.T) { func TestAnalyzeRenderOutputPathIsRecorded(t *testing.T) { env, m, fake := setupAnalyzeEnv(t) paths := env.ArtifactStore.SessionPaths(m.SessionID) - writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[]}`) + writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`) env.Config.Pipeline.Scriptorium.RenderDebug = true result, err := (analyzeStage{}).Run(context.Background(), env, m) @@ -155,7 +155,7 @@ func TestAnalyzeRenderOutputPathIsRecorded(t *testing.T) { func TestAnalyzeRenderFailurePreventsRun(t *testing.T) { env, m, fake := setupAnalyzeEnv(t) paths := env.ArtifactStore.SessionPaths(m.SessionID) - writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[]}`) + writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`) env.Config.Pipeline.Scriptorium.RenderDebug = true fake.RenderErr = errors.New("render boom") @@ -177,7 +177,7 @@ func TestAnalyzeRenderFailurePreventsRun(t *testing.T) { func TestAnalyzeRenderInvalidJSONFailsClearly(t *testing.T) { env, m, _ := setupAnalyzeEnv(t) paths := env.ArtifactStore.SessionPaths(m.SessionID) - writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[]}`) + writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`) env.Config.Pipeline.Scriptorium.RenderDebug = true runner := &orderedScriptoriumRunner{ RenderBody: `not-json`, @@ -200,7 +200,7 @@ func TestAnalyzeRenderInvalidJSONFailsClearly(t *testing.T) { func TestAnalyzeRunStillSucceedsWhenRenderSucceeds(t *testing.T) { env, m, fake := setupAnalyzeEnv(t) paths := env.ArtifactStore.SessionPaths(m.SessionID) - writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[]}`) + writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`) env.Config.Pipeline.Scriptorium.RenderDebug = true result, err := (analyzeStage{}).Run(context.Background(), env, m) @@ -391,7 +391,7 @@ func TestAnalyzeFailsWhenRequiredPreviousRecapMissing(t *testing.T) { func TestAnalyzeFailsWhenOutputPathMissing(t *testing.T) { env, m, fake := setupAnalyzeEnv(t) paths := env.ArtifactStore.SessionPaths(m.SessionID) - writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[]}`) + writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`) artifact := env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] artifact.OutputPath = "" @@ -409,21 +409,48 @@ func TestAnalyzeFailsWhenOutputPathMissing(t *testing.T) { } } -func TestAnalyzeFailsWhenProcessedTranscriptMissing(t *testing.T) { +func TestAnalyzeFailsWhenTrimmedTranscriptMissing(t *testing.T) { env, m, _ := setupAnalyzeEnv(t) _, err := (analyzeStage{}).Run(context.Background(), env, m) if err == nil { t.Fatal("expected error, got nil") } - if !strings.Contains(err.Error(), "processed transcript input is required") { - t.Fatalf("error = %q, want missing processed transcript context", err.Error()) + if !strings.Contains(err.Error(), "trimmed transcript input is unavailable") { + t.Fatalf("error = %q, want missing trimmed transcript context", err.Error()) + } + if !strings.Contains(err.Error(), "run trim stage first") { + t.Fatalf("error = %q, want guidance to run trim stage first", err.Error()) + } +} + +func TestAnalyzeSupportsProcessedTranscriptSourceWhenConfigured(t *testing.T) { + env, m, fake := setupAnalyzeEnv(t) + paths := env.ArtifactStore.SessionPaths(m.SessionID) + writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[]}`) + + artifact := env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] + artifact.Inputs["transcript"] = config.ScriptoriumInputConfig{ + Source: "processed_transcript", + Required: true, + } + env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] = artifact + + _, err := (analyzeStage{}).Run(context.Background(), env, m) + if err != nil { + t.Fatalf("Run() error = %v", err) + } + if len(fake.RunRequests) != 1 { + t.Fatalf("run requests = %d, want 1", len(fake.RunRequests)) + } + if fake.RunRequests[0].InputPaths["transcript"] != filepath.Join(paths.TranscriptsDir, "processed.json") { + t.Fatalf("transcript input = %q, want processed transcript path", fake.RunRequests[0].InputPaths["transcript"]) } } func TestAnalyzeFailsWhenProcessedTranscriptInvalidJSON(t *testing.T) { env, m, _ := setupAnalyzeEnv(t) paths := env.ArtifactStore.SessionPaths(m.SessionID) - writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{not-json`) + writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{not-json`) _, err := (analyzeStage{}).Run(context.Background(), env, m) if err == nil { @@ -437,7 +464,7 @@ func TestAnalyzeFailsWhenProcessedTranscriptInvalidJSON(t *testing.T) { func TestAnalyzeFailsWhenProcessedTranscriptMissingSegmentsArray(t *testing.T) { env, m, _ := setupAnalyzeEnv(t) paths := env.ArtifactStore.SessionPaths(m.SessionID) - writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"not_segments":[]}`) + writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"not_segments":[]}`) _, err := (analyzeStage{}).Run(context.Background(), env, m) if err == nil { @@ -451,7 +478,7 @@ func TestAnalyzeFailsWhenProcessedTranscriptMissingSegmentsArray(t *testing.T) { func TestAnalyzeRecordsRefsAndMetadata(t *testing.T) { env, m, _ := setupAnalyzeEnv(t) paths := env.ArtifactStore.SessionPaths(m.SessionID) - writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[]}`) + writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`) result, err := (analyzeStage{}).Run(context.Background(), env, m) if err != nil { @@ -480,7 +507,7 @@ func TestAnalyzeRecordsRefsAndMetadata(t *testing.T) { func TestAnalyzeHandlesAdapterError(t *testing.T) { env, m, fake := setupAnalyzeEnv(t) paths := env.ArtifactStore.SessionPaths(m.SessionID) - writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[]}`) + writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`) fake.RunErr = errors.New("adapter boom") _, err := (analyzeStage{}).Run(context.Background(), env, m) @@ -495,7 +522,7 @@ func TestAnalyzeHandlesAdapterError(t *testing.T) { func TestAnalyzeHandlesValidationFailedResultAsError(t *testing.T) { env, m, fake := setupAnalyzeEnv(t) paths := env.ArtifactStore.SessionPaths(m.SessionID) - writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[]}`) + writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`) fake.RunResult = scriptorium.ArtifactResult{ ValidationFailed: true, ExitCode: 2, @@ -514,7 +541,7 @@ func TestAnalyzeHandlesValidationFailedResultAsError(t *testing.T) { func TestAnalyzeSkipsWhenNoEnabledScriptoriumArtifactsConfigured(t *testing.T) { env, m, _ := setupAnalyzeEnv(t) paths := env.ArtifactStore.SessionPaths(m.SessionID) - writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[]}`) + writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "trimmed.json"), `{"segments":[]}`) env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] = config.ScriptoriumArtifactConfig{ Enabled: false, @@ -558,7 +585,7 @@ func setupAnalyzeEnv(t *testing.T) (*Env, *manifest.Manifest, *scriptorium.FakeR Timeout: "2m", Inputs: map[string]config.ScriptoriumInputConfig{ "transcript": { - Source: "processed_transcript", + Source: "trimmed_transcript", Required: true, }, "previous_recap": {