eric/narratio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 27 Wiki Activity

Require manifest-backed local artifacts

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-02 14:09:32 +00:00
parent 62b1f9b0ae
commit 57719b7d81
28 changed files with 392 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
internal/app/artifact_evidence_test.go Normal file
View File

@@ -0,0 +1,31 @@
package app
import (
"context"
"testing"
"time"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
func setCurrentBuiltInEvidence(t *testing.T, m *manifest.Manifest, stage, kind, sourceID, path string) {
t.Helper()
checksum, err := artifacts.SHA256File(path)
if err != nil {
t.Fatal(err)
}
m.MarkStageSucceeded(stage, time.Date(2026, 5, 16, 1, 2, 3, 0, time.UTC), []manifest.ArtifactRecord{{
Kind: kind, SourceID: sourceID, LocalPath: path, ProducerRunID: stage + "-run", Checksum: checksum,
}})
}
func seedCurrentBuiltInEvidence(t *testing.T, manifestPath, sessionID, campaign, stage, kind, sourceID, path string) {
t.Helper()
m := manifest.New(sessionID, time.Now().UTC())
m.Campaign = campaign
setCurrentBuiltInEvidence(t, m, stage, kind, sourceID, path)
if err := (&manifest.LocalStore{}).Save(context.Background(), manifestPath, m); err != nil {
t.Fatal(err)
}
}

9
internal/app/commands_test.go
View File

@@ -12,6 +12,7 @@ import (
"testing"
"time"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
@@ -126,7 +127,9 @@ func TestExecuteRunStageNormalizeIsAccepted(t *testing.T) {
workspaceRoot := t.TempDir()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "polished.json"), `{"segments":[{"id":1}]}`)
polishedPath := filepath.Join(workRoot, "transcripts", "polished.json")
mustWriteTestFile(t, polishedPath, `{"segments":[{"id":1}]}`)
seedCurrentBuiltInEvidence(t, filepath.Join(workRoot, "manifest.json"), "2026-05-03", "sample-campaign", "polish", artifacts.TranscriptOutputKindPolished, artifacts.ArtifactTranscriptPolished, polishedPath)
var stdout bytes.Buffer
var stderr bytes.Buffer
@@ -247,7 +250,9 @@ inputs:
})
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", sessionID)
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "base.json"), `{"schema":"seriatim-intermediate","segments":[]}`)
basePath := filepath.Join(workRoot, "transcripts", "base.json")
mustWriteTestFile(t, basePath, `{"schema":"seriatim-intermediate","segments":[]}`)
seedCurrentBuiltInEvidence(t, filepath.Join(workRoot, "manifest.json"), sessionID, "sample-campaign", "merge", artifacts.TranscriptOutputKindBase, artifacts.ArtifactTranscriptBase, basePath)
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "[]\n")
var stdout bytes.Buffer

37
internal/app/extract_lifecycle_test.go
View File

@@ -156,7 +156,7 @@ func TestExtractLifecyclePrepareBindsVerifiedReferenceSnapshots(t *testing.T) {
if err != nil {
t.Fatalf("executeStages() error = %v", err)
}
if len(summary.Executed) != 2 || len(runner.requests) != 1 {
if len(summary.Executed) != 3 || len(runner.requests) != 1 {
t.Fatalf("summary = %#v requests=%d", summary, len(runner.requests))
}
paths, err := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).EnsureLayoutFor(cfg.Session.Campaign, cfg.Session.SessionID)
@@ -510,12 +510,12 @@ func TestExtractLifecycleRerunsAfterDirectTranscriptChange(t *testing.T) {
t.Fatalf("Save(mutated) error = %v", err)
}
rerun, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
if err != nil {
t.Fatalf("rerun executeStages() error = %v", err)
_, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
if err == nil || !strings.Contains(err.Error(), "checksum mismatch") {
t.Fatalf("rerun executeStages() error = %v, want stale transcript evidence rejection", err)
}
if len(rerun.Executed) != 2 || len(rerun.Skipped) != 0 || len(runner.requests) != 2 || analyzeRuns != 2 {
t.Fatalf("rerun summary = %#v requests=%d analyze=%d", rerun, len(runner.requests), analyzeRuns)
if len(runner.requests) != 1 || analyzeRuns != 1 {
t.Fatalf("stale transcript should prevent extraction, requests=%d analyze=%d", len(runner.requests), analyzeRuns)
}
}
@@ -645,7 +645,24 @@ func prepareExtractLifecyclePlan(t *testing.T) []stage.Stage {
if err != nil {
t.Fatalf("BuildSingleStagePlan(extract) error = %v", err)
}
return append(prepare, extract...)
return append(append(prepare, lifecycleTrimEvidenceStage{}), extract...)
}
type lifecycleTrimEvidenceStage struct{}
func (lifecycleTrimEvidenceStage) Name() string { return "trim" }
func (lifecycleTrimEvidenceStage) Run(_ context.Context, env *stage.Env, m *manifest.Manifest) (*stage.StageResult, error) {
paths := artifacts.NewLocalStore(env.Config.Pipeline.Workspace.Root).SessionPathsFor(m.Campaign, m.SessionID)
path := filepath.Join(paths.ArtifactsDir, "trimmed.from-manifest.json")
checksum, err := artifacts.SHA256File(path)
if err != nil {
return nil, err
}
return &stage.StageResult{Outputs: []artifacts.Ref{{
Kind: artifactmodel.TranscriptOutputKindFinalTrimmed, SourceID: artifactmodel.SourceTranscriptFinalTrimmed,
AbsolutePath: path, Checksum: checksum,
}}}, nil
}
func configureLifecycleReferences(t *testing.T, cfg *config.Config) map[string]string {
@@ -738,9 +755,13 @@ func extractionLifecycleFixture(t *testing.T, enabled bool) (*config.Config, *st
}
m := manifest.New(cfg.Session.SessionID, time.Now().UTC())
m.Campaign = cfg.Session.Campaign
inputChecksum, err := artifacts.SHA256File(inputPath)
if err != nil {
t.Fatalf("SHA256File(input) error = %v", err)
}
m.MarkStageSucceeded("trim", time.Now().UTC(), []manifest.ArtifactRecord{{
Kind: artifactmodel.TranscriptOutputKindFinalTrimmed, SourceID: artifactmodel.SourceTranscriptFinalTrimmed,
LocalPath: inputPath,
LocalPath: inputPath, ProducerRunID: "trim-run", Checksum: inputChecksum,
}})
if err := (&manifest.LocalStore{}).Save(context.Background(), paths.ManifestPath, m); err != nil {
t.Fatalf("Save(seed) error = %v", err)

33
internal/app/operator_helpers_test.go
View File

@@ -1276,13 +1276,32 @@ func TestExecutePublishLoadsRemoteLocks(t *testing.T) {
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render", "analyze"} {
// The publish stage only checks the manifest statuses and source files.
_ = stageName
}
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "final.trimmed.json"), `{"segments":[]}`)
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "final.md"), "# final\n")
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "final.trimmed.md"), "# final trimmed\n")
m := manifest.New("2026-05-03", time.Now().UTC())
m.Campaign = "sample-campaign"
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "extract", "analyze"} {
m.MarkStageSucceeded(stageName, time.Now().UTC(), nil)
}
setCurrentBuiltInEvidence(t, m, "trim", artifacts.TranscriptOutputKindFinalTrimmed, artifacts.ArtifactTranscriptFinalTrimmed, filepath.Join(workRoot, "transcripts", "final.trimmed.json"))
finalMarkdown := filepath.Join(workRoot, "transcripts", "final.md")
trimmedMarkdown := filepath.Join(workRoot, "transcripts", "final.trimmed.md")
finalChecksum, err := artifacts.SHA256File(finalMarkdown)
if err != nil {
t.Fatal(err)
}
trimmedChecksum, err := artifacts.SHA256File(trimmedMarkdown)
if err != nil {
t.Fatal(err)
}
m.MarkStageSucceeded("render", time.Now().UTC(), []manifest.ArtifactRecord{
{Kind: artifacts.TranscriptOutputKindFinalMarkdown, SourceID: artifacts.ArtifactTranscriptFinalMarkdown, LocalPath: finalMarkdown, ProducerRunID: "render-run", Checksum: finalChecksum},
{Kind: artifacts.TranscriptOutputKindFinalTrimmedMarkdown, SourceID: artifacts.ArtifactTranscriptFinalTrimmedMarkdown, LocalPath: trimmedMarkdown, ProducerRunID: "render-run", Checksum: trimmedChecksum},
})
if err := (&manifest.LocalStore{}).Save(context.Background(), filepath.Join(workRoot, "manifest.json"), m); err != nil {
t.Fatal(err)
}
var stdout bytes.Buffer
var stderr bytes.Buffer
@@ -1357,10 +1376,14 @@ func writeOperatorExtractionManifest(t *testing.T, workspaceRoot string) string
}
m := manifest.New("2026-05-03", time.Now().UTC())
m.Campaign = "sample-campaign"
trimmedChecksum, err := artifacts.SHA256File(trimmedPath)
if err != nil {
t.Fatal(err)
}
m.Stages["trim"] = &manifest.StageRecord{
Name: "trim", Status: manifest.StatusSucceeded,
Outputs: []manifest.ArtifactRecord{{
Kind: artifactmodel.TranscriptOutputKindFinalTrimmed, LocalPath: trimmedPath, ProducerRunID: "trim-run-1",
Kind: artifactmodel.TranscriptOutputKindFinalTrimmed, SourceID: artifactmodel.SourceTranscriptFinalTrimmed, LocalPath: trimmedPath, ProducerRunID: "trim-run-1", Checksum: trimmedChecksum,
}},
}
input, err := artifacts.ResolveExtractionInputIdentity(paths, m)

1
internal/app/post_publish_cleanup_test.go
View File

@@ -542,6 +542,7 @@ func publishStageCleanupFixture(t *testing.T) (*config.Config, cleanupSeed, stri
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render", "analyze"} {
seedManifest.MarkStageSucceeded(name, time.Now().UTC(), nil)
}
setCurrentBuiltInEvidence(t, seedManifest, "trim", artifacts.TranscriptOutputKindFinalTrimmed, artifacts.ArtifactTranscriptFinalTrimmed, filepath.Join(artifacts.SessionWorkDirForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID), "transcripts", "final.trimmed.json"))
setAppAnalyzeEvidence(seedManifest, "session_recap", "artifacts/session_recap.md", []byte("# recap\n"))
seedManifest.S3SessionPrefix = artifacts.S3SessionPrefix("dnd", cfg.Session.Campaign, cfg.Session.SessionID)
seedManifest.S3RunPrefix = artifacts.S3RunPrefix(seedManifest.S3SessionPrefix, runID)

8
internal/app/restore_workflow_test.go
View File

@@ -206,6 +206,14 @@ previous_session_id: 2026-04-26
sessionRoot := artifacts.SessionWorkDirForCampaign(workspaceRoot, cfg.Session.Campaign, cfg.Session.SessionID)
mustReadEquals(t, filepath.Join(sessionRoot, "transcripts", "final.trimmed.json"), `{"segments":[]}`+"\n")
localManifest, err := (&manifest.LocalStore{}).Load(context.Background(), filepath.Join(sessionRoot, "manifest.json"))
if err != nil {
t.Fatalf("load restored manifest: %v", err)
}
setCurrentBuiltInEvidence(t, localManifest, "trim", artifacts.TranscriptOutputKindFinalTrimmed, artifacts.ArtifactTranscriptFinalTrimmed, filepath.Join(sessionRoot, "transcripts", "final.trimmed.json"))
if err := (&manifest.LocalStore{}).Save(context.Background(), filepath.Join(sessionRoot, "manifest.json"), localManifest); err != nil {
t.Fatalf("save manifest evidence: %v", err)
}
previousManifestBytes, err := os.ReadFile(filepath.Join(sessionRoot, "previous", "manifest.json"))
if err != nil {
t.Fatalf("read restored previous manifest: %v", err)

35
internal/app/run_stage_test.go
View File

@@ -10,6 +10,7 @@ import (
"testing"
"time"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
@@ -121,7 +122,9 @@ func TestRunStageExecutesOnlySelectedStage(t *testing.T) {
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "base.json"), `{"segments":[]}`)
basePath := filepath.Join(workRoot, "transcripts", "base.json")
mustWriteTestFile(t, basePath, `{"segments":[]}`)
seedCurrentBuiltInEvidence(t, manifestPath, "2026-05-03", "sample-campaign", "merge", artifacts.TranscriptOutputKindBase, artifacts.ArtifactTranscriptBase, basePath)
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "terms: []\n")
var out bytes.Buffer
@@ -172,11 +175,13 @@ func TestRunStageSkipAndForce(t *testing.T) {
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "base.json"), `{"segments":[]}`)
basePath := filepath.Join(workRoot, "transcripts", "base.json")
mustWriteTestFile(t, basePath, `{"segments":[]}`)
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "terms: []\n")
store := &manifest.LocalStore{}
m := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
setCurrentBuiltInEvidence(t, m, "merge", artifacts.TranscriptOutputKindBase, artifacts.ArtifactTranscriptBase, basePath)
m.MarkStageSucceeded("polish", time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
seedCurrentSemanticEvidence(t, loadConfigForSemanticEvidence(t, pipelinePath, campaignPath, sessionPath), m, "polish")
if err := store.Save(context.Background(), manifestPath, m); err != nil {
@@ -207,7 +212,9 @@ func TestRunStageForceMarksDownstreamStaleAndRunContinuesFromStale(t *testing.T)
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "base.json"), `{"segments":[]}`)
basePath := filepath.Join(workRoot, "transcripts", "base.json")
mustWriteTestFile(t, basePath, `{"segments":[]}`)
seedCurrentBuiltInEvidence(t, manifestPath, "2026-05-03", "sample-campaign", "merge", artifacts.TranscriptOutputKindBase, artifacts.ArtifactTranscriptBase, basePath)
store := &manifest.LocalStore{}
materializePrepareResumeFixture(t, pipelinePath, campaignPath, sessionPath)
@@ -218,6 +225,7 @@ func TestRunStageForceMarksDownstreamStaleAndRunContinuesFromStale(t *testing.T)
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render", "analyze", "publish", "notify"} {
seed.MarkStageSucceeded(name, time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
}
setCurrentBuiltInEvidence(t, seed, "merge", artifacts.TranscriptOutputKindBase, artifacts.ArtifactTranscriptBase, basePath)
seedCurrentSemanticEvidence(t, loadConfigForSemanticEvidence(t, pipelinePath, campaignPath, sessionPath), seed, "prepare", "transcribe", "merge")
if err := store.Save(context.Background(), manifestPath, seed); err != nil {
t.Fatalf("save manifest: %v", err)
@@ -257,7 +265,9 @@ func TestRunStageTrimExecutes(t *testing.T) {
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "final.json"), `{"segments":[{"id":1},{"id":2}]}`)
finalPath := filepath.Join(workRoot, "transcripts", "final.json")
mustWriteTestFile(t, finalPath, `{"segments":[{"id":1},{"id":2}]}`)
seedCurrentBuiltInEvidence(t, manifestPath, "2026-05-03", "sample-campaign", "normalize", artifacts.TranscriptOutputKindFinal, artifacts.ArtifactTranscriptFinal, finalPath)
var out bytes.Buffer
err := RunStage(context.Background(), []string{"trim", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &out)
@@ -286,7 +296,9 @@ func TestRunStageNormalizeExecutes(t *testing.T) {
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "polished.json"), `{"segments":[{"id":1},{"id":2}]}`)
polishedPath := filepath.Join(workRoot, "transcripts", "polished.json")
mustWriteTestFile(t, polishedPath, `{"segments":[{"id":1},{"id":2}]}`)
seedCurrentBuiltInEvidence(t, manifestPath, "2026-05-03", "sample-campaign", "polish", artifacts.TranscriptOutputKindPolished, artifacts.ArtifactTranscriptPolished, polishedPath)
var out bytes.Buffer
err := RunStage(context.Background(), []string{"normalize", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &out)
@@ -312,8 +324,17 @@ func TestRunStageRenderExecutes(t *testing.T) {
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "final.json"), `{"segments":[{"id":1}]}`)
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "final.trimmed.json"), `{"segments":[{"id":2}]}`)
finalPath := filepath.Join(workRoot, "transcripts", "final.json")
trimmedPath := filepath.Join(workRoot, "transcripts", "final.trimmed.json")
mustWriteTestFile(t, finalPath, `{"segments":[{"id":1}]}`)
mustWriteTestFile(t, trimmedPath, `{"segments":[{"id":2}]}`)
seed := manifest.New("2026-05-03", time.Now().UTC())
seed.Campaign = "sample-campaign"
setCurrentBuiltInEvidence(t, seed, "normalize", artifacts.TranscriptOutputKindFinal, artifacts.ArtifactTranscriptFinal, finalPath)
setCurrentBuiltInEvidence(t, seed, "trim", artifacts.TranscriptOutputKindFinalTrimmed, artifacts.ArtifactTranscriptFinalTrimmed, trimmedPath)
if err := (&manifest.LocalStore{}).Save(context.Background(), manifestPath, seed); err != nil {
t.Fatal(err)
}
var out bytes.Buffer
err := RunStage(context.Background(), []string{"render", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &out)