Add in-memory pipeline run command

2026-07-04 01:06:04 +00:00
parent 361b1f53f4
commit ae218d7c57
4 changed files with 569 additions and 3 deletions
--- a/internal/cli/run_test.go
+++ b/internal/cli/run_test.go
@@ -3,12 +3,15 @@ package cli
 import (
 	"bytes"
 	"context"
+	"encoding/json"
+	"errors"
 	"os"
 	"path/filepath"
 	"reflect"
 	"strings"
 	"testing"

+	"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
 	"gitea.maximumdirect.net/eric/notarius/internal/core/config"
 	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
 	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
@@ -495,6 +498,211 @@ func TestProductionLLMClientFactoryReturnsScheduledClientAndManifestMetadata(t *
 	}
 }

+func TestRunPipelineMissingPipelineID(t *testing.T) {
+	configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "dnd/spells"))
+	inputPath := writeSeriatimInput(t)
+	var stdout bytes.Buffer
+	var stderr bytes.Buffer
+
+	code := RunWithOptions([]string{"run", "--config", configPath, "--input", inputPath}, &stdout, &stderr, Options{})
+
+	if code != 2 {
+		t.Fatalf("RunWithOptions() code = %d, want 2", code)
+	}
+	if !strings.Contains(stderr.String(), "pipeline ID") {
+		t.Fatalf("stderr = %q, want missing pipeline ID error", stderr.String())
+	}
+}
+
+func TestRunPipelineMissingInputFlag(t *testing.T) {
+	configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "dnd/spells"))
+	var stdout bytes.Buffer
+	var stderr bytes.Buffer
+
+	code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath}, &stdout, &stderr, Options{})
+
+	if code != 2 {
+		t.Fatalf("RunWithOptions() code = %d, want 2", code)
+	}
+	if !strings.Contains(stderr.String(), "--input") {
+		t.Fatalf("stderr = %q, want missing input error", stderr.String())
+	}
+}
+
+func TestRunPipelineRejectsUnknownFlag(t *testing.T) {
+	var stdout bytes.Buffer
+	var stderr bytes.Buffer
+
+	code := RunWithOptions([]string{"run", "dnd-session", "--extractor", "dnd/spells"}, &stdout, &stderr, Options{})
+
+	if code != 2 {
+		t.Fatalf("RunWithOptions() code = %d, want 2", code)
+	}
+	if !strings.Contains(stderr.String(), "flag provided but not defined") {
+		t.Fatalf("stderr = %q, want invalid flag error", stderr.String())
+	}
+}
+
+func TestRunPipelineUnknownPipeline(t *testing.T) {
+	configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "dnd/spells"))
+	inputPath := writeSeriatimInput(t)
+	var stdout bytes.Buffer
+	var stderr bytes.Buffer
+
+	code := RunWithOptions([]string{"run", "missing", "--config", configPath, "--input", inputPath}, &stdout, &stderr, Options{
+		LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil),
+	})
+
+	if code != 1 {
+		t.Fatalf("RunWithOptions() code = %d, want 1", code)
+	}
+	if !strings.Contains(stderr.String(), "pipeline \"missing\" is not configured") {
+		t.Fatalf("stderr = %q, want unknown pipeline error", stderr.String())
+	}
+}
+
+func TestRunPipelineUnknownOnlyLane(t *testing.T) {
+	configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "dnd/spells"))
+	inputPath := writeSeriatimInput(t)
+	var stdout bytes.Buffer
+	var stderr bytes.Buffer
+
+	code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--only", "missing"}, &stdout, &stderr, Options{
+		LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil),
+	})
+
+	if code != 1 {
+		t.Fatalf("RunWithOptions() code = %d, want 1", code)
+	}
+	if !strings.Contains(stderr.String(), "selected artifact lane") {
+		t.Fatalf("stderr = %q, want selected lane error", stderr.String())
+	}
+}
+
+func TestRunPipelineInvalidInputPath(t *testing.T) {
+	configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "dnd/spells"))
+	inputPath := filepath.Join(t.TempDir(), "missing.json")
+	var stdout bytes.Buffer
+	var stderr bytes.Buffer
+
+	code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath}, &stdout, &stderr, Options{
+		LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil),
+	})
+
+	if code != 1 {
+		t.Fatalf("RunWithOptions() code = %d, want 1", code)
+	}
+	if !strings.Contains(stderr.String(), "read input") {
+		t.Fatalf("stderr = %q, want input read error", stderr.String())
+	}
+}
+
+func TestRunPipelineSuccessUsesProductionRegistriesAndFakeLLM(t *testing.T) {
+	configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "dnd/spells"))
+	inputPath := writeSeriatimInput(t)
+	client := newFakeRunLLMClient(false)
+	var stdout bytes.Buffer
+	var stderr bytes.Buffer
+
+	code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath}, &stdout, &stderr, Options{
+		LLMClientFactory: fakeLLMFactory(client, nil),
+	})
+
+	if code != 0 {
+		t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
+	}
+	if client.calls != 1 {
+		t.Fatalf("LLM calls = %d, want 1", client.calls)
+	}
+	for _, want := range []string{"dnd-session", "approved=1", "rejected=0"} {
+		if !strings.Contains(stdout.String(), want) {
+			t.Fatalf("stdout = %q, want substring %q", stdout.String(), want)
+		}
+	}
+	if stderr.Len() != 0 {
+		t.Fatalf("stderr = %q, want empty", stderr.String())
+	}
+}
+
+func TestRunPipelineOnlySelectsRequestedLane(t *testing.T) {
+	configPath := writeTestConfig(t, mvpConfigYAMLForLanes("dnd-session", "spells", "rituals"))
+	inputPath := writeSeriatimInput(t)
+	client := newFakeRunLLMClient(false)
+	var stdout bytes.Buffer
+	var stderr bytes.Buffer
+
+	code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--only", "spells"}, &stdout, &stderr, Options{
+		LLMClientFactory: fakeLLMFactory(client, nil),
+	})
+
+	if code != 0 {
+		t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
+	}
+	if client.calls != 1 {
+		t.Fatalf("LLM calls = %d, want only selected lane to run once", client.calls)
+	}
+	if !strings.Contains(stdout.String(), "approved=1") {
+		t.Fatalf("stdout = %q, want approved count", stdout.String())
+	}
+}
+
+func TestRunPipelineLLMFactoryFailure(t *testing.T) {
+	configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "dnd/spells"))
+	inputPath := writeSeriatimInput(t)
+	var stdout bytes.Buffer
+	var stderr bytes.Buffer
+
+	code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath}, &stdout, &stderr, Options{
+		LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), errors.New("factory unavailable")),
+	})
+
+	if code != 1 {
+		t.Fatalf("RunWithOptions() code = %d, want 1", code)
+	}
+	if !strings.Contains(stderr.String(), "create LLM client") || !strings.Contains(stderr.String(), "factory unavailable") {
+		t.Fatalf("stderr = %q, want LLM factory error", stderr.String())
+	}
+}
+
+func TestRunPipelineValidationRejectionCompletesSuccessfully(t *testing.T) {
+	configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "dnd/spells"))
+	inputPath := writeSeriatimInput(t)
+	client := newFakeRunLLMClient(true)
+	var stdout bytes.Buffer
+	var stderr bytes.Buffer
+
+	code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath}, &stdout, &stderr, Options{
+		LLMClientFactory: fakeLLMFactory(client, nil),
+	})
+
+	if code != 0 {
+		t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
+	}
+	if !strings.Contains(stdout.String(), "approved=0") || !strings.Contains(stdout.String(), "rejected=1") {
+		t.Fatalf("stdout = %q, want rejection counts", stdout.String())
+	}
+}
+
+func TestRunPipelineLLMProfileOverrideSelectsFactoryProfile(t *testing.T) {
+	configPath := writeTestConfig(t, mvpConfigYAMLWithProfiles("dnd-session"))
+	inputPath := writeSeriatimInput(t)
+	client := newFakeRunLLMClient(false)
+	factory := &recordingLLMFactory{client: client}
+	var stdout bytes.Buffer
+	var stderr bytes.Buffer
+
+	code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--llm-profile", "runtime"}, &stdout, &stderr, Options{
+		LLMClientFactory: factory.build,
+	})
+
+	if code != 0 {
+		t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
+	}
+	if got, want := strings.Join(factory.profileIDs, ","), "runtime"; got != want {
+		t.Fatalf("factory profile IDs = %q, want %q", got, want)
+	}
+}
+
 func writeTestConfig(t *testing.T, content string) string {
 	t.Helper()
 	return writeFile(t, "config.yml", content)
@@ -540,6 +748,115 @@ pipelines:
 `
 }

+func mvpConfigYAMLForLanes(pipelineID string, laneIDs ...string) string {
+	var b strings.Builder
+	b.WriteString("version: 1\n")
+	b.WriteString("pipelines:\n")
+	b.WriteString("  " + pipelineID + ":\n")
+	b.WriteString("    input: seriatim\n")
+	b.WriteString("    artifacts:\n")
+	for _, laneID := range laneIDs {
+		b.WriteString("      " + laneID + ":\n")
+		b.WriteString("        extract: dnd/spells\n")
+	}
+	return b.String()
+}
+
+func mvpConfigYAMLWithProfiles(pipelineID string) string {
+	return `version: 1
+llm_profiles:
+  default:
+    provider: openai-compatible
+  runtime:
+    provider: openai-compatible
+pipelines:
+  ` + pipelineID + `:
+    input: seriatim
+    artifacts:
+      spells:
+        extract: dnd/spells
+`
+}
+
+func writeSeriatimInput(t *testing.T) string {
+	t.Helper()
+	return writeFile(t, "source.json", `{
+  "metadata": {
+    "id": "session-alpha"
+  },
+  "segments": [
+    {
+      "id": "seg-001",
+      "start": 0,
+      "end": 1,
+      "speaker": "Aria",
+      "text": "Aria casts Cure Wounds."
+    }
+  ]
+}`)
+}
+
+type fakeRunLLMClient struct {
+	invalidSourceRef bool
+	calls            int
+}
+
+func newFakeRunLLMClient(invalidSourceRef bool) *fakeRunLLMClient {
+	return &fakeRunLLMClient{invalidSourceRef: invalidSourceRef}
+}
+
+func (client *fakeRunLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
+	client.calls++
+	startUnitID := "seg-001"
+	if client.invalidSourceRef {
+		startUnitID = "missing-segment"
+	}
+	payload := map[string]any{
+		"spell_casts": []map[string]any{
+			{
+				"caster":                "Aria",
+				"spell":                 "Cure Wounds",
+				"effect":                "Heals a wounded ally.",
+				"narrative_description": "Aria casts Cure Wounds.",
+				"source_refs": []map[string]string{
+					{
+						"source_id":     "session-alpha",
+						"start_unit_id": startUnitID,
+						"end_unit_id":   "seg-001",
+					},
+				},
+			},
+		},
+	}
+	encoded, err := json.Marshal(payload)
+	if err != nil {
+		return contracts.StructuredCompletionResponse{}, err
+	}
+	if err := json.Unmarshal(encoded, out); err != nil {
+		return contracts.StructuredCompletionResponse{}, err
+	}
+	return contracts.StructuredCompletionResponse{Content: encoded}, nil
+}
+
+func fakeLLMFactory(client contracts.StructuredLLMClient, err error) LLMClientFactory {
+	return func(ctx context.Context, cfg config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
+		if err != nil {
+			return nil, nil, err
+		}
+		return client, []artifacts.LLMProfileManifest{{ID: strings.TrimSpace(profileID)}}, nil
+	}
+}
+
+type recordingLLMFactory struct {
+	client     contracts.StructuredLLMClient
+	profileIDs []string
+}
+
+func (factory *recordingLLMFactory) build(ctx context.Context, cfg config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
+	factory.profileIDs = append(factory.profileIDs, strings.TrimSpace(profileID))
+	return factory.client, []artifacts.LLMProfileManifest{{ID: strings.TrimSpace(profileID)}}, nil
+}
+
 func fakeCatalog(t *testing.T) pipeline.ModuleCatalog {
 	t.Helper()
 	inputs := pipeline.NewInputAdapterRegistry()