Implement a library profile API and built-in profile docs

2026-07-04 17:23:34 -05:00
parent 4669b73d38
commit d60ef66f53
10 changed files with 493 additions and 4 deletions
--- a/engine_test.go
+++ b/engine_test.go
@@ -864,6 +864,236 @@ model: profile-file-model
 	}
 }

+func TestPrepareWorksWithInMemoryProfilesWithoutProfileFiles(t *testing.T) {
+	engine, err := scriptorium.NewEngine(scriptorium.Config{
+		PromptDir: "./examples/prompts",
+		SchemaDir: "./examples/schemas",
+	}, scriptorium.WithProfiles(scriptorium.Profile{
+		ID:       "memory-profile",
+		Endpoint: "http://memory-profile/v1",
+		Model:    "memory-model",
+	}))
+	if err != nil {
+		t.Fatalf("expected engine construction to succeed, got %v", err)
+	}
+
+	prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
+		PromptID:  "generic.markdown_summary",
+		ProfileID: "memory-profile",
+		Inputs: map[string]scriptorium.ArtifactRef{
+			"transcript": scriptorium.Inline("Rin opens the gate."),
+			"glossary":   scriptorium.Inline("gate: A guarded passage."),
+		},
+	})
+	if err != nil {
+		t.Fatalf("expected prepare to succeed, got %v", err)
+	}
+	if prepared.EffectiveModelParams.Model != "memory-model" {
+		t.Fatalf("expected in-memory profile model, got %q", prepared.EffectiveModelParams.Model)
+	}
+}
+
+func TestInMemoryProfilesOverrideBuiltInsAndProfileSources(t *testing.T) {
+	profileFS := fstest.MapFS{
+		"profiles/mistral-small-3.yaml": &fstest.MapFile{Data: []byte(`
+id: mistral-small-3
+endpoint: http://profile-fs/v1
+model: profile-fs-model
+`)},
+	}
+
+	engine, err := scriptorium.NewEngine(scriptorium.Config{
+		PromptDir: "./examples/prompts",
+		SchemaDir: "./examples/schemas",
+	},
+		scriptorium.WithProfileFS(profileFS, "profiles"),
+		scriptorium.WithProfiles(scriptorium.Profile{
+			ID:       "mistral-small-3",
+			Endpoint: "http://memory-profile/v1",
+			Model:    "memory-profile-model",
+		}),
+	)
+	if err != nil {
+		t.Fatalf("expected engine construction to succeed, got %v", err)
+	}
+
+	prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
+		PromptID:  "generic.markdown_summary",
+		ProfileID: "mistral-small-3",
+		Inputs: map[string]scriptorium.ArtifactRef{
+			"transcript": scriptorium.Inline("Rin opens the gate."),
+			"glossary":   scriptorium.Inline("gate: A guarded passage."),
+		},
+	})
+	if err != nil {
+		t.Fatalf("expected prepare to succeed, got %v", err)
+	}
+	if prepared.EffectiveModelParams.Model != "memory-profile-model" {
+		t.Fatalf("expected in-memory profile to have highest precedence, got %q", prepared.EffectiveModelParams.Model)
+	}
+}
+
+func TestWithProfilesRejectsDuplicateIDs(t *testing.T) {
+	_, err := scriptorium.NewEngine(scriptorium.Config{PromptDir: "./examples/prompts"},
+		scriptorium.WithProfiles(
+			scriptorium.Profile{ID: "duplicate", Endpoint: "http://one/v1", Model: "one"},
+			scriptorium.Profile{ID: "duplicate", Endpoint: "http://two/v1", Model: "two"},
+		),
+	)
+	if !errors.Is(err, scriptorium.ErrInvalidConfig) {
+		t.Fatalf("expected ErrInvalidConfig, got %v", err)
+	}
+}
+
+func TestOpenAICompatibleProfileRunsThroughNormalProfilePath(t *testing.T) {
+	fake := &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}}
+	prof := scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
+		ID:             "template-profile",
+		Endpoint:       "http://template/v1",
+		Model:          "template-model",
+		APIKeyRequired: true,
+		ExtraParams: map[string]any{
+			"provider": "template",
+		},
+	})
+
+	engine, err := scriptorium.NewEngine(scriptorium.Config{
+		PromptDir: "./examples/prompts",
+		SchemaDir: "./examples/schemas",
+	}, scriptorium.WithProfiles(prof), scriptorium.WithLLMClient(fake))
+	if err != nil {
+		t.Fatalf("expected engine construction to succeed, got %v", err)
+	}
+
+	_, err = engine.Run(context.Background(), scriptorium.RunRequest{
+		PromptID:  "generic.markdown_summary",
+		ProfileID: "template-profile",
+		APIKey:    "template-key",
+		Inputs: map[string]scriptorium.ArtifactRef{
+			"transcript": scriptorium.Inline("Rin opens the gate."),
+			"glossary":   scriptorium.Inline("gate: A guarded passage."),
+		},
+	})
+	if err != nil {
+		t.Fatalf("expected run to succeed, got %v", err)
+	}
+	if len(fake.requests) != 1 {
+		t.Fatalf("expected one request, got %d", len(fake.requests))
+	}
+	if fake.requests[0].Target.Model != "template-model" || fake.requests[0].APIKey != "template-key" {
+		t.Fatalf("unexpected generated request: %+v", fake.requests[0])
+	}
+	if !reflect.DeepEqual(fake.requests[0].Target.ExtraParams, map[string]any{"provider": "template"}) {
+		t.Fatalf("unexpected extra params: %#v", fake.requests[0].Target.ExtraParams)
+	}
+}
+
+func TestInMemoryProfileAPIKeyRequiredBehavior(t *testing.T) {
+	engine, err := scriptorium.NewEngine(scriptorium.Config{
+		PromptDir: "./examples/prompts",
+		SchemaDir: "./examples/schemas",
+	}, scriptorium.WithProfiles(scriptorium.Profile{
+		ID:             "requires-key",
+		Endpoint:       "http://requires-key/v1",
+		Model:          "requires-key-model",
+		APIKeyRequired: true,
+	}))
+	if err != nil {
+		t.Fatalf("expected engine construction to succeed, got %v", err)
+	}
+
+	req := scriptorium.RunRequest{
+		PromptID:  "generic.markdown_summary",
+		ProfileID: "requires-key",
+		Inputs: map[string]scriptorium.ArtifactRef{
+			"transcript": scriptorium.Inline("Rin opens the gate."),
+			"glossary":   scriptorium.Inline("gate: A guarded passage."),
+		},
+	}
+	_, err = engine.Prepare(context.Background(), req)
+	if !errors.Is(err, scriptorium.ErrInvalidRequest) {
+		t.Fatalf("expected ErrInvalidRequest without API key, got %v", err)
+	}
+	req.APIKey = "direct-required-key"
+	if _, err := engine.Prepare(context.Background(), req); err != nil {
+		t.Fatalf("expected direct API key to satisfy APIKeyRequired, got %v", err)
+	}
+}
+
+func TestInMemoryProfileWithoutAPIKeyRequiredWorksWithoutKey(t *testing.T) {
+	engine, err := scriptorium.NewEngine(scriptorium.Config{
+		PromptDir: "./examples/prompts",
+		SchemaDir: "./examples/schemas",
+	}, scriptorium.WithProfiles(scriptorium.Profile{
+		ID:       "no-key-required",
+		Endpoint: "http://no-key/v1",
+		Model:    "no-key-model",
+	}))
+	if err != nil {
+		t.Fatalf("expected engine construction to succeed, got %v", err)
+	}
+
+	_, err = engine.Prepare(context.Background(), scriptorium.RunRequest{
+		PromptID:  "generic.markdown_summary",
+		ProfileID: "no-key-required",
+		Inputs: map[string]scriptorium.ArtifactRef{
+			"transcript": scriptorium.Inline("Rin opens the gate."),
+			"glossary":   scriptorium.Inline("gate: A guarded passage."),
+		},
+	})
+	if err != nil {
+		t.Fatalf("expected prepare without API key to succeed, got %v", err)
+	}
+}
+
+func TestInMemoryProfileExtraParamsAreCopiedAcrossPublicBoundary(t *testing.T) {
+	fake := &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}}
+	labels := map[string]string{"route": "primary"}
+	ids := []int{1, 2, 3}
+	extraParams := map[string]any{
+		"labels": labels,
+		"ids":    ids,
+	}
+
+	engine, err := scriptorium.NewEngine(scriptorium.Config{
+		PromptDir: "./examples/prompts",
+		SchemaDir: "./examples/schemas",
+	},
+		scriptorium.WithProfiles(scriptorium.Profile{
+			ID:          "copy-profile",
+			Endpoint:    "http://copy/v1",
+			Model:       "copy-model",
+			ExtraParams: extraParams,
+		}),
+		scriptorium.WithLLMClient(fake),
+	)
+	if err != nil {
+		t.Fatalf("expected engine construction to succeed, got %v", err)
+	}
+	labels["route"] = "mutated-before-run"
+	ids[0] = 99
+	extraParams["added"] = "mutated"
+
+	_, err = engine.Run(context.Background(), scriptorium.RunRequest{
+		PromptID:  "generic.markdown_summary",
+		ProfileID: "copy-profile",
+		Inputs: map[string]scriptorium.ArtifactRef{
+			"transcript": scriptorium.Inline("Rin opens the gate."),
+			"glossary":   scriptorium.Inline("gate: A guarded passage."),
+		},
+	})
+	if err != nil {
+		t.Fatalf("expected run to succeed, got %v", err)
+	}
+	want := map[string]any{
+		"labels": map[string]string{"route": "primary"},
+		"ids":    []int{1, 2, 3},
+	}
+	if !reflect.DeepEqual(fake.requests[0].Target.ExtraParams, want) {
+		t.Fatalf("captured extra params changed after mutation:\ngot=%#v\nwant=%#v", fake.requests[0].Target.ExtraParams, want)
+	}
+}
+
 func TestRunStructuredOutputWorksWithSchemaFS(t *testing.T) {
 	fake := &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: `{"events":[]}`}}
 	engine, err := scriptorium.NewEngine(scriptorium.Config{