eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Switch config to Scriptorium profiles

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-05 18:03:23 +00:00
parent 49d94cc2e9
commit 0fc740470f
22 changed files with 319 additions and 797 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
internal/cli/catalog.go
View File

@@ -8,7 +8,6 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/dnd/scenes"
"gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/generic"
@@ -130,47 +129,5 @@ func productionLLMClientFactory(ctx context.Context, cfg config.Config, profileI
return nil, nil, err
}
trimmedID := strings.TrimSpace(profileID)
if trimmedID == "" {
trimmedID = pipeline.DefaultLLMProfile
}
profile, ok := cfg.LLMProfile(trimmedID)
if !ok {
return nil, nil, fmt.Errorf("LLM profile %q is not configured", trimmedID)
}
clientCfg, err := cfg.OpenAICompatibleClientConfig(trimmedID)
if err != nil {
return nil, nil, err
}
client, err := llm.NewOpenAICompatibleClient(clientCfg)
if err != nil {
return nil, nil, fmt.Errorf("create LLM client for profile %q: %w", trimmedID, err)
}
scheduler, err := llm.NewScheduler(effectiveLLMConcurrency(cfg, profile))
if err != nil {
return nil, nil, fmt.Errorf("create LLM scheduler for profile %q: %w", trimmedID, err)
}
provider := strings.TrimSpace(profile.Provider)
if provider == "" {
provider = "openai-compatible"
}
metadata := []artifacts.LLMProfileManifest{
{
ID: trimmedID,
Provider: provider,
Model: strings.TrimSpace(profile.Model),
},
}
return llm.NewScheduledClient(client, scheduler), metadata, nil
}
func effectiveLLMConcurrency(cfg config.Config, profile config.LLMProfile) int {
if profile.MaxConcurrency > 0 {
return profile.MaxConcurrency
}
if cfg.Concurrency.TotalLLM > 0 {
return cfg.Concurrency.TotalLLM
}
return 1
return nil, nil, fmt.Errorf("create Scriptorium-backed LLM client for profile %q: not implemented yet", trimmedID)
}

26
internal/cli/run.go
View File

@@ -194,6 +194,10 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
if err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
}
profileIDs := effectiveLLMProfileIDs(effective.ResolvedPipeline)
if err := validateExplicitScriptoriumProfiles(context.Background(), effective.Config, profileIDs); err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
}
workingDir, err := os.Getwd()
if err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("resolve working directory: %w", err))
@@ -220,11 +224,6 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics resolved references: %w", err))
}
profileIDs := effectiveLLMProfileIDs(effective.ResolvedPipeline)
if len(profileIDs) != 1 {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("pipeline %q uses %d distinct LLM profiles; current runs require exactly one: %s", pipelineID, len(profileIDs), strings.Join(profileIDs, ", ")))
}
rawInput, err := os.ReadFile(strings.TrimSpace(*inputPath))
if err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("read input %q: %w", strings.TrimSpace(*inputPath), err))
@@ -236,9 +235,13 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
}
ctx := context.Background()
llmClient, llmProfiles, err := opts.LLMClientFactory(ctx, effective.Config, profileIDs[0])
factoryProfileID := ""
if len(profileIDs) == 1 {
factoryProfileID = profileIDs[0]
}
llmClient, llmProfiles, err := opts.LLMClientFactory(ctx, effective.Config, factoryProfileID)
if err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("create LLM client for profile %q: %w", profileIDs[0], err))
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("create LLM client for profile %q: %w", factoryProfileID, err))
}
output, err := pipeline.New(registries).Run(ctx, pipeline.RunInput{
@@ -571,11 +574,16 @@ func runConfigValidate(args []string, stdout, stderr io.Writer, opts Options) in
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 1
}
if _, err := cfg.Resolve(config.ResolveInput{
effective, err := cfg.Resolve(config.ResolveInput{
PipelineID: *pipelineID,
Only: only,
Catalog: catalog,
}); err != nil {
})
if err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 1
}
if err := validateExplicitScriptoriumProfiles(context.Background(), effective.Config, effectiveLLMProfileIDs(effective.ResolvedPipeline)); err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 1
}

160
internal/cli/run_test.go
View File

@@ -228,7 +228,7 @@ func TestRunConfigValidateUnknownProductionModuleIncludesContext(t *testing.T) {
}
func TestRunConfigValidateReportsParseErrors(t *testing.T) {
configPath := writeFile(t, "config.yml", "version: 2\n")
configPath := writeFile(t, "config.yml", "version: 1\n")
var stdout bytes.Buffer
var stderr bytes.Buffer
@@ -372,11 +372,10 @@ func TestRunUsesNotariusConfigWhenConfigFlagAbsent(t *testing.T) {
}
}
func TestRunConfigValidateResolvesAPIKeyEnvThroughOptions(t *testing.T) {
configPath := writeTestConfig(t, `version: 1
func TestRunConfigValidateRejectsStaleLLMProfiles(t *testing.T) {
configPath := writeTestConfig(t, `version: 2
llm_profiles:
default:
api_key_env: NOTARIUS_TEST_API_KEY
default: {}
pipelines:
example:
input: fake/input
@@ -387,12 +386,13 @@ pipelines:
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{"config", "validate", "--config", configPath}, &stdout, &stderr, Options{
LookupEnv: mapLookup(map[string]string{"NOTARIUS_TEST_API_KEY": "secret"}),
})
code := RunWithOptions([]string{"config", "validate", "--config", configPath}, &stdout, &stderr, Options{})
if code != 0 {
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
if code != 1 {
t.Fatalf("RunWithOptions() code = %d, want 1", code)
}
if !strings.Contains(stderr.String(), "llm_profiles") {
t.Fatalf("stderr = %q, want stale llm_profiles error", stderr.String())
}
}
@@ -445,80 +445,15 @@ func TestRunInvalidFlagsExitTwo(t *testing.T) {
}
}
func TestProductionLLMClientFactoryRejectsMissingProfile(t *testing.T) {
func TestProductionLLMClientFactoryReportsPendingScriptoriumRuntime(t *testing.T) {
cfg := config.Default()
_, _, err := productionLLMClientFactory(context.Background(), cfg, "missing")
_, _, err := productionLLMClientFactory(context.Background(), cfg, "mistral-small-3")
if err == nil {
t.Fatal("productionLLMClientFactory() error = nil, want error")
}
if !strings.Contains(err.Error(), "LLM profile") || !strings.Contains(err.Error(), "missing") {
t.Fatalf("error = %q, want missing profile context", err.Error())
}
}
func TestProductionLLMClientFactoryRejectsInvalidProfile(t *testing.T) {
tests := []struct {
name string
profile config.LLMProfile
want string
}{
{
name: "unsupported provider",
profile: config.LLMProfile{Provider: "other", BaseURL: "https://example.test", Model: "model"},
want: "not supported",
},
{
name: "missing base url",
profile: config.LLMProfile{Provider: "openai-compatible", Model: "model"},
want: "base URL",
},
{
name: "missing model",
profile: config.LLMProfile{Provider: "openai-compatible", BaseURL: "https://example.test"},
want: "model",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cfg := config.Default()
cfg.LLMProfiles = map[string]config.LLMProfile{"default": test.profile}
_, _, err := productionLLMClientFactory(context.Background(), cfg, "default")
if err == nil {
t.Fatal("productionLLMClientFactory() error = nil, want error")
}
if !strings.Contains(err.Error(), test.want) {
t.Fatalf("error = %q, want substring %q", err.Error(), test.want)
}
})
}
}
func TestProductionLLMClientFactoryReturnsScheduledClientAndManifestMetadata(t *testing.T) {
cfg := config.Default()
cfg.LLMProfiles = map[string]config.LLMProfile{
"default": {
Provider: "openai-compatible",
BaseURL: "https://example.test",
Model: "model-a",
MaxConcurrency: 2,
},
}
client, metadata, err := productionLLMClientFactory(context.Background(), cfg, "default")
if err != nil {
t.Fatalf("productionLLMClientFactory() error = %v, want nil", err)
}
if client == nil {
t.Fatal("client = nil, want scheduled client")
}
if len(metadata) != 1 {
t.Fatalf("len(metadata) = %d, want 1", len(metadata))
}
if metadata[0].ID != "default" || metadata[0].Provider != "openai-compatible" || metadata[0].Model != "model-a" {
t.Fatalf("metadata = %#v, want profile-safe model metadata", metadata)
if !strings.Contains(err.Error(), "Scriptorium-backed LLM client") || !strings.Contains(err.Error(), "not implemented yet") {
t.Fatalf("error = %q, want pending Scriptorium runtime context", err.Error())
}
}
@@ -718,7 +653,8 @@ func TestRunPipelineValidationRejectionCompletesSuccessfully(t *testing.T) {
}
func TestRunPipelineLLMProfileOverrideSelectsFactoryProfile(t *testing.T) {
configPath := writeTestConfig(t, mvpConfigYAMLWithProfiles("dnd-session"))
profilePath := writeScriptoriumProfileFile(t, "runtime", "http://profile.test/v1", "test-model")
configPath := writeTestConfig(t, mvpConfigYAMLWithProfileFile("dnd-session", profilePath))
inputPath := writeSeriatimInput(t)
outputDir := t.TempDir()
diagnosticsDir := t.TempDir()
@@ -739,6 +675,35 @@ func TestRunPipelineLLMProfileOverrideSelectsFactoryProfile(t *testing.T) {
}
}
func TestRunConfigValidateRejectsUnknownExplicitScriptoriumProfile(t *testing.T) {
profilePath := writeScriptoriumProfileFile(t, "known", "http://profile.test/v1", "test-model")
configPath := writeTestConfig(t, `version: 2
scriptorium:
profile_file: `+profilePath+`
pipelines:
example:
input: fake/input
artifacts:
events:
extract:
module: fake/extract
llm_profile: missing
`)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "example"}, &stdout, &stderr, Options{
Catalog: fakeCatalog(t),
})
if code != 1 {
t.Fatalf("RunWithOptions() code = %d, want 1", code)
}
if !strings.Contains(stderr.String(), "Scriptorium profile") || !strings.Contains(stderr.String(), "missing") {
t.Fatalf("stderr = %q, want unknown Scriptorium profile", stderr.String())
}
}
func TestRunPipelineSessionIDFlagRecordsExplicitTrimmedValue(t *testing.T) {
configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "dnd/spells"))
inputPath := writeSeriatimInput(t)
@@ -2117,7 +2082,7 @@ func testConfigYAML(pipelineID string, laneIDs ...string) string {
func testConfigYAMLForPipelines(pipelines map[string][]string) string {
var b strings.Builder
b.WriteString("version: 1\n")
b.WriteString("version: 2\n")
b.WriteString("pipelines:\n")
for pipelineID, laneIDs := range pipelines {
b.WriteString(" " + pipelineID + ":\n")
@@ -2133,7 +2098,7 @@ func testConfigYAMLForPipelines(pipelines map[string][]string) string {
func testConfigYAMLWithReferences(pipelineID string, laneID string, references map[string]string) string {
var b strings.Builder
b.WriteString("version: 1\n")
b.WriteString("version: 2\n")
b.WriteString("pipelines:\n")
b.WriteString(" " + pipelineID + ":\n")
b.WriteString(" input: fake/input\n")
@@ -2154,7 +2119,7 @@ func testConfigYAMLWithReferences(pipelineID string, laneID string, references m
func testConfigYAMLWithPipelineReferences(pipelineID string, laneID string, references map[string]string) string {
var b strings.Builder
b.WriteString("version: 1\n")
b.WriteString("version: 2\n")
b.WriteString("pipelines:\n")
b.WriteString(" " + pipelineID + ":\n")
b.WriteString(" input: fake/input\n")
@@ -2175,7 +2140,7 @@ func testConfigYAMLWithPipelineReferences(pipelineID string, laneID string, refe
func testConfigYAMLWithReferencesAndDiagnostics(pipelineID string, laneID string, diagnosticsDir string, references map[string]string) string {
var b strings.Builder
b.WriteString("version: 1\n")
b.WriteString("version: 2\n")
b.WriteString("diagnostics:\n")
b.WriteString(" work_dir: " + diagnosticsDir + "\n")
b.WriteString(" retention: always\n")
@@ -2198,7 +2163,7 @@ func testConfigYAMLWithReferencesAndDiagnostics(pipelineID string, laneID string
}
func mvpConfigYAML(pipelineID string, extractor string) string {
return `version: 1
return `version: 2
pipelines:
` + pipelineID + `:
input: seriatim
@@ -2209,7 +2174,7 @@ pipelines:
}
func mvpConfigYAMLWithChunk(pipelineID string, chunker string, extractor string) string {
return `version: 1
return `version: 2
pipelines:
` + pipelineID + `:
input: seriatim
@@ -2222,7 +2187,7 @@ pipelines:
func mvpConfigYAMLForLanes(pipelineID string, laneIDs ...string) string {
var b strings.Builder
b.WriteString("version: 1\n")
b.WriteString("version: 2\n")
b.WriteString("pipelines:\n")
b.WriteString(" " + pipelineID + ":\n")
b.WriteString(" input: seriatim\n")
@@ -2234,13 +2199,10 @@ func mvpConfigYAMLForLanes(pipelineID string, laneIDs ...string) string {
return b.String()
}
func mvpConfigYAMLWithProfiles(pipelineID string) string {
return `version: 1
llm_profiles:
default:
provider: openai-compatible
runtime:
provider: openai-compatible
func mvpConfigYAMLWithProfileFile(pipelineID string, profileFile string) string {
return `version: 2
scriptorium:
profile_file: ` + profileFile + `
pipelines:
` + pipelineID + `:
input: seriatim
@@ -2250,8 +2212,16 @@ pipelines:
`
}
func writeScriptoriumProfileFile(t *testing.T, id string, endpoint string, model string) string {
t.Helper()
return writeFile(t, id+".profile.yml", `id: `+id+`
endpoint: `+endpoint+`
model: `+model+`
`)
}
func mvpConfigYAMLWithDiagnostics(pipelineID, diagnosticsDir, retention string) string {
return `version: 1
return `version: 2
diagnostics:
work_dir: ` + diagnosticsDir + `
retention: ` + retention + `

68
internal/cli/scriptorium_profiles.go Normal file
View File

@@ -0,0 +1,68 @@
package cli
import (
"context"
"errors"
"fmt"
"testing/fstest"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/scriptorium"
)
const profileCheckPromptID = "notarius.profile.check"
var profileCheckPromptFS = fstest.MapFS{
"prompts/profile-check.yaml": &fstest.MapFile{Data: []byte(`id: notarius.profile.check
version: "1.0.0"
default_profile: mistral-small-3
inputs:
- name: transcript
required: true
messages:
- role: user
content: "{{input \"transcript\"}}"
output:
format: text
validation_mode: none
repair_attempts: 0
`)},
}
func validateExplicitScriptoriumProfiles(ctx context.Context, cfg config.Config, profileIDs []string) error {
if len(profileIDs) == 0 {
return nil
}
engine, err := newProfileValidationEngine(cfg)
if err != nil {
return fmt.Errorf("load Scriptorium profiles: %w", err)
}
for _, profileID := range profileIDs {
if _, err := engine.Prepare(ctx, scriptorium.RunRequest{
PromptID: profileCheckPromptID,
ProfileID: profileID,
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline("profile check"),
},
}); err != nil {
if errors.Is(err, scriptorium.ErrProfileNotFound) {
return fmt.Errorf("Scriptorium profile %q is not configured", profileID)
}
return fmt.Errorf("validate Scriptorium profile %q: %w", profileID, err)
}
}
return nil
}
func newProfileValidationEngine(cfg config.Config) (*scriptorium.Engine, error) {
opts := []scriptorium.Option{
scriptorium.WithPromptFS(profileCheckPromptFS, "prompts"),
}
if cfg.Scriptorium.ProfileFile != "" {
opts = append(opts, scriptorium.WithProfileFile(cfg.Scriptorium.ProfileFile))
}
return scriptorium.NewEngine(scriptorium.Config{
PromptDir: "unused",
ProfileDir: cfg.Scriptorium.ProfileDir,
}, opts...)
}