eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Switch config to Scriptorium profiles

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-05 18:03:23 +00:00
parent 49d94cc2e9
commit 0fc740470f
22 changed files with 319 additions and 797 deletions
Download Patch File Download Diff File Expand all files Collapse all files

160
internal/cli/run_test.go
View File

@@ -228,7 +228,7 @@ func TestRunConfigValidateUnknownProductionModuleIncludesContext(t *testing.T) {
}
func TestRunConfigValidateReportsParseErrors(t *testing.T) {
configPath := writeFile(t, "config.yml", "version: 2\n")
configPath := writeFile(t, "config.yml", "version: 1\n")
var stdout bytes.Buffer
var stderr bytes.Buffer
@@ -372,11 +372,10 @@ func TestRunUsesNotariusConfigWhenConfigFlagAbsent(t *testing.T) {
}
}
func TestRunConfigValidateResolvesAPIKeyEnvThroughOptions(t *testing.T) {
configPath := writeTestConfig(t, `version: 1
func TestRunConfigValidateRejectsStaleLLMProfiles(t *testing.T) {
configPath := writeTestConfig(t, `version: 2
llm_profiles:
default:
api_key_env: NOTARIUS_TEST_API_KEY
default: {}
pipelines:
example:
input: fake/input
@@ -387,12 +386,13 @@ pipelines:
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{"config", "validate", "--config", configPath}, &stdout, &stderr, Options{
LookupEnv: mapLookup(map[string]string{"NOTARIUS_TEST_API_KEY": "secret"}),
})
code := RunWithOptions([]string{"config", "validate", "--config", configPath}, &stdout, &stderr, Options{})
if code != 0 {
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
if code != 1 {
t.Fatalf("RunWithOptions() code = %d, want 1", code)
}
if !strings.Contains(stderr.String(), "llm_profiles") {
t.Fatalf("stderr = %q, want stale llm_profiles error", stderr.String())
}
}
@@ -445,80 +445,15 @@ func TestRunInvalidFlagsExitTwo(t *testing.T) {
}
}
func TestProductionLLMClientFactoryRejectsMissingProfile(t *testing.T) {
func TestProductionLLMClientFactoryReportsPendingScriptoriumRuntime(t *testing.T) {
cfg := config.Default()
_, _, err := productionLLMClientFactory(context.Background(), cfg, "missing")
_, _, err := productionLLMClientFactory(context.Background(), cfg, "mistral-small-3")
if err == nil {
t.Fatal("productionLLMClientFactory() error = nil, want error")
}
if !strings.Contains(err.Error(), "LLM profile") || !strings.Contains(err.Error(), "missing") {
t.Fatalf("error = %q, want missing profile context", err.Error())
}
}
func TestProductionLLMClientFactoryRejectsInvalidProfile(t *testing.T) {
tests := []struct {
name string
profile config.LLMProfile
want string
}{
{
name: "unsupported provider",
profile: config.LLMProfile{Provider: "other", BaseURL: "https://example.test", Model: "model"},
want: "not supported",
},
{
name: "missing base url",
profile: config.LLMProfile{Provider: "openai-compatible", Model: "model"},
want: "base URL",
},
{
name: "missing model",
profile: config.LLMProfile{Provider: "openai-compatible", BaseURL: "https://example.test"},
want: "model",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cfg := config.Default()
cfg.LLMProfiles = map[string]config.LLMProfile{"default": test.profile}
_, _, err := productionLLMClientFactory(context.Background(), cfg, "default")
if err == nil {
t.Fatal("productionLLMClientFactory() error = nil, want error")
}
if !strings.Contains(err.Error(), test.want) {
t.Fatalf("error = %q, want substring %q", err.Error(), test.want)
}
})
}
}
func TestProductionLLMClientFactoryReturnsScheduledClientAndManifestMetadata(t *testing.T) {
cfg := config.Default()
cfg.LLMProfiles = map[string]config.LLMProfile{
"default": {
Provider: "openai-compatible",
BaseURL: "https://example.test",
Model: "model-a",
MaxConcurrency: 2,
},
}
client, metadata, err := productionLLMClientFactory(context.Background(), cfg, "default")
if err != nil {
t.Fatalf("productionLLMClientFactory() error = %v, want nil", err)
}
if client == nil {
t.Fatal("client = nil, want scheduled client")
}
if len(metadata) != 1 {
t.Fatalf("len(metadata) = %d, want 1", len(metadata))
}
if metadata[0].ID != "default" || metadata[0].Provider != "openai-compatible" || metadata[0].Model != "model-a" {
t.Fatalf("metadata = %#v, want profile-safe model metadata", metadata)
if !strings.Contains(err.Error(), "Scriptorium-backed LLM client") || !strings.Contains(err.Error(), "not implemented yet") {
t.Fatalf("error = %q, want pending Scriptorium runtime context", err.Error())
}
}
@@ -718,7 +653,8 @@ func TestRunPipelineValidationRejectionCompletesSuccessfully(t *testing.T) {
}
func TestRunPipelineLLMProfileOverrideSelectsFactoryProfile(t *testing.T) {
configPath := writeTestConfig(t, mvpConfigYAMLWithProfiles("dnd-session"))
profilePath := writeScriptoriumProfileFile(t, "runtime", "http://profile.test/v1", "test-model")
configPath := writeTestConfig(t, mvpConfigYAMLWithProfileFile("dnd-session", profilePath))
inputPath := writeSeriatimInput(t)
outputDir := t.TempDir()
diagnosticsDir := t.TempDir()
@@ -739,6 +675,35 @@ func TestRunPipelineLLMProfileOverrideSelectsFactoryProfile(t *testing.T) {
}
}
func TestRunConfigValidateRejectsUnknownExplicitScriptoriumProfile(t *testing.T) {
profilePath := writeScriptoriumProfileFile(t, "known", "http://profile.test/v1", "test-model")
configPath := writeTestConfig(t, `version: 2
scriptorium:
profile_file: `+profilePath+`
pipelines:
example:
input: fake/input
artifacts:
events:
extract:
module: fake/extract
llm_profile: missing
`)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "example"}, &stdout, &stderr, Options{
Catalog: fakeCatalog(t),
})
if code != 1 {
t.Fatalf("RunWithOptions() code = %d, want 1", code)
}
if !strings.Contains(stderr.String(), "Scriptorium profile") || !strings.Contains(stderr.String(), "missing") {
t.Fatalf("stderr = %q, want unknown Scriptorium profile", stderr.String())
}
}
func TestRunPipelineSessionIDFlagRecordsExplicitTrimmedValue(t *testing.T) {
configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "dnd/spells"))
inputPath := writeSeriatimInput(t)
@@ -2117,7 +2082,7 @@ func testConfigYAML(pipelineID string, laneIDs ...string) string {
func testConfigYAMLForPipelines(pipelines map[string][]string) string {
var b strings.Builder
b.WriteString("version: 1\n")
b.WriteString("version: 2\n")
b.WriteString("pipelines:\n")
for pipelineID, laneIDs := range pipelines {
b.WriteString(" " + pipelineID + ":\n")
@@ -2133,7 +2098,7 @@ func testConfigYAMLForPipelines(pipelines map[string][]string) string {
func testConfigYAMLWithReferences(pipelineID string, laneID string, references map[string]string) string {
var b strings.Builder
b.WriteString("version: 1\n")
b.WriteString("version: 2\n")
b.WriteString("pipelines:\n")
b.WriteString(" " + pipelineID + ":\n")
b.WriteString(" input: fake/input\n")
@@ -2154,7 +2119,7 @@ func testConfigYAMLWithReferences(pipelineID string, laneID string, references m
func testConfigYAMLWithPipelineReferences(pipelineID string, laneID string, references map[string]string) string {
var b strings.Builder
b.WriteString("version: 1\n")
b.WriteString("version: 2\n")
b.WriteString("pipelines:\n")
b.WriteString(" " + pipelineID + ":\n")
b.WriteString(" input: fake/input\n")
@@ -2175,7 +2140,7 @@ func testConfigYAMLWithPipelineReferences(pipelineID string, laneID string, refe
func testConfigYAMLWithReferencesAndDiagnostics(pipelineID string, laneID string, diagnosticsDir string, references map[string]string) string {
var b strings.Builder
b.WriteString("version: 1\n")
b.WriteString("version: 2\n")
b.WriteString("diagnostics:\n")
b.WriteString(" work_dir: " + diagnosticsDir + "\n")
b.WriteString(" retention: always\n")
@@ -2198,7 +2163,7 @@ func testConfigYAMLWithReferencesAndDiagnostics(pipelineID string, laneID string
}
func mvpConfigYAML(pipelineID string, extractor string) string {
return `version: 1
return `version: 2
pipelines:
` + pipelineID + `:
input: seriatim
@@ -2209,7 +2174,7 @@ pipelines:
}
func mvpConfigYAMLWithChunk(pipelineID string, chunker string, extractor string) string {
return `version: 1
return `version: 2
pipelines:
` + pipelineID + `:
input: seriatim
@@ -2222,7 +2187,7 @@ pipelines:
func mvpConfigYAMLForLanes(pipelineID string, laneIDs ...string) string {
var b strings.Builder
b.WriteString("version: 1\n")
b.WriteString("version: 2\n")
b.WriteString("pipelines:\n")
b.WriteString(" " + pipelineID + ":\n")
b.WriteString(" input: seriatim\n")
@@ -2234,13 +2199,10 @@ func mvpConfigYAMLForLanes(pipelineID string, laneIDs ...string) string {
return b.String()
}
func mvpConfigYAMLWithProfiles(pipelineID string) string {
return `version: 1
llm_profiles:
default:
provider: openai-compatible
runtime:
provider: openai-compatible
func mvpConfigYAMLWithProfileFile(pipelineID string, profileFile string) string {
return `version: 2
scriptorium:
profile_file: ` + profileFile + `
pipelines:
` + pipelineID + `:
input: seriatim
@@ -2250,8 +2212,16 @@ pipelines:
`
}
func writeScriptoriumProfileFile(t *testing.T, id string, endpoint string, model string) string {
t.Helper()
return writeFile(t, id+".profile.yml", `id: `+id+`
endpoint: `+endpoint+`
model: `+model+`
`)
}
func mvpConfigYAMLWithDiagnostics(pipelineID, diagnosticsDir, retention string) string {
return `version: 1
return `version: 2
diagnostics:
work_dir: ` + diagnosticsDir + `
retention: ` + retention + `