Switch config to Scriptorium profiles
This commit is contained in:
@@ -8,7 +8,6 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/dnd/scenes"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/generic"
|
||||
@@ -130,47 +129,5 @@ func productionLLMClientFactory(ctx context.Context, cfg config.Config, profileI
|
||||
return nil, nil, err
|
||||
}
|
||||
trimmedID := strings.TrimSpace(profileID)
|
||||
if trimmedID == "" {
|
||||
trimmedID = pipeline.DefaultLLMProfile
|
||||
}
|
||||
|
||||
profile, ok := cfg.LLMProfile(trimmedID)
|
||||
if !ok {
|
||||
return nil, nil, fmt.Errorf("LLM profile %q is not configured", trimmedID)
|
||||
}
|
||||
clientCfg, err := cfg.OpenAICompatibleClientConfig(trimmedID)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
client, err := llm.NewOpenAICompatibleClient(clientCfg)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("create LLM client for profile %q: %w", trimmedID, err)
|
||||
}
|
||||
|
||||
scheduler, err := llm.NewScheduler(effectiveLLMConcurrency(cfg, profile))
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("create LLM scheduler for profile %q: %w", trimmedID, err)
|
||||
}
|
||||
provider := strings.TrimSpace(profile.Provider)
|
||||
if provider == "" {
|
||||
provider = "openai-compatible"
|
||||
}
|
||||
metadata := []artifacts.LLMProfileManifest{
|
||||
{
|
||||
ID: trimmedID,
|
||||
Provider: provider,
|
||||
Model: strings.TrimSpace(profile.Model),
|
||||
},
|
||||
}
|
||||
return llm.NewScheduledClient(client, scheduler), metadata, nil
|
||||
}
|
||||
|
||||
func effectiveLLMConcurrency(cfg config.Config, profile config.LLMProfile) int {
|
||||
if profile.MaxConcurrency > 0 {
|
||||
return profile.MaxConcurrency
|
||||
}
|
||||
if cfg.Concurrency.TotalLLM > 0 {
|
||||
return cfg.Concurrency.TotalLLM
|
||||
}
|
||||
return 1
|
||||
return nil, nil, fmt.Errorf("create Scriptorium-backed LLM client for profile %q: not implemented yet", trimmedID)
|
||||
}
|
||||
|
||||
@@ -194,6 +194,10 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
|
||||
}
|
||||
profileIDs := effectiveLLMProfileIDs(effective.ResolvedPipeline)
|
||||
if err := validateExplicitScriptoriumProfiles(context.Background(), effective.Config, profileIDs); err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
|
||||
}
|
||||
workingDir, err := os.Getwd()
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("resolve working directory: %w", err))
|
||||
@@ -220,11 +224,6 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics resolved references: %w", err))
|
||||
}
|
||||
|
||||
profileIDs := effectiveLLMProfileIDs(effective.ResolvedPipeline)
|
||||
if len(profileIDs) != 1 {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("pipeline %q uses %d distinct LLM profiles; current runs require exactly one: %s", pipelineID, len(profileIDs), strings.Join(profileIDs, ", ")))
|
||||
}
|
||||
|
||||
rawInput, err := os.ReadFile(strings.TrimSpace(*inputPath))
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("read input %q: %w", strings.TrimSpace(*inputPath), err))
|
||||
@@ -236,9 +235,13 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
llmClient, llmProfiles, err := opts.LLMClientFactory(ctx, effective.Config, profileIDs[0])
|
||||
factoryProfileID := ""
|
||||
if len(profileIDs) == 1 {
|
||||
factoryProfileID = profileIDs[0]
|
||||
}
|
||||
llmClient, llmProfiles, err := opts.LLMClientFactory(ctx, effective.Config, factoryProfileID)
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("create LLM client for profile %q: %w", profileIDs[0], err))
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("create LLM client for profile %q: %w", factoryProfileID, err))
|
||||
}
|
||||
|
||||
output, err := pipeline.New(registries).Run(ctx, pipeline.RunInput{
|
||||
@@ -571,11 +574,16 @@ func runConfigValidate(args []string, stdout, stderr io.Writer, opts Options) in
|
||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
if _, err := cfg.Resolve(config.ResolveInput{
|
||||
effective, err := cfg.Resolve(config.ResolveInput{
|
||||
PipelineID: *pipelineID,
|
||||
Only: only,
|
||||
Catalog: catalog,
|
||||
}); err != nil {
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
if err := validateExplicitScriptoriumProfiles(context.Background(), effective.Config, effectiveLLMProfileIDs(effective.ResolvedPipeline)); err != nil {
|
||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
|
||||
@@ -228,7 +228,7 @@ func TestRunConfigValidateUnknownProductionModuleIncludesContext(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunConfigValidateReportsParseErrors(t *testing.T) {
|
||||
configPath := writeFile(t, "config.yml", "version: 2\n")
|
||||
configPath := writeFile(t, "config.yml", "version: 1\n")
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
@@ -372,11 +372,10 @@ func TestRunUsesNotariusConfigWhenConfigFlagAbsent(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunConfigValidateResolvesAPIKeyEnvThroughOptions(t *testing.T) {
|
||||
configPath := writeTestConfig(t, `version: 1
|
||||
func TestRunConfigValidateRejectsStaleLLMProfiles(t *testing.T) {
|
||||
configPath := writeTestConfig(t, `version: 2
|
||||
llm_profiles:
|
||||
default:
|
||||
api_key_env: NOTARIUS_TEST_API_KEY
|
||||
default: {}
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
@@ -387,12 +386,13 @@ pipelines:
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
code := RunWithOptions([]string{"config", "validate", "--config", configPath}, &stdout, &stderr, Options{
|
||||
LookupEnv: mapLookup(map[string]string{"NOTARIUS_TEST_API_KEY": "secret"}),
|
||||
})
|
||||
code := RunWithOptions([]string{"config", "validate", "--config", configPath}, &stdout, &stderr, Options{})
|
||||
|
||||
if code != 0 {
|
||||
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
|
||||
if code != 1 {
|
||||
t.Fatalf("RunWithOptions() code = %d, want 1", code)
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "llm_profiles") {
|
||||
t.Fatalf("stderr = %q, want stale llm_profiles error", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -445,80 +445,15 @@ func TestRunInvalidFlagsExitTwo(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProductionLLMClientFactoryRejectsMissingProfile(t *testing.T) {
|
||||
func TestProductionLLMClientFactoryReportsPendingScriptoriumRuntime(t *testing.T) {
|
||||
cfg := config.Default()
|
||||
|
||||
_, _, err := productionLLMClientFactory(context.Background(), cfg, "missing")
|
||||
_, _, err := productionLLMClientFactory(context.Background(), cfg, "mistral-small-3")
|
||||
if err == nil {
|
||||
t.Fatal("productionLLMClientFactory() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "LLM profile") || !strings.Contains(err.Error(), "missing") {
|
||||
t.Fatalf("error = %q, want missing profile context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestProductionLLMClientFactoryRejectsInvalidProfile(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
profile config.LLMProfile
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "unsupported provider",
|
||||
profile: config.LLMProfile{Provider: "other", BaseURL: "https://example.test", Model: "model"},
|
||||
want: "not supported",
|
||||
},
|
||||
{
|
||||
name: "missing base url",
|
||||
profile: config.LLMProfile{Provider: "openai-compatible", Model: "model"},
|
||||
want: "base URL",
|
||||
},
|
||||
{
|
||||
name: "missing model",
|
||||
profile: config.LLMProfile{Provider: "openai-compatible", BaseURL: "https://example.test"},
|
||||
want: "model",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
cfg := config.Default()
|
||||
cfg.LLMProfiles = map[string]config.LLMProfile{"default": test.profile}
|
||||
|
||||
_, _, err := productionLLMClientFactory(context.Background(), cfg, "default")
|
||||
if err == nil {
|
||||
t.Fatal("productionLLMClientFactory() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), test.want) {
|
||||
t.Fatalf("error = %q, want substring %q", err.Error(), test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProductionLLMClientFactoryReturnsScheduledClientAndManifestMetadata(t *testing.T) {
|
||||
cfg := config.Default()
|
||||
cfg.LLMProfiles = map[string]config.LLMProfile{
|
||||
"default": {
|
||||
Provider: "openai-compatible",
|
||||
BaseURL: "https://example.test",
|
||||
Model: "model-a",
|
||||
MaxConcurrency: 2,
|
||||
},
|
||||
}
|
||||
|
||||
client, metadata, err := productionLLMClientFactory(context.Background(), cfg, "default")
|
||||
if err != nil {
|
||||
t.Fatalf("productionLLMClientFactory() error = %v, want nil", err)
|
||||
}
|
||||
if client == nil {
|
||||
t.Fatal("client = nil, want scheduled client")
|
||||
}
|
||||
if len(metadata) != 1 {
|
||||
t.Fatalf("len(metadata) = %d, want 1", len(metadata))
|
||||
}
|
||||
if metadata[0].ID != "default" || metadata[0].Provider != "openai-compatible" || metadata[0].Model != "model-a" {
|
||||
t.Fatalf("metadata = %#v, want profile-safe model metadata", metadata)
|
||||
if !strings.Contains(err.Error(), "Scriptorium-backed LLM client") || !strings.Contains(err.Error(), "not implemented yet") {
|
||||
t.Fatalf("error = %q, want pending Scriptorium runtime context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -718,7 +653,8 @@ func TestRunPipelineValidationRejectionCompletesSuccessfully(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunPipelineLLMProfileOverrideSelectsFactoryProfile(t *testing.T) {
|
||||
configPath := writeTestConfig(t, mvpConfigYAMLWithProfiles("dnd-session"))
|
||||
profilePath := writeScriptoriumProfileFile(t, "runtime", "http://profile.test/v1", "test-model")
|
||||
configPath := writeTestConfig(t, mvpConfigYAMLWithProfileFile("dnd-session", profilePath))
|
||||
inputPath := writeSeriatimInput(t)
|
||||
outputDir := t.TempDir()
|
||||
diagnosticsDir := t.TempDir()
|
||||
@@ -739,6 +675,35 @@ func TestRunPipelineLLMProfileOverrideSelectsFactoryProfile(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunConfigValidateRejectsUnknownExplicitScriptoriumProfile(t *testing.T) {
|
||||
profilePath := writeScriptoriumProfileFile(t, "known", "http://profile.test/v1", "test-model")
|
||||
configPath := writeTestConfig(t, `version: 2
|
||||
scriptorium:
|
||||
profile_file: `+profilePath+`
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
artifacts:
|
||||
events:
|
||||
extract:
|
||||
module: fake/extract
|
||||
llm_profile: missing
|
||||
`)
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "example"}, &stdout, &stderr, Options{
|
||||
Catalog: fakeCatalog(t),
|
||||
})
|
||||
|
||||
if code != 1 {
|
||||
t.Fatalf("RunWithOptions() code = %d, want 1", code)
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "Scriptorium profile") || !strings.Contains(stderr.String(), "missing") {
|
||||
t.Fatalf("stderr = %q, want unknown Scriptorium profile", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPipelineSessionIDFlagRecordsExplicitTrimmedValue(t *testing.T) {
|
||||
configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "dnd/spells"))
|
||||
inputPath := writeSeriatimInput(t)
|
||||
@@ -2117,7 +2082,7 @@ func testConfigYAML(pipelineID string, laneIDs ...string) string {
|
||||
|
||||
func testConfigYAMLForPipelines(pipelines map[string][]string) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("version: 1\n")
|
||||
b.WriteString("version: 2\n")
|
||||
b.WriteString("pipelines:\n")
|
||||
for pipelineID, laneIDs := range pipelines {
|
||||
b.WriteString(" " + pipelineID + ":\n")
|
||||
@@ -2133,7 +2098,7 @@ func testConfigYAMLForPipelines(pipelines map[string][]string) string {
|
||||
|
||||
func testConfigYAMLWithReferences(pipelineID string, laneID string, references map[string]string) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("version: 1\n")
|
||||
b.WriteString("version: 2\n")
|
||||
b.WriteString("pipelines:\n")
|
||||
b.WriteString(" " + pipelineID + ":\n")
|
||||
b.WriteString(" input: fake/input\n")
|
||||
@@ -2154,7 +2119,7 @@ func testConfigYAMLWithReferences(pipelineID string, laneID string, references m
|
||||
|
||||
func testConfigYAMLWithPipelineReferences(pipelineID string, laneID string, references map[string]string) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("version: 1\n")
|
||||
b.WriteString("version: 2\n")
|
||||
b.WriteString("pipelines:\n")
|
||||
b.WriteString(" " + pipelineID + ":\n")
|
||||
b.WriteString(" input: fake/input\n")
|
||||
@@ -2175,7 +2140,7 @@ func testConfigYAMLWithPipelineReferences(pipelineID string, laneID string, refe
|
||||
|
||||
func testConfigYAMLWithReferencesAndDiagnostics(pipelineID string, laneID string, diagnosticsDir string, references map[string]string) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("version: 1\n")
|
||||
b.WriteString("version: 2\n")
|
||||
b.WriteString("diagnostics:\n")
|
||||
b.WriteString(" work_dir: " + diagnosticsDir + "\n")
|
||||
b.WriteString(" retention: always\n")
|
||||
@@ -2198,7 +2163,7 @@ func testConfigYAMLWithReferencesAndDiagnostics(pipelineID string, laneID string
|
||||
}
|
||||
|
||||
func mvpConfigYAML(pipelineID string, extractor string) string {
|
||||
return `version: 1
|
||||
return `version: 2
|
||||
pipelines:
|
||||
` + pipelineID + `:
|
||||
input: seriatim
|
||||
@@ -2209,7 +2174,7 @@ pipelines:
|
||||
}
|
||||
|
||||
func mvpConfigYAMLWithChunk(pipelineID string, chunker string, extractor string) string {
|
||||
return `version: 1
|
||||
return `version: 2
|
||||
pipelines:
|
||||
` + pipelineID + `:
|
||||
input: seriatim
|
||||
@@ -2222,7 +2187,7 @@ pipelines:
|
||||
|
||||
func mvpConfigYAMLForLanes(pipelineID string, laneIDs ...string) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("version: 1\n")
|
||||
b.WriteString("version: 2\n")
|
||||
b.WriteString("pipelines:\n")
|
||||
b.WriteString(" " + pipelineID + ":\n")
|
||||
b.WriteString(" input: seriatim\n")
|
||||
@@ -2234,13 +2199,10 @@ func mvpConfigYAMLForLanes(pipelineID string, laneIDs ...string) string {
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func mvpConfigYAMLWithProfiles(pipelineID string) string {
|
||||
return `version: 1
|
||||
llm_profiles:
|
||||
default:
|
||||
provider: openai-compatible
|
||||
runtime:
|
||||
provider: openai-compatible
|
||||
func mvpConfigYAMLWithProfileFile(pipelineID string, profileFile string) string {
|
||||
return `version: 2
|
||||
scriptorium:
|
||||
profile_file: ` + profileFile + `
|
||||
pipelines:
|
||||
` + pipelineID + `:
|
||||
input: seriatim
|
||||
@@ -2250,8 +2212,16 @@ pipelines:
|
||||
`
|
||||
}
|
||||
|
||||
func writeScriptoriumProfileFile(t *testing.T, id string, endpoint string, model string) string {
|
||||
t.Helper()
|
||||
return writeFile(t, id+".profile.yml", `id: `+id+`
|
||||
endpoint: `+endpoint+`
|
||||
model: `+model+`
|
||||
`)
|
||||
}
|
||||
|
||||
func mvpConfigYAMLWithDiagnostics(pipelineID, diagnosticsDir, retention string) string {
|
||||
return `version: 1
|
||||
return `version: 2
|
||||
diagnostics:
|
||||
work_dir: ` + diagnosticsDir + `
|
||||
retention: ` + retention + `
|
||||
|
||||
68
internal/cli/scriptorium_profiles.go
Normal file
68
internal/cli/scriptorium_profiles.go
Normal file
@@ -0,0 +1,68 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing/fstest"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
)
|
||||
|
||||
const profileCheckPromptID = "notarius.profile.check"
|
||||
|
||||
var profileCheckPromptFS = fstest.MapFS{
|
||||
"prompts/profile-check.yaml": &fstest.MapFile{Data: []byte(`id: notarius.profile.check
|
||||
version: "1.0.0"
|
||||
default_profile: mistral-small-3
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
messages:
|
||||
- role: user
|
||||
content: "{{input \"transcript\"}}"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
repair_attempts: 0
|
||||
`)},
|
||||
}
|
||||
|
||||
func validateExplicitScriptoriumProfiles(ctx context.Context, cfg config.Config, profileIDs []string) error {
|
||||
if len(profileIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
engine, err := newProfileValidationEngine(cfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("load Scriptorium profiles: %w", err)
|
||||
}
|
||||
for _, profileID := range profileIDs {
|
||||
if _, err := engine.Prepare(ctx, scriptorium.RunRequest{
|
||||
PromptID: profileCheckPromptID,
|
||||
ProfileID: profileID,
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.Inline("profile check"),
|
||||
},
|
||||
}); err != nil {
|
||||
if errors.Is(err, scriptorium.ErrProfileNotFound) {
|
||||
return fmt.Errorf("Scriptorium profile %q is not configured", profileID)
|
||||
}
|
||||
return fmt.Errorf("validate Scriptorium profile %q: %w", profileID, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func newProfileValidationEngine(cfg config.Config) (*scriptorium.Engine, error) {
|
||||
opts := []scriptorium.Option{
|
||||
scriptorium.WithPromptFS(profileCheckPromptFS, "prompts"),
|
||||
}
|
||||
if cfg.Scriptorium.ProfileFile != "" {
|
||||
opts = append(opts, scriptorium.WithProfileFile(cfg.Scriptorium.ProfileFile))
|
||||
}
|
||||
return scriptorium.NewEngine(scriptorium.Config{
|
||||
PromptDir: "unused",
|
||||
ProfileDir: cfg.Scriptorium.ProfileDir,
|
||||
}, opts...)
|
||||
}
|
||||
@@ -5,24 +5,18 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
const SupportedFileConfigVersion = 1
|
||||
const SupportedFileConfigVersion = 2
|
||||
|
||||
type Config struct {
|
||||
LLMProfiles map[string]LLMProfile `json:"llm_profiles"`
|
||||
Scriptorium ScriptoriumConfig `json:"scriptorium,omitempty"`
|
||||
Pipelines map[string]pipeline.PipelineProfile `json:"pipelines"`
|
||||
Concurrency ConcurrencyConfig `json:"concurrency"`
|
||||
Diagnostics DiagnosticsConfig `json:"diagnostics"`
|
||||
}
|
||||
|
||||
type LLMProfile struct {
|
||||
Provider string `json:"provider,omitempty"`
|
||||
BaseURL string `json:"base_url,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
APIKey string `json:"api_key,omitempty"`
|
||||
APIKeyEnv string `json:"api_key_env,omitempty"`
|
||||
TimeoutSeconds int `json:"timeout_seconds,omitempty"`
|
||||
MaxRetries int `json:"max_retries,omitempty"`
|
||||
MaxConcurrency int `json:"max_concurrency,omitempty"`
|
||||
type ScriptoriumConfig struct {
|
||||
ProfileDir string `json:"profile_dir,omitempty"`
|
||||
ProfileFile string `json:"profile_file,omitempty"`
|
||||
}
|
||||
|
||||
type ConcurrencyConfig struct {
|
||||
@@ -36,14 +30,6 @@ type DiagnosticsConfig struct {
|
||||
|
||||
func Default() Config {
|
||||
return Config{
|
||||
LLMProfiles: map[string]LLMProfile{
|
||||
pipeline.DefaultLLMProfile: {
|
||||
Provider: "openai-compatible",
|
||||
TimeoutSeconds: 600,
|
||||
MaxRetries: 3,
|
||||
MaxConcurrency: 1,
|
||||
},
|
||||
},
|
||||
Pipelines: map[string]pipeline.PipelineProfile{},
|
||||
Concurrency: ConcurrencyConfig{
|
||||
TotalLLM: 1,
|
||||
@@ -57,10 +43,6 @@ func Default() Config {
|
||||
|
||||
func cloneConfig(in Config) Config {
|
||||
out := in
|
||||
out.LLMProfiles = make(map[string]LLMProfile, len(in.LLMProfiles))
|
||||
for key, profile := range in.LLMProfiles {
|
||||
out.LLMProfiles[key] = profile
|
||||
}
|
||||
out.Pipelines = make(map[string]pipeline.PipelineProfile, len(in.Pipelines))
|
||||
for key, profile := range in.Pipelines {
|
||||
out.Pipelines[key] = clonePipelineProfile(profile)
|
||||
|
||||
@@ -4,24 +4,13 @@ import (
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestDefaultValues(t *testing.T) {
|
||||
cfg := Default()
|
||||
|
||||
defaultProfile, ok := cfg.LLMProfiles[pipeline.DefaultLLMProfile]
|
||||
if !ok {
|
||||
t.Fatalf("expected default LLM profile")
|
||||
}
|
||||
if defaultProfile.Provider != "openai-compatible" {
|
||||
t.Fatalf("unexpected provider: %q", defaultProfile.Provider)
|
||||
}
|
||||
if defaultProfile.BaseURL != "" || defaultProfile.Model != "" {
|
||||
t.Fatalf("default profile should not require base URL/model yet: %+v", defaultProfile)
|
||||
}
|
||||
if defaultProfile.TimeoutSeconds != 600 || defaultProfile.MaxRetries != 3 || defaultProfile.MaxConcurrency != 1 {
|
||||
t.Fatalf("unexpected default LLM operational values: %+v", defaultProfile)
|
||||
if cfg.Scriptorium.ProfileDir != "" || cfg.Scriptorium.ProfileFile != "" {
|
||||
t.Fatalf("unexpected Scriptorium profile source defaults: %+v", cfg.Scriptorium)
|
||||
}
|
||||
if len(cfg.Pipelines) != 0 {
|
||||
t.Fatalf("expected no built-in pipeline profiles, got %v", cfg.Pipelines)
|
||||
@@ -39,10 +28,9 @@ func TestDefaultValues(t *testing.T) {
|
||||
|
||||
func TestApplyFileConfigMergesWithDefaults(t *testing.T) {
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(`
|
||||
version: 1
|
||||
llm_profiles:
|
||||
default:
|
||||
model: test-model
|
||||
version: 2
|
||||
scriptorium:
|
||||
profile_dir: ./profiles
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
@@ -59,12 +47,8 @@ pipelines:
|
||||
t.Fatalf("ApplyFileConfig: %v", err)
|
||||
}
|
||||
|
||||
profile := cfg.LLMProfiles[pipeline.DefaultLLMProfile]
|
||||
if profile.Model != "test-model" {
|
||||
t.Fatalf("expected file model, got %+v", profile)
|
||||
}
|
||||
if profile.Provider != "openai-compatible" || profile.TimeoutSeconds != 600 || profile.MaxRetries != 3 {
|
||||
t.Fatalf("expected default LLM fields to be preserved, got %+v", profile)
|
||||
if cfg.Scriptorium.ProfileDir != "./profiles" {
|
||||
t.Fatalf("expected Scriptorium profile dir, got %+v", cfg.Scriptorium)
|
||||
}
|
||||
if cfg.Concurrency.TotalLLM != 1 {
|
||||
t.Fatalf("expected default concurrency preserved, got %d", cfg.Concurrency.TotalLLM)
|
||||
|
||||
@@ -3,9 +3,7 @@ package config
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
@@ -44,9 +42,6 @@ func (c Config) Resolve(input ResolveInput) (EffectiveConfig, error) {
|
||||
profile = clonePipelineProfile(profile)
|
||||
profile.ID = pipelineID
|
||||
if override := strings.TrimSpace(input.LLMProfileOverride); override != "" {
|
||||
if !hasLLMProfile(c.LLMProfiles, override) {
|
||||
return EffectiveConfig{}, fmt.Errorf("LLM profile override %q is not configured", override)
|
||||
}
|
||||
applyLLMProfileOverride(&profile, override)
|
||||
}
|
||||
|
||||
@@ -93,36 +88,3 @@ func lookupPipelineProfile(profiles map[string]pipeline.PipelineProfile, pipelin
|
||||
}
|
||||
return pipeline.PipelineProfile{}, false
|
||||
}
|
||||
|
||||
func (c Config) OpenAICompatibleClientConfig(profileID string) (llm.OpenAICompatibleClientConfig, error) {
|
||||
trimmedID := strings.TrimSpace(profileID)
|
||||
profile, ok := c.LLMProfile(trimmedID)
|
||||
if !ok {
|
||||
return llm.OpenAICompatibleClientConfig{}, fmt.Errorf("LLM profile %q is not configured", trimmedID)
|
||||
}
|
||||
|
||||
provider := strings.TrimSpace(profile.Provider)
|
||||
if provider == "" {
|
||||
provider = providerOpenAICompatible
|
||||
}
|
||||
if provider != providerOpenAICompatible {
|
||||
return llm.OpenAICompatibleClientConfig{}, fmt.Errorf("LLM profile %q provider %q is not supported", trimmedID, provider)
|
||||
}
|
||||
|
||||
baseURL := strings.TrimSpace(profile.BaseURL)
|
||||
if baseURL == "" {
|
||||
return llm.OpenAICompatibleClientConfig{}, fmt.Errorf("LLM profile %q base URL must not be empty", trimmedID)
|
||||
}
|
||||
model := strings.TrimSpace(profile.Model)
|
||||
if model == "" {
|
||||
return llm.OpenAICompatibleClientConfig{}, fmt.Errorf("LLM profile %q model must not be empty", trimmedID)
|
||||
}
|
||||
|
||||
return llm.OpenAICompatibleClientConfig{
|
||||
BaseURL: baseURL,
|
||||
Model: model,
|
||||
APIKey: profile.APIKey,
|
||||
MaxRetries: profile.MaxRetries,
|
||||
RequestTimeout: time.Duration(profile.TimeoutSeconds) * time.Second,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ package config
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
@@ -157,7 +156,6 @@ func TestResolveDigestChangesWhenEffectiveConfigChanges(t *testing.T) {
|
||||
|
||||
func TestResolveLLMProfileOverrideAppliesBeforeDigest(t *testing.T) {
|
||||
cfg := validConfig()
|
||||
cfg.LLMProfiles["runtime"] = LLMProfile{Provider: "openai-compatible"}
|
||||
|
||||
base, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: fakeCatalog(t)})
|
||||
if err != nil {
|
||||
@@ -180,15 +178,6 @@ func TestResolveLLMProfileOverrideAppliesBeforeDigest(t *testing.T) {
|
||||
t.Fatalf("binding profile = %q, want runtime", binding.LLMProfile)
|
||||
}
|
||||
}
|
||||
|
||||
_, err = cfg.Resolve(ResolveInput{
|
||||
PipelineID: "example",
|
||||
Catalog: fakeCatalog(t),
|
||||
LLMProfileOverride: "missing",
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "LLM profile override") {
|
||||
t.Fatalf("expected override profile error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func resolvedBindings(resolved pipeline.ResolvedPipeline) []pipeline.ModuleBinding {
|
||||
@@ -199,53 +188,3 @@ func resolvedBindings(resolved pipeline.ResolvedPipeline) []pipeline.ModuleBindi
|
||||
}
|
||||
return bindings
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientConfigRejectsIncompleteDefaultProfile(t *testing.T) {
|
||||
cfg := Default()
|
||||
|
||||
_, err := cfg.OpenAICompatibleClientConfig("default")
|
||||
if err == nil || !strings.Contains(err.Error(), "base URL") {
|
||||
t.Fatalf("expected incomplete profile error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientConfigSuccess(t *testing.T) {
|
||||
cfg := validConfig()
|
||||
profile := cfg.LLMProfiles["default"]
|
||||
profile.APIKey = "secret"
|
||||
profile.TimeoutSeconds = 45
|
||||
profile.MaxRetries = 4
|
||||
cfg.LLMProfiles["default"] = profile
|
||||
|
||||
llmCfg, err := cfg.OpenAICompatibleClientConfig(" default ")
|
||||
if err != nil {
|
||||
t.Fatalf("OpenAICompatibleClientConfig: %v", err)
|
||||
}
|
||||
|
||||
if llmCfg.BaseURL != "https://example.invalid/v1" || llmCfg.Model != "test-model" || llmCfg.APIKey != "secret" {
|
||||
t.Fatalf("unexpected client config strings: %+v", llmCfg)
|
||||
}
|
||||
if llmCfg.MaxRetries != 4 {
|
||||
t.Fatalf("unexpected max retries: %d", llmCfg.MaxRetries)
|
||||
}
|
||||
if llmCfg.RequestTimeout != 45*time.Second {
|
||||
t.Fatalf("unexpected timeout: %s", llmCfg.RequestTimeout)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientConfigRejectsUnknownAndUnsupportedProfiles(t *testing.T) {
|
||||
_, err := validConfig().OpenAICompatibleClientConfig("missing")
|
||||
if err == nil || !strings.Contains(err.Error(), "not configured") {
|
||||
t.Fatalf("expected unknown profile error, got %v", err)
|
||||
}
|
||||
|
||||
cfg := validConfig()
|
||||
profile := cfg.LLMProfiles["default"]
|
||||
profile.Provider = "unsupported"
|
||||
cfg.LLMProfiles["default"] = profile
|
||||
|
||||
_, err = cfg.OpenAICompatibleClientConfig("default")
|
||||
if err == nil || !strings.Contains(err.Error(), "provider") {
|
||||
t.Fatalf("expected unsupported provider error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func LoadFromEnv() (Config, error) {
|
||||
@@ -30,43 +29,6 @@ func (c *Config) applyEnvOverridesWithLookup(lookup func(string) (string, bool))
|
||||
if c == nil {
|
||||
return fmt.Errorf("config must not be nil")
|
||||
}
|
||||
if c.LLMProfiles == nil {
|
||||
c.LLMProfiles = map[string]LLMProfile{}
|
||||
}
|
||||
|
||||
defaultProfile := c.LLMProfiles[pipeline.DefaultLLMProfile]
|
||||
if raw, ok := lookup("NOTARIUS_LLM_DEFAULT_API_KEY"); ok {
|
||||
defaultProfile.APIKey = raw
|
||||
}
|
||||
if raw, ok := lookup("NOTARIUS_LLM_DEFAULT_BASE_URL"); ok {
|
||||
defaultProfile.BaseURL = strings.TrimSpace(raw)
|
||||
}
|
||||
if raw, ok := lookup("NOTARIUS_LLM_DEFAULT_MODEL"); ok {
|
||||
defaultProfile.Model = strings.TrimSpace(raw)
|
||||
}
|
||||
if raw, ok := lookup("NOTARIUS_LLM_DEFAULT_TIMEOUT_SECONDS"); ok {
|
||||
value, err := parseIntEnv("NOTARIUS_LLM_DEFAULT_TIMEOUT_SECONDS", raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defaultProfile.TimeoutSeconds = value
|
||||
}
|
||||
if raw, ok := lookup("NOTARIUS_LLM_DEFAULT_MAX_RETRIES"); ok {
|
||||
value, err := parseIntEnv("NOTARIUS_LLM_DEFAULT_MAX_RETRIES", raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defaultProfile.MaxRetries = value
|
||||
}
|
||||
if raw, ok := lookup("NOTARIUS_LLM_DEFAULT_MAX_CONCURRENCY"); ok {
|
||||
value, err := parseIntEnv("NOTARIUS_LLM_DEFAULT_MAX_CONCURRENCY", raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defaultProfile.MaxConcurrency = value
|
||||
}
|
||||
c.LLMProfiles[pipeline.DefaultLLMProfile] = defaultProfile
|
||||
|
||||
if raw, ok := lookup("NOTARIUS_TOTAL_LLM_CONCURRENCY"); ok {
|
||||
value, err := parseIntEnv("NOTARIUS_TOTAL_LLM_CONCURRENCY", raw)
|
||||
if err != nil {
|
||||
|
||||
@@ -8,32 +8,22 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestApplyEnvOverridesOperationalAndLLMValues(t *testing.T) {
|
||||
func TestApplyEnvOverridesOperationalValues(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.Pipelines["example"] = pipeline.PipelineProfile{ID: "example", Input: pipeline.Binding("before")}
|
||||
|
||||
err := cfg.applyEnvOverridesWithLookup(mapLookup(map[string]string{
|
||||
"NOTARIUS_LLM_DEFAULT_API_KEY": "secret",
|
||||
"NOTARIUS_LLM_DEFAULT_BASE_URL": "https://example.invalid/v1",
|
||||
"NOTARIUS_LLM_DEFAULT_MODEL": "test-model",
|
||||
"NOTARIUS_LLM_DEFAULT_TIMEOUT_SECONDS": "120",
|
||||
"NOTARIUS_LLM_DEFAULT_MAX_RETRIES": "5",
|
||||
"NOTARIUS_LLM_DEFAULT_MAX_CONCURRENCY": "2",
|
||||
"NOTARIUS_TOTAL_LLM_CONCURRENCY": "3",
|
||||
"NOTARIUS_WORK_DIR": "/tmp/notarius-env",
|
||||
"NOTARIUS_DIAGNOSTICS_RETENTION": "never",
|
||||
"NOTARIUS_PIPELINE_INPUT": "after",
|
||||
"NOTARIUS_TOTAL_LLM_CONCURRENCY": "3",
|
||||
"NOTARIUS_WORK_DIR": "/tmp/notarius-env",
|
||||
"NOTARIUS_DIAGNOSTICS_RETENTION": "never",
|
||||
"NOTARIUS_PIPELINE_INPUT": "after",
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatalf("ApplyEnvOverrides: %v", err)
|
||||
}
|
||||
|
||||
profile := cfg.LLMProfiles[pipeline.DefaultLLMProfile]
|
||||
if profile.APIKey != "secret" || profile.BaseURL != "https://example.invalid/v1" || profile.Model != "test-model" {
|
||||
t.Fatalf("unexpected LLM profile strings: %+v", profile)
|
||||
}
|
||||
if profile.TimeoutSeconds != 120 || profile.MaxRetries != 5 || profile.MaxConcurrency != 2 {
|
||||
t.Fatalf("unexpected LLM profile numeric values: %+v", profile)
|
||||
if cfg.Scriptorium.ProfileDir != "" || cfg.Scriptorium.ProfileFile != "" {
|
||||
t.Fatalf("LLM environment overrides must not change Scriptorium config: %+v", cfg.Scriptorium)
|
||||
}
|
||||
if cfg.Concurrency.TotalLLM != 3 {
|
||||
t.Fatalf("unexpected total concurrency: %d", cfg.Concurrency.TotalLLM)
|
||||
@@ -57,13 +47,16 @@ func TestApplyEnvOverridesRejectsInvalidIntegers(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestLoadFromEnvUsesDefaultConfig(t *testing.T) {
|
||||
t.Setenv("NOTARIUS_LLM_DEFAULT_MODEL", "env-model")
|
||||
t.Setenv("NOTARIUS_TOTAL_LLM_CONCURRENCY", "2")
|
||||
|
||||
cfg, err := LoadFromEnv()
|
||||
if err != nil {
|
||||
t.Fatalf("LoadFromEnv: %v", err)
|
||||
}
|
||||
if cfg.LLMProfiles[pipeline.DefaultLLMProfile].Model != "env-model" {
|
||||
t.Fatalf("expected env model, got %+v", cfg.LLMProfiles[pipeline.DefaultLLMProfile])
|
||||
if cfg.Scriptorium.ProfileDir != "" || cfg.Scriptorium.ProfileFile != "" {
|
||||
t.Fatalf("unexpected Scriptorium config from env: %+v", cfg.Scriptorium)
|
||||
}
|
||||
if cfg.Concurrency.TotalLLM != 2 {
|
||||
t.Fatalf("expected env concurrency override, got %+v", cfg.Concurrency)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,34 +4,25 @@ import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
var envVarNamePattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||
|
||||
type FileConfig struct {
|
||||
Version int `yaml:"version"`
|
||||
LLMProfiles map[string]FileLLMProfile `yaml:"llm_profiles,omitempty"`
|
||||
Scriptorium *FileScriptoriumConfig `yaml:"scriptorium,omitempty"`
|
||||
Pipelines map[string]FilePipelineProfile `yaml:"pipelines,omitempty"`
|
||||
Concurrency *FileConcurrencyConfig `yaml:"concurrency,omitempty"`
|
||||
Diagnostics *FileDiagnosticsConfig `yaml:"diagnostics,omitempty"`
|
||||
}
|
||||
|
||||
type FileLLMProfile struct {
|
||||
Provider *string `yaml:"provider,omitempty"`
|
||||
BaseURL *string `yaml:"base_url,omitempty"`
|
||||
Model *string `yaml:"model,omitempty"`
|
||||
APIKeyEnv *string `yaml:"api_key_env,omitempty"`
|
||||
Timeout *fileDurationSeconds `yaml:"timeout,omitempty"`
|
||||
MaxRetries *int `yaml:"max_retries,omitempty"`
|
||||
MaxConcurrency *int `yaml:"max_concurrency,omitempty"`
|
||||
type FileScriptoriumConfig struct {
|
||||
ProfileDir *string `yaml:"profile_dir,omitempty"`
|
||||
ProfileFile *string `yaml:"profile_file,omitempty"`
|
||||
}
|
||||
|
||||
type FilePipelineProfile struct {
|
||||
@@ -59,42 +50,6 @@ type FileDiagnosticsConfig struct {
|
||||
Retention *string `yaml:"retention,omitempty"`
|
||||
}
|
||||
|
||||
type fileDurationSeconds struct {
|
||||
seconds int
|
||||
}
|
||||
|
||||
func (d *fileDurationSeconds) UnmarshalYAML(node *yaml.Node) error {
|
||||
if node.Kind != yaml.ScalarNode {
|
||||
return fmt.Errorf("must be an integer seconds value or duration string")
|
||||
}
|
||||
if node.Tag == "!!int" {
|
||||
var seconds int
|
||||
if err := node.Decode(&seconds); err != nil {
|
||||
return fmt.Errorf("must be an integer seconds value or duration string")
|
||||
}
|
||||
d.seconds = seconds
|
||||
return nil
|
||||
}
|
||||
|
||||
var raw string
|
||||
if err := node.Decode(&raw); err != nil {
|
||||
return fmt.Errorf("must be an integer seconds value or duration string")
|
||||
}
|
||||
duration, err := time.ParseDuration(strings.TrimSpace(raw))
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid duration %q", raw)
|
||||
}
|
||||
if duration%time.Second != 0 {
|
||||
return fmt.Errorf("duration %q must resolve to whole seconds", raw)
|
||||
}
|
||||
d.seconds = int(duration / time.Second)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d fileDurationSeconds) Seconds() int {
|
||||
return d.seconds
|
||||
}
|
||||
|
||||
type fileModuleBinding struct {
|
||||
Module string
|
||||
LLMProfile string
|
||||
@@ -196,23 +151,17 @@ func (c *Config) ApplyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
|
||||
}
|
||||
|
||||
func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(string) (string, bool)) error {
|
||||
_ = lookup
|
||||
if c == nil {
|
||||
return fmt.Errorf("config must not be nil")
|
||||
}
|
||||
if fileCfg.Version != SupportedFileConfigVersion {
|
||||
return fmt.Errorf("unsupported config version %d", fileCfg.Version)
|
||||
}
|
||||
if c.LLMProfiles == nil {
|
||||
c.LLMProfiles = map[string]LLMProfile{}
|
||||
}
|
||||
if c.Pipelines == nil {
|
||||
c.Pipelines = map[string]pipeline.PipelineProfile{}
|
||||
}
|
||||
|
||||
profileIDs, rawLLMProfileIDs, err := normalizedMapKeys(fileCfg.LLMProfiles, "llm profile id")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pipelineIDs, rawPipelineIDs, err := normalizedMapKeys(fileCfg.Pipelines, "pipeline id")
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -267,36 +216,21 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
|
||||
}
|
||||
}
|
||||
|
||||
for _, profileID := range profileIDs {
|
||||
fileProfile := fileCfg.LLMProfiles[rawLLMProfileIDs[profileID]]
|
||||
profile := c.LLMProfiles[profileID]
|
||||
if fileProfile.Provider != nil {
|
||||
profile.Provider = strings.TrimSpace(*fileProfile.Provider)
|
||||
}
|
||||
if fileProfile.BaseURL != nil {
|
||||
profile.BaseURL = strings.TrimSpace(*fileProfile.BaseURL)
|
||||
}
|
||||
if fileProfile.Model != nil {
|
||||
profile.Model = strings.TrimSpace(*fileProfile.Model)
|
||||
}
|
||||
if fileProfile.APIKeyEnv != nil {
|
||||
apiKey, err := resolveAPIKeyEnv(*fileProfile.APIKeyEnv, lookup)
|
||||
if err != nil {
|
||||
return fmt.Errorf("llm_profiles.%s.api_key_env: %w", profileID, err)
|
||||
if fileCfg.Scriptorium != nil {
|
||||
if fileCfg.Scriptorium.ProfileDir != nil {
|
||||
value := strings.TrimSpace(*fileCfg.Scriptorium.ProfileDir)
|
||||
if value == "" {
|
||||
return fmt.Errorf("scriptorium.profile_dir must not be empty when set")
|
||||
}
|
||||
profile.APIKeyEnv = strings.TrimSpace(*fileProfile.APIKeyEnv)
|
||||
profile.APIKey = apiKey
|
||||
c.Scriptorium.ProfileDir = value
|
||||
}
|
||||
if fileProfile.Timeout != nil {
|
||||
profile.TimeoutSeconds = fileProfile.Timeout.Seconds()
|
||||
if fileCfg.Scriptorium.ProfileFile != nil {
|
||||
value := strings.TrimSpace(*fileCfg.Scriptorium.ProfileFile)
|
||||
if value == "" {
|
||||
return fmt.Errorf("scriptorium.profile_file must not be empty when set")
|
||||
}
|
||||
c.Scriptorium.ProfileFile = value
|
||||
}
|
||||
if fileProfile.MaxRetries != nil {
|
||||
profile.MaxRetries = *fileProfile.MaxRetries
|
||||
}
|
||||
if fileProfile.MaxConcurrency != nil {
|
||||
profile.MaxConcurrency = *fileProfile.MaxConcurrency
|
||||
}
|
||||
c.LLMProfiles[profileID] = profile
|
||||
}
|
||||
|
||||
for _, pipelineID := range pipelineIDs {
|
||||
@@ -408,21 +342,6 @@ func mergeStringMaps(base map[string]string, override map[string]string) map[str
|
||||
return out
|
||||
}
|
||||
|
||||
func resolveAPIKeyEnv(envName string, lookup func(string) (string, bool)) (string, error) {
|
||||
name := strings.TrimSpace(envName)
|
||||
if name == "" {
|
||||
return "", fmt.Errorf("must not be empty")
|
||||
}
|
||||
if !envVarNamePattern.MatchString(name) {
|
||||
return "", fmt.Errorf("must be an environment variable name")
|
||||
}
|
||||
value, ok := lookup(name)
|
||||
if !ok {
|
||||
return "", fmt.Errorf("%s is not set", name)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func normalizeOptions(options map[string]any) map[string]any {
|
||||
if len(options) == 0 {
|
||||
return nil
|
||||
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
|
||||
func TestParseMinimalValidConfig(t *testing.T) {
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(`
|
||||
version: 1
|
||||
version: 2
|
||||
`))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML: %v", err)
|
||||
@@ -24,7 +24,7 @@ version: 1
|
||||
|
||||
func TestLoadFileConfig(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "config.yml")
|
||||
if err := os.WriteFile(path, []byte("version: 1\n"), 0o644); err != nil {
|
||||
if err := os.WriteFile(path, []byte("version: 2\n"), 0o644); err != nil {
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
|
||||
@@ -39,7 +39,7 @@ func TestLoadFileConfig(t *testing.T) {
|
||||
|
||||
func TestParseFileConfigRejectsUnknownYAMLFields(t *testing.T) {
|
||||
_, err := ParseFileConfigYAML([]byte(`
|
||||
version: 1
|
||||
version: 2
|
||||
unexpected: true
|
||||
`))
|
||||
if err == nil || !strings.Contains(err.Error(), "field unexpected not found") {
|
||||
@@ -49,7 +49,7 @@ unexpected: true
|
||||
|
||||
func TestParseFileConfigRejectsUnknownModuleBindingFields(t *testing.T) {
|
||||
_, err := ParseFileConfigYAML([]byte(`
|
||||
version: 1
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input:
|
||||
@@ -70,8 +70,8 @@ func TestParseFileConfigRejectsMissingAndUnsupportedVersion(t *testing.T) {
|
||||
data string
|
||||
want string
|
||||
}{
|
||||
{name: "missing", data: `llm_profiles: {}`, want: "version is required"},
|
||||
{name: "unsupported", data: `version: 2`, want: "unsupported config version"},
|
||||
{name: "missing", data: `scriptorium: {}`, want: "version is required"},
|
||||
{name: "unsupported", data: `version: 1`, want: "unsupported config version"},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
@@ -84,9 +84,44 @@ func TestParseFileConfigRejectsMissingAndUnsupportedVersion(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFileConfigRejectsStaleLLMProfiles(t *testing.T) {
|
||||
_, err := ParseFileConfigYAML([]byte(`
|
||||
version: 2
|
||||
llm_profiles:
|
||||
default: {}
|
||||
`))
|
||||
if err == nil || !strings.Contains(err.Error(), "llm_profiles") {
|
||||
t.Fatalf("expected stale llm_profiles error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFileConfigScriptoriumProfileSources(t *testing.T) {
|
||||
t.Run("profile dir", func(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 2
|
||||
scriptorium:
|
||||
profile_dir: ./profiles
|
||||
`)
|
||||
if cfg.Scriptorium.ProfileDir != "./profiles" || cfg.Scriptorium.ProfileFile != "" {
|
||||
t.Fatalf("Scriptorium = %+v, want profile_dir", cfg.Scriptorium)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("profile file", func(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 2
|
||||
scriptorium:
|
||||
profile_file: ./profiles.yml
|
||||
`)
|
||||
if cfg.Scriptorium.ProfileFile != "./profiles.yml" || cfg.Scriptorium.ProfileDir != "" {
|
||||
t.Fatalf("Scriptorium = %+v, want profile_file", cfg.Scriptorium)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestParseFileConfigModuleBindingForms(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 1
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
@@ -146,7 +181,7 @@ pipelines:
|
||||
|
||||
func TestParseFileConfigReferenceMaps(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 1
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
@@ -171,7 +206,7 @@ pipelines:
|
||||
|
||||
func TestParseFileConfigStageLocalReferenceMaps(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 1
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
@@ -218,7 +253,7 @@ pipelines:
|
||||
|
||||
func TestParseFileConfigValidatorMixedBindingForms(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 1
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
@@ -248,87 +283,9 @@ pipelines:
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFileConfigDurationParsing(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
raw string
|
||||
want int
|
||||
}{
|
||||
{name: "integer seconds", raw: "600", want: 600},
|
||||
{name: "duration string", raw: "10m", want: 600},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 1
|
||||
llm_profiles:
|
||||
default:
|
||||
timeout: `+tc.raw+`
|
||||
`)
|
||||
if got := cfg.LLMProfiles["default"].TimeoutSeconds; got != tc.want {
|
||||
t.Fatalf("TimeoutSeconds = %d, want %d", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFileConfigRejectsSubsecondDuration(t *testing.T) {
|
||||
_, err := ParseFileConfigYAML([]byte(`
|
||||
version: 1
|
||||
llm_profiles:
|
||||
default:
|
||||
timeout: 1500ms
|
||||
`))
|
||||
if err == nil || !strings.Contains(err.Error(), "whole seconds") {
|
||||
t.Fatalf("expected whole-seconds duration error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigResolvesAPIKeyEnv(t *testing.T) {
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(`
|
||||
version: 1
|
||||
llm_profiles:
|
||||
default:
|
||||
api_key_env: NOTARIUS_TEST_API_KEY
|
||||
`))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML: %v", err)
|
||||
}
|
||||
|
||||
cfg := Default()
|
||||
if err := cfg.applyFileConfigWithLookup(fileCfg, mapLookup(map[string]string{"NOTARIUS_TEST_API_KEY": "secret"})); err != nil {
|
||||
t.Fatalf("ApplyFileConfig: %v", err)
|
||||
}
|
||||
profile := cfg.LLMProfiles["default"]
|
||||
if profile.APIKeyEnv != "NOTARIUS_TEST_API_KEY" || profile.APIKey != "secret" {
|
||||
t.Fatalf("unexpected resolved API key: %+v", profile)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigRejectsDuplicateTrimmedLLMProfileIDs(t *testing.T) {
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(`
|
||||
version: 1
|
||||
llm_profiles:
|
||||
default:
|
||||
model: first
|
||||
" default ":
|
||||
model: second
|
||||
`))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML: %v", err)
|
||||
}
|
||||
|
||||
cfg := Default()
|
||||
err = cfg.applyFileConfigWithLookup(fileCfg, emptyLookup)
|
||||
if err == nil || !strings.Contains(err.Error(), "llm profile id") || !strings.Contains(err.Error(), "duplicated") {
|
||||
t.Fatalf("expected duplicate LLM profile ID error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigRejectsDuplicateTrimmedPipelineIDs(t *testing.T) {
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(`
|
||||
version: 1
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
@@ -348,7 +305,7 @@ pipelines:
|
||||
|
||||
func TestApplyFileConfigRejectsDuplicateTrimmedArtifactLaneIDs(t *testing.T) {
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(`
|
||||
version: 1
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
@@ -378,7 +335,7 @@ func TestApplyFileConfigRejectsDuplicateTrimmedReferenceSlots(t *testing.T) {
|
||||
{
|
||||
name: "pipeline",
|
||||
raw: `
|
||||
version: 1
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
@@ -391,7 +348,7 @@ pipelines:
|
||||
{
|
||||
name: "lane",
|
||||
raw: `
|
||||
version: 1
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
@@ -407,7 +364,7 @@ pipelines:
|
||||
{
|
||||
name: "chunk",
|
||||
raw: `
|
||||
version: 1
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
@@ -422,7 +379,7 @@ pipelines:
|
||||
{
|
||||
name: "extract",
|
||||
raw: `
|
||||
version: 1
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
@@ -439,7 +396,7 @@ pipelines:
|
||||
{
|
||||
name: "normalize",
|
||||
raw: `
|
||||
version: 1
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
@@ -471,49 +428,32 @@ pipelines:
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigAllowsRetryOnlyLLMProfile(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 1
|
||||
llm_profiles:
|
||||
retry-only:
|
||||
max_retries: 3
|
||||
`)
|
||||
|
||||
profile := cfg.LLMProfiles["retry-only"]
|
||||
if profile.MaxRetries != 3 {
|
||||
t.Fatalf("unexpected max retries: %d", profile.MaxRetries)
|
||||
}
|
||||
if profile.TimeoutSeconds != 0 {
|
||||
t.Fatalf("expected unset timeout, got %d", profile.TimeoutSeconds)
|
||||
}
|
||||
if profile.MaxConcurrency != 0 {
|
||||
t.Fatalf("expected unset max concurrency, got %d", profile.MaxConcurrency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigRejectsInvalidAPIKeyEnv(t *testing.T) {
|
||||
func TestApplyFileConfigRejectsInvalidScriptoriumSources(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
env string
|
||||
raw string
|
||||
want string
|
||||
}{
|
||||
{name: "invalid name", env: "NOTARIUS-KEY", want: "environment variable name"},
|
||||
{name: "not set", env: "NOTARIUS_TEST_API_KEY", want: "is not set"},
|
||||
{name: "empty profile dir", raw: "profile_dir: ' '", want: "profile_dir"},
|
||||
{name: "empty profile file", raw: "profile_file: ' '", want: "profile_file"},
|
||||
{name: "both sources", raw: "profile_dir: ./profiles\n profile_file: ./profiles.yml", want: "mutually exclusive"},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
cfg := Default()
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(`
|
||||
version: 1
|
||||
llm_profiles:
|
||||
default:
|
||||
api_key_env: ` + tc.env + `
|
||||
version: 2
|
||||
scriptorium:
|
||||
` + tc.raw + `
|
||||
`))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML: %v", err)
|
||||
}
|
||||
cfg := Default()
|
||||
err = cfg.applyFileConfigWithLookup(fileCfg, emptyLookup)
|
||||
if err == nil {
|
||||
err = cfg.Validate()
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), tc.want) {
|
||||
t.Fatalf("expected error containing %q, got %v", tc.want, err)
|
||||
}
|
||||
@@ -523,7 +463,7 @@ llm_profiles:
|
||||
|
||||
func TestApplyFileConfigOperationalSections(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 1
|
||||
version: 2
|
||||
concurrency:
|
||||
total_llm: 4
|
||||
diagnostics:
|
||||
|
||||
@@ -2,17 +2,8 @@ package config
|
||||
|
||||
import "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
|
||||
const redactedSecret = "[REDACTED]"
|
||||
|
||||
func (c Config) Redacted() Config {
|
||||
redacted := cloneConfig(c)
|
||||
for id, profile := range redacted.LLMProfiles {
|
||||
if profile.APIKey != "" {
|
||||
profile.APIKey = redactedSecret
|
||||
}
|
||||
redacted.LLMProfiles[id] = profile
|
||||
}
|
||||
return redacted
|
||||
return cloneConfig(c)
|
||||
}
|
||||
|
||||
func (c Config) RedactedDiagnosticsPayload() any {
|
||||
|
||||
@@ -7,63 +7,36 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestRedactedConfigRemovesAPIKeyValues(t *testing.T) {
|
||||
func TestRedactedConfigCopiesScriptoriumConfig(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.LLMProfiles[pipeline.DefaultLLMProfile] = LLMProfile{
|
||||
Provider: "openai-compatible",
|
||||
BaseURL: "https://example.invalid/v1",
|
||||
Model: "test-model",
|
||||
APIKey: "secret",
|
||||
APIKeyEnv: "NOTARIUS_TEST_API_KEY",
|
||||
TimeoutSeconds: 600,
|
||||
MaxRetries: 3,
|
||||
MaxConcurrency: 1,
|
||||
}
|
||||
cfg.LLMProfiles["other"] = LLMProfile{APIKey: "other-secret", Model: "other-model"}
|
||||
cfg.Scriptorium.ProfileDir = "./profiles"
|
||||
|
||||
redacted := cfg.Redacted()
|
||||
|
||||
if redacted.LLMProfiles[pipeline.DefaultLLMProfile].APIKey != redactedSecret {
|
||||
t.Fatalf("expected default API key redacted, got %+v", redacted.LLMProfiles[pipeline.DefaultLLMProfile])
|
||||
if redacted.Scriptorium.ProfileDir != "./profiles" {
|
||||
t.Fatalf("expected Scriptorium profile source preserved, got %+v", redacted.Scriptorium)
|
||||
}
|
||||
if redacted.LLMProfiles["other"].APIKey != redactedSecret {
|
||||
t.Fatalf("expected other API key redacted, got %+v", redacted.LLMProfiles["other"])
|
||||
}
|
||||
if redacted.LLMProfiles[pipeline.DefaultLLMProfile].Model != "test-model" {
|
||||
t.Fatalf("expected non-secret fields preserved, got %+v", redacted.LLMProfiles[pipeline.DefaultLLMProfile])
|
||||
}
|
||||
if cfg.LLMProfiles[pipeline.DefaultLLMProfile].APIKey != "secret" {
|
||||
redacted.Scriptorium.ProfileDir = "./changed"
|
||||
if cfg.Scriptorium.ProfileDir != "./profiles" {
|
||||
t.Fatalf("redaction mutated original config")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigRedactedDiagnosticsPayloadRedactsAPIKeys(t *testing.T) {
|
||||
func TestConfigRedactedDiagnosticsPayloadCopiesConfig(t *testing.T) {
|
||||
cfg := Default()
|
||||
profile := cfg.LLMProfiles[pipeline.DefaultLLMProfile]
|
||||
profile.APIKey = "secret"
|
||||
profile.Model = "test-model"
|
||||
cfg.LLMProfiles[pipeline.DefaultLLMProfile] = profile
|
||||
cfg.Scriptorium.ProfileFile = "./profiles.yml"
|
||||
|
||||
payload, ok := cfg.RedactedDiagnosticsPayload().(Config)
|
||||
if !ok {
|
||||
t.Fatalf("expected Config payload, got %T", cfg.RedactedDiagnosticsPayload())
|
||||
}
|
||||
if payload.LLMProfiles[pipeline.DefaultLLMProfile].APIKey != redactedSecret {
|
||||
t.Fatalf("expected API key redacted, got %+v", payload.LLMProfiles[pipeline.DefaultLLMProfile])
|
||||
}
|
||||
if payload.LLMProfiles[pipeline.DefaultLLMProfile].Model != "test-model" {
|
||||
t.Fatalf("expected non-secret fields preserved, got %+v", payload.LLMProfiles[pipeline.DefaultLLMProfile])
|
||||
}
|
||||
if cfg.LLMProfiles[pipeline.DefaultLLMProfile].APIKey != "secret" {
|
||||
t.Fatalf("redacted diagnostics payload mutated original config")
|
||||
if payload.Scriptorium.ProfileFile != "./profiles.yml" {
|
||||
t.Fatalf("expected Scriptorium profile file preserved, got %+v", payload.Scriptorium)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectiveConfigRedactedDiagnosticsPayloadRedactsAndCopies(t *testing.T) {
|
||||
func TestEffectiveConfigRedactedDiagnosticsPayloadCopies(t *testing.T) {
|
||||
cfg := validConfig()
|
||||
profile := cfg.LLMProfiles[pipeline.DefaultLLMProfile]
|
||||
profile.APIKey = "secret"
|
||||
cfg.LLMProfiles[pipeline.DefaultLLMProfile] = profile
|
||||
lane := cfg.Pipelines["example"].Artifacts["events"]
|
||||
lane.Extract.Options = map[string]any{"temperature": 0.2}
|
||||
lane.References = map[string]string{"roster": "./roster.yml"}
|
||||
@@ -116,12 +89,6 @@ func TestEffectiveConfigRedactedDiagnosticsPayloadRedactsAndCopies(t *testing.T)
|
||||
if !ok {
|
||||
t.Fatalf("expected EffectiveConfig payload, got %T", effective.RedactedDiagnosticsPayload())
|
||||
}
|
||||
if payload.Config.LLMProfiles[pipeline.DefaultLLMProfile].APIKey != redactedSecret {
|
||||
t.Fatalf("expected nested API key redacted, got %+v", payload.Config.LLMProfiles[pipeline.DefaultLLMProfile])
|
||||
}
|
||||
if cfg.LLMProfiles[pipeline.DefaultLLMProfile].APIKey != "secret" {
|
||||
t.Fatalf("redacted diagnostics payload mutated source config")
|
||||
}
|
||||
if payload.PipelineID != effective.PipelineID || payload.ResolvedPipeline.Digest != effective.ResolvedPipeline.Digest {
|
||||
t.Fatalf("expected pipeline metadata preserved, got %+v", payload)
|
||||
}
|
||||
|
||||
@@ -8,10 +8,8 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
const providerOpenAICompatible = "openai-compatible"
|
||||
|
||||
func (c Config) Validate() error {
|
||||
if err := validateLLMProfiles(c.LLMProfiles); err != nil {
|
||||
if err := validateScriptorium(c.Scriptorium); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateDiagnostics(c.Diagnostics); err != nil {
|
||||
@@ -20,44 +18,12 @@ func (c Config) Validate() error {
|
||||
if c.Concurrency.TotalLLM <= 0 {
|
||||
return fmt.Errorf("total LLM concurrency must be greater than zero")
|
||||
}
|
||||
return validatePipelineProfiles(c.Pipelines, c.LLMProfiles)
|
||||
return validatePipelineProfiles(c.Pipelines)
|
||||
}
|
||||
|
||||
func (c Config) LLMProfile(id string) (LLMProfile, bool) {
|
||||
trimmedID := strings.TrimSpace(id)
|
||||
for rawID, profile := range c.LLMProfiles {
|
||||
if strings.TrimSpace(rawID) == trimmedID {
|
||||
return profile, true
|
||||
}
|
||||
}
|
||||
return LLMProfile{}, false
|
||||
}
|
||||
|
||||
func validateLLMProfiles(profiles map[string]LLMProfile) error {
|
||||
seen := make(map[string]struct{}, len(profiles))
|
||||
for rawID, profile := range profiles {
|
||||
id := strings.TrimSpace(rawID)
|
||||
if id == "" {
|
||||
return fmt.Errorf("LLM profile id must not be empty")
|
||||
}
|
||||
if _, ok := seen[id]; ok {
|
||||
return fmt.Errorf("LLM profile id %q is duplicated after trimming", id)
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
|
||||
provider := strings.TrimSpace(profile.Provider)
|
||||
if provider != "" && provider != providerOpenAICompatible {
|
||||
return fmt.Errorf("LLM profile %q provider %q is not supported", id, provider)
|
||||
}
|
||||
if profile.TimeoutSeconds < 0 {
|
||||
return fmt.Errorf("LLM profile %q timeout seconds must not be negative", id)
|
||||
}
|
||||
if profile.MaxRetries < 0 {
|
||||
return fmt.Errorf("LLM profile %q max retries must not be negative", id)
|
||||
}
|
||||
if profile.MaxConcurrency < 0 {
|
||||
return fmt.Errorf("LLM profile %q max concurrency must not be negative", id)
|
||||
}
|
||||
func validateScriptorium(cfg ScriptoriumConfig) error {
|
||||
if strings.TrimSpace(cfg.ProfileDir) != "" && strings.TrimSpace(cfg.ProfileFile) != "" {
|
||||
return fmt.Errorf("scriptorium profile_dir and profile_file are mutually exclusive")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -74,7 +40,7 @@ func validateDiagnostics(cfg DiagnosticsConfig) error {
|
||||
}
|
||||
}
|
||||
|
||||
func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile, llmProfiles map[string]LLMProfile) error {
|
||||
func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile) error {
|
||||
seen := make(map[string]struct{}, len(profiles))
|
||||
for rawID, profile := range profiles {
|
||||
id := strings.TrimSpace(rawID)
|
||||
@@ -89,13 +55,13 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile, llmP
|
||||
if profile.ID != "" && strings.TrimSpace(profile.ID) != id {
|
||||
return fmt.Errorf("pipeline %q profile id %q does not match map key", id, profile.ID)
|
||||
}
|
||||
if err := validateBinding(id, "", "input", profile.Input, llmProfiles, false); err != nil {
|
||||
if err := validateBinding(id, "", "input", profile.Input, false); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateBinding(id, "", "chunk", profile.Chunk, llmProfiles, true); err != nil {
|
||||
if err := validateBinding(id, "", "chunk", profile.Chunk, true); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateBinding(id, "", "output", profile.Output, llmProfiles, false); err != nil {
|
||||
if err := validateBinding(id, "", "output", profile.Output, false); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateReferenceMap(id, "", profile.References); err != nil {
|
||||
@@ -109,17 +75,17 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile, llmP
|
||||
if err := validateReferenceMap(id, laneID, lane.References); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateBinding(id, laneID, "extract", lane.Extract, llmProfiles, true); err != nil {
|
||||
if err := validateBinding(id, laneID, "extract", lane.Extract, true); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateBinding(id, laneID, "merge", lane.Merge, llmProfiles, false); err != nil {
|
||||
if err := validateBinding(id, laneID, "merge", lane.Merge, false); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateBinding(id, laneID, "normalize", lane.Normalize, llmProfiles, true); err != nil {
|
||||
if err := validateBinding(id, laneID, "normalize", lane.Normalize, true); err != nil {
|
||||
return err
|
||||
}
|
||||
for i, validator := range lane.Validators {
|
||||
if err := validateBinding(id, laneID, fmt.Sprintf("validator[%d]", i), validator, llmProfiles, false); err != nil {
|
||||
if err := validateBinding(id, laneID, fmt.Sprintf("validator[%d]", i), validator, false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -133,10 +99,9 @@ func validateBinding(
|
||||
laneID string,
|
||||
slot string,
|
||||
binding pipeline.ModuleBinding,
|
||||
profiles map[string]LLMProfile,
|
||||
referencesAllowed bool,
|
||||
) error {
|
||||
if err := validateBindingLLMProfile(pipelineID, laneID, slot, binding, profiles); err != nil {
|
||||
if err := validateBindingLLMProfile(pipelineID, laneID, slot, binding); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(binding.References) == 0 {
|
||||
@@ -191,27 +156,12 @@ func validateBindingLLMProfile(
|
||||
laneID string,
|
||||
slot string,
|
||||
binding pipeline.ModuleBinding,
|
||||
profiles map[string]LLMProfile,
|
||||
) error {
|
||||
profileID := strings.TrimSpace(binding.LLMProfile)
|
||||
if profileID == "" {
|
||||
profileID = pipeline.DefaultLLMProfile
|
||||
}
|
||||
if hasLLMProfile(profiles, profileID) {
|
||||
return nil
|
||||
}
|
||||
if laneID != "" {
|
||||
return fmt.Errorf("pipeline %q lane %q %s references unknown LLM profile %q", pipelineID, laneID, slot, profileID)
|
||||
}
|
||||
return fmt.Errorf("pipeline %q %s references unknown LLM profile %q", pipelineID, slot, profileID)
|
||||
}
|
||||
|
||||
func hasLLMProfile(profiles map[string]LLMProfile, profileID string) bool {
|
||||
profileID = strings.TrimSpace(profileID)
|
||||
for rawID := range profiles {
|
||||
if strings.TrimSpace(rawID) == profileID {
|
||||
return true
|
||||
if binding.LLMProfile != "" && strings.TrimSpace(binding.LLMProfile) == "" {
|
||||
if laneID != "" {
|
||||
return fmt.Errorf("pipeline %q lane %q %s llm_profile must not be empty when set", pipelineID, laneID, slot)
|
||||
}
|
||||
return fmt.Errorf("pipeline %q %s llm_profile must not be empty when set", pipelineID, slot)
|
||||
}
|
||||
return false
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -17,27 +17,26 @@ func TestValidateSuccessForValidConfig(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRejectsUnknownLLMProfileReferencedByBinding(t *testing.T) {
|
||||
func TestValidateAllowsExplicitScriptoriumProfileIDOnBinding(t *testing.T) {
|
||||
cfg := validConfig()
|
||||
lane := cfg.Pipelines["example"].Artifacts["events"]
|
||||
lane.Extract.LLMProfile = "missing"
|
||||
lane.Extract.LLMProfile = "scriptorium-profile"
|
||||
cfg.Pipelines["example"].Artifacts["events"] = lane
|
||||
|
||||
err := cfg.Validate()
|
||||
if err == nil || !strings.Contains(err.Error(), "unknown LLM profile") || !strings.Contains(err.Error(), "events") {
|
||||
t.Fatalf("expected unknown LLM profile error with lane context, got %v", err)
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("Validate() error = %v, want nil", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRejectsInvalidProvider(t *testing.T) {
|
||||
func TestValidateRejectsWhitespaceOnlyExplicitLLMProfile(t *testing.T) {
|
||||
cfg := validConfig()
|
||||
profile := cfg.LLMProfiles["default"]
|
||||
profile.Provider = "unsupported"
|
||||
cfg.LLMProfiles["default"] = profile
|
||||
lane := cfg.Pipelines["example"].Artifacts["events"]
|
||||
lane.Extract.LLMProfile = " "
|
||||
cfg.Pipelines["example"].Artifacts["events"] = lane
|
||||
|
||||
err := cfg.Validate()
|
||||
if err == nil || !strings.Contains(err.Error(), "provider") {
|
||||
t.Fatalf("expected provider error, got %v", err)
|
||||
if err == nil || !strings.Contains(err.Error(), "llm_profile") || !strings.Contains(err.Error(), "events") {
|
||||
t.Fatalf("expected llm_profile error with lane context, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,36 +54,6 @@ func TestValidateRejectsInvalidNumericFields(t *testing.T) {
|
||||
},
|
||||
want: "total LLM concurrency",
|
||||
},
|
||||
{
|
||||
name: "timeout",
|
||||
mutate: func(cfg Config) Config {
|
||||
profile := cfg.LLMProfiles["default"]
|
||||
profile.TimeoutSeconds = -1
|
||||
cfg.LLMProfiles["default"] = profile
|
||||
return cfg
|
||||
},
|
||||
want: "timeout",
|
||||
},
|
||||
{
|
||||
name: "max retries",
|
||||
mutate: func(cfg Config) Config {
|
||||
profile := cfg.LLMProfiles["default"]
|
||||
profile.MaxRetries = -1
|
||||
cfg.LLMProfiles["default"] = profile
|
||||
return cfg
|
||||
},
|
||||
want: "max retries",
|
||||
},
|
||||
{
|
||||
name: "max concurrency",
|
||||
mutate: func(cfg Config) Config {
|
||||
profile := cfg.LLMProfiles["default"]
|
||||
profile.MaxConcurrency = -1
|
||||
cfg.LLMProfiles["default"] = profile
|
||||
return cfg
|
||||
},
|
||||
want: "max concurrency",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
@@ -97,12 +66,14 @@ func TestValidateRejectsInvalidNumericFields(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateAllowsPartialLLMProfileNumericConfig(t *testing.T) {
|
||||
func TestValidateRejectsMutuallyExclusiveScriptoriumProfileSources(t *testing.T) {
|
||||
cfg := validConfig()
|
||||
cfg.LLMProfiles["retry-only"] = LLMProfile{MaxRetries: 3}
|
||||
cfg.Scriptorium.ProfileDir = "./profiles"
|
||||
cfg.Scriptorium.ProfileFile = "./profiles.yml"
|
||||
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("Validate: %v", err)
|
||||
err := cfg.Validate()
|
||||
if err == nil || !strings.Contains(err.Error(), "mutually exclusive") {
|
||||
t.Fatalf("expected Scriptorium source conflict, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -327,14 +298,6 @@ func TestValidateRejectsEmptyIDs(t *testing.T) {
|
||||
mutate func(Config) Config
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "LLM profile",
|
||||
mutate: func(cfg Config) Config {
|
||||
cfg.LLMProfiles[" "] = LLMProfile{}
|
||||
return cfg
|
||||
},
|
||||
want: "LLM profile id",
|
||||
},
|
||||
{
|
||||
name: "pipeline",
|
||||
mutate: func(cfg Config) Config {
|
||||
@@ -361,14 +324,6 @@ func TestValidateRejectsIDsDuplicatedAfterTrimming(t *testing.T) {
|
||||
mutate func(Config) Config
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "LLM profile",
|
||||
mutate: func(cfg Config) Config {
|
||||
cfg.LLMProfiles[" default "] = cfg.LLMProfiles["default"]
|
||||
return cfg
|
||||
},
|
||||
want: "duplicated",
|
||||
},
|
||||
{
|
||||
name: "pipeline",
|
||||
mutate: func(cfg Config) Config {
|
||||
@@ -389,25 +344,8 @@ func TestValidateRejectsIDsDuplicatedAfterTrimming(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateUsesTrimmedLLMProfileIDs(t *testing.T) {
|
||||
cfg := validConfig()
|
||||
cfg.LLMProfiles[" default "] = cfg.LLMProfiles["default"]
|
||||
delete(cfg.LLMProfiles, "default")
|
||||
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("Validate: %v", err)
|
||||
}
|
||||
if _, ok := cfg.LLMProfile("default"); !ok {
|
||||
t.Fatalf("expected trimmed LLM profile lookup to succeed")
|
||||
}
|
||||
}
|
||||
|
||||
func validConfig() Config {
|
||||
cfg := Default()
|
||||
profile := cfg.LLMProfiles["default"]
|
||||
profile.BaseURL = "https://example.invalid/v1"
|
||||
profile.Model = "test-model"
|
||||
cfg.LLMProfiles["default"] = profile
|
||||
cfg.Pipelines["example"] = pipeline.PipelineProfile{
|
||||
Input: pipeline.Binding("fake/input"),
|
||||
Artifacts: map[string]pipeline.ArtifactLaneProfile{
|
||||
|
||||
@@ -597,9 +597,6 @@ func resolveBinding(binding ModuleBinding, defaultModule string) ModuleBinding {
|
||||
module = defaultModule
|
||||
}
|
||||
llmProfile := strings.TrimSpace(binding.LLMProfile)
|
||||
if llmProfile == "" {
|
||||
llmProfile = DefaultLLMProfile
|
||||
}
|
||||
return ModuleBinding{
|
||||
Module: module,
|
||||
LLMProfile: llmProfile,
|
||||
|
||||
@@ -49,8 +49,8 @@ func TestResolvePipelineWithExplicitModules(t *testing.T) {
|
||||
if !reflect.DeepEqual(resolved.Input, ModuleBinding{Module: "text", LLMProfile: "fast"}) {
|
||||
t.Fatalf("Input = %#v, want trimmed explicit input", resolved.Input)
|
||||
}
|
||||
if resolved.Chunk.Module != "window" || resolved.Chunk.LLMProfile != DefaultLLMProfile {
|
||||
t.Fatalf("Chunk = %#v, want explicit module and default LLM profile", resolved.Chunk)
|
||||
if resolved.Chunk.Module != "window" || resolved.Chunk.LLMProfile != "" {
|
||||
t.Fatalf("Chunk = %#v, want explicit module and empty LLM profile", resolved.Chunk)
|
||||
}
|
||||
if resolved.Chunk.Options["size"] != 10 {
|
||||
t.Fatalf("Chunk.Options = %#v, want size option", resolved.Chunk.Options)
|
||||
@@ -91,24 +91,24 @@ func TestResolvePipelineAppliesDefaults(t *testing.T) {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
if resolved.Input.LLMProfile != DefaultLLMProfile {
|
||||
t.Fatalf("Input.LLMProfile = %q, want %q", resolved.Input.LLMProfile, DefaultLLMProfile)
|
||||
if resolved.Input.LLMProfile != "" {
|
||||
t.Fatalf("Input.LLMProfile = %q, want empty", resolved.Input.LLMProfile)
|
||||
}
|
||||
if !reflect.DeepEqual(resolved.Chunk, ModuleBinding{Module: DefaultChunkModule, LLMProfile: DefaultLLMProfile}) {
|
||||
if !reflect.DeepEqual(resolved.Chunk, ModuleBinding{Module: DefaultChunkModule}) {
|
||||
t.Fatalf("Chunk = %#v, want default chunk binding", resolved.Chunk)
|
||||
}
|
||||
if !reflect.DeepEqual(resolved.Output, ModuleBinding{Module: DefaultOutputModule, LLMProfile: DefaultLLMProfile}) {
|
||||
if !reflect.DeepEqual(resolved.Output, ModuleBinding{Module: DefaultOutputModule}) {
|
||||
t.Fatalf("Output = %#v, want default output binding", resolved.Output)
|
||||
}
|
||||
lane := resolved.ArtifactLanes[0]
|
||||
if !reflect.DeepEqual(lane.Merge, ModuleBinding{Module: DefaultMergeModule, LLMProfile: DefaultLLMProfile}) {
|
||||
if !reflect.DeepEqual(lane.Merge, ModuleBinding{Module: DefaultMergeModule}) {
|
||||
t.Fatalf("Merge = %#v, want default merge binding", lane.Merge)
|
||||
}
|
||||
if !reflect.DeepEqual(lane.Normalize, ModuleBinding{Module: DefaultNormalizeModule, LLMProfile: DefaultLLMProfile}) {
|
||||
if !reflect.DeepEqual(lane.Normalize, ModuleBinding{Module: DefaultNormalizeModule}) {
|
||||
t.Fatalf("Normalize = %#v, want default normalize binding", lane.Normalize)
|
||||
}
|
||||
if lane.Extract.LLMProfile != DefaultLLMProfile {
|
||||
t.Fatalf("Extract.LLMProfile = %q, want %q", lane.Extract.LLMProfile, DefaultLLMProfile)
|
||||
if lane.Extract.LLMProfile != "" {
|
||||
t.Fatalf("Extract.LLMProfile = %q, want empty", lane.Extract.LLMProfile)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"manifest": {
|
||||
"pipeline_id": "walking-skeleton",
|
||||
"pipeline_digest": "sha256:25084e39a0cadace375c896551d1752413755c1a6772f6b24cfe91c029ea2631",
|
||||
"pipeline_digest": "sha256:437d7ff486c336ddba38654ea00b7ef9dd6e49ce09f468698202ad8fc459bdcd",
|
||||
"validation_status": "approved",
|
||||
"artifact_lanes": [
|
||||
{
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: 1
|
||||
version: 2
|
||||
pipelines:
|
||||
dnd-spells-fixture:
|
||||
input: seriatim
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: 1
|
||||
version: 2
|
||||
pipelines:
|
||||
seriatim-fixture:
|
||||
input: seriatim
|
||||
|
||||
Reference in New Issue
Block a user