Add extract worker configuration controls

This commit is contained in:
2026-07-17 08:21:24 +00:00
parent adfe3825ee
commit 4023c66508
17 changed files with 343 additions and 22 deletions

View File

@@ -41,6 +41,7 @@ rejected; execution profiles now come from Scriptorium.
Built-in defaults: Built-in defaults:
- `concurrency.total_llm`: `1` - `concurrency.total_llm`: `1`
- `concurrency.stage_workers.extract`: effective `concurrency.total_llm`
- `diagnostics.work_dir`: `/tmp/notarius` - `diagnostics.work_dir`: `/tmp/notarius`
- `diagnostics.retention`: `auto` - `diagnostics.retention`: `auto`
- `workspace.directory`: unset - `workspace.directory`: unset
@@ -88,6 +89,7 @@ These environment variables are applied after the config file:
- `NOTARIUS_CONFIG`: config discovery path. - `NOTARIUS_CONFIG`: config discovery path.
- `NOTARIUS_TOTAL_LLM_CONCURRENCY`: integer global LLM concurrency. - `NOTARIUS_TOTAL_LLM_CONCURRENCY`: integer global LLM concurrency.
- `NOTARIUS_STAGE_WORKERS_EXTRACT`: integer extract worker limit.
- `NOTARIUS_WORKSPACE_DIR`: workspace directory. - `NOTARIUS_WORKSPACE_DIR`: workspace directory.
- `NOTARIUS_WORKSPACE_DIAGNOSTICS_ENABLED`: boolean diagnostics enablement. - `NOTARIUS_WORKSPACE_DIAGNOSTICS_ENABLED`: boolean diagnostics enablement.
- `NOTARIUS_WORKSPACE_DIAGNOSTICS_RETENTION`: workspace diagnostics retention - `NOTARIUS_WORKSPACE_DIAGNOSTICS_RETENTION`: workspace diagnostics retention
@@ -107,6 +109,24 @@ The removed `NOTARIUS_LLM_DEFAULT_*` variables are not read. Configure provider
endpoint, model, and credential environment variable names through Scriptorium endpoint, model, and credential environment variable names through Scriptorium
profiles. profiles.
## Concurrency
`concurrency` fields:
- `total_llm`: positive integer ceiling on concurrent provider calls.
- `stage_workers`: optional map of framework worker limits. The only supported
key is `extract`.
`stage_workers.extract` defaults to the effective `total_llm` value after file
and environment precedence. It must be between `1` and `total_llm`, inclusive.
Unknown or empty stage-worker keys are rejected. The environment override
`NOTARIUS_STAGE_WORKERS_EXTRACT` takes precedence over the file value, as does
`NOTARIUS_TOTAL_LLM_CONCURRENCY` for the global ceiling.
The worker value is present in effective and redacted configuration. The
current runner remains serial; this setting does not yet change execution
scheduling.
## Pipelines ## Pipelines
A pipeline selects implementations for the fixed workflow defined by A pipeline selects implementations for the fixed workflow defined by
@@ -262,9 +282,9 @@ production validators do not call the LLM and must not set `llm_profile`.
| input | `seriatim` | Reads Seriatim transcript JSON. | | input | `seriatim` | Reads Seriatim transcript JSON. |
| chunk | `generic` | Splits source units into ordered chunks. | | chunk | `generic` | Splits source units into ordered chunks. |
| chunk | `dnd/scenes` | Uses an LLM to split transcript source units into D&D scenes. | | chunk | `dnd/scenes` | Uses an LLM to split transcript source units into D&D scenes. |
| extract | `dnd/spells` | Extracts D&D spell raw outputs. | | extract | `dnd/spells` | Extracts typed D&D spell-list artifacts. |
| merge | `appendorder` | Merges JSON raw extract outputs in chunk order. | | merge | `appendorder` | Combines typed artifacts in chunk order. |
| normalize | `noop` | Passes merged raw outputs through unchanged. | | normalize | `noop` | Passes merged typed artifacts through unchanged. |
| output | `json` | Produces JSON output files for normalized `application/json` lanes. | | output | `json` | Produces JSON output files for normalized `application/json` lanes. |
## Implemented Production Validators ## Implemented Production Validators
@@ -275,7 +295,7 @@ production validators do not call the LLM and must not set `llm_profile`.
| `generic/always_reject` | deterministic | Rejects returned module output with reason `always_reject`. | | `generic/always_reject` | deterministic | Rejects returned module output with reason `always_reject`. |
| `generic/valid_json` | deterministic | Rejects payloads that are not syntactically valid JSON. | | `generic/valid_json` | deterministic | Rejects payloads that are not syntactically valid JSON. |
| `generic/valid_json_schema` | deterministic | Rejects invalid JSON or JSON that does not conform to the module response schema. | | `generic/valid_json_schema` | deterministic | Rejects invalid JSON or JSON that does not conform to the module response schema. |
| `extract/dnd/spells/shape` | deterministic | Rejects malformed D&D spell-cast JSON payloads. | | `extract/dnd/spells/shape` | deterministic | Rejects malformed D&D spell-list artifacts. |
| `extract/dnd/spells/source_refs` | deterministic | Rejects missing or invalid D&D spell source references. | | `extract/dnd/spells/source_refs` | deterministic | Rejects missing or invalid D&D spell source references. |
| `extract/dnd/spells/source_relatedness` | deterministic | Emits warnings when a spell name is not found near its cited source text. | | `extract/dnd/spells/source_relatedness` | deterministic | Emits warnings when a spell name is not found near its cited source text. |
@@ -366,6 +386,8 @@ Configuration validation checks:
- mutually exclusive `scriptorium.profile_dir` and `scriptorium.profile_file`; - mutually exclusive `scriptorium.profile_dir` and `scriptorium.profile_file`;
- non-empty, non-duplicated IDs after trimming; - non-empty, non-duplicated IDs after trimming;
- positive global LLM concurrency; - positive global LLM concurrency;
- supported stage-worker keys and an effective extract worker count in the
inclusive range `1..concurrency.total_llm`;
- supported diagnostics retention and non-empty work directory; - supported diagnostics retention and non-empty work directory;
- stale removed fields such as `llm_profiles`. - stale removed fields such as `llm_profiles`.

View File

@@ -209,8 +209,8 @@ does not inventory implementations.
## Tests To Inspect ## Tests To Inspect
- Package-local `*_test.go` files under the module or validator being changed. - Package-local `*_test.go` files under the module or validator being changed.
- `internal/framework/pipeline/registry_integration_test.go`: registry and spec - `internal/framework/pipeline/typed_resolution_test.go`: typed registry, spec,
composition. and heterogeneous artifact composition.
- `internal/framework/pipeline/default_modules_test.go`: framework binding - `internal/framework/pipeline/default_modules_test.go`: framework binding
defaults. defaults.
- `internal/cli/run_test.go`: production catalog, config resolution, and - `internal/cli/run_test.go`: production catalog, config resolution, and

View File

@@ -245,9 +245,10 @@ durable manifest and logical file schemas are defined in the
construction order, dependency failures, and the before-source-work boundary. construction order, dependency failures, and the before-source-work boundary.
- `internal/framework/pipeline/references_test.go`: target resolution and - `internal/framework/pipeline/references_test.go`: target resolution and
materialization. materialization.
- `internal/framework/pipeline/runner_test.go`: stage transitions, retries, - `internal/cli/run_test.go`: production stage transitions, retries, rejections,
rejections, warnings, checkpoints, debug hooks, and manifests. warnings, debug hooks, manifests, and end-to-end composition.
- `internal/framework/pipeline/walking_skeleton_test.go`: fake-backed complete - `internal/modules/integration/*_test.go` and
workflow composition. `internal/modules/seriatim/input/transcript/runner_test.go`: typed runner
composition across concrete module families.
- `internal/framework/checkpoint/*_test.go`: checkpoint serialization and reuse - `internal/framework/checkpoint/*_test.go`: checkpoint serialization and reuse
collaborators. collaborators.

View File

@@ -111,8 +111,8 @@ file placement, checkpoints, or diagnostics.
## Validation ## Validation
Validation is a framework-managed boundary around raw outputs from chunk, Validation is a framework-managed boundary around outputs from chunk, extract,
extract, merge, and normalize stages. Validators receive immutable stage output merge, and normalize stages. Validators receive immutable stage output
and make an explicit whole-output decision: approve, approve with warnings, or and make an explicit whole-output decision: approve, approve with warnings, or
reject. reject.

View File

@@ -1,6 +1,8 @@
version: 2 version: 2
concurrency: concurrency:
total_llm: 1 total_llm: 1
stage_workers:
extract: 1
workspace: workspace:
directory: /var/lib/notarius directory: /var/lib/notarius
diagnostics: diagnostics:

View File

@@ -172,8 +172,8 @@ func TestProductionCompatibilitySnapshot(t *testing.T) {
t.Fatalf("Resolve(production example) error = %v, want nil", err) t.Fatalf("Resolve(production example) error = %v, want nil", err)
} }
productionResolved := productionEffective.ResolvedPipeline productionResolved := productionEffective.ResolvedPipeline
if productionConfig.Concurrency.TotalLLM != 1 || !reflect.DeepEqual(productionResolved.Chunk.Options, map[string]any{"max_units": 50}) { if productionConfig.Concurrency.TotalLLM != 1 || productionConfig.Concurrency.StageWorkers["extract"] != 1 || !reflect.DeepEqual(productionResolved.Chunk.Options, map[string]any{"max_units": 50}) {
t.Fatalf("production example concurrency/options = %d/%#v, want compatibility snapshot", productionConfig.Concurrency.TotalLLM, productionResolved.Chunk.Options) t.Fatalf("production example concurrency/options = %#v/%#v, want compatibility snapshot", productionConfig.Concurrency, productionResolved.Chunk.Options)
} }
bindings := productionResolved.ArtifactLanes[0].ExtractReferences.Bindings bindings := productionResolved.ArtifactLanes[0].ExtractReferences.Bindings
if len(bindings) != 2 || bindings[0].SlotName != "glossary" || bindings[0].Source != "./dnd-spells-glossary.txt" || bindings[1].SlotName != "party" || bindings[1].Source != "./dnd-spells-roster.txt" { if len(bindings) != 2 || bindings[0].SlotName != "glossary" || bindings[0].Source != "./dnd-spells-glossary.txt" || bindings[1].SlotName != "party" || bindings[1].Source != "./dnd-spells-roster.txt" {

View File

@@ -25,6 +25,10 @@ type ScriptoriumConfig struct {
type ConcurrencyConfig struct { type ConcurrencyConfig struct {
TotalLLM int `json:"total_llm"` TotalLLM int `json:"total_llm"`
StageWorkers map[string]int `json:"stage_workers"`
extractWorkersConfigured bool
defaultedExtractWorkers int
} }
type DiagnosticsConfig struct { type DiagnosticsConfig struct {
@@ -59,6 +63,8 @@ func Default() Config {
Pipelines: map[string]pipeline.PipelineProfile{}, Pipelines: map[string]pipeline.PipelineProfile{},
Concurrency: ConcurrencyConfig{ Concurrency: ConcurrencyConfig{
TotalLLM: 1, TotalLLM: 1,
StageWorkers: map[string]int{"extract": 1},
defaultedExtractWorkers: 1,
}, },
Diagnostics: DiagnosticsConfig{ Diagnostics: DiagnosticsConfig{
WorkDir: "/tmp/notarius", WorkDir: "/tmp/notarius",
@@ -101,6 +107,7 @@ func (c Config) workspaceDirectory() string {
func cloneConfig(in Config) Config { func cloneConfig(in Config) Config {
out := in out := in
out.Concurrency.StageWorkers = cloneIntMap(in.Concurrency.StageWorkers)
out.Pipelines = make(map[string]pipeline.PipelineProfile, len(in.Pipelines)) out.Pipelines = make(map[string]pipeline.PipelineProfile, len(in.Pipelines))
for key, profile := range in.Pipelines { for key, profile := range in.Pipelines {
out.Pipelines[key] = clonePipelineProfile(profile) out.Pipelines[key] = clonePipelineProfile(profile)
@@ -108,6 +115,34 @@ func cloneConfig(in Config) Config {
return out return out
} }
func cloneIntMap(in map[string]int) map[string]int {
if len(in) == 0 {
return nil
}
out := make(map[string]int, len(in))
for key, value := range in {
out[key] = value
}
return out
}
func (c *ConcurrencyConfig) recomputeStageWorkerDefaults() {
if c == nil {
return
}
if c.StageWorkers == nil {
c.StageWorkers = make(map[string]int)
}
if !c.extractWorkersConfigured {
if value, ok := c.StageWorkers["extract"]; ok && (c.defaultedExtractWorkers == 0 || value != c.defaultedExtractWorkers) {
c.extractWorkersConfigured = true
return
}
c.StageWorkers["extract"] = c.TotalLLM
c.defaultedExtractWorkers = c.TotalLLM
}
}
func clonePipelineProfile(in pipeline.PipelineProfile) pipeline.PipelineProfile { func clonePipelineProfile(in pipeline.PipelineProfile) pipeline.PipelineProfile {
out := in out := in
out.Input = cloneModuleBinding(in.Input) out.Input = cloneModuleBinding(in.Input)

View File

@@ -18,6 +18,9 @@ func TestDefaultValues(t *testing.T) {
if cfg.Concurrency.TotalLLM != 1 { if cfg.Concurrency.TotalLLM != 1 {
t.Fatalf("unexpected total LLM concurrency: %d", cfg.Concurrency.TotalLLM) t.Fatalf("unexpected total LLM concurrency: %d", cfg.Concurrency.TotalLLM)
} }
if got := cfg.Concurrency.StageWorkers["extract"]; got != 1 {
t.Fatalf("unexpected extract workers: %d", got)
}
if cfg.Diagnostics.WorkDir != "/tmp/notarius" { if cfg.Diagnostics.WorkDir != "/tmp/notarius" {
t.Fatalf("unexpected diagnostics work dir: %q", cfg.Diagnostics.WorkDir) t.Fatalf("unexpected diagnostics work dir: %q", cfg.Diagnostics.WorkDir)
} }
@@ -68,6 +71,9 @@ pipelines:
if cfg.Concurrency.TotalLLM != 1 { if cfg.Concurrency.TotalLLM != 1 {
t.Fatalf("expected default concurrency preserved, got %d", cfg.Concurrency.TotalLLM) t.Fatalf("expected default concurrency preserved, got %d", cfg.Concurrency.TotalLLM)
} }
if got := cfg.Concurrency.StageWorkers["extract"]; got != 1 {
t.Fatalf("expected default extract workers preserved, got %d", got)
}
if cfg.Diagnostics.Retention != diagnostics.RetentionAuto { if cfg.Diagnostics.Retention != diagnostics.RetentionAuto {
t.Fatalf("expected default diagnostics retention preserved, got %q", cfg.Diagnostics.Retention) t.Fatalf("expected default diagnostics retention preserved, got %q", cfg.Diagnostics.Retention)
} }

View File

@@ -26,6 +26,7 @@ type EffectiveConfig struct {
} }
func (c Config) Resolve(input ResolveInput) (EffectiveConfig, error) { func (c Config) Resolve(input ResolveInput) (EffectiveConfig, error) {
c.Concurrency.recomputeStageWorkerDefaults()
if err := c.Validate(); err != nil { if err := c.Validate(); err != nil {
return EffectiveConfig{}, err return EffectiveConfig{}, err
} }

View File

@@ -27,6 +27,19 @@ func TestResolveRejectsEmptyAndUnknownPipelineID(t *testing.T) {
} }
} }
func TestResolveMaterializesDefaultExtractWorkersFromEffectiveTotal(t *testing.T) {
cfg := validConfig()
cfg.Concurrency.TotalLLM = 4
effective, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: fakeCatalog(t)})
if err != nil {
t.Fatalf("Resolve() error = %v, want nil", err)
}
if got := effective.Config.Concurrency.StageWorkers["extract"]; got != 4 {
t.Fatalf("effective extract workers = %d, want total concurrency 4", got)
}
}
func TestResolveLaneFilteringSuccessAndFailure(t *testing.T) { func TestResolveLaneFilteringSuccessAndFailure(t *testing.T) {
effective, err := validConfig().Resolve(ResolveInput{ effective, err := validConfig().Resolve(ResolveInput{
PipelineID: " example ", PipelineID: " example ",

View File

@@ -36,6 +36,18 @@ func (c *Config) applyEnvOverridesWithLookup(lookup func(string) (string, bool))
} }
c.Concurrency.TotalLLM = value c.Concurrency.TotalLLM = value
} }
if raw, ok := lookup("NOTARIUS_STAGE_WORKERS_EXTRACT"); ok {
value, err := parseIntEnv("NOTARIUS_STAGE_WORKERS_EXTRACT", raw)
if err != nil {
return err
}
if c.Concurrency.StageWorkers == nil {
c.Concurrency.StageWorkers = make(map[string]int)
}
c.Concurrency.StageWorkers["extract"] = value
c.Concurrency.extractWorkersConfigured = true
}
c.Concurrency.recomputeStageWorkerDefaults()
if raw, ok := lookup("NOTARIUS_WORK_DIR"); ok { if raw, ok := lookup("NOTARIUS_WORK_DIR"); ok {
c.Diagnostics.WorkDir = strings.TrimSpace(raw) c.Diagnostics.WorkDir = strings.TrimSpace(raw)
} }

View File

@@ -14,6 +14,7 @@ func TestApplyEnvOverridesOperationalValues(t *testing.T) {
err := cfg.applyEnvOverridesWithLookup(mapLookup(map[string]string{ err := cfg.applyEnvOverridesWithLookup(mapLookup(map[string]string{
"NOTARIUS_TOTAL_LLM_CONCURRENCY": "3", "NOTARIUS_TOTAL_LLM_CONCURRENCY": "3",
"NOTARIUS_STAGE_WORKERS_EXTRACT": "2",
"NOTARIUS_WORK_DIR": "/tmp/notarius-env", "NOTARIUS_WORK_DIR": "/tmp/notarius-env",
"NOTARIUS_DIAGNOSTICS_RETENTION": "never", "NOTARIUS_DIAGNOSTICS_RETENTION": "never",
"NOTARIUS_WORKSPACE_DIR": "/var/lib/notarius-env", "NOTARIUS_WORKSPACE_DIR": "/var/lib/notarius-env",
@@ -33,6 +34,9 @@ func TestApplyEnvOverridesOperationalValues(t *testing.T) {
if cfg.Concurrency.TotalLLM != 3 { if cfg.Concurrency.TotalLLM != 3 {
t.Fatalf("unexpected total concurrency: %d", cfg.Concurrency.TotalLLM) t.Fatalf("unexpected total concurrency: %d", cfg.Concurrency.TotalLLM)
} }
if got := cfg.Concurrency.StageWorkers["extract"]; got != 2 {
t.Fatalf("extract workers = %d, want 2", got)
}
if cfg.Workspace.Directory != "/var/lib/notarius-env" { if cfg.Workspace.Directory != "/var/lib/notarius-env" {
t.Fatalf("unexpected workspace directory: %q", cfg.Workspace.Directory) t.Fatalf("unexpected workspace directory: %q", cfg.Workspace.Directory)
} }
@@ -54,13 +58,75 @@ func TestApplyEnvOverridesOperationalValues(t *testing.T) {
} }
func TestApplyEnvOverridesRejectsInvalidIntegers(t *testing.T) { func TestApplyEnvOverridesRejectsInvalidIntegers(t *testing.T) {
for _, name := range []string{"NOTARIUS_TOTAL_LLM_CONCURRENCY", "NOTARIUS_STAGE_WORKERS_EXTRACT"} {
t.Run(name, func(t *testing.T) {
cfg := Default() cfg := Default()
err := cfg.applyEnvOverridesWithLookup(mapLookup(map[string]string{ err := cfg.applyEnvOverridesWithLookup(mapLookup(map[string]string{name: "many"}))
"NOTARIUS_TOTAL_LLM_CONCURRENCY": "many", if err == nil || !strings.Contains(err.Error(), name) {
}))
if err == nil || !strings.Contains(err.Error(), "NOTARIUS_TOTAL_LLM_CONCURRENCY") {
t.Fatalf("expected named integer error, got %v", err) t.Fatalf("expected named integer error, got %v", err)
} }
})
}
}
func TestStageWorkerEnvironmentPrecedenceAndDefaulting(t *testing.T) {
fileCfg, err := ParseFileConfigYAML([]byte(`
version: 2
concurrency:
total_llm: 4
stage_workers:
extract: 2
`))
if err != nil {
t.Fatalf("ParseFileConfigYAML() error = %v", err)
}
cfg := Default()
if err := cfg.applyFileConfigWithLookup(fileCfg, emptyLookup); err != nil {
t.Fatalf("ApplyFileConfig() error = %v", err)
}
if err := cfg.applyEnvOverridesWithLookup(mapLookup(map[string]string{
"NOTARIUS_TOTAL_LLM_CONCURRENCY": "5",
"NOTARIUS_STAGE_WORKERS_EXTRACT": "3",
})); err != nil {
t.Fatalf("ApplyEnvOverrides() error = %v", err)
}
if cfg.Concurrency.TotalLLM != 5 || cfg.Concurrency.StageWorkers["extract"] != 3 {
t.Fatalf("effective concurrency = %#v, want total 5 and extract 3", cfg.Concurrency)
}
if err := cfg.Validate(); err != nil {
t.Fatalf("Validate(overridden) error = %v, want nil", err)
}
defaulted := Default()
if err := defaulted.applyEnvOverridesWithLookup(mapLookup(map[string]string{"NOTARIUS_TOTAL_LLM_CONCURRENCY": "6"})); err != nil {
t.Fatalf("ApplyEnvOverrides(defaulted) error = %v", err)
}
if got := defaulted.Concurrency.StageWorkers["extract"]; got != 6 {
t.Fatalf("defaulted extract workers = %d, want effective total 6", got)
}
}
func TestStageWorkerRangeValidationUsesFinalEnvironmentTotal(t *testing.T) {
fileCfg, err := ParseFileConfigYAML([]byte(`
version: 2
concurrency:
total_llm: 4
stage_workers:
extract: 5
`))
if err != nil {
t.Fatalf("ParseFileConfigYAML() error = %v", err)
}
cfg := Default()
if err := cfg.applyFileConfigWithLookup(fileCfg, emptyLookup); err != nil {
t.Fatalf("ApplyFileConfig() error = %v", err)
}
if err := cfg.applyEnvOverridesWithLookup(mapLookup(map[string]string{"NOTARIUS_TOTAL_LLM_CONCURRENCY": "6"})); err != nil {
t.Fatalf("ApplyEnvOverrides() error = %v", err)
}
if err := cfg.Validate(); err != nil {
t.Fatalf("Validate() error = %v, want final total to make extract workers valid", err)
}
} }
func TestApplyEnvOverridesRejectsInvalidBooleans(t *testing.T) { func TestApplyEnvOverridesRejectsInvalidBooleans(t *testing.T) {
@@ -111,4 +177,7 @@ func TestLoadFromEnvUsesDefaultConfig(t *testing.T) {
if cfg.Concurrency.TotalLLM != 2 { if cfg.Concurrency.TotalLLM != 2 {
t.Fatalf("expected env concurrency override, got %+v", cfg.Concurrency) t.Fatalf("expected env concurrency override, got %+v", cfg.Concurrency)
} }
if got := cfg.Concurrency.StageWorkers["extract"]; got != 2 {
t.Fatalf("expected extract workers to default to total, got %d", got)
}
} }

View File

@@ -44,6 +44,7 @@ type FileArtifactLaneProfile struct {
type FileConcurrencyConfig struct { type FileConcurrencyConfig struct {
TotalLLM *int `yaml:"total_llm,omitempty"` TotalLLM *int `yaml:"total_llm,omitempty"`
StageWorkers map[string]int `yaml:"stage_workers,omitempty"`
} }
type FileDiagnosticsConfig struct { type FileDiagnosticsConfig struct {
@@ -316,6 +317,15 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
if fileCfg.Concurrency != nil && fileCfg.Concurrency.TotalLLM != nil { if fileCfg.Concurrency != nil && fileCfg.Concurrency.TotalLLM != nil {
c.Concurrency.TotalLLM = *fileCfg.Concurrency.TotalLLM c.Concurrency.TotalLLM = *fileCfg.Concurrency.TotalLLM
} }
if fileCfg.Concurrency != nil && fileCfg.Concurrency.StageWorkers != nil {
workers, configured, err := normalizeStageWorkers(fileCfg.Concurrency.StageWorkers)
if err != nil {
return err
}
c.Concurrency.StageWorkers = workers
c.Concurrency.extractWorkersConfigured = configured
}
c.Concurrency.recomputeStageWorkerDefaults()
if fileCfg.Diagnostics != nil { if fileCfg.Diagnostics != nil {
if fileCfg.Diagnostics.WorkDir != nil { if fileCfg.Diagnostics.WorkDir != nil {
c.Diagnostics.WorkDir = strings.TrimSpace(*fileCfg.Diagnostics.WorkDir) c.Diagnostics.WorkDir = strings.TrimSpace(*fileCfg.Diagnostics.WorkDir)
@@ -350,6 +360,26 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
return nil return nil
} }
func normalizeStageWorkers(values map[string]int) (map[string]int, bool, error) {
workers := make(map[string]int, len(values))
configured := false
for rawKey, value := range values {
key := strings.TrimSpace(rawKey)
if key == "" {
return nil, false, fmt.Errorf("concurrency.stage_workers key must not be empty")
}
if key != "extract" {
return nil, false, fmt.Errorf("concurrency.stage_workers key %q is not supported", rawKey)
}
if _, exists := workers[key]; exists {
return nil, false, fmt.Errorf("concurrency.stage_workers key %q is duplicated after trimming", key)
}
workers[key] = value
configured = true
}
return workers, configured, nil
}
func normalizedMapKeys[T any](values map[string]T, keyName string) ([]string, map[string]string, error) { func normalizedMapKeys[T any](values map[string]T, keyName string) ([]string, map[string]string, error) {
keys := make([]string, 0, len(values)) keys := make([]string, 0, len(values))
rawByNormalized := make(map[string]string, len(values)) rawByNormalized := make(map[string]string, len(values))

View File

@@ -546,6 +546,9 @@ diagnostics:
if cfg.Concurrency.TotalLLM != 4 { if cfg.Concurrency.TotalLLM != 4 {
t.Fatalf("unexpected total concurrency: %d", cfg.Concurrency.TotalLLM) t.Fatalf("unexpected total concurrency: %d", cfg.Concurrency.TotalLLM)
} }
if got := cfg.Concurrency.StageWorkers["extract"]; got != 4 {
t.Fatalf("default extract workers = %d, want total concurrency", got)
}
if cfg.Diagnostics.WorkDir != "/tmp/notarius-test" { if cfg.Diagnostics.WorkDir != "/tmp/notarius-test" {
t.Fatalf("unexpected work dir: %q", cfg.Diagnostics.WorkDir) t.Fatalf("unexpected work dir: %q", cfg.Diagnostics.WorkDir)
} }
@@ -554,6 +557,58 @@ diagnostics:
} }
} }
func TestApplyFileConfigStageWorkers(t *testing.T) {
cfg := parseAndApplyConfig(t, `
version: 2
concurrency:
total_llm: 4
stage_workers:
extract: 3
`)
if got := cfg.Concurrency.StageWorkers["extract"]; got != 3 {
t.Fatalf("extract workers = %d, want 3", got)
}
if err := cfg.Validate(); err != nil {
t.Fatalf("Validate() error = %v, want nil", err)
}
}
func TestApplyFileConfigEmptyStageWorkersDefaultsExtractToTotal(t *testing.T) {
cfg := parseAndApplyConfig(t, `
version: 2
concurrency:
total_llm: 4
stage_workers: {}
`)
if got := cfg.Concurrency.StageWorkers["extract"]; got != 4 {
t.Fatalf("extract workers = %d, want total concurrency 4", got)
}
}
func TestApplyFileConfigRejectsUnsupportedStageWorkerKeys(t *testing.T) {
for _, test := range []struct {
name string
key string
want string
}{
{name: "empty", key: "' '", want: "must not be empty"},
{name: "unknown", key: "merge", want: "not supported"},
} {
t.Run(test.name, func(t *testing.T) {
fileCfg, err := ParseFileConfigYAML([]byte("version: 2\nconcurrency:\n stage_workers:\n " + test.key + ": 1\n"))
if err != nil {
t.Fatalf("ParseFileConfigYAML() error = %v", err)
}
cfg := Default()
err = cfg.applyFileConfigWithLookup(fileCfg, emptyLookup)
if err == nil || !strings.Contains(err.Error(), test.want) {
t.Fatalf("ApplyFileConfig() error = %v, want %q", err, test.want)
}
})
}
}
func TestApplyFileConfigWorkspaceSection(t *testing.T) { func TestApplyFileConfigWorkspaceSection(t *testing.T) {
cfg := parseAndApplyConfig(t, ` cfg := parseAndApplyConfig(t, `
version: 2 version: 2

View File

@@ -12,6 +12,7 @@ func TestRedactedConfigCopiesScriptoriumConfig(t *testing.T) {
cfg.Scriptorium.ProfileDir = "./profiles" cfg.Scriptorium.ProfileDir = "./profiles"
cfg.Workspace.Directory = "/var/lib/notarius" cfg.Workspace.Directory = "/var/lib/notarius"
cfg.Workspace.Resume.Enabled = true cfg.Workspace.Resume.Enabled = true
cfg.Concurrency.StageWorkers["extract"] = 1
redacted := cfg.Redacted() redacted := cfg.Redacted()
@@ -29,6 +30,10 @@ func TestRedactedConfigCopiesScriptoriumConfig(t *testing.T) {
if cfg.Workspace.Directory != "/var/lib/notarius" { if cfg.Workspace.Directory != "/var/lib/notarius" {
t.Fatalf("redaction mutated original workspace config") t.Fatalf("redaction mutated original workspace config")
} }
redacted.Concurrency.StageWorkers["extract"] = 9
if cfg.Concurrency.StageWorkers["extract"] != 1 {
t.Fatalf("redaction aliased stage worker map")
}
} }
func TestConfigRedactedDiagnosticsPayloadCopiesConfig(t *testing.T) { func TestConfigRedactedDiagnosticsPayloadCopiesConfig(t *testing.T) {
@@ -46,6 +51,8 @@ func TestConfigRedactedDiagnosticsPayloadCopiesConfig(t *testing.T) {
func TestEffectiveConfigRedactedDiagnosticsPayloadCopies(t *testing.T) { func TestEffectiveConfigRedactedDiagnosticsPayloadCopies(t *testing.T) {
cfg := validConfig() cfg := validConfig()
cfg.Concurrency.TotalLLM = 4
cfg.Concurrency.StageWorkers["extract"] = 2
lane := cfg.Pipelines["example"].Artifacts["events"] lane := cfg.Pipelines["example"].Artifacts["events"]
lane.Extract.Options = map[string]any{"temperature": 0.2} lane.Extract.Options = map[string]any{"temperature": 0.2}
lane.References = map[string]string{"roster": "./roster.yml"} lane.References = map[string]string{"roster": "./roster.yml"}
@@ -101,6 +108,10 @@ func TestEffectiveConfigRedactedDiagnosticsPayloadCopies(t *testing.T) {
if payload.PipelineID != effective.PipelineID || payload.ResolvedPipeline.Digest != effective.ResolvedPipeline.Digest { if payload.PipelineID != effective.PipelineID || payload.ResolvedPipeline.Digest != effective.ResolvedPipeline.Digest {
t.Fatalf("expected pipeline metadata preserved, got %+v", payload) t.Fatalf("expected pipeline metadata preserved, got %+v", payload)
} }
payload.Config.Concurrency.StageWorkers["extract"] = 4
if effective.Config.Concurrency.StageWorkers["extract"] != 2 {
t.Fatalf("expected effective stage worker map to be copied")
}
payload.Only[0] = "changed" payload.Only[0] = "changed"
if effective.Only[0] != "events" { if effective.Only[0] != "events" {

View File

@@ -2,6 +2,7 @@ package config
import ( import (
"fmt" "fmt"
"sort"
"strings" "strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics" "gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
@@ -9,6 +10,7 @@ import (
) )
func (c Config) Validate() error { func (c Config) Validate() error {
c.Concurrency.recomputeStageWorkerDefaults()
if err := validateScriptorium(c.Scriptorium); err != nil { if err := validateScriptorium(c.Scriptorium); err != nil {
return err return err
} }
@@ -21,9 +23,36 @@ func (c Config) Validate() error {
if c.Concurrency.TotalLLM <= 0 { if c.Concurrency.TotalLLM <= 0 {
return fmt.Errorf("total LLM concurrency must be greater than zero") return fmt.Errorf("total LLM concurrency must be greater than zero")
} }
if err := validateStageWorkers(c.Concurrency); err != nil {
return err
}
return validatePipelineProfiles(c.Pipelines) return validatePipelineProfiles(c.Pipelines)
} }
func validateStageWorkers(cfg ConcurrencyConfig) error {
keys := make([]string, 0, len(cfg.StageWorkers))
for key := range cfg.StageWorkers {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
if strings.TrimSpace(key) == "" {
return fmt.Errorf("concurrency.stage_workers key must not be empty")
}
if key != "extract" {
return fmt.Errorf("concurrency.stage_workers key %q is not supported", key)
}
}
extractWorkers, ok := cfg.StageWorkers["extract"]
if !ok {
extractWorkers = cfg.TotalLLM
}
if extractWorkers < 1 || extractWorkers > cfg.TotalLLM {
return fmt.Errorf("concurrency.stage_workers.extract must be between 1 and concurrency.total_llm (%d)", cfg.TotalLLM)
}
return nil
}
func validateScriptorium(cfg ScriptoriumConfig) error { func validateScriptorium(cfg ScriptoriumConfig) error {
if strings.TrimSpace(cfg.ProfileDir) != "" && strings.TrimSpace(cfg.ProfileFile) != "" { if strings.TrimSpace(cfg.ProfileDir) != "" && strings.TrimSpace(cfg.ProfileFile) != "" {
return fmt.Errorf("scriptorium profile_dir and profile_file are mutually exclusive") return fmt.Errorf("scriptorium profile_dir and profile_file are mutually exclusive")

View File

@@ -79,6 +79,41 @@ func TestValidateRejectsInvalidNumericFields(t *testing.T) {
} }
} }
func TestValidateStageWorkerBoundaries(t *testing.T) {
for _, test := range []struct {
name string
workers int
wantErr bool
}{
{name: "below minimum", workers: 0, wantErr: true},
{name: "minimum", workers: 1},
{name: "maximum", workers: 4},
{name: "above maximum", workers: 5, wantErr: true},
} {
t.Run(test.name, func(t *testing.T) {
cfg := validConfig()
cfg.Concurrency.TotalLLM = 4
cfg.Concurrency.StageWorkers["extract"] = test.workers
err := cfg.Validate()
if test.wantErr && (err == nil || !strings.Contains(err.Error(), "stage_workers.extract")) {
t.Fatalf("Validate() error = %v, want extract worker range error", err)
}
if !test.wantErr && err != nil {
t.Fatalf("Validate() error = %v, want nil", err)
}
})
}
}
func TestValidateRejectsUnknownEffectiveStageWorkerKey(t *testing.T) {
cfg := validConfig()
cfg.Concurrency.StageWorkers["merge"] = 1
err := cfg.Validate()
if err == nil || !strings.Contains(err.Error(), "stage_workers key") || !strings.Contains(err.Error(), "merge") {
t.Fatalf("Validate() error = %v, want unknown stage worker key", err)
}
}
func TestValidateRejectsMutuallyExclusiveScriptoriumProfileSources(t *testing.T) { func TestValidateRejectsMutuallyExclusiveScriptoriumProfileSources(t *testing.T) {
cfg := validConfig() cfg := validConfig()
cfg.Scriptorium.ProfileDir = "./profiles" cfg.Scriptorium.ProfileDir = "./profiles"