Preserve explicit worker counts when normalizing keys
This commit is contained in:
@@ -100,6 +100,9 @@ chunk/extract/merge/normalize and valid reference forms remain accepted.
|
|||||||
|
|
||||||
## Finding 3 — Restore Worker-Key Normalization Without Losing Explicitness
|
## Finding 3 — Restore Worker-Key Normalization Without Losing Explicitness
|
||||||
|
|
||||||
|
Status: Addressed. Worker keys normalize before defaulting; original input
|
||||||
|
presence preserves explicit counts. Regression tests cover defaults and collisions.
|
||||||
|
|
||||||
Priority: P2. The migration introduced an unplanned restriction.
|
Priority: P2. The migration introduced an unplanned restriction.
|
||||||
|
|
||||||
The former `normalizeStageWorkers` trimmed keys and rejected normalized
|
The former `normalizeStageWorkers` trimmed keys and rejected normalized
|
||||||
|
|||||||
@@ -18,7 +18,12 @@ func resolveDocumentOperations(document *Document, resolution *configkit.Resolut
|
|||||||
normalizeDocumentState(document, resolution)
|
normalizeDocumentState(document, resolution)
|
||||||
normalizeDocumentPipelines(document, resolution, state)
|
normalizeDocumentPipelines(document, resolution, state)
|
||||||
normalizeDocumentPaths(document, resolution, state)
|
normalizeDocumentPaths(document, resolution, state)
|
||||||
deriveDocumentExtractWorkers(document, resolution)
|
if document.Concurrency != nil {
|
||||||
|
document.Concurrency.StageWorkers = normalizeDocumentMap(document.Concurrency.StageWorkers,
|
||||||
|
configkit.Keys("concurrency", "stage_workers"), "stage worker identifier",
|
||||||
|
documentNormalizer{Resolution: resolution}, state)
|
||||||
|
}
|
||||||
|
deriveDocumentExtractWorkers(document, resolution, state)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -74,7 +79,7 @@ func normalizeDocumentString(resolution *configkit.Resolution, path configkit.Pa
|
|||||||
configkit.Normalize(resolution, path, target, documentValue(strings.TrimSpace(**target)), reason)
|
configkit.Normalize(resolution, path, target, documentValue(strings.TrimSpace(**target)), reason)
|
||||||
}
|
}
|
||||||
|
|
||||||
func deriveDocumentExtractWorkers(document *Document, resolution *configkit.Resolution) {
|
func deriveDocumentExtractWorkers(document *Document, resolution *configkit.Resolution, state *documentLoadState) {
|
||||||
if document.Concurrency == nil || document.Concurrency.TotalLLM == nil {
|
if document.Concurrency == nil || document.Concurrency.TotalLLM == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -82,6 +87,11 @@ func deriveDocumentExtractWorkers(document *Document, resolution *configkit.Reso
|
|||||||
document.Concurrency.StageWorkers = make(map[string]int)
|
document.Concurrency.StageWorkers = make(map[string]int)
|
||||||
}
|
}
|
||||||
path := configkit.Keys("concurrency", "stage_workers", "extract")
|
path := configkit.Keys("concurrency", "stage_workers", "extract")
|
||||||
|
// Presence belongs to the input spelling, not the key produced by trimming.
|
||||||
|
// A moved explicit count must never be replaced by a derived default.
|
||||||
|
if resolution.IsExplicit(state.originalPath(path)) {
|
||||||
|
return
|
||||||
|
}
|
||||||
workers := document.Concurrency.StageWorkers
|
workers := document.Concurrency.StageWorkers
|
||||||
value := workers["extract"]
|
value := workers["extract"]
|
||||||
configkit.Default(resolution, path, &value, *document.Concurrency.TotalLLM,
|
configkit.Default(resolution, path, &value, *document.Concurrency.TotalLLM,
|
||||||
|
|||||||
@@ -1,14 +1,54 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"errors"
|
"errors"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func TestWorkerKeyNormalizationPreservesExplicitCount(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
for _, test := range []struct {
|
||||||
|
name, workers string
|
||||||
|
want int
|
||||||
|
errorPath string
|
||||||
|
}{
|
||||||
|
{name: "omitted", workers: "{}", want: 4},
|
||||||
|
{name: "padded", workers: "{' extract ': 2}", want: 2},
|
||||||
|
{name: "explicit invalid", workers: "{' extract ': 0}", errorPath: "/concurrency/stage_workers/extract"},
|
||||||
|
{name: "collision", workers: "{' extract ': 2, extract: 3}", errorPath: "/concurrency/stage_workers/extract"},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
loaded, err := loadConfigSource(configkit.Bytes("workers.yml", []byte(
|
||||||
|
"version: 5\nconcurrency:\n total_llm: 4\n stage_workers: "+test.workers+"\n")), LoadOptions{})
|
||||||
|
if test.errorPath != "" {
|
||||||
|
assertDocumentDiagnosticAt(t, err, test.errorPath)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := loaded.Config.Concurrency.StageWorkers["extract"]; got != test.want {
|
||||||
|
t.Fatalf("workers = %d, want %d", got, test.want)
|
||||||
|
}
|
||||||
|
var explanation bytes.Buffer
|
||||||
|
if err := loaded.WriteExplanation(&explanation, "/concurrency/stage_workers"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if test.name == "padded" && (!strings.Contains(explanation.String(), "normalization") ||
|
||||||
|
!strings.Contains(explanation.String(), " extract ") ||
|
||||||
|
strings.Contains(explanation.String(), "derived extraction workers")) {
|
||||||
|
t.Fatalf("explicit worker provenance = %s", explanation.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestDocumentLoadAppliesOperationalDefaultsAndNormalization(t *testing.T) {
|
func TestDocumentLoadAppliesOperationalDefaultsAndNormalization(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user