Apply spell normalization follow-up fixes
This commit is contained in:
@@ -161,6 +161,130 @@ func TestProductionSpellValidatorsPrepareFromMaterializedCatalog(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProductionSpellNormalizerRejectsInvalidCatalogReferencesBeforeExecution(t *testing.T) {
|
||||
components := productionTestComponents(t)
|
||||
configPath := repositoryPath("examples", "dnd-spells-production.config.yml")
|
||||
resolve := func(t *testing.T) pipeline.ResolvedPipeline {
|
||||
t.Helper()
|
||||
effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
|
||||
if err != nil {
|
||||
t.Fatalf("resolve production spell configuration: %v", err)
|
||||
}
|
||||
return effective.ResolvedPipeline
|
||||
}
|
||||
materialize := func(resolved pipeline.ResolvedPipeline) (pipeline.ResolvedPipeline, error) {
|
||||
materialized, _, err := pipeline.MaterializeReferences(resolved, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{
|
||||
ConfigPath: configPath,
|
||||
WorkingDir: filepath.Dir(configPath),
|
||||
})
|
||||
return materialized, err
|
||||
}
|
||||
|
||||
t.Run("malformed catalog fails preparation", func(t *testing.T) {
|
||||
catalogPath := filepath.Join(t.TempDir(), "malformed.json")
|
||||
if err := os.WriteFile(catalogPath, []byte(`{"schema_version":`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resolved := resolve(t)
|
||||
setNormalizeSpellCatalogSource(t, &resolved, catalogPath)
|
||||
materialized, err := materialize(resolved)
|
||||
if err != nil {
|
||||
t.Fatalf("MaterializeReferences() error = %v, want malformed JSON to reach preparation", err)
|
||||
}
|
||||
_, err = pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}})
|
||||
for _, fragment := range []string{`pipeline "dnd-session"`, `lane "spells"`, "normalize", `module "dnd/spells"`, "decode spell catalog overlay"} {
|
||||
if err == nil || !strings.Contains(err.Error(), fragment) {
|
||||
t.Fatalf("Prepare() error = %v, want context fragment %q", err, fragment)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("multiple catalog items fail preparation", func(t *testing.T) {
|
||||
materialized, err := materialize(resolve(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
slot := materialized.ArtifactLanes[0].NormalizeReferences.ReferenceSet.Slots["spell_catalog"]
|
||||
slot.Items = append(slot.Items, slot.Items[0])
|
||||
materialized.ArtifactLanes[0].NormalizeReferences.ReferenceSet.Slots["spell_catalog"] = slot
|
||||
_, err = pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}})
|
||||
for _, fragment := range []string{"normalize", `module "dnd/spells"`, "zero or one item"} {
|
||||
if err == nil || !strings.Contains(err.Error(), fragment) {
|
||||
t.Fatalf("Prepare() error = %v, want context fragment %q", err, fragment)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("oversized catalog fails materialization", func(t *testing.T) {
|
||||
catalogPath := filepath.Join(t.TempDir(), "oversized.json")
|
||||
if err := os.WriteFile(catalogPath, []byte(strings.Repeat("x", 1048577)), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
checkpointRoot := filepath.Join(t.TempDir(), "checkpoints")
|
||||
content := string(readRepositoryFile(t, "examples", "dnd-spells-production.config.yml"))
|
||||
content = replaceRequiredOnce(t, content, "./dnd-spells-roster.txt", repositoryPath("examples", "dnd-spells-roster.txt"))
|
||||
content = replaceRequiredOnce(t, content, "./dnd-spells-glossary.txt", repositoryPath("examples", "dnd-spells-glossary.txt"))
|
||||
content = strings.Replace(content, "./dnd-spells-catalog.json", repositoryPath("examples", "dnd-spells-catalog.json"), 1)
|
||||
content = replaceRequiredOnce(t, content, "./dnd-spells-catalog.json", catalogPath)
|
||||
content = replaceRequiredOnce(t, content, " enabled: false\n directory: /var/cache/notarius/checkpoints", " enabled: true\n directory: "+checkpointRoot)
|
||||
configFile := filepath.Join(t.TempDir(), "config.yml")
|
||||
if err := os.WriteFile(configFile, []byte(content), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
llmConstructed := false
|
||||
chunkStoreConstructed := false
|
||||
options := Options{
|
||||
Catalog: catalogFromRegistries(components.registries),
|
||||
Registries: components.registries,
|
||||
LLMClientFactory: func(context.Context, config.Config, string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||
llmConstructed = true
|
||||
return nil, nil, errors.New("LLM client must not be constructed")
|
||||
},
|
||||
ChunkPlanStoreFactory: func(string) (pipeline.ChunkPlanStore, error) {
|
||||
chunkStoreConstructed = true
|
||||
return nil, errors.New("chunk-plan store must not be constructed")
|
||||
},
|
||||
}
|
||||
var stdout, stderr strings.Builder
|
||||
code := RunWithOptions([]string{
|
||||
"run", "dnd-session", "--config", configFile,
|
||||
"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
|
||||
}, &stdout, &stderr, options)
|
||||
errText := stderr.String()
|
||||
for _, fragment := range []string{"normalize", `lane "spells"`, `reference slot "spell_catalog"`, "1048577 bytes", "limit 1048576"} {
|
||||
if code == 0 || !strings.Contains(errText, fragment) {
|
||||
t.Fatalf("RunWithOptions() code = %d stderr = %q, want context fragment %q", code, errText, fragment)
|
||||
}
|
||||
}
|
||||
if llmConstructed || chunkStoreConstructed {
|
||||
t.Fatalf("runtime construction = LLM %t, chunk store %t; want materialization failure first", llmConstructed, chunkStoreConstructed)
|
||||
}
|
||||
if _, err := os.Stat(checkpointRoot); !errors.Is(err, fs.ErrNotExist) {
|
||||
t.Fatalf("checkpoint root stat error = %v, want no checkpoint allocation", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func setNormalizeSpellCatalogSource(t *testing.T, resolved *pipeline.ResolvedPipeline, sourcePath string) {
|
||||
t.Helper()
|
||||
if resolved == nil || len(resolved.ArtifactLanes) != 1 {
|
||||
t.Fatalf("resolved pipeline = %#v, want one artifact lane", resolved)
|
||||
}
|
||||
bindings := resolved.ArtifactLanes[0].NormalizeReferences.Bindings
|
||||
matches := 0
|
||||
for index := range bindings {
|
||||
if bindings[index].SlotName == "spell_catalog" {
|
||||
bindings[index].Source = sourcePath
|
||||
matches++
|
||||
}
|
||||
}
|
||||
if matches != 1 {
|
||||
t.Fatalf("normalize reference bindings = %#v, want exactly one spell_catalog binding", bindings)
|
||||
}
|
||||
resolved.ArtifactLanes[0].NormalizeReferences.Bindings = bindings
|
||||
}
|
||||
|
||||
func TestProductionLLMClientFactoriesBuildOfflineRuntime(t *testing.T) {
|
||||
components := productionTestComponents(t)
|
||||
factories := []struct {
|
||||
|
||||
Reference in New Issue
Block a user