Compose production modules through family registrars

internal/cli/catalog.go

@@ -9,23 +9,17 @@ import (
 	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
 	"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
 	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
-	"gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/dnd/scenes"
-	"gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/generic"
-	"gitea.maximumdirect.net/eric/notarius/internal/modules/extract/dnd/spells"
-	"gitea.maximumdirect.net/eric/notarius/internal/modules/input/seriatim"
-	"gitea.maximumdirect.net/eric/notarius/internal/modules/merge/appendorder"
-	"gitea.maximumdirect.net/eric/notarius/internal/modules/normalize/noop"
-	jsonoutput "gitea.maximumdirect.net/eric/notarius/internal/modules/output/json"
-	spellshape "gitea.maximumdirect.net/eric/notarius/internal/validators/extract/dnd/spells/shape"
-	spellsourcerefs "gitea.maximumdirect.net/eric/notarius/internal/validators/extract/dnd/spells/source_refs"
-	spellrelatedness "gitea.maximumdirect.net/eric/notarius/internal/validators/extract/dnd/spells/source_relatedness"
-	alwaysaccept "gitea.maximumdirect.net/eric/notarius/internal/validators/generic/always_accept"
-	alwaysreject "gitea.maximumdirect.net/eric/notarius/internal/validators/generic/always_reject"
-	validjson "gitea.maximumdirect.net/eric/notarius/internal/validators/generic/valid_json"
-	validjsonschema "gitea.maximumdirect.net/eric/notarius/internal/validators/generic/valid_json_schema"
+	dndregister "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/register"
+	genericregister "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/register"
+	seriatimregister "gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/register"
 )
 
-func productionRegistries() (pipeline.Registries, error) {
+type productionComponents struct {
+	registries pipeline.Registries
+	assets     *llm.AssetRegistry
+}
+
+func newProductionComponents() (productionComponents, error) {
 	registries := pipeline.Registries{
 		Inputs:          pipeline.NewInputAdapterRegistry(),
 		Chunkers:        pipeline.NewChunkerRegistry(),
@@ -36,72 +30,26 @@ func productionRegistries() (pipeline.Registries, error) {
 		ValidatorChains: pipeline.NewValidatorChainRegistry(),
 		Outputs:         pipeline.NewOutputEncoderRegistry(),
 	}
-	if err := seriatim.Register(registries.Inputs); err != nil {
-		return pipeline.Registries{}, fmt.Errorf("register seriatim input: %w", err)
-	}
-	if err := generic.Register(registries.Chunkers); err != nil {
-		return pipeline.Registries{}, fmt.Errorf("register generic chunker: %w", err)
-	}
-	if err := scenes.Register(registries.Chunkers); err != nil {
-		return pipeline.Registries{}, fmt.Errorf("register dnd scenes chunker: %w", err)
-	}
-	if err := spells.Register(registries.Extractors); err != nil {
-		return pipeline.Registries{}, fmt.Errorf("register dnd spells extractor: %w", err)
-	}
-	if err := appendorder.Register(registries.Mergers); err != nil {
-		return pipeline.Registries{}, fmt.Errorf("register appendorder merger: %w", err)
-	}
-	if err := noop.Register(registries.Normalizers); err != nil {
-		return pipeline.Registries{}, fmt.Errorf("register noop normalizer: %w", err)
-	}
-	if err := registerProductionValidators(registries.Validators); err != nil {
-		return pipeline.Registries{}, err
-	}
-	if err := registerProductionValidatorChains(registries.ValidatorChains); err != nil {
-		return pipeline.Registries{}, err
-	}
-	if err := jsonoutput.Register(registries.Outputs); err != nil {
-		return pipeline.Registries{}, fmt.Errorf("register json output encoder: %w", err)
-	}
-	return registries, nil
-}
-
-func registerProductionValidators(registry *pipeline.ValidatorRegistry) error {
-	registrations := []struct {
+	assets := llm.NewAssetRegistry()
+	registrars := []struct {
 		name     string
-		register func(*pipeline.ValidatorRegistry) error
+		register func(pipeline.Registries, *llm.AssetRegistry) error
 	}{
-		{name: "generic always accept validator", register: alwaysaccept.Register},
-		{name: "generic always reject validator", register: alwaysreject.Register},
-		{name: "generic valid json validator", register: validjson.Register},
-		{name: "generic valid json schema validator", register: validjsonschema.Register},
-		{name: "dnd spell shape validator", register: spellshape.Register},
-		{name: "dnd spell source references validator", register: spellsourcerefs.Register},
-		{name: "dnd spell source relatedness validator", register: spellrelatedness.Register},
+		{name: "generic", register: genericregister.Register},
+		{name: "seriatim", register: seriatimregister.Register},
+		{name: "dnd", register: dndregister.Register},
 	}
-	for _, registration := range registrations {
-		if err := registration.register(registry); err != nil {
-			return fmt.Errorf("register %s: %w", registration.name, err)
+	for _, registrar := range registrars {
+		if err := registrar.register(registries, assets); err != nil {
+			return productionComponents{}, fmt.Errorf("register %s module family: %w", registrar.name, err)
 		}
 	}
-	return nil
+	return productionComponents{registries: registries, assets: assets}, nil
 }
 
-func registerProductionValidatorChains(registry *pipeline.ValidatorChainRegistry) error {
-	if err := registry.Register(pipeline.ValidatorChainMapping{
-		Stage:  pipeline.StageExtract,
-		Module: spells.Key,
-		Validators: []pipeline.ModuleBinding{
-			pipeline.Binding(validjson.Key),
-			pipeline.Binding(validjsonschema.Key),
-			pipeline.Binding(spellshape.Key),
-			pipeline.Binding(spellsourcerefs.Key),
-			pipeline.Binding(spellrelatedness.Key),
-		},
-	}); err != nil {
-		return fmt.Errorf("register dnd spells validator chain: %w", err)
-	}
-	return nil
+func productionRegistries() (pipeline.Registries, error) {
+	components, err := newProductionComponents()
+	return components.registries, err
 }
 
 func productionCatalog() (pipeline.ModuleCatalog, error) {
@@ -113,14 +61,8 @@ func productionCatalog() (pipeline.ModuleCatalog, error) {
 }
 
 func productionPromptAssets() (*llm.AssetRegistry, error) {
-	registry := llm.NewAssetRegistry()
-	if err := scenes.RegisterPromptAssets(registry); err != nil {
-		return nil, fmt.Errorf("register dnd scenes prompt assets: %w", err)
-	}
-	if err := spells.RegisterPromptAssets(registry); err != nil {
-		return nil, fmt.Errorf("register dnd spells prompt assets: %w", err)
-	}
-	return registry, nil
+	components, err := newProductionComponents()
+	return components.assets, err
 }
 
 func effectiveCatalog(opts Options) (pipeline.ModuleCatalog, error) {
@@ -199,6 +141,22 @@ func productionLLMClientFactory(ctx context.Context, cfg config.Config, profileI
 	if err != nil {
 		return nil, nil, err
 	}
+	return buildProductionLLMClient(ctx, cfg, profileID, assets)
+}
+
+func productionLLMClientFactoryWithAssets(assets *llm.AssetRegistry) LLMClientFactory {
+	return func(ctx context.Context, cfg config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
+		return buildProductionLLMClient(ctx, cfg, profileID, assets)
+	}
+}
+
+func buildProductionLLMClient(ctx context.Context, cfg config.Config, profileID string, assets *llm.AssetRegistry) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
+	if err := ctx.Err(); err != nil {
+		return nil, nil, err
+	}
+	if assets == nil {
+		return nil, nil, fmt.Errorf("production asset registry must not be nil")
+	}
 	recorder := llm.NewLLMProfileRecorder()
 	client, err := llm.NewScriptoriumClient(llm.ScriptoriumClientConfig{
 		ProfileDir:  cfg.Scriptorium.ProfileDir,

internal/cli/compatibility_test.go

@@ -28,6 +28,24 @@ import (
 )
 
 func TestProductionCompatibilitySnapshot(t *testing.T) {
+	normalizedOptions, err := normalizeOptions(Options{})
+	if err != nil {
+		t.Fatalf("normalizeOptions() error = %v, want nil", err)
+	}
+	if normalizedOptions.Catalog.Inputs != normalizedOptions.Registries.Inputs ||
+		normalizedOptions.Catalog.Chunkers != normalizedOptions.Registries.Chunkers ||
+		normalizedOptions.Catalog.Extractors != normalizedOptions.Registries.Extractors ||
+		normalizedOptions.Catalog.Mergers != normalizedOptions.Registries.Mergers ||
+		normalizedOptions.Catalog.Normalizers != normalizedOptions.Registries.Normalizers ||
+		normalizedOptions.Catalog.Validators != normalizedOptions.Registries.Validators ||
+		normalizedOptions.Catalog.ValidatorChains != normalizedOptions.Registries.ValidatorChains ||
+		normalizedOptions.Catalog.Outputs != normalizedOptions.Registries.Outputs {
+		t.Fatal("production catalog and execution registries do not share one composition")
+	}
+	if normalizedOptions.LLMClientFactory == nil {
+		t.Fatal("production LLM client factory is nil")
+	}
+
 	registries, err := productionRegistries()
 	if err != nil {
 		t.Fatalf("productionRegistries() error = %v, want nil", err)

internal/cli/run.go

@@ -51,7 +51,12 @@ func Run(args []string, stdout, stderr io.Writer) int {
 }
 
 func RunWithOptions(args []string, stdout, stderr io.Writer, opts Options) int {
-	opts = normalizeOptions(opts)
+	var err error
+	opts, err = normalizeOptions(opts)
+	if err != nil {
+		fmt.Fprintf(stderr, "notarius: %v\n", err)
+		return 1
+	}
 	if len(args) == 0 {
 		writeUsage(stdout)
 		return 0
@@ -78,17 +83,28 @@ func writeUsage(w io.Writer) {
 	fmt.Fprint(w, usage)
 }
 
-func normalizeOptions(opts Options) Options {
+func normalizeOptions(opts Options) (Options, error) {
 	if opts.LookupEnv == nil {
 		opts.LookupEnv = os.LookupEnv
 	}
 	if opts.Now == nil {
 		opts.Now = time.Now
 	}
+	if isEmptyCatalog(opts.Catalog) && isEmptyRegistries(opts.Registries) {
+		components, err := newProductionComponents()
+		if err != nil {
+			return Options{}, err
+		}
+		opts.Registries = components.registries
+		opts.Catalog = catalogFromRegistries(components.registries)
+		if opts.LLMClientFactory == nil {
+			opts.LLMClientFactory = productionLLMClientFactoryWithAssets(components.assets)
+		}
+	}
 	if opts.LLMClientFactory == nil {
 		opts.LLMClientFactory = productionLLMClientFactory
 	}
-	return opts
+	return opts, nil
 }
 
 func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) int {