Demonstrate overlay model routing in prototype

examples/library-adoption/main.go

@@ -3,7 +3,11 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"errors"
+	"flag"
 	"fmt"
+	"io"
+	"net/http"
 	"os"
 
 	ck "gitea.maximumdirect.net/eric/ConfigKit"
@@ -16,31 +20,70 @@ func main() {
 		fmt.Fprintln(os.Stderr, err)
 		os.Exit(1)
 	}
-	loaded, err := loadConfig(ck.File("config.yaml"), "")
-	if err != nil {
-		fmt.Fprintln(os.Stderr, err)
-		os.Exit(1)
-	}
-	runtime, err := newRuntime(loaded.Config, ".", demonstrationClient())
-	if err != nil {
-		fmt.Fprintln(os.Stderr, err)
-		os.Exit(1)
-	}
-	recorder := &Recorder{}
-	result, err := runtime.workflow(context.Background(), recorder.observer)
-	if err != nil {
-		fmt.Fprintln(os.Stderr, "offline workflow failed")
-		os.Exit(1)
-	}
-	attempts, retries, repairs := recorder.totals()
-	output := struct {
-		Result   WorkflowResult `json:"result"`
-		Attempts uint64         `json:"attempts"`
-		Retries  uint64         `json:"transient_retries"`
-		Repairs  uint64         `json:"structural_repairs"`
-	}{result, attempts, retries, repairs}
-	if err := json.NewEncoder(os.Stdout).Encode(output); err != nil {
+	if err := run(os.Args[1:], os.Stdout, demonstrationClient()); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return
+		}
 		fmt.Fprintln(os.Stderr, err)
 		os.Exit(1)
 	}
 }
+
+func run(args []string, output io.Writer, client *http.Client) error {
+	flags := flag.NewFlagSet("library-adoption", flag.ContinueOnError)
+	flags.SetOutput(os.Stderr)
+	overlay := flags.String("overlay", "", "ConfigKit overlay")
+	proposal := flags.String("proposal-profile", "", "proposal profile")
+	validation := flags.String("validation-profile", "", "validation profile")
+	if err := flags.Parse(args); err != nil {
+		return err
+	}
+	if flags.NArg() != 0 {
+		return fmt.Errorf("unexpected positional arguments: %v", flags.Args())
+	}
+	var overrides []ck.Override
+	flags.Visit(func(f *flag.Flag) {
+		switch f.Name {
+		case "proposal-profile":
+			overrides = append(overrides, ck.Override{Path: ck.Keys("llm", "proposal_profile"), Value: *proposal, Flag: "--proposal-profile"})
+		case "validation-profile":
+			overrides = append(overrides, ck.Override{Path: ck.Keys("llm", "validation_profile"), Value: *validation, Flag: "--validation-profile"})
+		}
+	})
+	loaded, err := loadConfig(ck.File("config.yaml"), *overlay, overrides...)
+	if err != nil {
+		return err
+	}
+	runtime, err := newRuntime(loaded.Config, ".", client)
+	if err != nil {
+		return err
+	}
+	recorder := &Recorder{}
+	result, err := runtime.workflow(context.Background(), recorder.observer)
+	if err != nil {
+		return fmt.Errorf("offline workflow failed: %w", err)
+	}
+	attempts, retries, repairs := recorder.totals()
+	type selection struct {
+		Profile string `json:"profile"`
+		Backend string `json:"backend"`
+		Model   string `json:"model"`
+	}
+	selected := func(role string) selection {
+		p := runtime.Profiles[role]
+		return selection{p.ProfileID, p.EffectiveModelParams.BackendID, p.EffectiveModelParams.Model}
+	}
+	summary := struct {
+		Overlay    string         `json:"overlay"`
+		Proposal   selection      `json:"proposal"`
+		Assessment selection      `json:"assessment"`
+		Result     WorkflowResult `json:"result"`
+		Attempts   uint64         `json:"attempts"`
+		Retries    uint64         `json:"transient_retries"`
+		Repairs    uint64         `json:"structural_repairs"`
+	}{*overlay, selected("proposal"), selected("assessment"), result, attempts, retries, repairs}
+	if err := json.NewEncoder(output).Encode(summary); err != nil {
+		return err
+	}
+	return nil
+}