78 lines
2.8 KiB
Go
78 lines
2.8 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"embed"
|
|
"fmt"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
|
|
pk "gitea.maximumdirect.net/eric/promptkit"
|
|
)
|
|
|
|
//go:embed prompts schemas
|
|
var assets embed.FS
|
|
|
|
type Runtime struct {
|
|
Config Config
|
|
Engine *pk.Engine
|
|
Profiles map[string]pk.ProfileInspection
|
|
}
|
|
|
|
func newRuntime(c Config, configDir string, client *http.Client) (*Runtime, error) {
|
|
profileDir := c.ProfilesDir
|
|
if !filepath.IsAbs(profileDir) {
|
|
profileDir = filepath.Join(configDir, profileDir)
|
|
}
|
|
opts := []pk.Option{pk.WithPromptFS(assets, "prompts"), pk.WithSchemaFS(assets, "schemas"), pk.WithProfileFS(os.DirFS(profileDir), ".")}
|
|
for id, b := range c.Backends {
|
|
opts = append(opts, pk.WithBackend(pk.Backend{ID: id, Endpoint: b.Endpoint, ConcurrencyLimit: b.Concurrency, QueueCapacity: &b.QueueCapacity}))
|
|
}
|
|
engine, err := pk.NewEngine(pk.Config{HTTPClient: client, TransientRetries: &c.LLM.TransientRetries}, opts...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
r := &Runtime{Config: c, Engine: engine, Profiles: make(map[string]pk.ProfileInspection)}
|
|
for role, id := range map[string]string{"proposal": c.LLM.ProposalProfile, "assessment": c.LLM.ValidationProfile} {
|
|
profile, err := engine.InspectProfile(context.Background(), id)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
target := profile.EffectiveModelParams
|
|
backend, ok := c.Backends[target.BackendID]
|
|
if !ok {
|
|
return nil, fmt.Errorf("%s profile must select an application backend", role)
|
|
}
|
|
if strings.TrimRight(strings.TrimSpace(target.Endpoint), "/") != strings.TrimRight(strings.TrimSpace(backend.Endpoint), "/") || target.APIKeyEnv != "" {
|
|
return nil, fmt.Errorf("%s profile must use its backend endpoint and ConfigKit credentials", role)
|
|
}
|
|
if profile.APIKeyRequired && backend.Credential == nil {
|
|
return nil, fmt.Errorf("%s profile requires a backend credential", role)
|
|
}
|
|
r.Profiles[role] = *profile
|
|
}
|
|
return r, nil
|
|
}
|
|
|
|
func (r *Runtime) call(ctx context.Context, role string, vars map[string]string, observer pk.ExecutionObserver) (*pk.RunResult, error) {
|
|
profile, ok := r.Profiles[role]
|
|
if !ok {
|
|
return nil, fmt.Errorf("unknown role")
|
|
}
|
|
id, schema := "modules.grammar.proposal", "correction_set.v1.json"
|
|
if role == "assessment" {
|
|
id, schema = "validators.grammar_review", "validator_decision_set.v1.json"
|
|
}
|
|
key := ""
|
|
if secret := r.Config.Backends[profile.EffectiveModelParams.BackendID].Credential; secret != nil {
|
|
key = secret.Value()
|
|
}
|
|
ctx, cancel := context.WithTimeout(ctx, r.Config.LLM.OperationTimeout)
|
|
defer cancel()
|
|
return r.Engine.Run(ctx, pk.RunRequest{PromptID: id, PromptVersion: "v1", ProfileID: profile.ProfileID, APIKey: key, Vars: vars, Observer: observer,
|
|
Validation: &pk.OutputContract{Format: pk.FormatJSON, ValidationMode: pk.ValidationJSONSchema, SchemaPath: schema, RepairAttempts: r.Config.LLM.RepairAttempts},
|
|
})
|
|
}
|