Add D&D extraction fallback profile
This commit is contained in:
@@ -13,7 +13,7 @@ owns prompt, profile, and schema file contracts.
|
||||
Notarius relies on the root `promptkit` package to:
|
||||
|
||||
- construct an `Engine` with filesystem-backed prompt, schema, and optional
|
||||
profile sources;
|
||||
operator and application-fallback profile sources;
|
||||
- prepare one frozen execution from a `RunRequest` with named inline artifacts,
|
||||
variables, a direct session ID, prompt identity, and profile selection, then
|
||||
record credential-redacted details and run that exact execution;
|
||||
@@ -69,6 +69,8 @@ error type, and leaves retries to the calling pipeline stage.
|
||||
module assets, maps its transport-neutral completion contract, prepares and
|
||||
executes requests, validates output, records provenance, captures debug
|
||||
material, redacts errors, and preserves timeout ownership.
|
||||
[D&D Module Internals](../internal/dnd.md) owns the embedded
|
||||
`dnd-extraction` fallback profile and the maintained D&D prompt defaults.
|
||||
[Configuration](../config.md#promptkit-profiles) defines how a Notarius
|
||||
configuration selects one PromptKit profile source and optionally registers
|
||||
the conventional local backend.
|
||||
|
||||
@@ -22,10 +22,11 @@ does not repeat their JSON shapes or schemas.
|
||||
## Family Composition
|
||||
|
||||
The D&D registrar registers the family’s artifact codecs, extractors, typed
|
||||
append-order mergers, normalizers, validators, prompt assets, and default
|
||||
validator chains. Each extractor and normalizer has a stable module spec,
|
||||
strict option decoding, and a typed builder. Configuration remains the
|
||||
canonical owner of the exact keys and validator order.
|
||||
append-order mergers, normalizers, validators, prompt assets, fallback LLM
|
||||
profile asset, and default validator chains. Each extractor and normalizer has
|
||||
a stable module spec, strict option decoding, and a typed builder.
|
||||
Configuration remains the canonical owner of the exact keys and validator
|
||||
order.
|
||||
|
||||
Private structured-LLM response schemas are deliberately minimal. They reject
|
||||
invalid JSON structure, missing required fields, incompatible types, and
|
||||
@@ -41,6 +42,13 @@ reference, and transcript assets instead of copying their text into individual
|
||||
modules. A manifest’s declared sequence, including cache-control placement, is
|
||||
part of the prompt behavior.
|
||||
|
||||
Every maintained D&D LLM prompt selects `dnd-extraction` as its default
|
||||
profile. The D&D registrar embeds that fallback profile with the maintained
|
||||
OpenRouter model, timeout, and service-tier policy. An operator may provide a
|
||||
complete profile with the same ID through the configured PromptKit source; that
|
||||
definition replaces the fallback rather than merging with it. The fallback
|
||||
leaves reasoning and optional sampling controls unspecified.
|
||||
|
||||
All extraction prompts share this four-message rendered prefix: the system
|
||||
message without cache control, the identity message without cache control, the
|
||||
campaign-reference message with ephemeral cache control, and the chunk
|
||||
|
||||
@@ -66,9 +66,11 @@ profile-source construction to apply the configured profile directory or file,
|
||||
the optional registered fallback profile assets, and the optional conventional
|
||||
`local` backend. Preflight therefore resolves the same profile sources and
|
||||
backend membership as runtime without performing generation. Fallback assets
|
||||
are mounted only when at least one source is registered. PromptKit owns source
|
||||
precedence and profile parsing: an operator-provided matching profile takes
|
||||
precedence over a fallback profile without Notarius merging either document.
|
||||
are mounted only when at least one source is registered. The production D&D
|
||||
registrar contributes its `dnd-extraction` fallback, and the maintained D&D
|
||||
prompts select that logical ID by default. PromptKit owns source precedence and
|
||||
profile parsing: an operator-provided matching profile takes precedence over a
|
||||
fallback profile without Notarius merging either document.
|
||||
When the registration is absent, a profile selecting `backend: local` fails
|
||||
inspection instead of falling back to a built-in or endpoint-only target.
|
||||
|
||||
|
||||
@@ -201,6 +201,76 @@ func TestDefaultCLICompositionValidatesRepresentativeConfiguration(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProductionAssetsResolveDNDExtractionProfile(t *testing.T) {
|
||||
components := productionTestComponents(t)
|
||||
newEngine := func(profileFile string) (*promptkit.Engine, error) {
|
||||
t.Helper()
|
||||
options, err := components.assets.PromptKitOptions()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if profileFile != "" {
|
||||
options = append(options, promptkit.WithProfileFile(profileFile))
|
||||
}
|
||||
return promptkit.NewEngine(promptkit.Config{}, options...)
|
||||
}
|
||||
|
||||
t.Run("fallback", func(t *testing.T) {
|
||||
engine, err := newEngine("")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inspection, err := engine.InspectProfile(context.Background(), "dnd-extraction")
|
||||
if err != nil {
|
||||
t.Fatalf("InspectProfile() error = %v, want fallback profile", err)
|
||||
}
|
||||
params := inspection.EffectiveModelParams
|
||||
if params.BackendID != "openrouter" || params.Model != "openai/gpt-5.6-luna" || params.TimeoutSeconds != 240 || params.ServiceTier != "flex" {
|
||||
t.Fatalf("fallback profile parameters = %#v", params)
|
||||
}
|
||||
if params.ReasoningEffort != "" || params.Temperature != 0 || params.MaxTokens != 0 || params.TopP != 0 {
|
||||
t.Fatalf("fallback profile selected optional provider controls: %#v", params)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("valid operator profile wins", func(t *testing.T) {
|
||||
profilePath := filepath.Join(t.TempDir(), "profiles.yaml")
|
||||
if err := os.WriteFile(profilePath, []byte(`id: dnd-extraction
|
||||
endpoint: http://operator.example.test/v1
|
||||
model: operator-model
|
||||
timeout_seconds: 75
|
||||
`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine, err := newEngine(profilePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inspection, err := engine.InspectProfile(context.Background(), "dnd-extraction")
|
||||
if err != nil {
|
||||
t.Fatalf("InspectProfile() error = %v, want operator profile", err)
|
||||
}
|
||||
params := inspection.EffectiveModelParams
|
||||
if params.BackendID != "" || params.Endpoint != "http://operator.example.test/v1" || params.Model != "operator-model" || params.TimeoutSeconds != 75 || params.ServiceTier != "" {
|
||||
t.Fatalf("operator profile parameters = %#v, want complete replacement", params)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("invalid operator profile does not fall through", func(t *testing.T) {
|
||||
profilePath := filepath.Join(t.TempDir(), "profiles.yaml")
|
||||
if err := os.WriteFile(profilePath, []byte("id: dnd-extraction\nendpoint: http://operator.example.test/v1\nmodel: operator-model\nunknown: value\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine, err := newEngine(profilePath)
|
||||
if err == nil {
|
||||
_, err = engine.InspectProfile(context.Background(), "dnd-extraction")
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatal("operator profile error = nil, want failure instead of fallback")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestProductionPromptAssetsPrepareWithoutProviderCredentials(t *testing.T) {
|
||||
components := productionTestComponents(t)
|
||||
cfg := config.Default()
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
id: dnd.scenes
|
||||
version: "v1"
|
||||
default_profile: gemini-2-flash
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
id: dnd.combat_turns
|
||||
version: "v1"
|
||||
default_profile: gemini-2-flash
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
id: dnd.item_events
|
||||
version: "v1"
|
||||
default_profile: gemini-2-flash
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
id: dnd.npc_interactions
|
||||
version: "v1"
|
||||
default_profile: gemini-2-flash
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
id: dnd.npcs
|
||||
version: "v1"
|
||||
default_profile: gemini-2-flash
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
id: dnd.scene_descriptions
|
||||
version: "v1"
|
||||
default_profile: gemini-2-flash
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
id: dnd.spells
|
||||
version: "v1"
|
||||
default_profile: gemini-2-flash
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
id: dnd.npcs.normalize
|
||||
version: "v1"
|
||||
default_profile: gemini-2-flash
|
||||
default_profile: dnd-extraction
|
||||
inputs:
|
||||
- name: candidates
|
||||
required: true
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
id: dnd-extraction
|
||||
backend: openrouter
|
||||
model: openai/gpt-5.6-luna
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
14
internal/modules/dnd/register/profiles.go
Normal file
14
internal/modules/dnd/register/profiles.go
Normal file
@@ -0,0 +1,14 @@
|
||||
package register
|
||||
|
||||
import (
|
||||
"embed"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
)
|
||||
|
||||
//go:embed assets/profiles/*.yaml
|
||||
var embeddedProfileAssets embed.FS
|
||||
|
||||
func registerFallbackProfiles(assets *llm.AssetRegistry) error {
|
||||
return assets.RegisterFallbackProfileFS(embeddedProfileAssets, "assets/profiles")
|
||||
}
|
||||
@@ -30,6 +30,9 @@ func Register(registries pipeline.Registries, assets *llm.AssetRegistry) error {
|
||||
if err := registerPromptAssets(assets); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := registerFallbackProfiles(assets); err != nil {
|
||||
return err
|
||||
}
|
||||
return registerDefaultChains(registries.ValidatorChains)
|
||||
}
|
||||
|
||||
|
||||
@@ -39,6 +39,31 @@ func TestRegisterAddsDNDFamily(t *testing.T) {
|
||||
if _, err := fs.ReadFile(promptFS, "dnd.npcs.normalize/dnd.npcs.normalize.yaml"); err != nil {
|
||||
t.Fatalf("normalization prompt asset = %v, want registered private prompt", err)
|
||||
}
|
||||
for _, name := range []string{
|
||||
"dnd.scenes/dnd.scenes.yaml",
|
||||
"dnd.spells/dnd.spells.yaml",
|
||||
"dnd.npcs/dnd.npcs.yaml",
|
||||
"dnd.combat_turns/dnd.combat_turns.yaml",
|
||||
"dnd.item_events/dnd.item_events.yaml",
|
||||
"dnd.npc_interactions/dnd.npc_interactions.yaml",
|
||||
"dnd.scene_descriptions/dnd.scene_descriptions.yaml",
|
||||
"dnd.npcs.normalize/dnd.npcs.normalize.yaml",
|
||||
} {
|
||||
content, err := fs.ReadFile(promptFS, name)
|
||||
if err != nil {
|
||||
t.Fatalf("read prompt asset %q: %v", name, err)
|
||||
}
|
||||
if !strings.Contains(string(content), "default_profile: dnd-extraction") {
|
||||
t.Fatalf("prompt asset %q does not select dnd-extraction", name)
|
||||
}
|
||||
}
|
||||
fallbackFS, err := assets.FallbackProfileFS()
|
||||
if err != nil {
|
||||
t.Fatalf("FallbackProfileFS() error = %v", err)
|
||||
}
|
||||
if _, err := fs.ReadFile(fallbackFS, "dnd-extraction.yaml"); err != nil {
|
||||
t.Fatalf("fallback profile asset = %v, want registered D&D profile", err)
|
||||
}
|
||||
schemaFS, err := assets.SchemaFS()
|
||||
if err != nil {
|
||||
t.Fatalf("SchemaFS() error = %v", err)
|
||||
|
||||
Reference in New Issue
Block a user