diff --git a/construction_contract_test.go b/construction_contract_test.go index f0895d7..18df25e 100644 --- a/construction_contract_test.go +++ b/construction_contract_test.go @@ -191,19 +191,24 @@ func TestPublicConstructionRejectsSuppliedInvalidMainInput(t *testing.T) { } } -func TestPublicConstructionCanOverlayMaintainedBackend(t *testing.T) { - endpoint := "http://local-openrouter.example/v1" +func TestPublicConstructionUsesProgrammaticBackendAndProfileSources(t *testing.T) { + endpoint, backendID, model := "http://application.example/v1", "application-backend", "synthetic-model" engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ - Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.BackendSource("operator backend", map[string]promptkit.BackendConfig{ - promptkit.BackendOpenRouter: {Endpoint: &endpoint}, - })}}}}, + Underlays: []configkit.Underlay{{Inputs: []configkit.Input{ + {Source: promptkit.BackendSource("operator backend", map[string]promptkit.BackendConfig{ + backendID: {Endpoint: &endpoint}, + })}, + {Source: promptkit.ProfileSource("operator profile", map[string]promptkit.ProfileConfig{ + "application-profile": {Backend: &backendID, Model: &model}, + })}, + }}}, }}) if err != nil { t.Fatalf("construct engine with backend overlay: %v", err) } - inspection, err := engine.InspectProfile(context.Background(), "mistral-small-3") + inspection, err := engine.InspectProfile(context.Background(), "application-profile") if err != nil { - t.Fatalf("inspect catalog profile: %v", err) + t.Fatalf("inspect configured profile: %v", err) } if inspection.EffectiveModelParams.Endpoint != endpoint { t.Fatalf("effective endpoint = %q, want %q", inspection.EffectiveModelParams.Endpoint, endpoint) diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 25815b3..1507bb9 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -532,6 +532,14 @@ Scope: owned fixtures and examples that currently assume automatic provider data Validation: the affected root, engine, and usecase tests plus examples. Ordinary coverage should now express its own backend, profile, and prompt requirements. +Completion: ordinary application tests now provide the backend, profile, and +smoke metadata they exercise instead of relying on bundled provider entries. +The listing and inheritance contracts use synthetic resources; catalog-specific +assembly tests remain isolated for the later integration removal. Maintained +examples already supplied their own definitions and needed no changes. +Validation passed: `GOWORK=off go test ./...`, +`GOWORK=off python3 scripts/check-examples.py`, and `git diff --check`. + ## Stage 10: Remove Automatic Catalogs And Built-In Profiles Scope: engine composition, dependency/notices cleanup, and the retired promise. diff --git a/internal/engine/smoke_test.go b/internal/engine/smoke_test.go index 5674256..51eb35d 100644 --- a/internal/engine/smoke_test.go +++ b/internal/engine/smoke_test.go @@ -7,9 +7,26 @@ import ( "strings" "testing" + "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/promptkit/internal/domain" ) +func engineSmokeBackendInput() configkit.Input { + return configkit.Input{Source: configkit.Mapping("smoke-test backend", map[string]any{ + "backends": map[string]any{"test-backend": map[string]any{ + "endpoint": "https://test-backend.example/v1", + "smoke_tests": map[string]any{ + "default_profile": "test-profile", + "authentication": map[string]any{"kind": "none"}, + }, + }}, + "profiles": map[string]any{"test-profile": map[string]any{ + "backend": "test-backend", + "model": "synthetic-model", + }}, + })} +} + func TestEngineSmokeTestUsesInjectedGenerationWithoutHTTPSupport(t *testing.T) { calls := 0 options := Options{} @@ -20,13 +37,14 @@ func TestEngineSmokeTestUsesInjectedGenerationWithoutHTTPSupport(t *testing.T) { } return &domain.GenerateResponse{Content: "engine response"}, nil })) - assembled, err := New(Config{}, options) + input := engineSmokeBackendInput() + assembled, err := New(Config{Sources: ConfigurationSources{Main: &input}}, options) if err != nil { t.Fatal(err) } report, err := assembled.SmokeTest(context.Background(), domain.SmokeTestRequest{ - Kind: domain.SmokeTestTextInference, BackendID: "rakestrawhome", + Kind: domain.SmokeTestTextInference, BackendID: "test-backend", }) if err != nil { t.Fatal(err) @@ -35,7 +53,7 @@ func TestEngineSmokeTestUsesInjectedGenerationWithoutHTTPSupport(t *testing.T) { report.Facts.GenerationCompleted == nil || !*report.Facts.GenerationCompleted || report.Facts.TextReturned == nil || !*report.Facts.TextReturned || report.Facts.ExpectedTextMatched == nil || *report.Facts.ExpectedTextMatched || - report.Facts.EndpointReachable != nil || report.Target.BackendID != "rakestrawhome" { + report.Facts.EndpointReachable != nil || report.Target.BackendID != "test-backend" { t.Fatalf("injected smoke report calls=%d: %+v", calls, report) } if report.Target.Endpoint != "" || report.Target.EffectiveSettings.Endpoint == "" || len(report.Attempts) != 1 || @@ -44,7 +62,7 @@ func TestEngineSmokeTestUsesInjectedGenerationWithoutHTTPSupport(t *testing.T) { } authentication, err := assembled.SmokeTest(context.Background(), domain.SmokeTestRequest{ - Kind: domain.SmokeTestAuthentication, BackendID: "rakestrawhome", + Kind: domain.SmokeTestAuthentication, BackendID: "test-backend", }) if err != nil { t.Fatal(err) @@ -59,7 +77,8 @@ func TestEngineSmokeTestUsesBuiltInHTTPClientForInferenceEvidence(t *testing.T) var observedURL string var authorization string options := Options{} - assembled, err := New(Config{HTTPClient: &http.Client{Transport: engineRoundTripperFunc(func(request *http.Request) (*http.Response, error) { + input := engineSmokeBackendInput() + assembled, err := New(Config{Sources: ConfigurationSources{Main: &input}, HTTPClient: &http.Client{Transport: engineRoundTripperFunc(func(request *http.Request) (*http.Response, error) { observedURL = request.URL.String() authorization = request.Header.Get("Authorization") return &http.Response{ @@ -73,7 +92,7 @@ func TestEngineSmokeTestUsesBuiltInHTTPClientForInferenceEvidence(t *testing.T) } report, err := assembled.SmokeTest(context.Background(), domain.SmokeTestRequest{ - Kind: domain.SmokeTestTextInference, BackendID: "rakestrawhome", + Kind: domain.SmokeTestTextInference, BackendID: "test-backend", }) if err != nil { t.Fatal(err) diff --git a/listing_contract_test.go b/listing_contract_test.go index 97e445f..89a27d7 100644 --- a/listing_contract_test.go +++ b/listing_contract_test.go @@ -147,7 +147,6 @@ func TestListIdentitiesUseEffectiveLayeredMembership(t *testing.T) { } for _, id := range []string{ "shared/base", "shared/child", "shared/overlay", - "promptkit-openrouter-standard", "promptkit-openrouter-auto", } { if !containsListingProfile(profiles, id) { t.Errorf("profile listing %#v does not include %q", profiles, id) diff --git a/profile_inheritance_contract_test.go b/profile_inheritance_contract_test.go index 085d118..e82472c 100644 --- a/profile_inheritance_contract_test.go +++ b/profile_inheritance_contract_test.go @@ -20,18 +20,19 @@ func aliasPrompt() map[string]any { })}} } -func TestProfileInheritanceBuiltInAliasWorkflow(t *testing.T) { - standardID, reasoning, timeout := "promptkit-openrouter-standard", "high", 120 +func TestProfileInheritanceConfiguredBaseAliasWorkflow(t *testing.T) { + baseID, baseEndpoint, baseModel, reasoning, timeout := "base", "https://profile.example/v1", "synthetic-model", "high", 120 engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("application alias", map[string]promptkit.ProfileConfig{ - "child": {Extends: &standardID, ReasoningEffort: &reasoning, TimeoutSeconds: &timeout}, + "base": {Endpoint: &baseEndpoint, Model: &baseModel}, + "child": {Extends: &baseID, ReasoningEffort: &reasoning, TimeoutSeconds: &timeout}, })}}}}, Main: &configkit.Input{Source: configkit.Mapping("prompt", aliasPrompt())}, }}) if err != nil { t.Fatalf("construct alias engine: %v", err) } - base, err := engine.InspectProfile(context.Background(), standardID) + base, err := engine.InspectProfile(context.Background(), baseID) if err != nil { t.Fatalf("inspect base: %v", err) } @@ -57,9 +58,9 @@ func TestProfileInheritanceBuiltInAliasWorkflow(t *testing.T) { } } -func TestProfileInheritanceYAMLAliasOfBuiltIn(t *testing.T) { +func TestProfileInheritanceYAMLAliasOfConfiguredProfile(t *testing.T) { inputs, err := promptkit.FilesystemInputs("profile alias", fstest.MapFS{ - "config/alias.yaml": &fstest.MapFile{Data: []byte("profiles:\n yaml-alias:\n extends: promptkit-openrouter-standard\n")}, + "config/alias.yaml": &fstest.MapFile{Data: []byte("profiles:\n base:\n model: synthetic-model\n yaml-alias:\n extends: base\n")}, }, "config") if err != nil { t.Fatal(err)