eric/promptkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Decouple application tests from bundled profiles

Browse Source
This commit is contained in:
Eric Rakestraw
2026-10-17 06:05:27 +00:00
parent 473fc1f6d7
commit ee40a1bdd2
5 changed files with 52 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
internal/engine/smoke_test.go
View File

@@ -7,9 +7,26 @@ import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
func engineSmokeBackendInput() configkit.Input {
return configkit.Input{Source: configkit.Mapping("smoke-test backend", map[string]any{
"backends": map[string]any{"test-backend": map[string]any{
"endpoint": "https://test-backend.example/v1",
"smoke_tests": map[string]any{
"default_profile": "test-profile",
"authentication": map[string]any{"kind": "none"},
},
}},
"profiles": map[string]any{"test-profile": map[string]any{
"backend": "test-backend",
"model": "synthetic-model",
}},
})}
}
func TestEngineSmokeTestUsesInjectedGenerationWithoutHTTPSupport(t *testing.T) {
calls := 0
options := Options{}
@@ -20,13 +37,14 @@ func TestEngineSmokeTestUsesInjectedGenerationWithoutHTTPSupport(t *testing.T) {
}
return &domain.GenerateResponse{Content: "engine response"}, nil
}))
assembled, err := New(Config{}, options)
input := engineSmokeBackendInput()
assembled, err := New(Config{Sources: ConfigurationSources{Main: &input}}, options)
if err != nil {
t.Fatal(err)
}
report, err := assembled.SmokeTest(context.Background(), domain.SmokeTestRequest{
Kind: domain.SmokeTestTextInference, BackendID: "rakestrawhome",
Kind: domain.SmokeTestTextInference, BackendID: "test-backend",
})
if err != nil {
t.Fatal(err)
@@ -35,7 +53,7 @@ func TestEngineSmokeTestUsesInjectedGenerationWithoutHTTPSupport(t *testing.T) {
report.Facts.GenerationCompleted == nil || !*report.Facts.GenerationCompleted ||
report.Facts.TextReturned == nil || !*report.Facts.TextReturned ||
report.Facts.ExpectedTextMatched == nil || *report.Facts.ExpectedTextMatched ||
report.Facts.EndpointReachable != nil || report.Target.BackendID != "rakestrawhome" {
report.Facts.EndpointReachable != nil || report.Target.BackendID != "test-backend" {
t.Fatalf("injected smoke report calls=%d: %+v", calls, report)
}
if report.Target.Endpoint != "" || report.Target.EffectiveSettings.Endpoint == "" || len(report.Attempts) != 1 ||
@@ -44,7 +62,7 @@ func TestEngineSmokeTestUsesInjectedGenerationWithoutHTTPSupport(t *testing.T) {
}
authentication, err := assembled.SmokeTest(context.Background(), domain.SmokeTestRequest{
Kind: domain.SmokeTestAuthentication, BackendID: "rakestrawhome",
Kind: domain.SmokeTestAuthentication, BackendID: "test-backend",
})
if err != nil {
t.Fatal(err)
@@ -59,7 +77,8 @@ func TestEngineSmokeTestUsesBuiltInHTTPClientForInferenceEvidence(t *testing.T)
var observedURL string
var authorization string
options := Options{}
assembled, err := New(Config{HTTPClient: &http.Client{Transport: engineRoundTripperFunc(func(request *http.Request) (*http.Response, error) {
input := engineSmokeBackendInput()
assembled, err := New(Config{Sources: ConfigurationSources{Main: &input}, HTTPClient: &http.Client{Transport: engineRoundTripperFunc(func(request *http.Request) (*http.Response, error) {
observedURL = request.URL.String()
authorization = request.Header.Get("Authorization")
return &http.Response{
@@ -73,7 +92,7 @@ func TestEngineSmokeTestUsesBuiltInHTTPClientForInferenceEvidence(t *testing.T)
}
report, err := assembled.SmokeTest(context.Background(), domain.SmokeTestRequest{
Kind: domain.SmokeTestTextInference, BackendID: "rakestrawhome",
Kind: domain.SmokeTestTextInference, BackendID: "test-backend",
})
if err != nil {
t.Fatal(err)