Relaxed CLI requirements when defaults are specified in the profile or application defaults

This commit is contained in:
2026-05-05 08:41:07 -05:00
parent 281202e313
commit ca4d939fdc
13 changed files with 311 additions and 74 deletions

View File

@@ -7,6 +7,7 @@ import (
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
@@ -33,6 +34,7 @@ func TestHandlerPostRunsSuccess(t *testing.T) {
start := time.Now().UTC()
end := start.Add(2 * time.Second)
r := &fakeRunner{result: &domain.RunResult{
RunID: "11111111-1111-4111-8111-111111111111",
Artifact: domain.Artifact{
Name: "output",
ContentType: "text/plain",
@@ -43,14 +45,24 @@ func TestHandlerPostRunsSuccess(t *testing.T) {
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
ProfileID: "p1",
ProfileVersion: "1.0.0",
ProfileHash: "phash",
ModelName: "m1",
Endpoint: "http://llm/v1",
InputHashes: map[string]string{"transcript": "h1"},
PromptHash: "ph",
Usage: domain.TokenUsage{PromptTokens: 1, CompletionTokens: 2, TotalTokens: 3},
StartTime: start,
EndTime: end,
RawOutput: "hello",
ModelParams: domain.ModelTarget{
Endpoint: "http://llm/v1",
Model: "m1",
Temperature: 0.2,
MaxTokens: 42,
TopP: 0.9,
TimeoutSeconds: 120,
},
InputHashes: map[string]string{"transcript": "h1"},
PromptHash: "ph",
Usage: domain.TokenUsage{PromptTokens: 1, CompletionTokens: 2, TotalTokens: 3},
StartTime: start,
EndTime: end,
Duration: 2 * time.Second,
RawOutput: "hello",
}}
h := NewHandler(r)
@@ -86,10 +98,26 @@ func TestHandlerPostRunsSuccess(t *testing.T) {
t.Fatalf("expected validation.status passed, got %#v", validation["status"])
}
metadata := resp["metadata"].(map[string]any)
if metadata["run_id"] != "11111111-1111-4111-8111-111111111111" {
t.Fatalf("unexpected metadata.run_id: %#v", metadata["run_id"])
}
if metadata["profile_hash"] != "phash" {
t.Fatalf("unexpected metadata.profile_hash: %#v", metadata["profile_hash"])
}
usage := metadata["usage"].(map[string]any)
if usage["total_tokens"] != float64(3) {
t.Fatalf("expected usage.total_tokens=3, got %#v", usage["total_tokens"])
}
if metadata["duration_ms"] != float64(2000) {
t.Fatalf("expected duration_ms=2000, got %#v", metadata["duration_ms"])
}
if metadata["validation_mode"] != "basic" || metadata["validation_status"] != "passed" {
t.Fatalf("unexpected validation metadata: mode=%#v status=%#v", metadata["validation_mode"], metadata["validation_status"])
}
modelParams := metadata["model_params"].(map[string]any)
if modelParams["model"] != "m1" {
t.Fatalf("unexpected model_params.model: %#v", modelParams["model"])
}
if resp["raw_model_output"] != "hello" {
t.Fatalf("expected raw model output hello, got %#v", resp["raw_model_output"])
}
@@ -153,6 +181,20 @@ func TestHandlerUsecaseErrorMapping(t *testing.T) {
if w.Code != tc.status {
t.Fatalf("expected %d, got %d body=%s", tc.status, w.Code, w.Body.String())
}
var resp map[string]any
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatalf("invalid JSON response: %v", err)
}
errBody := resp["error"].(map[string]any)
if _, ok := errBody["code"].(string); !ok {
t.Fatalf("expected error code string, got %#v", errBody["code"])
}
if msg, ok := errBody["message"].(string); !ok || msg == "" {
t.Fatalf("expected non-empty error message, got %#v", errBody["message"])
}
if strings.Contains(w.Body.String(), "read failed") || strings.Contains(w.Body.String(), "render failed") || strings.Contains(w.Body.String(), "llm failed") {
t.Fatalf("expected response to avoid leaking internal cause details, got %s", w.Body.String())
}
})
}
}