208 lines
10 KiB
Go
208 lines
10 KiB
Go
package state
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
|
)
|
|
|
|
func TestPromptPreparationArtifactValidation(t *testing.T) {
|
|
valid := validPreparationArtifact()
|
|
if err := valid.Validate(); err != nil {
|
|
t.Fatalf("valid successful preparation: %v", err)
|
|
}
|
|
failed := valid
|
|
failed.Status = PromptPreparationFailed
|
|
failed.Preparation = nil
|
|
failed.Error = &PromptArtifactError{Category: promptexec.Generation, Message: "provider unavailable"}
|
|
if err := failed.Validate(); err != nil {
|
|
t.Fatalf("valid failed preparation: %v", err)
|
|
}
|
|
|
|
tests := []struct {
|
|
name string
|
|
mutate func(*PromptPreparationArtifact)
|
|
}{
|
|
{"report id", func(a *PromptPreparationArtifact) { a.ReportID = "" }},
|
|
{"run id", func(a *PromptPreparationArtifact) { a.RunID = "" }},
|
|
{"prompt id", func(a *PromptPreparationArtifact) { a.PromptID = "" }},
|
|
{"prompt id for another report", func(a *PromptPreparationArtifact) { a.PromptID = "weather.hourly_generated_text" }},
|
|
{"prompt version", func(a *PromptPreparationArtifact) { a.PromptVersion = "latest" }},
|
|
{"data package", func(a *PromptPreparationArtifact) { a.DataPackagePath = "" }},
|
|
{"start time", func(a *PromptPreparationArtifact) { a.StartedAt = time.Time{} }},
|
|
{"end time", func(a *PromptPreparationArtifact) { a.EndedAt = time.Time{} }},
|
|
{"negative duration", func(a *PromptPreparationArtifact) { a.Duration = -time.Second }},
|
|
{"reversed times", func(a *PromptPreparationArtifact) { a.EndedAt = a.StartedAt.Add(-time.Second) }},
|
|
{"missing provenance", func(a *PromptPreparationArtifact) { a.Preparation = nil }},
|
|
{"success error", func(a *PromptPreparationArtifact) {
|
|
a.Error = &PromptArtifactError{Category: promptexec.Generation, Message: "failed"}
|
|
}},
|
|
{"provenance prompt id", func(a *PromptPreparationArtifact) { a.Preparation.PromptID = "other" }},
|
|
{"provenance prompt version", func(a *PromptPreparationArtifact) { a.Preparation.PromptVersion = "other" }},
|
|
{"provenance data package", func(a *PromptPreparationArtifact) { a.Preparation.DataPackagePath = "/other/data.yaml" }},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
artifact := validPreparationArtifact()
|
|
test.mutate(&artifact)
|
|
if err := artifact.Validate(); err == nil {
|
|
t.Fatalf("Validate() error = nil for %#v", artifact)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestFailedPromptPreparationRejectsContradictoryDetails(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
mutate func(*PromptPreparationArtifact)
|
|
}{
|
|
{"missing error", func(a *PromptPreparationArtifact) { a.Error = nil }},
|
|
{"unknown category", func(a *PromptPreparationArtifact) { a.Error.Category = promptexec.ErrorCategory("other") }},
|
|
{"empty message", func(a *PromptPreparationArtifact) { a.Error.Message = " " }},
|
|
{"oversized message", func(a *PromptPreparationArtifact) { a.Error.Message = strings.Repeat("x", promptArtifactErrorLimit+1) }},
|
|
{"invented provenance", func(a *PromptPreparationArtifact) { a.Preparation = validPreparationArtifact().Preparation }},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
artifact := validPreparationArtifact()
|
|
artifact.Status = PromptPreparationFailed
|
|
artifact.Preparation = nil
|
|
artifact.Error = &PromptArtifactError{Category: promptexec.Generation, Message: "provider unavailable"}
|
|
test.mutate(&artifact)
|
|
if err := artifact.Validate(); err == nil {
|
|
t.Fatalf("Validate() error = nil for %#v", artifact)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestPromptExecutionArtifactValidation(t *testing.T) {
|
|
valid := validExecutionArtifact()
|
|
valid.Provenance.RunID = "provider-run-different-from-weatherreporter"
|
|
valid.Provenance.GeneratedHash = ""
|
|
valid.Provenance.Usage = promptexec.TokenUsage{}
|
|
if err := valid.Validate(); err != nil {
|
|
t.Fatalf("valid completed execution with provider run identity and omitted counters: %v", err)
|
|
}
|
|
rejected := validExecutionArtifact()
|
|
rejected.Status = PromptExecutionValidationRejected
|
|
validation := promptexec.NewValidation(promptexec.ValidationFailed, "json_schema", "daily.generated_text.schema.json", []string{"schema mismatch"})
|
|
rejected.Validation = &validation
|
|
if err := rejected.Validate(); err != nil {
|
|
t.Fatalf("valid validation rejection: %v", err)
|
|
}
|
|
failed := validFailedExecutionArtifact()
|
|
if err := failed.Validate(); err != nil {
|
|
t.Fatalf("valid operational failure: %v", err)
|
|
}
|
|
|
|
tests := []struct {
|
|
name string
|
|
mutate func(*PromptExecutionArtifact)
|
|
}{
|
|
{"report id", func(a *PromptExecutionArtifact) { a.ReportID = "" }},
|
|
{"run id", func(a *PromptExecutionArtifact) { a.RunID = "" }},
|
|
{"prompt id", func(a *PromptExecutionArtifact) { a.PromptID = "" }},
|
|
{"prompt version", func(a *PromptExecutionArtifact) { a.PromptVersion = "latest" }},
|
|
{"start time", func(a *PromptExecutionArtifact) { a.StartedAt = time.Time{} }},
|
|
{"end time", func(a *PromptExecutionArtifact) { a.EndedAt = time.Time{} }},
|
|
{"negative duration", func(a *PromptExecutionArtifact) { a.Duration = -time.Second }},
|
|
{"reversed times", func(a *PromptExecutionArtifact) { a.EndedAt = a.StartedAt.Add(-time.Second) }},
|
|
{"missing provenance", func(a *PromptExecutionArtifact) { a.Provenance = nil }},
|
|
{"missing validation", func(a *PromptExecutionArtifact) { a.Validation = nil }},
|
|
{"wrong validation", func(a *PromptExecutionArtifact) {
|
|
value := promptexec.NewValidation(promptexec.ValidationFailed, "json_schema", "schema.json", nil)
|
|
a.Validation = &value
|
|
}},
|
|
{"operational error", func(a *PromptExecutionArtifact) {
|
|
a.Error = &PromptArtifactError{Category: promptexec.Generation, Message: "failed"}
|
|
}},
|
|
{"provenance prompt id", func(a *PromptExecutionArtifact) { a.Provenance.PromptID = "other" }},
|
|
{"provenance prompt version", func(a *PromptExecutionArtifact) { a.Provenance.PromptVersion = "other" }},
|
|
{"provenance run id", func(a *PromptExecutionArtifact) { a.Provenance.RunID = "" }},
|
|
{"prompt hash", func(a *PromptExecutionArtifact) { a.Provenance.PromptHash = "" }},
|
|
{"rendered hash", func(a *PromptExecutionArtifact) { a.Provenance.RenderedPromptHash = "" }},
|
|
{"profile id", func(a *PromptExecutionArtifact) { a.Provenance.ProfileID = "" }},
|
|
{"backend id", func(a *PromptExecutionArtifact) { a.Provenance.BackendID = "" }},
|
|
{"model name", func(a *PromptExecutionArtifact) { a.Provenance.ModelName = "" }},
|
|
{"data package", func(a *PromptExecutionArtifact) { a.Provenance.DataPackagePath = "" }},
|
|
{"provenance start time", func(a *PromptExecutionArtifact) { a.Provenance.StartedAt = time.Time{} }},
|
|
{"provenance end time", func(a *PromptExecutionArtifact) { a.Provenance.EndedAt = time.Time{} }},
|
|
{"provenance negative duration", func(a *PromptExecutionArtifact) { a.Provenance.Duration = -time.Second }},
|
|
{"provenance reversed times", func(a *PromptExecutionArtifact) { a.Provenance.EndedAt = a.Provenance.StartedAt.Add(-time.Second) }},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
artifact := validExecutionArtifact()
|
|
test.mutate(&artifact)
|
|
if err := artifact.Validate(); err == nil {
|
|
t.Fatalf("Validate() error = nil for %#v", artifact)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestFailedPromptExecutionRejectsContradictoryDetails(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
mutate func(*PromptExecutionArtifact)
|
|
}{
|
|
{"missing error", func(a *PromptExecutionArtifact) { a.Error = nil }},
|
|
{"unknown category", func(a *PromptExecutionArtifact) { a.Error.Category = promptexec.ErrorCategory("other") }},
|
|
{"provenance", func(a *PromptExecutionArtifact) { a.Provenance = validExecutionArtifact().Provenance }},
|
|
{"completed validation", func(a *PromptExecutionArtifact) { a.Validation = validExecutionArtifact().Validation }},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
artifact := validFailedExecutionArtifact()
|
|
test.mutate(&artifact)
|
|
if err := artifact.Validate(); err == nil {
|
|
t.Fatalf("Validate() error = nil for %#v", artifact)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func validPreparationArtifact() PromptPreparationArtifact {
|
|
started := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
|
|
return PromptPreparationArtifact{
|
|
SchemaVersion: PromptPreparationSchemaVersion, Status: PromptPreparationSucceeded,
|
|
ReportID: report.Daily, RunID: "weatherreporter-run", PromptID: "weather.daily_generated_text",
|
|
PromptVersion: "1.0.0", DataPackagePath: "/workspace/data.yaml",
|
|
Preparation: &promptexec.Preparation{
|
|
PromptID: "weather.daily_generated_text", PromptVersion: "1.0.0",
|
|
DataPackagePath: "/workspace/data.yaml",
|
|
},
|
|
StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second,
|
|
}
|
|
}
|
|
|
|
func validExecutionArtifact() PromptExecutionArtifact {
|
|
started := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
|
|
validation := promptexec.NewValidation(promptexec.ValidationPassed, "json_schema", "daily.generated_text.schema.json", nil)
|
|
return PromptExecutionArtifact{
|
|
SchemaVersion: PromptExecutionSchemaVersion, Status: PromptExecutionSucceeded,
|
|
ReportID: report.Daily, RunID: "weatherreporter-run", PromptID: "weather.daily_generated_text", PromptVersion: "1.0.0",
|
|
Provenance: &PromptExecutionProvenance{
|
|
RunID: "provider-run", PromptID: "weather.daily_generated_text", PromptVersion: "1.0.0",
|
|
PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: "profile",
|
|
BackendID: "backend", ModelName: "model", DataPackagePath: "/workspace/data.yaml",
|
|
StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second,
|
|
},
|
|
Validation: &validation, StartedAt: started, EndedAt: started.Add(time.Second), Duration: time.Second,
|
|
}
|
|
}
|
|
|
|
func validFailedExecutionArtifact() PromptExecutionArtifact {
|
|
started := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
|
|
return PromptExecutionArtifact{
|
|
SchemaVersion: PromptExecutionSchemaVersion, Status: PromptExecutionFailed,
|
|
ReportID: report.Daily, RunID: "weatherreporter-run", PromptID: "weather.daily_generated_text", PromptVersion: "1.0.0",
|
|
StartedAt: started, EndedAt: started, Error: &PromptArtifactError{Category: promptexec.Generation, Message: "provider unavailable"},
|
|
}
|
|
}
|