Add run control contract tests
This commit is contained in:
@@ -322,6 +322,15 @@ func TestRunUsesOneInjectedIdentityForDebugOutputAndManifest(t *testing.T) {
|
||||
if manifest.RunID != runID {
|
||||
t.Fatalf("manifest run ID = %q, want %q", manifest.RunID, runID)
|
||||
}
|
||||
wantStartedAt := time.Unix(1, 0).UTC()
|
||||
if manifest.StartedAt == nil || !manifest.StartedAt.Equal(wantStartedAt) {
|
||||
t.Fatalf("manifest started at = %v, want %v", manifest.StartedAt, wantStartedAt)
|
||||
}
|
||||
var invocation debugbundle.Invocation
|
||||
readStateTestSummaryJSON(t, debugPath, "invocation.json", &invocation)
|
||||
if invocation.RunID != runID || !invocation.StartedAt.Equal(wantStartedAt) {
|
||||
t.Fatalf("debug invocation identity = %#v, want run %q at %v", invocation, runID, wantStartedAt)
|
||||
}
|
||||
report := readStateTestRunReport(t, debugPath)
|
||||
if !report.Succeeded || report.RunID != runID || report.PipelineID != "sample" || report.OutputPath != outputPath || report.DebugPath != debugPath || report.OutputCount != 1 || report.RejectedCount != 0 || report.WarningCount != 0 || report.ValidationStatus != "approved" {
|
||||
t.Fatalf("success report = %#v", report)
|
||||
@@ -799,11 +808,15 @@ type stateTestHarness struct {
|
||||
runIDCalls uint64
|
||||
extractErr error
|
||||
chunkWarnings []contracts.Warning
|
||||
moduleProfiles []string
|
||||
sessionIDs []string
|
||||
outputWarnings []contracts.Warning
|
||||
includeWarnings bool
|
||||
}
|
||||
|
||||
func newStateTestHarness() *stateTestHarness { return &stateTestHarness{} }
|
||||
func (h *stateTestHarness) options() Options {
|
||||
registries := pipeline.Registries{Inputs: pipeline.NewInputAdapterRegistry(), Chunkers: pipeline.NewChunkerRegistry(), ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), Extractors: pipeline.NewExtractorRegistry(), Mergers: pipeline.NewMergerRegistry(), Normalizers: pipeline.NewNormalizerRegistry(), Outputs: pipeline.NewOutputEncoderRegistry()}
|
||||
registries := pipeline.Registries{Inputs: pipeline.NewInputAdapterRegistry(), Chunkers: pipeline.NewChunkerRegistry(), ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), Extractors: pipeline.NewExtractorRegistry(), Mergers: pipeline.NewMergerRegistry(), Normalizers: pipeline.NewNormalizerRegistry(), Validators: pipeline.NewValidatorRegistry(), ValidatorChains: pipeline.NewValidatorChainRegistry(), Outputs: pipeline.NewOutputEncoderRegistry()}
|
||||
if err := pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, stateTestCodec{}); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
@@ -816,13 +829,15 @@ func (h *stateTestHarness) options() Options {
|
||||
if err := pipeline.RegisterExtractor(registries.Extractors, pipeline.ModuleSpec{Key: "test/extract", Stage: pipeline.StageExtract, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: stateTestArtifactKind}, func() (contracts.Extractor[stateTestArtifact], error) { return stateTestExtractor{h}, nil }); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if err := pipeline.RegisterMerger(registries.Mergers, pipeline.ModuleSpec{Key: "test/merge", Stage: pipeline.StageMerge, Requires: []string{"artifact"}, Provides: []string{"merged"}, ArtifactKind: stateTestArtifactKind}, func() (contracts.Merger[stateTestArtifact], error) { return stateTestMerger{}, nil }); err != nil {
|
||||
if err := pipeline.RegisterMerger(registries.Mergers, pipeline.ModuleSpec{Key: "test/merge", Stage: pipeline.StageMerge, Requires: []string{"artifact"}, Provides: []string{"merged"}, ArtifactKind: stateTestArtifactKind}, func() (contracts.Merger[stateTestArtifact], error) { return stateTestMerger{harness: h}, nil }); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if err := pipeline.RegisterNormalizer(registries.Normalizers, pipeline.ModuleSpec{Key: "test/normalize", Stage: pipeline.StageNormalize, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: stateTestArtifactKind}, func() (contracts.Normalizer[stateTestArtifact], error) { return stateTestNormalizer{}, nil }); err != nil {
|
||||
if err := pipeline.RegisterNormalizer(registries.Normalizers, pipeline.ModuleSpec{Key: "test/normalize", Stage: pipeline.StageNormalize, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: stateTestArtifactKind}, func() (contracts.Normalizer[stateTestArtifact], error) { return stateTestNormalizer{harness: h}, nil }); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if err := registries.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "test/output", Stage: pipeline.StageOutput, Requires: []string{"normalized"}, Provides: []string{"output"}}, func() (contracts.OutputEncoder, error) { return stateTestOutput{}, nil }); err != nil {
|
||||
if err := registries.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "test/output", Stage: pipeline.StageOutput, Requires: []string{"normalized"}, Provides: []string{"output"}}, func() (contracts.OutputEncoder, error) {
|
||||
return stateTestOutput{harness: h, includeWarnings: h.includeWarnings}, nil
|
||||
}); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return Options{Catalog: catalogFromRegistries(registries), Registries: registries, LookupEnv: emptyLookup, Now: func() time.Time { return time.Unix(1, 0) }, RunIDGenerator: func(startedAt time.Time) (string, error) {
|
||||
@@ -847,6 +862,10 @@ type stateTestChunker struct{ harness *stateTestHarness }
|
||||
func (stateTestChunker) Key() string { return "test/chunk" }
|
||||
func (stateTestChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
func (c stateTestChunker) Plan(_ context.Context, req contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
|
||||
c.harness.mu.Lock()
|
||||
c.harness.moduleProfiles = append(c.harness.moduleProfiles, req.LLMProfile)
|
||||
c.harness.sessionIDs = append(c.harness.sessionIDs, req.SessionID)
|
||||
c.harness.mu.Unlock()
|
||||
c.harness.mu.Lock()
|
||||
c.harness.chunkCalls++
|
||||
c.harness.mu.Unlock()
|
||||
@@ -879,36 +898,56 @@ type stateTestExtractor struct{ harness *stateTestHarness }
|
||||
|
||||
func (stateTestExtractor) Key() string { return "test/extract" }
|
||||
func (stateTestExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
func (e stateTestExtractor) Extract(context.Context, contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[stateTestArtifact], error) {
|
||||
func (e stateTestExtractor) Extract(_ context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[stateTestArtifact], error) {
|
||||
e.harness.mu.Lock()
|
||||
defer e.harness.mu.Unlock()
|
||||
e.harness.extractCalls++
|
||||
e.harness.moduleProfiles = append(e.harness.moduleProfiles, req.LLMProfile)
|
||||
e.harness.sessionIDs = append(e.harness.sessionIDs, req.SessionID)
|
||||
if e.harness.extractErr != nil {
|
||||
return contracts.TypedExtractionResult[stateTestArtifact]{}, e.harness.extractErr
|
||||
}
|
||||
return contracts.TypedExtractionResult[stateTestArtifact]{Value: stateTestArtifact{Value: "ok"}}, nil
|
||||
}
|
||||
|
||||
type stateTestMerger struct{}
|
||||
type stateTestMerger struct{ harness *stateTestHarness }
|
||||
|
||||
func (stateTestMerger) Key() string { return "test/merge" }
|
||||
func (stateTestMerger) Merge(_ context.Context, req contracts.TypedMergeRequest[stateTestArtifact]) (contracts.TypedMergeResult[stateTestArtifact], error) {
|
||||
func (m stateTestMerger) Merge(_ context.Context, req contracts.TypedMergeRequest[stateTestArtifact]) (contracts.TypedMergeResult[stateTestArtifact], error) {
|
||||
m.harness.mu.Lock()
|
||||
m.harness.moduleProfiles = append(m.harness.moduleProfiles, req.LLMProfile)
|
||||
m.harness.sessionIDs = append(m.harness.sessionIDs, req.SessionID)
|
||||
m.harness.mu.Unlock()
|
||||
return contracts.TypedMergeResult[stateTestArtifact]{Value: req.ExtractOutputs[0].Value}, nil
|
||||
}
|
||||
|
||||
type stateTestNormalizer struct{}
|
||||
type stateTestNormalizer struct{ harness *stateTestHarness }
|
||||
|
||||
func (stateTestNormalizer) Key() string { return "test/normalize" }
|
||||
func (stateTestNormalizer) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
func (stateTestNormalizer) Normalize(_ context.Context, req contracts.TypedNormalizeRequest[stateTestArtifact]) (contracts.TypedNormalizeResult[stateTestArtifact], error) {
|
||||
func (n stateTestNormalizer) Normalize(_ context.Context, req contracts.TypedNormalizeRequest[stateTestArtifact]) (contracts.TypedNormalizeResult[stateTestArtifact], error) {
|
||||
n.harness.mu.Lock()
|
||||
n.harness.moduleProfiles = append(n.harness.moduleProfiles, req.LLMProfile)
|
||||
n.harness.sessionIDs = append(n.harness.sessionIDs, req.SessionID)
|
||||
n.harness.mu.Unlock()
|
||||
return contracts.TypedNormalizeResult[stateTestArtifact]{Value: req.MergeOutput.Value}, nil
|
||||
}
|
||||
|
||||
type stateTestOutput struct{}
|
||||
type stateTestOutput struct {
|
||||
harness *stateTestHarness
|
||||
includeWarnings bool
|
||||
}
|
||||
|
||||
func (stateTestOutput) Key() string { return "test/output" }
|
||||
func (stateTestOutput) Encode(context.Context, contracts.OutputRequest) (contracts.OutputResult, error) {
|
||||
return contracts.OutputResult{Files: []contracts.OutputFile{{Name: "result.json", Bytes: []byte("{\"ok\":true}\n")}}}, nil
|
||||
func (o stateTestOutput) Key() string { return "test/output" }
|
||||
func (o stateTestOutput) Encode(_ context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
|
||||
o.harness.mu.Lock()
|
||||
o.harness.outputWarnings = append([]contracts.Warning(nil), req.Warnings...)
|
||||
o.harness.mu.Unlock()
|
||||
data := []byte("{\"ok\":true}\n")
|
||||
if o.includeWarnings && len(req.Warnings) > 0 {
|
||||
data = []byte(fmt.Sprintf("{\"ok\":true,\"warnings\":%q}\n", req.Warnings[0].ReasonCode))
|
||||
}
|
||||
return contracts.OutputResult{Files: []contracts.OutputFile{{Name: "result.json", Bytes: data}}}, nil
|
||||
}
|
||||
|
||||
type failingDebugRecorder struct{}
|
||||
|
||||
Reference in New Issue
Block a user