Remove legacy raw pipeline contracts
This commit is contained in:
@@ -25,14 +25,14 @@ type CheckpointRecorder interface {
|
||||
ChunkRejected(moduleKey string, sourceDigest string, rejected contracts.RejectedOutput) error
|
||||
ChunkFailed(moduleKey string, sourceDigest string, err error) error
|
||||
ExtractRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
|
||||
ExtractSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, outputs []contracts.ExtractOutput, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error
|
||||
ExtractSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error
|
||||
ExtractFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
|
||||
MergeRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
|
||||
MergeSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, output contracts.MergeOutput, warnings []contracts.Warning) error
|
||||
MergeSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, output CheckpointArtifact, warnings []contracts.Warning) error
|
||||
MergeRejected(laneID string, moduleKey string, dependencies []CheckpointFingerprint, rejected contracts.RejectedOutput) error
|
||||
MergeFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
|
||||
NormalizeRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
|
||||
NormalizeSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, output contracts.NormalizeOutput, warnings []contracts.Warning) error
|
||||
NormalizeSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, output CheckpointArtifact, warnings []contracts.Warning) error
|
||||
NormalizeRejected(laneID string, moduleKey string, dependencies []CheckpointFingerprint, rejected contracts.RejectedOutput) error
|
||||
NormalizeFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
|
||||
}
|
||||
@@ -59,25 +59,9 @@ type ChunkCheckpoint struct {
|
||||
Warnings []contracts.Warning
|
||||
}
|
||||
|
||||
type ExtractCheckpoint struct {
|
||||
Outputs []contracts.ExtractOutput
|
||||
Rejected []contracts.RejectedOutput
|
||||
Warnings []contracts.Warning
|
||||
}
|
||||
|
||||
type MergeCheckpoint struct {
|
||||
Output contracts.MergeOutput
|
||||
Warnings []contracts.Warning
|
||||
}
|
||||
|
||||
type NormalizeCheckpoint struct {
|
||||
Output contracts.NormalizeOutput
|
||||
Warnings []contracts.Warning
|
||||
}
|
||||
|
||||
// ArtifactCheckpointOutput is the durable, domain-neutral value stored at a
|
||||
// typed lane checkpoint boundary.
|
||||
type ArtifactCheckpointOutput struct {
|
||||
// CheckpointArtifact is the durable, domain-neutral value stored at a lane
|
||||
// checkpoint boundary.
|
||||
type CheckpointArtifact struct {
|
||||
LaneID string
|
||||
ModuleKey string
|
||||
SourceID string
|
||||
@@ -88,33 +72,21 @@ type ArtifactCheckpointOutput struct {
|
||||
SchemaDigest string
|
||||
}
|
||||
|
||||
type ArtifactExtractCheckpoint struct {
|
||||
Outputs []ArtifactCheckpointOutput
|
||||
type ExtractCheckpoint struct {
|
||||
Outputs []CheckpointArtifact
|
||||
Rejected []contracts.RejectedOutput
|
||||
Warnings []contracts.Warning
|
||||
}
|
||||
|
||||
type ArtifactMergeCheckpoint struct {
|
||||
Output ArtifactCheckpointOutput
|
||||
type MergeCheckpoint struct {
|
||||
Output CheckpointArtifact
|
||||
Warnings []contracts.Warning
|
||||
}
|
||||
type ArtifactNormalizeCheckpoint struct {
|
||||
Output ArtifactCheckpointOutput
|
||||
type NormalizeCheckpoint struct {
|
||||
Output CheckpointArtifact
|
||||
Warnings []contracts.Warning
|
||||
}
|
||||
|
||||
type ArtifactCheckpointRecorder interface {
|
||||
ArtifactExtractSucceeded(string, string, []CheckpointFingerprint, []ArtifactCheckpointOutput, []contracts.RejectedOutput, []contracts.Warning) error
|
||||
ArtifactMergeSucceeded(string, string, []CheckpointFingerprint, ArtifactCheckpointOutput, []contracts.Warning) error
|
||||
ArtifactNormalizeSucceeded(string, string, []CheckpointFingerprint, ArtifactCheckpointOutput, []contracts.Warning) error
|
||||
}
|
||||
|
||||
type ArtifactCheckpointLoader interface {
|
||||
ArtifactExtract(string, string, []CheckpointFingerprint) (ArtifactExtractCheckpoint, CheckpointDecision)
|
||||
ArtifactMerge(string, string, []CheckpointFingerprint) (ArtifactMergeCheckpoint, CheckpointDecision)
|
||||
ArtifactNormalize(string, string, []CheckpointFingerprint) (ArtifactNormalizeCheckpoint, CheckpointDecision)
|
||||
}
|
||||
|
||||
type CheckpointLoader interface {
|
||||
Enabled() bool
|
||||
Source(moduleKey string) (SourceCheckpoint, CheckpointDecision)
|
||||
@@ -144,14 +116,14 @@ func (noopCheckpointRecorder) ChunkFailed(string, string, error) error { return
|
||||
func (noopCheckpointRecorder) ExtractRunning(string, string, []CheckpointFingerprint) error {
|
||||
return nil
|
||||
}
|
||||
func (noopCheckpointRecorder) ExtractSucceeded(string, string, []CheckpointFingerprint, []contracts.ExtractOutput, []contracts.RejectedOutput, []contracts.Warning) error {
|
||||
func (noopCheckpointRecorder) ExtractSucceeded(string, string, []CheckpointFingerprint, []CheckpointArtifact, []contracts.RejectedOutput, []contracts.Warning) error {
|
||||
return nil
|
||||
}
|
||||
func (noopCheckpointRecorder) ExtractFailed(string, string, []CheckpointFingerprint, error) error {
|
||||
return nil
|
||||
}
|
||||
func (noopCheckpointRecorder) MergeRunning(string, string, []CheckpointFingerprint) error { return nil }
|
||||
func (noopCheckpointRecorder) MergeSucceeded(string, string, []CheckpointFingerprint, contracts.MergeOutput, []contracts.Warning) error {
|
||||
func (noopCheckpointRecorder) MergeSucceeded(string, string, []CheckpointFingerprint, CheckpointArtifact, []contracts.Warning) error {
|
||||
return nil
|
||||
}
|
||||
func (noopCheckpointRecorder) MergeRejected(string, string, []CheckpointFingerprint, contracts.RejectedOutput) error {
|
||||
@@ -163,7 +135,7 @@ func (noopCheckpointRecorder) MergeFailed(string, string, []CheckpointFingerprin
|
||||
func (noopCheckpointRecorder) NormalizeRunning(string, string, []CheckpointFingerprint) error {
|
||||
return nil
|
||||
}
|
||||
func (noopCheckpointRecorder) NormalizeSucceeded(string, string, []CheckpointFingerprint, contracts.NormalizeOutput, []contracts.Warning) error {
|
||||
func (noopCheckpointRecorder) NormalizeSucceeded(string, string, []CheckpointFingerprint, CheckpointArtifact, []contracts.Warning) error {
|
||||
return nil
|
||||
}
|
||||
func (noopCheckpointRecorder) NormalizeRejected(string, string, []CheckpointFingerprint, contracts.RejectedOutput) error {
|
||||
@@ -190,28 +162,6 @@ func (noopCheckpointLoader) Normalize(string, string, []CheckpointFingerprint) (
|
||||
return NormalizeCheckpoint{}, CheckpointDecision{Reason: "checkpoint loading disabled"}
|
||||
}
|
||||
|
||||
func rawOutputDigests(payloads []contracts.RawPayload) []CheckpointFingerprint {
|
||||
values := make([]CheckpointFingerprint, 0, len(payloads))
|
||||
for i, payload := range payloads {
|
||||
values = append(values, CheckpointFingerprint{
|
||||
Name: fmt.Sprintf("payload[%d]", i),
|
||||
Value: checkpointContentDigest(payload.Content),
|
||||
})
|
||||
}
|
||||
return normalizeCheckpointFingerprints(values)
|
||||
}
|
||||
|
||||
func extractPayloads(outputs []contracts.ExtractOutput) []contracts.RawPayload {
|
||||
if len(outputs) == 0 {
|
||||
return nil
|
||||
}
|
||||
payloads := make([]contracts.RawPayload, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
payloads = append(payloads, output.Payload)
|
||||
}
|
||||
return payloads
|
||||
}
|
||||
|
||||
func digestFingerprints(name string, digest string) []CheckpointFingerprint {
|
||||
digest = strings.TrimSpace(digest)
|
||||
if digest == "" {
|
||||
|
||||
@@ -32,7 +32,7 @@ func (r *ChunkerRegistry) RegisterWithSpec(spec ModuleSpec, constructor ChunkerC
|
||||
if constructor == nil {
|
||||
return fmt.Errorf("chunker constructor for %q must not be nil", strings.TrimSpace(spec.Key))
|
||||
}
|
||||
return r.RegisterBuilderWithSpec(spec, allowLegacyOptions, func(BuildRequest) (contracts.Chunker, error) {
|
||||
return r.RegisterBuilderWithSpec(spec, rejectUnconfiguredOptions, func(BuildRequest) (contracts.Chunker, error) {
|
||||
return constructor()
|
||||
})
|
||||
}
|
||||
|
||||
@@ -338,34 +338,6 @@ func (chunker registryChunker) Chunk(ctx context.Context, req contracts.ChunkReq
|
||||
return contracts.ChunkResult{}, nil
|
||||
}
|
||||
|
||||
type registryMerger struct {
|
||||
key string
|
||||
}
|
||||
|
||||
func (merger registryMerger) Key() string {
|
||||
return merger.key
|
||||
}
|
||||
|
||||
func (merger registryMerger) Merge(ctx context.Context, req contracts.MergeRequest) (contracts.MergeResult, error) {
|
||||
return contracts.MergeResult{}, nil
|
||||
}
|
||||
|
||||
type registryNormalizer struct {
|
||||
key string
|
||||
}
|
||||
|
||||
func (normalizer registryNormalizer) Key() string {
|
||||
return normalizer.key
|
||||
}
|
||||
|
||||
func (normalizer registryNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (normalizer registryNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
|
||||
return contracts.NormalizeResult{}, nil
|
||||
}
|
||||
|
||||
type registryOutputEncoder struct {
|
||||
key string
|
||||
}
|
||||
@@ -377,19 +349,3 @@ func (encoder registryOutputEncoder) Key() string {
|
||||
func (encoder registryOutputEncoder) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
|
||||
return contracts.OutputResult{}, nil
|
||||
}
|
||||
|
||||
type registryValidator struct {
|
||||
name string
|
||||
}
|
||||
|
||||
func (validator registryValidator) Name() string {
|
||||
return validator.name
|
||||
}
|
||||
|
||||
func (validator registryValidator) ExecutionClass() contracts.ExecutionClass {
|
||||
return contracts.ExecutionClassDeterministic
|
||||
}
|
||||
|
||||
func (validator registryValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
|
||||
@@ -24,7 +24,9 @@ type BuildRequest struct {
|
||||
// OptionValidator validates one module binding without constructing it.
|
||||
type OptionValidator func(map[string]any) error
|
||||
|
||||
func allowLegacyOptions(map[string]any) error { return nil }
|
||||
func rejectUnconfiguredOptions(options map[string]any) error {
|
||||
return RejectUnknownOptions(options)
|
||||
}
|
||||
|
||||
func validateRegisteredOptions(validator OptionValidator, options map[string]any) error {
|
||||
if validator == nil {
|
||||
|
||||
@@ -82,10 +82,6 @@ type debugBinaryEnvelope struct {
|
||||
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
type debugRawPayload struct {
|
||||
Content debugBinaryEnvelope `json:"content"`
|
||||
}
|
||||
|
||||
type debugSourceInput struct {
|
||||
SourceID string `json:"source_id,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
@@ -113,32 +109,6 @@ type debugSourceChunk struct {
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
type debugExtractOutput struct {
|
||||
LaneID string `json:"lane_id"`
|
||||
ExtractorKey string `json:"extractor_key"`
|
||||
SourceID string `json:"source_id"`
|
||||
ChunkID string `json:"chunk_id"`
|
||||
ChunkIndex int `json:"chunk_index"`
|
||||
Schema contracts.ResponseSchema `json:"schema,omitempty"`
|
||||
Payload debugBinaryEnvelope `json:"payload"`
|
||||
}
|
||||
|
||||
type debugMergeOutput struct {
|
||||
LaneID string `json:"lane_id"`
|
||||
MergerKey string `json:"merger_key"`
|
||||
SourceID string `json:"source_id,omitempty"`
|
||||
Schema contracts.ResponseSchema `json:"schema,omitempty"`
|
||||
Payload debugBinaryEnvelope `json:"payload"`
|
||||
}
|
||||
|
||||
type debugNormalizeOutput struct {
|
||||
LaneID string `json:"lane_id"`
|
||||
NormalizerKey string `json:"normalizer_key"`
|
||||
SourceID string `json:"source_id,omitempty"`
|
||||
Schema contracts.ResponseSchema `json:"schema,omitempty"`
|
||||
Payload debugBinaryEnvelope `json:"payload"`
|
||||
}
|
||||
|
||||
type debugSerializedOutput struct {
|
||||
LaneID string `json:"lane_id"`
|
||||
NormalizerKey string `json:"normalizer_key"`
|
||||
@@ -202,25 +172,6 @@ type debugLLMCallReference struct {
|
||||
Error bool `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type debugValidationRequest struct {
|
||||
Stage string `json:"stage"`
|
||||
LaneID string `json:"lane_id,omitempty"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
SourceID string `json:"source_id,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
Options map[string]any `json:"options,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
Schema contracts.ResponseSchema `json:"schema,omitempty"`
|
||||
Payload *debugBinaryEnvelope `json:"payload,omitempty"`
|
||||
ChunkID string `json:"chunk_id,omitempty"`
|
||||
ChunkIndex int `json:"chunk_index,omitempty"`
|
||||
Chunk *debugSourceChunk `json:"chunk,omitempty"`
|
||||
Chunks []debugSourceChunk `json:"chunks,omitempty"`
|
||||
ExtractOutputs []debugExtractOutput `json:"extract_outputs,omitempty"`
|
||||
MergeOutput *debugMergeOutput `json:"merge_output,omitempty"`
|
||||
}
|
||||
|
||||
type debugValidationCall struct {
|
||||
ValidatorName string `json:"validator_name"`
|
||||
Request any `json:"request"`
|
||||
@@ -448,10 +399,6 @@ func debugContentEnvelope(content []byte, mediaType string, metadata map[string]
|
||||
}
|
||||
}
|
||||
|
||||
func debugPayloadEnvelope(payload contracts.RawPayload) debugBinaryEnvelope {
|
||||
return debugContentEnvelope(payload.Content, payload.MediaType, payload.Metadata, payload.Warnings)
|
||||
}
|
||||
|
||||
func debugSourceDocumentEnvelope(doc *source.SourceDocument) *debugSourceDocument {
|
||||
if doc == nil {
|
||||
return nil
|
||||
@@ -489,63 +436,6 @@ func debugSourceChunkEnvelopes(chunks []source.Chunk) []debugSourceChunk {
|
||||
return out
|
||||
}
|
||||
|
||||
func debugExtractOutputEnvelope(output contracts.ExtractOutput) debugExtractOutput {
|
||||
output.Schema.JSONSchema = nil
|
||||
return debugExtractOutput{
|
||||
LaneID: output.LaneID,
|
||||
ExtractorKey: output.ExtractorKey,
|
||||
SourceID: output.SourceID,
|
||||
ChunkID: output.ChunkID,
|
||||
ChunkIndex: output.ChunkIndex,
|
||||
Schema: output.Schema,
|
||||
Payload: debugPayloadEnvelope(output.Payload),
|
||||
}
|
||||
}
|
||||
|
||||
func debugExtractOutputEnvelopes(outputs []contracts.ExtractOutput) []debugExtractOutput {
|
||||
if len(outputs) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]debugExtractOutput, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
out = append(out, debugExtractOutputEnvelope(output))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func debugMergeOutputEnvelope(output contracts.MergeOutput) debugMergeOutput {
|
||||
output.Schema.JSONSchema = nil
|
||||
return debugMergeOutput{
|
||||
LaneID: output.LaneID,
|
||||
MergerKey: output.MergerKey,
|
||||
SourceID: output.SourceID,
|
||||
Schema: output.Schema,
|
||||
Payload: debugPayloadEnvelope(output.Payload),
|
||||
}
|
||||
}
|
||||
|
||||
func debugNormalizeOutputEnvelope(output contracts.NormalizeOutput) debugNormalizeOutput {
|
||||
output.Schema.JSONSchema = nil
|
||||
return debugNormalizeOutput{
|
||||
LaneID: output.LaneID,
|
||||
NormalizerKey: output.NormalizerKey,
|
||||
SourceID: output.SourceID,
|
||||
Schema: output.Schema,
|
||||
Payload: debugPayloadEnvelope(output.Payload),
|
||||
}
|
||||
}
|
||||
|
||||
func debugNormalizeOutputEnvelopes(outputs []contracts.NormalizeOutput) []debugNormalizeOutput {
|
||||
if len(outputs) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]debugNormalizeOutput, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
out = append(out, debugNormalizeOutputEnvelope(output))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func debugSerializedOutputEnvelope(output contracts.SerializedOutput) debugSerializedOutput {
|
||||
schema := contracts.CloneArtifactSchema(output.Artifact.Schema)
|
||||
digest := contracts.DigestArtifactSchema(schema)
|
||||
@@ -714,36 +604,6 @@ func debugResponseModel(response contracts.StructuredCompletionResponse) string
|
||||
return response.Debug.Response.ModelName
|
||||
}
|
||||
|
||||
func debugValidationRequestEnvelope(req contracts.ValidationRequest) debugValidationRequest {
|
||||
req.Schema.JSONSchema = nil
|
||||
out := debugValidationRequest{
|
||||
Stage: req.Stage,
|
||||
LaneID: req.LaneID,
|
||||
ModuleKey: req.ModuleKey,
|
||||
SourceID: req.SourceID,
|
||||
SessionID: req.SessionID,
|
||||
LLMProfile: req.LLMProfile,
|
||||
Options: redactSensitiveMap(req.Options),
|
||||
Metadata: redactSensitiveMap(req.Metadata),
|
||||
Schema: req.Schema,
|
||||
ChunkID: req.ChunkID,
|
||||
ChunkIndex: req.ChunkIndex,
|
||||
}
|
||||
payload := debugPayloadEnvelope(req.Payload)
|
||||
out.Payload = &payload
|
||||
if req.Chunk != nil {
|
||||
chunk := debugSourceChunkEnvelope(*req.Chunk)
|
||||
out.Chunk = &chunk
|
||||
}
|
||||
out.Chunks = debugSourceChunkEnvelopes(req.Chunks)
|
||||
out.ExtractOutputs = debugExtractOutputEnvelopes(req.ExtractOutputs)
|
||||
if len(req.MergeOutput.Payload.Content) > 0 || req.MergeOutput.LaneID != "" {
|
||||
merge := debugMergeOutputEnvelope(req.MergeOutput)
|
||||
out.MergeOutput = &merge
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func debugValidationResultEnvelope(result contracts.ValidationResult) contracts.ValidationResult {
|
||||
result.Message = string(redactSecretBytes([]byte(result.Message)))
|
||||
result.DiagnosticArtifactPath = string(redactSecretBytes([]byte(result.DiagnosticArtifactPath)))
|
||||
|
||||
@@ -2,6 +2,7 @@ package pipeline_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
@@ -71,30 +72,40 @@ func defaultModuleCatalog(t *testing.T) pipeline.ModuleCatalog {
|
||||
if err := units.Register(chunkers); err != nil {
|
||||
t.Fatalf("register generic chunker: %v", err)
|
||||
}
|
||||
if err := extractors.RegisterLegacyRawWithSpec(pipeline.ModuleSpec{
|
||||
Key: "extract",
|
||||
Stage: pipeline.StageExtract,
|
||||
Requires: []string{"chunks"},
|
||||
Provides: []string{"records"},
|
||||
}, func() (contracts.LegacyRawExtractor, error) {
|
||||
if err := pipeline.RegisterExtractor[defaultArtifact](extractors, pipeline.ModuleSpec{
|
||||
Key: "extract",
|
||||
Stage: pipeline.StageExtract,
|
||||
ArtifactKind: defaultArtifactKind,
|
||||
Requires: []string{"chunks"},
|
||||
Provides: []string{"records"},
|
||||
}, func() (contracts.Extractor[defaultArtifact], error) {
|
||||
return defaultExtractor{}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register extractor: %v", err)
|
||||
}
|
||||
if err := appendorder.Register(mergers); err != nil {
|
||||
if err := appendorder.RegisterTyped(mergers, defaultArtifactKind, func(values []defaultArtifact) (defaultArtifact, error) {
|
||||
if len(values) == 0 {
|
||||
return defaultArtifact{}, nil
|
||||
}
|
||||
return values[0], nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register appendorder merger: %v", err)
|
||||
}
|
||||
if err := noop.Register(normalizers); err != nil {
|
||||
if err := noop.RegisterTyped[defaultArtifact](normalizers, defaultArtifactKind); err != nil {
|
||||
t.Fatalf("register noop normalizer: %v", err)
|
||||
}
|
||||
if err := jsonoutput.Register(outputs); err != nil {
|
||||
t.Fatalf("register json output: %v", err)
|
||||
}
|
||||
|
||||
codecs := pipeline.NewArtifactCodecRegistry()
|
||||
if err := pipeline.RegisterArtifactCodec(codecs, defaultArtifactCodec{}); err != nil {
|
||||
t.Fatalf("register artifact codec: %v", err)
|
||||
}
|
||||
return pipeline.ModuleCatalog{
|
||||
Inputs: inputs,
|
||||
Chunkers: chunkers,
|
||||
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(),
|
||||
ArtifactCodecs: codecs,
|
||||
Extractors: extractors,
|
||||
Mergers: mergers,
|
||||
Normalizers: normalizers,
|
||||
@@ -117,6 +128,25 @@ func (defaultExtractor) Key() string { return "extract" }
|
||||
|
||||
func (defaultExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
|
||||
func (defaultExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
|
||||
return contracts.ExtractionResult{}, nil
|
||||
func (defaultExtractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[defaultArtifact], error) {
|
||||
return contracts.TypedExtractionResult[defaultArtifact]{}, nil
|
||||
}
|
||||
|
||||
const defaultArtifactKind contracts.ArtifactKind = "test/default"
|
||||
|
||||
type defaultArtifact struct {
|
||||
Value string `json:"value"`
|
||||
}
|
||||
type defaultArtifactCodec struct{}
|
||||
|
||||
func (defaultArtifactCodec) Kind() contracts.ArtifactKind { return defaultArtifactKind }
|
||||
func (defaultArtifactCodec) Schema() contracts.ArtifactSchema {
|
||||
return contracts.ArtifactSchema{ID: "urn:notarius:test:default", Name: "default", Version: "1", JSONSchema: []byte(`{"type":"object"}`)}
|
||||
}
|
||||
func (defaultArtifactCodec) MediaType() string { return "application/json" }
|
||||
func (defaultArtifactCodec) Encode(value defaultArtifact) ([]byte, error) { return json.Marshal(value) }
|
||||
func (defaultArtifactCodec) Decode(content []byte) (defaultArtifact, error) {
|
||||
var value defaultArtifact
|
||||
err := json.Unmarshal(content, &value)
|
||||
return value, err
|
||||
}
|
||||
|
||||
@@ -9,14 +9,9 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
type LegacyRawExtractorConstructor func() (contracts.LegacyRawExtractor, error)
|
||||
type LegacyRawExtractorBuilder func(BuildRequest) (contracts.LegacyRawExtractor, error)
|
||||
|
||||
type ExtractorRegistry struct {
|
||||
legacyBuilders map[string]LegacyRawExtractorBuilder
|
||||
legacyValidators map[string]OptionValidator
|
||||
typedEntries map[string]typedExtractorEntry
|
||||
specs map[string]ModuleSpec
|
||||
typedEntries map[string]typedExtractorEntry
|
||||
specs map[string]ModuleSpec
|
||||
}
|
||||
|
||||
type typedExtractorEntry struct {
|
||||
@@ -25,201 +20,79 @@ type typedExtractorEntry struct {
|
||||
validateOptions OptionValidator
|
||||
builder func(BuildRequest) (any, error)
|
||||
extract typedExtractOperation
|
||||
rawBuilder LegacyRawExtractorBuilder
|
||||
}
|
||||
|
||||
func NewExtractorRegistry() *ExtractorRegistry {
|
||||
return &ExtractorRegistry{
|
||||
legacyBuilders: make(map[string]LegacyRawExtractorBuilder),
|
||||
legacyValidators: make(map[string]OptionValidator),
|
||||
typedEntries: make(map[string]typedExtractorEntry),
|
||||
specs: make(map[string]ModuleSpec),
|
||||
}
|
||||
}
|
||||
|
||||
func (r *ExtractorRegistry) RegisterLegacyRaw(key string, constructor LegacyRawExtractorConstructor) error {
|
||||
return r.RegisterLegacyRawWithSpec(defaultModuleSpec(key, StageExtract), constructor)
|
||||
}
|
||||
|
||||
func (r *ExtractorRegistry) RegisterLegacyRawWithSpec(spec ModuleSpec, constructor LegacyRawExtractorConstructor) error {
|
||||
if constructor == nil {
|
||||
return fmt.Errorf("extractor constructor for %q must not be nil", strings.TrimSpace(spec.Key))
|
||||
}
|
||||
return r.RegisterLegacyRawBuilderWithSpec(spec, allowLegacyOptions, func(BuildRequest) (contracts.LegacyRawExtractor, error) {
|
||||
return constructor()
|
||||
})
|
||||
}
|
||||
|
||||
func (r *ExtractorRegistry) RegisterLegacyRawBuilderWithSpec(spec ModuleSpec, validateOptions OptionValidator, builder LegacyRawExtractorBuilder) error {
|
||||
if r == nil {
|
||||
return fmt.Errorf("extractor registry must not be nil")
|
||||
}
|
||||
normalizedSpec := normalizeModuleSpec(spec)
|
||||
if err := validateModuleSpec("extractor", StageExtract, normalizedSpec); err != nil {
|
||||
return err
|
||||
}
|
||||
if normalizedSpec.ArtifactKind != "" {
|
||||
return fmt.Errorf("legacy raw extractor %q must not declare an artifact kind", normalizedSpec.Key)
|
||||
}
|
||||
if validateOptions == nil {
|
||||
return fmt.Errorf("extractor option validator for %q must not be nil", normalizedSpec.Key)
|
||||
}
|
||||
if builder == nil {
|
||||
return fmt.Errorf("extractor builder for %q must not be nil", normalizedSpec.Key)
|
||||
}
|
||||
if _, ok := r.specs[normalizedSpec.Key]; ok {
|
||||
return fmt.Errorf("extractor %q is already registered", normalizedSpec.Key)
|
||||
}
|
||||
if r.legacyBuilders == nil {
|
||||
r.legacyBuilders = make(map[string]LegacyRawExtractorBuilder)
|
||||
}
|
||||
if r.legacyValidators == nil {
|
||||
r.legacyValidators = make(map[string]OptionValidator)
|
||||
}
|
||||
if r.specs == nil {
|
||||
r.specs = make(map[string]ModuleSpec)
|
||||
}
|
||||
r.legacyBuilders[normalizedSpec.Key] = builder
|
||||
r.legacyValidators[normalizedSpec.Key] = validateOptions
|
||||
r.specs[normalizedSpec.Key] = cloneModuleSpec(normalizedSpec)
|
||||
return nil
|
||||
return &ExtractorRegistry{typedEntries: map[string]typedExtractorEntry{}, specs: map[string]ModuleSpec{}}
|
||||
}
|
||||
|
||||
func RegisterExtractor[T any](registry *ExtractorRegistry, spec ModuleSpec, constructor func() (contracts.Extractor[T], error)) error {
|
||||
if constructor == nil {
|
||||
return fmt.Errorf("extractor constructor for %q must not be nil", strings.TrimSpace(spec.Key))
|
||||
}
|
||||
return RegisterExtractorBuilder(registry, spec, allowLegacyOptions, func(BuildRequest) (contracts.Extractor[T], error) {
|
||||
return constructor()
|
||||
})
|
||||
return RegisterExtractorBuilder(registry, spec, rejectUnconfiguredOptions, func(BuildRequest) (contracts.Extractor[T], error) { return constructor() })
|
||||
}
|
||||
|
||||
func RegisterExtractorBuilder[T any](registry *ExtractorRegistry, spec ModuleSpec, validateOptions OptionValidator, builder func(BuildRequest) (contracts.Extractor[T], error)) error {
|
||||
return registerExtractorBuilder(registry, spec, validateOptions, builder, nil)
|
||||
}
|
||||
|
||||
// RegisterExtractorBuilderWithRawAdapter registers a typed extractor while a
|
||||
// raw downstream remains in use. Resolution selects the adapter until the
|
||||
// registration is replaced with the typed-only builder.
|
||||
func RegisterExtractorBuilderWithRawAdapter[T any](registry *ExtractorRegistry, spec ModuleSpec, validateOptions OptionValidator, builder func(BuildRequest) (contracts.Extractor[T], error), rawBuilder LegacyRawExtractorBuilder) error {
|
||||
if rawBuilder == nil {
|
||||
return fmt.Errorf("extractor raw adapter builder for %q must not be nil", strings.TrimSpace(spec.Key))
|
||||
}
|
||||
return registerExtractorBuilder(registry, spec, validateOptions, builder, rawBuilder)
|
||||
}
|
||||
|
||||
func registerExtractorBuilder[T any](registry *ExtractorRegistry, spec ModuleSpec, validateOptions OptionValidator, builder func(BuildRequest) (contracts.Extractor[T], error), rawBuilder LegacyRawExtractorBuilder) error {
|
||||
if registry == nil {
|
||||
return fmt.Errorf("extractor registry must not be nil")
|
||||
}
|
||||
normalizedSpec := normalizeModuleSpec(spec)
|
||||
if err := validateModuleSpec("extractor", StageExtract, normalizedSpec); err != nil {
|
||||
normalized := normalizeModuleSpec(spec)
|
||||
if err := validateModuleSpec("extractor", StageExtract, normalized); err != nil {
|
||||
return err
|
||||
}
|
||||
if normalizedSpec.ArtifactKind == "" {
|
||||
return fmt.Errorf("typed extractor %q artifact kind must not be empty", normalizedSpec.Key)
|
||||
if normalized.ArtifactKind == "" {
|
||||
return fmt.Errorf("typed extractor %q artifact kind must not be empty", normalized.Key)
|
||||
}
|
||||
if validateOptions == nil {
|
||||
return fmt.Errorf("extractor option validator for %q must not be nil", normalizedSpec.Key)
|
||||
return fmt.Errorf("extractor option validator for %q must not be nil", normalized.Key)
|
||||
}
|
||||
if builder == nil {
|
||||
return fmt.Errorf("extractor builder for %q must not be nil", normalizedSpec.Key)
|
||||
return fmt.Errorf("extractor builder for %q must not be nil", normalized.Key)
|
||||
}
|
||||
if _, ok := registry.specs[normalizedSpec.Key]; ok {
|
||||
return fmt.Errorf("extractor %q is already registered", normalizedSpec.Key)
|
||||
}
|
||||
entry := typedExtractorEntry{
|
||||
spec: cloneModuleSpec(normalizedSpec),
|
||||
valueType: reflect.TypeFor[T](),
|
||||
validateOptions: validateOptions,
|
||||
builder: func(request BuildRequest) (any, error) {
|
||||
return builder(cloneBuildRequest(request))
|
||||
},
|
||||
extract: func(ctx context.Context, implementation any, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
extractor, ok := implementation.(contracts.Extractor[T])
|
||||
if !ok {
|
||||
return erasedTypedResult{}, fmt.Errorf("extractor %q has incompatible implementation %T", normalizedSpec.Key, implementation)
|
||||
}
|
||||
result, err := extractor.Extract(ctx, request)
|
||||
if err != nil {
|
||||
return erasedTypedResult{}, err
|
||||
}
|
||||
return erasedTypedResult{Value: result.Value, Warnings: result.Warnings}, nil
|
||||
},
|
||||
rawBuilder: rawBuilder,
|
||||
if _, ok := registry.specs[normalized.Key]; ok {
|
||||
return fmt.Errorf("extractor %q is already registered", normalized.Key)
|
||||
}
|
||||
entry := typedExtractorEntry{spec: cloneModuleSpec(normalized), valueType: reflect.TypeFor[T](), validateOptions: validateOptions, builder: func(request BuildRequest) (any, error) { return builder(cloneBuildRequest(request)) }, extract: func(ctx context.Context, implementation any, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
extractor, ok := implementation.(contracts.Extractor[T])
|
||||
if !ok {
|
||||
return erasedTypedResult{}, fmt.Errorf("extractor %q has incompatible implementation %T", normalized.Key, implementation)
|
||||
}
|
||||
result, err := extractor.Extract(ctx, request)
|
||||
if err != nil {
|
||||
return erasedTypedResult{}, err
|
||||
}
|
||||
return erasedTypedResult{Value: result.Value, Warnings: result.Warnings}, nil
|
||||
}}
|
||||
if registry.typedEntries == nil {
|
||||
registry.typedEntries = make(map[string]typedExtractorEntry)
|
||||
registry.typedEntries = map[string]typedExtractorEntry{}
|
||||
}
|
||||
if registry.specs == nil {
|
||||
registry.specs = make(map[string]ModuleSpec)
|
||||
registry.specs = map[string]ModuleSpec{}
|
||||
}
|
||||
registry.typedEntries[normalizedSpec.Key] = entry
|
||||
registry.specs[normalizedSpec.Key] = cloneModuleSpec(normalizedSpec)
|
||||
registry.typedEntries[normalized.Key] = entry
|
||||
registry.specs[normalized.Key] = cloneModuleSpec(normalized)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *ExtractorRegistry) BuildLegacyRaw(key string) (contracts.LegacyRawExtractor, error) {
|
||||
return r.BuildLegacyRawWithRequest(key, BuildRequest{})
|
||||
}
|
||||
|
||||
func (r *ExtractorRegistry) BuildLegacyRawWithRequest(key string, request BuildRequest) (contracts.LegacyRawExtractor, error) {
|
||||
if r == nil {
|
||||
return nil, fmt.Errorf("extractor registry must not be nil")
|
||||
}
|
||||
normalizedKey := strings.TrimSpace(key)
|
||||
if normalizedKey == "" {
|
||||
return nil, fmt.Errorf("extractor key must not be empty")
|
||||
}
|
||||
builder, ok := r.legacyBuilders[normalizedKey]
|
||||
if !ok {
|
||||
if entry, typedOK := r.typedEntries[normalizedKey]; typedOK && entry.rawBuilder != nil {
|
||||
builder = entry.rawBuilder
|
||||
ok = true
|
||||
}
|
||||
}
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("legacy raw extractor %q is not registered", normalizedKey)
|
||||
}
|
||||
extractor, err := builder(cloneBuildRequest(request))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build extractor %q: %w", normalizedKey, err)
|
||||
}
|
||||
if extractor == nil {
|
||||
return nil, fmt.Errorf("extractor %q constructor returned nil", normalizedKey)
|
||||
}
|
||||
if extractor.Key() != normalizedKey {
|
||||
return nil, fmt.Errorf("extractor %q returned key %q", normalizedKey, extractor.Key())
|
||||
}
|
||||
return extractor, nil
|
||||
}
|
||||
|
||||
func (r *ExtractorRegistry) validateOptions(key string, options map[string]any) error {
|
||||
if r == nil {
|
||||
return fmt.Errorf("extractor registry must not be nil")
|
||||
}
|
||||
normalizedKey := strings.TrimSpace(key)
|
||||
if entry, ok := r.typedEntries[normalizedKey]; ok {
|
||||
return validateRegisteredOptions(entry.validateOptions, options)
|
||||
}
|
||||
validator, ok := r.legacyValidators[normalizedKey]
|
||||
normalized := strings.TrimSpace(key)
|
||||
entry, ok := r.typedEntries[normalized]
|
||||
if !ok {
|
||||
return fmt.Errorf("extractor %q is not registered", normalizedKey)
|
||||
return fmt.Errorf("extractor %q is not registered", normalized)
|
||||
}
|
||||
return validateRegisteredOptions(validator, options)
|
||||
return validateRegisteredOptions(entry.validateOptions, options)
|
||||
}
|
||||
|
||||
func (r *ExtractorRegistry) Spec(key string) (ModuleSpec, bool) {
|
||||
if r == nil {
|
||||
return ModuleSpec{}, false
|
||||
}
|
||||
spec, ok := r.specs[strings.TrimSpace(key)]
|
||||
if !ok {
|
||||
return ModuleSpec{}, false
|
||||
}
|
||||
return cloneModuleSpec(spec), true
|
||||
return cloneModuleSpec(spec), ok
|
||||
}
|
||||
|
||||
func (r *ExtractorRegistry) typedEntry(key string) (typedExtractorEntry, bool) {
|
||||
if r == nil {
|
||||
return typedExtractorEntry{}, false
|
||||
@@ -227,12 +100,6 @@ func (r *ExtractorRegistry) typedEntry(key string) (typedExtractorEntry, bool) {
|
||||
entry, ok := r.typedEntries[strings.TrimSpace(key)]
|
||||
return entry, ok
|
||||
}
|
||||
|
||||
func (r *ExtractorRegistry) usesRawAdapter(key string) bool {
|
||||
entry, ok := r.typedEntry(key)
|
||||
return ok && entry.rawBuilder != nil
|
||||
}
|
||||
|
||||
func (r *ExtractorRegistry) RegisteredKeys() []string {
|
||||
if r == nil {
|
||||
return nil
|
||||
|
||||
@@ -1,414 +0,0 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func TestExtractorRegistryRegisterAndBuild(t *testing.T) {
|
||||
registry := NewExtractorRegistry()
|
||||
|
||||
if err := registry.RegisterLegacyRaw("generic-extractor", fakeExtractorConstructor("generic-extractor")); err != nil {
|
||||
t.Fatalf("Register() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
extractor, err := registry.BuildLegacyRaw("generic-extractor")
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v, want nil", err)
|
||||
}
|
||||
if extractor.Key() != "generic-extractor" {
|
||||
t.Fatalf("extractor.Key() = %q, want generic-extractor", extractor.Key())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractorRegistryRegisterAndBuildTrimKeys(t *testing.T) {
|
||||
registry := NewExtractorRegistry()
|
||||
|
||||
if err := registry.RegisterLegacyRaw(" generic-extractor ", fakeExtractorConstructor("generic-extractor")); err != nil {
|
||||
t.Fatalf("Register() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
extractor, err := registry.BuildLegacyRaw("\tgeneric-extractor\n")
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v, want nil", err)
|
||||
}
|
||||
if extractor.Key() != "generic-extractor" {
|
||||
t.Fatalf("extractor.Key() = %q, want generic-extractor", extractor.Key())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractorRegistryRegisterWithSpecStoresMetadata(t *testing.T) {
|
||||
registry := NewExtractorRegistry()
|
||||
spec := ModuleSpec{
|
||||
Key: " generic-extractor ",
|
||||
Stage: StageExtract,
|
||||
Provides: []string{" generic-artifact ", "source-citations", "generic-artifact", ""},
|
||||
Requires: []string{" source-document ", "source-document", ""},
|
||||
ReferenceSlots: []contracts.ReferenceSlot{
|
||||
{
|
||||
Name: " glossary ",
|
||||
Description: " Supporting terms ",
|
||||
AcceptedMediaTypes: []string{" text/plain ", "text/markdown", "text/plain", ""},
|
||||
MaxBytes: 1024,
|
||||
},
|
||||
{
|
||||
Name: " roster ",
|
||||
Description: " Characters ",
|
||||
Required: true,
|
||||
Multiple: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
if err := registry.RegisterLegacyRawWithSpec(spec, fakeExtractorConstructor("generic-extractor")); err != nil {
|
||||
t.Fatalf("RegisterWithSpec() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
got, ok := registry.Spec("\tgeneric-extractor\n")
|
||||
if !ok {
|
||||
t.Fatal("Spec() ok = false, want true")
|
||||
}
|
||||
want := ModuleSpec{
|
||||
Key: "generic-extractor",
|
||||
Stage: StageExtract,
|
||||
Provides: []string{"generic-artifact", "source-citations"},
|
||||
Requires: []string{"source-document"},
|
||||
ReferenceSlots: []contracts.ReferenceSlot{
|
||||
{
|
||||
Name: "glossary",
|
||||
Description: "Supporting terms",
|
||||
AcceptedMediaTypes: []string{"text/markdown", "text/plain"},
|
||||
MaxBytes: 1024,
|
||||
},
|
||||
{
|
||||
Name: "roster",
|
||||
Description: "Characters",
|
||||
Required: true,
|
||||
Multiple: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("Spec() = %#v, want %#v", got, want)
|
||||
}
|
||||
|
||||
got.Provides[0] = "changed"
|
||||
got.ReferenceSlots[0].Name = "changed"
|
||||
got.ReferenceSlots[0].AcceptedMediaTypes[0] = "changed"
|
||||
again, ok := registry.Spec("generic-extractor")
|
||||
if !ok {
|
||||
t.Fatal("Spec() after caller mutation ok = false, want true")
|
||||
}
|
||||
if !reflect.DeepEqual(again, want) {
|
||||
t.Fatalf("Spec() after caller mutation = %#v, want %#v", again, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractorRegistryRegisterStoresDefaultSpec(t *testing.T) {
|
||||
registry := NewExtractorRegistry()
|
||||
|
||||
if err := registry.RegisterLegacyRaw(" generic-extractor ", fakeExtractorConstructor("generic-extractor")); err != nil {
|
||||
t.Fatalf("Register() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
got, ok := registry.Spec("generic-extractor")
|
||||
if !ok {
|
||||
t.Fatal("Spec() ok = false, want true")
|
||||
}
|
||||
want := ModuleSpec{Key: "generic-extractor", Stage: StageExtract}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("Spec() = %#v, want %#v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractorRegistryRegisterWithSpecRejectsWrongStage(t *testing.T) {
|
||||
registry := NewExtractorRegistry()
|
||||
|
||||
err := registry.RegisterLegacyRawWithSpec(ModuleSpec{Key: "generic-extractor", Stage: StageInput}, fakeExtractorConstructor("generic-extractor"))
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("RegisterWithSpec() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "stage") {
|
||||
t.Fatalf("RegisterWithSpec() error = %q, want stage error", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractorRegistryRejectsInvalidReferenceSlots(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
slots []contracts.ReferenceSlot
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "empty name",
|
||||
slots: []contracts.ReferenceSlot{{Name: " "}},
|
||||
want: "name",
|
||||
},
|
||||
{
|
||||
name: "duplicate name after trim",
|
||||
slots: []contracts.ReferenceSlot{
|
||||
{Name: "roster"},
|
||||
{Name: " roster "},
|
||||
},
|
||||
want: "duplicated",
|
||||
},
|
||||
{
|
||||
name: "negative max bytes",
|
||||
slots: []contracts.ReferenceSlot{{Name: "roster", MaxBytes: -1}},
|
||||
want: "max_bytes",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
registry := NewExtractorRegistry()
|
||||
err := registry.RegisterLegacyRawWithSpec(ModuleSpec{
|
||||
Key: "generic-extractor",
|
||||
Stage: StageExtract,
|
||||
ReferenceSlots: test.slots,
|
||||
}, fakeExtractorConstructor("generic-extractor"))
|
||||
if err == nil {
|
||||
t.Fatal("RegisterWithSpec() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), test.want) {
|
||||
t.Fatalf("RegisterWithSpec() error = %q, want %q", err.Error(), test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractorRegistrySpecRejectsUnknownKey(t *testing.T) {
|
||||
registry := NewExtractorRegistry()
|
||||
|
||||
if _, ok := registry.Spec("missing-extractor"); ok {
|
||||
t.Fatal("Spec() ok = true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractorRegistryRegisterRejectsEmptyKey(t *testing.T) {
|
||||
registry := NewExtractorRegistry()
|
||||
|
||||
err := registry.RegisterLegacyRaw(" \t", fakeExtractorConstructor("generic-extractor"))
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("Register() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "key must not be empty") {
|
||||
t.Fatalf("Register() error = %q, want empty key error", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractorRegistryRegisterRejectsDuplicateKey(t *testing.T) {
|
||||
registry := NewExtractorRegistry()
|
||||
if err := registry.RegisterLegacyRaw("generic-extractor", fakeExtractorConstructor("generic-extractor")); err != nil {
|
||||
t.Fatalf("Register() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
err := registry.RegisterLegacyRaw(" generic-extractor ", fakeExtractorConstructor("generic-extractor"))
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("Register() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "already registered") {
|
||||
t.Fatalf("Register() error = %q, want duplicate key error", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractorRegistryRegisterRejectsNilConstructor(t *testing.T) {
|
||||
registry := NewExtractorRegistry()
|
||||
|
||||
err := registry.RegisterLegacyRaw("generic-extractor", nil)
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("Register() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "constructor") {
|
||||
t.Fatalf("Register() error = %q, want constructor error", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractorRegistryBuildRejectsUnknownKey(t *testing.T) {
|
||||
registry := NewExtractorRegistry()
|
||||
|
||||
_, err := registry.BuildLegacyRaw("missing-extractor")
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("Build() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "not registered") {
|
||||
t.Fatalf("Build() error = %q, want unknown key error", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractorRegistryBuildWrapsConstructorError(t *testing.T) {
|
||||
registry := NewExtractorRegistry()
|
||||
constructorErr := errors.New("constructor failed")
|
||||
if err := registry.RegisterLegacyRaw("generic-extractor", func() (contracts.LegacyRawExtractor, error) {
|
||||
return nil, constructorErr
|
||||
}); err != nil {
|
||||
t.Fatalf("Register() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
_, err := registry.BuildLegacyRaw("generic-extractor")
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("Build() error = nil, want error")
|
||||
}
|
||||
if !errors.Is(err, constructorErr) {
|
||||
t.Fatalf("Build() error = %v, want wrapped constructor error", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "generic-extractor") {
|
||||
t.Fatalf("Build() error = %q, want key context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractorRegistryBuildRejectsNilExtractor(t *testing.T) {
|
||||
registry := NewExtractorRegistry()
|
||||
if err := registry.RegisterLegacyRaw("generic-extractor", func() (contracts.LegacyRawExtractor, error) {
|
||||
return nil, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("Register() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
_, err := registry.BuildLegacyRaw("generic-extractor")
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("Build() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "returned nil") {
|
||||
t.Fatalf("Build() error = %q, want nil extractor error", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractorRegistryBuildRejectsExtractorKeyMismatch(t *testing.T) {
|
||||
registry := NewExtractorRegistry()
|
||||
if err := registry.RegisterLegacyRaw("generic-extractor", fakeExtractorConstructor("other-extractor")); err != nil {
|
||||
t.Fatalf("Register() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
_, err := registry.BuildLegacyRaw("generic-extractor")
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("Build() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "returned key") {
|
||||
t.Fatalf("Build() error = %q, want key mismatch error", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractorRegistryRegisteredKeysReturnsSortedCopy(t *testing.T) {
|
||||
registry := NewExtractorRegistry()
|
||||
for _, key := range []string{"zeta", "alpha", "middle"} {
|
||||
if err := registry.RegisterLegacyRaw(key, fakeExtractorConstructor(key)); err != nil {
|
||||
t.Fatalf("Register(%q) error = %v, want nil", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
keys := registry.RegisteredKeys()
|
||||
|
||||
want := []string{"alpha", "middle", "zeta"}
|
||||
if !reflect.DeepEqual(keys, want) {
|
||||
t.Fatalf("RegisteredKeys() = %#v, want %#v", keys, want)
|
||||
}
|
||||
|
||||
keys[0] = "changed"
|
||||
if got := registry.RegisteredKeys(); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("RegisteredKeys() after caller mutation = %#v, want %#v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractorRegistryNilRegistryBehavior(t *testing.T) {
|
||||
var registry *ExtractorRegistry
|
||||
|
||||
if err := registry.RegisterLegacyRaw("generic-extractor", fakeExtractorConstructor("generic-extractor")); err == nil {
|
||||
t.Fatal("Register() error = nil, want error")
|
||||
}
|
||||
if _, err := registry.BuildLegacyRaw("generic-extractor"); err == nil {
|
||||
t.Fatal("Build() error = nil, want error")
|
||||
}
|
||||
if _, ok := registry.Spec("generic-extractor"); ok {
|
||||
t.Fatal("Spec() ok = true, want false")
|
||||
}
|
||||
if keys := registry.RegisteredKeys(); keys != nil {
|
||||
t.Fatalf("RegisteredKeys() = %#v, want nil", keys)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractorRegistryBuildRejectsEmptyKey(t *testing.T) {
|
||||
registry := NewExtractorRegistry()
|
||||
|
||||
_, err := registry.BuildLegacyRaw(" \n")
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("Build() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "key must not be empty") {
|
||||
t.Fatalf("Build() error = %q, want empty key error", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractorRegistryTypedRegistrationCanProvideRawAdapter(t *testing.T) {
|
||||
registry := NewExtractorRegistry()
|
||||
spec := ModuleSpec{Key: "typed-extractor", Stage: StageExtract, ArtifactKind: "test/value"}
|
||||
if err := RegisterExtractorBuilderWithRawAdapter(registry, spec, func(map[string]any) error { return nil },
|
||||
func(BuildRequest) (contracts.Extractor[registryTypedValue], error) {
|
||||
return registryTypedExtractor{key: spec.Key}, nil
|
||||
},
|
||||
func(BuildRequest) (contracts.LegacyRawExtractor, error) {
|
||||
return registryFakeExtractor{key: spec.Key}, nil
|
||||
},
|
||||
); err != nil {
|
||||
t.Fatalf("RegisterExtractorBuilderWithRawAdapter() error = %v", err)
|
||||
}
|
||||
if !registry.usesRawAdapter(spec.Key) {
|
||||
t.Fatal("usesRawAdapter() = false, want true")
|
||||
}
|
||||
if _, ok := registry.typedEntry(spec.Key); !ok {
|
||||
t.Fatal("typedEntry() ok = false, want true")
|
||||
}
|
||||
adapter, err := registry.BuildLegacyRaw(spec.Key)
|
||||
if err != nil || adapter.Key() != spec.Key {
|
||||
t.Fatalf("BuildLegacyRaw() = %#v, %v", adapter, err)
|
||||
}
|
||||
if err := RegisterExtractorBuilderWithRawAdapter[registryTypedValue](NewExtractorRegistry(), spec, func(map[string]any) error { return nil }, nil, nil); err == nil || !strings.Contains(err.Error(), "raw adapter") {
|
||||
t.Fatalf("nil raw builder error = %v, want raw adapter context", err)
|
||||
}
|
||||
}
|
||||
|
||||
type registryFakeExtractor struct {
|
||||
key string
|
||||
}
|
||||
|
||||
type registryTypedValue struct{ Value string }
|
||||
|
||||
type registryTypedExtractor struct{ key string }
|
||||
|
||||
func (extractor registryTypedExtractor) Key() string { return extractor.key }
|
||||
func (registryTypedExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
func (registryTypedExtractor) Extract(context.Context, contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[registryTypedValue], error) {
|
||||
return contracts.TypedExtractionResult[registryTypedValue]{}, nil
|
||||
}
|
||||
|
||||
func fakeExtractorConstructor(key string) LegacyRawExtractorConstructor {
|
||||
return func() (contracts.LegacyRawExtractor, error) {
|
||||
return registryFakeExtractor{key: key}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (extractor registryFakeExtractor) Key() string {
|
||||
return extractor.key
|
||||
}
|
||||
|
||||
func (extractor registryFakeExtractor) ReferenceSlots() []contracts.ReferenceSlot {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (extractor registryFakeExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
|
||||
return contracts.ExtractionResult{}, nil
|
||||
}
|
||||
@@ -32,7 +32,7 @@ func (r *InputAdapterRegistry) RegisterWithSpec(spec ModuleSpec, constructor Inp
|
||||
if constructor == nil {
|
||||
return fmt.Errorf("input adapter constructor for %q must not be nil", strings.TrimSpace(spec.Key))
|
||||
}
|
||||
return r.RegisterBuilderWithSpec(spec, allowLegacyOptions, func(BuildRequest) (contracts.InputAdapter, error) {
|
||||
return r.RegisterBuilderWithSpec(spec, rejectUnconfiguredOptions, func(BuildRequest) (contracts.InputAdapter, error) {
|
||||
return constructor()
|
||||
})
|
||||
}
|
||||
|
||||
@@ -9,19 +9,13 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
type LegacyRawMergerConstructor func() (contracts.LegacyRawMerger, error)
|
||||
type LegacyRawMergerBuilder func(BuildRequest) (contracts.LegacyRawMerger, error)
|
||||
|
||||
type artifactVariantKey struct {
|
||||
module string
|
||||
kind contracts.ArtifactKind
|
||||
}
|
||||
|
||||
type MergerRegistry struct {
|
||||
legacyBuilders map[string]LegacyRawMergerBuilder
|
||||
legacyValidators map[string]OptionValidator
|
||||
legacySpecs map[string]ModuleSpec
|
||||
typedEntries map[artifactVariantKey]typedMergerEntry
|
||||
typedEntries map[artifactVariantKey]typedMergerEntry
|
||||
}
|
||||
|
||||
type typedMergerEntry struct {
|
||||
@@ -34,66 +28,15 @@ type typedMergerEntry struct {
|
||||
|
||||
func NewMergerRegistry() *MergerRegistry {
|
||||
return &MergerRegistry{
|
||||
legacyBuilders: make(map[string]LegacyRawMergerBuilder),
|
||||
legacyValidators: make(map[string]OptionValidator),
|
||||
legacySpecs: make(map[string]ModuleSpec),
|
||||
typedEntries: make(map[artifactVariantKey]typedMergerEntry),
|
||||
typedEntries: make(map[artifactVariantKey]typedMergerEntry),
|
||||
}
|
||||
}
|
||||
|
||||
func (r *MergerRegistry) RegisterLegacyRaw(key string, constructor LegacyRawMergerConstructor) error {
|
||||
return r.RegisterLegacyRawWithSpec(defaultModuleSpec(key, StageMerge), constructor)
|
||||
}
|
||||
|
||||
func (r *MergerRegistry) RegisterLegacyRawWithSpec(spec ModuleSpec, constructor LegacyRawMergerConstructor) error {
|
||||
if constructor == nil {
|
||||
return fmt.Errorf("merger constructor for %q must not be nil", strings.TrimSpace(spec.Key))
|
||||
}
|
||||
return r.RegisterLegacyRawBuilderWithSpec(spec, allowLegacyOptions, func(BuildRequest) (contracts.LegacyRawMerger, error) {
|
||||
return constructor()
|
||||
})
|
||||
}
|
||||
|
||||
func (r *MergerRegistry) RegisterLegacyRawBuilderWithSpec(spec ModuleSpec, validateOptions OptionValidator, builder LegacyRawMergerBuilder) error {
|
||||
if r == nil {
|
||||
return fmt.Errorf("merger registry must not be nil")
|
||||
}
|
||||
normalizedSpec := normalizeModuleSpec(spec)
|
||||
if err := validateModuleSpec("merger", StageMerge, normalizedSpec); err != nil {
|
||||
return err
|
||||
}
|
||||
if normalizedSpec.ArtifactKind != "" {
|
||||
return fmt.Errorf("legacy raw merger %q must not declare an artifact kind", normalizedSpec.Key)
|
||||
}
|
||||
if validateOptions == nil {
|
||||
return fmt.Errorf("merger option validator for %q must not be nil", normalizedSpec.Key)
|
||||
}
|
||||
if builder == nil {
|
||||
return fmt.Errorf("merger builder for %q must not be nil", normalizedSpec.Key)
|
||||
}
|
||||
if _, ok := r.legacyBuilders[normalizedSpec.Key]; ok {
|
||||
return fmt.Errorf("legacy raw merger %q is already registered", normalizedSpec.Key)
|
||||
}
|
||||
if r.legacyBuilders == nil {
|
||||
r.legacyBuilders = make(map[string]LegacyRawMergerBuilder)
|
||||
}
|
||||
if r.legacyValidators == nil {
|
||||
r.legacyValidators = make(map[string]OptionValidator)
|
||||
}
|
||||
if r.legacySpecs == nil {
|
||||
r.legacySpecs = make(map[string]ModuleSpec)
|
||||
}
|
||||
r.legacyBuilders[normalizedSpec.Key] = builder
|
||||
r.legacyValidators[normalizedSpec.Key] = validateOptions
|
||||
r.legacySpecs[normalizedSpec.Key] = cloneModuleSpec(normalizedSpec)
|
||||
return nil
|
||||
}
|
||||
|
||||
func RegisterMerger[T any](registry *MergerRegistry, spec ModuleSpec, constructor func() (contracts.Merger[T], error)) error {
|
||||
if constructor == nil {
|
||||
return fmt.Errorf("merger constructor for %q must not be nil", strings.TrimSpace(spec.Key))
|
||||
}
|
||||
return RegisterMergerBuilder(registry, spec, allowLegacyOptions, func(BuildRequest) (contracts.Merger[T], error) {
|
||||
return RegisterMergerBuilder(registry, spec, rejectUnconfiguredOptions, func(BuildRequest) (contracts.Merger[T], error) {
|
||||
return constructor()
|
||||
})
|
||||
}
|
||||
@@ -152,63 +95,29 @@ func RegisterMergerBuilder[T any](registry *MergerRegistry, spec ModuleSpec, val
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *MergerRegistry) BuildLegacyRaw(key string) (contracts.LegacyRawMerger, error) {
|
||||
return r.BuildLegacyRawWithRequest(key, BuildRequest{})
|
||||
}
|
||||
|
||||
func (r *MergerRegistry) BuildLegacyRawWithRequest(key string, request BuildRequest) (contracts.LegacyRawMerger, error) {
|
||||
if r == nil {
|
||||
return nil, fmt.Errorf("merger registry must not be nil")
|
||||
}
|
||||
normalizedKey := strings.TrimSpace(key)
|
||||
if normalizedKey == "" {
|
||||
return nil, fmt.Errorf("merger key must not be empty")
|
||||
}
|
||||
builder, ok := r.legacyBuilders[normalizedKey]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("legacy raw merger %q is not registered", normalizedKey)
|
||||
}
|
||||
merger, err := builder(cloneBuildRequest(request))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build merger %q: %w", normalizedKey, err)
|
||||
}
|
||||
if merger == nil {
|
||||
return nil, fmt.Errorf("merger %q constructor returned nil", normalizedKey)
|
||||
}
|
||||
if merger.Key() != normalizedKey {
|
||||
return nil, fmt.Errorf("merger %q returned key %q", normalizedKey, merger.Key())
|
||||
}
|
||||
return merger, nil
|
||||
}
|
||||
|
||||
func (r *MergerRegistry) validateOptions(key string, kind contracts.ArtifactKind, options map[string]any) error {
|
||||
if r == nil {
|
||||
return fmt.Errorf("merger registry must not be nil")
|
||||
}
|
||||
normalizedKey := strings.TrimSpace(key)
|
||||
if kind != "" {
|
||||
entry, ok := r.typedEntry(normalizedKey, kind)
|
||||
if !ok {
|
||||
return fmt.Errorf("merger %q variant for artifact kind %q is not registered", normalizedKey, kind)
|
||||
}
|
||||
return validateRegisteredOptions(entry.validateOptions, options)
|
||||
}
|
||||
validator, ok := r.legacyValidators[normalizedKey]
|
||||
entry, ok := r.typedEntry(normalizedKey, kind)
|
||||
if !ok {
|
||||
return fmt.Errorf("legacy raw merger %q is not registered", normalizedKey)
|
||||
return fmt.Errorf("merger %q variant for artifact kind %q is not registered", normalizedKey, kind)
|
||||
}
|
||||
return validateRegisteredOptions(validator, options)
|
||||
return validateRegisteredOptions(entry.validateOptions, options)
|
||||
}
|
||||
|
||||
func (r *MergerRegistry) Spec(key string) (ModuleSpec, bool) {
|
||||
if r == nil {
|
||||
return ModuleSpec{}, false
|
||||
}
|
||||
spec, ok := r.legacySpecs[strings.TrimSpace(key)]
|
||||
if !ok {
|
||||
return ModuleSpec{}, false
|
||||
module := strings.TrimSpace(key)
|
||||
for variant, entry := range r.typedEntries {
|
||||
if variant.module == module {
|
||||
return cloneModuleSpec(entry.spec), true
|
||||
}
|
||||
}
|
||||
return cloneModuleSpec(spec), true
|
||||
return ModuleSpec{}, false
|
||||
}
|
||||
|
||||
func (r *MergerRegistry) typedEntry(key string, kind contracts.ArtifactKind) (typedMergerEntry, bool) {
|
||||
@@ -238,10 +147,7 @@ func (r *MergerRegistry) RegisteredKeys() []string {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
keys := make(map[string]struct{}, len(r.legacySpecs)+len(r.typedEntries))
|
||||
for key := range r.legacySpecs {
|
||||
keys[key] = struct{}{}
|
||||
}
|
||||
keys := make(map[string]struct{}, len(r.typedEntries))
|
||||
for key := range r.typedEntries {
|
||||
keys[key.module] = struct{}{}
|
||||
}
|
||||
|
||||
@@ -1,58 +0,0 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func TestMergerRegistryBehavior(t *testing.T) {
|
||||
runRegistryBehaviorTests(t, registryBehaviorCase[contracts.LegacyRawMerger]{
|
||||
name: "MergerRegistry",
|
||||
key: "generic-merger",
|
||||
stage: StageMerge,
|
||||
wrongStage: StageExtract,
|
||||
newRegistry: func() any {
|
||||
return NewMergerRegistry()
|
||||
},
|
||||
register: func(registry any, key string, constructor func() (contracts.LegacyRawMerger, error)) error {
|
||||
return registry.(*MergerRegistry).RegisterLegacyRaw(key, constructor)
|
||||
},
|
||||
registerWithSpec: func(registry any, spec ModuleSpec, constructor func() (contracts.LegacyRawMerger, error)) error {
|
||||
return registry.(*MergerRegistry).RegisterLegacyRawWithSpec(spec, constructor)
|
||||
},
|
||||
build: func(registry any, key string) (contracts.LegacyRawMerger, error) {
|
||||
return registry.(*MergerRegistry).BuildLegacyRaw(key)
|
||||
},
|
||||
spec: func(registry any, key string) (ModuleSpec, bool) {
|
||||
return registry.(*MergerRegistry).Spec(key)
|
||||
},
|
||||
registeredKeys: func(registry any) []string {
|
||||
return registry.(*MergerRegistry).RegisteredKeys()
|
||||
},
|
||||
nilRegister: func(key string, constructor func() (contracts.LegacyRawMerger, error)) error {
|
||||
var registry *MergerRegistry
|
||||
return registry.RegisterLegacyRaw(key, constructor)
|
||||
},
|
||||
nilBuild: func(key string) (contracts.LegacyRawMerger, error) {
|
||||
var registry *MergerRegistry
|
||||
return registry.BuildLegacyRaw(key)
|
||||
},
|
||||
nilSpec: func(key string) (ModuleSpec, bool) {
|
||||
var registry *MergerRegistry
|
||||
return registry.Spec(key)
|
||||
},
|
||||
nilRegisteredKey: func() []string {
|
||||
var registry *MergerRegistry
|
||||
return registry.RegisteredKeys()
|
||||
},
|
||||
constructor: func(key string) func() (contracts.LegacyRawMerger, error) {
|
||||
return func() (contracts.LegacyRawMerger, error) {
|
||||
return registryMerger{key: key}, nil
|
||||
}
|
||||
},
|
||||
moduleKey: func(module contracts.LegacyRawMerger) string {
|
||||
return module.Key()
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -9,14 +9,8 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
type LegacyRawNormalizerConstructor func() (contracts.LegacyRawNormalizer, error)
|
||||
type LegacyRawNormalizerBuilder func(BuildRequest) (contracts.LegacyRawNormalizer, error)
|
||||
|
||||
type NormalizerRegistry struct {
|
||||
legacyBuilders map[string]LegacyRawNormalizerBuilder
|
||||
legacyValidators map[string]OptionValidator
|
||||
legacySpecs map[string]ModuleSpec
|
||||
typedEntries map[artifactVariantKey]typedNormalizerEntry
|
||||
typedEntries map[artifactVariantKey]typedNormalizerEntry
|
||||
}
|
||||
|
||||
type typedNormalizerEntry struct {
|
||||
@@ -29,66 +23,15 @@ type typedNormalizerEntry struct {
|
||||
|
||||
func NewNormalizerRegistry() *NormalizerRegistry {
|
||||
return &NormalizerRegistry{
|
||||
legacyBuilders: make(map[string]LegacyRawNormalizerBuilder),
|
||||
legacyValidators: make(map[string]OptionValidator),
|
||||
legacySpecs: make(map[string]ModuleSpec),
|
||||
typedEntries: make(map[artifactVariantKey]typedNormalizerEntry),
|
||||
typedEntries: make(map[artifactVariantKey]typedNormalizerEntry),
|
||||
}
|
||||
}
|
||||
|
||||
func (r *NormalizerRegistry) RegisterLegacyRaw(key string, constructor LegacyRawNormalizerConstructor) error {
|
||||
return r.RegisterLegacyRawWithSpec(defaultModuleSpec(key, StageNormalize), constructor)
|
||||
}
|
||||
|
||||
func (r *NormalizerRegistry) RegisterLegacyRawWithSpec(spec ModuleSpec, constructor LegacyRawNormalizerConstructor) error {
|
||||
if constructor == nil {
|
||||
return fmt.Errorf("normalizer constructor for %q must not be nil", strings.TrimSpace(spec.Key))
|
||||
}
|
||||
return r.RegisterLegacyRawBuilderWithSpec(spec, allowLegacyOptions, func(BuildRequest) (contracts.LegacyRawNormalizer, error) {
|
||||
return constructor()
|
||||
})
|
||||
}
|
||||
|
||||
func (r *NormalizerRegistry) RegisterLegacyRawBuilderWithSpec(spec ModuleSpec, validateOptions OptionValidator, builder LegacyRawNormalizerBuilder) error {
|
||||
if r == nil {
|
||||
return fmt.Errorf("normalizer registry must not be nil")
|
||||
}
|
||||
normalizedSpec := normalizeModuleSpec(spec)
|
||||
if err := validateModuleSpec("normalizer", StageNormalize, normalizedSpec); err != nil {
|
||||
return err
|
||||
}
|
||||
if normalizedSpec.ArtifactKind != "" {
|
||||
return fmt.Errorf("legacy raw normalizer %q must not declare an artifact kind", normalizedSpec.Key)
|
||||
}
|
||||
if validateOptions == nil {
|
||||
return fmt.Errorf("normalizer option validator for %q must not be nil", normalizedSpec.Key)
|
||||
}
|
||||
if builder == nil {
|
||||
return fmt.Errorf("normalizer builder for %q must not be nil", normalizedSpec.Key)
|
||||
}
|
||||
if _, ok := r.legacyBuilders[normalizedSpec.Key]; ok {
|
||||
return fmt.Errorf("legacy raw normalizer %q is already registered", normalizedSpec.Key)
|
||||
}
|
||||
if r.legacyBuilders == nil {
|
||||
r.legacyBuilders = make(map[string]LegacyRawNormalizerBuilder)
|
||||
}
|
||||
if r.legacyValidators == nil {
|
||||
r.legacyValidators = make(map[string]OptionValidator)
|
||||
}
|
||||
if r.legacySpecs == nil {
|
||||
r.legacySpecs = make(map[string]ModuleSpec)
|
||||
}
|
||||
r.legacyBuilders[normalizedSpec.Key] = builder
|
||||
r.legacyValidators[normalizedSpec.Key] = validateOptions
|
||||
r.legacySpecs[normalizedSpec.Key] = cloneModuleSpec(normalizedSpec)
|
||||
return nil
|
||||
}
|
||||
|
||||
func RegisterNormalizer[T any](registry *NormalizerRegistry, spec ModuleSpec, constructor func() (contracts.Normalizer[T], error)) error {
|
||||
if constructor == nil {
|
||||
return fmt.Errorf("normalizer constructor for %q must not be nil", strings.TrimSpace(spec.Key))
|
||||
}
|
||||
return RegisterNormalizerBuilder(registry, spec, allowLegacyOptions, func(BuildRequest) (contracts.Normalizer[T], error) {
|
||||
return RegisterNormalizerBuilder(registry, spec, rejectUnconfiguredOptions, func(BuildRequest) (contracts.Normalizer[T], error) {
|
||||
return constructor()
|
||||
})
|
||||
}
|
||||
@@ -143,63 +86,29 @@ func RegisterNormalizerBuilder[T any](registry *NormalizerRegistry, spec ModuleS
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *NormalizerRegistry) BuildLegacyRaw(key string) (contracts.LegacyRawNormalizer, error) {
|
||||
return r.BuildLegacyRawWithRequest(key, BuildRequest{})
|
||||
}
|
||||
|
||||
func (r *NormalizerRegistry) BuildLegacyRawWithRequest(key string, request BuildRequest) (contracts.LegacyRawNormalizer, error) {
|
||||
if r == nil {
|
||||
return nil, fmt.Errorf("normalizer registry must not be nil")
|
||||
}
|
||||
normalizedKey := strings.TrimSpace(key)
|
||||
if normalizedKey == "" {
|
||||
return nil, fmt.Errorf("normalizer key must not be empty")
|
||||
}
|
||||
builder, ok := r.legacyBuilders[normalizedKey]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("legacy raw normalizer %q is not registered", normalizedKey)
|
||||
}
|
||||
normalizer, err := builder(cloneBuildRequest(request))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build normalizer %q: %w", normalizedKey, err)
|
||||
}
|
||||
if normalizer == nil {
|
||||
return nil, fmt.Errorf("normalizer %q constructor returned nil", normalizedKey)
|
||||
}
|
||||
if normalizer.Key() != normalizedKey {
|
||||
return nil, fmt.Errorf("normalizer %q returned key %q", normalizedKey, normalizer.Key())
|
||||
}
|
||||
return normalizer, nil
|
||||
}
|
||||
|
||||
func (r *NormalizerRegistry) validateOptions(key string, kind contracts.ArtifactKind, options map[string]any) error {
|
||||
if r == nil {
|
||||
return fmt.Errorf("normalizer registry must not be nil")
|
||||
}
|
||||
normalizedKey := strings.TrimSpace(key)
|
||||
if kind != "" {
|
||||
entry, ok := r.typedEntry(normalizedKey, kind)
|
||||
if !ok {
|
||||
return fmt.Errorf("normalizer %q variant for artifact kind %q is not registered", normalizedKey, kind)
|
||||
}
|
||||
return validateRegisteredOptions(entry.validateOptions, options)
|
||||
}
|
||||
validator, ok := r.legacyValidators[normalizedKey]
|
||||
entry, ok := r.typedEntry(normalizedKey, kind)
|
||||
if !ok {
|
||||
return fmt.Errorf("legacy raw normalizer %q is not registered", normalizedKey)
|
||||
return fmt.Errorf("normalizer %q variant for artifact kind %q is not registered", normalizedKey, kind)
|
||||
}
|
||||
return validateRegisteredOptions(validator, options)
|
||||
return validateRegisteredOptions(entry.validateOptions, options)
|
||||
}
|
||||
|
||||
func (r *NormalizerRegistry) Spec(key string) (ModuleSpec, bool) {
|
||||
if r == nil {
|
||||
return ModuleSpec{}, false
|
||||
}
|
||||
spec, ok := r.legacySpecs[strings.TrimSpace(key)]
|
||||
if !ok {
|
||||
return ModuleSpec{}, false
|
||||
module := strings.TrimSpace(key)
|
||||
for variant, entry := range r.typedEntries {
|
||||
if variant.module == module {
|
||||
return cloneModuleSpec(entry.spec), true
|
||||
}
|
||||
}
|
||||
return cloneModuleSpec(spec), true
|
||||
return ModuleSpec{}, false
|
||||
}
|
||||
|
||||
func (r *NormalizerRegistry) typedEntry(key string, kind contracts.ArtifactKind) (typedNormalizerEntry, bool) {
|
||||
@@ -229,10 +138,7 @@ func (r *NormalizerRegistry) RegisteredKeys() []string {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
keys := make(map[string]struct{}, len(r.legacySpecs)+len(r.typedEntries))
|
||||
for key := range r.legacySpecs {
|
||||
keys[key] = struct{}{}
|
||||
}
|
||||
keys := make(map[string]struct{}, len(r.typedEntries))
|
||||
for key := range r.typedEntries {
|
||||
keys[key.module] = struct{}{}
|
||||
}
|
||||
|
||||
@@ -1,58 +0,0 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func TestNormalizerRegistryBehavior(t *testing.T) {
|
||||
runRegistryBehaviorTests(t, registryBehaviorCase[contracts.LegacyRawNormalizer]{
|
||||
name: "NormalizerRegistry",
|
||||
key: "generic-normalizer",
|
||||
stage: StageNormalize,
|
||||
wrongStage: StageExtract,
|
||||
newRegistry: func() any {
|
||||
return NewNormalizerRegistry()
|
||||
},
|
||||
register: func(registry any, key string, constructor func() (contracts.LegacyRawNormalizer, error)) error {
|
||||
return registry.(*NormalizerRegistry).RegisterLegacyRaw(key, constructor)
|
||||
},
|
||||
registerWithSpec: func(registry any, spec ModuleSpec, constructor func() (contracts.LegacyRawNormalizer, error)) error {
|
||||
return registry.(*NormalizerRegistry).RegisterLegacyRawWithSpec(spec, constructor)
|
||||
},
|
||||
build: func(registry any, key string) (contracts.LegacyRawNormalizer, error) {
|
||||
return registry.(*NormalizerRegistry).BuildLegacyRaw(key)
|
||||
},
|
||||
spec: func(registry any, key string) (ModuleSpec, bool) {
|
||||
return registry.(*NormalizerRegistry).Spec(key)
|
||||
},
|
||||
registeredKeys: func(registry any) []string {
|
||||
return registry.(*NormalizerRegistry).RegisteredKeys()
|
||||
},
|
||||
nilRegister: func(key string, constructor func() (contracts.LegacyRawNormalizer, error)) error {
|
||||
var registry *NormalizerRegistry
|
||||
return registry.RegisterLegacyRaw(key, constructor)
|
||||
},
|
||||
nilBuild: func(key string) (contracts.LegacyRawNormalizer, error) {
|
||||
var registry *NormalizerRegistry
|
||||
return registry.BuildLegacyRaw(key)
|
||||
},
|
||||
nilSpec: func(key string) (ModuleSpec, bool) {
|
||||
var registry *NormalizerRegistry
|
||||
return registry.Spec(key)
|
||||
},
|
||||
nilRegisteredKey: func() []string {
|
||||
var registry *NormalizerRegistry
|
||||
return registry.RegisteredKeys()
|
||||
},
|
||||
constructor: func(key string) func() (contracts.LegacyRawNormalizer, error) {
|
||||
return func() (contracts.LegacyRawNormalizer, error) {
|
||||
return registryNormalizer{key: key}, nil
|
||||
}
|
||||
},
|
||||
moduleKey: func(module contracts.LegacyRawNormalizer) string {
|
||||
return module.Key()
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -32,7 +32,7 @@ func (r *OutputEncoderRegistry) RegisterWithSpec(spec ModuleSpec, constructor Ou
|
||||
if constructor == nil {
|
||||
return fmt.Errorf("output encoder constructor for %q must not be nil", strings.TrimSpace(spec.Key))
|
||||
}
|
||||
return r.RegisterBuilderWithSpec(spec, allowLegacyOptions, func(BuildRequest) (contracts.OutputEncoder, error) {
|
||||
return r.RegisterBuilderWithSpec(spec, rejectUnconfiguredOptions, func(BuildRequest) (contracts.OutputEncoder, error) {
|
||||
return constructor()
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
@@ -143,7 +145,7 @@ func constructionProfile() PipelineProfile {
|
||||
}
|
||||
}
|
||||
|
||||
func constructionRegistries(t *testing.T, built *[]string, failure *constructionFailure) (Registries, *runnerInputAdapter) {
|
||||
func constructionRegistries(t *testing.T, built *[]string, failure *constructionFailure) (Registries, *constructionInput) {
|
||||
t.Helper()
|
||||
if built == nil {
|
||||
built = &[]string{}
|
||||
@@ -153,48 +155,64 @@ func constructionRegistries(t *testing.T, built *[]string, failure *construction
|
||||
}
|
||||
record := func(name string) { *built = append(*built, name) }
|
||||
strict := func(options map[string]any) error { return RejectUnknownOptions(options, "known") }
|
||||
modules := defaultRunnerModules()
|
||||
input := &constructionInput{key: "input"}
|
||||
registries := Registries{
|
||||
Inputs: NewInputAdapterRegistry(), Chunkers: NewChunkerRegistry(), ArtifactCodecs: NewArtifactCodecRegistry(),
|
||||
Extractors: NewExtractorRegistry(), Mergers: NewMergerRegistry(), Normalizers: NewNormalizerRegistry(),
|
||||
Validators: NewValidatorRegistry(), ValidatorChains: NewValidatorChainRegistry(), Outputs: NewOutputEncoderRegistry(),
|
||||
}
|
||||
if err := RegisterArtifactCodec(registries.ArtifactCodecs, notesCodec()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := registries.Inputs.RegisterBuilderWithSpec(defaultModuleSpec("input", StageInput), strict, func(BuildRequest) (contracts.InputAdapter, error) {
|
||||
record("input")
|
||||
return modules.input, nil
|
||||
return input, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := registries.Chunkers.RegisterBuilderWithSpec(defaultModuleSpec("chunk", StageChunk), strict, func(BuildRequest) (contracts.Chunker, error) {
|
||||
record("chunk")
|
||||
return modules.chunker, nil
|
||||
return &typedTestChunker{key: "chunk"}, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := registries.Extractors.RegisterLegacyRawBuilderWithSpec(defaultModuleSpec("extract", StageExtract), strict, func(request BuildRequest) (contracts.LegacyRawExtractor, error) {
|
||||
extractSpec := defaultModuleSpec("extract", StageExtract)
|
||||
extractSpec.ArtifactKind = "test/notes"
|
||||
if err := RegisterExtractorBuilder(registries.Extractors, extractSpec, strict, func(request BuildRequest) (contracts.Extractor[codecNotes], error) {
|
||||
record("extract")
|
||||
if failure.requireExtractorLLM && request.Dependencies.LLM == nil {
|
||||
return nil, errors.New("structured LLM client is required")
|
||||
}
|
||||
return &runnerExtractor{key: "extract"}, nil
|
||||
return typedTestExtractor[codecNotes]{key: "extract"}, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := registries.Mergers.RegisterLegacyRawBuilderWithSpec(defaultModuleSpec("merge", StageMerge), strict, func(BuildRequest) (contracts.LegacyRawMerger, error) {
|
||||
mergeSpec := defaultModuleSpec("merge", StageMerge)
|
||||
mergeSpec.ArtifactKind = "test/notes"
|
||||
if err := RegisterMergerBuilder(registries.Mergers, mergeSpec, strict, func(BuildRequest) (contracts.Merger[codecNotes], error) {
|
||||
record("merge")
|
||||
return modules.mergers["merge"], nil
|
||||
return typedTestMerger[codecNotes]{key: "merge"}, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := registries.Normalizers.RegisterLegacyRawBuilderWithSpec(defaultModuleSpec("normalize", StageNormalize), strict, func(BuildRequest) (contracts.LegacyRawNormalizer, error) {
|
||||
normalizeSpec := defaultModuleSpec("normalize", StageNormalize)
|
||||
normalizeSpec.ArtifactKind = "test/notes"
|
||||
if err := RegisterNormalizerBuilder(registries.Normalizers, normalizeSpec, strict, func(BuildRequest) (contracts.Normalizer[codecNotes], error) {
|
||||
record("normalize")
|
||||
return modules.normalizers["normalize"], nil
|
||||
return typedTestNormalizer[codecNotes]{key: "normalize"}, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := registries.Validators.RegisterLegacyRawBuilderWithSpec(ValidatorSpec{Key: "configured", ExecutionClass: contracts.ExecutionClassDeterministic}, strict, func(BuildRequest) (contracts.LegacyRawValidator, error) {
|
||||
validatorSpec := ValidatorSpec{Key: "configured", ExecutionClass: contracts.ExecutionClassDeterministic}
|
||||
if err := RegisterChunkValidatorBuilder(registries.Validators, validatorSpec, strict, func(BuildRequest) (contracts.ChunkValidator, error) {
|
||||
record("validator")
|
||||
return modules.validators["configured"], nil
|
||||
return typedTestChunkValidator{key: "configured"}, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := RegisterTypedValidatorBuilder(registries.Validators, "test/notes", validatorSpec, strict, func(BuildRequest) (contracts.TypedValidator[codecNotes], error) {
|
||||
record("validator")
|
||||
return typedTestValidator[codecNotes]{key: "configured"}, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -203,9 +221,20 @@ func constructionRegistries(t *testing.T, built *[]string, failure *construction
|
||||
if failure.output != nil {
|
||||
return nil, failure.output
|
||||
}
|
||||
return modules.output, nil
|
||||
return &typedTestOutput{key: "output"}, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return registries, modules.input
|
||||
return registries, input
|
||||
}
|
||||
|
||||
type constructionInput struct {
|
||||
key string
|
||||
requests []contracts.ParseRequest
|
||||
}
|
||||
|
||||
func (input *constructionInput) Key() string { return input.key }
|
||||
func (input *constructionInput) Parse(_ context.Context, request contracts.ParseRequest) (*source.SourceDocument, error) {
|
||||
input.requests = append(input.requests, request)
|
||||
return typedTestDocument(), nil
|
||||
}
|
||||
|
||||
@@ -32,19 +32,12 @@ type PreparedArtifactLane struct {
|
||||
|
||||
type preparedLaneExecutor struct {
|
||||
resolved ResolvedArtifactLane
|
||||
legacy *preparedLegacyLane
|
||||
typed *preparedTypedLane
|
||||
extractValidators preparedValidatorChain
|
||||
mergeValidators preparedValidatorChain
|
||||
normalizeValidators preparedValidatorChain
|
||||
}
|
||||
|
||||
type preparedLegacyLane struct {
|
||||
extractor contracts.LegacyRawExtractor
|
||||
merger contracts.LegacyRawMerger
|
||||
normalizer contracts.LegacyRawNormalizer
|
||||
}
|
||||
|
||||
type preparedTypedLane struct {
|
||||
extractor any
|
||||
merger any
|
||||
@@ -62,7 +55,6 @@ type preparedValidatorChain struct {
|
||||
|
||||
type preparedValidator struct {
|
||||
resolved ResolvedValidator
|
||||
legacy contracts.LegacyRawValidator
|
||||
typed any
|
||||
typedValidate typedValidateOperation
|
||||
chunk contracts.ChunkValidator
|
||||
@@ -130,75 +122,50 @@ func prepareLane(pipeline ResolvedPipeline, lane ResolvedArtifactLane, registrie
|
||||
request := func(binding ModuleBinding) BuildRequest {
|
||||
return BuildRequest{Dependencies: deps, Options: cloneOptions(binding.Options)}
|
||||
}
|
||||
if lane.ArtifactKind == "" {
|
||||
extractor, err := registries.Extractors.BuildLegacyRawWithRequest(lane.Extract.Module, request(lane.Extract))
|
||||
if err != nil {
|
||||
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageExtract, lane.Extract.Module, "", err)
|
||||
}
|
||||
executor.legacy = &preparedLegacyLane{extractor: extractor}
|
||||
} else {
|
||||
entry, ok := registries.Extractors.typedEntry(lane.Extract.Module)
|
||||
if !ok {
|
||||
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageExtract, lane.Extract.Module, "", fmt.Errorf("typed construction entry is not registered"))
|
||||
}
|
||||
module, err := buildErasedModule(entry.builder, request(lane.Extract), lane.Extract.Module, "extractor")
|
||||
if err != nil {
|
||||
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageExtract, lane.Extract.Module, "", err)
|
||||
}
|
||||
codec, _, codecErr := registries.ArtifactCodecs.entry(lane.ArtifactKind)
|
||||
if codecErr != nil {
|
||||
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageExtract, lane.Extract.Module, "", codecErr)
|
||||
}
|
||||
executor.typed = &preparedTypedLane{extractor: module, extract: entry.extract, codec: codec}
|
||||
extractEntry, ok := registries.Extractors.typedEntry(lane.Extract.Module)
|
||||
if !ok {
|
||||
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageExtract, lane.Extract.Module, "", fmt.Errorf("typed construction entry is not registered"))
|
||||
}
|
||||
module, err := buildErasedModule(extractEntry.builder, request(lane.Extract), lane.Extract.Module, "extractor")
|
||||
if err != nil {
|
||||
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageExtract, lane.Extract.Module, "", err)
|
||||
}
|
||||
codec, _, codecErr := registries.ArtifactCodecs.entry(lane.ArtifactKind)
|
||||
if codecErr != nil {
|
||||
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageExtract, lane.Extract.Module, "", codecErr)
|
||||
}
|
||||
executor.typed = &preparedTypedLane{extractor: module, extract: extractEntry.extract, codec: codec}
|
||||
|
||||
var err error
|
||||
executor.extractValidators, err = prepareValidatorChain(pipeline, registries, deps, StageExtract, lane.ID, lane.Extract.Module)
|
||||
if err != nil {
|
||||
return preparedLaneExecutor{}, err
|
||||
}
|
||||
|
||||
if lane.ArtifactKind == "" {
|
||||
module, err := registries.Mergers.BuildLegacyRawWithRequest(lane.Merge.Module, request(lane.Merge))
|
||||
if err != nil {
|
||||
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageMerge, lane.Merge.Module, "", err)
|
||||
}
|
||||
executor.legacy.merger = module
|
||||
} else {
|
||||
entry, ok := registries.Mergers.typedEntry(lane.Merge.Module, lane.ArtifactKind)
|
||||
if !ok {
|
||||
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageMerge, lane.Merge.Module, "", fmt.Errorf("typed construction entry is not registered"))
|
||||
}
|
||||
module, err := buildErasedModule(entry.builder, request(lane.Merge), lane.Merge.Module, "merger")
|
||||
if err != nil {
|
||||
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageMerge, lane.Merge.Module, "", err)
|
||||
}
|
||||
executor.typed.merger = module
|
||||
executor.typed.merge = entry.merge
|
||||
mergeEntry, ok := registries.Mergers.typedEntry(lane.Merge.Module, lane.ArtifactKind)
|
||||
if !ok {
|
||||
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageMerge, lane.Merge.Module, "", fmt.Errorf("typed construction entry is not registered"))
|
||||
}
|
||||
module, err = buildErasedModule(mergeEntry.builder, request(lane.Merge), lane.Merge.Module, "merger")
|
||||
if err != nil {
|
||||
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageMerge, lane.Merge.Module, "", err)
|
||||
}
|
||||
executor.typed.merger = module
|
||||
executor.typed.merge = mergeEntry.merge
|
||||
executor.mergeValidators, err = prepareValidatorChain(pipeline, registries, deps, StageMerge, lane.ID, lane.Merge.Module)
|
||||
if err != nil {
|
||||
return preparedLaneExecutor{}, err
|
||||
}
|
||||
|
||||
if lane.ArtifactKind == "" {
|
||||
module, err := registries.Normalizers.BuildLegacyRawWithRequest(lane.Normalize.Module, request(lane.Normalize))
|
||||
if err != nil {
|
||||
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageNormalize, lane.Normalize.Module, "", err)
|
||||
}
|
||||
executor.legacy.normalizer = module
|
||||
} else {
|
||||
entry, ok := registries.Normalizers.typedEntry(lane.Normalize.Module, lane.ArtifactKind)
|
||||
if !ok {
|
||||
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageNormalize, lane.Normalize.Module, "", fmt.Errorf("typed construction entry is not registered"))
|
||||
}
|
||||
module, err := buildErasedModule(entry.builder, request(lane.Normalize), lane.Normalize.Module, "normalizer")
|
||||
if err != nil {
|
||||
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageNormalize, lane.Normalize.Module, "", err)
|
||||
}
|
||||
executor.typed.normalizer = module
|
||||
executor.typed.normalize = entry.normalize
|
||||
normalizeEntry, ok := registries.Normalizers.typedEntry(lane.Normalize.Module, lane.ArtifactKind)
|
||||
if !ok {
|
||||
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageNormalize, lane.Normalize.Module, "", fmt.Errorf("typed construction entry is not registered"))
|
||||
}
|
||||
module, err = buildErasedModule(normalizeEntry.builder, request(lane.Normalize), lane.Normalize.Module, "normalizer")
|
||||
if err != nil {
|
||||
return preparedLaneExecutor{}, constructionError(pipeline.ID, lane.ID, StageNormalize, lane.Normalize.Module, "", err)
|
||||
}
|
||||
executor.typed.normalizer = module
|
||||
executor.typed.normalize = normalizeEntry.normalize
|
||||
executor.normalizeValidators, err = prepareValidatorChain(pipeline, registries, deps, StageNormalize, lane.ID, lane.Normalize.Module)
|
||||
if err != nil {
|
||||
return preparedLaneExecutor{}, err
|
||||
@@ -249,8 +216,7 @@ func buildPreparedValidator(registry *ValidatorRegistry, resolved ResolvedValida
|
||||
prepared.serialized, err = entry.builder(cloneBuildRequest(request))
|
||||
implementation = prepared.serialized
|
||||
default:
|
||||
prepared.legacy, err = registry.BuildLegacyRawWithRequest(key, request)
|
||||
implementation = prepared.legacy
|
||||
return preparedValidator{}, fmt.Errorf("validator construction target %q is not supported", resolved.Target)
|
||||
}
|
||||
if err != nil {
|
||||
return preparedValidator{}, err
|
||||
|
||||
@@ -384,7 +384,7 @@ func configuredValidatorsError(pipelineID string, laneID string) error {
|
||||
|
||||
func resolveArtifactIdentity(pipelineID, laneID string, lane *ResolvedArtifactLane, extractSpec ModuleSpec, catalog ModuleCatalog) (reflect.Type, error) {
|
||||
if extractSpec.ArtifactKind == "" {
|
||||
return nil, nil
|
||||
return nil, fmt.Errorf("pipeline %q lane %q extract module %q does not declare an artifact kind", pipelineID, laneID, lane.Extract.Module)
|
||||
}
|
||||
if catalog.Extractors == nil {
|
||||
return nil, fmt.Errorf("pipeline %q lane %q extractor registry must not be nil", pipelineID, laneID)
|
||||
@@ -393,9 +393,6 @@ func resolveArtifactIdentity(pipelineID, laneID string, lane *ResolvedArtifactLa
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("pipeline %q lane %q extract module %q declares artifact kind %q without a typed registration", pipelineID, laneID, lane.Extract.Module, extractSpec.ArtifactKind)
|
||||
}
|
||||
if catalog.Extractors.usesRawAdapter(lane.Extract.Module) {
|
||||
return nil, nil
|
||||
}
|
||||
if catalog.ArtifactCodecs == nil {
|
||||
return nil, fmt.Errorf("pipeline %q lane %q artifact codec registry must not be nil for kind %q", pipelineID, laneID, extractSpec.ArtifactKind)
|
||||
}
|
||||
@@ -470,7 +467,7 @@ func validatorSpecForTarget(registry *ValidatorRegistry, stage ModuleStage, key
|
||||
if spec, ok := registry.Spec(key); ok {
|
||||
return spec, "", nil
|
||||
}
|
||||
return ValidatorSpec{}, "", fmt.Errorf("references unknown validator %q on legacy raw path", key)
|
||||
return ValidatorSpec{}, "", fmt.Errorf("references unknown validator %q without an artifact kind", key)
|
||||
}
|
||||
if entry, ok := registry.typedEntry(key, kind); ok {
|
||||
if entry.valueType != expectedType {
|
||||
@@ -646,9 +643,6 @@ func validatePipelineReferenceDefaults(
|
||||
merge := resolveBinding(laneProfile.Merge, DefaultMergeModule)
|
||||
var artifactType reflect.Type
|
||||
artifactKind := extractSpec.ArtifactKind
|
||||
if catalog.Extractors != nil && catalog.Extractors.usesRawAdapter(extract.Module) {
|
||||
artifactKind = ""
|
||||
}
|
||||
if artifactKind != "" && catalog.Extractors != nil {
|
||||
if entry, ok := catalog.Extractors.typedEntry(extract.Module); ok {
|
||||
artifactType = entry.valueType
|
||||
|
||||
@@ -766,20 +766,22 @@ func TestResolvePipelineUsesReferenceSlotsFromSpecWithoutConstructingExtractor(t
|
||||
profile := baselineProfile()
|
||||
profile.References = map[string]string{"roster": "./roster.yml"}
|
||||
catalog := emptyProfileCatalog()
|
||||
mustRegisterArtifactCodec(t, catalog.ArtifactCodecs, notesCodec())
|
||||
for _, spec := range defaultProfileSpecs() {
|
||||
if spec.Key != "event-extractor" {
|
||||
registerProfileSpecs(t, catalog, spec)
|
||||
}
|
||||
}
|
||||
if err := catalog.Extractors.RegisterLegacyRawWithSpec(ModuleSpec{
|
||||
Key: "event-extractor",
|
||||
Stage: StageExtract,
|
||||
Requires: []string{"chunk"},
|
||||
Provides: []string{"candidate"},
|
||||
if err := RegisterExtractor[codecNotes](catalog.Extractors, ModuleSpec{
|
||||
Key: "event-extractor",
|
||||
Stage: StageExtract,
|
||||
ArtifactKind: "test/notes",
|
||||
Requires: []string{"chunk"},
|
||||
Provides: []string{"candidate"},
|
||||
ReferenceSlots: []contracts.ReferenceSlot{
|
||||
{Name: "roster", Required: true},
|
||||
},
|
||||
}, func() (contracts.LegacyRawExtractor, error) {
|
||||
}, func() (contracts.Extractor[codecNotes], error) {
|
||||
return nil, errors.New("constructor should not run")
|
||||
}); err != nil {
|
||||
t.Fatalf("RegisterWithSpec() error = %v, want nil", err)
|
||||
@@ -1177,6 +1179,7 @@ func newProfileCatalog(t *testing.T) ModuleCatalog {
|
||||
t.Helper()
|
||||
|
||||
catalog := emptyProfileCatalog()
|
||||
mustRegisterArtifactCodec(t, catalog.ArtifactCodecs, notesCodec())
|
||||
registerProfileSpecs(t, catalog, defaultProfileSpecs()...)
|
||||
return catalog
|
||||
}
|
||||
@@ -1192,6 +1195,9 @@ func newProfileCatalogWithOverrides(t *testing.T, overrides ...ModuleSpec) Modul
|
||||
|
||||
specs := defaultProfileSpecs()
|
||||
for _, override := range overrides {
|
||||
if override.ArtifactKind == "" && (override.Stage == StageExtract || override.Stage == StageMerge || override.Stage == StageNormalize) {
|
||||
override.ArtifactKind = "test/notes"
|
||||
}
|
||||
replaced := false
|
||||
for index, spec := range specs {
|
||||
if spec.Stage == override.Stage && spec.Key == override.Key {
|
||||
@@ -1206,6 +1212,7 @@ func newProfileCatalogWithOverrides(t *testing.T, overrides ...ModuleSpec) Modul
|
||||
}
|
||||
|
||||
catalog := emptyProfileCatalog()
|
||||
mustRegisterArtifactCodec(t, catalog.ArtifactCodecs, notesCodec())
|
||||
registerProfileSpecs(t, catalog, specs...)
|
||||
return catalog
|
||||
}
|
||||
@@ -1228,10 +1235,10 @@ func defaultProfileSpecs() []ModuleSpec {
|
||||
return []ModuleSpec{
|
||||
ModuleSpec{Key: "text", Stage: StageInput, Provides: []string{"source"}},
|
||||
ModuleSpec{Key: "generic", Stage: StageChunk, Requires: []string{"source"}, Provides: []string{"chunk"}},
|
||||
ModuleSpec{Key: "event-extractor", Stage: StageExtract, Requires: []string{"chunk"}, Provides: []string{"candidate"}},
|
||||
ModuleSpec{Key: "note-extractor", Stage: StageExtract, Requires: []string{"chunk"}, Provides: []string{"candidate"}},
|
||||
ModuleSpec{Key: "appendorder", Stage: StageMerge, Requires: []string{"candidate"}, Provides: []string{"merged"}},
|
||||
ModuleSpec{Key: "noop", Stage: StageNormalize, Requires: []string{"merged"}, Provides: []string{"normalized"}},
|
||||
ModuleSpec{Key: "event-extractor", Stage: StageExtract, ArtifactKind: "test/notes", Requires: []string{"chunk"}, Provides: []string{"candidate"}},
|
||||
ModuleSpec{Key: "note-extractor", Stage: StageExtract, ArtifactKind: "test/notes", Requires: []string{"chunk"}, Provides: []string{"candidate"}},
|
||||
ModuleSpec{Key: "appendorder", Stage: StageMerge, ArtifactKind: "test/notes", Requires: []string{"candidate"}, Provides: []string{"merged"}},
|
||||
ModuleSpec{Key: "noop", Stage: StageNormalize, ArtifactKind: "test/notes", Requires: []string{"merged"}, Provides: []string{"normalized"}},
|
||||
ModuleSpec{Key: "grounded", Stage: StageValidate, Requires: []string{"normalized"}, Provides: []string{"validated"}},
|
||||
ModuleSpec{Key: "json", Stage: StageOutput, Requires: []string{"normalized"}, Provides: []string{"encoded"}},
|
||||
}
|
||||
@@ -1241,30 +1248,40 @@ func registerProfileSpecs(t *testing.T, catalog ModuleCatalog, specs ...ModuleSp
|
||||
t.Helper()
|
||||
|
||||
for _, spec := range specs {
|
||||
if spec.ArtifactKind == "" && (spec.Stage == StageExtract || spec.Stage == StageMerge || spec.Stage == StageNormalize) {
|
||||
spec.ArtifactKind = "test/notes"
|
||||
}
|
||||
switch spec.Stage {
|
||||
case StageInput:
|
||||
if err := catalog.Inputs.RegisterWithSpec(spec, profileInputConstructor(spec.Key)); err != nil {
|
||||
t.Fatalf("register input spec %#v: %v", spec, err)
|
||||
}
|
||||
case StageChunk:
|
||||
if err := catalog.Chunkers.RegisterWithSpec(spec, profileChunkerConstructor(spec.Key)); err != nil {
|
||||
validateOptions := func(options map[string]any) error { return RejectUnknownOptions(options, "a", "b", "size") }
|
||||
if err := catalog.Chunkers.RegisterBuilderWithSpec(spec, validateOptions, func(BuildRequest) (contracts.Chunker, error) { return &typedTestChunker{key: spec.Key}, nil }); err != nil {
|
||||
t.Fatalf("register chunk spec %#v: %v", spec, err)
|
||||
}
|
||||
case StageExtract:
|
||||
if err := catalog.Extractors.RegisterLegacyRawWithSpec(spec, profileExtractorConstructor(spec.Key)); err != nil {
|
||||
if err := RegisterExtractor(catalog.Extractors, spec, func() (contracts.Extractor[codecNotes], error) {
|
||||
return typedTestExtractor[codecNotes]{key: spec.Key}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register extractor spec %#v: %v", spec, err)
|
||||
}
|
||||
case StageMerge:
|
||||
if err := catalog.Mergers.RegisterLegacyRawWithSpec(spec, profileMergerConstructor(spec.Key)); err != nil {
|
||||
if err := RegisterMerger(catalog.Mergers, spec, func() (contracts.Merger[codecNotes], error) { return typedTestMerger[codecNotes]{key: spec.Key}, nil }); err != nil {
|
||||
t.Fatalf("register merger spec %#v: %v", spec, err)
|
||||
}
|
||||
case StageNormalize:
|
||||
if err := catalog.Normalizers.RegisterLegacyRawWithSpec(spec, profileNormalizerConstructor(spec.Key)); err != nil {
|
||||
if err := RegisterNormalizer(catalog.Normalizers, spec, func() (contracts.Normalizer[codecNotes], error) {
|
||||
return typedTestNormalizer[codecNotes]{key: spec.Key}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register normalizer spec %#v: %v", spec, err)
|
||||
}
|
||||
case StageValidate:
|
||||
validatorSpec := ValidatorSpec{Key: spec.Key, ExecutionClass: contracts.ExecutionClassDeterministic}
|
||||
if err := catalog.Validators.RegisterLegacyRawWithSpec(validatorSpec, profileValidatorConstructor(spec.Key)); err != nil {
|
||||
if err := RegisterTypedValidator(catalog.Validators, "test/notes", validatorSpec, func() (contracts.TypedValidator[codecNotes], error) {
|
||||
return typedTestValidator[codecNotes]{key: spec.Key}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register validator spec %#v: %v", spec, err)
|
||||
}
|
||||
case StageOutput:
|
||||
@@ -1279,7 +1296,9 @@ func registerProfileSpecs(t *testing.T, catalog ModuleCatalog, specs ...ModuleSp
|
||||
|
||||
func registerProfileValidatorSpec(t *testing.T, catalog ModuleCatalog, spec ValidatorSpec) {
|
||||
t.Helper()
|
||||
if err := catalog.Validators.RegisterLegacyRawWithSpec(spec, profileValidatorConstructor(spec.Key)); err != nil {
|
||||
if err := RegisterTypedValidator(catalog.Validators, "test/notes", spec, func() (contracts.TypedValidator[codecNotes], error) {
|
||||
return typedTestValidator[codecNotes]{key: spec.Key}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register validator spec %#v: %v", spec, err)
|
||||
}
|
||||
}
|
||||
@@ -1302,36 +1321,6 @@ func (adapter profileInputAdapter) Parse(ctx context.Context, req contracts.Pars
|
||||
return &source.SourceDocument{}, nil
|
||||
}
|
||||
|
||||
func profileChunkerConstructor(key string) ChunkerConstructor {
|
||||
return func() (contracts.Chunker, error) {
|
||||
return registryChunker{key: key}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func profileExtractorConstructor(key string) LegacyRawExtractorConstructor {
|
||||
return func() (contracts.LegacyRawExtractor, error) {
|
||||
return registryFakeExtractor{key: key}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func profileMergerConstructor(key string) LegacyRawMergerConstructor {
|
||||
return func() (contracts.LegacyRawMerger, error) {
|
||||
return registryMerger{key: key}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func profileNormalizerConstructor(key string) LegacyRawNormalizerConstructor {
|
||||
return func() (contracts.LegacyRawNormalizer, error) {
|
||||
return registryNormalizer{key: key}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func profileValidatorConstructor(key string) LegacyRawValidatorConstructor {
|
||||
return func() (contracts.LegacyRawValidator, error) {
|
||||
return registryValidator{name: key}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func profileOutputConstructor(key string) OutputEncoderConstructor {
|
||||
return func() (contracts.OutputEncoder, error) {
|
||||
return registryOutputEncoder{key: key}, nil
|
||||
|
||||
14
internal/framework/pipeline/reference_test_helpers_test.go
Normal file
14
internal/framework/pipeline/reference_test_helpers_test.go
Normal file
@@ -0,0 +1,14 @@
|
||||
package pipeline
|
||||
|
||||
import "gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
|
||||
func validSourceDocument() *source.SourceDocument {
|
||||
return &source.SourceDocument{
|
||||
ID: "source-1", Kind: "document", Format: "text/plain", Digest: "sha256:source",
|
||||
Units: []source.SourceUnit{
|
||||
{ID: 1, Kind: "unit", Text: "Source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}},
|
||||
{ID: 2, Kind: "unit", Text: "Second source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 2, EndUnitID: 2}},
|
||||
{ID: 3, Kind: "unit", Text: "Third source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 3, EndUnitID: 3}},
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -1,265 +0,0 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func TestRunnerUsesRegistries(t *testing.T) {
|
||||
var built []string
|
||||
var executed []string
|
||||
registries := integrationRegistries(t, &built, &executed)
|
||||
|
||||
output, err := newPreparedRunner(t, registries).Run(context.Background(), RunInput{
|
||||
pipeline: integrationPipeline(),
|
||||
SourceID: "source-1",
|
||||
RawInput: []byte("source text"),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
wantBuilt := []string{"input", "chunk", "extract-first", "merge", "normalize", "extract-second", "merge", "normalize", "output"}
|
||||
if !reflect.DeepEqual(built, wantBuilt) {
|
||||
t.Fatalf("built = %#v, want %#v", built, wantBuilt)
|
||||
}
|
||||
if !reflect.DeepEqual(executed, []string{"extract-first:chunk-0", "extract-second:chunk-0"}) {
|
||||
t.Fatalf("executed = %#v, want extractor chunk execution", executed)
|
||||
}
|
||||
if got := normalizeOutputKeys(output.NormalizeOutputs); !reflect.DeepEqual(got, []string{"normalize", "normalize"}) {
|
||||
t.Fatalf("normalize output keys = %#v, want one output from each lane", got)
|
||||
}
|
||||
if len(output.Rejected) != 0 {
|
||||
t.Fatalf("len(Rejected) = %d, want none", len(output.Rejected))
|
||||
}
|
||||
}
|
||||
|
||||
func integrationRegistries(t *testing.T, built, executed *[]string) Registries {
|
||||
t.Helper()
|
||||
|
||||
registries := Registries{
|
||||
Inputs: NewInputAdapterRegistry(),
|
||||
Chunkers: NewChunkerRegistry(),
|
||||
ArtifactCodecs: NewArtifactCodecRegistry(),
|
||||
Extractors: NewExtractorRegistry(),
|
||||
Mergers: NewMergerRegistry(),
|
||||
Normalizers: NewNormalizerRegistry(),
|
||||
Outputs: NewOutputEncoderRegistry(),
|
||||
}
|
||||
if err := registries.Inputs.Register("input", func() (contracts.InputAdapter, error) {
|
||||
*built = append(*built, "input")
|
||||
return integrationInput{}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register input: %v", err)
|
||||
}
|
||||
if err := registries.Chunkers.Register("chunk", func() (contracts.Chunker, error) {
|
||||
*built = append(*built, "chunk")
|
||||
return integrationChunker{}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register chunker: %v", err)
|
||||
}
|
||||
registerIntegrationExtractor(t, registries.Extractors, "extract-first", built, executed)
|
||||
registerIntegrationExtractor(t, registries.Extractors, "extract-second", built, executed)
|
||||
if err := registries.Mergers.RegisterLegacyRaw("merge", func() (contracts.LegacyRawMerger, error) {
|
||||
*built = append(*built, "merge")
|
||||
return integrationMerger{}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register merger: %v", err)
|
||||
}
|
||||
if err := registries.Normalizers.RegisterLegacyRaw("normalize", func() (contracts.LegacyRawNormalizer, error) {
|
||||
*built = append(*built, "normalize")
|
||||
return integrationNormalizer{}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register normalizer: %v", err)
|
||||
}
|
||||
if err := registries.Outputs.Register("output", func() (contracts.OutputEncoder, error) {
|
||||
*built = append(*built, "output")
|
||||
return integrationOutput{}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register output: %v", err)
|
||||
}
|
||||
return registries
|
||||
}
|
||||
|
||||
func registerIntegrationExtractor(t *testing.T, registry *ExtractorRegistry, key string, built, executed *[]string) {
|
||||
t.Helper()
|
||||
|
||||
if err := registry.RegisterLegacyRaw(key, func() (contracts.LegacyRawExtractor, error) {
|
||||
*built = append(*built, key)
|
||||
return integrationExtractor{key: key, executed: executed}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("Register(%q) error = %v, want nil", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
type integrationInput struct{}
|
||||
|
||||
func (input integrationInput) Key() string {
|
||||
return "input"
|
||||
}
|
||||
|
||||
func (input integrationInput) Parse(ctx context.Context, req contracts.ParseRequest) (*source.SourceDocument, error) {
|
||||
return integrationSourceDocument(), nil
|
||||
}
|
||||
|
||||
type integrationChunker struct{}
|
||||
|
||||
func (chunker integrationChunker) Key() string {
|
||||
return "chunk"
|
||||
}
|
||||
|
||||
func (chunker integrationChunker) ReferenceSlots() []contracts.ReferenceSlot {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (chunker integrationChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
||||
return contracts.ChunkResult{
|
||||
Chunks: []source.Chunk{
|
||||
{
|
||||
ID: "chunk-0",
|
||||
SourceID: req.Source.ID,
|
||||
Index: 0,
|
||||
Ref: source.SourceRef{
|
||||
SourceID: req.Source.ID,
|
||||
StartUnitID: req.Source.Units[0].ID,
|
||||
EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID,
|
||||
},
|
||||
Content: []byte(`{"units":[1]}`),
|
||||
MediaType: "application/json",
|
||||
Units: req.Source.Units,
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
type integrationExtractor struct {
|
||||
key string
|
||||
executed *[]string
|
||||
}
|
||||
|
||||
func (extractor integrationExtractor) Key() string {
|
||||
return extractor.key
|
||||
}
|
||||
|
||||
func (extractor integrationExtractor) ReferenceSlots() []contracts.ReferenceSlot {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (extractor integrationExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
|
||||
*extractor.executed = append(*extractor.executed, extractor.key+":"+req.Chunk.ID)
|
||||
return contracts.ExtractionResult{
|
||||
Output: contracts.ExtractOutput{
|
||||
Schema: contracts.ResponseSchema{ID: "integration", Name: "integration", Version: "v1"},
|
||||
Payload: contracts.RawPayload{
|
||||
Content: []byte(`{"value":true}`),
|
||||
MediaType: "application/json",
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
type integrationNormalizer struct{}
|
||||
|
||||
type integrationMerger struct{}
|
||||
|
||||
func (merger integrationMerger) Key() string {
|
||||
return "merge"
|
||||
}
|
||||
|
||||
func (merger integrationMerger) Merge(ctx context.Context, req contracts.MergeRequest) (contracts.MergeResult, error) {
|
||||
output := contracts.MergeOutput{
|
||||
LaneID: req.LaneID,
|
||||
Schema: contracts.ResponseSchema{ID: "integration", Name: "integration", Version: "v1"},
|
||||
Payload: contracts.RawPayload{
|
||||
Content: []byte(`{"merged":true}`),
|
||||
MediaType: "application/json",
|
||||
},
|
||||
}
|
||||
if len(req.ExtractOutputs) > 0 {
|
||||
output.SourceID = req.ExtractOutputs[0].SourceID
|
||||
output.Schema = req.ExtractOutputs[0].Schema
|
||||
output.Payload = req.ExtractOutputs[0].Payload
|
||||
}
|
||||
return contracts.MergeResult{Output: output}, nil
|
||||
}
|
||||
|
||||
func (normalizer integrationNormalizer) Key() string {
|
||||
return "normalize"
|
||||
}
|
||||
|
||||
func (normalizer integrationNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (normalizer integrationNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
|
||||
return contracts.NormalizeResult{
|
||||
Output: contracts.NormalizeOutput{
|
||||
LaneID: req.LaneID,
|
||||
SourceID: req.MergeOutput.SourceID,
|
||||
Schema: req.MergeOutput.Schema,
|
||||
Payload: req.MergeOutput.Payload,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
type integrationOutput struct{}
|
||||
|
||||
func (output integrationOutput) Key() string {
|
||||
return "output"
|
||||
}
|
||||
|
||||
func (output integrationOutput) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
|
||||
return contracts.OutputResult{
|
||||
Files: []contracts.OutputFile{
|
||||
{Name: "output.json", ContentType: "application/json", Bytes: []byte(`{}`)},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func integrationPipeline() ResolvedPipeline {
|
||||
return ResolvedPipeline{
|
||||
ID: "pipeline-1",
|
||||
Digest: "sha256:pipeline",
|
||||
Input: Binding("input"),
|
||||
Chunk: Binding("chunk"),
|
||||
ArtifactLanes: []ResolvedArtifactLane{
|
||||
{
|
||||
ID: "first",
|
||||
Extract: Binding("extract-first"),
|
||||
Merge: Binding("merge"),
|
||||
Normalize: Binding("normalize"),
|
||||
},
|
||||
{
|
||||
ID: "second",
|
||||
Extract: Binding("extract-second"),
|
||||
Merge: Binding("merge"),
|
||||
Normalize: Binding("normalize"),
|
||||
},
|
||||
},
|
||||
Output: Binding("output"),
|
||||
}
|
||||
}
|
||||
|
||||
func integrationSourceDocument() *source.SourceDocument {
|
||||
return &source.SourceDocument{
|
||||
ID: "source-1",
|
||||
Kind: "document",
|
||||
Format: "text/plain",
|
||||
Digest: "sha256:abc123",
|
||||
Units: []source.SourceUnit{
|
||||
{ID: 1, Kind: "unit", Text: "Source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeOutputKeys(outputs []contracts.SerializedOutput) []string {
|
||||
keys := make([]string, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
keys = append(keys, output.NormalizerKey)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
@@ -245,7 +245,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
|
||||
}, llmScope))
|
||||
return false, nil, err
|
||||
}
|
||||
validationWarnings, rejection, err := r.validateChunks(attemptCtx, doc, chunker.Key(), chunks, sourceInput, sessionID, input.pipeline.ChunkReferences.ReferenceSet, input.llmClient, input.Metadata, input.Prepared.chunkValidators, attempt, input.Debug)
|
||||
validationWarnings, rejection, err := r.validateChunks(attemptCtx, doc, chunker.Key(), chunks, sourceInput, sessionID, input.pipeline.ChunkReferences.ReferenceSet, input.Metadata, input.Prepared.chunkValidators, attempt, input.Debug)
|
||||
if err != nil || rejection != nil {
|
||||
_ = writeDebugTimed(debugRecorder, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
|
||||
Stage: string(StageChunk),
|
||||
@@ -323,7 +323,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
|
||||
} else {
|
||||
output.Manifest.ValidationStatus = "approved"
|
||||
}
|
||||
populateRawOutputManifest(&output)
|
||||
populateOutputManifest(&output)
|
||||
output.Manifest.CompletedAt = timePtr(time.Now().UTC())
|
||||
|
||||
encoder := input.Prepared.output
|
||||
@@ -377,534 +377,16 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
|
||||
}
|
||||
|
||||
func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, checkpointLoader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, prepared preparedLaneExecutor, output *RunOutput) error {
|
||||
if prepared.typed != nil {
|
||||
return r.runTypedLane(ctx, input, checkpoints, checkpointLoader, doc, sourceInput, sessionID, chunks, prepared, output)
|
||||
}
|
||||
return r.runLegacyLane(ctx, input, checkpoints, checkpointLoader, doc, sourceInput, sessionID, chunks, prepared, output)
|
||||
}
|
||||
|
||||
func (r *Runner) runLegacyLane(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, checkpointLoader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, prepared preparedLaneExecutor, output *RunOutput) error {
|
||||
lane := prepared.resolved
|
||||
if prepared.legacy == nil {
|
||||
return fmt.Errorf("resolved pipeline lane %q uses typed artifact kind %q, which the legacy raw runner cannot execute", lane.ID, lane.ArtifactKind)
|
||||
}
|
||||
extractor := prepared.legacy.extractor
|
||||
merger := prepared.legacy.merger
|
||||
normalizer := prepared.legacy.normalizer
|
||||
setLaneManifestMetadata(output, lane.ID, extractor, merger, normalizer)
|
||||
|
||||
extractOutputs := make([]contracts.ExtractOutput, 0, len(chunks))
|
||||
extractWarnings := []contracts.Warning{}
|
||||
extractRejectedStart := len(output.Rejected)
|
||||
chunksDigest, err := joinedChunkDigest(chunks)
|
||||
if err != nil {
|
||||
return fmt.Errorf("digest chunks for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
extractDependencies := digestFingerprints("chunks", chunksDigest)
|
||||
extractCheckpoint, extractDecision := checkpointLoader.Extract(lane.ID, extractor.Key(), extractDependencies)
|
||||
recordCheckpointEvent(output, checkpointLoader, string(StageExtract), lane.ID, extractor.Key(), extractDecision)
|
||||
extractStarted := time.Now().UTC()
|
||||
if err := writeDebugTimed(input.Debug, path.Join("extract", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{
|
||||
Stage: string(StageExtract),
|
||||
LaneID: lane.ID,
|
||||
ModuleKey: extractor.Key(),
|
||||
StartedAt: extractStarted,
|
||||
Payload: map[string]any{
|
||||
"reused": extractDecision.Reused,
|
||||
"decision": extractDecision,
|
||||
"source": debugSourceDocumentEnvelope(doc),
|
||||
"chunks": debugSourceChunkEnvelopes(chunks),
|
||||
"options": redactSensitiveMap(lane.Extract.Options),
|
||||
"metadata": redactSensitiveMap(input.Metadata),
|
||||
},
|
||||
}); err != nil {
|
||||
return fmt.Errorf("write extract debug artifact for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
if extractDecision.Reused {
|
||||
extractOutputs = cloneExtractOutputs(extractCheckpoint.Outputs)
|
||||
extractWarnings = cloneWarnings(extractCheckpoint.Warnings)
|
||||
output.Rejected = append(output.Rejected, cloneRejectedOutputs(extractCheckpoint.Rejected)...)
|
||||
output.Warnings = append(output.Warnings, extractWarnings...)
|
||||
} else {
|
||||
if err := checkpoints.ExtractRunning(lane.ID, extractor.Key(), extractDependencies); err != nil {
|
||||
return fmt.Errorf("write extract checkpoint for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
for index := range chunks {
|
||||
chunk := chunks[index]
|
||||
var acceptedOutput contracts.ExtractOutput
|
||||
var acceptedWarnings []contracts.Warning
|
||||
accepted, rejection, err := runWithRetry(ctx, lane.Extract.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
|
||||
attemptStarted := time.Now().UTC()
|
||||
attemptPath := path.Join("extract", debugPathComponent(lane.ID), fmt.Sprintf("chunk-%06d", chunk.Index+1), fmt.Sprintf("attempt-%02d", attempt))
|
||||
attemptCtx, llmScope := withDebugLLMScope(ctx, attemptPath)
|
||||
result, err := extractor.Extract(attemptCtx, contracts.ExtractionRequest{
|
||||
Source: doc,
|
||||
Chunk: &chunk,
|
||||
SourceInput: chunkInputMaterial(sourceInput, chunk),
|
||||
SessionID: sessionID,
|
||||
References: CloneReferenceSet(lane.ExtractReferences.ReferenceSet),
|
||||
LLMClient: input.llmClient,
|
||||
LLMProfile: lane.Extract.LLMProfile,
|
||||
Options: cloneOptions(lane.Extract.Options),
|
||||
Metadata: input.Metadata,
|
||||
})
|
||||
if err != nil {
|
||||
_ = writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
|
||||
Stage: string(StageExtract),
|
||||
LaneID: lane.ID,
|
||||
ModuleKey: extractor.Key(),
|
||||
Attempt: attempt,
|
||||
StartedAt: attemptStarted,
|
||||
Error: err.Error(),
|
||||
}, llmScope))
|
||||
return false, nil, fmt.Errorf("extract lane %q chunk %q with extractor %q: %w", lane.ID, chunk.ID, extractor.Key(), err)
|
||||
}
|
||||
extractOutput := result.Output
|
||||
extractOutput.LaneID = lane.ID
|
||||
extractOutput.ExtractorKey = extractor.Key()
|
||||
extractOutput.SourceID = doc.ID
|
||||
extractOutput.ChunkID = chunk.ID
|
||||
extractOutput.ChunkIndex = chunk.Index
|
||||
extractOutput.Payload.Warnings = append(extractOutput.Payload.Warnings, result.Warnings...)
|
||||
validationWarnings, rejection, err := r.validateRaw(attemptCtx, rawValidationTarget{
|
||||
stage: StageExtract,
|
||||
laneID: lane.ID,
|
||||
moduleKey: extractor.Key(),
|
||||
source: doc,
|
||||
sourceID: doc.ID,
|
||||
chunkID: chunk.ID,
|
||||
chunkIndex: chunk.Index,
|
||||
chunk: &chunk,
|
||||
sourceInput: chunkInputMaterial(sourceInput, chunk),
|
||||
sessionID: sessionID,
|
||||
references: lane.ExtractReferences.ReferenceSet,
|
||||
llmClient: input.llmClient,
|
||||
schema: extractOutput.Schema,
|
||||
payload: extractOutput.Payload,
|
||||
metadata: input.Metadata,
|
||||
prepared: prepared.extractValidators,
|
||||
attempt: attempt,
|
||||
debug: input.Debug,
|
||||
})
|
||||
if err != nil || rejection != nil {
|
||||
_ = writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
|
||||
Stage: string(StageExtract),
|
||||
LaneID: lane.ID,
|
||||
ModuleKey: extractor.Key(),
|
||||
Attempt: attempt,
|
||||
StartedAt: attemptStarted,
|
||||
Payload: map[string]any{
|
||||
"output": debugExtractOutputEnvelope(extractOutput),
|
||||
"warnings": append(cloneWarnings(result.Warnings), validationWarnings...),
|
||||
"rejection": debugRejectedOutputPtr(rejection),
|
||||
},
|
||||
}, llmScope))
|
||||
return false, rejection, err
|
||||
}
|
||||
acceptedOutput = cloneExtractOutput(extractOutput)
|
||||
acceptedWarnings = append(cloneWarnings(result.Warnings), validationWarnings...)
|
||||
if err := writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
|
||||
Stage: string(StageExtract),
|
||||
LaneID: lane.ID,
|
||||
ModuleKey: extractor.Key(),
|
||||
Attempt: attempt,
|
||||
StartedAt: attemptStarted,
|
||||
Payload: map[string]any{
|
||||
"output": debugExtractOutputEnvelope(extractOutput),
|
||||
"warnings": acceptedWarnings,
|
||||
},
|
||||
}, llmScope)); err != nil {
|
||||
return false, nil, err
|
||||
}
|
||||
return true, nil, nil
|
||||
})
|
||||
if err != nil {
|
||||
_ = checkpoints.ExtractFailed(lane.ID, extractor.Key(), extractDependencies, err)
|
||||
return err
|
||||
}
|
||||
if !accepted {
|
||||
output.Rejected = append(output.Rejected, *rejection)
|
||||
continue
|
||||
}
|
||||
output.Warnings = append(output.Warnings, acceptedWarnings...)
|
||||
extractWarnings = append(extractWarnings, acceptedWarnings...)
|
||||
extractOutputs = append(extractOutputs, acceptedOutput)
|
||||
}
|
||||
extractRejected := cloneRejectedOutputs(output.Rejected[extractRejectedStart:])
|
||||
if err := checkpoints.ExtractSucceeded(lane.ID, extractor.Key(), extractDependencies, extractOutputs, extractRejected, extractWarnings); err != nil {
|
||||
return fmt.Errorf("write extract checkpoint for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
}
|
||||
if err := writeDebugTimed(input.Debug, path.Join("extract", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{
|
||||
Stage: string(StageExtract),
|
||||
LaneID: lane.ID,
|
||||
ModuleKey: extractor.Key(),
|
||||
StartedAt: extractStarted,
|
||||
Payload: map[string]any{
|
||||
"reused": extractDecision.Reused,
|
||||
"outputs": debugExtractOutputEnvelopes(extractOutputs),
|
||||
"rejected": debugRejectedOutputEnvelopes(output.Rejected[extractRejectedStart:]),
|
||||
"warnings": extractWarnings,
|
||||
},
|
||||
}); err != nil {
|
||||
return fmt.Errorf("write extract debug artifact for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
|
||||
if len(extractOutputs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var acceptedMerge contracts.MergeOutput
|
||||
var mergeWarnings []contracts.Warning
|
||||
mergeDependencies := rawOutputDigests(extractPayloads(extractOutputs))
|
||||
mergeCheckpoint, mergeDecision := checkpointLoader.Merge(lane.ID, merger.Key(), mergeDependencies)
|
||||
recordCheckpointEvent(output, checkpointLoader, string(StageMerge), lane.ID, merger.Key(), mergeDecision)
|
||||
mergeStarted := time.Now().UTC()
|
||||
if err := writeDebugTimed(input.Debug, path.Join("merge", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{
|
||||
Stage: string(StageMerge),
|
||||
LaneID: lane.ID,
|
||||
ModuleKey: merger.Key(),
|
||||
StartedAt: mergeStarted,
|
||||
Payload: map[string]any{
|
||||
"reused": mergeDecision.Reused,
|
||||
"decision": mergeDecision,
|
||||
"source": debugSourceDocumentEnvelope(doc),
|
||||
"extract_outputs": debugExtractOutputEnvelopes(extractOutputs),
|
||||
"options": redactSensitiveMap(lane.Merge.Options),
|
||||
"metadata": redactSensitiveMap(input.Metadata),
|
||||
},
|
||||
}); err != nil {
|
||||
return fmt.Errorf("write merge debug artifact for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
if mergeDecision.Reused {
|
||||
acceptedMerge = cloneMergeOutput(mergeCheckpoint.Output)
|
||||
mergeWarnings = cloneWarnings(mergeCheckpoint.Warnings)
|
||||
output.Warnings = append(output.Warnings, mergeWarnings...)
|
||||
} else {
|
||||
if err := checkpoints.MergeRunning(lane.ID, merger.Key(), mergeDependencies); err != nil {
|
||||
return fmt.Errorf("write merge checkpoint for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
mergeAccepted, mergeRejection, err := runWithRetry(ctx, lane.Merge.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
|
||||
attemptStarted := time.Now().UTC()
|
||||
attemptPath := path.Join("merge", debugPathComponent(lane.ID), fmt.Sprintf("attempt-%02d", attempt))
|
||||
attemptCtx, llmScope := withDebugLLMScope(ctx, attemptPath)
|
||||
mergeResult, err := merger.Merge(attemptCtx, contracts.MergeRequest{
|
||||
Source: doc,
|
||||
LaneID: lane.ID,
|
||||
ExtractOutputs: cloneExtractOutputs(extractOutputs),
|
||||
SourceInput: sourceInput.Clone(),
|
||||
SessionID: sessionID,
|
||||
References: CloneReferenceSet(lane.MergeReferences.ReferenceSet),
|
||||
LLMClient: input.llmClient,
|
||||
LLMProfile: lane.Merge.LLMProfile,
|
||||
Options: cloneOptions(lane.Merge.Options),
|
||||
Metadata: input.Metadata,
|
||||
})
|
||||
if err != nil {
|
||||
_ = writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
|
||||
Stage: string(StageMerge),
|
||||
LaneID: lane.ID,
|
||||
ModuleKey: merger.Key(),
|
||||
Attempt: attempt,
|
||||
StartedAt: attemptStarted,
|
||||
Error: err.Error(),
|
||||
}, llmScope))
|
||||
return false, nil, fmt.Errorf("merge lane %q with merger %q: %w", lane.ID, merger.Key(), err)
|
||||
}
|
||||
mergeOutput := mergeResult.Output
|
||||
mergeOutput.LaneID = lane.ID
|
||||
mergeOutput.MergerKey = merger.Key()
|
||||
mergeOutput.SourceID = doc.ID
|
||||
mergeOutput.Payload.Warnings = append(mergeOutput.Payload.Warnings, mergeResult.Warnings...)
|
||||
validationWarnings, rejection, err := r.validateRaw(attemptCtx, rawValidationTarget{
|
||||
stage: StageMerge,
|
||||
laneID: lane.ID,
|
||||
moduleKey: merger.Key(),
|
||||
source: doc,
|
||||
sourceID: doc.ID,
|
||||
sourceInput: sourceInput.Clone(),
|
||||
sessionID: sessionID,
|
||||
references: lane.MergeReferences.ReferenceSet,
|
||||
llmClient: input.llmClient,
|
||||
schema: mergeOutput.Schema,
|
||||
payload: mergeOutput.Payload,
|
||||
extractOutputs: extractOutputs,
|
||||
metadata: input.Metadata,
|
||||
prepared: prepared.mergeValidators,
|
||||
attempt: attempt,
|
||||
debug: input.Debug,
|
||||
})
|
||||
if err != nil || rejection != nil {
|
||||
_ = writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
|
||||
Stage: string(StageMerge),
|
||||
LaneID: lane.ID,
|
||||
ModuleKey: merger.Key(),
|
||||
Attempt: attempt,
|
||||
StartedAt: attemptStarted,
|
||||
Payload: map[string]any{
|
||||
"output": debugMergeOutputEnvelope(mergeOutput),
|
||||
"warnings": append(cloneWarnings(mergeResult.Warnings), validationWarnings...),
|
||||
"rejection": debugRejectedOutputPtr(rejection),
|
||||
},
|
||||
}, llmScope))
|
||||
return false, rejection, err
|
||||
}
|
||||
acceptedMerge = cloneMergeOutput(mergeOutput)
|
||||
mergeWarnings = append(cloneWarnings(mergeResult.Warnings), validationWarnings...)
|
||||
if err := writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
|
||||
Stage: string(StageMerge),
|
||||
LaneID: lane.ID,
|
||||
ModuleKey: merger.Key(),
|
||||
Attempt: attempt,
|
||||
StartedAt: attemptStarted,
|
||||
Payload: map[string]any{
|
||||
"output": debugMergeOutputEnvelope(mergeOutput),
|
||||
"warnings": mergeWarnings,
|
||||
},
|
||||
}, llmScope)); err != nil {
|
||||
return false, nil, err
|
||||
}
|
||||
return true, nil, nil
|
||||
})
|
||||
if err != nil {
|
||||
_ = checkpoints.MergeFailed(lane.ID, merger.Key(), mergeDependencies, err)
|
||||
return err
|
||||
}
|
||||
if !mergeAccepted {
|
||||
output.Rejected = append(output.Rejected, *mergeRejection)
|
||||
if err := checkpoints.MergeRejected(lane.ID, merger.Key(), mergeDependencies, *mergeRejection); err != nil {
|
||||
return fmt.Errorf("write merge checkpoint for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
if err := writeDebugTimed(input.Debug, path.Join("merge", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{
|
||||
Stage: string(StageMerge),
|
||||
LaneID: lane.ID,
|
||||
ModuleKey: merger.Key(),
|
||||
StartedAt: mergeStarted,
|
||||
Payload: map[string]any{
|
||||
"accepted": false,
|
||||
"rejection": debugRejectedOutputEnvelope(*mergeRejection),
|
||||
"warnings": mergeWarnings,
|
||||
},
|
||||
}); err != nil {
|
||||
return fmt.Errorf("write merge debug artifact for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
output.Warnings = append(output.Warnings, mergeWarnings...)
|
||||
if err := checkpoints.MergeSucceeded(lane.ID, merger.Key(), mergeDependencies, acceptedMerge, mergeWarnings); err != nil {
|
||||
return fmt.Errorf("write merge checkpoint for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
}
|
||||
if err := writeDebugTimed(input.Debug, path.Join("merge", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{
|
||||
Stage: string(StageMerge),
|
||||
LaneID: lane.ID,
|
||||
ModuleKey: merger.Key(),
|
||||
StartedAt: mergeStarted,
|
||||
Payload: map[string]any{
|
||||
"reused": mergeDecision.Reused,
|
||||
"accepted": true,
|
||||
"output": debugMergeOutputEnvelope(acceptedMerge),
|
||||
"warnings": mergeWarnings,
|
||||
},
|
||||
}); err != nil {
|
||||
return fmt.Errorf("write merge debug artifact for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
|
||||
var acceptedNormalize contracts.NormalizeOutput
|
||||
var normalizeWarnings []contracts.Warning
|
||||
normalizeDependencies := rawOutputDigests([]contracts.RawPayload{acceptedMerge.Payload})
|
||||
normalizeCheckpoint, normalizeDecision := checkpointLoader.Normalize(lane.ID, normalizer.Key(), normalizeDependencies)
|
||||
recordCheckpointEvent(output, checkpointLoader, string(StageNormalize), lane.ID, normalizer.Key(), normalizeDecision)
|
||||
normalizeStarted := time.Now().UTC()
|
||||
if err := writeDebugTimed(input.Debug, path.Join("normalize", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{
|
||||
Stage: string(StageNormalize),
|
||||
LaneID: lane.ID,
|
||||
ModuleKey: normalizer.Key(),
|
||||
StartedAt: normalizeStarted,
|
||||
Payload: map[string]any{
|
||||
"reused": normalizeDecision.Reused,
|
||||
"decision": normalizeDecision,
|
||||
"source": debugSourceDocumentEnvelope(doc),
|
||||
"merge_output": debugMergeOutputEnvelope(acceptedMerge),
|
||||
"options": redactSensitiveMap(lane.Normalize.Options),
|
||||
"metadata": redactSensitiveMap(input.Metadata),
|
||||
},
|
||||
}); err != nil {
|
||||
return fmt.Errorf("write normalize debug artifact for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
if normalizeDecision.Reused {
|
||||
acceptedNormalize = cloneNormalizeOutput(normalizeCheckpoint.Output)
|
||||
normalizeWarnings = cloneWarnings(normalizeCheckpoint.Warnings)
|
||||
output.Warnings = append(output.Warnings, normalizeWarnings...)
|
||||
} else {
|
||||
if err := checkpoints.NormalizeRunning(lane.ID, normalizer.Key(), normalizeDependencies); err != nil {
|
||||
return fmt.Errorf("write normalize checkpoint for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
normalizeAccepted, normalizeRejection, err := runWithRetry(ctx, lane.Normalize.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
|
||||
attemptStarted := time.Now().UTC()
|
||||
attemptPath := path.Join("normalize", debugPathComponent(lane.ID), fmt.Sprintf("attempt-%02d", attempt))
|
||||
attemptCtx, llmScope := withDebugLLMScope(ctx, attemptPath)
|
||||
normalizeResult, err := normalizer.Normalize(attemptCtx, contracts.NormalizeRequest{
|
||||
Source: doc,
|
||||
LaneID: lane.ID,
|
||||
MergeOutput: cloneMergeOutput(acceptedMerge),
|
||||
SourceInput: sourceInput.Clone(),
|
||||
SessionID: sessionID,
|
||||
References: CloneReferenceSet(lane.NormalizeReferences.ReferenceSet),
|
||||
LLMClient: input.llmClient,
|
||||
LLMProfile: lane.Normalize.LLMProfile,
|
||||
Options: cloneOptions(lane.Normalize.Options),
|
||||
Metadata: input.Metadata,
|
||||
})
|
||||
if err != nil {
|
||||
_ = writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
|
||||
Stage: string(StageNormalize),
|
||||
LaneID: lane.ID,
|
||||
ModuleKey: normalizer.Key(),
|
||||
Attempt: attempt,
|
||||
StartedAt: attemptStarted,
|
||||
Error: err.Error(),
|
||||
}, llmScope))
|
||||
return false, nil, fmt.Errorf("normalize lane %q with normalizer %q: %w", lane.ID, normalizer.Key(), err)
|
||||
}
|
||||
normalizeOutput := normalizeResult.Output
|
||||
normalizeOutput.LaneID = lane.ID
|
||||
normalizeOutput.NormalizerKey = normalizer.Key()
|
||||
normalizeOutput.SourceID = doc.ID
|
||||
normalizeOutput.Payload.Warnings = append(normalizeOutput.Payload.Warnings, normalizeResult.Warnings...)
|
||||
validationWarnings, rejection, err := r.validateRaw(attemptCtx, rawValidationTarget{
|
||||
stage: StageNormalize,
|
||||
laneID: lane.ID,
|
||||
moduleKey: normalizer.Key(),
|
||||
source: doc,
|
||||
sourceID: doc.ID,
|
||||
sourceInput: sourceInput.Clone(),
|
||||
sessionID: sessionID,
|
||||
references: lane.NormalizeReferences.ReferenceSet,
|
||||
llmClient: input.llmClient,
|
||||
schema: normalizeOutput.Schema,
|
||||
payload: normalizeOutput.Payload,
|
||||
mergeOutput: acceptedMerge,
|
||||
metadata: input.Metadata,
|
||||
prepared: prepared.normalizeValidators,
|
||||
attempt: attempt,
|
||||
debug: input.Debug,
|
||||
})
|
||||
if err != nil || rejection != nil {
|
||||
_ = writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
|
||||
Stage: string(StageNormalize),
|
||||
LaneID: lane.ID,
|
||||
ModuleKey: normalizer.Key(),
|
||||
Attempt: attempt,
|
||||
StartedAt: attemptStarted,
|
||||
Payload: map[string]any{
|
||||
"output": debugNormalizeOutputEnvelope(normalizeOutput),
|
||||
"warnings": append(cloneWarnings(normalizeResult.Warnings), validationWarnings...),
|
||||
"rejection": debugRejectedOutputPtr(rejection),
|
||||
},
|
||||
}, llmScope))
|
||||
return false, rejection, err
|
||||
}
|
||||
acceptedNormalize = cloneNormalizeOutput(normalizeOutput)
|
||||
normalizeWarnings = append(cloneWarnings(normalizeResult.Warnings), validationWarnings...)
|
||||
if err := writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
|
||||
Stage: string(StageNormalize),
|
||||
LaneID: lane.ID,
|
||||
ModuleKey: normalizer.Key(),
|
||||
Attempt: attempt,
|
||||
StartedAt: attemptStarted,
|
||||
Payload: map[string]any{
|
||||
"output": debugNormalizeOutputEnvelope(normalizeOutput),
|
||||
"warnings": normalizeWarnings,
|
||||
},
|
||||
}, llmScope)); err != nil {
|
||||
return false, nil, err
|
||||
}
|
||||
return true, nil, nil
|
||||
})
|
||||
if err != nil {
|
||||
_ = checkpoints.NormalizeFailed(lane.ID, normalizer.Key(), normalizeDependencies, err)
|
||||
return err
|
||||
}
|
||||
if !normalizeAccepted {
|
||||
output.Rejected = append(output.Rejected, *normalizeRejection)
|
||||
if err := checkpoints.NormalizeRejected(lane.ID, normalizer.Key(), normalizeDependencies, *normalizeRejection); err != nil {
|
||||
return fmt.Errorf("write normalize checkpoint for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
if err := writeDebugTimed(input.Debug, path.Join("normalize", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{
|
||||
Stage: string(StageNormalize),
|
||||
LaneID: lane.ID,
|
||||
ModuleKey: normalizer.Key(),
|
||||
StartedAt: normalizeStarted,
|
||||
Payload: map[string]any{
|
||||
"accepted": false,
|
||||
"rejection": debugRejectedOutputEnvelope(*normalizeRejection),
|
||||
"warnings": normalizeWarnings,
|
||||
},
|
||||
}); err != nil {
|
||||
return fmt.Errorf("write normalize debug artifact for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
output.Warnings = append(output.Warnings, normalizeWarnings...)
|
||||
if err := checkpoints.NormalizeSucceeded(lane.ID, normalizer.Key(), normalizeDependencies, acceptedNormalize, normalizeWarnings); err != nil {
|
||||
return fmt.Errorf("write normalize checkpoint for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
}
|
||||
if err := writeDebugTimed(input.Debug, path.Join("normalize", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{
|
||||
Stage: string(StageNormalize),
|
||||
LaneID: lane.ID,
|
||||
ModuleKey: normalizer.Key(),
|
||||
StartedAt: normalizeStarted,
|
||||
Payload: map[string]any{
|
||||
"reused": normalizeDecision.Reused,
|
||||
"accepted": true,
|
||||
"output": debugNormalizeOutputEnvelope(acceptedNormalize),
|
||||
"warnings": normalizeWarnings,
|
||||
},
|
||||
}); err != nil {
|
||||
return fmt.Errorf("write normalize debug artifact for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
output.NormalizeOutputs = append(output.NormalizeOutputs, serializedOutputFromLegacy(acceptedNormalize))
|
||||
return nil
|
||||
}
|
||||
|
||||
type rawValidationTarget struct {
|
||||
stage ModuleStage
|
||||
laneID string
|
||||
moduleKey string
|
||||
source *source.SourceDocument
|
||||
sourceID string
|
||||
sourceInput contracts.LLMInputMaterial
|
||||
sessionID string
|
||||
references contracts.ReferenceSet
|
||||
llmClient contracts.StructuredLLMClient
|
||||
chunkID string
|
||||
chunkIndex int
|
||||
chunk *source.Chunk
|
||||
chunks []source.Chunk
|
||||
schema contracts.ResponseSchema
|
||||
payload contracts.RawPayload
|
||||
extractOutputs []contracts.ExtractOutput
|
||||
mergeOutput contracts.MergeOutput
|
||||
metadata map[string]any
|
||||
prepared preparedValidatorChain
|
||||
attempt int
|
||||
debug DebugRecorder
|
||||
return r.runTypedLane(ctx, input, checkpoints, checkpointLoader, doc, sourceInput, sessionID, chunks, prepared, output)
|
||||
}
|
||||
|
||||
func runWithRetry(ctx context.Context, retries int, run func(attempt int) (bool, *contracts.RejectedOutput, error)) (bool, *contracts.RejectedOutput, error) {
|
||||
attempts := 1
|
||||
if retries > 0 {
|
||||
attempts += retries
|
||||
}
|
||||
|
||||
var lastRejection *contracts.RejectedOutput
|
||||
attempts := retries + 1
|
||||
var last *contracts.RejectedOutput
|
||||
for attempt := 1; attempt <= attempts; attempt++ {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return false, nil, err
|
||||
}
|
||||
|
||||
accepted, rejection, err := run(attempt)
|
||||
if err != nil {
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
@@ -920,55 +402,22 @@ func runWithRetry(ctx context.Context, retries int, run func(attempt int) (bool,
|
||||
}
|
||||
if rejection != nil {
|
||||
rejection.AttemptCount = attempt
|
||||
lastRejection = rejection
|
||||
last = rejection
|
||||
}
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return false, nil, ctxErr
|
||||
}
|
||||
if attempt == attempts {
|
||||
if lastRejection == nil {
|
||||
lastRejection = &contracts.RejectedOutput{
|
||||
ReasonCode: "raw_output_rejected",
|
||||
Message: "raw output rejected",
|
||||
AttemptCount: attempt,
|
||||
}
|
||||
if last == nil {
|
||||
last = &contracts.RejectedOutput{ReasonCode: "output_rejected", Message: "output rejected", AttemptCount: attempt}
|
||||
}
|
||||
return false, lastRejection, nil
|
||||
return false, last, nil
|
||||
}
|
||||
}
|
||||
|
||||
return false, lastRejection, nil
|
||||
return false, last, nil
|
||||
}
|
||||
|
||||
func (r *Runner) validateChunksRaw(ctx context.Context, doc *source.SourceDocument, moduleKey string, chunks []source.Chunk, sourceInput contracts.LLMInputMaterial, sessionID string, references contracts.ReferenceSet, llmClient contracts.StructuredLLMClient, metadata map[string]any, prepared preparedValidatorChain, attempt int, debug DebugRecorder) ([]contracts.Warning, *contracts.RejectedOutput, error) {
|
||||
return r.validateRaw(ctx, rawValidationTarget{
|
||||
stage: StageChunk,
|
||||
moduleKey: moduleKey,
|
||||
source: doc,
|
||||
sourceID: doc.ID,
|
||||
sourceInput: sourceInput.Clone(),
|
||||
sessionID: sessionID,
|
||||
references: references,
|
||||
llmClient: llmClient,
|
||||
chunks: chunks,
|
||||
metadata: metadata,
|
||||
prepared: prepared,
|
||||
attempt: attempt,
|
||||
debug: debug,
|
||||
})
|
||||
}
|
||||
|
||||
func (r *Runner) validateChunks(ctx context.Context, doc *source.SourceDocument, moduleKey string, chunks []source.Chunk, sourceInput contracts.LLMInputMaterial, sessionID string, references contracts.ReferenceSet, llmClient contracts.StructuredLLMClient, metadata map[string]any, prepared preparedValidatorChain, attempt int, debug DebugRecorder) ([]contracts.Warning, *contracts.RejectedOutput, error) {
|
||||
allLegacy := true
|
||||
for _, item := range prepared.validators {
|
||||
if item.resolved.Target != ValidatorTargetLegacyRaw && item.resolved.Target != "" {
|
||||
allLegacy = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if allLegacy {
|
||||
return r.validateChunksRaw(ctx, doc, moduleKey, chunks, sourceInput, sessionID, references, llmClient, metadata, prepared, attempt, debug)
|
||||
}
|
||||
func (r *Runner) validateChunks(ctx context.Context, doc *source.SourceDocument, moduleKey string, chunks []source.Chunk, sourceInput contracts.LLMInputMaterial, sessionID string, references contracts.ReferenceSet, metadata map[string]any, prepared preparedValidatorChain, attempt int, debug DebugRecorder) ([]contracts.Warning, *contracts.RejectedOutput, error) {
|
||||
content, err := json.Marshal(chunks)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("encode canonical chunks for validation: %w", err)
|
||||
@@ -989,8 +438,9 @@ func (r *Runner) validateChunks(ctx context.Context, doc *source.SourceDocument,
|
||||
default:
|
||||
return nil, nil, fmt.Errorf("validator %q is incompatible with chunk validation", binding.Module)
|
||||
}
|
||||
debugRequest := contracts.ValidationRequest{Stage: string(StageChunk), ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, Metadata: cloneMetadata(metadata), Chunks: cloneSourceChunks(chunks), Schema: contracts.ResponseSchema{ID: schema.ID, Name: schema.Name, Version: schema.Version, JSONSchema: append([]byte(nil), schema.JSONSchema...)}, Payload: contracts.RawPayload{Content: content, MediaType: "application/json"}}
|
||||
debugCall := debugValidationCall{ValidatorName: binding.Module, Request: debugValidationRequestEnvelope(debugRequest), Result: debugValidationResultEnvelope(result)}
|
||||
debugContent := debugContentEnvelope(content, "application/json", nil, nil)
|
||||
debugContent.ContentDigest = debugContentDigest(content)
|
||||
debugCall := debugValidationCall{ValidatorName: binding.Module, Request: map[string]any{"stage": string(StageChunk), "module_key": moduleKey, "source_id": doc.ID, "schema": schema, "schema_digest": contracts.DigestArtifactSchema(schema), "content": debugContent, "metadata": redactSensitiveMap(metadata)}, Result: debugValidationResultEnvelope(result)}
|
||||
if err != nil {
|
||||
debugCall.Error = err.Error()
|
||||
}
|
||||
@@ -1003,11 +453,11 @@ func (r *Runner) validateChunks(ctx context.Context, doc *source.SourceDocument,
|
||||
if !result.Approved {
|
||||
reason := result.ReasonCode
|
||||
if reason == "" {
|
||||
reason = "raw_output_rejected"
|
||||
reason = "output_rejected"
|
||||
}
|
||||
message := result.Message
|
||||
if message == "" {
|
||||
message = "raw output rejected"
|
||||
message = "output rejected"
|
||||
}
|
||||
return nil, &contracts.RejectedOutput{Stage: string(StageChunk), ModuleKey: moduleKey, ValidatorName: binding.Module, ReasonCode: reason, Message: message, AttemptCount: attempt, DiagnosticArtifactPath: result.DiagnosticArtifactPath}, nil
|
||||
}
|
||||
@@ -1016,97 +466,6 @@ func (r *Runner) validateChunks(ctx context.Context, doc *source.SourceDocument,
|
||||
return warnings, nil, nil
|
||||
}
|
||||
|
||||
func (r *Runner) validateRaw(ctx context.Context, target rawValidationTarget) ([]contracts.Warning, *contracts.RejectedOutput, error) {
|
||||
if len(target.prepared.validators) == 0 {
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
var warnings []contracts.Warning
|
||||
for index, preparedValidator := range target.prepared.validators {
|
||||
validator := preparedValidator.legacy
|
||||
validatorBinding := preparedValidator.resolved
|
||||
if validator == nil {
|
||||
return nil, nil, fmt.Errorf("validator %q is not available on the legacy raw path", validatorBinding.Binding.Module)
|
||||
}
|
||||
request := target.validationRequest(validatorBinding.Binding)
|
||||
started := time.Now().UTC()
|
||||
attemptPath := path.Join("validate", debugPathComponent(string(target.stage)), debugPathComponent(target.laneID), debugPathComponent(target.moduleKey), fmt.Sprintf("%02d-%s-attempt-%02d", index+1, debugPathComponent(validator.Name()), target.attempt))
|
||||
validatorCtx, llmScope := withDebugLLMScope(ctx, attemptPath)
|
||||
result, err := validator.Validate(validatorCtx, request)
|
||||
debugPayload := debugValidationCall{
|
||||
ValidatorName: validator.Name(),
|
||||
Request: debugValidationRequestEnvelope(request),
|
||||
Result: debugValidationResultEnvelope(result),
|
||||
}
|
||||
if err != nil {
|
||||
debugPayload.Error = err.Error()
|
||||
}
|
||||
if debugErr := writeDebugTimed(target.debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
|
||||
Stage: string(target.stage),
|
||||
LaneID: target.laneID,
|
||||
ModuleKey: target.moduleKey,
|
||||
Attempt: target.attempt,
|
||||
StartedAt: started,
|
||||
Payload: debugPayload,
|
||||
Error: debugPayload.Error,
|
||||
}, llmScope)); debugErr != nil {
|
||||
return nil, nil, fmt.Errorf("write validation debug artifact: %w", debugErr)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("validate raw %s output with validator %q: %w", target.stage, validator.Name(), err)
|
||||
}
|
||||
if !result.Approved {
|
||||
reasonCode := strings.TrimSpace(result.ReasonCode)
|
||||
if reasonCode == "" {
|
||||
reasonCode = "raw_output_rejected"
|
||||
}
|
||||
message := strings.TrimSpace(result.Message)
|
||||
if message == "" {
|
||||
message = "raw output rejected"
|
||||
}
|
||||
return nil, &contracts.RejectedOutput{
|
||||
Stage: string(target.stage),
|
||||
LaneID: target.laneID,
|
||||
ModuleKey: target.moduleKey,
|
||||
ChunkID: target.chunkID,
|
||||
ChunkIndex: target.chunkIndex,
|
||||
ValidatorName: validator.Name(),
|
||||
ReasonCode: reasonCode,
|
||||
Message: message,
|
||||
AttemptCount: target.attempt,
|
||||
DiagnosticArtifactPath: result.DiagnosticArtifactPath,
|
||||
}, nil
|
||||
}
|
||||
warnings = append(warnings, result.Warnings...)
|
||||
}
|
||||
return warnings, nil, nil
|
||||
}
|
||||
|
||||
func (target rawValidationTarget) validationRequest(binding ModuleBinding) contracts.ValidationRequest {
|
||||
return contracts.ValidationRequest{
|
||||
Stage: string(target.stage),
|
||||
LaneID: target.laneID,
|
||||
ModuleKey: target.moduleKey,
|
||||
Source: target.source,
|
||||
SourceID: target.sourceID,
|
||||
SourceInput: target.sourceInput.Clone(),
|
||||
SessionID: target.sessionID,
|
||||
References: CloneReferenceSet(target.references),
|
||||
LLMClient: target.llmClient,
|
||||
LLMProfile: binding.LLMProfile,
|
||||
Options: cloneOptions(binding.Options),
|
||||
Metadata: cloneMetadata(target.metadata),
|
||||
Schema: cloneResponseSchema(target.schema),
|
||||
Payload: cloneRawPayload(target.payload),
|
||||
ChunkID: target.chunkID,
|
||||
ChunkIndex: target.chunkIndex,
|
||||
Chunk: cloneSourceChunkPtr(target.chunk),
|
||||
Chunks: cloneSourceChunks(target.chunks),
|
||||
ExtractOutputs: cloneExtractOutputs(target.extractOutputs),
|
||||
MergeOutput: cloneMergeOutput(target.mergeOutput),
|
||||
}
|
||||
}
|
||||
|
||||
func resolvedValidatorChain(stage ModuleStage, laneID string, moduleKey string, chains []ResolvedValidatorChain) ResolvedValidatorChain {
|
||||
for _, chain := range chains {
|
||||
if chain.Stage != stage {
|
||||
@@ -1250,7 +609,7 @@ func validatorChainManifests(chains []ResolvedValidatorChain) []artifacts.Valida
|
||||
|
||||
func failOutput(output RunOutput) RunOutput {
|
||||
if output.Manifest.PipelineID != "" {
|
||||
populateRawOutputManifest(&output)
|
||||
populateOutputManifest(&output)
|
||||
output.Manifest.ValidationStatus = "failed"
|
||||
output.Manifest.CompletedAt = timePtr(time.Now().UTC())
|
||||
}
|
||||
@@ -1274,7 +633,7 @@ func recordCheckpointEvent(output *RunOutput, loader CheckpointLoader, stage str
|
||||
})
|
||||
}
|
||||
|
||||
func populateRawOutputManifest(output *RunOutput) {
|
||||
func populateOutputManifest(output *RunOutput) {
|
||||
if output == nil {
|
||||
return
|
||||
}
|
||||
@@ -1325,59 +684,18 @@ func rejectedOutputManifests(rejected []contracts.RejectedOutput) []artifacts.Re
|
||||
return manifests
|
||||
}
|
||||
|
||||
func setLaneManifestMetadata(output *RunOutput, laneID string, modules ...any) {
|
||||
if output == nil {
|
||||
return
|
||||
}
|
||||
for i := range output.Manifest.ArtifactLanes {
|
||||
if output.Manifest.ArtifactLanes[i].ID != laneID {
|
||||
continue
|
||||
}
|
||||
|
||||
metadata := make(map[string]any)
|
||||
for _, module := range modules {
|
||||
moduleMetadata, ok := moduleManifestMetadata(module)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
key := manifestMetadataKey(module)
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
metadata[key] = moduleMetadata
|
||||
}
|
||||
if len(metadata) > 0 {
|
||||
output.Manifest.ArtifactLanes[i].Metadata = metadata
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func attachModuleManifestMetadata(output *RunOutput, moduleKey string, module any) {
|
||||
if output == nil {
|
||||
return
|
||||
}
|
||||
moduleMetadata, ok := moduleManifestMetadata(module)
|
||||
metadata, ok := moduleManifestMetadata(module)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if output.Manifest.ModuleMetadata == nil {
|
||||
output.Manifest.ModuleMetadata = make(map[string]map[string]any)
|
||||
}
|
||||
output.Manifest.ModuleMetadata[moduleKey] = moduleMetadata
|
||||
}
|
||||
|
||||
func manifestMetadataKey(module any) string {
|
||||
switch module.(type) {
|
||||
case contracts.LegacyRawExtractor:
|
||||
return "extractor"
|
||||
case contracts.LegacyRawMerger:
|
||||
return "merger"
|
||||
case contracts.LegacyRawNormalizer:
|
||||
return "normalizer"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
output.Manifest.ModuleMetadata[moduleKey] = metadata
|
||||
}
|
||||
|
||||
func moduleManifestMetadata(module any) (map[string]any, bool) {
|
||||
@@ -1557,20 +875,6 @@ func cloneWarnings(warnings []contracts.Warning) []contracts.Warning {
|
||||
return append([]contracts.Warning(nil), warnings...)
|
||||
}
|
||||
|
||||
func cloneRawPayload(payload contracts.RawPayload) contracts.RawPayload {
|
||||
return contracts.RawPayload{
|
||||
Content: append([]byte(nil), payload.Content...),
|
||||
MediaType: payload.MediaType,
|
||||
Metadata: cloneMetadata(payload.Metadata),
|
||||
Warnings: cloneWarnings(payload.Warnings),
|
||||
}
|
||||
}
|
||||
|
||||
func cloneResponseSchema(schema contracts.ResponseSchema) contracts.ResponseSchema {
|
||||
schema.JSONSchema = append([]byte(nil), schema.JSONSchema...)
|
||||
return schema
|
||||
}
|
||||
|
||||
func cloneSourceChunkPtr(chunk *source.Chunk) *source.Chunk {
|
||||
if chunk == nil {
|
||||
return nil
|
||||
@@ -1608,35 +912,6 @@ func cloneSourceUnits(units []source.SourceUnit) []source.SourceUnit {
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneExtractOutput(output contracts.ExtractOutput) contracts.ExtractOutput {
|
||||
output.Schema = cloneResponseSchema(output.Schema)
|
||||
output.Payload = cloneRawPayload(output.Payload)
|
||||
return output
|
||||
}
|
||||
|
||||
func cloneExtractOutputs(outputs []contracts.ExtractOutput) []contracts.ExtractOutput {
|
||||
if len(outputs) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]contracts.ExtractOutput, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
out = append(out, cloneExtractOutput(output))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneMergeOutput(output contracts.MergeOutput) contracts.MergeOutput {
|
||||
output.Schema = cloneResponseSchema(output.Schema)
|
||||
output.Payload = cloneRawPayload(output.Payload)
|
||||
return output
|
||||
}
|
||||
|
||||
func cloneNormalizeOutput(output contracts.NormalizeOutput) contracts.NormalizeOutput {
|
||||
output.Schema = cloneResponseSchema(output.Schema)
|
||||
output.Payload = cloneRawPayload(output.Payload)
|
||||
return output
|
||||
}
|
||||
|
||||
func cloneSerializedOutputs(outputs []contracts.SerializedOutput) []contracts.SerializedOutput {
|
||||
if len(outputs) == 0 {
|
||||
return nil
|
||||
@@ -1648,16 +923,6 @@ func cloneSerializedOutputs(outputs []contracts.SerializedOutput) []contracts.Se
|
||||
return out
|
||||
}
|
||||
|
||||
func serializedOutputFromLegacy(output contracts.NormalizeOutput) contracts.SerializedOutput {
|
||||
return contracts.SerializedOutput{
|
||||
LaneID: output.LaneID, NormalizerKey: output.NormalizerKey, SourceID: output.SourceID,
|
||||
Artifact: contracts.SerializedArtifact{
|
||||
Schema: contracts.ArtifactSchema{ID: output.Schema.ID, Name: output.Schema.Name, Version: output.Schema.Version, JSONSchema: append([]byte(nil), output.Schema.JSONSchema...)},
|
||||
MediaType: output.Payload.MediaType, Content: append([]byte(nil), output.Payload.Content...), Metadata: cloneMetadata(output.Payload.Metadata),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func cloneRejectedOutputs(rejected []contracts.RejectedOutput) []contracts.RejectedOutput {
|
||||
if len(rejected) == 0 {
|
||||
return nil
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -13,54 +13,30 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func loadArtifactExtract(loader CheckpointLoader, laneID, moduleKey string, deps []CheckpointFingerprint) (ArtifactExtractCheckpoint, CheckpointDecision) {
|
||||
typed, ok := loader.(ArtifactCheckpointLoader)
|
||||
if !ok {
|
||||
return ArtifactExtractCheckpoint{}, CheckpointDecision{Reason: "artifact checkpoint loading is unavailable"}
|
||||
}
|
||||
return typed.ArtifactExtract(laneID, moduleKey, deps)
|
||||
func loadExtract(loader CheckpointLoader, laneID, moduleKey string, deps []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
|
||||
return loader.Extract(laneID, moduleKey, deps)
|
||||
}
|
||||
func loadArtifactMerge(loader CheckpointLoader, laneID, moduleKey string, deps []CheckpointFingerprint) (ArtifactMergeCheckpoint, CheckpointDecision) {
|
||||
typed, ok := loader.(ArtifactCheckpointLoader)
|
||||
if !ok {
|
||||
return ArtifactMergeCheckpoint{}, CheckpointDecision{Reason: "artifact checkpoint loading is unavailable"}
|
||||
}
|
||||
return typed.ArtifactMerge(laneID, moduleKey, deps)
|
||||
func loadMerge(loader CheckpointLoader, laneID, moduleKey string, deps []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) {
|
||||
return loader.Merge(laneID, moduleKey, deps)
|
||||
}
|
||||
func loadArtifactNormalize(loader CheckpointLoader, laneID, moduleKey string, deps []CheckpointFingerprint) (ArtifactNormalizeCheckpoint, CheckpointDecision) {
|
||||
typed, ok := loader.(ArtifactCheckpointLoader)
|
||||
if !ok {
|
||||
return ArtifactNormalizeCheckpoint{}, CheckpointDecision{Reason: "artifact checkpoint loading is unavailable"}
|
||||
}
|
||||
return typed.ArtifactNormalize(laneID, moduleKey, deps)
|
||||
func loadNormalize(loader CheckpointLoader, laneID, moduleKey string, deps []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) {
|
||||
return loader.Normalize(laneID, moduleKey, deps)
|
||||
}
|
||||
func recordArtifactExtract(recorder CheckpointRecorder, laneID, moduleKey string, deps []CheckpointFingerprint, outputs []ArtifactCheckpointOutput, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error {
|
||||
typed, ok := recorder.(ArtifactCheckpointRecorder)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return typed.ArtifactExtractSucceeded(laneID, moduleKey, deps, outputs, rejected, warnings)
|
||||
func recordExtract(recorder CheckpointRecorder, laneID, moduleKey string, deps []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error {
|
||||
return recorder.ExtractSucceeded(laneID, moduleKey, deps, outputs, rejected, warnings)
|
||||
}
|
||||
func recordArtifactMerge(recorder CheckpointRecorder, laneID, moduleKey string, deps []CheckpointFingerprint, output ArtifactCheckpointOutput, warnings []contracts.Warning) error {
|
||||
typed, ok := recorder.(ArtifactCheckpointRecorder)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return typed.ArtifactMergeSucceeded(laneID, moduleKey, deps, output, warnings)
|
||||
func recordMerge(recorder CheckpointRecorder, laneID, moduleKey string, deps []CheckpointFingerprint, output CheckpointArtifact, warnings []contracts.Warning) error {
|
||||
return recorder.MergeSucceeded(laneID, moduleKey, deps, output, warnings)
|
||||
}
|
||||
func recordArtifactNormalize(recorder CheckpointRecorder, laneID, moduleKey string, deps []CheckpointFingerprint, output ArtifactCheckpointOutput, warnings []contracts.Warning) error {
|
||||
typed, ok := recorder.(ArtifactCheckpointRecorder)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return typed.ArtifactNormalizeSucceeded(laneID, moduleKey, deps, output, warnings)
|
||||
func recordNormalize(recorder CheckpointRecorder, laneID, moduleKey string, deps []CheckpointFingerprint, output CheckpointArtifact, warnings []contracts.Warning) error {
|
||||
return recorder.NormalizeSucceeded(laneID, moduleKey, deps, output, warnings)
|
||||
}
|
||||
func cloneArtifactCheckpointOutput(output ArtifactCheckpointOutput) ArtifactCheckpointOutput {
|
||||
func cloneCheckpointArtifact(output CheckpointArtifact) CheckpointArtifact {
|
||||
output.Artifact = contracts.CloneSerializedArtifact(output.Artifact)
|
||||
return output
|
||||
}
|
||||
|
||||
func hydrateCheckpointArtifact(codec artifactCodecEntry, output ArtifactCheckpointOutput, value any) ArtifactCheckpointOutput {
|
||||
func hydrateCheckpointArtifact(codec artifactCodecEntry, output CheckpointArtifact, value any) CheckpointArtifact {
|
||||
output.Artifact.Schema = contracts.CloneArtifactSchema(codec.spec.Schema)
|
||||
if codec.metadata != nil {
|
||||
output.Artifact.Metadata = cloneMetadata(codec.metadata(value))
|
||||
@@ -69,7 +45,7 @@ func hydrateCheckpointArtifact(codec artifactCodecEntry, output ArtifactCheckpoi
|
||||
}
|
||||
return output
|
||||
}
|
||||
func artifactCheckpointDigests(outputs []ArtifactCheckpointOutput) []CheckpointFingerprint {
|
||||
func artifactCheckpointDigests(outputs []CheckpointArtifact) []CheckpointFingerprint {
|
||||
values := make([]CheckpointFingerprint, 0, len(outputs))
|
||||
for i, output := range outputs {
|
||||
sum := sha256.Sum256(output.Artifact.Content)
|
||||
@@ -77,7 +53,7 @@ func artifactCheckpointDigests(outputs []ArtifactCheckpointOutput) []CheckpointF
|
||||
}
|
||||
return normalizeCheckpointFingerprints(values)
|
||||
}
|
||||
func debugArtifactCheckpointOutput(output ArtifactCheckpointOutput) map[string]any {
|
||||
func debugCheckpointArtifact(output CheckpointArtifact) map[string]any {
|
||||
artifact := output.Artifact
|
||||
schema := contracts.CloneArtifactSchema(artifact.Schema)
|
||||
digest := output.SchemaDigest
|
||||
@@ -89,13 +65,13 @@ func debugArtifactCheckpointOutput(output ArtifactCheckpointOutput) map[string]a
|
||||
content.ContentDigest = debugContentDigest(artifact.Content)
|
||||
return map[string]any{"lane_id": output.LaneID, "module_key": output.ModuleKey, "source_id": output.SourceID, "chunk_id": output.ChunkID, "chunk_index": output.ChunkIndex, "chunk_ref": output.ChunkRef, "artifact_kind": artifact.Kind, "schema": schema, "schema_digest": digest, "content": content}
|
||||
}
|
||||
func debugArtifactCheckpointOutputs(outputs []ArtifactCheckpointOutput) []map[string]any {
|
||||
func debugCheckpointArtifacts(outputs []CheckpointArtifact) []map[string]any {
|
||||
if len(outputs) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]map[string]any, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
out = append(out, debugArtifactCheckpointOutput(output))
|
||||
out = append(out, debugCheckpointArtifact(output))
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -117,7 +93,7 @@ func serializeArtifact(codec artifactCodecEntry, value any, candidate bool) (con
|
||||
return contracts.SerializedArtifact{Kind: codec.spec.Kind, Schema: contracts.CloneArtifactSchema(schema), MediaType: codec.spec.MediaType, Content: append([]byte(nil), content...), Metadata: cloneMetadata(metadata)}, nil
|
||||
}
|
||||
|
||||
func decodeCheckpointArtifact(codec artifactCodecEntry, artifact ArtifactCheckpointOutput) (any, error) {
|
||||
func decodeCheckpointArtifact(codec artifactCodecEntry, artifact CheckpointArtifact) (any, error) {
|
||||
expectedDigest := contracts.DigestArtifactSchema(codec.spec.Schema)
|
||||
if artifact.Artifact.Kind != codec.spec.Kind {
|
||||
return nil, fmt.Errorf("artifact kind %q does not match codec %q", artifact.Artifact.Kind, codec.spec.Kind)
|
||||
@@ -134,12 +110,12 @@ func decodeCheckpointArtifact(codec artifactCodecEntry, artifact ArtifactCheckpo
|
||||
return codec.decode(append([]byte(nil), artifact.Artifact.Content...))
|
||||
}
|
||||
|
||||
func checkpointArtifact(codec artifactCodecEntry, laneID, moduleKey, sourceID string, value any) (ArtifactCheckpointOutput, error) {
|
||||
func checkpointArtifact(codec artifactCodecEntry, laneID, moduleKey, sourceID string, value any) (CheckpointArtifact, error) {
|
||||
serialized, err := serializeArtifact(codec, value, false)
|
||||
if err != nil {
|
||||
return ArtifactCheckpointOutput{}, err
|
||||
return CheckpointArtifact{}, err
|
||||
}
|
||||
return ArtifactCheckpointOutput{LaneID: laneID, ModuleKey: moduleKey, SourceID: sourceID, Artifact: serialized, SchemaDigest: contracts.DigestArtifactSchema(serialized.Schema)}, nil
|
||||
return CheckpointArtifact{LaneID: laneID, ModuleKey: moduleKey, SourceID: sourceID, Artifact: serialized, SchemaDigest: contracts.DigestArtifactSchema(serialized.Schema)}, nil
|
||||
}
|
||||
|
||||
func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, prepared preparedLaneExecutor, output *RunOutput) error {
|
||||
@@ -150,7 +126,7 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
|
||||
setTypedLaneManifestMetadata(output, lane.ID, typed.extractor, typed.merger, typed.normalizer)
|
||||
|
||||
values := make([]erasedExtractArtifact, 0, len(chunks))
|
||||
serializedExtracts := make([]ArtifactCheckpointOutput, 0, len(chunks))
|
||||
serializedExtracts := make([]CheckpointArtifact, 0, len(chunks))
|
||||
extractWarnings := []contracts.Warning{}
|
||||
rejectedStart := len(output.Rejected)
|
||||
chunksDigest, err := joinedChunkDigest(chunks)
|
||||
@@ -158,7 +134,7 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
|
||||
return fmt.Errorf("digest chunks for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
extractDeps := digestFingerprints("chunks", chunksDigest)
|
||||
cp, decision := loadArtifactExtract(loader, lane.ID, lane.Extract.Module, extractDeps)
|
||||
cp, decision := loadExtract(loader, lane.ID, lane.Extract.Module, extractDeps)
|
||||
if decision.Reused {
|
||||
for _, stored := range cp.Outputs {
|
||||
if _, decodeErr := decodeCheckpointArtifact(typed.codec, stored); decodeErr != nil {
|
||||
@@ -183,7 +159,7 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
|
||||
artifact.ChunkRef = chunks[stored.ChunkIndex].Ref
|
||||
}
|
||||
values = append(values, artifact)
|
||||
serializedExtracts = append(serializedExtracts, cloneArtifactCheckpointOutput(stored))
|
||||
serializedExtracts = append(serializedExtracts, cloneCheckpointArtifact(stored))
|
||||
}
|
||||
extractWarnings = cloneWarnings(cp.Warnings)
|
||||
output.Warnings = append(output.Warnings, extractWarnings...)
|
||||
@@ -195,7 +171,7 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
|
||||
for i := range chunks {
|
||||
chunk := chunks[i]
|
||||
var accepted erasedExtractArtifact
|
||||
var serializedAccepted ArtifactCheckpointOutput
|
||||
var serializedAccepted CheckpointArtifact
|
||||
var acceptedWarnings []contracts.Warning
|
||||
ok, rejection, runErr := runWithRetry(ctx, lane.Extract.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
|
||||
started := time.Now().UTC()
|
||||
@@ -218,7 +194,7 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
|
||||
stored.ChunkID, stored.ChunkIndex, stored.ChunkRef = artifact.ChunkID, artifact.ChunkIndex, artifact.ChunkRef
|
||||
accepted, serializedAccepted = artifact, stored
|
||||
acceptedWarnings = append(cloneWarnings(result.Warnings), warnings...)
|
||||
if debugErr := writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{Stage: string(StageExtract), LaneID: lane.ID, ModuleKey: lane.Extract.Module, Attempt: attempt, StartedAt: started, Payload: map[string]any{"output": debugArtifactCheckpointOutput(stored), "warnings": debugWarningEnvelopes(acceptedWarnings)}}, llmScope)); debugErr != nil {
|
||||
if debugErr := writeDebugTimed(input.Debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{Stage: string(StageExtract), LaneID: lane.ID, ModuleKey: lane.Extract.Module, Attempt: attempt, StartedAt: started, Payload: map[string]any{"output": debugCheckpointArtifact(stored), "warnings": debugWarningEnvelopes(acceptedWarnings)}}, llmScope)); debugErr != nil {
|
||||
return false, nil, debugErr
|
||||
}
|
||||
return true, nil, nil
|
||||
@@ -236,13 +212,13 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
|
||||
extractWarnings = append(extractWarnings, acceptedWarnings...)
|
||||
output.Warnings = append(output.Warnings, acceptedWarnings...)
|
||||
}
|
||||
if err := recordArtifactExtract(checkpoints, lane.ID, lane.Extract.Module, extractDeps, serializedExtracts, cloneRejectedOutputs(output.Rejected[rejectedStart:]), extractWarnings); err != nil {
|
||||
if err := recordExtract(checkpoints, lane.ID, lane.Extract.Module, extractDeps, serializedExtracts, cloneRejectedOutputs(output.Rejected[rejectedStart:]), extractWarnings); err != nil {
|
||||
return fmt.Errorf("write extract checkpoint for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
}
|
||||
sort.SliceStable(values, func(i, j int) bool { return values[i].ChunkIndex < values[j].ChunkIndex })
|
||||
sort.SliceStable(serializedExtracts, func(i, j int) bool { return serializedExtracts[i].ChunkIndex < serializedExtracts[j].ChunkIndex })
|
||||
if err := writeDebugTimed(input.Debug, path.Join("extract", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageExtract), LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": decision.Reused, "outputs": debugArtifactCheckpointOutputs(serializedExtracts), "rejected": debugRejectedOutputEnvelopes(output.Rejected[rejectedStart:]), "warnings": debugWarningEnvelopes(extractWarnings)}}); err != nil {
|
||||
if err := writeDebugTimed(input.Debug, path.Join("extract", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageExtract), LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": decision.Reused, "outputs": debugCheckpointArtifacts(serializedExtracts), "rejected": debugRejectedOutputEnvelopes(output.Rejected[rejectedStart:]), "warnings": debugWarningEnvelopes(extractWarnings)}}); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(values) == 0 {
|
||||
@@ -254,18 +230,18 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
|
||||
mergeInputs[i] = contracts.ExtractArtifact[any]{LaneID: value.LaneID, ExtractorKey: value.ExtractorKey, SourceID: value.SourceID, ChunkID: value.ChunkID, ChunkIndex: value.ChunkIndex, ChunkRef: value.ChunkRef, Value: value.Value}
|
||||
}
|
||||
mergeDeps := artifactCheckpointDigests(serializedExtracts)
|
||||
mergeCP, mergeDecision := loadArtifactMerge(loader, lane.ID, lane.Merge.Module, mergeDeps)
|
||||
mergeCP, mergeDecision := loadMerge(loader, lane.ID, lane.Merge.Module, mergeDeps)
|
||||
if mergeDecision.Reused {
|
||||
if _, decodeErr := decodeCheckpointArtifact(typed.codec, mergeCP.Output); decodeErr != nil {
|
||||
mergeDecision = CheckpointDecision{Reason: "merge artifact checkpoint codec is incompatible: " + decodeErr.Error()}
|
||||
}
|
||||
}
|
||||
recordCheckpointEvent(output, loader, string(StageMerge), lane.ID, lane.Merge.Module, mergeDecision)
|
||||
if err := writeDebugTimed(input.Debug, path.Join("merge", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageMerge), LaneID: lane.ID, ModuleKey: lane.Merge.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": mergeDecision.Reused, "decision": mergeDecision, "source": debugSourceDocumentEnvelope(doc), "extract_outputs": debugArtifactCheckpointOutputs(serializedExtracts), "options": redactSensitiveMap(lane.Merge.Options), "metadata": redactSensitiveMap(input.Metadata)}}); err != nil {
|
||||
if err := writeDebugTimed(input.Debug, path.Join("merge", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageMerge), LaneID: lane.ID, ModuleKey: lane.Merge.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": mergeDecision.Reused, "decision": mergeDecision, "source": debugSourceDocumentEnvelope(doc), "extract_outputs": debugCheckpointArtifacts(serializedExtracts), "options": redactSensitiveMap(lane.Merge.Options), "metadata": redactSensitiveMap(input.Metadata)}}); err != nil {
|
||||
return err
|
||||
}
|
||||
var merged erasedMergeArtifact
|
||||
var serializedMerge ArtifactCheckpointOutput
|
||||
var serializedMerge CheckpointArtifact
|
||||
var mergeWarnings []contracts.Warning
|
||||
if mergeDecision.Reused {
|
||||
value, decodeErr := decodeCheckpointArtifact(typed.codec, mergeCP.Output)
|
||||
@@ -273,7 +249,7 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
|
||||
return fmt.Errorf("decode merge checkpoint for lane %q: %w", lane.ID, decodeErr)
|
||||
}
|
||||
merged = erasedMergeArtifact{LaneID: lane.ID, MergerKey: lane.Merge.Module, SourceID: doc.ID, Value: value}
|
||||
serializedMerge = hydrateCheckpointArtifact(typed.codec, cloneArtifactCheckpointOutput(mergeCP.Output), value)
|
||||
serializedMerge = hydrateCheckpointArtifact(typed.codec, cloneCheckpointArtifact(mergeCP.Output), value)
|
||||
mergeWarnings = cloneWarnings(mergeCP.Warnings)
|
||||
output.Warnings = append(output.Warnings, mergeWarnings...)
|
||||
} else {
|
||||
@@ -310,33 +286,33 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
|
||||
return nil
|
||||
}
|
||||
output.Warnings = append(output.Warnings, mergeWarnings...)
|
||||
if err := recordArtifactMerge(checkpoints, lane.ID, lane.Merge.Module, mergeDeps, serializedMerge, mergeWarnings); err != nil {
|
||||
if err := recordMerge(checkpoints, lane.ID, lane.Merge.Module, mergeDeps, serializedMerge, mergeWarnings); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := writeDebugTimed(input.Debug, path.Join("merge", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageMerge), LaneID: lane.ID, ModuleKey: lane.Merge.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": mergeDecision.Reused, "accepted": true, "output": debugArtifactCheckpointOutput(serializedMerge), "warnings": debugWarningEnvelopes(mergeWarnings)}}); err != nil {
|
||||
if err := writeDebugTimed(input.Debug, path.Join("merge", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageMerge), LaneID: lane.ID, ModuleKey: lane.Merge.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": mergeDecision.Reused, "accepted": true, "output": debugCheckpointArtifact(serializedMerge), "warnings": debugWarningEnvelopes(mergeWarnings)}}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
normalizeDeps := artifactCheckpointDigests([]ArtifactCheckpointOutput{serializedMerge})
|
||||
normalizeCP, normalizeDecision := loadArtifactNormalize(loader, lane.ID, lane.Normalize.Module, normalizeDeps)
|
||||
normalizeDeps := artifactCheckpointDigests([]CheckpointArtifact{serializedMerge})
|
||||
normalizeCP, normalizeDecision := loadNormalize(loader, lane.ID, lane.Normalize.Module, normalizeDeps)
|
||||
if normalizeDecision.Reused {
|
||||
if _, decodeErr := decodeCheckpointArtifact(typed.codec, normalizeCP.Output); decodeErr != nil {
|
||||
normalizeDecision = CheckpointDecision{Reason: "normalize artifact checkpoint codec is incompatible: " + decodeErr.Error()}
|
||||
}
|
||||
}
|
||||
recordCheckpointEvent(output, loader, string(StageNormalize), lane.ID, lane.Normalize.Module, normalizeDecision)
|
||||
if err := writeDebugTimed(input.Debug, path.Join("normalize", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageNormalize), LaneID: lane.ID, ModuleKey: lane.Normalize.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": normalizeDecision.Reused, "decision": normalizeDecision, "source": debugSourceDocumentEnvelope(doc), "merge_output": debugArtifactCheckpointOutput(serializedMerge), "options": redactSensitiveMap(lane.Normalize.Options), "metadata": redactSensitiveMap(input.Metadata)}}); err != nil {
|
||||
if err := writeDebugTimed(input.Debug, path.Join("normalize", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageNormalize), LaneID: lane.ID, ModuleKey: lane.Normalize.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": normalizeDecision.Reused, "decision": normalizeDecision, "source": debugSourceDocumentEnvelope(doc), "merge_output": debugCheckpointArtifact(serializedMerge), "options": redactSensitiveMap(lane.Normalize.Options), "metadata": redactSensitiveMap(input.Metadata)}}); err != nil {
|
||||
return err
|
||||
}
|
||||
var serializedNormalize ArtifactCheckpointOutput
|
||||
var serializedNormalize CheckpointArtifact
|
||||
var normalizeWarnings []contracts.Warning
|
||||
if normalizeDecision.Reused {
|
||||
value, decodeErr := decodeCheckpointArtifact(typed.codec, normalizeCP.Output)
|
||||
if decodeErr != nil {
|
||||
return fmt.Errorf("decode normalize checkpoint for lane %q: %w", lane.ID, decodeErr)
|
||||
}
|
||||
serializedNormalize, normalizeWarnings = hydrateCheckpointArtifact(typed.codec, cloneArtifactCheckpointOutput(normalizeCP.Output), value), cloneWarnings(normalizeCP.Warnings)
|
||||
serializedNormalize, normalizeWarnings = hydrateCheckpointArtifact(typed.codec, cloneCheckpointArtifact(normalizeCP.Output), value), cloneWarnings(normalizeCP.Warnings)
|
||||
output.Warnings = append(output.Warnings, normalizeWarnings...)
|
||||
} else {
|
||||
if err := checkpoints.NormalizeRunning(lane.ID, lane.Normalize.Module, normalizeDeps); err != nil {
|
||||
@@ -371,11 +347,11 @@ func (r *Runner) runTypedLane(ctx context.Context, input RunInput, checkpoints C
|
||||
return nil
|
||||
}
|
||||
output.Warnings = append(output.Warnings, normalizeWarnings...)
|
||||
if err := recordArtifactNormalize(checkpoints, lane.ID, lane.Normalize.Module, normalizeDeps, serializedNormalize, normalizeWarnings); err != nil {
|
||||
if err := recordNormalize(checkpoints, lane.ID, lane.Normalize.Module, normalizeDeps, serializedNormalize, normalizeWarnings); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := writeDebugTimed(input.Debug, path.Join("normalize", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageNormalize), LaneID: lane.ID, ModuleKey: lane.Normalize.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": normalizeDecision.Reused, "accepted": true, "output": debugArtifactCheckpointOutput(serializedNormalize), "warnings": debugWarningEnvelopes(normalizeWarnings)}}); err != nil {
|
||||
if err := writeDebugTimed(input.Debug, path.Join("normalize", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageNormalize), LaneID: lane.ID, ModuleKey: lane.Normalize.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": normalizeDecision.Reused, "accepted": true, "output": debugCheckpointArtifact(serializedNormalize), "warnings": debugWarningEnvelopes(normalizeWarnings)}}); err != nil {
|
||||
return err
|
||||
}
|
||||
output.NormalizeOutputs = append(output.NormalizeOutputs, contracts.SerializedOutput{LaneID: lane.ID, NormalizerKey: lane.Normalize.Module, SourceID: doc.ID, Artifact: contracts.CloneSerializedArtifact(serializedNormalize.Artifact)})
|
||||
@@ -430,7 +406,7 @@ func (r *Runner) validateTypedArtifact(ctx context.Context, codec artifactCodecE
|
||||
return nil, nil, fmt.Errorf("validator %q is incompatible with typed artifact validation", binding.Module)
|
||||
}
|
||||
artifact, _ := serializeArtifact(codec, target.value, true)
|
||||
debugCall := debugValidationCall{ValidatorName: binding.Module, Request: map[string]any{"stage": string(target.stage), "lane_id": target.laneID, "module_key": target.moduleKey, "source_id": target.sourceID, "artifact": debugArtifactCheckpointOutput(ArtifactCheckpointOutput{Artifact: artifact, SchemaDigest: contracts.DigestArtifactSchema(artifact.Schema)}), "metadata": redactSensitiveMap(target.metadata)}, Result: debugValidationResultEnvelope(result)}
|
||||
debugCall := debugValidationCall{ValidatorName: binding.Module, Request: map[string]any{"stage": string(target.stage), "lane_id": target.laneID, "module_key": target.moduleKey, "source_id": target.sourceID, "artifact": debugCheckpointArtifact(CheckpointArtifact{Artifact: artifact, SchemaDigest: contracts.DigestArtifactSchema(artifact.Schema)}), "metadata": redactSensitiveMap(target.metadata)}, Result: debugValidationResultEnvelope(result)}
|
||||
if err != nil {
|
||||
debugCall.Error = err.Error()
|
||||
}
|
||||
|
||||
@@ -20,22 +20,22 @@ func TestDecodeCheckpointArtifactRejectsIncompatibleCodecIdentityAndBytes(t *tes
|
||||
if err != nil {
|
||||
t.Fatalf("serializeArtifact: %v", err)
|
||||
}
|
||||
base := ArtifactCheckpointOutput{Artifact: artifact, SchemaDigest: contracts.DigestArtifactSchema(artifact.Schema)}
|
||||
base := CheckpointArtifact{Artifact: artifact, SchemaDigest: contracts.DigestArtifactSchema(artifact.Schema)}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(*ArtifactCheckpointOutput)
|
||||
mutate func(*CheckpointArtifact)
|
||||
want string
|
||||
}{
|
||||
{name: "missing kind", mutate: func(v *ArtifactCheckpointOutput) { v.Artifact.Kind = "" }, want: "artifact kind"},
|
||||
{name: "schema version", mutate: func(v *ArtifactCheckpointOutput) { v.Artifact.Schema.Version = "v999" }, want: "does not match codec schema"},
|
||||
{name: "schema digest", mutate: func(v *ArtifactCheckpointOutput) { v.SchemaDigest = "sha256:different" }, want: "schema digest"},
|
||||
{name: "media type", mutate: func(v *ArtifactCheckpointOutput) { v.Artifact.MediaType = "text/plain" }, want: "media type"},
|
||||
{name: "decode failure", mutate: func(v *ArtifactCheckpointOutput) { v.Artifact.Content = []byte(`{"items":[`) }, want: "unexpected EOF"},
|
||||
{name: "missing kind", mutate: func(v *CheckpointArtifact) { v.Artifact.Kind = "" }, want: "artifact kind"},
|
||||
{name: "schema version", mutate: func(v *CheckpointArtifact) { v.Artifact.Schema.Version = "v999" }, want: "does not match codec schema"},
|
||||
{name: "schema digest", mutate: func(v *CheckpointArtifact) { v.SchemaDigest = "sha256:different" }, want: "schema digest"},
|
||||
{name: "media type", mutate: func(v *CheckpointArtifact) { v.Artifact.MediaType = "text/plain" }, want: "media type"},
|
||||
{name: "decode failure", mutate: func(v *CheckpointArtifact) { v.Artifact.Content = []byte(`{"items":[`) }, want: "unexpected EOF"},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
stored := cloneArtifactCheckpointOutput(base)
|
||||
stored := cloneCheckpointArtifact(base)
|
||||
test.mutate(&stored)
|
||||
if _, err := decodeCheckpointArtifact(codec, stored); err == nil || !strings.Contains(err.Error(), test.want) {
|
||||
t.Fatalf("decode error = %v, want %q", err, test.want)
|
||||
|
||||
@@ -55,6 +55,40 @@ func (typedTestChunkValidator) Validate(context.Context, contracts.ChunkValidati
|
||||
|
||||
type typedTestSerializedValidator struct{ key string }
|
||||
|
||||
type typedTestInput struct {
|
||||
key string
|
||||
doc *source.SourceDocument
|
||||
}
|
||||
|
||||
func (v *typedTestInput) Key() string { return v.key }
|
||||
func (v *typedTestInput) Parse(context.Context, contracts.ParseRequest) (*source.SourceDocument, error) {
|
||||
return v.doc, nil
|
||||
}
|
||||
|
||||
type typedTestChunker struct {
|
||||
key string
|
||||
chunks []source.Chunk
|
||||
}
|
||||
|
||||
func (v *typedTestChunker) Key() string { return v.key }
|
||||
func (v *typedTestChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
func (v *typedTestChunker) Chunk(context.Context, contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
||||
return contracts.ChunkResult{Chunks: v.chunks}, nil
|
||||
}
|
||||
|
||||
type typedTestOutput struct{ key string }
|
||||
|
||||
func (v *typedTestOutput) Key() string { return v.key }
|
||||
func (v *typedTestOutput) Encode(context.Context, contracts.OutputRequest) (contracts.OutputResult, error) {
|
||||
return contracts.OutputResult{}, nil
|
||||
}
|
||||
func typedTestDocument() *source.SourceDocument {
|
||||
ref := source.SourceRef{SourceID: "source", StartUnitID: 1, EndUnitID: 1}
|
||||
doc := &source.SourceDocument{ID: "source", Kind: "document", Format: "text/plain", Units: []source.SourceUnit{{ID: 1, Kind: "line", Text: "text", Ref: ref}}}
|
||||
doc.Digest, _ = source.DigestDocument(doc)
|
||||
return doc
|
||||
}
|
||||
|
||||
func (v typedTestSerializedValidator) Name() string { return v.key }
|
||||
func (typedTestSerializedValidator) ExecutionClass() contracts.ExecutionClass {
|
||||
return contracts.ExecutionClassDeterministic
|
||||
@@ -177,20 +211,6 @@ func TestResolveTypedLaneRejectsIncompatibleComposition(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLegacyRawRegistrationCannotSatisfyTypedLane(t *testing.T) {
|
||||
options := completeTypedCatalogOptions()
|
||||
options.registerScoreMerger = false
|
||||
catalog := typedResolutionCatalog(t, options)
|
||||
if err := catalog.Mergers.RegisterLegacyRaw("typed/merge", func() (contracts.LegacyRawMerger, error) { return nil, nil }); err != nil {
|
||||
t.Fatalf("RegisterLegacyRaw() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
_, err := ResolvePipeline(typedResolutionProfile(), ResolveOptions{}, catalog)
|
||||
if err == nil || !strings.Contains(err.Error(), `no typed variant for artifact kind "test/score"`) {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want typed variant error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTypedVariantRegistrationRejectsDuplicates(t *testing.T) {
|
||||
registry := NewMergerRegistry()
|
||||
spec := ModuleSpec{Key: "typed/merge", Stage: StageMerge, ArtifactKind: "test/notes"}
|
||||
@@ -205,6 +225,48 @@ func TestTypedVariantRegistrationRejectsDuplicates(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestConstructorRegistrationsRejectUnconfiguredOptions(t *testing.T) {
|
||||
extractors := NewExtractorRegistry()
|
||||
if err := RegisterExtractor(extractors, ModuleSpec{Key: "typed/extract", Stage: StageExtract, ArtifactKind: "test/notes"}, func() (contracts.Extractor[codecNotes], error) {
|
||||
return typedTestExtractor[codecNotes]{key: "typed/extract"}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("RegisterExtractor() error = %v", err)
|
||||
}
|
||||
if err := extractors.validateOptions("typed/extract", map[string]any{"unexpected": true}); err == nil || !strings.Contains(err.Error(), "unknown option") {
|
||||
t.Fatalf("extractor option validation error = %v, want unknown option", err)
|
||||
}
|
||||
|
||||
mergers := NewMergerRegistry()
|
||||
if err := RegisterMerger(mergers, ModuleSpec{Key: "typed/merge", Stage: StageMerge, ArtifactKind: "test/notes"}, func() (contracts.Merger[codecNotes], error) {
|
||||
return typedTestMerger[codecNotes]{key: "typed/merge"}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("RegisterMerger() error = %v", err)
|
||||
}
|
||||
if err := mergers.validateOptions("typed/merge", "test/notes", map[string]any{"unexpected": true}); err == nil || !strings.Contains(err.Error(), "unknown option") {
|
||||
t.Fatalf("merger option validation error = %v, want unknown option", err)
|
||||
}
|
||||
|
||||
normalizers := NewNormalizerRegistry()
|
||||
if err := RegisterNormalizer(normalizers, ModuleSpec{Key: "typed/normalize", Stage: StageNormalize, ArtifactKind: "test/notes"}, func() (contracts.Normalizer[codecNotes], error) {
|
||||
return typedTestNormalizer[codecNotes]{key: "typed/normalize"}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("RegisterNormalizer() error = %v", err)
|
||||
}
|
||||
if err := normalizers.validateOptions("typed/normalize", "test/notes", map[string]any{"unexpected": true}); err == nil || !strings.Contains(err.Error(), "unknown option") {
|
||||
t.Fatalf("normalizer option validation error = %v, want unknown option", err)
|
||||
}
|
||||
|
||||
validators := NewValidatorRegistry()
|
||||
if err := RegisterTypedValidator(validators, "test/notes", ValidatorSpec{Key: "typed/check", ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.TypedValidator[codecNotes], error) {
|
||||
return typedTestValidator[codecNotes]{key: "typed/check"}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("RegisterTypedValidator() error = %v", err)
|
||||
}
|
||||
if err := validators.validateOptions(ResolvedValidator{Binding: ModuleBinding{Module: "typed/check", Options: map[string]any{"unexpected": true}}, Target: ValidatorTargetTyped, ArtifactKind: "test/notes"}); err == nil || !strings.Contains(err.Error(), "unknown option") {
|
||||
t.Fatalf("validator option validation error = %v, want unknown option", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvedPipelineDigestIncludesArtifactSchemaIdentity(t *testing.T) {
|
||||
baseOptions := completeTypedCatalogOptions()
|
||||
base, err := ResolvePipeline(typedResolutionProfile(), ResolveOptions{}, typedResolutionCatalog(t, baseOptions))
|
||||
@@ -302,18 +364,18 @@ func typedResolutionCatalog(t *testing.T, options typedCatalogOptions) ModuleCat
|
||||
func mustRegisterTypedTestBase(t *testing.T, catalog ModuleCatalog) {
|
||||
t.Helper()
|
||||
if err := catalog.Inputs.RegisterWithSpec(ModuleSpec{Key: "typed/input", Stage: StageInput}, func() (contracts.InputAdapter, error) {
|
||||
return &runnerInputAdapter{key: "typed/input", doc: validSourceDocument()}, nil
|
||||
return &typedTestInput{key: "typed/input", doc: typedTestDocument()}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register input: %v", err)
|
||||
}
|
||||
if err := catalog.Chunkers.RegisterWithSpec(ModuleSpec{Key: "typed/chunk", Stage: StageChunk}, func() (contracts.Chunker, error) {
|
||||
doc := validSourceDocument()
|
||||
return &runnerChunker{key: "typed/chunk", chunks: []source.Chunk{{ID: "chunk-1", SourceID: doc.ID, Index: 0, Ref: doc.Units[0].Ref, Content: []byte(`{"chunk":1}`), MediaType: "application/json", Units: []source.SourceUnit{doc.Units[0]}}}}, nil
|
||||
doc := typedTestDocument()
|
||||
return &typedTestChunker{key: "typed/chunk", chunks: []source.Chunk{{ID: "chunk-1", SourceID: doc.ID, Index: 0, Ref: doc.Units[0].Ref, Content: []byte(`{"chunk":1}`), MediaType: "application/json", Units: []source.SourceUnit{doc.Units[0]}}}}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register chunker: %v", err)
|
||||
}
|
||||
if err := catalog.Outputs.RegisterWithSpec(ModuleSpec{Key: "typed/output", Stage: StageOutput}, func() (contracts.OutputEncoder, error) {
|
||||
return &runnerOutputEncoder{key: "typed/output"}, nil
|
||||
return &typedTestOutput{key: "typed/output"}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register output: %v", err)
|
||||
}
|
||||
|
||||
@@ -10,9 +10,6 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
type LegacyRawValidatorConstructor func() (contracts.LegacyRawValidator, error)
|
||||
type LegacyRawValidatorBuilder func(BuildRequest) (contracts.LegacyRawValidator, error)
|
||||
|
||||
type ValidatorSpec struct {
|
||||
Key string `json:"key"`
|
||||
ExecutionClass contracts.ExecutionClass `json:"execution_class"`
|
||||
@@ -27,16 +24,12 @@ type SerializedValidatorSpec struct {
|
||||
type ValidatorTarget string
|
||||
|
||||
const (
|
||||
ValidatorTargetLegacyRaw ValidatorTarget = "legacy_raw"
|
||||
ValidatorTargetChunk ValidatorTarget = "chunk"
|
||||
ValidatorTargetSerialized ValidatorTarget = "serialized"
|
||||
ValidatorTargetTyped ValidatorTarget = "typed"
|
||||
)
|
||||
|
||||
type ValidatorRegistry struct {
|
||||
legacyBuilders map[string]LegacyRawValidatorBuilder
|
||||
legacyValidators map[string]OptionValidator
|
||||
legacySpecs map[string]ValidatorSpec
|
||||
typedEntries map[artifactVariantKey]typedValidatorEntry
|
||||
chunkEntries map[string]chunkValidatorEntry
|
||||
serializedEntries map[string]serializedValidatorEntry
|
||||
@@ -65,65 +58,17 @@ type serializedValidatorEntry struct {
|
||||
|
||||
func NewValidatorRegistry() *ValidatorRegistry {
|
||||
return &ValidatorRegistry{
|
||||
legacyBuilders: make(map[string]LegacyRawValidatorBuilder),
|
||||
legacyValidators: make(map[string]OptionValidator),
|
||||
legacySpecs: make(map[string]ValidatorSpec),
|
||||
typedEntries: make(map[artifactVariantKey]typedValidatorEntry),
|
||||
chunkEntries: make(map[string]chunkValidatorEntry),
|
||||
serializedEntries: make(map[string]serializedValidatorEntry),
|
||||
}
|
||||
}
|
||||
|
||||
func (r *ValidatorRegistry) RegisterLegacyRaw(key string, constructor LegacyRawValidatorConstructor) error {
|
||||
return r.RegisterLegacyRawWithSpec(ValidatorSpec{Key: key, ExecutionClass: contracts.ExecutionClassDeterministic}, constructor)
|
||||
}
|
||||
|
||||
func (r *ValidatorRegistry) RegisterLegacyRawWithSpec(spec ValidatorSpec, constructor LegacyRawValidatorConstructor) error {
|
||||
if constructor == nil {
|
||||
return fmt.Errorf("validator constructor for %q must not be nil", strings.TrimSpace(spec.Key))
|
||||
}
|
||||
return r.RegisterLegacyRawBuilderWithSpec(spec, allowLegacyOptions, func(BuildRequest) (contracts.LegacyRawValidator, error) {
|
||||
return constructor()
|
||||
})
|
||||
}
|
||||
|
||||
func (r *ValidatorRegistry) RegisterLegacyRawBuilderWithSpec(spec ValidatorSpec, validateOptions OptionValidator, builder LegacyRawValidatorBuilder) error {
|
||||
if r == nil {
|
||||
return fmt.Errorf("validator registry must not be nil")
|
||||
}
|
||||
normalizedSpec, err := normalizeValidatorSpec(spec)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if validateOptions == nil {
|
||||
return fmt.Errorf("validator option validator for %q must not be nil", normalizedSpec.Key)
|
||||
}
|
||||
if builder == nil {
|
||||
return fmt.Errorf("validator builder for %q must not be nil", normalizedSpec.Key)
|
||||
}
|
||||
if _, ok := r.legacyBuilders[normalizedSpec.Key]; ok {
|
||||
return fmt.Errorf("legacy raw validator %q is already registered", normalizedSpec.Key)
|
||||
}
|
||||
if r.legacyBuilders == nil {
|
||||
r.legacyBuilders = make(map[string]LegacyRawValidatorBuilder)
|
||||
}
|
||||
if r.legacyValidators == nil {
|
||||
r.legacyValidators = make(map[string]OptionValidator)
|
||||
}
|
||||
if r.legacySpecs == nil {
|
||||
r.legacySpecs = make(map[string]ValidatorSpec)
|
||||
}
|
||||
r.legacyBuilders[normalizedSpec.Key] = builder
|
||||
r.legacyValidators[normalizedSpec.Key] = validateOptions
|
||||
r.legacySpecs[normalizedSpec.Key] = normalizedSpec
|
||||
return nil
|
||||
}
|
||||
|
||||
func RegisterTypedValidator[T any](registry *ValidatorRegistry, kind contracts.ArtifactKind, spec ValidatorSpec, constructor func() (contracts.TypedValidator[T], error)) error {
|
||||
if constructor == nil {
|
||||
return fmt.Errorf("validator constructor for %q must not be nil", strings.TrimSpace(spec.Key))
|
||||
}
|
||||
return RegisterTypedValidatorBuilder(registry, kind, spec, allowLegacyOptions, func(BuildRequest) (contracts.TypedValidator[T], error) {
|
||||
return RegisterTypedValidatorBuilder(registry, kind, spec, rejectUnconfiguredOptions, func(BuildRequest) (contracts.TypedValidator[T], error) {
|
||||
return constructor()
|
||||
})
|
||||
}
|
||||
@@ -180,7 +125,7 @@ func RegisterChunkValidator(registry *ValidatorRegistry, spec ValidatorSpec, con
|
||||
if constructor == nil {
|
||||
return fmt.Errorf("validator constructor for %q must not be nil", strings.TrimSpace(spec.Key))
|
||||
}
|
||||
return RegisterChunkValidatorBuilder(registry, spec, allowLegacyOptions, func(BuildRequest) (contracts.ChunkValidator, error) {
|
||||
return RegisterChunkValidatorBuilder(registry, spec, rejectUnconfiguredOptions, func(BuildRequest) (contracts.ChunkValidator, error) {
|
||||
return constructor()
|
||||
})
|
||||
}
|
||||
@@ -213,7 +158,7 @@ func RegisterSerializedValidator(registry *ValidatorRegistry, spec SerializedVal
|
||||
if constructor == nil {
|
||||
return fmt.Errorf("validator constructor for %q must not be nil", strings.TrimSpace(spec.Key))
|
||||
}
|
||||
return RegisterSerializedValidatorBuilder(registry, spec, allowLegacyOptions, func(BuildRequest) (contracts.SerializedValidator, error) {
|
||||
return RegisterSerializedValidatorBuilder(registry, spec, rejectUnconfiguredOptions, func(BuildRequest) (contracts.SerializedValidator, error) {
|
||||
return constructor()
|
||||
})
|
||||
}
|
||||
@@ -246,39 +191,6 @@ func RegisterSerializedValidatorBuilder(registry *ValidatorRegistry, spec Serial
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *ValidatorRegistry) BuildLegacyRaw(key string) (contracts.LegacyRawValidator, error) {
|
||||
return r.BuildLegacyRawWithRequest(key, BuildRequest{})
|
||||
}
|
||||
|
||||
func (r *ValidatorRegistry) BuildLegacyRawWithRequest(key string, request BuildRequest) (contracts.LegacyRawValidator, error) {
|
||||
if r == nil {
|
||||
return nil, fmt.Errorf("validator registry must not be nil")
|
||||
}
|
||||
normalizedKey := strings.TrimSpace(key)
|
||||
if normalizedKey == "" {
|
||||
return nil, fmt.Errorf("validator key must not be empty")
|
||||
}
|
||||
builder, ok := r.legacyBuilders[normalizedKey]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("legacy raw validator %q is not registered", normalizedKey)
|
||||
}
|
||||
validator, err := builder(cloneBuildRequest(request))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build validator %q: %w", normalizedKey, err)
|
||||
}
|
||||
if validator == nil {
|
||||
return nil, fmt.Errorf("validator %q constructor returned nil", normalizedKey)
|
||||
}
|
||||
if validator.Name() != normalizedKey {
|
||||
return nil, fmt.Errorf("validator %q returned name %q", normalizedKey, validator.Name())
|
||||
}
|
||||
spec := r.legacySpecs[normalizedKey]
|
||||
if validator.ExecutionClass() != spec.ExecutionClass {
|
||||
return nil, fmt.Errorf("validator %q returned execution class %q, want %q", normalizedKey, validator.ExecutionClass(), spec.ExecutionClass)
|
||||
}
|
||||
return validator, nil
|
||||
}
|
||||
|
||||
func (r *ValidatorRegistry) validateOptions(resolved ResolvedValidator) error {
|
||||
if r == nil {
|
||||
return fmt.Errorf("validator registry must not be nil")
|
||||
@@ -301,8 +213,6 @@ func (r *ValidatorRegistry) validateOptions(resolved ResolvedValidator) error {
|
||||
if ok {
|
||||
validator = entry.validateOptions
|
||||
}
|
||||
default:
|
||||
validator = r.legacyValidators[key]
|
||||
}
|
||||
if validator == nil {
|
||||
return fmt.Errorf("validator %q construction entry is not registered", key)
|
||||
@@ -314,10 +224,6 @@ func (r *ValidatorRegistry) Spec(key string) (ValidatorSpec, bool) {
|
||||
if r == nil {
|
||||
return ValidatorSpec{}, false
|
||||
}
|
||||
spec, ok := r.legacySpecs[strings.TrimSpace(key)]
|
||||
if ok {
|
||||
return spec, true
|
||||
}
|
||||
normalized := strings.TrimSpace(key)
|
||||
if entry, found := r.chunkEntries[normalized]; found {
|
||||
return entry.spec, true
|
||||
@@ -329,7 +235,7 @@ func (r *ValidatorRegistry) Spec(key string) (ValidatorSpec, bool) {
|
||||
entry, found := r.typedEntry(normalized, kinds[0])
|
||||
return entry.spec, found
|
||||
}
|
||||
return spec, ok
|
||||
return ValidatorSpec{}, false
|
||||
}
|
||||
|
||||
func (r *ValidatorRegistry) typedEntry(key string, kind contracts.ArtifactKind) (typedValidatorEntry, bool) {
|
||||
@@ -372,13 +278,15 @@ func (r *ValidatorRegistry) registeredTypedKinds(key string) []contracts.Artifac
|
||||
}
|
||||
|
||||
func (r *ValidatorRegistry) RegisteredSpecs() []ValidatorSpec {
|
||||
if r == nil || len(r.legacySpecs) == 0 {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
keys := sortedRegistryKeys(r.legacySpecs)
|
||||
keys := r.RegisteredKeys()
|
||||
specs := make([]ValidatorSpec, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
specs = append(specs, r.legacySpecs[key])
|
||||
if spec, ok := r.Spec(key); ok {
|
||||
specs = append(specs, spec)
|
||||
}
|
||||
}
|
||||
return specs
|
||||
}
|
||||
@@ -388,9 +296,6 @@ func (r *ValidatorRegistry) RegisteredKeys() []string {
|
||||
return nil
|
||||
}
|
||||
keys := make(map[string]struct{})
|
||||
for key := range r.legacySpecs {
|
||||
keys[key] = struct{}{}
|
||||
}
|
||||
for key := range r.typedEntries {
|
||||
keys[key.module] = struct{}{}
|
||||
}
|
||||
|
||||
@@ -1,117 +0,0 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func TestValidatorRegistryBehavior(t *testing.T) {
|
||||
registry := NewValidatorRegistry()
|
||||
if err := registry.RegisterLegacyRaw(" generic-validator ", validatorConstructor("generic-validator", contracts.ExecutionClassDeterministic)); err != nil {
|
||||
t.Fatalf("Register() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
validator, err := registry.BuildLegacyRaw("generic-validator")
|
||||
if err != nil {
|
||||
t.Fatalf("Build() error = %v, want nil", err)
|
||||
}
|
||||
if validator.Name() != "generic-validator" {
|
||||
t.Fatalf("validator name = %q, want generic-validator", validator.Name())
|
||||
}
|
||||
|
||||
spec, ok := registry.Spec(" generic-validator ")
|
||||
if !ok {
|
||||
t.Fatal("Spec() ok = false, want true")
|
||||
}
|
||||
want := ValidatorSpec{Key: "generic-validator", ExecutionClass: contracts.ExecutionClassDeterministic}
|
||||
if !reflect.DeepEqual(spec, want) {
|
||||
t.Fatalf("Spec() = %#v, want %#v", spec, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatorRegistryRegistersSpecs(t *testing.T) {
|
||||
registry := NewValidatorRegistry()
|
||||
spec := ValidatorSpec{Key: " llm-validator ", ExecutionClass: contracts.ExecutionClassLLMBacked}
|
||||
if err := registry.RegisterLegacyRawWithSpec(spec, validatorConstructor("llm-validator", contracts.ExecutionClassLLMBacked)); err != nil {
|
||||
t.Fatalf("RegisterWithSpec() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
got, ok := registry.Spec("llm-validator")
|
||||
if !ok {
|
||||
t.Fatal("Spec() ok = false, want true")
|
||||
}
|
||||
want := ValidatorSpec{Key: "llm-validator", ExecutionClass: contracts.ExecutionClassLLMBacked}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("Spec() = %#v, want %#v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatorRegistryRegisteredSpecsAreSorted(t *testing.T) {
|
||||
registry := NewValidatorRegistry()
|
||||
for _, key := range []string{"zeta", "alpha"} {
|
||||
if err := registry.RegisterLegacyRaw(key, validatorConstructor(key, contracts.ExecutionClassDeterministic)); err != nil {
|
||||
t.Fatalf("Register(%q) error = %v", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
specs := registry.RegisteredSpecs()
|
||||
if len(specs) != 2 || specs[0].Key != "alpha" || specs[1].Key != "zeta" {
|
||||
t.Fatalf("RegisteredSpecs() = %#v, want sorted specs", specs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatorRegistryRejectsUnsupportedExecutionClass(t *testing.T) {
|
||||
registry := NewValidatorRegistry()
|
||||
err := registry.RegisterLegacyRawWithSpec(
|
||||
ValidatorSpec{Key: "invalid-validator", ExecutionClass: contracts.ExecutionClass("unsupported")},
|
||||
validatorConstructor("invalid-validator", contracts.ExecutionClass("unsupported")),
|
||||
)
|
||||
if err == nil {
|
||||
t.Fatal("RegisterWithSpec() error = nil, want unsupported execution class error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatorRegistryRejectsConstructorExecutionClassMismatch(t *testing.T) {
|
||||
registry := NewValidatorRegistry()
|
||||
if err := registry.RegisterLegacyRawWithSpec(
|
||||
ValidatorSpec{Key: "validator", ExecutionClass: contracts.ExecutionClassDeterministic},
|
||||
validatorConstructor("validator", contracts.ExecutionClassLLMBacked),
|
||||
); err != nil {
|
||||
t.Fatalf("RegisterWithSpec() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
_, err := registry.BuildLegacyRaw("validator")
|
||||
if err == nil {
|
||||
t.Fatal("Build() error = nil, want execution class mismatch")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "execution class") {
|
||||
t.Fatalf("Build() error = %q, want execution class context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
type testValidator struct {
|
||||
name string
|
||||
executionClass contracts.ExecutionClass
|
||||
}
|
||||
|
||||
func validatorConstructor(name string, executionClass contracts.ExecutionClass) LegacyRawValidatorConstructor {
|
||||
return func() (contracts.LegacyRawValidator, error) {
|
||||
return testValidator{name: name, executionClass: executionClass}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (validator testValidator) Name() string {
|
||||
return validator.name
|
||||
}
|
||||
|
||||
func (validator testValidator) ExecutionClass() contracts.ExecutionClass {
|
||||
return validator.executionClass
|
||||
}
|
||||
|
||||
func (validator testValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
@@ -1,467 +0,0 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func TestWalkingSkeletonFixture(t *testing.T) {
|
||||
inputBytes := readTestFixture(t, "testdata/walking_skeleton_input.json")
|
||||
expectedBytes := readTestFixture(t, "testdata/walking_skeleton_output.json")
|
||||
llmClient := &walkingSkeletonLLMClient{}
|
||||
|
||||
resolved, err := ResolvePipeline(walkingSkeletonProfile(), ResolveOptions{}, walkingSkeletonCatalog(t))
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
output, err := newPreparedRunner(t, walkingSkeletonRegistries(t)).Run(context.Background(), RunInput{
|
||||
pipeline: resolved,
|
||||
SourceID: "fixture-source",
|
||||
Path: "walking_skeleton_input.json",
|
||||
RawInput: inputBytes,
|
||||
llmClient: llmClient,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
if len(output.OutputFiles) != 1 {
|
||||
t.Fatalf("len(OutputFiles) = %d, want 1", len(output.OutputFiles))
|
||||
}
|
||||
if output.OutputFiles[0].ContentType != "application/json" {
|
||||
t.Fatalf("ContentType = %q, want application/json", output.OutputFiles[0].ContentType)
|
||||
}
|
||||
assertStructuralJSONEqual(t, output.OutputFiles[0].Bytes, expectedBytes)
|
||||
if llmClient.calls != 3 {
|
||||
t.Fatalf("LLM calls = %d, want extractor calls plus normalizer call", llmClient.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWalkingSkeletonResolutionRejectsMissingCapability(t *testing.T) {
|
||||
catalog := walkingSkeletonCatalog(t)
|
||||
catalog.Extractors = NewExtractorRegistry()
|
||||
if err := catalog.Extractors.RegisterLegacyRawWithSpec(ModuleSpec{
|
||||
Key: "fake/extract",
|
||||
Stage: StageExtract,
|
||||
Requires: []string{"missing"},
|
||||
Provides: []string{"fake_artifacts"},
|
||||
}, func() (contracts.LegacyRawExtractor, error) {
|
||||
return walkingSkeletonExtractor{}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("RegisterWithSpec() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
_, err := ResolvePipeline(walkingSkeletonProfile(), ResolveOptions{}, catalog)
|
||||
if err == nil {
|
||||
t.Fatal("ResolvePipeline() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "missing") {
|
||||
t.Fatalf("ResolvePipeline() error = %q, want missing capability", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestWalkingSkeletonResolutionRejectsUnknownOnlyLane(t *testing.T) {
|
||||
_, err := ResolvePipeline(walkingSkeletonProfile(), ResolveOptions{Only: []string{"missing"}}, walkingSkeletonCatalog(t))
|
||||
if err == nil {
|
||||
t.Fatal("ResolvePipeline() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "missing") || !strings.Contains(err.Error(), "not declared") {
|
||||
t.Fatalf("ResolvePipeline() error = %q, want unknown lane error", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func walkingSkeletonProfile() PipelineProfile {
|
||||
return PipelineProfile{
|
||||
ID: "walking-skeleton",
|
||||
Input: Binding("fake/input"),
|
||||
Chunk: Binding("fake/chunk"),
|
||||
Output: Binding("json"),
|
||||
Artifacts: map[string]ArtifactLaneProfile{
|
||||
"events": {Extract: Binding("fake/extract")},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func walkingSkeletonCatalog(t *testing.T) ModuleCatalog {
|
||||
t.Helper()
|
||||
|
||||
catalog := ModuleCatalog{
|
||||
Inputs: NewInputAdapterRegistry(),
|
||||
Chunkers: NewChunkerRegistry(),
|
||||
ArtifactCodecs: NewArtifactCodecRegistry(),
|
||||
Extractors: NewExtractorRegistry(),
|
||||
Mergers: NewMergerRegistry(),
|
||||
Normalizers: NewNormalizerRegistry(),
|
||||
ValidatorChains: NewValidatorChainRegistry(),
|
||||
Outputs: NewOutputEncoderRegistry(),
|
||||
}
|
||||
if err := catalog.Inputs.RegisterWithSpec(ModuleSpec{
|
||||
Key: "fake/input",
|
||||
Stage: StageInput,
|
||||
Provides: []string{"plain_text"},
|
||||
}, func() (contracts.InputAdapter, error) {
|
||||
return walkingSkeletonInput{}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register fake input: %v", err)
|
||||
}
|
||||
if err := catalog.Chunkers.RegisterWithSpec(ModuleSpec{
|
||||
Key: "fake/chunk",
|
||||
Stage: StageChunk,
|
||||
Requires: []string{"plain_text"},
|
||||
Provides: []string{"chunks"},
|
||||
}, func() (contracts.Chunker, error) {
|
||||
return walkingSkeletonChunker{}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register fake chunker: %v", err)
|
||||
}
|
||||
if err := catalog.Extractors.RegisterLegacyRawWithSpec(ModuleSpec{
|
||||
Key: "fake/extract",
|
||||
Stage: StageExtract,
|
||||
Requires: []string{"chunks"},
|
||||
Provides: []string{"fake_artifacts"},
|
||||
}, func() (contracts.LegacyRawExtractor, error) {
|
||||
return walkingSkeletonExtractor{}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register fake extractor: %v", err)
|
||||
}
|
||||
if err := catalog.Mergers.RegisterLegacyRawWithSpec(ModuleSpec{
|
||||
Key: DefaultMergeModule,
|
||||
Stage: StageMerge,
|
||||
Requires: []string{"fake_artifacts"},
|
||||
}, func() (contracts.LegacyRawMerger, error) {
|
||||
return walkingSkeletonMerger{}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register append-order merger: %v", err)
|
||||
}
|
||||
if err := catalog.Normalizers.RegisterLegacyRawWithSpec(ModuleSpec{
|
||||
Key: DefaultNormalizeModule,
|
||||
Stage: StageNormalize,
|
||||
}, func() (contracts.LegacyRawNormalizer, error) {
|
||||
return walkingSkeletonNormalizer{}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register no-op normalizer: %v", err)
|
||||
}
|
||||
if err := catalog.Outputs.RegisterWithSpec(ModuleSpec{
|
||||
Key: "json",
|
||||
Stage: StageOutput,
|
||||
}, func() (contracts.OutputEncoder, error) {
|
||||
return walkingSkeletonOutput{}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register fake output: %v", err)
|
||||
}
|
||||
return catalog
|
||||
}
|
||||
|
||||
func walkingSkeletonRegistries(t *testing.T) Registries {
|
||||
t.Helper()
|
||||
|
||||
catalog := walkingSkeletonCatalog(t)
|
||||
return Registries{
|
||||
Inputs: catalog.Inputs,
|
||||
Chunkers: catalog.Chunkers,
|
||||
ArtifactCodecs: catalog.ArtifactCodecs,
|
||||
Extractors: catalog.Extractors,
|
||||
Mergers: catalog.Mergers,
|
||||
Normalizers: catalog.Normalizers,
|
||||
Outputs: catalog.Outputs,
|
||||
}
|
||||
}
|
||||
|
||||
type walkingSkeletonInput struct{}
|
||||
|
||||
func (input walkingSkeletonInput) Key() string {
|
||||
return "fake/input"
|
||||
}
|
||||
|
||||
func (input walkingSkeletonInput) Parse(ctx context.Context, req contracts.ParseRequest) (*source.SourceDocument, error) {
|
||||
var fixture struct {
|
||||
ID string `json:"id"`
|
||||
Units []struct {
|
||||
ID int `json:"id"`
|
||||
Text string `json:"text"`
|
||||
} `json:"units"`
|
||||
}
|
||||
if err := json.Unmarshal(req.Raw, &fixture); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
units := make([]source.SourceUnit, 0, len(fixture.Units))
|
||||
for _, unit := range fixture.Units {
|
||||
units = append(units, source.SourceUnit{
|
||||
ID: unit.ID,
|
||||
Kind: "unit",
|
||||
Text: unit.Text,
|
||||
Ref: source.SourceRef{
|
||||
SourceID: fixture.ID,
|
||||
StartUnitID: unit.ID,
|
||||
EndUnitID: unit.ID,
|
||||
},
|
||||
})
|
||||
}
|
||||
return &source.SourceDocument{
|
||||
ID: fixture.ID,
|
||||
Kind: "fixture",
|
||||
Format: "application/json",
|
||||
Digest: rawDigest(req.Raw),
|
||||
Units: units,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type walkingSkeletonChunker struct{}
|
||||
|
||||
func (chunker walkingSkeletonChunker) Key() string {
|
||||
return "fake/chunk"
|
||||
}
|
||||
|
||||
func (chunker walkingSkeletonChunker) ReferenceSlots() []contracts.ReferenceSlot {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (chunker walkingSkeletonChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
||||
if len(req.Source.Units) < 3 {
|
||||
return contracts.ChunkResult{}, fmt.Errorf("fixture source must contain at least three units")
|
||||
}
|
||||
return contracts.ChunkResult{
|
||||
Chunks: []source.Chunk{
|
||||
{
|
||||
ID: req.Source.ID + ":chunk:0",
|
||||
SourceID: req.Source.ID,
|
||||
Index: 0,
|
||||
Ref: source.SourceRef{SourceID: req.Source.ID, StartUnitID: req.Source.Units[0].ID, EndUnitID: req.Source.Units[1].ID},
|
||||
Content: []byte(`{"units":[1,2]}`),
|
||||
MediaType: "application/json",
|
||||
Units: append([]source.SourceUnit(nil), req.Source.Units[:2]...),
|
||||
},
|
||||
{
|
||||
ID: req.Source.ID + ":chunk:1",
|
||||
SourceID: req.Source.ID,
|
||||
Index: 1,
|
||||
Ref: source.SourceRef{SourceID: req.Source.ID, StartUnitID: req.Source.Units[2].ID, EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID},
|
||||
Content: []byte(`{"units":[3]}`),
|
||||
MediaType: "application/json",
|
||||
Units: append([]source.SourceUnit(nil), req.Source.Units[2:]...),
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
type walkingSkeletonExtractor struct{}
|
||||
|
||||
func (extractor walkingSkeletonExtractor) Key() string {
|
||||
return "fake/extract"
|
||||
}
|
||||
|
||||
func (extractor walkingSkeletonExtractor) ReferenceSlots() []contracts.ReferenceSlot {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (extractor walkingSkeletonExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
|
||||
var response struct {
|
||||
Call int `json:"call"`
|
||||
}
|
||||
if _, err := req.LLMClient.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
StageName: "fake/extract",
|
||||
PromptID: "fake.event",
|
||||
PromptVersion: "v1",
|
||||
}, &response); err != nil {
|
||||
return contracts.ExtractionResult{}, err
|
||||
}
|
||||
|
||||
payload, err := json.Marshal(map[string]any{
|
||||
"chunk_id": req.Chunk.ID,
|
||||
"llm_call": response.Call,
|
||||
"text": chunkText(req.Chunk.Units),
|
||||
})
|
||||
if err != nil {
|
||||
return contracts.ExtractionResult{}, err
|
||||
}
|
||||
|
||||
return contracts.ExtractionResult{
|
||||
Output: contracts.ExtractOutput{
|
||||
Schema: contracts.ResponseSchema{ID: "fake_event", Name: "fake_event", Version: "v1"},
|
||||
Payload: contracts.RawPayload{
|
||||
Content: payload,
|
||||
MediaType: "application/json",
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
type walkingSkeletonLLMClient struct {
|
||||
calls int
|
||||
}
|
||||
|
||||
func (client *walkingSkeletonLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
client.calls++
|
||||
if response, ok := out.(*struct {
|
||||
Call int `json:"call"`
|
||||
}); ok {
|
||||
response.Call = client.calls
|
||||
}
|
||||
content, err := json.Marshal(map[string]any{"call": client.calls})
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{
|
||||
Content: content,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type walkingSkeletonMerger struct{}
|
||||
|
||||
func (merger walkingSkeletonMerger) Key() string {
|
||||
return DefaultMergeModule
|
||||
}
|
||||
|
||||
func (merger walkingSkeletonMerger) Merge(ctx context.Context, req contracts.MergeRequest) (contracts.MergeResult, error) {
|
||||
outputs := make([]json.RawMessage, 0, len(req.ExtractOutputs))
|
||||
for _, output := range req.ExtractOutputs {
|
||||
outputs = append(outputs, json.RawMessage(output.Payload.Content))
|
||||
}
|
||||
content, err := json.Marshal(map[string]any{"outputs": outputs})
|
||||
if err != nil {
|
||||
return contracts.MergeResult{}, err
|
||||
}
|
||||
return contracts.MergeResult{
|
||||
Output: contracts.MergeOutput{
|
||||
LaneID: req.LaneID,
|
||||
SourceID: req.Source.ID,
|
||||
Schema: contracts.ResponseSchema{ID: "fake_event", Name: "fake_event", Version: "v1"},
|
||||
Payload: contracts.RawPayload{
|
||||
Content: content,
|
||||
MediaType: "application/json",
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
type walkingSkeletonNormalizer struct{}
|
||||
|
||||
func (normalizer walkingSkeletonNormalizer) Key() string {
|
||||
return DefaultNormalizeModule
|
||||
}
|
||||
|
||||
func (normalizer walkingSkeletonNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (normalizer walkingSkeletonNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
|
||||
var response struct {
|
||||
Call int `json:"call"`
|
||||
}
|
||||
if _, err := req.LLMClient.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
StageName: "fake/normalize",
|
||||
PromptID: "fake.normalize",
|
||||
PromptVersion: "v1",
|
||||
}, &response); err != nil {
|
||||
return contracts.NormalizeResult{}, err
|
||||
}
|
||||
return contracts.NormalizeResult{
|
||||
Output: contracts.NormalizeOutput{
|
||||
LaneID: req.LaneID,
|
||||
SourceID: req.MergeOutput.SourceID,
|
||||
Schema: req.MergeOutput.Schema,
|
||||
Payload: req.MergeOutput.Payload,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
type walkingSkeletonOutput struct{}
|
||||
|
||||
func (output walkingSkeletonOutput) Key() string {
|
||||
return "json"
|
||||
}
|
||||
|
||||
func (output walkingSkeletonOutput) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
|
||||
type rawOutput struct {
|
||||
LaneID string `json:"lane_id"`
|
||||
NormalizerKey string `json:"normalizer_key"`
|
||||
SourceID string `json:"source_id"`
|
||||
Schema contracts.ResponseSchema `json:"schema"`
|
||||
MediaType string `json:"media_type"`
|
||||
Content json.RawMessage `json:"content"`
|
||||
}
|
||||
rawOutputs := make([]rawOutput, 0, len(req.NormalizeOutputs))
|
||||
for _, output := range req.NormalizeOutputs {
|
||||
rawOutputs = append(rawOutputs, rawOutput{
|
||||
LaneID: output.LaneID,
|
||||
NormalizerKey: output.NormalizerKey,
|
||||
SourceID: output.SourceID,
|
||||
Schema: contracts.ResponseSchema{ID: output.Artifact.Schema.ID, Name: output.Artifact.Schema.Name, Version: output.Artifact.Schema.Version},
|
||||
MediaType: output.Artifact.MediaType,
|
||||
Content: json.RawMessage(output.Artifact.Content),
|
||||
})
|
||||
}
|
||||
encoded, err := json.Marshal(struct {
|
||||
Manifest artifacts.RunManifest `json:"manifest"`
|
||||
NormalizeOutputs []rawOutput `json:"normalize_outputs"`
|
||||
}{
|
||||
Manifest: artifacts.RunManifest{
|
||||
PipelineID: req.Manifest.PipelineID,
|
||||
PipelineDigest: req.Manifest.PipelineDigest,
|
||||
ArtifactLanes: req.Manifest.ArtifactLanes,
|
||||
ValidationStatus: req.Manifest.ValidationStatus,
|
||||
},
|
||||
NormalizeOutputs: rawOutputs,
|
||||
})
|
||||
if err != nil {
|
||||
return contracts.OutputResult{}, err
|
||||
}
|
||||
return contracts.OutputResult{
|
||||
Files: []contracts.OutputFile{
|
||||
{Name: "output.json", ContentType: "application/json", Bytes: encoded},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func readTestFixture(t *testing.T, path string) []byte {
|
||||
t.Helper()
|
||||
|
||||
bytes, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read fixture %q: %v", path, err)
|
||||
}
|
||||
return bytes
|
||||
}
|
||||
|
||||
func assertStructuralJSONEqual(t *testing.T, gotBytes, wantBytes []byte) {
|
||||
t.Helper()
|
||||
|
||||
var got any
|
||||
if err := json.Unmarshal(gotBytes, &got); err != nil {
|
||||
t.Fatalf("unmarshal actual JSON: %v\n%s", err, gotBytes)
|
||||
}
|
||||
var want any
|
||||
if err := json.Unmarshal(wantBytes, &want); err != nil {
|
||||
t.Fatalf("unmarshal expected JSON: %v\n%s", err, wantBytes)
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
gotFormatted, _ := json.MarshalIndent(got, "", " ")
|
||||
wantFormatted, _ := json.MarshalIndent(want, "", " ")
|
||||
t.Fatalf("actual JSON:\n%s\nwant:\n%s", gotFormatted, wantFormatted)
|
||||
}
|
||||
}
|
||||
|
||||
func chunkText(units []source.SourceUnit) string {
|
||||
parts := make([]string, 0, len(units))
|
||||
for _, unit := range units {
|
||||
parts = append(parts, unit.Text)
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
func rawDigest(raw []byte) string {
|
||||
sum := sha256.Sum256(raw)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
Reference in New Issue
Block a user