304 lines
9.1 KiB
Go
304 lines
9.1 KiB
Go
package scenes
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
)
|
|
|
|
const Key = "dnd/scenes"
|
|
|
|
var requiredCapabilities = []string{
|
|
"source.transcript",
|
|
}
|
|
|
|
var providedCapabilities = []string{
|
|
"chunks",
|
|
"chunks.scenes",
|
|
}
|
|
|
|
var _ contracts.Chunker = (*Chunker)(nil)
|
|
var _ contracts.ManifestMetadataProvider = (*Chunker)(nil)
|
|
|
|
type Chunker struct{}
|
|
|
|
func New() *Chunker {
|
|
return &Chunker{}
|
|
}
|
|
|
|
func (c *Chunker) Key() string {
|
|
return Key
|
|
}
|
|
|
|
func (c *Chunker) ManifestMetadata() map[string]any {
|
|
promptMetadata := scenesPromptBundle.Metadata()
|
|
metadata := map[string]any{
|
|
"prompt_id": PromptID,
|
|
"prompt_version": promptMetadata.PromptVersion,
|
|
"prompt_sha256": promptMetadata.SHA256,
|
|
"response_schema_key": string(ResponseSchemaKey),
|
|
"response_schema_id": ResponseSchemaID,
|
|
"response_schema_name": ResponseSchemaName,
|
|
}
|
|
if schema, err := loadResponseSchema(); err == nil {
|
|
metadata["response_schema_version"] = schema.Version
|
|
metadata["response_schema_sha256"] = schema.SHA256
|
|
}
|
|
return metadata
|
|
}
|
|
|
|
func (c *Chunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
|
if c == nil {
|
|
return contracts.ChunkResult{}, chunkerErrorf("chunker must not be nil")
|
|
}
|
|
if ctx == nil {
|
|
return contracts.ChunkResult{}, chunkerErrorf("context must not be nil")
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return contracts.ChunkResult{}, chunkerErrorf("context error before chunking: %w", err)
|
|
}
|
|
if req.Source == nil {
|
|
return contracts.ChunkResult{}, chunkerErrorf("source must not be nil")
|
|
}
|
|
if len(req.Source.Units) == 0 {
|
|
return contracts.ChunkResult{}, chunkerErrorf("source units must not be empty")
|
|
}
|
|
if err := source.ValidateDocument(req.Source); err != nil {
|
|
return contracts.ChunkResult{}, chunkerErrorf("validate source document: %w", err)
|
|
}
|
|
if req.LLMClient == nil {
|
|
return contracts.ChunkResult{}, chunkerErrorf("LLM client must not be nil")
|
|
}
|
|
if len(req.Options) > 0 {
|
|
return contracts.ChunkResult{}, chunkerErrorf("options are not supported")
|
|
}
|
|
|
|
system, user, _, err := renderPrompt(req)
|
|
if err != nil {
|
|
return contracts.ChunkResult{}, chunkerErrorf("render prompt: %w", err)
|
|
}
|
|
schema, err := loadResponseSchema()
|
|
if err != nil {
|
|
return contracts.ChunkResult{}, chunkerErrorf("load response schema %q: %w", ResponseSchemaKey, err)
|
|
}
|
|
|
|
var response chunkResponse
|
|
if _, err := req.LLMClient.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
|
StageName: Key,
|
|
Messages: []contracts.LLMMessage{
|
|
{Role: "system", Content: system},
|
|
{Role: "user", Content: user},
|
|
},
|
|
ResponseSchemaName: schema.Name,
|
|
ResponseSchema: schema.JSONSchema,
|
|
}, &response); err != nil {
|
|
return contracts.ChunkResult{}, chunkerErrorf("complete structured output: %w", err)
|
|
}
|
|
|
|
chunks, err := chunksFromResponse(req.Source, response)
|
|
if err != nil {
|
|
return contracts.ChunkResult{}, chunkerErrorf("malformed structured output: %w", err)
|
|
}
|
|
return contracts.ChunkResult{
|
|
Chunks: chunks,
|
|
Warnings: warningsFromCaveats(response.BoundaryCaveats),
|
|
}, nil
|
|
}
|
|
|
|
func ModuleSpec() pipeline.ModuleSpec {
|
|
return pipeline.ModuleSpec{
|
|
Key: Key,
|
|
Stage: pipeline.StageChunk,
|
|
Requires: append([]string(nil), requiredCapabilities...),
|
|
Provides: append([]string(nil), providedCapabilities...),
|
|
}
|
|
}
|
|
|
|
func Register(registry *pipeline.ChunkerRegistry) error {
|
|
return registry.RegisterWithSpec(ModuleSpec(), func() (contracts.Chunker, error) {
|
|
return New(), nil
|
|
})
|
|
}
|
|
|
|
func chunksFromResponse(doc *source.SourceDocument, response chunkResponse) ([]contracts.SourceChunk, error) {
|
|
if response.Scenes == nil {
|
|
return nil, fmt.Errorf("scenes must be present")
|
|
}
|
|
if len(response.Scenes) == 0 {
|
|
return nil, fmt.Errorf("scenes must not be empty")
|
|
}
|
|
|
|
unitIndexes := make(map[string]int, len(doc.Units))
|
|
for i, unit := range doc.Units {
|
|
unitIndexes[unit.ID] = i
|
|
}
|
|
|
|
chunks := make([]contracts.SourceChunk, 0, len(response.Scenes))
|
|
previousEnd := -1
|
|
for i, scene := range response.Scenes {
|
|
normalized, err := normalizeScene(i, scene)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
startIndex, ok := unitIndexes[normalized.StartUnitID]
|
|
if !ok {
|
|
return nil, fmt.Errorf("scene[%d] start_unit_id %q was not found", i, normalized.StartUnitID)
|
|
}
|
|
endIndex, ok := unitIndexes[normalized.EndUnitID]
|
|
if !ok {
|
|
return nil, fmt.Errorf("scene[%d] end_unit_id %q was not found", i, normalized.EndUnitID)
|
|
}
|
|
if startIndex > endIndex {
|
|
return nil, fmt.Errorf("scene[%d] start_unit_id %q appears after end_unit_id %q", i, normalized.StartUnitID, normalized.EndUnitID)
|
|
}
|
|
|
|
if i == 0 && startIndex != 0 {
|
|
return nil, fmt.Errorf("first scene must start at first source unit %q", doc.Units[0].ID)
|
|
}
|
|
if i > 0 {
|
|
if startIndex <= previousEnd {
|
|
return nil, fmt.Errorf("scene[%d] overlaps previous scene", i)
|
|
}
|
|
if startIndex > previousEnd+1 {
|
|
return nil, fmt.Errorf("scene[%d] leaves a gap after previous scene", i)
|
|
}
|
|
}
|
|
previousEnd = endIndex
|
|
|
|
units := cloneUnits(doc.Units[startIndex : endIndex+1])
|
|
chunks = append(chunks, contracts.SourceChunk{
|
|
ID: fmt.Sprintf("scene-%06d", i+1),
|
|
SourceID: doc.ID,
|
|
Index: i,
|
|
Units: units,
|
|
Metadata: map[string]any{
|
|
"scene_title": normalized.ShortTitle,
|
|
"primary_mode": normalized.PrimaryMode,
|
|
"main_participants": append([]string(nil), normalized.MainParticipants...),
|
|
"summary": normalized.Summary,
|
|
"boundary_note": normalized.BoundaryNote,
|
|
"boundary_confidence": normalized.BoundaryConfidence,
|
|
"start_unit_id": normalized.StartUnitID,
|
|
"end_unit_id": normalized.EndUnitID,
|
|
"unit_count": len(units),
|
|
},
|
|
})
|
|
}
|
|
|
|
if previousEnd != len(doc.Units)-1 {
|
|
return nil, fmt.Errorf("final scene must end at final source unit %q", doc.Units[len(doc.Units)-1].ID)
|
|
}
|
|
return chunks, nil
|
|
}
|
|
|
|
func normalizeScene(index int, scene sceneResponse) (sceneResponse, error) {
|
|
out := sceneResponse{
|
|
StartUnitID: strings.TrimSpace(scene.StartUnitID),
|
|
EndUnitID: strings.TrimSpace(scene.EndUnitID),
|
|
ShortTitle: strings.TrimSpace(scene.ShortTitle),
|
|
PrimaryMode: strings.TrimSpace(scene.PrimaryMode),
|
|
Summary: strings.TrimSpace(scene.Summary),
|
|
BoundaryNote: strings.TrimSpace(scene.BoundaryNote),
|
|
BoundaryConfidence: strings.TrimSpace(scene.BoundaryConfidence),
|
|
}
|
|
|
|
required := map[string]string{
|
|
"start_unit_id": out.StartUnitID,
|
|
"end_unit_id": out.EndUnitID,
|
|
"short_title": out.ShortTitle,
|
|
"primary_mode": out.PrimaryMode,
|
|
"summary": out.Summary,
|
|
"boundary_note": out.BoundaryNote,
|
|
"boundary_confidence": out.BoundaryConfidence,
|
|
}
|
|
for field, value := range required {
|
|
if value == "" {
|
|
return sceneResponse{}, fmt.Errorf("scene[%d] %s must not be empty", index, field)
|
|
}
|
|
}
|
|
if !validPrimaryMode(out.PrimaryMode) {
|
|
return sceneResponse{}, fmt.Errorf("scene[%d] primary_mode %q is not supported", index, out.PrimaryMode)
|
|
}
|
|
if !validBoundaryConfidence(out.BoundaryConfidence) {
|
|
return sceneResponse{}, fmt.Errorf("scene[%d] boundary_confidence %q is not supported", index, out.BoundaryConfidence)
|
|
}
|
|
if len(scene.MainParticipants) == 0 {
|
|
return sceneResponse{}, fmt.Errorf("scene[%d] main_participants must not be empty", index)
|
|
}
|
|
out.MainParticipants = make([]string, 0, len(scene.MainParticipants))
|
|
for participantIndex, participant := range scene.MainParticipants {
|
|
trimmed := strings.TrimSpace(participant)
|
|
if trimmed == "" {
|
|
return sceneResponse{}, fmt.Errorf("scene[%d] main_participants[%d] must not be empty", index, participantIndex)
|
|
}
|
|
out.MainParticipants = append(out.MainParticipants, trimmed)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func validPrimaryMode(value string) bool {
|
|
switch value {
|
|
case "Recap", "Discussion", "Combat", "Narrative":
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func validBoundaryConfidence(value string) bool {
|
|
switch value {
|
|
case "High", "Medium", "Low":
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func warningsFromCaveats(caveats []string) []contracts.Warning {
|
|
if len(caveats) == 0 {
|
|
return nil
|
|
}
|
|
warnings := make([]contracts.Warning, 0, len(caveats))
|
|
for _, caveat := range caveats {
|
|
warnings = append(warnings, contracts.Warning{
|
|
Scope: Key,
|
|
ReasonCode: "scene_boundary_caveat",
|
|
Message: caveat,
|
|
})
|
|
}
|
|
return warnings
|
|
}
|
|
|
|
func cloneUnits(units []source.SourceUnit) []source.SourceUnit {
|
|
out := make([]source.SourceUnit, 0, len(units))
|
|
for _, unit := range units {
|
|
out = append(out, source.SourceUnit{
|
|
ID: unit.ID,
|
|
Kind: unit.Kind,
|
|
Text: unit.Text,
|
|
Metadata: cloneMetadata(unit.Metadata),
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
func cloneMetadata(metadata map[string]any) map[string]any {
|
|
if len(metadata) == 0 {
|
|
return nil
|
|
}
|
|
out := make(map[string]any, len(metadata))
|
|
for key, value := range metadata {
|
|
out[key] = value
|
|
}
|
|
return out
|
|
}
|
|
|
|
func chunkerErrorf(format string, args ...any) error {
|
|
return fmt.Errorf("dnd scenes chunker: "+format, args...)
|
|
}
|