narratio/internal/stage/analyze.go

package stage

import (
	"context"
	"encoding/json"
	"fmt"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
	"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
	"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
	"gitea.maximumdirect.net/eric/narratio/internal/config"
	"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)

type analyzeStage struct{}

func (analyzeStage) Name() string { return "analyze" }

type analyzeArtifactExecutionPlan struct {
	Name string
	Cfg  config.ScriptoriumArtifactConfig
}

type analyzeArtifactExecutionResult struct {
	Output           artifacts.Ref
	OutputSize       int64
	Scriptorium      manifest.AnalyzeArtifactProvenance
	Logs             []string
	GeneratedConfigs []string
	Metadata         map[string]any
	ReusedArtifacts  []map[string]any
}

type analyzeExecutionContext struct {
	Env            *Env
	Manifest       *manifest.Manifest
	Paths          artifacts.SessionPaths
	RunLayout      runStageLayout
	SessionID      string
	TranscriptRefs analyzeTranscriptInputs
	Catalog        *artifacts.ArtifactCatalog
}

func (analyzeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*StageResult, error) {
	if env == nil || env.Config == nil {
		return nil, fmt.Errorf("analyze: stage environment config is required")
	}
	if env.ArtifactStore == nil {
		return nil, fmt.Errorf("analyze: artifact store is required")
	}
	if env.Config.Pipeline == nil || env.Config.Session == nil {
		return nil, fmt.Errorf("analyze: resolved config must include pipeline and session")
	}
	var sessionID string
	if m != nil {
		sessionID = strings.TrimSpace(m.SessionID)
	}
	if sessionID == "" {
		sessionID = strings.TrimSpace(env.Config.Session.SessionID)
	}
	if sessionID == "" {
		return nil, fmt.Errorf("analyze: session id is required")
	}

	paths := sessionPathsForEnv(env, sessionID)
	runLayout, err := resolveRunStageLayout(env, m, paths, sessionID, "analyze")
	if err != nil {
		return nil, fmt.Errorf("analyze: resolve run-stage layout: %w", err)
	}
	if env.Config.Pipeline.Scriptorium == nil {
		return &StageResult{Metadata: map[string]any{
			"stage":   "analyze",
			"skipped": true,
			"reason":  "pipeline.scriptorium is not configured",
		}}, nil
	}
	if len(env.Config.Pipeline.Scriptorium.Artifacts) == 0 {
		return &StageResult{Metadata: map[string]any{
			"stage":   "analyze",
			"skipped": true,
			"reason":  "no scriptorium artifacts configured",
		}}, nil
	}

	effective := env.EffectiveArtifacts
	if !effective.Resolved() {
		effective, err = artifacts.ResolveEffectiveArtifactSet(
			artifacts.ConfiguredArtifactDefinitions(env.Config.Pipeline.Scriptorium.Artifacts),
			env.SelectedArtifactKeys,
		)
		if err != nil {
			return nil, fmt.Errorf("analyze: resolve effective artifacts: %w", err)
		}
	}

	runtimeCatalog, err := buildAnalyzeRuntimeArtifactCatalog(
		paths,
		m,
		env.Config.Pipeline.Scriptorium,
		env.Config.Pipeline.Notarius,
		effective,
	)
	if err != nil {
		return nil, fmt.Errorf("analyze: build runtime artifact catalog: %w", err)
	}

	if len(effective.Keys()) == 0 {
		return &StageResult{Metadata: map[string]any{
			"stage":   "analyze",
			"skipped": true,
			"reason":  "no selected scriptorium artifacts to execute",
		}}, nil
	}

	execution := analyzeExecutionContext{
		Env:            env,
		Manifest:       m,
		Paths:          paths,
		RunLayout:      runLayout,
		SessionID:      sessionID,
		TranscriptRefs: discoverAnalyzeTranscriptRefs(m, paths),
		Catalog:        runtimeCatalog,
	}
	reconciliation, err := reconcileAnalyzeArtifacts(env.Config.Pipeline.Scriptorium, execution)
	if err != nil {
		return nil, fmt.Errorf("analyze: reconcile configured artifacts: %w", err)
	}
	workPlan, err := planAnalyzeWork(
		env.Config.Pipeline.Scriptorium,
		env.SelectedArtifactKeys,
		env.Force,
		reconciliation,
	)
	if err != nil {
		return nil, fmt.Errorf("analyze: plan configured artifacts: %w", err)
	}
	if len(workPlan.ExecutionOrder) > 0 && env.Scriptorium == nil {
		return nil, fmt.Errorf("analyze: scriptorium adapter is required")
	}

	state := newAnalyzeExecutionState(execution, workPlan, reconciliation)
	if err := state.checkpointStartingCollection(ctx); err != nil {
		return nil, fmt.Errorf("analyze: checkpoint reconciled artifact collection before execution: %w", err)
	}

	for _, item := range workPlan.ExecutionOrder {
		artifactCfg := env.Config.Pipeline.Scriptorium.Artifacts[item.Key]
		fingerprint, _, _, err := computeAnalyzeArtifactFingerprint(
			item.Key,
			env.Config.Pipeline.Scriptorium,
			execution,
		)
		if err != nil {
			executionErr := fmt.Errorf("analyze: compute execution fingerprint for artifact %q: %w", item.Key, err)
			return state.failedResult(item, artifactCfg, "", executionErr), executionErr
		}
		plan := analyzeArtifactExecutionPlan{Name: item.Key, Cfg: artifactCfg}
		artifactResult, err := executeAnalyzeArtifact(ctx, execution, plan)
		if err != nil {
			return state.failedResult(item, artifactCfg, fingerprint, err), err
		}
		if err := state.acceptArtifact(ctx, plan, fingerprint, artifactResult); err != nil {
			return state.failedResult(item, artifactCfg, fingerprint, err), err
		}
	}
	return state.successResult(), nil
}

func analyzeArtifactOrigin(execution analyzeExecutionContext, key string) (config.ArtifactFamilyMemberOrigin, bool) {
	if execution.Env == nil || execution.Env.Config == nil || execution.Env.Config.Pipeline == nil {
		return config.ArtifactFamilyMemberOrigin{}, false
	}
	origin, ok := config.ArtifactFamilies(execution.Env.Config.Pipeline).Members[key]
	return origin, ok
}

func artifactCfgMap(execution analyzeExecutionContext) map[string]config.ScriptoriumArtifactConfig {
	if execution.Env == nil || execution.Env.Config == nil || execution.Env.Config.Pipeline == nil ||
		execution.Env.Config.Pipeline.Scriptorium == nil {
		return nil
	}
	return execution.Env.Config.Pipeline.Scriptorium.Artifacts
}

func staleAnalyzeDependents(
	configured map[string]config.ScriptoriumArtifactConfig,
	changed string,
	records map[string]manifest.AnalyzeArtifactRecord,
) {
	staleAnalyzeDependentClosure(configured, changed, records, nil)
}

func currentAnalyzeArtifactRecord(
	execution analyzeExecutionContext,
	plan analyzeArtifactExecutionPlan,
	fingerprint string,
	result *analyzeArtifactExecutionResult,
) (manifest.AnalyzeArtifactRecord, error) {
	if result == nil {
		return manifest.AnalyzeArtifactRecord{}, fmt.Errorf("execution result is required")
	}
	producerRunID := analyzeProducerRunID(execution)
	relativePath, err := normalizedAnalyzeOutputIdentity(plan.Cfg.OutputPath)
	if err != nil {
		return manifest.AnalyzeArtifactRecord{}, err
	}
	if relativePath == "" {
		return manifest.AnalyzeArtifactRecord{}, fmt.Errorf("configured output path is required")
	}
	contract := result.Output.Contract
	if contract == nil {
		return manifest.AnalyzeArtifactRecord{}, fmt.Errorf("validated output contract is required")
	}
	record := manifest.AnalyzeArtifactRecord{
		Key:                plan.Name,
		Status:             manifest.AnalyzeArtifactCurrent,
		FingerprintVersion: manifest.AnalyzeFingerprintContractVersion,
		Fingerprint:        fingerprint,
		Dependencies:       normalizedAnalyzeDependencyKeys(plan.Cfg.DependsOn),
		Output: &manifest.ArtifactRecord{
			Kind:          "scriptorium_artifact",
			SourceID:      artifacts.ConfiguredArtifactSourceID(plan.Name),
			LocalPath:     relativePath,
			Contract:      cloneAnalyzeOutputContract(contract),
			ProducerRunID: producerRunID,
			Checksum:      result.Output.Checksum,
		},
		OutputSize:       result.OutputSize,
		ProducerRunID:    producerRunID,
		UpdatedAt:        time.Now().UTC(),
		Scriptorium:      &result.Scriptorium,
		Logs:             dedupeAndSortPaths(result.Logs),
		GeneratedConfigs: dedupeAndSortPaths(result.GeneratedConfigs),
	}
	if origin, ok := analyzeArtifactOrigin(execution, plan.Name); ok {
		record.Family = origin.Family
		record.CharacterID = origin.CharacterID
	}
	if err := manifest.ValidateAnalyzeArtifactCollection(
		manifest.AnalyzeStateContractVersion,
		map[string]manifest.AnalyzeArtifactRecord{plan.Name: record},
	); err != nil {
		return manifest.AnalyzeArtifactRecord{}, err
	}
	return record, nil
}

func analyzeProducerRunID(execution analyzeExecutionContext) string {
	if execution.Manifest != nil {
		if runID := strings.TrimSpace(execution.Manifest.RunID); runID != "" {
			return runID
		}
	}
	// Direct stage callers use a stable synthetic ID. Application-owned execution
	// always supplies the actual run identity.
	return "direct-analyze"
}

func sameAnalyzeOutputIdentity(left, right manifest.AnalyzeArtifactRecord) bool {
	if left.Status != manifest.AnalyzeArtifactCurrent || right.Status != manifest.AnalyzeArtifactCurrent ||
		left.Output == nil || right.Output == nil || left.Output.Contract == nil || right.Output.Contract == nil {
		return false
	}
	return left.OutputSize == right.OutputSize &&
		left.Output.Checksum == right.Output.Checksum &&
		*left.Output.Contract == *right.Output.Contract
}

func staleUnscheduledAnalyzeDependents(
	configured map[string]config.ScriptoriumArtifactConfig,
	changed string,
	currentKeys map[string]struct{},
	records map[string]manifest.AnalyzeArtifactRecord,
) {
	staleAnalyzeDependentClosure(configured, changed, records, func(key string) bool {
		_, evaluated := currentKeys[key]
		return !evaluated
	})
}

func staleAnalyzeDependentClosure(
	configured map[string]config.ScriptoriumArtifactConfig,
	changed string,
	records map[string]manifest.AnalyzeArtifactRecord,
	eligible func(string) bool,
) {
	reverse := make(map[string][]string, len(configured))
	for key, artifactCfg := range configured {
		for _, dependency := range normalizedAnalyzeDependencyKeys(artifactCfg.DependsOn) {
			reverse[dependency] = append(reverse[dependency], key)
		}
	}
	for key := range reverse {
		sort.Strings(reverse[key])
	}
	queue := append([]string(nil), reverse[changed]...)
	seen := make(map[string]struct{}, len(queue))
	for len(queue) > 0 {
		key := queue[0]
		queue = queue[1:]
		if _, visited := seen[key]; visited {
			continue
		}
		seen[key] = struct{}{}
		if eligible != nil && !eligible(key) {
			continue
		}
		staleAnalyzeRecord(records, key)
		queue = append(queue, reverse[key]...)
	}
}

func analyzePlanKeysForMetadata(items []analyzePlanItem) []string {
	if len(items) == 0 {
		return nil
	}
	keys := make([]string, 0, len(items))
	for _, item := range items {
		keys = append(keys, item.Key)
	}
	return keys
}

func cloneAnalyzeOutputContract(value *artifactmodel.ContractMetadata) *artifactmodel.ContractMetadata {
	if value == nil {
		return nil
	}
	cloned := *value
	return &cloned
}

func configuredArtifactNameFromSourceID(sourceID string) string {
	name, _ := artifactpolicy.ParseConfiguredSource(sourceID)
	return name
}

func discoverProcessedTranscript(m *manifest.Manifest, paths artifacts.SessionPaths) (string, string, error) {
	return resolveSingletonTranscript(m, paths, artifacts.ArtifactTranscriptPolished)
}

type analyzeTranscriptInputs struct {
	ProcessedPath    string
	ProcessedSource  string
	NormalizedPath   string
	NormalizedSource string
	TrimmedPath      string
	TrimmedSource    string
}

func discoverAnalyzeTranscriptRefs(m *manifest.Manifest, paths artifacts.SessionPaths) analyzeTranscriptInputs {
	processedPath, processedSource := discoverAnalyzeArtifactRef(m, paths, artifacts.ArtifactTranscriptPolished)
	normalizedPath, normalizedSource := discoverAnalyzeArtifactRef(m, paths, artifacts.ArtifactTranscriptFinal)
	trimmedPath, trimmedSource := discoverAnalyzeArtifactRef(m, paths, artifacts.ArtifactTranscriptFinalTrimmed)
	return analyzeTranscriptInputs{
		ProcessedPath:    processedPath,
		ProcessedSource:  processedSource,
		NormalizedPath:   normalizedPath,
		NormalizedSource: normalizedSource,
		TrimmedPath:      trimmedPath,
		TrimmedSource:    trimmedSource,
	}
}

func discoverAnalyzeArtifactRef(m *manifest.Manifest, paths artifacts.SessionPaths, source string) (string, string) {
	resolved, err := artifacts.ResolveSessionArtifact(paths, m, source)
	if err != nil {
		return "", ""
	}
	return resolved.Path, resolved.Provenance
}

func requiredBuiltInInputError(source string, execution analyzeExecutionContext) error {
	entry, ok := execution.Catalog.Lookup(source)
	if !ok || strings.TrimSpace(entry.ProducerStage) == "" {
		return fmt.Errorf("required built-in source %q is unavailable", source)
	}
	return fmt.Errorf(
		"required built-in source %q is unavailable; run narratio run-stage %s %s --force",
		source,
		entry.ProducerStage,
		execution.SessionID,
	)
}

func buildAnalyzeRuntimeArtifactCatalog(
	paths artifacts.SessionPaths,
	m *manifest.Manifest,
	scriptoriumCfg *config.ScriptoriumConfig,
	notariusCfg *config.NotariusConfig,
	effective artifacts.EffectiveArtifactSet,
) (*artifacts.ArtifactCatalog, error) {
	configured := artifacts.ConfiguredArtifactDefinitions(nil)
	if scriptoriumCfg != nil {
		configured = artifacts.ConfiguredArtifactDefinitions(scriptoriumCfg.Artifacts)
	}
	extractionDefinitions := artifacts.ExtractionDefinitionsFromConfig(notariusCfg)
	catalog, err := artifacts.BootstrapRuntimeCatalog(configured, effective, extractionDefinitions)
	if err != nil {
		return nil, err
	}
	if notariusCfg != nil && notariusCfg.Enabled {
		catalog.HydrateExtractionArtifacts(paths, m, extractionDefinitions)
	}
	catalog.HydrateAnalyzeArtifacts(paths, m, configured)

	return catalog, nil
}

func resolveScriptoriumOutputPath(paths artifacts.SessionPaths, configured string) (string, error) {
	outputPath := strings.TrimSpace(configured)
	if outputPath == "" {
		return "", fmt.Errorf("scriptorium artifact output path is required")
	}
	if filepath.IsAbs(outputPath) {
		return filepath.Clean(outputPath), nil
	}

	rel := filepath.Clean(outputPath)
	if rel == "." || rel == "" {
		return "", fmt.Errorf("relative output path is required")
	}
	if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
		return "", fmt.Errorf("relative output path escapes session root: %q", outputPath)
	}
	return filepath.Join(paths.Root, rel), nil
}

func resolveScriptoriumTimeout(topLevel, artifact string) (time.Duration, error) {
	raw := strings.TrimSpace(artifact)
	if raw == "" {
		raw = strings.TrimSpace(topLevel)
	}
	if raw == "" {
		raw = "10m"
	}
	d, err := time.ParseDuration(raw)
	if err != nil {
		return 0, fmt.Errorf("parse duration %q: %w", raw, err)
	}
	if d <= 0 {
		return 0, fmt.Errorf("duration must be > 0")
	}
	return d, nil
}

func buildScriptoriumVars(varsCfg map[string]any, session *config.SessionConfig) (map[string]string, error) {
	if len(varsCfg) == 0 {
		return nil, nil
	}
	vars := map[string]string{}
	keys := make([]string, 0, len(varsCfg))
	for key := range varsCfg {
		keys = append(keys, key)
	}
	sort.Strings(keys)

	for _, key := range keys {
		value := varsCfg[key]
		switch typed := value.(type) {
		case bool:
			if !typed {
				continue
			}
			derived, ok, err := deriveSessionVarValue(key, session)
			if err != nil {
				return nil, err
			}
			if ok {
				vars[key] = derived
			}
		case string:
			vars[key] = typed
		default:
			return nil, fmt.Errorf("var %q has unsupported type %T", key, value)
		}
	}
	if len(vars) == 0 {
		return nil, nil
	}
	return vars, nil
}

func deriveSessionVarValue(name string, session *config.SessionConfig) (string, bool, error) {
	switch name {
	case "session_id":
		if session == nil || strings.TrimSpace(session.SessionID) == "" {
			return "", false, nil
		}
		return strings.TrimSpace(session.SessionID), true, nil
	case "session_date":
		if session == nil || strings.TrimSpace(session.Date) == "" {
			return "", false, nil
		}
		return strings.TrimSpace(session.Date), true, nil
	case "campaign_name":
		if session == nil || strings.TrimSpace(session.Campaign) == "" {
			return "", false, nil
		}
		return strings.TrimSpace(session.Campaign), true, nil
	case "previous_session_id":
		return "", false, nil
	default:
		return "", false, fmt.Errorf("unsupported boolean var %q", name)
	}
}

func requireNonEmptyFile(path string, label string) error {
	data, err := readExternalResult(path, label)
	if err != nil {
		return err
	}
	if len(data) == 0 {
		return fmt.Errorf("%s %q is empty", label, path)
	}
	return nil
}

func sortedScriptoriumInputNames(inputs map[string]config.ScriptoriumInputConfig) []string {
	if len(inputs) == 0 {
		return nil
	}
	names := make([]string, 0, len(inputs))
	for name := range inputs {
		names = append(names, name)
	}
	sort.Strings(names)
	return names
}

func sortedMapKeys(values map[string]string) []string {
	if len(values) == 0 {
		return nil
	}
	keys := make([]string, 0, len(values))
	for key := range values {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	return keys
}

func coalesceString(primary, fallback string) string {
	if strings.TrimSpace(primary) != "" {
		return primary
	}
	return fallback
}

func resolveRenderDebugEnabled(global bool, perArtifact *bool) bool {
	if perArtifact == nil {
		return global
	}
	return *perArtifact
}

func validateJSONFile(path string) error {
	data, err := readExternalResult(path, "scriptorium artifact result")
	if err != nil {
		return err
	}
	var payload any
	if err := json.Unmarshal(data, &payload); err != nil {
		return fmt.Errorf("decode json: %w", err)
	}
	return nil
}