package stage import ( "context" "encoding/json" "fmt" "path/filepath" "sort" "strings" "time" "gitea.maximumdirect.net/eric/narratio/internal/artifactmodel" "gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy" "gitea.maximumdirect.net/eric/narratio/internal/artifacts" "gitea.maximumdirect.net/eric/narratio/internal/config" "gitea.maximumdirect.net/eric/narratio/internal/manifest" ) type analyzeStage struct{} func (analyzeStage) Name() string { return "analyze" } type analyzeArtifactExecutionPlan struct { Name string Cfg config.ScriptoriumArtifactConfig } type analyzeArtifactExecutionResult struct { Output artifacts.Ref OutputSize int64 Scriptorium manifest.AnalyzeArtifactProvenance Logs []string GeneratedConfigs []string Metadata map[string]any ReusedArtifacts []map[string]any } type analyzeExecutionContext struct { Env *Env Manifest *manifest.Manifest Paths artifacts.SessionPaths RunLayout runStageLayout SessionID string TranscriptRefs analyzeTranscriptInputs Catalog *artifacts.ArtifactCatalog } func (analyzeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*StageResult, error) { if env == nil || env.Config == nil { return nil, fmt.Errorf("analyze: stage environment config is required") } if env.ArtifactStore == nil { return nil, fmt.Errorf("analyze: artifact store is required") } if env.Config.Pipeline == nil || env.Config.Session == nil { return nil, fmt.Errorf("analyze: resolved config must include pipeline and session") } var sessionID string if m != nil { sessionID = strings.TrimSpace(m.SessionID) } if sessionID == "" { sessionID = strings.TrimSpace(env.Config.Session.SessionID) } if sessionID == "" { return nil, fmt.Errorf("analyze: session id is required") } paths := sessionPathsForEnv(env, sessionID) runLayout, err := resolveRunStageLayout(env, m, paths, sessionID, "analyze") if err != nil { return nil, fmt.Errorf("analyze: resolve run-stage layout: %w", err) } if env.Config.Pipeline.Scriptorium == nil { return &StageResult{Metadata: map[string]any{ "stage": "analyze", "skipped": true, "reason": "pipeline.scriptorium is not configured", }}, nil } if len(env.Config.Pipeline.Scriptorium.Artifacts) == 0 { return &StageResult{Metadata: map[string]any{ "stage": "analyze", "skipped": true, "reason": "no scriptorium artifacts configured", }}, nil } effective := env.EffectiveArtifacts if !effective.Resolved() { effective, err = artifacts.ResolveEffectiveArtifactSet( artifacts.ConfiguredArtifactDefinitions(env.Config.Pipeline.Scriptorium.Artifacts), env.SelectedArtifactKeys, ) if err != nil { return nil, fmt.Errorf("analyze: resolve effective artifacts: %w", err) } } runtimeCatalog, err := buildAnalyzeRuntimeArtifactCatalog( paths, m, env.Config.Pipeline.Scriptorium, env.Config.Pipeline.Notarius, effective, ) if err != nil { return nil, fmt.Errorf("analyze: build runtime artifact catalog: %w", err) } if len(effective.Keys()) == 0 { return &StageResult{Metadata: map[string]any{ "stage": "analyze", "skipped": true, "reason": "no selected scriptorium artifacts to execute", }}, nil } execution := analyzeExecutionContext{ Env: env, Manifest: m, Paths: paths, RunLayout: runLayout, SessionID: sessionID, TranscriptRefs: discoverAnalyzeTranscriptRefs(m, paths), Catalog: runtimeCatalog, } reconciliation, err := reconcileAnalyzeArtifacts(env.Config.Pipeline.Scriptorium, execution) if err != nil { return nil, fmt.Errorf("analyze: reconcile configured artifacts: %w", err) } workPlan, err := planAnalyzeWork( env.Config.Pipeline.Scriptorium, env.SelectedArtifactKeys, env.Force, reconciliation, ) if err != nil { return nil, fmt.Errorf("analyze: plan configured artifacts: %w", err) } if len(workPlan.ExecutionOrder) > 0 && env.Scriptorium == nil { return nil, fmt.Errorf("analyze: scriptorium adapter is required") } state := newAnalyzeExecutionState(execution, workPlan, reconciliation) if err := state.checkpointStartingCollection(ctx); err != nil { return nil, fmt.Errorf("analyze: checkpoint reconciled artifact collection before execution: %w", err) } for _, item := range workPlan.ExecutionOrder { artifactCfg := env.Config.Pipeline.Scriptorium.Artifacts[item.Key] fingerprint, _, _, err := computeAnalyzeArtifactFingerprint( item.Key, env.Config.Pipeline.Scriptorium, execution, ) if err != nil { executionErr := fmt.Errorf("analyze: compute execution fingerprint for artifact %q: %w", item.Key, err) return state.failedResult(item, artifactCfg, "", executionErr), executionErr } plan := analyzeArtifactExecutionPlan{Name: item.Key, Cfg: artifactCfg} artifactResult, err := executeAnalyzeArtifact(ctx, execution, plan) if err != nil { return state.failedResult(item, artifactCfg, fingerprint, err), err } if err := state.acceptArtifact(ctx, plan, fingerprint, artifactResult); err != nil { return state.failedResult(item, artifactCfg, fingerprint, err), err } } return state.successResult(), nil } func analyzeArtifactOrigin(execution analyzeExecutionContext, key string) (config.ArtifactFamilyMemberOrigin, bool) { if execution.Env == nil || execution.Env.Config == nil || execution.Env.Config.Pipeline == nil { return config.ArtifactFamilyMemberOrigin{}, false } origin, ok := config.ArtifactFamilies(execution.Env.Config.Pipeline).Members[key] return origin, ok } func artifactCfgMap(execution analyzeExecutionContext) map[string]config.ScriptoriumArtifactConfig { if execution.Env == nil || execution.Env.Config == nil || execution.Env.Config.Pipeline == nil || execution.Env.Config.Pipeline.Scriptorium == nil { return nil } return execution.Env.Config.Pipeline.Scriptorium.Artifacts } func staleAnalyzeDependents( configured map[string]config.ScriptoriumArtifactConfig, changed string, records map[string]manifest.AnalyzeArtifactRecord, ) { staleAnalyzeDependentClosure(configured, changed, records, nil) } func currentAnalyzeArtifactRecord( execution analyzeExecutionContext, plan analyzeArtifactExecutionPlan, fingerprint string, result *analyzeArtifactExecutionResult, ) (manifest.AnalyzeArtifactRecord, error) { if result == nil { return manifest.AnalyzeArtifactRecord{}, fmt.Errorf("execution result is required") } producerRunID := analyzeProducerRunID(execution) relativePath, err := normalizedAnalyzeOutputIdentity(plan.Cfg.OutputPath) if err != nil { return manifest.AnalyzeArtifactRecord{}, err } if relativePath == "" { return manifest.AnalyzeArtifactRecord{}, fmt.Errorf("configured output path is required") } contract := result.Output.Contract if contract == nil { return manifest.AnalyzeArtifactRecord{}, fmt.Errorf("validated output contract is required") } record := manifest.AnalyzeArtifactRecord{ Key: plan.Name, Status: manifest.AnalyzeArtifactCurrent, FingerprintVersion: manifest.AnalyzeFingerprintContractVersion, Fingerprint: fingerprint, Dependencies: normalizedAnalyzeDependencyKeys(plan.Cfg.DependsOn), Output: &manifest.ArtifactRecord{ Kind: "scriptorium_artifact", SourceID: artifacts.ConfiguredArtifactSourceID(plan.Name), LocalPath: relativePath, Contract: cloneAnalyzeOutputContract(contract), ProducerRunID: producerRunID, Checksum: result.Output.Checksum, }, OutputSize: result.OutputSize, ProducerRunID: producerRunID, UpdatedAt: time.Now().UTC(), Scriptorium: &result.Scriptorium, Logs: dedupeAndSortPaths(result.Logs), GeneratedConfigs: dedupeAndSortPaths(result.GeneratedConfigs), } if origin, ok := analyzeArtifactOrigin(execution, plan.Name); ok { record.Family = origin.Family record.CharacterID = origin.CharacterID } if err := manifest.ValidateAnalyzeArtifactCollection( manifest.AnalyzeStateContractVersion, map[string]manifest.AnalyzeArtifactRecord{plan.Name: record}, ); err != nil { return manifest.AnalyzeArtifactRecord{}, err } return record, nil } func analyzeProducerRunID(execution analyzeExecutionContext) string { if execution.Manifest != nil { if runID := strings.TrimSpace(execution.Manifest.RunID); runID != "" { return runID } } // Direct stage callers use a stable synthetic ID. Application-owned execution // always supplies the actual run identity. return "direct-analyze" } func sameAnalyzeOutputIdentity(left, right manifest.AnalyzeArtifactRecord) bool { if left.Status != manifest.AnalyzeArtifactCurrent || right.Status != manifest.AnalyzeArtifactCurrent || left.Output == nil || right.Output == nil || left.Output.Contract == nil || right.Output.Contract == nil { return false } return left.OutputSize == right.OutputSize && left.Output.Checksum == right.Output.Checksum && *left.Output.Contract == *right.Output.Contract } func staleUnscheduledAnalyzeDependents( configured map[string]config.ScriptoriumArtifactConfig, changed string, currentKeys map[string]struct{}, records map[string]manifest.AnalyzeArtifactRecord, ) { staleAnalyzeDependentClosure(configured, changed, records, func(key string) bool { _, evaluated := currentKeys[key] return !evaluated }) } func staleAnalyzeDependentClosure( configured map[string]config.ScriptoriumArtifactConfig, changed string, records map[string]manifest.AnalyzeArtifactRecord, eligible func(string) bool, ) { reverse := make(map[string][]string, len(configured)) for key, artifactCfg := range configured { for _, dependency := range normalizedAnalyzeDependencyKeys(artifactCfg.DependsOn) { reverse[dependency] = append(reverse[dependency], key) } } for key := range reverse { sort.Strings(reverse[key]) } queue := append([]string(nil), reverse[changed]...) seen := make(map[string]struct{}, len(queue)) for len(queue) > 0 { key := queue[0] queue = queue[1:] if _, visited := seen[key]; visited { continue } seen[key] = struct{}{} if eligible != nil && !eligible(key) { continue } staleAnalyzeRecord(records, key) queue = append(queue, reverse[key]...) } } func analyzePlanKeysForMetadata(items []analyzePlanItem) []string { if len(items) == 0 { return nil } keys := make([]string, 0, len(items)) for _, item := range items { keys = append(keys, item.Key) } return keys } func cloneAnalyzeOutputContract(value *artifactmodel.ContractMetadata) *artifactmodel.ContractMetadata { if value == nil { return nil } cloned := *value return &cloned } func configuredArtifactNameFromSourceID(sourceID string) string { name, _ := artifactpolicy.ParseConfiguredSource(sourceID) return name } func discoverProcessedTranscript(m *manifest.Manifest, paths artifacts.SessionPaths) (string, string, error) { return resolveSingletonTranscript(m, paths, artifacts.ArtifactTranscriptPolished) } type analyzeTranscriptInputs struct { ProcessedPath string ProcessedSource string NormalizedPath string NormalizedSource string TrimmedPath string TrimmedSource string } func discoverAnalyzeTranscriptRefs(m *manifest.Manifest, paths artifacts.SessionPaths) analyzeTranscriptInputs { processedPath, processedSource := discoverAnalyzeArtifactRef(m, paths, artifacts.ArtifactTranscriptPolished) normalizedPath, normalizedSource := discoverAnalyzeArtifactRef(m, paths, artifacts.ArtifactTranscriptFinal) trimmedPath, trimmedSource := discoverAnalyzeArtifactRef(m, paths, artifacts.ArtifactTranscriptFinalTrimmed) return analyzeTranscriptInputs{ ProcessedPath: processedPath, ProcessedSource: processedSource, NormalizedPath: normalizedPath, NormalizedSource: normalizedSource, TrimmedPath: trimmedPath, TrimmedSource: trimmedSource, } } func discoverAnalyzeArtifactRef(m *manifest.Manifest, paths artifacts.SessionPaths, source string) (string, string) { resolved, err := artifacts.ResolveSessionArtifact(paths, m, source) if err != nil { return "", "" } return resolved.Path, resolved.Provenance } func requiredBuiltInInputError(source string, execution analyzeExecutionContext) error { entry, ok := execution.Catalog.Lookup(source) if !ok || strings.TrimSpace(entry.ProducerStage) == "" { return fmt.Errorf("required built-in source %q is unavailable", source) } return fmt.Errorf( "required built-in source %q is unavailable; run narratio run-stage %s %s --force", source, entry.ProducerStage, execution.SessionID, ) } func buildAnalyzeRuntimeArtifactCatalog( paths artifacts.SessionPaths, m *manifest.Manifest, scriptoriumCfg *config.ScriptoriumConfig, notariusCfg *config.NotariusConfig, effective artifacts.EffectiveArtifactSet, ) (*artifacts.ArtifactCatalog, error) { configured := artifacts.ConfiguredArtifactDefinitions(nil) if scriptoriumCfg != nil { configured = artifacts.ConfiguredArtifactDefinitions(scriptoriumCfg.Artifacts) } extractionDefinitions := artifacts.ExtractionDefinitionsFromConfig(notariusCfg) catalog, err := artifacts.BootstrapRuntimeCatalog(configured, effective, extractionDefinitions) if err != nil { return nil, err } if notariusCfg != nil && notariusCfg.Enabled { catalog.HydrateExtractionArtifacts(paths, m, extractionDefinitions) } catalog.HydrateAnalyzeArtifacts(paths, m, configured) return catalog, nil } func resolveScriptoriumOutputPath(paths artifacts.SessionPaths, configured string) (string, error) { outputPath := strings.TrimSpace(configured) if outputPath == "" { return "", fmt.Errorf("scriptorium artifact output path is required") } if filepath.IsAbs(outputPath) { return filepath.Clean(outputPath), nil } rel := filepath.Clean(outputPath) if rel == "." || rel == "" { return "", fmt.Errorf("relative output path is required") } if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { return "", fmt.Errorf("relative output path escapes session root: %q", outputPath) } return filepath.Join(paths.Root, rel), nil } func resolveScriptoriumTimeout(topLevel, artifact string) (time.Duration, error) { raw := strings.TrimSpace(artifact) if raw == "" { raw = strings.TrimSpace(topLevel) } if raw == "" { raw = "10m" } d, err := time.ParseDuration(raw) if err != nil { return 0, fmt.Errorf("parse duration %q: %w", raw, err) } if d <= 0 { return 0, fmt.Errorf("duration must be > 0") } return d, nil } func buildScriptoriumVars(varsCfg map[string]any, session *config.SessionConfig) (map[string]string, error) { if len(varsCfg) == 0 { return nil, nil } vars := map[string]string{} keys := make([]string, 0, len(varsCfg)) for key := range varsCfg { keys = append(keys, key) } sort.Strings(keys) for _, key := range keys { value := varsCfg[key] switch typed := value.(type) { case bool: if !typed { continue } derived, ok, err := deriveSessionVarValue(key, session) if err != nil { return nil, err } if ok { vars[key] = derived } case string: vars[key] = typed default: return nil, fmt.Errorf("var %q has unsupported type %T", key, value) } } if len(vars) == 0 { return nil, nil } return vars, nil } func deriveSessionVarValue(name string, session *config.SessionConfig) (string, bool, error) { switch name { case "session_id": if session == nil || strings.TrimSpace(session.SessionID) == "" { return "", false, nil } return strings.TrimSpace(session.SessionID), true, nil case "session_date": if session == nil || strings.TrimSpace(session.Date) == "" { return "", false, nil } return strings.TrimSpace(session.Date), true, nil case "campaign_name": if session == nil || strings.TrimSpace(session.Campaign) == "" { return "", false, nil } return strings.TrimSpace(session.Campaign), true, nil case "previous_session_id": return "", false, nil default: return "", false, fmt.Errorf("unsupported boolean var %q", name) } } func requireNonEmptyFile(path string, label string) error { data, err := readExternalResult(path, label) if err != nil { return err } if len(data) == 0 { return fmt.Errorf("%s %q is empty", label, path) } return nil } func sortedScriptoriumInputNames(inputs map[string]config.ScriptoriumInputConfig) []string { if len(inputs) == 0 { return nil } names := make([]string, 0, len(inputs)) for name := range inputs { names = append(names, name) } sort.Strings(names) return names } func sortedMapKeys(values map[string]string) []string { if len(values) == 0 { return nil } keys := make([]string, 0, len(values)) for key := range values { keys = append(keys, key) } sort.Strings(keys) return keys } func coalesceString(primary, fallback string) string { if strings.TrimSpace(primary) != "" { return primary } return fallback } func resolveRenderDebugEnabled(global bool, perArtifact *bool) bool { if perArtifact == nil { return global } return *perArtifact } func validateJSONFile(path string) error { data, err := readExternalResult(path, "scriptorium artifact result") if err != nil { return err } var payload any if err := json.Unmarshal(data, &payload); err != nil { return fmt.Errorf("decode json: %w", err) } return nil }