1331 lines
43 KiB
Go
1331 lines
43 KiB
Go
package cli
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"flag"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/workspace"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
)
|
|
|
|
const defaultConfigPath = "/usr/local/etc/notarius/config.yml"
|
|
const defaultOutputRoot = "./notarius-output"
|
|
|
|
const usage = `Usage:
|
|
notarius help
|
|
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--resume] [--session-id id] [--reference selector=path] [--without-reference selector]
|
|
notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b]
|
|
notarius pipelines list --config path/to/config.yml [--json]
|
|
`
|
|
|
|
type Options struct {
|
|
Catalog pipeline.ModuleCatalog
|
|
Registries pipeline.Registries
|
|
LLMClientFactory LLMClientFactory
|
|
LookupEnv func(string) (string, bool)
|
|
Now func() time.Time
|
|
}
|
|
|
|
type LLMClientFactory func(ctx context.Context, cfg config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error)
|
|
|
|
// Run executes the command-line interface and returns a process exit code.
|
|
func Run(args []string, stdout, stderr io.Writer) int {
|
|
return RunWithOptions(args, stdout, stderr, Options{})
|
|
}
|
|
|
|
func RunWithOptions(args []string, stdout, stderr io.Writer, opts Options) int {
|
|
opts = normalizeOptions(opts)
|
|
if len(args) == 0 {
|
|
writeUsage(stdout)
|
|
return 0
|
|
}
|
|
|
|
switch args[0] {
|
|
case "help", "--help", "-h":
|
|
writeUsage(stdout)
|
|
return 0
|
|
case "config":
|
|
return runConfig(args[1:], stdout, stderr, opts)
|
|
case "pipelines":
|
|
return runPipelines(args[1:], stdout, stderr, opts)
|
|
case "run":
|
|
return runPipelineCommand(args[1:], stdout, stderr, opts)
|
|
default:
|
|
fmt.Fprintf(stderr, "notarius: unknown command %q\n", args[0])
|
|
writeUsage(stderr)
|
|
return 2
|
|
}
|
|
}
|
|
|
|
func writeUsage(w io.Writer) {
|
|
fmt.Fprint(w, usage)
|
|
}
|
|
|
|
func normalizeOptions(opts Options) Options {
|
|
if opts.LookupEnv == nil {
|
|
opts.LookupEnv = os.LookupEnv
|
|
}
|
|
if opts.Now == nil {
|
|
opts.Now = time.Now
|
|
}
|
|
if opts.LLMClientFactory == nil {
|
|
opts.LLMClientFactory = productionLLMClientFactory
|
|
}
|
|
return opts
|
|
}
|
|
|
|
func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) int {
|
|
fs := flag.NewFlagSet("run", flag.ContinueOnError)
|
|
fs.SetOutput(io.Discard)
|
|
configPath := fs.String("config", "", "config file path")
|
|
inputPath := fs.String("input", "", "source input file path")
|
|
onlyRaw := fs.String("only", "", "comma-separated artifact lanes")
|
|
outputDir := fs.String("output-dir", "", "output directory")
|
|
diagnosticsDir := fs.String("diagnostics-dir", "", "diagnostics directory")
|
|
llmProfile := fs.String("llm-profile", "", "LLM profile override")
|
|
resume := fs.Bool("resume", false, "reuse valid workspace checkpoints")
|
|
sessionID := sessionIDFlag{}
|
|
referenceFlags := stringListFlag{}
|
|
withoutReferenceFlags := stringListFlag{}
|
|
fs.Var(&sessionID, "session-id", "prompt session identifier")
|
|
fs.Var(&referenceFlags, "reference", "reference binding, as slot=path, chunk.slot=path, merge.slot=path, lane.slot=path, lane.extract.slot=path, lane.merge.slot=path, or lane.normalize.slot=path")
|
|
fs.Var(&withoutReferenceFlags, "without-reference", "unbind a reference, using the same selector forms as --reference")
|
|
if err := validateRunFlagValues(args); err != nil {
|
|
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
|
return 2
|
|
}
|
|
if err := fs.Parse(reorderRunArgs(args)); err != nil {
|
|
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
|
return 2
|
|
}
|
|
if fs.NArg() == 0 {
|
|
fmt.Fprintln(stderr, "notarius: run requires a pipeline ID")
|
|
return 2
|
|
}
|
|
if fs.NArg() > 1 {
|
|
fmt.Fprintf(stderr, "notarius: unexpected argument %q\n", fs.Arg(1))
|
|
return 2
|
|
}
|
|
pipelineID := strings.TrimSpace(fs.Arg(0))
|
|
if pipelineID == "" {
|
|
fmt.Fprintln(stderr, "notarius: run requires a pipeline ID")
|
|
return 2
|
|
}
|
|
if strings.TrimSpace(*inputPath) == "" {
|
|
fmt.Fprintln(stderr, "notarius: run requires --input")
|
|
return 2
|
|
}
|
|
if sessionID.set && strings.TrimSpace(sessionID.value) == "" {
|
|
fmt.Fprintln(stderr, "notarius: --session-id must not be empty")
|
|
return 2
|
|
}
|
|
only, err := parseOnly(*onlyRaw)
|
|
if err != nil {
|
|
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
|
return 2
|
|
}
|
|
referenceRequests, err := parseReferenceFlags(referenceFlags)
|
|
if err != nil {
|
|
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
|
return 2
|
|
}
|
|
referenceUnbindRequests, err := parseReferenceUnbindFlags(withoutReferenceFlags)
|
|
if err != nil {
|
|
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
|
return 2
|
|
}
|
|
|
|
cfg, loadedConfigPath, err := loadConfig(*configPath, opts)
|
|
if err != nil {
|
|
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
|
return 1
|
|
}
|
|
workspaceSettings := workspace.FromConfig(cfg)
|
|
if dir := strings.TrimSpace(*diagnosticsDir); dir != "" {
|
|
workspaceSettings.DiagnosticsRoot = dir
|
|
}
|
|
|
|
startedAt := opts.Now().UTC()
|
|
runID := fmt.Sprintf("run-%d", startedAt.UnixNano())
|
|
var runDir *diagnostics.RunDirectory
|
|
if workspaceSettings.DiagnosticsEnabled {
|
|
var err error
|
|
runDir, err = diagnostics.NewRunDirectory(workspaceSettings.DiagnosticsRoot, cfg.Diagnostics.Retention)
|
|
if err != nil {
|
|
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
|
return 1
|
|
}
|
|
runID = runDir.RunID()
|
|
}
|
|
invocation := diagnostics.InvocationMetadata{
|
|
Operation: "run",
|
|
PipelineID: pipelineID,
|
|
InputPath: strings.TrimSpace(*inputPath),
|
|
ConfigPath: loadedConfigPath,
|
|
ConfigSource: configSource(*configPath),
|
|
OnlyLanes: append([]string(nil), only...),
|
|
Resume: *resume,
|
|
RunID: runID,
|
|
StartedAt: startedAt,
|
|
}
|
|
if err := writeDiagnostics(runDir, func() error { return runDir.WriteInvocationMetadata(invocation) }); err != nil {
|
|
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics invocation metadata: %w", err))
|
|
}
|
|
if *resume && !workspaceSettings.ResumeEnabled {
|
|
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("--resume requires workspace.resume.enabled: true"))
|
|
}
|
|
|
|
catalog, err := effectiveCatalog(opts)
|
|
if err != nil {
|
|
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
|
|
}
|
|
referenceOverrides, referenceUnbinds, err := resolveCLIReferenceRequests(cfg, pipelineID, only, catalog, referenceRequests, referenceUnbindRequests)
|
|
if err != nil {
|
|
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
|
|
}
|
|
effective, err := cfg.Resolve(config.ResolveInput{
|
|
PipelineID: pipelineID,
|
|
Only: only,
|
|
Catalog: catalog,
|
|
LLMProfileOverride: *llmProfile,
|
|
ReferenceOverrides: referenceOverrides,
|
|
ReferenceUnbinds: referenceUnbinds,
|
|
})
|
|
if err != nil {
|
|
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
|
|
}
|
|
profileIDs := effectiveLLMProfileIDs(effective.ResolvedPipeline)
|
|
if err := validateExplicitScriptoriumProfiles(context.Background(), effective.Config, profileIDs); err != nil {
|
|
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
|
|
}
|
|
workingDir, err := os.Getwd()
|
|
if err != nil {
|
|
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("resolve working directory: %w", err))
|
|
}
|
|
materialized, referenceWarnings, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{
|
|
ConfigPath: loadedConfigPath,
|
|
WorkingDir: workingDir,
|
|
})
|
|
if err != nil {
|
|
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
|
|
}
|
|
effective.ResolvedPipeline = materialized
|
|
invocation.PipelineDigest = effective.ResolvedPipeline.Digest
|
|
if err := writeDiagnostics(runDir, func() error { return runDir.WriteInvocationMetadata(invocation) }); err != nil {
|
|
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics invocation metadata: %w", err))
|
|
}
|
|
if err := writeDiagnostics(runDir, func() error { return runDir.WriteRedactedEffectiveConfig(effective) }); err != nil {
|
|
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics effective config: %w", err))
|
|
}
|
|
if err := writeDiagnostics(runDir, func() error { return runDir.WriteResolvedPipeline(effective.ResolvedPipeline) }); err != nil {
|
|
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics resolved pipeline: %w", err))
|
|
}
|
|
if err := writeDiagnostics(runDir, func() error {
|
|
return runDir.WriteResolvedReferences(pipeline.ReferenceProvenance(effective.ResolvedPipeline))
|
|
}); err != nil {
|
|
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics resolved references: %w", err))
|
|
}
|
|
|
|
rawInput, err := os.ReadFile(strings.TrimSpace(*inputPath))
|
|
if err != nil {
|
|
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("read input %q: %w", strings.TrimSpace(*inputPath), err))
|
|
}
|
|
|
|
registries, err := effectiveRegistries(opts)
|
|
if err != nil {
|
|
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
|
|
}
|
|
|
|
ctx := context.Background()
|
|
factoryProfileID := ""
|
|
if len(profileIDs) == 1 {
|
|
factoryProfileID = profileIDs[0]
|
|
}
|
|
llmClient, llmProfiles, err := opts.LLMClientFactory(ctx, effective.Config, factoryProfileID)
|
|
if err != nil {
|
|
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("create LLM client for profile %q: %w", factoryProfileID, err))
|
|
}
|
|
checkpointRecorder, checkpointLoader, err := checkpointHandlersForRun(workspaceSettings, effective.ResolvedPipeline, rawInput, only, llmProfiles, strings.TrimSpace(*llmProfile), strings.TrimSpace(sessionID.value), *resume)
|
|
if err != nil {
|
|
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
|
|
}
|
|
|
|
output, err := pipeline.New(registries).Run(ctx, pipeline.RunInput{
|
|
Pipeline: effective.ResolvedPipeline,
|
|
Path: strings.TrimSpace(*inputPath),
|
|
RawInput: rawInput,
|
|
LLMClient: llmClient,
|
|
SessionID: strings.TrimSpace(sessionID.value),
|
|
RunID: runID,
|
|
StartedAt: startedAt,
|
|
LLMProfiles: llmProfiles,
|
|
Metadata: runMetadata(*outputDir, *diagnosticsDir),
|
|
Warnings: referenceWarnings,
|
|
Checkpoints: checkpointRecorder,
|
|
Checkpoint: checkpointLoader,
|
|
})
|
|
if err != nil {
|
|
if output.Manifest.PipelineID != "" && runDir != nil {
|
|
_ = runDir.WriteRunManifest(output.Manifest)
|
|
_ = runDir.WriteCheckpointEvents(output.CheckpointEvents)
|
|
}
|
|
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("run pipeline %q: %w", pipelineID, err))
|
|
}
|
|
|
|
runOutputDir := filepath.Join(outputRoot(*outputDir), runID)
|
|
if err := writeDiagnostics(runDir, func() error { return runDir.WriteRunManifest(output.Manifest) }); err != nil {
|
|
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics run manifest: %w", err))
|
|
}
|
|
if err := writeDiagnostics(runDir, func() error { return runDir.WriteWarnings(output.Warnings) }); err != nil {
|
|
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics warnings: %w", err))
|
|
}
|
|
if err := writeDiagnostics(runDir, func() error { return runDir.WriteCheckpointEvents(output.CheckpointEvents) }); err != nil {
|
|
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics checkpoint events: %w", err))
|
|
}
|
|
if err := writeDiagnostics(runDir, func() error {
|
|
return runDir.WriteRunReport(runReport{
|
|
RunID: runDir.RunID(),
|
|
PipelineID: effective.PipelineID,
|
|
OutputPath: runOutputDir,
|
|
DiagnosticsPath: runDir.Path(),
|
|
OutputCount: len(output.NormalizeOutputs),
|
|
RejectedCount: len(output.Rejected),
|
|
WarningCount: len(output.Warnings),
|
|
ValidationStatus: output.Manifest.ValidationStatus,
|
|
})
|
|
}); err != nil {
|
|
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics run report: %w", err))
|
|
}
|
|
if err := writeOutputFiles(runOutputDir, output.OutputFiles); err != nil {
|
|
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
|
|
}
|
|
if err := writeDiagnostics(runDir, func() error {
|
|
return runDir.ApplyRetention(diagnostics.RetentionDecisionInput{
|
|
RetentionMode: cfg.Diagnostics.Retention,
|
|
RunSucceeded: true,
|
|
HasWarnings: len(output.Warnings) > 0,
|
|
})
|
|
}); err != nil {
|
|
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("apply diagnostics retention: %w", err))
|
|
}
|
|
|
|
fmt.Fprintf(stdout, "pipeline %q complete: outputs=%d rejected=%d output=%s\n", effective.PipelineID, len(output.NormalizeOutputs), len(output.Rejected), runOutputDir)
|
|
if len(output.Warnings) > 0 {
|
|
fmt.Fprintf(stderr, "notarius: run completed with %d warning(s)\n", len(output.Warnings))
|
|
}
|
|
return 0
|
|
}
|
|
|
|
type runReport struct {
|
|
RunID string `json:"run_id"`
|
|
PipelineID string `json:"pipeline_id"`
|
|
OutputPath string `json:"output_path"`
|
|
DiagnosticsPath string `json:"diagnostics_path,omitempty"`
|
|
OutputCount int `json:"output_count"`
|
|
RejectedCount int `json:"rejected_count"`
|
|
WarningCount int `json:"warning_count"`
|
|
ValidationStatus string `json:"validation_status,omitempty"`
|
|
}
|
|
|
|
func failPipelineCommand(stderr io.Writer, runDir *diagnostics.RunDirectory, retention diagnostics.RetentionMode, err error) int {
|
|
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
|
if runDir != nil {
|
|
if logErr := runDir.WriteErrorLog(err.Error()); logErr != nil {
|
|
fmt.Fprintf(stderr, "notarius: write diagnostics error log: %v\n", logErr)
|
|
}
|
|
if retentionErr := runDir.ApplyRetention(diagnostics.RetentionDecisionInput{
|
|
RetentionMode: retention,
|
|
RunSucceeded: false,
|
|
}); retentionErr != nil {
|
|
fmt.Fprintf(stderr, "notarius: apply diagnostics retention: %v\n", retentionErr)
|
|
}
|
|
}
|
|
return 1
|
|
}
|
|
|
|
func writeDiagnostics(runDir *diagnostics.RunDirectory, write func() error) error {
|
|
if runDir == nil {
|
|
return nil
|
|
}
|
|
return write()
|
|
}
|
|
|
|
func checkpointHandlersForRun(
|
|
settings workspace.Settings,
|
|
resolved pipeline.ResolvedPipeline,
|
|
rawInput []byte,
|
|
only []string,
|
|
llmProfiles []artifacts.LLMProfileManifest,
|
|
llmProfileOverride string,
|
|
sessionID string,
|
|
resume bool,
|
|
) (pipeline.CheckpointRecorder, pipeline.CheckpointLoader, error) {
|
|
identity, err := workspace.NewCheckpointIdentity(workspace.CheckpointIdentityInput{
|
|
Pipeline: resolved,
|
|
InputKey: resolved.Input.Module,
|
|
RawInputDigest: rawInputDigest(rawInput),
|
|
SelectedLanes: only,
|
|
RuntimeOverrides: runtimeOverrideFingerprints(llmProfileOverride, sessionID),
|
|
References: pipeline.ReferenceProvenance(resolved),
|
|
ProvenanceFingerprints: llmProfileFingerprints(llmProfiles),
|
|
})
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("create checkpoint identity: %w", err)
|
|
}
|
|
recorder, err := checkpoint.NewWorkspaceRecorder(settings, identity)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("create checkpoint recorder: %w", err)
|
|
}
|
|
loader := pipeline.NoopCheckpointLoader()
|
|
if resume {
|
|
loader, err = checkpoint.NewWorkspaceLoader(settings, identity)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("create checkpoint loader: %w", err)
|
|
}
|
|
}
|
|
return recorder, loader, nil
|
|
}
|
|
|
|
func rawInputDigest(data []byte) string {
|
|
sum := sha256.Sum256(data)
|
|
return "sha256:" + hex.EncodeToString(sum[:])
|
|
}
|
|
|
|
func runtimeOverrideFingerprints(llmProfileOverride string, sessionID string) []workspace.Fingerprint {
|
|
var values []workspace.Fingerprint
|
|
if strings.TrimSpace(llmProfileOverride) != "" {
|
|
values = append(values, workspace.Fingerprint{Name: "llm_profile_override", Value: strings.TrimSpace(llmProfileOverride)})
|
|
}
|
|
if strings.TrimSpace(sessionID) != "" {
|
|
values = append(values, workspace.Fingerprint{Name: "session_id", Value: strings.TrimSpace(sessionID)})
|
|
}
|
|
return values
|
|
}
|
|
|
|
func llmProfileFingerprints(profiles []artifacts.LLMProfileManifest) []workspace.Fingerprint {
|
|
if len(profiles) == 0 {
|
|
return nil
|
|
}
|
|
values := make([]workspace.Fingerprint, 0, len(profiles))
|
|
for _, profile := range profiles {
|
|
id := strings.TrimSpace(profile.ID)
|
|
if id == "" {
|
|
continue
|
|
}
|
|
values = append(values, workspace.Fingerprint{
|
|
Name: "llm_profile:" + id,
|
|
Value: strings.TrimSpace(profile.Provider) + ":" + strings.TrimSpace(profile.Model),
|
|
})
|
|
}
|
|
return values
|
|
}
|
|
|
|
func configSource(configPath string) string {
|
|
if strings.TrimSpace(configPath) != "" {
|
|
return "flag"
|
|
}
|
|
return "discovered"
|
|
}
|
|
|
|
func outputRoot(outputDir string) string {
|
|
if dir := strings.TrimSpace(outputDir); dir != "" {
|
|
return dir
|
|
}
|
|
return defaultOutputRoot
|
|
}
|
|
|
|
func writeOutputFiles(runOutputDir string, files []contracts.OutputFile) error {
|
|
type outputTarget struct {
|
|
path string
|
|
file contracts.OutputFile
|
|
}
|
|
targets := make([]outputTarget, 0, len(files))
|
|
for _, file := range files {
|
|
targetPath, err := outputFilePath(runOutputDir, file.Name)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
targets = append(targets, outputTarget{path: targetPath, file: file})
|
|
}
|
|
|
|
if err := os.MkdirAll(runOutputDir, 0o755); err != nil {
|
|
return fmt.Errorf("create output directory %q: %w", runOutputDir, err)
|
|
}
|
|
for _, target := range targets {
|
|
if err := os.MkdirAll(filepath.Dir(target.path), 0o755); err != nil {
|
|
return fmt.Errorf("create output directory %q: %w", filepath.Dir(target.path), err)
|
|
}
|
|
if err := writeFileAtomic(target.path, target.file.Bytes, 0o644); err != nil {
|
|
return fmt.Errorf("write output file %q: %w", target.file.Name, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func outputFilePath(runOutputDir, logicalName string) (string, error) {
|
|
name := strings.TrimSpace(logicalName)
|
|
if name == "" {
|
|
return "", fmt.Errorf("output file name must not be empty")
|
|
}
|
|
if strings.Contains(name, `\`) {
|
|
return "", fmt.Errorf("output file name %q must use slash-separated relative paths", name)
|
|
}
|
|
if path.IsAbs(name) || filepath.IsAbs(name) {
|
|
return "", fmt.Errorf("output file name %q must be relative", name)
|
|
}
|
|
if strings.Contains(name, "..") {
|
|
return "", fmt.Errorf("output file name %q must not contain ..", name)
|
|
}
|
|
cleaned := path.Clean(name)
|
|
if cleaned == "." || cleaned != name {
|
|
return "", fmt.Errorf("output file name %q must be clean", name)
|
|
}
|
|
|
|
root, err := filepath.Abs(runOutputDir)
|
|
if err != nil {
|
|
return "", fmt.Errorf("resolve output directory %q: %w", runOutputDir, err)
|
|
}
|
|
target, err := filepath.Abs(filepath.Join(root, filepath.FromSlash(cleaned)))
|
|
if err != nil {
|
|
return "", fmt.Errorf("resolve output file %q: %w", name, err)
|
|
}
|
|
rel, err := filepath.Rel(root, target)
|
|
if err != nil {
|
|
return "", fmt.Errorf("resolve output file %q: %w", name, err)
|
|
}
|
|
if rel == "." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || rel == ".." {
|
|
return "", fmt.Errorf("output file name %q resolves outside output directory", name)
|
|
}
|
|
return target, nil
|
|
}
|
|
|
|
func writeFileAtomic(path string, data []byte, perm os.FileMode) error {
|
|
dir := filepath.Dir(path)
|
|
temp, err := os.CreateTemp(dir, "."+filepath.Base(path)+".tmp-*")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
tempPath := temp.Name()
|
|
removeTemp := true
|
|
defer func() {
|
|
if removeTemp {
|
|
_ = os.Remove(tempPath)
|
|
}
|
|
}()
|
|
|
|
if _, err := temp.Write(data); err != nil {
|
|
_ = temp.Close()
|
|
return err
|
|
}
|
|
if err := temp.Chmod(perm); err != nil {
|
|
_ = temp.Close()
|
|
return err
|
|
}
|
|
if err := temp.Close(); err != nil {
|
|
return err
|
|
}
|
|
if err := os.Rename(tempPath, path); err != nil {
|
|
return err
|
|
}
|
|
removeTemp = false
|
|
return nil
|
|
}
|
|
|
|
func reorderRunArgs(args []string) []string {
|
|
var flags []string
|
|
var positionals []string
|
|
for i := 0; i < len(args); i++ {
|
|
arg := args[i]
|
|
if arg == "--" {
|
|
positionals = append(positionals, args[i+1:]...)
|
|
break
|
|
}
|
|
if strings.HasPrefix(arg, "-") {
|
|
flags = append(flags, arg)
|
|
if runFlagTakesValue(arg) && !strings.Contains(arg, "=") && i+1 < len(args) {
|
|
i++
|
|
flags = append(flags, args[i])
|
|
}
|
|
continue
|
|
}
|
|
positionals = append(positionals, arg)
|
|
}
|
|
return append(flags, positionals...)
|
|
}
|
|
|
|
func runFlagTakesValue(arg string) bool {
|
|
switch arg {
|
|
case "--config", "--input", "--only", "--output-dir", "--diagnostics-dir", "--llm-profile", "--session-id", "--reference", "--without-reference":
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func validateRunFlagValues(args []string) error {
|
|
for i, arg := range args {
|
|
if arg != "--session-id" {
|
|
continue
|
|
}
|
|
if i+1 >= len(args) || strings.HasPrefix(args[i+1], "-") {
|
|
return fmt.Errorf("flag needs an argument: --session-id")
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func effectiveLLMProfileIDs(resolved pipeline.ResolvedPipeline) []string {
|
|
seen := make(map[string]struct{})
|
|
add := func(binding pipeline.ModuleBinding) {
|
|
id := strings.TrimSpace(binding.LLMProfile)
|
|
if id != "" {
|
|
seen[id] = struct{}{}
|
|
}
|
|
}
|
|
add(resolved.Chunk)
|
|
for _, lane := range resolved.ArtifactLanes {
|
|
add(lane.Extract)
|
|
add(lane.Merge)
|
|
add(lane.Normalize)
|
|
}
|
|
for _, chain := range resolved.ValidatorChains {
|
|
for _, validator := range chain.Validators {
|
|
if validator.ExecutionClass == contracts.ExecutionClassLLMBacked {
|
|
add(validator.Binding)
|
|
}
|
|
}
|
|
}
|
|
ids := make([]string, 0, len(seen))
|
|
for id := range seen {
|
|
ids = append(ids, id)
|
|
}
|
|
sort.Strings(ids)
|
|
return ids
|
|
}
|
|
|
|
func runMetadata(outputDir, diagnosticsDir string) map[string]any {
|
|
metadata := make(map[string]any)
|
|
if dir := strings.TrimSpace(outputDir); dir != "" {
|
|
metadata["output_dir"] = dir
|
|
}
|
|
if dir := strings.TrimSpace(diagnosticsDir); dir != "" {
|
|
metadata["diagnostics_dir"] = dir
|
|
}
|
|
if len(metadata) == 0 {
|
|
return nil
|
|
}
|
|
return metadata
|
|
}
|
|
|
|
func runConfig(args []string, stdout, stderr io.Writer, opts Options) int {
|
|
if len(args) == 0 {
|
|
fmt.Fprintln(stderr, "notarius: config requires a subcommand")
|
|
writeUsage(stderr)
|
|
return 2
|
|
}
|
|
switch args[0] {
|
|
case "validate":
|
|
return runConfigValidate(args[1:], stdout, stderr, opts)
|
|
default:
|
|
fmt.Fprintf(stderr, "notarius: unknown config subcommand %q\n", args[0])
|
|
writeUsage(stderr)
|
|
return 2
|
|
}
|
|
}
|
|
|
|
func runConfigValidate(args []string, stdout, stderr io.Writer, opts Options) int {
|
|
fs := flag.NewFlagSet("config validate", flag.ContinueOnError)
|
|
fs.SetOutput(io.Discard)
|
|
configPath := fs.String("config", "", "config file path")
|
|
pipelineID := fs.String("pipeline", "", "pipeline ID")
|
|
onlyRaw := fs.String("only", "", "comma-separated artifact lanes")
|
|
if err := fs.Parse(args); err != nil {
|
|
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
|
return 2
|
|
}
|
|
if fs.NArg() != 0 {
|
|
fmt.Fprintf(stderr, "notarius: unexpected argument %q\n", fs.Arg(0))
|
|
return 2
|
|
}
|
|
if strings.TrimSpace(*onlyRaw) != "" && strings.TrimSpace(*pipelineID) == "" {
|
|
fmt.Fprintln(stderr, "notarius: --only requires --pipeline")
|
|
return 2
|
|
}
|
|
only, err := parseOnly(*onlyRaw)
|
|
if err != nil {
|
|
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
|
return 2
|
|
}
|
|
|
|
cfg, path, err := loadConfig(*configPath, opts)
|
|
if err != nil {
|
|
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
|
return 1
|
|
}
|
|
|
|
if strings.TrimSpace(*pipelineID) != "" {
|
|
catalog, err := effectiveCatalog(opts)
|
|
if err != nil {
|
|
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
|
return 1
|
|
}
|
|
effective, err := cfg.Resolve(config.ResolveInput{
|
|
PipelineID: *pipelineID,
|
|
Only: only,
|
|
Catalog: catalog,
|
|
})
|
|
if err != nil {
|
|
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
|
return 1
|
|
}
|
|
if err := validateExplicitScriptoriumProfiles(context.Background(), effective.Config, effectiveLLMProfileIDs(effective.ResolvedPipeline)); err != nil {
|
|
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
|
return 1
|
|
}
|
|
fmt.Fprintf(stdout, "config %q is valid for pipeline %q\n", path, strings.TrimSpace(*pipelineID))
|
|
return 0
|
|
}
|
|
|
|
if err := cfg.Validate(); err != nil {
|
|
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
|
return 1
|
|
}
|
|
fmt.Fprintf(stdout, "config %q is valid\n", path)
|
|
return 0
|
|
}
|
|
|
|
func runPipelines(args []string, stdout, stderr io.Writer, opts Options) int {
|
|
if len(args) == 0 {
|
|
fmt.Fprintln(stderr, "notarius: pipelines requires a subcommand")
|
|
writeUsage(stderr)
|
|
return 2
|
|
}
|
|
switch args[0] {
|
|
case "list":
|
|
return runPipelinesList(args[1:], stdout, stderr, opts)
|
|
default:
|
|
fmt.Fprintf(stderr, "notarius: unknown pipelines subcommand %q\n", args[0])
|
|
writeUsage(stderr)
|
|
return 2
|
|
}
|
|
}
|
|
|
|
func runPipelinesList(args []string, stdout, stderr io.Writer, opts Options) int {
|
|
fs := flag.NewFlagSet("pipelines list", flag.ContinueOnError)
|
|
fs.SetOutput(io.Discard)
|
|
configPath := fs.String("config", "", "config file path")
|
|
jsonOutput := fs.Bool("json", false, "write JSON output")
|
|
if err := fs.Parse(args); err != nil {
|
|
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
|
return 2
|
|
}
|
|
if fs.NArg() != 0 {
|
|
fmt.Fprintf(stderr, "notarius: unexpected argument %q\n", fs.Arg(0))
|
|
return 2
|
|
}
|
|
|
|
cfg, _, err := loadConfig(*configPath, opts)
|
|
if err != nil {
|
|
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
|
return 1
|
|
}
|
|
if err := cfg.Validate(); err != nil {
|
|
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
|
return 1
|
|
}
|
|
|
|
ids := sortedPipelineIDs(cfg)
|
|
if *jsonOutput {
|
|
payload := struct {
|
|
Pipelines []string `json:"pipelines"`
|
|
}{Pipelines: ids}
|
|
encoded, err := json.Marshal(payload)
|
|
if err != nil {
|
|
fmt.Fprintf(stderr, "notarius: marshal pipeline list: %v\n", err)
|
|
return 1
|
|
}
|
|
fmt.Fprintf(stdout, "%s\n", encoded)
|
|
return 0
|
|
}
|
|
|
|
for _, id := range ids {
|
|
fmt.Fprintln(stdout, id)
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func loadConfig(configPath string, opts Options) (config.Config, string, error) {
|
|
path, err := discoverConfigPath(configPath, opts)
|
|
if err != nil {
|
|
return config.Config{}, "", err
|
|
}
|
|
|
|
fileCfg, err := config.LoadFileConfig(path)
|
|
if err != nil {
|
|
return config.Config{}, "", err
|
|
}
|
|
cfg := config.Default()
|
|
if err := cfg.ApplyFileConfigWithLookup(fileCfg, opts.LookupEnv); err != nil {
|
|
return config.Config{}, "", err
|
|
}
|
|
if err := cfg.ApplyEnvOverridesWithLookup(opts.LookupEnv); err != nil {
|
|
return config.Config{}, "", err
|
|
}
|
|
return cfg, path, nil
|
|
}
|
|
|
|
func discoverConfigPath(configPath string, opts Options) (string, error) {
|
|
if path := strings.TrimSpace(configPath); path != "" {
|
|
if err := requireConfigFile(path); err != nil {
|
|
return "", err
|
|
}
|
|
return path, nil
|
|
}
|
|
if path, ok := opts.LookupEnv("NOTARIUS_CONFIG"); ok && strings.TrimSpace(path) != "" {
|
|
path = strings.TrimSpace(path)
|
|
if err := requireConfigFile(path); err != nil {
|
|
return "", err
|
|
}
|
|
return path, nil
|
|
}
|
|
if _, err := os.Stat(defaultConfigPath); err == nil {
|
|
return defaultConfigPath, nil
|
|
} else if err != nil && !os.IsNotExist(err) {
|
|
return "", fmt.Errorf("check default config %q: %w", defaultConfigPath, err)
|
|
}
|
|
return "", fmt.Errorf("config file not found; pass --config or set NOTARIUS_CONFIG")
|
|
}
|
|
|
|
func requireConfigFile(path string) error {
|
|
info, err := os.Stat(path)
|
|
if err != nil {
|
|
return fmt.Errorf("config file %q is not available: %w", path, err)
|
|
}
|
|
if info.IsDir() {
|
|
return fmt.Errorf("config file %q is a directory", path)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func parseOnly(raw string) ([]string, error) {
|
|
if strings.TrimSpace(raw) == "" {
|
|
return nil, nil
|
|
}
|
|
parts := strings.Split(raw, ",")
|
|
result := make([]string, 0, len(parts))
|
|
for _, part := range parts {
|
|
trimmed := strings.TrimSpace(part)
|
|
if trimmed == "" {
|
|
return nil, fmt.Errorf("--only must contain comma-separated non-empty artifact lane IDs")
|
|
}
|
|
result = append(result, trimmed)
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
type stringListFlag []string
|
|
|
|
func (flag *stringListFlag) String() string {
|
|
if flag == nil {
|
|
return ""
|
|
}
|
|
return strings.Join(*flag, ",")
|
|
}
|
|
|
|
func (flag *stringListFlag) Set(value string) error {
|
|
*flag = append(*flag, value)
|
|
return nil
|
|
}
|
|
|
|
type sessionIDFlag struct {
|
|
value string
|
|
set bool
|
|
}
|
|
|
|
func (flag *sessionIDFlag) String() string {
|
|
if flag == nil {
|
|
return ""
|
|
}
|
|
return flag.value
|
|
}
|
|
|
|
func (flag *sessionIDFlag) Set(value string) error {
|
|
flag.value = value
|
|
flag.set = true
|
|
return nil
|
|
}
|
|
|
|
type cliReferenceRequest struct {
|
|
Selector cliReferenceSelector
|
|
Source string
|
|
}
|
|
|
|
type cliReferenceUnbindRequest struct {
|
|
Selector cliReferenceSelector
|
|
}
|
|
|
|
type cliReferenceSelector struct {
|
|
LaneID string
|
|
Stage pipeline.ModuleStage
|
|
SlotName string
|
|
}
|
|
|
|
func parseReferenceFlags(values []string) ([]cliReferenceRequest, error) {
|
|
if len(values) == 0 {
|
|
return nil, nil
|
|
}
|
|
requests := make([]cliReferenceRequest, 0, len(values))
|
|
for _, raw := range values {
|
|
name, source, ok := strings.Cut(raw, "=")
|
|
if !ok {
|
|
return nil, fmt.Errorf("--reference must use slot=path or lane.slot=path")
|
|
}
|
|
if strings.TrimSpace(source) == "" {
|
|
return nil, fmt.Errorf("--reference path must not be empty; use --without-reference to unbind")
|
|
}
|
|
selector, err := parseReferenceSelector(name, "--reference")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
requests = append(requests, cliReferenceRequest{
|
|
Selector: selector,
|
|
Source: strings.TrimSpace(source),
|
|
})
|
|
}
|
|
return requests, nil
|
|
}
|
|
|
|
func parseReferenceUnbindFlags(values []string) ([]cliReferenceUnbindRequest, error) {
|
|
if len(values) == 0 {
|
|
return nil, nil
|
|
}
|
|
requests := make([]cliReferenceUnbindRequest, 0, len(values))
|
|
for _, raw := range values {
|
|
if strings.Contains(raw, "=") {
|
|
return nil, fmt.Errorf("--without-reference must use a reference selector without =path")
|
|
}
|
|
selector, err := parseReferenceSelector(raw, "--without-reference")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
requests = append(requests, cliReferenceUnbindRequest{
|
|
Selector: selector,
|
|
})
|
|
}
|
|
return requests, nil
|
|
}
|
|
|
|
func parseReferenceSelector(raw string, flagName string) (cliReferenceSelector, error) {
|
|
selector := strings.TrimSpace(raw)
|
|
if selector == "" {
|
|
return cliReferenceSelector{}, fmt.Errorf("%s reference slot must not be empty", flagName)
|
|
}
|
|
parts := strings.Split(selector, ".")
|
|
for _, part := range parts {
|
|
if strings.TrimSpace(part) == "" {
|
|
return cliReferenceSelector{}, fmt.Errorf("%s must use non-empty reference selector values", flagName)
|
|
}
|
|
}
|
|
switch len(parts) {
|
|
case 1:
|
|
return cliReferenceSelector{SlotName: strings.TrimSpace(parts[0])}, nil
|
|
case 2:
|
|
first := strings.TrimSpace(parts[0])
|
|
slotName := strings.TrimSpace(parts[1])
|
|
if first == string(pipeline.StageChunk) {
|
|
return cliReferenceSelector{Stage: pipeline.StageChunk, SlotName: slotName}, nil
|
|
}
|
|
if first == string(pipeline.StageMerge) {
|
|
return cliReferenceSelector{Stage: pipeline.StageMerge, SlotName: slotName}, nil
|
|
}
|
|
return cliReferenceSelector{LaneID: first, SlotName: slotName}, nil
|
|
case 3:
|
|
laneID := strings.TrimSpace(parts[0])
|
|
stage := pipeline.ModuleStage(strings.TrimSpace(parts[1]))
|
|
slotName := strings.TrimSpace(parts[2])
|
|
if stage != pipeline.StageExtract && stage != pipeline.StageMerge && stage != pipeline.StageNormalize {
|
|
return cliReferenceSelector{}, fmt.Errorf("%s lane-qualified selector must use lane.extract.slot, lane.merge.slot, or lane.normalize.slot", flagName)
|
|
}
|
|
return cliReferenceSelector{LaneID: laneID, Stage: stage, SlotName: slotName}, nil
|
|
default:
|
|
return cliReferenceSelector{}, fmt.Errorf("%s must use slot, chunk.slot, merge.slot, lane.slot, lane.extract.slot, lane.merge.slot, or lane.normalize.slot", flagName)
|
|
}
|
|
}
|
|
|
|
func resolveCLIReferenceRequests(
|
|
cfg config.Config,
|
|
pipelineID string,
|
|
only []string,
|
|
catalog pipeline.ModuleCatalog,
|
|
referenceRequests []cliReferenceRequest,
|
|
unbindRequests []cliReferenceUnbindRequest,
|
|
) ([]pipeline.ReferenceBinding, []pipeline.ReferenceUnbind, error) {
|
|
if len(referenceRequests) == 0 && len(unbindRequests) == 0 {
|
|
return nil, nil, nil
|
|
}
|
|
|
|
targets, err := selectedReferenceTargets(cfg, pipelineID, only, catalog)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
overrides := make([]pipeline.ReferenceBinding, 0, len(referenceRequests))
|
|
for _, request := range referenceRequests {
|
|
target, err := resolveCLIReferenceTarget(targets, request.Selector)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
overrides = append(overrides, pipeline.ReferenceBinding{
|
|
Stage: target.stage,
|
|
LaneID: target.laneID,
|
|
SlotName: request.Selector.SlotName,
|
|
Source: request.Source,
|
|
BindingSource: contracts.ReferenceBindingSourceCLI,
|
|
})
|
|
}
|
|
|
|
unbinds := make([]pipeline.ReferenceUnbind, 0, len(unbindRequests))
|
|
for _, request := range unbindRequests {
|
|
target, err := resolveCLIReferenceTarget(targets, request.Selector)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
unbinds = append(unbinds, pipeline.ReferenceUnbind{
|
|
Stage: target.stage,
|
|
LaneID: target.laneID,
|
|
SlotName: request.Selector.SlotName,
|
|
})
|
|
}
|
|
|
|
return overrides, unbinds, nil
|
|
}
|
|
|
|
type selectedReferenceTarget struct {
|
|
laneID string
|
|
stage pipeline.ModuleStage
|
|
module string
|
|
slots map[string]struct{}
|
|
}
|
|
|
|
func selectedReferenceTargets(cfg config.Config, pipelineID string, only []string, catalog pipeline.ModuleCatalog) ([]selectedReferenceTarget, error) {
|
|
profile, ok := lookupCLIReferencePipeline(cfg.Pipelines, pipelineID)
|
|
if !ok {
|
|
return nil, fmt.Errorf("pipeline %q is not configured", strings.TrimSpace(pipelineID))
|
|
}
|
|
|
|
lanesByID := make(map[string]pipeline.ArtifactLaneProfile, len(profile.Artifacts))
|
|
for rawLaneID, lane := range profile.Artifacts {
|
|
laneID := strings.TrimSpace(rawLaneID)
|
|
if laneID == "" {
|
|
return nil, fmt.Errorf("pipeline %q artifact lane id must not be empty", strings.TrimSpace(pipelineID))
|
|
}
|
|
if _, ok := lanesByID[laneID]; ok {
|
|
return nil, fmt.Errorf("pipeline %q artifact lane %q is duplicated after trimming", strings.TrimSpace(pipelineID), laneID)
|
|
}
|
|
lanesByID[laneID] = lane
|
|
}
|
|
|
|
selectedIDs := make([]string, 0, len(lanesByID))
|
|
if len(only) == 0 {
|
|
for laneID := range lanesByID {
|
|
selectedIDs = append(selectedIDs, laneID)
|
|
}
|
|
} else {
|
|
seen := make(map[string]struct{}, len(only))
|
|
for _, rawLaneID := range only {
|
|
laneID := strings.TrimSpace(rawLaneID)
|
|
if laneID == "" {
|
|
return nil, fmt.Errorf("pipeline %q selected artifact lane id must not be empty", strings.TrimSpace(pipelineID))
|
|
}
|
|
if _, ok := lanesByID[laneID]; !ok {
|
|
return nil, fmt.Errorf("pipeline %q selected artifact lane %q is not declared", strings.TrimSpace(pipelineID), laneID)
|
|
}
|
|
if _, ok := seen[laneID]; !ok {
|
|
selectedIDs = append(selectedIDs, laneID)
|
|
seen[laneID] = struct{}{}
|
|
}
|
|
}
|
|
}
|
|
sort.Strings(selectedIDs)
|
|
|
|
targets := make([]selectedReferenceTarget, 0, 1+len(selectedIDs)*3)
|
|
chunk := pipeline.Binding(profile.Chunk.Module)
|
|
chunk.Module = strings.TrimSpace(profile.Chunk.Module)
|
|
if chunk.Module == "" {
|
|
chunk.Module = pipeline.DefaultChunkModule
|
|
}
|
|
chunkSpec, err := cliReferenceChunkerSpec(catalog, chunk.Module)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("pipeline %q chunk module %q: %w", strings.TrimSpace(pipelineID), chunk.Module, err)
|
|
}
|
|
targets = append(targets, selectedReferenceTarget{
|
|
stage: pipeline.StageChunk,
|
|
module: chunk.Module,
|
|
slots: referenceSlotSet(chunkSpec.ReferenceSlots),
|
|
})
|
|
|
|
for _, laneID := range selectedIDs {
|
|
lane := lanesByID[laneID]
|
|
extractModule := strings.TrimSpace(lane.Extract.Module)
|
|
if extractModule == "" {
|
|
return nil, fmt.Errorf("pipeline %q lane %q extract module must not be empty", strings.TrimSpace(pipelineID), laneID)
|
|
}
|
|
extractSpec, err := cliReferenceExtractorSpec(catalog, extractModule)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("pipeline %q lane %q extract module %q: %w", strings.TrimSpace(pipelineID), laneID, extractModule, err)
|
|
}
|
|
targets = append(targets, selectedReferenceTarget{
|
|
laneID: laneID,
|
|
stage: pipeline.StageExtract,
|
|
module: extractModule,
|
|
slots: referenceSlotSet(extractSpec.ReferenceSlots),
|
|
})
|
|
|
|
mergeModule := strings.TrimSpace(lane.Merge.Module)
|
|
if mergeModule == "" {
|
|
mergeModule = pipeline.DefaultMergeModule
|
|
}
|
|
mergeSpec, err := cliReferenceMergerSpec(catalog, mergeModule)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("pipeline %q lane %q merge module %q: %w", strings.TrimSpace(pipelineID), laneID, mergeModule, err)
|
|
}
|
|
targets = append(targets, selectedReferenceTarget{
|
|
laneID: laneID,
|
|
stage: pipeline.StageMerge,
|
|
module: mergeModule,
|
|
slots: referenceSlotSet(mergeSpec.ReferenceSlots),
|
|
})
|
|
|
|
normalizeModule := strings.TrimSpace(lane.Normalize.Module)
|
|
if normalizeModule == "" {
|
|
normalizeModule = pipeline.DefaultNormalizeModule
|
|
}
|
|
normalizeSpec, err := cliReferenceNormalizerSpec(catalog, normalizeModule)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("pipeline %q lane %q normalize module %q: %w", strings.TrimSpace(pipelineID), laneID, normalizeModule, err)
|
|
}
|
|
targets = append(targets, selectedReferenceTarget{
|
|
laneID: laneID,
|
|
stage: pipeline.StageNormalize,
|
|
module: normalizeModule,
|
|
slots: referenceSlotSet(normalizeSpec.ReferenceSlots),
|
|
})
|
|
}
|
|
return targets, nil
|
|
}
|
|
|
|
func lookupCLIReferencePipeline(profiles map[string]pipeline.PipelineProfile, pipelineID string) (pipeline.PipelineProfile, bool) {
|
|
pipelineID = strings.TrimSpace(pipelineID)
|
|
for rawID, profile := range profiles {
|
|
if strings.TrimSpace(rawID) == pipelineID {
|
|
return profile, true
|
|
}
|
|
}
|
|
return pipeline.PipelineProfile{}, false
|
|
}
|
|
|
|
func cliReferenceChunkerSpec(catalog pipeline.ModuleCatalog, module string) (pipeline.ModuleSpec, error) {
|
|
if catalog.Chunkers == nil {
|
|
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
|
|
}
|
|
spec, ok := catalog.Chunkers.Spec(module)
|
|
if !ok {
|
|
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
|
|
}
|
|
return spec, nil
|
|
}
|
|
|
|
func cliReferenceExtractorSpec(catalog pipeline.ModuleCatalog, module string) (pipeline.ModuleSpec, error) {
|
|
if catalog.Extractors == nil {
|
|
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
|
|
}
|
|
spec, ok := catalog.Extractors.Spec(module)
|
|
if !ok {
|
|
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
|
|
}
|
|
return spec, nil
|
|
}
|
|
|
|
func cliReferenceMergerSpec(catalog pipeline.ModuleCatalog, module string) (pipeline.ModuleSpec, error) {
|
|
if catalog.Mergers == nil {
|
|
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
|
|
}
|
|
spec, ok := catalog.Mergers.Spec(module)
|
|
if !ok {
|
|
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
|
|
}
|
|
return spec, nil
|
|
}
|
|
|
|
func cliReferenceNormalizerSpec(catalog pipeline.ModuleCatalog, module string) (pipeline.ModuleSpec, error) {
|
|
if catalog.Normalizers == nil {
|
|
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
|
|
}
|
|
spec, ok := catalog.Normalizers.Spec(module)
|
|
if !ok {
|
|
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
|
|
}
|
|
return spec, nil
|
|
}
|
|
|
|
func referenceSlotSet(slots []contracts.ReferenceSlot) map[string]struct{} {
|
|
slotSet := make(map[string]struct{}, len(slots))
|
|
for _, slot := range slots {
|
|
slotSet[slot.Name] = struct{}{}
|
|
}
|
|
return slotSet
|
|
}
|
|
|
|
func resolveCLIReferenceTarget(targets []selectedReferenceTarget, selector cliReferenceSelector) (selectedReferenceTarget, error) {
|
|
slotName := strings.TrimSpace(selector.SlotName)
|
|
if slotName == "" {
|
|
return selectedReferenceTarget{}, fmt.Errorf("reference slot must not be empty")
|
|
}
|
|
if selector.Stage == pipeline.StageChunk {
|
|
for _, target := range targets {
|
|
if target.stage != pipeline.StageChunk {
|
|
continue
|
|
}
|
|
if _, ok := target.slots[slotName]; !ok {
|
|
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by chunk module %q", slotName, target.module)
|
|
}
|
|
return target, nil
|
|
}
|
|
return selectedReferenceTarget{}, fmt.Errorf("reference chunk target is not selected")
|
|
}
|
|
if selector.Stage == pipeline.StageExtract || selector.Stage == pipeline.StageMerge || selector.Stage == pipeline.StageNormalize {
|
|
if selector.LaneID == "" && selector.Stage == pipeline.StageMerge {
|
|
return resolveCLIReferenceStageTarget(targets, selector.Stage, slotName)
|
|
}
|
|
for _, target := range targets {
|
|
if target.laneID == selector.LaneID && target.stage == selector.Stage {
|
|
if _, ok := target.slots[slotName]; !ok {
|
|
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by selected %s target %q", slotName, selector.Stage, targetLabel(target))
|
|
}
|
|
return target, nil
|
|
}
|
|
}
|
|
return selectedReferenceTarget{}, fmt.Errorf("reference lane %q is not selected", selector.LaneID)
|
|
}
|
|
if strings.TrimSpace(selector.LaneID) != "" {
|
|
return resolveCLIReferenceLaneTarget(targets, strings.TrimSpace(selector.LaneID), slotName)
|
|
}
|
|
return resolveCLIReferenceFlatTarget(targets, slotName)
|
|
}
|
|
|
|
func resolveCLIReferenceStageTarget(targets []selectedReferenceTarget, stage pipeline.ModuleStage, slotName string) (selectedReferenceTarget, error) {
|
|
matches := make([]selectedReferenceTarget, 0, 2)
|
|
for _, target := range targets {
|
|
if target.stage != stage {
|
|
continue
|
|
}
|
|
if _, ok := target.slots[slotName]; ok {
|
|
matches = append(matches, target)
|
|
}
|
|
}
|
|
switch len(matches) {
|
|
case 0:
|
|
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by any selected %s target", slotName, stage)
|
|
case 1:
|
|
return matches[0], nil
|
|
default:
|
|
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is declared by multiple selected %s targets (%s); use a more specific selector such as %s", slotName, stage, targetList(matches), selectorSuggestions(matches, slotName))
|
|
}
|
|
}
|
|
|
|
func resolveCLIReferenceLaneTarget(targets []selectedReferenceTarget, laneID string, slotName string) (selectedReferenceTarget, error) {
|
|
laneSelected := false
|
|
matches := make([]selectedReferenceTarget, 0, 2)
|
|
for _, target := range targets {
|
|
if target.laneID != laneID {
|
|
continue
|
|
}
|
|
laneSelected = true
|
|
if _, ok := target.slots[slotName]; ok {
|
|
matches = append(matches, target)
|
|
}
|
|
}
|
|
if !laneSelected {
|
|
return selectedReferenceTarget{}, fmt.Errorf("reference lane %q is not selected", laneID)
|
|
}
|
|
switch len(matches) {
|
|
case 0:
|
|
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by selected lane %q", slotName, laneID)
|
|
case 1:
|
|
return matches[0], nil
|
|
default:
|
|
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is declared by multiple selected targets in lane %q (%s); use a more specific selector such as %s", slotName, laneID, targetList(matches), selectorSuggestions(matches, slotName))
|
|
}
|
|
}
|
|
|
|
func resolveCLIReferenceFlatTarget(targets []selectedReferenceTarget, slotName string) (selectedReferenceTarget, error) {
|
|
matches := make([]selectedReferenceTarget, 0, 2)
|
|
for _, target := range targets {
|
|
if _, ok := target.slots[slotName]; ok {
|
|
matches = append(matches, target)
|
|
}
|
|
}
|
|
switch len(matches) {
|
|
case 0:
|
|
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by any selected reference target", slotName)
|
|
case 1:
|
|
return matches[0], nil
|
|
default:
|
|
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is declared by multiple selected targets (%s); use a more specific selector such as %s", slotName, targetList(matches), selectorSuggestions(matches, slotName))
|
|
}
|
|
}
|
|
|
|
func targetList(targets []selectedReferenceTarget) string {
|
|
labels := make([]string, 0, len(targets))
|
|
for _, target := range targets {
|
|
labels = append(labels, targetLabel(target))
|
|
}
|
|
sort.Strings(labels)
|
|
return strings.Join(labels, ", ")
|
|
}
|
|
|
|
func targetLabel(target selectedReferenceTarget) string {
|
|
if target.stage == pipeline.StageChunk {
|
|
return "chunk"
|
|
}
|
|
return target.laneID + "." + string(target.stage)
|
|
}
|
|
|
|
func selectorSuggestions(targets []selectedReferenceTarget, slotName string) string {
|
|
suggestions := make([]string, 0, len(targets))
|
|
for _, target := range targets {
|
|
if target.stage == pipeline.StageChunk {
|
|
suggestions = append(suggestions, "chunk."+slotName)
|
|
continue
|
|
}
|
|
suggestions = append(suggestions, target.laneID+"."+string(target.stage)+"."+slotName)
|
|
}
|
|
sort.Strings(suggestions)
|
|
return strings.Join(suggestions, " or ")
|
|
}
|
|
|
|
func sortedPipelineIDs(cfg config.Config) []string {
|
|
ids := make([]string, 0, len(cfg.Pipelines))
|
|
for id := range cfg.Pipelines {
|
|
ids = append(ids, strings.TrimSpace(id))
|
|
}
|
|
sort.Strings(ids)
|
|
return ids
|
|
}
|