Unify application configuration loading
This commit is contained in:
@@ -36,6 +36,22 @@ This ordering preserves monolithic configuration behavior. Moving a field to
|
|||||||
an imported fragment changes its source ownership, not its path base, default,
|
an imported fragment changes its source ownership, not its path base, default,
|
||||||
or schema semantics.
|
or schema semantics.
|
||||||
|
|
||||||
|
## Loaded Context Resolution
|
||||||
|
|
||||||
|
`LoadedPipelineCampaign` carries one already composed pipeline and its selected
|
||||||
|
campaign into session resolution. `LoadSessionWithPipelineCampaignOptions`
|
||||||
|
loads a local session against that context, while
|
||||||
|
`ResolveLoadedPipelineCampaign` also accepts an already loaded remote session
|
||||||
|
or no session while a caller retrieves one. Compatibility loaders route through
|
||||||
|
these functions after their initial pipeline and campaign reads.
|
||||||
|
|
||||||
|
Application commands own pipeline and campaign discovery, campaign-file versus
|
||||||
|
registry selection, and the corresponding mutual-exclusion rules. Once they
|
||||||
|
have a `LoadedPipelineCampaign`, local session discovery and remote-session
|
||||||
|
download retain that exact pipeline object and its private provenance. Removing
|
||||||
|
a temporary downloaded session file therefore cannot invalidate the resolved
|
||||||
|
pipeline or campaign context.
|
||||||
|
|
||||||
## Diagnostics And Runtime Metadata
|
## Diagnostics And Runtime Metadata
|
||||||
|
|
||||||
Syntax, duplicate-key, composition, conflict, and schema failures include the
|
Syntax, duplicate-key, composition, conflict, and schema failures include the
|
||||||
@@ -66,5 +82,7 @@ the public schema. `pipeline_composition_test.go` exercises explicit imports,
|
|||||||
confinement, conflicts, strict decoding, metadata, and root-relative path
|
confinement, conflicts, strict decoding, metadata, and root-relative path
|
||||||
behavior through `LoadPipeline`. `pipeline_profiles_test.go` covers selection,
|
behavior through `LoadPipeline`. `pipeline_profiles_test.go` covers selection,
|
||||||
all-overlay validation, overlay behavior, provenance, option propagation, and
|
all-overlay validation, overlay behavior, provenance, option propagation, and
|
||||||
effective-digest stability. Other configuration tests continue to protect
|
effective-digest stability. Application configuration-loader tests protect the
|
||||||
defaults and validation after assembly.
|
single-read boundary by changing the pipeline file after its initial load and
|
||||||
|
confirming local session resolution retains the original pipeline. Other
|
||||||
|
configuration tests continue to protect defaults and validation after assembly.
|
||||||
|
|||||||
@@ -47,9 +47,11 @@ Pipeline execution and `session plan` share the same inclusive contiguous-range
|
|||||||
model. Planning clones session state and applies selected-stage transitions and
|
model. Planning clones session state and applies selected-stage transitions and
|
||||||
resume validation in memory; it does not create invocation state or initialize
|
resume validation in memory; it does not create invocation state or initialize
|
||||||
stage-execution adapters. Command configuration loading can still retrieve a
|
stage-execution adapters. Command configuration loading can still retrieve a
|
||||||
missing session file through configured remote storage. Analyze planning
|
missing session file through configured remote storage. It retains the initially
|
||||||
additionally exposes the artifact closure's targets, prerequisite rebuilds,
|
composed pipeline and selected campaign while resolving either a local or
|
||||||
execution order, and current reuse.
|
downloaded remote session, so one invocation cannot mix pipeline revisions.
|
||||||
|
Analyze planning additionally exposes the artifact closure's targets,
|
||||||
|
prerequisite rebuilds, execution order, and current reuse.
|
||||||
|
|
||||||
## Pipeline Stage Set
|
## Pipeline Stage Set
|
||||||
|
|
||||||
|
|||||||
@@ -497,7 +497,7 @@ configuration provenance within `internal/config`.
|
|||||||
|
|
||||||
## Stage 8 — One Production Configuration Loading Path
|
## Stage 8 — One Production Configuration Loading Path
|
||||||
|
|
||||||
**Status: Pending**
|
**Status: Completed**
|
||||||
|
|
||||||
### Goal
|
### Goal
|
||||||
|
|
||||||
|
|||||||
@@ -88,11 +88,7 @@ func cleanAllLocal(flags commonConfigFlags, dryRun, clearCache bool, out io.Writ
|
|||||||
strings.TrimSpace(flags.previousSessionID) != "" {
|
strings.TrimSpace(flags.previousSessionID) != "" {
|
||||||
return fmt.Errorf("clean: --all cannot be combined with --campaign, --campaign-file, --session, a session_id, or --previous-session-id")
|
return fmt.Errorf("clean: --all cannot be combined with --campaign, --campaign-file, --session, a session_id, or --previous-session-id")
|
||||||
}
|
}
|
||||||
resolvedPipelinePath, err := resolvePipelineConfigPath(flags.pipelinePath)
|
_, pipelineCfg, err := loadPipelineConfig(flags.pipelinePath)
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("clean: %w", err)
|
|
||||||
}
|
|
||||||
pipelineCfg, err := config.LoadPipeline(resolvedPipelinePath)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("clean: %w", err)
|
return fmt.Errorf("clean: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,14 +14,10 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||||
)
|
)
|
||||||
|
|
||||||
type pipelineCampaignConfig struct {
|
type pipelineCampaignConfig = config.LoadedPipelineCampaign
|
||||||
PipelinePath string
|
|
||||||
CampaignPath string
|
|
||||||
Pipeline *config.PipelineConfig
|
|
||||||
Campaign *config.CampaignConfig
|
|
||||||
}
|
|
||||||
|
|
||||||
var downloadObjectToTempFn = storage.DownloadObjectToTemp
|
var downloadObjectToTempFn = storage.DownloadObjectToTemp
|
||||||
|
var loadPipelineConfigFn = config.LoadPipeline
|
||||||
|
|
||||||
type commandConfig struct {
|
type commandConfig struct {
|
||||||
Config *config.Config
|
Config *config.Config
|
||||||
@@ -58,7 +54,7 @@ func loadCommandConfig(ctx context.Context, pipelineFlag, campaignFlag, campaign
|
|||||||
}
|
}
|
||||||
|
|
||||||
if explicitSession := strings.TrimSpace(sessionFlag); explicitSession != "" {
|
if explicitSession := strings.TrimSpace(sessionFlag); explicitSession != "" {
|
||||||
cfg, err := config.LoadWithSessionOptions(base.PipelinePath, base.CampaignPath, explicitSession, sessionOpts)
|
cfg, err := config.LoadSessionWithPipelineCampaignOptions(*base, explicitSession, sessionOpts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -70,7 +66,7 @@ func loadCommandConfig(ctx context.Context, pipelineFlag, campaignFlag, campaign
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if discoveredSession.Path != "" {
|
if discoveredSession.Path != "" {
|
||||||
cfg, err := config.LoadWithSessionOptions(base.PipelinePath, base.CampaignPath, discoveredSession.Path, sessionOpts)
|
cfg, err := config.LoadSessionWithPipelineCampaignOptions(*base, discoveredSession.Path, sessionOpts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -88,11 +84,9 @@ func loadCommandConfig(ctx context.Context, pipelineFlag, campaignFlag, campaign
|
|||||||
}
|
}
|
||||||
sessionPrefix := artifacts.S3SessionPrefix(rootPrefix, config.CampaignID(base.Campaign), sessionID)
|
sessionPrefix := artifacts.S3SessionPrefix(rootPrefix, config.CampaignID(base.Campaign), sessionID)
|
||||||
remoteKey := artifacts.S3SessionConfigKey(sessionPrefix)
|
remoteKey := artifacts.S3SessionConfigKey(sessionPrefix)
|
||||||
partialCfg := &config.Config{
|
partialCfg, err := config.ResolveLoadedPipelineCampaign(*base, "", nil, config.SessionSource{})
|
||||||
Pipeline: base.Pipeline,
|
if err != nil {
|
||||||
Campaign: base.Campaign,
|
return nil, err
|
||||||
PipelinePath: base.PipelinePath,
|
|
||||||
CampaignPath: base.CampaignPath,
|
|
||||||
}
|
}
|
||||||
store, err := newCommandObjectStore(ctx, partialCfg, nil)
|
store, err := newCommandObjectStore(ctx, partialCfg, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -122,11 +116,8 @@ func loadCommandConfig(ctx context.Context, pipelineFlag, campaignFlag, campaign
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg, err := config.Resolve(
|
cfg, err := config.ResolveLoadedPipelineCampaign(
|
||||||
base.PipelinePath,
|
*base,
|
||||||
base.Pipeline,
|
|
||||||
base.CampaignPath,
|
|
||||||
base.Campaign,
|
|
||||||
sessionTempPath,
|
sessionTempPath,
|
||||||
sessionCfg,
|
sessionCfg,
|
||||||
config.SessionSource{
|
config.SessionSource{
|
||||||
@@ -147,11 +138,7 @@ func loadCommandConfig(ctx context.Context, pipelineFlag, campaignFlag, campaign
|
|||||||
}
|
}
|
||||||
|
|
||||||
func loadPipelineCampaignConfig(pipelineFlag, campaignFlag, campaignFileFlag string) (*pipelineCampaignConfig, error) {
|
func loadPipelineCampaignConfig(pipelineFlag, campaignFlag, campaignFileFlag string) (*pipelineCampaignConfig, error) {
|
||||||
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelineFlag)
|
loadedPipelinePath, pipelineCfg, err := loadPipelineConfig(pipelineFlag)
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
pipelineCfg, err := config.LoadPipeline(resolvedPipelinePath)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -169,13 +156,25 @@ func loadPipelineCampaignConfig(pipelineFlag, campaignFlag, campaignFileFlag str
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
return &pipelineCampaignConfig{
|
return &pipelineCampaignConfig{
|
||||||
PipelinePath: resolvedPipelinePath,
|
PipelinePath: loadedPipelinePath,
|
||||||
CampaignPath: resolvedCampaignPath,
|
CampaignPath: resolvedCampaignPath,
|
||||||
Pipeline: pipelineCfg,
|
Pipeline: pipelineCfg,
|
||||||
Campaign: campaignCfg,
|
Campaign: campaignCfg,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func loadPipelineConfig(pipelineFlag string) (string, *config.PipelineConfig, error) {
|
||||||
|
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelineFlag)
|
||||||
|
if err != nil {
|
||||||
|
return "", nil, err
|
||||||
|
}
|
||||||
|
pipelineCfg, err := loadPipelineConfigFn(resolvedPipelinePath)
|
||||||
|
if err != nil {
|
||||||
|
return "", nil, err
|
||||||
|
}
|
||||||
|
return resolvedPipelinePath, pipelineCfg, nil
|
||||||
|
}
|
||||||
|
|
||||||
func findRemoteSessionConfig(ctx context.Context, store storage.ObjectStore, sessionPrefix, remoteKey string) (storage.ObjectInfo, error) {
|
func findRemoteSessionConfig(ctx context.Context, store storage.ObjectStore, sessionPrefix, remoteKey string) (storage.ObjectInfo, error) {
|
||||||
objects, err := store.List(ctx, sessionPrefix)
|
objects, err := store.List(ctx, sessionPrefix)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
50
internal/app/config_loader_test.go
Normal file
50
internal/app/config_loader_test.go
Normal file
@@ -0,0 +1,50 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestLoadCommandConfigRetainsInitiallyLoadedPipeline(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
|
originalLoader := loadPipelineConfigFn
|
||||||
|
loadCalls := 0
|
||||||
|
loadPipelineConfigFn = func(path string) (*config.PipelineConfig, error) {
|
||||||
|
loaded, err := originalLoader(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
loadCalls++
|
||||||
|
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
changedRoot := workspaceRoot + "-changed"
|
||||||
|
updated := strings.ReplaceAll(string(data), workspaceRoot, changedRoot)
|
||||||
|
if err := os.WriteFile(path, []byte(updated), 0o644); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return loaded, nil
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { loadPipelineConfigFn = originalLoader })
|
||||||
|
|
||||||
|
loaded, err := loadCommandConfig(context.Background(), pipelinePath, "", campaignPath, sessionPath, config.SessionLoadOptions{SessionID: "2026-05-03"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("loadCommandConfig() error = %v", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = loaded.Close() }()
|
||||||
|
|
||||||
|
if loadCalls != 1 {
|
||||||
|
t.Fatalf("pipeline load calls = %d, want 1", loadCalls)
|
||||||
|
}
|
||||||
|
if got := loaded.Config.Pipeline.Workspace.Root; got != workspaceRoot {
|
||||||
|
t.Fatalf("resolved workspace root = %q, want originally loaded %q", got, workspaceRoot)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -79,7 +79,7 @@ func SessionInit(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("session init: %w", err)
|
return fmt.Errorf("session init: %w", err)
|
||||||
}
|
}
|
||||||
cfg, err := config.Resolve(base.PipelinePath, base.Pipeline, base.CampaignPath, base.Campaign, label, sessionCfg, config.SessionSource{Source: "session_config", LocalPath: label})
|
cfg, err := config.ResolveLoadedPipelineCampaign(*base, label, sessionCfg, config.SessionSource{Source: "session_config", LocalPath: label})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("session init: %w", err)
|
return fmt.Errorf("session init: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -166,6 +166,9 @@ inputs:
|
|||||||
if err := loaded.Close(); err != nil {
|
if err := loaded.Close(); err != nil {
|
||||||
t.Fatalf("second Close() error = %v", err)
|
t.Fatalf("second Close() error = %v", err)
|
||||||
}
|
}
|
||||||
|
if loaded.Config == nil || loaded.Config.Pipeline == nil || loaded.Config.Campaign == nil || loaded.Config.Session == nil {
|
||||||
|
t.Fatal("closing remote session cleanup discarded retained configuration")
|
||||||
|
}
|
||||||
if _, err := os.Stat(downloadedPath); !errors.Is(err, os.ErrNotExist) {
|
if _, err := os.Stat(downloadedPath); !errors.Is(err, os.ErrNotExist) {
|
||||||
t.Fatalf("downloaded remote session path still exists or could not be inspected: %q, err=%v", downloadedPath, err)
|
t.Fatalf("downloaded remote session path still exists or could not be inspected: %q, err=%v", downloadedPath, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -167,42 +167,81 @@ func LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath string, sess
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
return LoadSessionWithPipelineCampaignOptions(LoadedPipelineCampaign{
|
||||||
|
PipelinePath: pipelinePath,
|
||||||
|
Pipeline: pipelineCfg,
|
||||||
|
CampaignPath: campaignPath,
|
||||||
|
Campaign: campaignCfg,
|
||||||
|
}, sessionPath, sessionOpts)
|
||||||
|
}
|
||||||
|
|
||||||
|
// LoadedPipelineCampaign retains one already loaded pipeline and campaign for
|
||||||
|
// subsequent local or remote session resolution. It prevents a command from
|
||||||
|
// reloading the root pipeline after campaign selection.
|
||||||
|
type LoadedPipelineCampaign struct {
|
||||||
|
PipelinePath string
|
||||||
|
Pipeline *PipelineConfig
|
||||||
|
CampaignPath string
|
||||||
|
Campaign *CampaignConfig
|
||||||
|
}
|
||||||
|
|
||||||
|
// LoadSessionWithPipelineCampaignOptions loads one local session and combines
|
||||||
|
// it with an already loaded pipeline and campaign.
|
||||||
|
func LoadSessionWithPipelineCampaignOptions(loaded LoadedPipelineCampaign, sessionPath string, sessionOpts SessionLoadOptions) (*Config, error) {
|
||||||
sessionCfg, err := LoadSessionWithOptions(sessionPath, sessionOpts)
|
sessionCfg, err := LoadSessionWithOptions(sessionPath, sessionOpts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
return ResolveLoadedPipelineCampaign(loaded, sessionPath, sessionCfg, SessionSource{
|
||||||
return Resolve(pipelinePath, pipelineCfg, campaignPath, campaignCfg, sessionPath, sessionCfg, SessionSource{
|
|
||||||
Source: "session_config",
|
Source: "session_config",
|
||||||
LocalPath: sessionPath,
|
LocalPath: sessionPath,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// Resolve builds final stage-facing configuration from already loaded
|
// ResolveLoadedPipelineCampaign combines an already loaded pipeline and
|
||||||
// pipeline, campaign, and session documents.
|
// campaign with optional already loaded session data. A nil session preserves
|
||||||
func Resolve(pipelinePath string, pipelineCfg *PipelineConfig, campaignPath string, campaignCfg *CampaignConfig, sessionPath string, sessionCfg *SessionConfig, sessionSource SessionSource) (*Config, error) {
|
// the resolved pipeline/campaign context for callers that need to locate or
|
||||||
stableInputs, err := mergeCampaignSession(campaignCfg, sessionCfg, campaignPath, sessionPath)
|
// retrieve a session without rereading the root pipeline.
|
||||||
|
func ResolveLoadedPipelineCampaign(loaded LoadedPipelineCampaign, sessionPath string, sessionCfg *SessionConfig, sessionSource SessionSource) (*Config, error) {
|
||||||
|
cfg := &Config{
|
||||||
|
Pipeline: loaded.Pipeline,
|
||||||
|
Campaign: loaded.Campaign,
|
||||||
|
Session: sessionCfg,
|
||||||
|
PipelinePath: loaded.PipelinePath,
|
||||||
|
CampaignPath: loaded.CampaignPath,
|
||||||
|
SessionPath: sessionPath,
|
||||||
|
SessionSource: sessionSource,
|
||||||
|
}
|
||||||
|
if sessionCfg == nil {
|
||||||
|
return cfg, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
stableInputs, err := mergeCampaignSession(loaded.Campaign, sessionCfg, loaded.CampaignPath, sessionPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(sessionSource.Source) == "" {
|
if strings.TrimSpace(cfg.SessionSource.Source) == "" {
|
||||||
sessionSource.Source = "session_config"
|
cfg.SessionSource.Source = "session_config"
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(sessionSource.LocalPath) == "" {
|
if strings.TrimSpace(cfg.SessionSource.LocalPath) == "" {
|
||||||
sessionSource.LocalPath = sessionPath
|
cfg.SessionSource.LocalPath = sessionPath
|
||||||
}
|
}
|
||||||
|
cfg.StableInputs = stableInputs
|
||||||
|
return cfg, nil
|
||||||
|
}
|
||||||
|
|
||||||
return &Config{
|
// Resolve builds final stage-facing configuration from already loaded
|
||||||
Pipeline: pipelineCfg,
|
// pipeline, campaign, and session documents.
|
||||||
Campaign: campaignCfg,
|
func Resolve(pipelinePath string, pipelineCfg *PipelineConfig, campaignPath string, campaignCfg *CampaignConfig, sessionPath string, sessionCfg *SessionConfig, sessionSource SessionSource) (*Config, error) {
|
||||||
Session: sessionCfg,
|
if sessionCfg == nil {
|
||||||
|
return nil, fmt.Errorf("session config is required")
|
||||||
|
}
|
||||||
|
return ResolveLoadedPipelineCampaign(LoadedPipelineCampaign{
|
||||||
PipelinePath: pipelinePath,
|
PipelinePath: pipelinePath,
|
||||||
|
Pipeline: pipelineCfg,
|
||||||
CampaignPath: campaignPath,
|
CampaignPath: campaignPath,
|
||||||
SessionPath: sessionPath,
|
Campaign: campaignCfg,
|
||||||
StableInputs: stableInputs,
|
}, sessionPath, sessionCfg, sessionSource)
|
||||||
SessionSource: sessionSource,
|
|
||||||
}, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func campaignSessionPaths(paths ...string) (campaignPath, sessionPath string, err error) {
|
func campaignSessionPaths(paths ...string) (campaignPath, sessionPath string, err error) {
|
||||||
|
|||||||
Reference in New Issue
Block a user