Add configured source validation and inspection
This commit is contained in:
@@ -3,9 +3,15 @@ package app
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/testutil"
|
||||
)
|
||||
|
||||
func TestValidateLocalBundle(t *testing.T) {
|
||||
@@ -31,9 +37,182 @@ func TestValidateExampleSourceBundle(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateConfiguredLocalSource(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
|
||||
var stdout bytes.Buffer
|
||||
|
||||
err := Validate(context.Background(), ValidateOptions{
|
||||
ConfigPath: testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot),
|
||||
PipelineID: "reports",
|
||||
Stdout: &stdout,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("Validate() configured source error = %v", err)
|
||||
}
|
||||
if got, want := stdout.String(), "Validated 1 bundle(s) for pipeline reports source local\n"; got != want {
|
||||
t.Fatalf("stdout = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateConfiguredSourceBundlePath(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
testutil.WriteSourceBundle(t, sourceRoot, "daily/one", testutil.BundleOptions{ID: "reports.one"})
|
||||
testutil.WriteSourceBundle(t, sourceRoot, "daily/two", testutil.BundleOptions{ID: "reports.two"})
|
||||
var stdout bytes.Buffer
|
||||
|
||||
err := Validate(context.Background(), ValidateOptions{
|
||||
ConfigPath: testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot),
|
||||
PipelineID: "reports",
|
||||
BundlePath: "daily/two",
|
||||
Stdout: &stdout,
|
||||
OutputFormat: OutputFormatJSON,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("Validate() configured bundle error = %v", err)
|
||||
}
|
||||
result := decodeAppResult(t, stdout.String())
|
||||
if result["pipeline_id"] != "reports" || result["source_backend"] != "local" || result["bundle_count"] != float64(1) {
|
||||
t.Fatalf("result = %#v, want configured source summary", result)
|
||||
}
|
||||
bundles, ok := result["bundles"].([]any)
|
||||
if !ok || len(bundles) != 1 {
|
||||
t.Fatalf("bundles = %#v, want one bundle", result["bundles"])
|
||||
}
|
||||
sourceBundle, ok := bundles[0].(map[string]any)
|
||||
if !ok || sourceBundle["path"] != "daily/two" || sourceBundle["id"] != "reports.two" {
|
||||
t.Fatalf("bundle = %#v, want narrowed bundle", sourceBundle)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateConfiguredRemoteSourcesThroughStorageAbstraction(t *testing.T) {
|
||||
s3Source := fake.New()
|
||||
testutil.WriteFakeSourceBundle(t, s3Source, "", testutil.BundleOptions{ID: "reports.s3"})
|
||||
sshSource := fake.New()
|
||||
testutil.WriteFakeSourceBundle(t, sshSource, "daily", testutil.BundleOptions{ID: "reports.ssh"})
|
||||
cfg := config.Config{Pipelines: []config.Pipeline{
|
||||
{
|
||||
ID: "s3-reports",
|
||||
Source: config.Backend{
|
||||
Backend: config.BackendS3,
|
||||
Endpoint: "http://s3.test",
|
||||
Bucket: "source-bucket",
|
||||
},
|
||||
Destinations: []config.Destination{{
|
||||
ID: "archive",
|
||||
Backend: config.BackendLocal,
|
||||
Path: t.TempDir(),
|
||||
}},
|
||||
},
|
||||
{
|
||||
ID: "ssh-reports",
|
||||
Source: config.Backend{
|
||||
Backend: config.BackendSSH,
|
||||
Host: "ssh.test",
|
||||
Path: "/source",
|
||||
},
|
||||
Destinations: []config.Destination{{
|
||||
ID: "archive",
|
||||
Backend: config.BackendLocal,
|
||||
Path: t.TempDir(),
|
||||
}},
|
||||
},
|
||||
}}
|
||||
config.ApplyDefaults(&cfg)
|
||||
provider := fakeBackendFactoryProvider(t, map[string]storage.Backend{
|
||||
"s3:source-bucket": s3Source,
|
||||
"ssh:/source": sshSource,
|
||||
})
|
||||
|
||||
var s3Stdout bytes.Buffer
|
||||
if err := validateConfigWithBackendFactory(context.Background(), cfg, ValidateOptions{
|
||||
PipelineID: "s3-reports",
|
||||
Stdout: &s3Stdout,
|
||||
OutputFormat: OutputFormatJSON,
|
||||
}, provider); err != nil {
|
||||
t.Fatalf("validate s3 source error = %v", err)
|
||||
}
|
||||
s3Result := decodeAppResult(t, s3Stdout.String())
|
||||
if s3Result["source_backend"] != "s3" || s3Result["bundle_count"] != float64(1) {
|
||||
t.Fatalf("s3 result = %#v, want one s3 bundle", s3Result)
|
||||
}
|
||||
|
||||
var sshStdout bytes.Buffer
|
||||
if err := validateConfigWithBackendFactory(context.Background(), cfg, ValidateOptions{
|
||||
PipelineID: "ssh-reports",
|
||||
BundlePath: "daily",
|
||||
Stdout: &sshStdout,
|
||||
}, provider); err != nil {
|
||||
t.Fatalf("validate ssh source error = %v", err)
|
||||
}
|
||||
if !strings.Contains(sshStdout.String(), "pipeline ssh-reports source ssh") {
|
||||
t.Fatalf("ssh stdout = %q, want ssh source summary", sshStdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateConfiguredSourceLoadsSecretsBeforeOpeningBackend(t *testing.T) {
|
||||
sourceRoot := filepath.Join(t.TempDir(), "missing-source")
|
||||
destinationRoot := t.TempDir()
|
||||
configPath := writeConfigFile(t, `
|
||||
secrets:
|
||||
directory: `+filepath.Join(t.TempDir(), "missing-secrets")+`
|
||||
pipelines:
|
||||
- id: reports
|
||||
source:
|
||||
backend: local
|
||||
path: `+sourceRoot+`
|
||||
destinations:
|
||||
- id: archive
|
||||
backend: local
|
||||
path: `+destinationRoot+`
|
||||
`)
|
||||
|
||||
err := Validate(context.Background(), ValidateOptions{ConfigPath: configPath, PipelineID: "reports"})
|
||||
if err == nil {
|
||||
t.Fatal("Validate() error = nil, want secrets directory error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "load secrets directory") {
|
||||
t.Fatalf("Validate() error = %v, want secrets directory error", err)
|
||||
}
|
||||
if strings.Contains(err.Error(), "missing-source") {
|
||||
t.Fatalf("Validate() error = %v, opened source before loading secrets", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateConfiguredSourceRequiresPipeline(t *testing.T) {
|
||||
sourceRoot := t.TempDir()
|
||||
destinationRoot := t.TempDir()
|
||||
testutil.WriteSourceBundle(t, sourceRoot, "", testutil.BundleOptions{})
|
||||
|
||||
err := Validate(context.Background(), ValidateOptions{
|
||||
ConfigPath: testutil.WriteMinimalLocalConfig(t, sourceRoot, destinationRoot),
|
||||
})
|
||||
|
||||
if err == nil || !strings.Contains(err.Error(), "requires --pipeline") {
|
||||
t.Fatalf("Validate() error = %v, want required pipeline", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRequiresPath(t *testing.T) {
|
||||
err := Validate(context.Background(), ValidateOptions{})
|
||||
if err == nil || !strings.Contains(err.Error(), "requires a path") {
|
||||
t.Fatalf("Validate() error = %v, want required path", err)
|
||||
}
|
||||
}
|
||||
|
||||
func decodeAppResult(t *testing.T, output string) map[string]any {
|
||||
t.Helper()
|
||||
var envelope map[string]any
|
||||
if err := json.Unmarshal([]byte(output), &envelope); err != nil {
|
||||
t.Fatalf("decode output: %v; output = %q", err, output)
|
||||
}
|
||||
result, ok := envelope["result"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("result = %#v, want object", envelope["result"])
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user