Add artifact workdir layout and locking
This commit is contained in:
@@ -13,7 +13,8 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func TestExecuteValidCommands(t *testing.T) {
|
func TestExecuteValidCommands(t *testing.T) {
|
||||||
pipelinePath, sessionPath := writeValidConfigFiles(t)
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
manifestPath := writeManifestPathForExecute(t)
|
manifestPath := writeManifestPathForExecute(t)
|
||||||
|
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
@@ -22,7 +23,7 @@ func TestExecuteValidCommands(t *testing.T) {
|
|||||||
wantOut string
|
wantOut string
|
||||||
}{
|
}{
|
||||||
{name: "run", args: []string{"run", "--config", pipelinePath, "--session", sessionPath}, wantOut: "narratio run: configuration loaded and valid"},
|
{name: "run", args: []string{"run", "--config", pipelinePath, "--session", sessionPath}, wantOut: "narratio run: configuration loaded and valid"},
|
||||||
{name: "plan", args: []string{"plan", "--config", pipelinePath, "--session", sessionPath}, wantOut: "narratio plan: configuration loaded and valid"},
|
{name: "plan", args: []string{"plan", "--config", pipelinePath, "--session", sessionPath}, wantOut: "narratio plan: configuration loaded and valid; workdir prepared at"},
|
||||||
{name: "status", args: []string{"status", "--manifest", manifestPath}, wantOut: "session_id: 2026-05-03"},
|
{name: "status", args: []string{"status", "--manifest", manifestPath}, wantOut: "session_id: 2026-05-03"},
|
||||||
{name: "resume", args: []string{"resume"}, wantOut: "narratio resume: not yet implemented"},
|
{name: "resume", args: []string{"resume"}, wantOut: "narratio resume: not yet implemented"},
|
||||||
{name: "run-stage", args: []string{"run-stage", "polish"}, wantOut: "narratio run-stage: not yet implemented"},
|
{name: "run-stage", args: []string{"run-stage", "polish"}, wantOut: "narratio run-stage: not yet implemented"},
|
||||||
@@ -113,7 +114,7 @@ func TestExecuteMissingCommand(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeValidConfigFiles(t *testing.T) (string, string) {
|
func writeValidConfigFiles(t *testing.T, workspaceRoot string) (string, string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
@@ -121,7 +122,7 @@ func writeValidConfigFiles(t *testing.T) (string, string) {
|
|||||||
sessionPath := filepath.Join(dir, "session.yml")
|
sessionPath := filepath.Join(dir, "session.yml")
|
||||||
|
|
||||||
pipelineYAML := `workspace:
|
pipelineYAML := `workspace:
|
||||||
root: /tmp/narratio
|
root: ` + workspaceRoot + `
|
||||||
storage:
|
storage:
|
||||||
backend: s3
|
backend: s3
|
||||||
whisperx:
|
whisperx:
|
||||||
|
|||||||
@@ -6,10 +6,11 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Plan validates configuration inputs and reports readiness for future planning.
|
// Plan validates configuration inputs and prepares the local session workdir.
|
||||||
func Plan(_ context.Context, args []string, out io.Writer) error {
|
func Plan(_ context.Context, args []string, out io.Writer) error {
|
||||||
fs := flag.NewFlagSet("plan", flag.ContinueOnError)
|
fs := flag.NewFlagSet("plan", flag.ContinueOnError)
|
||||||
fs.SetOutput(io.Discard)
|
fs.SetOutput(io.Discard)
|
||||||
@@ -37,6 +38,12 @@ func Plan(_ context.Context, args []string, out io.Writer) error {
|
|||||||
return fmt.Errorf("plan: %w", err)
|
return fmt.Errorf("plan: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = fmt.Fprintln(out, "narratio plan: configuration loaded and valid")
|
store := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root)
|
||||||
|
paths, err := store.EnsureLayout(cfg.Session.SessionID)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("plan: prepare workdir: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = fmt.Fprintf(out, "narratio plan: configuration loaded and valid; workdir prepared at %s\n", paths.Root)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
61
internal/app/plan_test.go
Normal file
61
internal/app/plan_test.go
Normal file
@@ -0,0 +1,61 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPlanCreatesAndReusesWorkdir(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
args := []string{"--config", pipelinePath, "--session", sessionPath}
|
||||||
|
|
||||||
|
if err := Plan(context.Background(), args, &out); err != nil {
|
||||||
|
t.Fatalf("first Plan() error = %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out.String(), "workdir prepared") {
|
||||||
|
t.Fatalf("first output = %q, want workdir prepared", out.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
sessionWorkdir := artifacts.SessionWorkDir(workspaceRoot, "2026-05-03")
|
||||||
|
expectedDirs := []string{
|
||||||
|
sessionWorkdir,
|
||||||
|
filepath.Join(sessionWorkdir, "inputs"),
|
||||||
|
filepath.Join(sessionWorkdir, "audio"),
|
||||||
|
filepath.Join(sessionWorkdir, "transcripts", "raw"),
|
||||||
|
filepath.Join(sessionWorkdir, "transcripts", "normalized"),
|
||||||
|
filepath.Join(sessionWorkdir, "artifacts"),
|
||||||
|
filepath.Join(sessionWorkdir, "config"),
|
||||||
|
filepath.Join(sessionWorkdir, "logs"),
|
||||||
|
}
|
||||||
|
for _, dir := range expectedDirs {
|
||||||
|
assertDir(t, dir)
|
||||||
|
}
|
||||||
|
|
||||||
|
out.Reset()
|
||||||
|
if err := Plan(context.Background(), args, &out); err != nil {
|
||||||
|
t.Fatalf("second Plan() error = %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out.String(), "workdir prepared") {
|
||||||
|
t.Fatalf("second output = %q, want workdir prepared", out.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertDir(t *testing.T, path string) {
|
||||||
|
t.Helper()
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Stat(%q) error = %v", path, err)
|
||||||
|
}
|
||||||
|
if !info.IsDir() {
|
||||||
|
t.Fatalf("%q is not a directory", path)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,31 +1,291 @@
|
|||||||
package artifacts
|
package artifacts
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
// LocalStore is a placeholder local filesystem artifact store.
|
// ErrLockConflict is returned when a session lock already exists.
|
||||||
|
var ErrLockConflict = errors.New("session workdir is already locked")
|
||||||
|
|
||||||
|
// LockHandle tracks a held lock for a session work directory.
|
||||||
|
type LockHandle struct {
|
||||||
|
path string
|
||||||
|
file *os.File
|
||||||
|
}
|
||||||
|
|
||||||
|
// LocalStore stores artifacts and session working state on the local filesystem.
|
||||||
type LocalStore struct {
|
type LocalStore struct {
|
||||||
RootDir string
|
WorkspaceRoot string
|
||||||
}
|
}
|
||||||
|
|
||||||
// WriteLocal returns a not-yet-implemented error in the scaffold.
|
// NewLocalStore constructs a local store rooted at workspaceRoot.
|
||||||
func (s *LocalStore) WriteLocal(_ context.Context, _ Ref, _ []byte) error {
|
func NewLocalStore(workspaceRoot string) *LocalStore {
|
||||||
return fmt.Errorf("artifacts local write: not yet implemented")
|
return &LocalStore{WorkspaceRoot: workspaceRoot}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ReadLocal returns a not-yet-implemented error in the scaffold.
|
// SessionPaths resolves canonical paths for a session workdir.
|
||||||
func (s *LocalStore) ReadLocal(_ context.Context, _ Ref) ([]byte, error) {
|
func (s *LocalStore) SessionPaths(sessionID string) SessionPaths {
|
||||||
return nil, fmt.Errorf("artifacts local read: not yet implemented")
|
return buildSessionPaths(s.WorkspaceRoot, sessionID)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExistsLocal returns a not-yet-implemented error in the scaffold.
|
// EnsureLayout creates and verifies the canonical session workdir directory layout.
|
||||||
func (s *LocalStore) ExistsLocal(_ context.Context, _ Ref) (bool, error) {
|
func (s *LocalStore) EnsureLayout(sessionID string) (SessionPaths, error) {
|
||||||
return false, fmt.Errorf("artifacts local exists: not yet implemented")
|
if strings.TrimSpace(s.WorkspaceRoot) == "" {
|
||||||
|
return SessionPaths{}, fmt.Errorf("workspace root is required")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(sessionID) == "" {
|
||||||
|
return SessionPaths{}, fmt.Errorf("sessionID is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
paths := s.SessionPaths(sessionID)
|
||||||
|
dirs := []string{
|
||||||
|
paths.Root,
|
||||||
|
paths.InputsDir,
|
||||||
|
paths.AudioDir,
|
||||||
|
paths.TranscriptsDir,
|
||||||
|
paths.TranscriptsRawDir,
|
||||||
|
paths.TranscriptsNormalizedDir,
|
||||||
|
paths.ArtifactsDir,
|
||||||
|
paths.ConfigDir,
|
||||||
|
paths.LogsDir,
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, dir := range dirs {
|
||||||
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||||
|
return SessionPaths{}, fmt.Errorf("ensure layout: create %q: %w", dir, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return paths, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Upload returns a not-yet-implemented error in the scaffold.
|
// CopyInput copies an input file into the session workdir under destRelativePath.
|
||||||
func (s *LocalStore) Upload(_ context.Context, _ Ref) (string, error) {
|
func (s *LocalStore) CopyInput(sessionID, srcPath, destRelativePath string) (Ref, error) {
|
||||||
return "", fmt.Errorf("artifacts local upload: not yet implemented")
|
paths, err := s.EnsureLayout(sessionID)
|
||||||
|
if err != nil {
|
||||||
|
return Ref{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
destAbs, err := resolveInRoot(paths.Root, destRelativePath)
|
||||||
|
if err != nil {
|
||||||
|
return Ref{}, fmt.Errorf("copy input: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := copyFileAtomic(srcPath, destAbs, 0o644); err != nil {
|
||||||
|
return Ref{}, fmt.Errorf("copy input %q -> %q: %w", srcPath, destAbs, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
checksum, err := s.Checksum(destAbs)
|
||||||
|
if err != nil {
|
||||||
|
return Ref{}, fmt.Errorf("copy input checksum %q: %w", destAbs, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return Ref{
|
||||||
|
Kind: "input",
|
||||||
|
Category: "inputs",
|
||||||
|
SessionID: sessionID,
|
||||||
|
RelativePath: filepath.Clean(destRelativePath),
|
||||||
|
AbsolutePath: destAbs,
|
||||||
|
Checksum: checksum,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Exists reports whether path exists.
|
||||||
|
func (s *LocalStore) Exists(path string) (bool, error) {
|
||||||
|
_, err := os.Stat(path)
|
||||||
|
if err == nil {
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
return false, fmt.Errorf("exists %q: %w", path, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExistsRef reports whether ref.AbsolutePath exists.
|
||||||
|
func (s *LocalStore) ExistsRef(ref Ref) (bool, error) {
|
||||||
|
if strings.TrimSpace(ref.AbsolutePath) == "" {
|
||||||
|
return false, fmt.Errorf("exists ref: absolute path is required")
|
||||||
|
}
|
||||||
|
return s.Exists(ref.AbsolutePath)
|
||||||
|
}
|
||||||
|
|
||||||
|
// WriteFileAtomic writes a file via temp-file + rename within the same directory.
|
||||||
|
func (s *LocalStore) WriteFileAtomic(path string, data []byte, perm os.FileMode) error {
|
||||||
|
if strings.TrimSpace(path) == "" {
|
||||||
|
return fmt.Errorf("write file atomic: path is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
dir := filepath.Dir(path)
|
||||||
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||||
|
return fmt.Errorf("write file atomic: create parent dir %q: %w", dir, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
base := filepath.Base(path)
|
||||||
|
tmp, err := os.CreateTemp(dir, "."+base+".tmp-*")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("write file atomic: create temp file: %w", err)
|
||||||
|
}
|
||||||
|
tmpName := tmp.Name()
|
||||||
|
removeTmp := true
|
||||||
|
defer func() {
|
||||||
|
if removeTmp {
|
||||||
|
_ = os.Remove(tmpName)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
if _, err := tmp.Write(data); err != nil {
|
||||||
|
_ = tmp.Close()
|
||||||
|
return fmt.Errorf("write file atomic: write temp file: %w", err)
|
||||||
|
}
|
||||||
|
if err := tmp.Sync(); err != nil {
|
||||||
|
_ = tmp.Close()
|
||||||
|
return fmt.Errorf("write file atomic: sync temp file: %w", err)
|
||||||
|
}
|
||||||
|
if err := tmp.Close(); err != nil {
|
||||||
|
return fmt.Errorf("write file atomic: close temp file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.Chmod(tmpName, perm); err != nil {
|
||||||
|
return fmt.Errorf("write file atomic: chmod temp file: %w", err)
|
||||||
|
}
|
||||||
|
if err := os.Rename(tmpName, path); err != nil {
|
||||||
|
return fmt.Errorf("write file atomic: rename temp file: %w", err)
|
||||||
|
}
|
||||||
|
removeTmp = false
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Checksum computes SHA-256 for a file path.
|
||||||
|
func (s *LocalStore) Checksum(path string) (string, error) {
|
||||||
|
digest, err := SHA256File(path)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("checksum %q: %w", path, err)
|
||||||
|
}
|
||||||
|
return digest, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// AcquireSessionLock acquires an exclusive lock file for a session workdir.
|
||||||
|
func (s *LocalStore) AcquireSessionLock(sessionID string) (*LockHandle, error) {
|
||||||
|
paths, err := s.EnsureLayout(sessionID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
f, err := os.OpenFile(paths.LockPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o644)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, os.ErrExist) {
|
||||||
|
return nil, fmt.Errorf("%w: %s", ErrLockConflict, paths.LockPath)
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("acquire lock %q: %w", paths.LockPath, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
metadata := "pid=" + strconv.Itoa(os.Getpid()) + "\nacquired_at=" + time.Now().UTC().Format(time.RFC3339Nano) + "\n"
|
||||||
|
if _, err := io.WriteString(f, metadata); err != nil {
|
||||||
|
_ = f.Close()
|
||||||
|
_ = os.Remove(paths.LockPath)
|
||||||
|
return nil, fmt.Errorf("acquire lock %q: write metadata: %w", paths.LockPath, err)
|
||||||
|
}
|
||||||
|
if err := f.Sync(); err != nil {
|
||||||
|
_ = f.Close()
|
||||||
|
_ = os.Remove(paths.LockPath)
|
||||||
|
return nil, fmt.Errorf("acquire lock %q: sync: %w", paths.LockPath, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &LockHandle{path: paths.LockPath, file: f}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReleaseSessionLock releases a previously acquired session lock.
|
||||||
|
func (s *LocalStore) ReleaseSessionLock(lock *LockHandle) error {
|
||||||
|
if lock == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var closeErr error
|
||||||
|
if lock.file != nil {
|
||||||
|
closeErr = lock.file.Close()
|
||||||
|
lock.file = nil
|
||||||
|
}
|
||||||
|
|
||||||
|
removeErr := os.Remove(lock.path)
|
||||||
|
if errors.Is(removeErr, os.ErrNotExist) {
|
||||||
|
removeErr = nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if closeErr != nil {
|
||||||
|
return fmt.Errorf("release lock %q: close: %w", lock.path, closeErr)
|
||||||
|
}
|
||||||
|
if removeErr != nil {
|
||||||
|
return fmt.Errorf("release lock %q: remove: %w", lock.path, removeErr)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveInRoot(root, relative string) (string, error) {
|
||||||
|
rel := filepath.Clean(relative)
|
||||||
|
if rel == "." || rel == "" {
|
||||||
|
return "", fmt.Errorf("relative destination path is required")
|
||||||
|
}
|
||||||
|
if filepath.IsAbs(rel) {
|
||||||
|
return "", fmt.Errorf("relative destination must not be absolute: %q", relative)
|
||||||
|
}
|
||||||
|
if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
|
||||||
|
return "", fmt.Errorf("relative destination escapes root: %q", relative)
|
||||||
|
}
|
||||||
|
return filepath.Join(root, rel), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyFileAtomic(srcPath, dstPath string, perm os.FileMode) error {
|
||||||
|
src, err := os.Open(srcPath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer src.Close()
|
||||||
|
|
||||||
|
if err := os.MkdirAll(filepath.Dir(dstPath), 0o755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
dir := filepath.Dir(dstPath)
|
||||||
|
base := filepath.Base(dstPath)
|
||||||
|
tmp, err := os.CreateTemp(dir, "."+base+".tmp-*")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
tmpName := tmp.Name()
|
||||||
|
removeTmp := true
|
||||||
|
defer func() {
|
||||||
|
if removeTmp {
|
||||||
|
_ = os.Remove(tmpName)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
if _, err := io.Copy(tmp, src); err != nil {
|
||||||
|
_ = tmp.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := tmp.Sync(); err != nil {
|
||||||
|
_ = tmp.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := tmp.Close(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := os.Chmod(tmpName, perm); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := os.Rename(tmpName, dstPath); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
removeTmp = false
|
||||||
|
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
174
internal/artifacts/local_test.go
Normal file
174
internal/artifacts/local_test.go
Normal file
@@ -0,0 +1,174 @@
|
|||||||
|
package artifacts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestEnsureLayoutCreatesExpectedDirectories(t *testing.T) {
|
||||||
|
store := NewLocalStore(t.TempDir())
|
||||||
|
paths, err := store.EnsureLayout("session-1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("EnsureLayout() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
checkDirExists(t, paths.Root)
|
||||||
|
checkDirExists(t, paths.InputsDir)
|
||||||
|
checkDirExists(t, paths.AudioDir)
|
||||||
|
checkDirExists(t, paths.TranscriptsDir)
|
||||||
|
checkDirExists(t, paths.TranscriptsRawDir)
|
||||||
|
checkDirExists(t, paths.TranscriptsNormalizedDir)
|
||||||
|
checkDirExists(t, paths.ArtifactsDir)
|
||||||
|
checkDirExists(t, paths.ConfigDir)
|
||||||
|
checkDirExists(t, paths.LogsDir)
|
||||||
|
|
||||||
|
if filepath.Base(paths.ManifestPath) != "manifest.json" {
|
||||||
|
t.Fatalf("ManifestPath = %q, want basename manifest.json", paths.ManifestPath)
|
||||||
|
}
|
||||||
|
if filepath.Base(paths.LockPath) != ".lock" {
|
||||||
|
t.Fatalf("LockPath = %q, want basename .lock", paths.LockPath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChecksumCalculation(t *testing.T) {
|
||||||
|
store := NewLocalStore(t.TempDir())
|
||||||
|
path := filepath.Join(t.TempDir(), "sample.txt")
|
||||||
|
if err := os.WriteFile(path, []byte("hello"), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
checksum, err := store.Checksum(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Checksum() error = %v", err)
|
||||||
|
}
|
||||||
|
const want = "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824"
|
||||||
|
if checksum != want {
|
||||||
|
t.Fatalf("checksum = %q, want %q", checksum, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLockAcquireRelease(t *testing.T) {
|
||||||
|
store := NewLocalStore(t.TempDir())
|
||||||
|
|
||||||
|
lock, err := store.AcquireSessionLock("session-1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("AcquireSessionLock() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
exists, err := store.Exists(lock.path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Exists() error = %v", err)
|
||||||
|
}
|
||||||
|
if !exists {
|
||||||
|
t.Fatalf("expected lock file %q to exist", lock.path)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := store.ReleaseSessionLock(lock); err != nil {
|
||||||
|
t.Fatalf("ReleaseSessionLock() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
exists, err = store.Exists(lock.path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Exists() error = %v", err)
|
||||||
|
}
|
||||||
|
if exists {
|
||||||
|
t.Fatalf("expected lock file %q to be removed", lock.path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLockConflict(t *testing.T) {
|
||||||
|
store := NewLocalStore(t.TempDir())
|
||||||
|
lock1, err := store.AcquireSessionLock("session-1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("first AcquireSessionLock() error = %v", err)
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
_ = store.ReleaseSessionLock(lock1)
|
||||||
|
}()
|
||||||
|
|
||||||
|
_, err = store.AcquireSessionLock("session-1")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected lock conflict error, got nil")
|
||||||
|
}
|
||||||
|
if !errors.Is(err, ErrLockConflict) {
|
||||||
|
t.Fatalf("error = %v, want ErrLockConflict", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWriteFileAtomicAndOverwrite(t *testing.T) {
|
||||||
|
store := NewLocalStore(t.TempDir())
|
||||||
|
path := filepath.Join(t.TempDir(), "out", "value.txt")
|
||||||
|
|
||||||
|
if err := store.WriteFileAtomic(path, []byte("one"), 0o644); err != nil {
|
||||||
|
t.Fatalf("first WriteFileAtomic() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := store.WriteFileAtomic(path, []byte("two"), 0o644); err != nil {
|
||||||
|
t.Fatalf("second WriteFileAtomic() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadFile() error = %v", err)
|
||||||
|
}
|
||||||
|
if string(data) != "two" {
|
||||||
|
t.Fatalf("file content = %q, want %q", string(data), "two")
|
||||||
|
}
|
||||||
|
|
||||||
|
entries, err := os.ReadDir(filepath.Dir(path))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadDir() error = %v", err)
|
||||||
|
}
|
||||||
|
for _, e := range entries {
|
||||||
|
if strings.Contains(e.Name(), ".tmp-") {
|
||||||
|
t.Fatalf("unexpected temp file residue: %s", e.Name())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCopyInput(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
store := NewLocalStore(root)
|
||||||
|
|
||||||
|
srcPath := filepath.Join(t.TempDir(), "speakers.yml")
|
||||||
|
content := "alice: alice.flac\n"
|
||||||
|
if err := os.WriteFile(srcPath, []byte(content), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ref, err := store.CopyInput("session-1", srcPath, "inputs/speakers.yml")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("CopyInput() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if ref.Kind != "input" {
|
||||||
|
t.Fatalf("ref.Kind = %q, want %q", ref.Kind, "input")
|
||||||
|
}
|
||||||
|
if ref.RelativePath != filepath.Clean("inputs/speakers.yml") {
|
||||||
|
t.Fatalf("ref.RelativePath = %q, want %q", ref.RelativePath, filepath.Clean("inputs/speakers.yml"))
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(ref.AbsolutePath, root) {
|
||||||
|
t.Fatalf("ref.AbsolutePath = %q, want under workspace root %q", ref.AbsolutePath, root)
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := os.ReadFile(ref.AbsolutePath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadFile() error = %v", err)
|
||||||
|
}
|
||||||
|
if string(data) != content {
|
||||||
|
t.Fatalf("copied content = %q, want %q", string(data), content)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func checkDirExists(t *testing.T, path string) {
|
||||||
|
t.Helper()
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Stat(%q) error = %v", path, err)
|
||||||
|
}
|
||||||
|
if !info.IsDir() {
|
||||||
|
t.Fatalf("%q is not a directory", path)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2,7 +2,40 @@ package artifacts
|
|||||||
|
|
||||||
import "path/filepath"
|
import "path/filepath"
|
||||||
|
|
||||||
|
// SessionPaths contains canonical local paths for one session work directory.
|
||||||
|
type SessionPaths struct {
|
||||||
|
Root string
|
||||||
|
InputsDir string
|
||||||
|
AudioDir string
|
||||||
|
TranscriptsDir string
|
||||||
|
TranscriptsRawDir string
|
||||||
|
TranscriptsNormalizedDir string
|
||||||
|
ArtifactsDir string
|
||||||
|
ConfigDir string
|
||||||
|
LogsDir string
|
||||||
|
ManifestPath string
|
||||||
|
LockPath string
|
||||||
|
}
|
||||||
|
|
||||||
// SessionWorkDir returns the work directory for one session.
|
// SessionWorkDir returns the work directory for one session.
|
||||||
func SessionWorkDir(rootDir, sessionID string) string {
|
func SessionWorkDir(rootDir, sessionID string) string {
|
||||||
return filepath.Join(rootDir, "work", sessionID)
|
return filepath.Join(rootDir, "work", sessionID)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func buildSessionPaths(workspaceRoot, sessionID string) SessionPaths {
|
||||||
|
root := SessionWorkDir(workspaceRoot, sessionID)
|
||||||
|
transcripts := filepath.Join(root, "transcripts")
|
||||||
|
return SessionPaths{
|
||||||
|
Root: root,
|
||||||
|
InputsDir: filepath.Join(root, "inputs"),
|
||||||
|
AudioDir: filepath.Join(root, "audio"),
|
||||||
|
TranscriptsDir: transcripts,
|
||||||
|
TranscriptsRawDir: filepath.Join(transcripts, "raw"),
|
||||||
|
TranscriptsNormalizedDir: filepath.Join(transcripts, "normalized"),
|
||||||
|
ArtifactsDir: filepath.Join(root, "artifacts"),
|
||||||
|
ConfigDir: filepath.Join(root, "config"),
|
||||||
|
LogsDir: filepath.Join(root, "logs"),
|
||||||
|
ManifestPath: filepath.Join(root, "manifest.json"),
|
||||||
|
LockPath: filepath.Join(root, ".lock"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,32 +1,57 @@
|
|||||||
package artifacts
|
package artifacts
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"os"
|
||||||
)
|
)
|
||||||
|
|
||||||
// S3Store is a placeholder S3-compatible artifact store.
|
// S3Store is a placeholder for future remote artifact persistence support.
|
||||||
type S3Store struct {
|
type S3Store struct {
|
||||||
Bucket string
|
Bucket string
|
||||||
Prefix string
|
Prefix string
|
||||||
}
|
}
|
||||||
|
|
||||||
// WriteLocal returns a not-yet-implemented error in the scaffold.
|
// SessionPaths is not implemented for S3-backed storage.
|
||||||
func (s *S3Store) WriteLocal(_ context.Context, _ Ref, _ []byte) error {
|
func (s *S3Store) SessionPaths(_ string) SessionPaths {
|
||||||
return fmt.Errorf("artifacts s3 write local: not yet implemented")
|
return SessionPaths{}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ReadLocal returns a not-yet-implemented error in the scaffold.
|
// EnsureLayout returns a not-yet-implemented error in the scaffold.
|
||||||
func (s *S3Store) ReadLocal(_ context.Context, _ Ref) ([]byte, error) {
|
func (s *S3Store) EnsureLayout(_ string) (SessionPaths, error) {
|
||||||
return nil, fmt.Errorf("artifacts s3 read local: not yet implemented")
|
return SessionPaths{}, fmt.Errorf("artifacts s3 ensure layout: not yet implemented")
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExistsLocal returns a not-yet-implemented error in the scaffold.
|
// CopyInput returns a not-yet-implemented error in the scaffold.
|
||||||
func (s *S3Store) ExistsLocal(_ context.Context, _ Ref) (bool, error) {
|
func (s *S3Store) CopyInput(_, _, _ string) (Ref, error) {
|
||||||
return false, fmt.Errorf("artifacts s3 exists local: not yet implemented")
|
return Ref{}, fmt.Errorf("artifacts s3 copy input: not yet implemented")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Upload returns a not-yet-implemented error in the scaffold.
|
// Exists returns a not-yet-implemented error in the scaffold.
|
||||||
func (s *S3Store) Upload(_ context.Context, _ Ref) (string, error) {
|
func (s *S3Store) Exists(_ string) (bool, error) {
|
||||||
return "", fmt.Errorf("artifacts s3 upload: not yet implemented")
|
return false, fmt.Errorf("artifacts s3 exists: not yet implemented")
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExistsRef returns a not-yet-implemented error in the scaffold.
|
||||||
|
func (s *S3Store) ExistsRef(_ Ref) (bool, error) {
|
||||||
|
return false, fmt.Errorf("artifacts s3 exists ref: not yet implemented")
|
||||||
|
}
|
||||||
|
|
||||||
|
// WriteFileAtomic returns a not-yet-implemented error in the scaffold.
|
||||||
|
func (s *S3Store) WriteFileAtomic(_ string, _ []byte, _ os.FileMode) error {
|
||||||
|
return fmt.Errorf("artifacts s3 write file atomic: not yet implemented")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Checksum returns a not-yet-implemented error in the scaffold.
|
||||||
|
func (s *S3Store) Checksum(_ string) (string, error) {
|
||||||
|
return "", fmt.Errorf("artifacts s3 checksum: not yet implemented")
|
||||||
|
}
|
||||||
|
|
||||||
|
// AcquireSessionLock returns a not-yet-implemented error in the scaffold.
|
||||||
|
func (s *S3Store) AcquireSessionLock(_ string) (*LockHandle, error) {
|
||||||
|
return nil, fmt.Errorf("artifacts s3 acquire lock: not yet implemented")
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReleaseSessionLock returns a not-yet-implemented error in the scaffold.
|
||||||
|
func (s *S3Store) ReleaseSessionLock(_ *LockHandle) error {
|
||||||
|
return fmt.Errorf("artifacts s3 release lock: not yet implemented")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,19 +1,27 @@
|
|||||||
package artifacts
|
package artifacts
|
||||||
|
|
||||||
import "context"
|
import "os"
|
||||||
|
|
||||||
// Ref identifies a pipeline artifact and optional remote location.
|
// Ref identifies a pipeline artifact and its local/remote coordinates.
|
||||||
type Ref struct {
|
type Ref struct {
|
||||||
Kind string
|
Kind string
|
||||||
LocalPath string
|
Category string
|
||||||
RemoteKey string
|
SessionID string
|
||||||
Checksum string
|
RelativePath string
|
||||||
|
AbsolutePath string
|
||||||
|
RemoteKey string
|
||||||
|
Checksum string
|
||||||
}
|
}
|
||||||
|
|
||||||
// Store is a placeholder artifact storage abstraction.
|
// Store is the local artifact/workdir abstraction used by orchestration code.
|
||||||
type Store interface {
|
type Store interface {
|
||||||
WriteLocal(ctx context.Context, ref Ref, data []byte) error
|
SessionPaths(sessionID string) SessionPaths
|
||||||
ReadLocal(ctx context.Context, ref Ref) ([]byte, error)
|
EnsureLayout(sessionID string) (SessionPaths, error)
|
||||||
ExistsLocal(ctx context.Context, ref Ref) (bool, error)
|
CopyInput(sessionID, srcPath, destRelativePath string) (Ref, error)
|
||||||
Upload(ctx context.Context, ref Ref) (string, error)
|
Exists(path string) (bool, error)
|
||||||
|
ExistsRef(ref Ref) (bool, error)
|
||||||
|
WriteFileAtomic(path string, data []byte, perm os.FileMode) error
|
||||||
|
Checksum(path string) (string, error)
|
||||||
|
AcquireSessionLock(sessionID string) (*LockHandle, error)
|
||||||
|
ReleaseSessionLock(lock *LockHandle) error
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user