fix(file)!: stream uploads through staged storage
- fix: replace multipart form parsing with streaming multipart reads and apply request body limits when max_upload_size is configured. - refactor: route uploads through staging paths before promotion to long-term data paths, keeping incomplete uploads out of durable storage records. - test: cover oversized uploads, unlimited uploads, staged cleanup, and local storage promotion boundaries. - docs: document the staged upload model and multipart parent_id query parameter.
This commit is contained in:
@@ -9,6 +9,11 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
dataPathPrefix = "data"
|
||||
stagingPathPrefix = "staging"
|
||||
)
|
||||
|
||||
// LocalStorage implements StorageBackend using the local filesystem.
|
||||
type LocalStorage struct {
|
||||
basePath string
|
||||
@@ -24,12 +29,15 @@ func NewLocalStorage(basePath string) (*LocalStorage, error) {
|
||||
return &LocalStorage{basePath: abs}, nil
|
||||
}
|
||||
|
||||
// Save writes the contents of reader to path under the storage root.
|
||||
func (s *LocalStorage) Save(_ context.Context, path string, reader io.Reader) (int64, error) {
|
||||
// SaveStaged writes the contents of reader to a staging path under the storage root.
|
||||
func (s *LocalStorage) SaveStaged(_ context.Context, path string, reader io.Reader) (int64, error) {
|
||||
fullPath := filepath.Join(s.basePath, path)
|
||||
if !isSubPath(s.basePath, fullPath) {
|
||||
return 0, fmt.Errorf("storage: path traversal attempt: %s", path)
|
||||
}
|
||||
if !hasPathPrefix(path, stagingPathPrefix) {
|
||||
return 0, fmt.Errorf("storage: staged path must be under %s/: %s", stagingPathPrefix, path)
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(fullPath), 0750); err != nil {
|
||||
return 0, fmt.Errorf("create parent directory: %w", err)
|
||||
@@ -51,6 +59,41 @@ func (s *LocalStorage) Save(_ context.Context, path string, reader io.Reader) (i
|
||||
return written, nil
|
||||
}
|
||||
|
||||
// PromoteStaged moves a staged file to its final path under the storage root.
|
||||
func (s *LocalStorage) PromoteStaged(_ context.Context, stagedPath, finalPath string) error {
|
||||
fullStagedPath := filepath.Join(s.basePath, stagedPath)
|
||||
if !isSubPath(s.basePath, fullStagedPath) {
|
||||
return fmt.Errorf("storage: path traversal attempt: %s", stagedPath)
|
||||
}
|
||||
if !hasPathPrefix(stagedPath, stagingPathPrefix) {
|
||||
return fmt.Errorf("storage: staged path must be under %s/: %s", stagingPathPrefix, stagedPath)
|
||||
}
|
||||
|
||||
fullFinalPath := filepath.Join(s.basePath, finalPath)
|
||||
if !isSubPath(s.basePath, fullFinalPath) {
|
||||
return fmt.Errorf("storage: path traversal attempt: %s", finalPath)
|
||||
}
|
||||
if !hasPathPrefix(finalPath, dataPathPrefix) {
|
||||
return fmt.Errorf("storage: final path must be under %s/: %s", dataPathPrefix, finalPath)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(fullFinalPath); err == nil {
|
||||
return fmt.Errorf("final file already exists: %s", finalPath)
|
||||
} else if !os.IsNotExist(err) {
|
||||
return fmt.Errorf("check final file: %w", err)
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(fullFinalPath), 0750); err != nil {
|
||||
return fmt.Errorf("create parent directory: %w", err)
|
||||
}
|
||||
|
||||
if err := os.Rename(fullStagedPath, fullFinalPath); err != nil {
|
||||
return fmt.Errorf("promote staged file: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Open returns a reader for the file at path under the storage root.
|
||||
func (s *LocalStorage) Open(_ context.Context, path string) (io.ReadCloser, error) {
|
||||
fullPath := filepath.Join(s.basePath, path)
|
||||
@@ -68,6 +111,23 @@ func (s *LocalStorage) Open(_ context.Context, path string) (io.ReadCloser, erro
|
||||
return file, nil
|
||||
}
|
||||
|
||||
// DeleteStaged removes a staged file under the storage root.
|
||||
func (s *LocalStorage) DeleteStaged(_ context.Context, path string) error {
|
||||
fullPath := filepath.Join(s.basePath, path)
|
||||
if !isSubPath(s.basePath, fullPath) {
|
||||
return fmt.Errorf("storage: path traversal attempt: %s", path)
|
||||
}
|
||||
if !hasPathPrefix(path, stagingPathPrefix) {
|
||||
return fmt.Errorf("storage: staged path must be under %s/: %s", stagingPathPrefix, path)
|
||||
}
|
||||
|
||||
err := os.Remove(fullPath)
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Delete removes the file at path under the storage root.
|
||||
func (s *LocalStorage) Delete(_ context.Context, path string) error {
|
||||
fullPath := filepath.Join(s.basePath, path)
|
||||
@@ -95,3 +155,8 @@ func isSubPath(base, target string) bool {
|
||||
// filepath.Rel returns ".." or starts with "../" for paths outside base.
|
||||
return rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator))
|
||||
}
|
||||
|
||||
func hasPathPrefix(path, prefix string) bool {
|
||||
clean := filepath.Clean(path)
|
||||
return clean == prefix || strings.HasPrefix(clean, prefix+string(filepath.Separator))
|
||||
}
|
||||
|
||||
@@ -17,15 +17,19 @@ func TestSaveAndOpen(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
content := "hello, mygo storage"
|
||||
written, err := store.Save(ctx, "test/hello.txt", strings.NewReader(content))
|
||||
written, err := store.SaveStaged(ctx, "staging/test/hello.txt", strings.NewReader(content))
|
||||
if err != nil {
|
||||
t.Fatalf("Save: %v", err)
|
||||
t.Fatalf("SaveStaged: %v", err)
|
||||
}
|
||||
if written != int64(len(content)) {
|
||||
t.Errorf("written = %d, want %d", written, len(content))
|
||||
}
|
||||
|
||||
reader, err := store.Open(ctx, "test/hello.txt")
|
||||
if err := store.PromoteStaged(ctx, "staging/test/hello.txt", "data/test/hello.txt"); err != nil {
|
||||
t.Fatalf("PromoteStaged: %v", err)
|
||||
}
|
||||
|
||||
reader, err := store.Open(ctx, "data/test/hello.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
@@ -40,7 +44,7 @@ func TestSaveAndOpen(t *testing.T) {
|
||||
}
|
||||
|
||||
// Verify the file exists on disk under the base path.
|
||||
expectedPath := dir + "/test/hello.txt"
|
||||
expectedPath := dir + "/data/test/hello.txt"
|
||||
if _, err := os.Stat(expectedPath); err != nil {
|
||||
t.Errorf("file not found on disk at %s: %v", expectedPath, err)
|
||||
}
|
||||
@@ -54,16 +58,19 @@ func TestDelete(t *testing.T) {
|
||||
}
|
||||
ctx := context.Background()
|
||||
|
||||
_, err = store.Save(ctx, "to_delete.txt", strings.NewReader("delete me"))
|
||||
_, err = store.SaveStaged(ctx, "staging/to_delete.txt", strings.NewReader("delete me"))
|
||||
if err != nil {
|
||||
t.Fatalf("Save: %v", err)
|
||||
t.Fatalf("SaveStaged: %v", err)
|
||||
}
|
||||
if err := store.PromoteStaged(ctx, "staging/to_delete.txt", "data/to_delete.txt"); err != nil {
|
||||
t.Fatalf("PromoteStaged: %v", err)
|
||||
}
|
||||
|
||||
if err := store.Delete(ctx, "to_delete.txt"); err != nil {
|
||||
if err := store.Delete(ctx, "data/to_delete.txt"); err != nil {
|
||||
t.Fatalf("Delete: %v", err)
|
||||
}
|
||||
|
||||
_, err = store.Open(ctx, "to_delete.txt")
|
||||
_, err = store.Open(ctx, "data/to_delete.txt")
|
||||
if err == nil {
|
||||
t.Error("Open should fail after delete")
|
||||
}
|
||||
@@ -91,9 +98,23 @@ func TestPathTraversalRejected(t *testing.T) {
|
||||
}
|
||||
ctx := context.Background()
|
||||
|
||||
_, err = store.Save(ctx, "../escape.txt", strings.NewReader("malicious"))
|
||||
_, err = store.SaveStaged(ctx, "../escape.txt", strings.NewReader("malicious"))
|
||||
if err == nil {
|
||||
t.Error("Save should reject path traversal")
|
||||
t.Error("SaveStaged should reject path traversal")
|
||||
} else if !strings.Contains(err.Error(), "path traversal") {
|
||||
t.Errorf("expected path traversal error, got: %v", err)
|
||||
}
|
||||
|
||||
err = store.PromoteStaged(ctx, "../escape.txt", "data/escape.txt")
|
||||
if err == nil {
|
||||
t.Error("PromoteStaged should reject staged path traversal")
|
||||
} else if !strings.Contains(err.Error(), "path traversal") {
|
||||
t.Errorf("expected path traversal error, got: %v", err)
|
||||
}
|
||||
|
||||
err = store.PromoteStaged(ctx, "staging/escape.txt", "../escape.txt")
|
||||
if err == nil {
|
||||
t.Error("PromoteStaged should reject final path traversal")
|
||||
} else if !strings.Contains(err.Error(), "path traversal") {
|
||||
t.Errorf("expected path traversal error, got: %v", err)
|
||||
}
|
||||
@@ -111,6 +132,51 @@ func TestPathTraversalRejected(t *testing.T) {
|
||||
} else if !strings.Contains(err.Error(), "path traversal") {
|
||||
t.Errorf("expected path traversal error, got: %v", err)
|
||||
}
|
||||
|
||||
err = store.DeleteStaged(ctx, "../escape.txt")
|
||||
if err == nil {
|
||||
t.Error("DeleteStaged should reject path traversal")
|
||||
} else if !strings.Contains(err.Error(), "path traversal") {
|
||||
t.Errorf("expected path traversal error, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagedPathsRequireStagingPrefix(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
store, err := NewLocalStorage(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("NewLocalStorage: %v", err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
|
||||
_, err = store.SaveStaged(ctx, "data/not-staged.txt", strings.NewReader("bad"))
|
||||
if err == nil {
|
||||
t.Fatal("SaveStaged should reject non-staging path")
|
||||
}
|
||||
|
||||
err = store.DeleteStaged(ctx, "data/not-staged.txt")
|
||||
if err == nil {
|
||||
t.Fatal("DeleteStaged should reject non-staging path")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromoteRequiresDataFinalPrefix(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
store, err := NewLocalStorage(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("NewLocalStorage: %v", err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
|
||||
_, err = store.SaveStaged(ctx, "staging/promote.txt", strings.NewReader("content"))
|
||||
if err != nil {
|
||||
t.Fatalf("SaveStaged: %v", err)
|
||||
}
|
||||
|
||||
err = store.PromoteStaged(ctx, "staging/promote.txt", "other/promote.txt")
|
||||
if err == nil {
|
||||
t.Fatal("PromoteStaged should reject non-data final path")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenMissingFile(t *testing.T) {
|
||||
|
||||
@@ -6,18 +6,24 @@ import (
|
||||
"io"
|
||||
)
|
||||
|
||||
// StorageBackend abstracts where file content is written, read, and deleted.
|
||||
// Paths passed to all methods are relative to the backend's root and must
|
||||
// not contain path traversal sequences.
|
||||
// StorageBackend abstracts where file content is staged, promoted, read, and
|
||||
// deleted. Paths passed to all methods are relative to the backend's root and
|
||||
// must not contain path traversal sequences.
|
||||
type StorageBackend interface {
|
||||
// Save writes the contents of reader to path, creating parent directories
|
||||
// as needed. It returns the number of bytes written.
|
||||
Save(ctx context.Context, path string, reader io.Reader) (int64, error)
|
||||
// SaveStaged writes the contents of reader to a staging path, creating
|
||||
// parent directories as needed. It returns the number of bytes written.
|
||||
SaveStaged(ctx context.Context, path string, reader io.Reader) (int64, error)
|
||||
|
||||
// PromoteStaged moves a staged object to its final path.
|
||||
PromoteStaged(ctx context.Context, stagedPath, finalPath string) error
|
||||
|
||||
// Open returns a reader for the file at path. The caller must close the
|
||||
// returned ReadCloser.
|
||||
Open(ctx context.Context, path string) (io.ReadCloser, error)
|
||||
|
||||
// DeleteStaged removes a staged object. It is idempotent.
|
||||
DeleteStaged(ctx context.Context, path string) error
|
||||
|
||||
// Delete removes the file at path. It is idempotent — deleting a
|
||||
// non-existent file is not an error.
|
||||
Delete(ctx context.Context, path string) error
|
||||
|
||||
Reference in New Issue
Block a user