fix(file)!: stream uploads through staged storage

- fix: replace multipart form parsing with streaming multipart reads and
  apply request body limits when max_upload_size is configured.
- refactor: route uploads through staging paths before promotion to
  long-term data paths, keeping incomplete uploads out of durable
  storage records.
- test: cover oversized uploads, unlimited uploads, staged cleanup, and
  local storage promotion boundaries.
- docs: document the staged upload model and multipart parent_id query
  parameter.
This commit is contained in:
2026-07-05 17:18:19 +08:00
parent bfeb4b26e4
commit b988b4b15e
10 changed files with 457 additions and 50 deletions
+76 -10
View File
@@ -17,15 +17,19 @@ func TestSaveAndOpen(t *testing.T) {
ctx := context.Background()
content := "hello, mygo storage"
written, err := store.Save(ctx, "test/hello.txt", strings.NewReader(content))
written, err := store.SaveStaged(ctx, "staging/test/hello.txt", strings.NewReader(content))
if err != nil {
t.Fatalf("Save: %v", err)
t.Fatalf("SaveStaged: %v", err)
}
if written != int64(len(content)) {
t.Errorf("written = %d, want %d", written, len(content))
}
reader, err := store.Open(ctx, "test/hello.txt")
if err := store.PromoteStaged(ctx, "staging/test/hello.txt", "data/test/hello.txt"); err != nil {
t.Fatalf("PromoteStaged: %v", err)
}
reader, err := store.Open(ctx, "data/test/hello.txt")
if err != nil {
t.Fatalf("Open: %v", err)
}
@@ -40,7 +44,7 @@ func TestSaveAndOpen(t *testing.T) {
}
// Verify the file exists on disk under the base path.
expectedPath := dir + "/test/hello.txt"
expectedPath := dir + "/data/test/hello.txt"
if _, err := os.Stat(expectedPath); err != nil {
t.Errorf("file not found on disk at %s: %v", expectedPath, err)
}
@@ -54,16 +58,19 @@ func TestDelete(t *testing.T) {
}
ctx := context.Background()
_, err = store.Save(ctx, "to_delete.txt", strings.NewReader("delete me"))
_, err = store.SaveStaged(ctx, "staging/to_delete.txt", strings.NewReader("delete me"))
if err != nil {
t.Fatalf("Save: %v", err)
t.Fatalf("SaveStaged: %v", err)
}
if err := store.PromoteStaged(ctx, "staging/to_delete.txt", "data/to_delete.txt"); err != nil {
t.Fatalf("PromoteStaged: %v", err)
}
if err := store.Delete(ctx, "to_delete.txt"); err != nil {
if err := store.Delete(ctx, "data/to_delete.txt"); err != nil {
t.Fatalf("Delete: %v", err)
}
_, err = store.Open(ctx, "to_delete.txt")
_, err = store.Open(ctx, "data/to_delete.txt")
if err == nil {
t.Error("Open should fail after delete")
}
@@ -91,9 +98,23 @@ func TestPathTraversalRejected(t *testing.T) {
}
ctx := context.Background()
_, err = store.Save(ctx, "../escape.txt", strings.NewReader("malicious"))
_, err = store.SaveStaged(ctx, "../escape.txt", strings.NewReader("malicious"))
if err == nil {
t.Error("Save should reject path traversal")
t.Error("SaveStaged should reject path traversal")
} else if !strings.Contains(err.Error(), "path traversal") {
t.Errorf("expected path traversal error, got: %v", err)
}
err = store.PromoteStaged(ctx, "../escape.txt", "data/escape.txt")
if err == nil {
t.Error("PromoteStaged should reject staged path traversal")
} else if !strings.Contains(err.Error(), "path traversal") {
t.Errorf("expected path traversal error, got: %v", err)
}
err = store.PromoteStaged(ctx, "staging/escape.txt", "../escape.txt")
if err == nil {
t.Error("PromoteStaged should reject final path traversal")
} else if !strings.Contains(err.Error(), "path traversal") {
t.Errorf("expected path traversal error, got: %v", err)
}
@@ -111,6 +132,51 @@ func TestPathTraversalRejected(t *testing.T) {
} else if !strings.Contains(err.Error(), "path traversal") {
t.Errorf("expected path traversal error, got: %v", err)
}
err = store.DeleteStaged(ctx, "../escape.txt")
if err == nil {
t.Error("DeleteStaged should reject path traversal")
} else if !strings.Contains(err.Error(), "path traversal") {
t.Errorf("expected path traversal error, got: %v", err)
}
}
func TestStagedPathsRequireStagingPrefix(t *testing.T) {
dir := t.TempDir()
store, err := NewLocalStorage(dir)
if err != nil {
t.Fatalf("NewLocalStorage: %v", err)
}
ctx := context.Background()
_, err = store.SaveStaged(ctx, "data/not-staged.txt", strings.NewReader("bad"))
if err == nil {
t.Fatal("SaveStaged should reject non-staging path")
}
err = store.DeleteStaged(ctx, "data/not-staged.txt")
if err == nil {
t.Fatal("DeleteStaged should reject non-staging path")
}
}
func TestPromoteRequiresDataFinalPrefix(t *testing.T) {
dir := t.TempDir()
store, err := NewLocalStorage(dir)
if err != nil {
t.Fatalf("NewLocalStorage: %v", err)
}
ctx := context.Background()
_, err = store.SaveStaged(ctx, "staging/promote.txt", strings.NewReader("content"))
if err != nil {
t.Fatalf("SaveStaged: %v", err)
}
err = store.PromoteStaged(ctx, "staging/promote.txt", "other/promote.txt")
if err == nil {
t.Fatal("PromoteStaged should reject non-data final path")
}
}
func TestOpenMissingFile(t *testing.T) {