fix(file)!: stream uploads through staged storage

- fix: replace multipart form parsing with streaming multipart reads and
  apply request body limits when max_upload_size is configured.
- refactor: route uploads through staging paths before promotion to
  long-term data paths, keeping incomplete uploads out of durable
  storage records.
- test: cover oversized uploads, unlimited uploads, staged cleanup, and
  local storage promotion boundaries.
- docs: document the staged upload model and multipart parent_id query
  parameter.
This commit is contained in:
2026-07-05 17:18:19 +08:00
parent bfeb4b26e4
commit b988b4b15e
10 changed files with 457 additions and 50 deletions
+86 -3
View File
@@ -8,6 +8,7 @@ import (
"mime/multipart"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/gin-gonic/gin"
@@ -29,7 +30,7 @@ func newInMemStore() *inMemStore {
return &inMemStore{files: make(map[string][]byte)}
}
func (s *inMemStore) Save(_ context.Context, path string, reader io.Reader) (int64, error) {
func (s *inMemStore) SaveStaged(_ context.Context, path string, reader io.Reader) (int64, error) {
data, err := io.ReadAll(reader)
if err != nil {
return 0, err
@@ -38,6 +39,16 @@ func (s *inMemStore) Save(_ context.Context, path string, reader io.Reader) (int
return int64(len(data)), nil
}
func (s *inMemStore) PromoteStaged(_ context.Context, stagedPath, finalPath string) error {
data, ok := s.files[stagedPath]
if !ok {
return io.ErrUnexpectedEOF
}
s.files[finalPath] = data
delete(s.files, stagedPath)
return nil
}
func (s *inMemStore) Open(_ context.Context, path string) (io.ReadCloser, error) {
data, ok := s.files[path]
if !ok {
@@ -46,6 +57,10 @@ func (s *inMemStore) Open(_ context.Context, path string) (io.ReadCloser, error)
return io.NopCloser(bytes.NewReader(data)), nil
}
func (s *inMemStore) DeleteStaged(ctx context.Context, path string) error {
return s.Delete(ctx, path)
}
func (s *inMemStore) Delete(_ context.Context, path string) error {
delete(s.files, path)
return nil
@@ -58,6 +73,10 @@ func (e *fsNotFoundError) Error() string { return "file not found on disk: " + e
var _ storage.StorageBackend = (*inMemStore)(nil)
func setupFileHandler(t *testing.T) *FileHandler {
return setupFileHandlerWithMaxUploadSize(t, 0)
}
func setupFileHandlerWithMaxUploadSize(t *testing.T, maxUploadSize int64) *FileHandler {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
@@ -70,15 +89,19 @@ func setupFileHandler(t *testing.T) *FileHandler {
fileRepo := repository.NewFileRepository(db)
store := newInMemStore()
fileService := service.NewFileService(fileRepo, store, 0, nil)
fileService := service.NewFileService(fileRepo, store, maxUploadSize, nil)
return NewFileHandler(fileService)
}
func setupFileRouter(t *testing.T) *gin.Engine {
return setupFileRouterWithMaxUploadSize(t, 0)
}
func setupFileRouterWithMaxUploadSize(t *testing.T, maxUploadSize int64) *gin.Engine {
t.Helper()
handler := setupFileHandler(t)
handler := setupFileHandlerWithMaxUploadSize(t, maxUploadSize)
gin.SetMode(gin.TestMode)
r := gin.New()
@@ -133,6 +156,66 @@ func TestFileHandler_Upload(t *testing.T) {
}
}
func TestFileHandler_UploadRejectsOversizedFile(t *testing.T) {
r := setupFileRouterWithMaxUploadSize(t, 5)
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
part, _ := writer.CreateFormFile("file", "too-large.txt")
part.Write([]byte("123456"))
writer.Close()
req := httptest.NewRequest(http.MethodPost, "/api/v1/files", body)
req.Header.Set("Content-Type", writer.FormDataContentType())
req.Header.Set("X-User-ID", "user1")
rec := httptest.NewRecorder()
r.ServeHTTP(rec, req)
if rec.Code != http.StatusRequestEntityTooLarge {
t.Errorf("status = %d, want %d; body = %s", rec.Code, http.StatusRequestEntityTooLarge, rec.Body.String())
}
}
func TestFileHandler_UploadAllowsExactMaxUploadSize(t *testing.T) {
r := setupFileRouterWithMaxUploadSize(t, 5)
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
part, _ := writer.CreateFormFile("file", "exact.txt")
part.Write([]byte("12345"))
writer.Close()
req := httptest.NewRequest(http.MethodPost, "/api/v1/files", body)
req.Header.Set("Content-Type", writer.FormDataContentType())
req.Header.Set("X-User-ID", "user1")
rec := httptest.NewRecorder()
r.ServeHTTP(rec, req)
if rec.Code != http.StatusCreated {
t.Errorf("status = %d, want %d; body = %s", rec.Code, http.StatusCreated, rec.Body.String())
}
}
func TestFileHandler_UploadUnlimitedWhenMaxUploadSizeIsZero(t *testing.T) {
r := setupFileRouter(t)
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
part, _ := writer.CreateFormFile("file", "unlimited.txt")
part.Write([]byte(strings.Repeat("a", 128)))
writer.Close()
req := httptest.NewRequest(http.MethodPost, "/api/v1/files", body)
req.Header.Set("Content-Type", writer.FormDataContentType())
req.Header.Set("X-User-ID", "user1")
rec := httptest.NewRecorder()
r.ServeHTTP(rec, req)
if rec.Code != http.StatusCreated {
t.Errorf("status = %d, want %d; body = %s", rec.Code, http.StatusCreated, rec.Body.String())
}
}
func TestFileHandler_UploadNoFile(t *testing.T) {
r := setupFileRouter(t)