MyGO/mygo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(file)!: stream uploads through staged storage

Browse Source 
- fix: replace multipart form parsing with streaming multipart reads and
  apply request body limits when max_upload_size is configured.
- refactor: route uploads through staging paths before promotion to
  long-term data paths, keeping incomplete uploads out of durable
  storage records.
- test: cover oversized uploads, unlimited uploads, staged cleanup, and
  local storage promotion boundaries.
- docs: document the staged upload model and multipart parent_id query
  parameter.
This commit is contained in:
Sunday
2026-07-05 17:18:19 +08:00
parent bfeb4b26e4
commit b988b4b15e
10 changed files with 457 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/handler/file.go
+65 -12
View File
@@ -1,10 +1,12 @@
package handler
import (
"errors"
"fmt"
"io"
"log/slog"
"mime"
"mime/multipart"
"net/http"
"net/url"
"strconv"
@@ -13,10 +15,15 @@ import (
"github.com/dhao2001/mygo/internal/api"
"github.com/dhao2001/mygo/internal/middleware"
"github.com/dhao2001/mygo/internal/model"
"github.com/dhao2001/mygo/internal/service"
)
const (
jsonUploadBodyLimit = 64 << 10
multipartUploadOverhead = 1 << 20
multipartFileField = "file"
)
// FileHandler handles file management endpoints.
type FileHandler struct {
fileService *service.FileService
@@ -47,9 +54,15 @@ func (h *FileHandler) Upload(c *gin.Context) {
// Directory creation via JSON.
mediaType, _, _ := mime.ParseMediaType(contentType)
if mediaType == "application/json" {
c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, jsonUploadBodyLimit)
var req createDirRequest
if err := c.ShouldBindJSON(&req); err != nil {
slog.DebugContext(c.Request.Context(), "create dir bind failed", "error", err)
if isRequestBodyTooLarge(err) {
api.Error(c, http.StatusRequestEntityTooLarge, "request body is too large")
return
}
api.Error(c, http.StatusBadRequest, "invalid request body")
return
}
@@ -65,33 +78,45 @@ func (h *FileHandler) Upload(c *gin.Context) {
}
// File upload via multipart.
if err := c.Request.ParseMultipartForm(32 << 20); err != nil {
slog.DebugContext(c.Request.Context(), "parse multipart form failed", "error", err)
api.Error(c, http.StatusBadRequest, "invalid multipart form")
if mediaType != "multipart/form-data" {
api.Error(c, http.StatusBadRequest, "unsupported content type")
return
}
parentIDStr := c.PostForm("parent_id")
if maxUploadSize := h.fileService.MaxUploadSize(); maxUploadSize > 0 {
c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, maxUploadSize+multipartUploadOverhead)
}
var parentID *string
parentIDStr := c.Query("parent_id")
if parentIDStr != "" {
parentID = &parentIDStr
}
header, err := c.FormFile("file")
reader, err := c.Request.MultipartReader()
if err != nil {
slog.DebugContext(c.Request.Context(), "form file missing", "error", err)
api.Error(c, http.StatusBadRequest, "missing file field")
slog.DebugContext(c.Request.Context(), "create multipart reader failed", "error", err)
if isRequestBodyTooLarge(err) {
api.Error(c, http.StatusRequestEntityTooLarge, "request body is too large")
return
}
api.Error(c, http.StatusBadRequest, "invalid multipart form")
return
}
file, err := header.Open()
part, fileName, err := nextUploadFilePart(reader)
if err != nil {
api.RespondError(c, model.NewInternalError("open uploaded file", err))
slog.DebugContext(c.Request.Context(), "read multipart file part failed", "error", err)
if isRequestBodyTooLarge(err) {
api.Error(c, http.StatusRequestEntityTooLarge, "request body is too large")
return
}
api.Error(c, http.StatusBadRequest, err.Error())
return
}
defer file.Close()
defer part.Close()
info, err := h.fileService.Upload(c.Request.Context(), userID, parentID, header.Filename, file)
info, err := h.fileService.Upload(c.Request.Context(), userID, parentID, fileName, part)
if err != nil {
api.RespondError(c, err)
return
@@ -100,6 +125,34 @@ func (h *FileHandler) Upload(c *gin.Context) {
c.JSON(http.StatusCreated, info)
}
func nextUploadFilePart(reader *multipart.Reader) (*multipart.Part, string, error) {
part, err := reader.NextPart()
if err != nil {
if err == io.EOF {
return nil, "", errors.New("missing file field")
}
return nil, "", err
}
if part.FormName() != multipartFileField {
part.Close()
return nil, "", fmt.Errorf("unexpected multipart field %q", part.FormName())
}
fileName := part.FileName()
if fileName == "" {
part.Close()
return nil, "", errors.New("missing file name")
}
return part, fileName, nil
}
func isRequestBodyTooLarge(err error) bool {
var maxBytesErr *http.MaxBytesError
return errors.As(err, &maxBytesErr)
}
// List handles GET /api/v1/files.
func (h *FileHandler) List(c *gin.Context) {
userID := middleware.MustGetUserID(c)
internal/handler/file_test.go
+86 -3
View File
@@ -8,6 +8,7 @@ import (
"mime/multipart"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/gin-gonic/gin"
@@ -29,7 +30,7 @@ func newInMemStore() *inMemStore {
return &inMemStore{files: make(map[string][]byte)}
}
func (s *inMemStore) Save(_ context.Context, path string, reader io.Reader) (int64, error) {
func (s *inMemStore) SaveStaged(_ context.Context, path string, reader io.Reader) (int64, error) {
data, err := io.ReadAll(reader)
if err != nil {
return 0, err
@@ -38,6 +39,16 @@ func (s *inMemStore) Save(_ context.Context, path string, reader io.Reader) (int
return int64(len(data)), nil
}
func (s *inMemStore) PromoteStaged(_ context.Context, stagedPath, finalPath string) error {
data, ok := s.files[stagedPath]
if !ok {
return io.ErrUnexpectedEOF
}
s.files[finalPath] = data
delete(s.files, stagedPath)
return nil
}
func (s *inMemStore) Open(_ context.Context, path string) (io.ReadCloser, error) {
data, ok := s.files[path]
if !ok {
@@ -46,6 +57,10 @@ func (s *inMemStore) Open(_ context.Context, path string) (io.ReadCloser, error)
return io.NopCloser(bytes.NewReader(data)), nil
}
func (s *inMemStore) DeleteStaged(ctx context.Context, path string) error {
return s.Delete(ctx, path)
}
func (s *inMemStore) Delete(_ context.Context, path string) error {
delete(s.files, path)
return nil
@@ -58,6 +73,10 @@ func (e *fsNotFoundError) Error() string { return "file not found on disk: " + e
var _ storage.StorageBackend = (*inMemStore)(nil)
func setupFileHandler(t *testing.T) *FileHandler {
return setupFileHandlerWithMaxUploadSize(t, 0)
}
func setupFileHandlerWithMaxUploadSize(t *testing.T, maxUploadSize int64) *FileHandler {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
@@ -70,15 +89,19 @@ func setupFileHandler(t *testing.T) *FileHandler {
fileRepo := repository.NewFileRepository(db)
store := newInMemStore()
fileService := service.NewFileService(fileRepo, store, 0, nil)
fileService := service.NewFileService(fileRepo, store, maxUploadSize, nil)
return NewFileHandler(fileService)
}
func setupFileRouter(t *testing.T) *gin.Engine {
return setupFileRouterWithMaxUploadSize(t, 0)
}
func setupFileRouterWithMaxUploadSize(t *testing.T, maxUploadSize int64) *gin.Engine {
t.Helper()
handler := setupFileHandler(t)
handler := setupFileHandlerWithMaxUploadSize(t, maxUploadSize)
gin.SetMode(gin.TestMode)
r := gin.New()
@@ -133,6 +156,66 @@ func TestFileHandler_Upload(t *testing.T) {
}
}
func TestFileHandler_UploadRejectsOversizedFile(t *testing.T) {
r := setupFileRouterWithMaxUploadSize(t, 5)
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
part, _ := writer.CreateFormFile("file", "too-large.txt")
part.Write([]byte("123456"))
writer.Close()
req := httptest.NewRequest(http.MethodPost, "/api/v1/files", body)
req.Header.Set("Content-Type", writer.FormDataContentType())
req.Header.Set("X-User-ID", "user1")
rec := httptest.NewRecorder()
r.ServeHTTP(rec, req)
if rec.Code != http.StatusRequestEntityTooLarge {
t.Errorf("status = %d, want %d; body = %s", rec.Code, http.StatusRequestEntityTooLarge, rec.Body.String())
}
}
func TestFileHandler_UploadAllowsExactMaxUploadSize(t *testing.T) {
r := setupFileRouterWithMaxUploadSize(t, 5)
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
part, _ := writer.CreateFormFile("file", "exact.txt")
part.Write([]byte("12345"))
writer.Close()
req := httptest.NewRequest(http.MethodPost, "/api/v1/files", body)
req.Header.Set("Content-Type", writer.FormDataContentType())
req.Header.Set("X-User-ID", "user1")
rec := httptest.NewRecorder()
r.ServeHTTP(rec, req)
if rec.Code != http.StatusCreated {
t.Errorf("status = %d, want %d; body = %s", rec.Code, http.StatusCreated, rec.Body.String())
}
}
func TestFileHandler_UploadUnlimitedWhenMaxUploadSizeIsZero(t *testing.T) {
r := setupFileRouter(t)
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
part, _ := writer.CreateFormFile("file", "unlimited.txt")
part.Write([]byte(strings.Repeat("a", 128)))
writer.Close()
req := httptest.NewRequest(http.MethodPost, "/api/v1/files", body)
req.Header.Set("Content-Type", writer.FormDataContentType())
req.Header.Set("X-User-ID", "user1")
rec := httptest.NewRecorder()
r.ServeHTTP(rec, req)
if rec.Code != http.StatusCreated {
t.Errorf("status = %d, want %d; body = %s", rec.Code, http.StatusCreated, rec.Body.String())
}
}
func TestFileHandler_UploadNoFile(t *testing.T) {
r := setupFileRouter(t)