fix(file)!: stream uploads through staged storage

- fix: replace multipart form parsing with streaming multipart reads and
  apply request body limits when max_upload_size is configured.
- refactor: route uploads through staging paths before promotion to
  long-term data paths, keeping incomplete uploads out of durable
  storage records.
- test: cover oversized uploads, unlimited uploads, staged cleanup, and
  local storage promotion boundaries.
- docs: document the staged upload model and multipart parent_id query
  parameter.
This commit is contained in:
2026-07-05 17:18:19 +08:00
parent bfeb4b26e4
commit b988b4b15e
10 changed files with 457 additions and 50 deletions
+65 -12
View File
@@ -1,10 +1,12 @@
package handler
import (
"errors"
"fmt"
"io"
"log/slog"
"mime"
"mime/multipart"
"net/http"
"net/url"
"strconv"
@@ -13,10 +15,15 @@ import (
"github.com/dhao2001/mygo/internal/api"
"github.com/dhao2001/mygo/internal/middleware"
"github.com/dhao2001/mygo/internal/model"
"github.com/dhao2001/mygo/internal/service"
)
const (
jsonUploadBodyLimit = 64 << 10
multipartUploadOverhead = 1 << 20
multipartFileField = "file"
)
// FileHandler handles file management endpoints.
type FileHandler struct {
fileService *service.FileService
@@ -47,9 +54,15 @@ func (h *FileHandler) Upload(c *gin.Context) {
// Directory creation via JSON.
mediaType, _, _ := mime.ParseMediaType(contentType)
if mediaType == "application/json" {
c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, jsonUploadBodyLimit)
var req createDirRequest
if err := c.ShouldBindJSON(&req); err != nil {
slog.DebugContext(c.Request.Context(), "create dir bind failed", "error", err)
if isRequestBodyTooLarge(err) {
api.Error(c, http.StatusRequestEntityTooLarge, "request body is too large")
return
}
api.Error(c, http.StatusBadRequest, "invalid request body")
return
}
@@ -65,33 +78,45 @@ func (h *FileHandler) Upload(c *gin.Context) {
}
// File upload via multipart.
if err := c.Request.ParseMultipartForm(32 << 20); err != nil {
slog.DebugContext(c.Request.Context(), "parse multipart form failed", "error", err)
api.Error(c, http.StatusBadRequest, "invalid multipart form")
if mediaType != "multipart/form-data" {
api.Error(c, http.StatusBadRequest, "unsupported content type")
return
}
parentIDStr := c.PostForm("parent_id")
if maxUploadSize := h.fileService.MaxUploadSize(); maxUploadSize > 0 {
c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, maxUploadSize+multipartUploadOverhead)
}
var parentID *string
parentIDStr := c.Query("parent_id")
if parentIDStr != "" {
parentID = &parentIDStr
}
header, err := c.FormFile("file")
reader, err := c.Request.MultipartReader()
if err != nil {
slog.DebugContext(c.Request.Context(), "form file missing", "error", err)
api.Error(c, http.StatusBadRequest, "missing file field")
slog.DebugContext(c.Request.Context(), "create multipart reader failed", "error", err)
if isRequestBodyTooLarge(err) {
api.Error(c, http.StatusRequestEntityTooLarge, "request body is too large")
return
}
api.Error(c, http.StatusBadRequest, "invalid multipart form")
return
}
file, err := header.Open()
part, fileName, err := nextUploadFilePart(reader)
if err != nil {
api.RespondError(c, model.NewInternalError("open uploaded file", err))
slog.DebugContext(c.Request.Context(), "read multipart file part failed", "error", err)
if isRequestBodyTooLarge(err) {
api.Error(c, http.StatusRequestEntityTooLarge, "request body is too large")
return
}
api.Error(c, http.StatusBadRequest, err.Error())
return
}
defer file.Close()
defer part.Close()
info, err := h.fileService.Upload(c.Request.Context(), userID, parentID, header.Filename, file)
info, err := h.fileService.Upload(c.Request.Context(), userID, parentID, fileName, part)
if err != nil {
api.RespondError(c, err)
return
@@ -100,6 +125,34 @@ func (h *FileHandler) Upload(c *gin.Context) {
c.JSON(http.StatusCreated, info)
}
func nextUploadFilePart(reader *multipart.Reader) (*multipart.Part, string, error) {
part, err := reader.NextPart()
if err != nil {
if err == io.EOF {
return nil, "", errors.New("missing file field")
}
return nil, "", err
}
if part.FormName() != multipartFileField {
part.Close()
return nil, "", fmt.Errorf("unexpected multipart field %q", part.FormName())
}
fileName := part.FileName()
if fileName == "" {
part.Close()
return nil, "", errors.New("missing file name")
}
return part, fileName, nil
}
func isRequestBodyTooLarge(err error) bool {
var maxBytesErr *http.MaxBytesError
return errors.As(err, &maxBytesErr)
}
// List handles GET /api/v1/files.
func (h *FileHandler) List(c *gin.Context) {
userID := middleware.MustGetUserID(c)