fix(file)!: stream uploads through staged storage

- fix: replace multipart form parsing with streaming multipart reads and
  apply request body limits when max_upload_size is configured.
- refactor: route uploads through staging paths before promotion to
  long-term data paths, keeping incomplete uploads out of durable
  storage records.
- test: cover oversized uploads, unlimited uploads, staged cleanup, and
  local storage promotion boundaries.
- docs: document the staged upload model and multipart parent_id query
  parameter.
This commit is contained in:
2026-07-05 17:18:19 +08:00
parent bfeb4b26e4
commit b988b4b15e
10 changed files with 457 additions and 50 deletions
+76 -10
View File
@@ -41,6 +41,8 @@ type FileList struct {
Total int64 `json:"total"`
}
var errUploadTooLarge = errors.New("upload exceeds maximum size")
// FileService handles file management business logic.
type FileService struct {
fileRepo repository.FileRepository
@@ -67,6 +69,12 @@ func NewFileService(
}
}
// MaxUploadSize returns the configured maximum upload size in bytes.
// A value of 0 means uploads are not size-limited by MyGO.
func (s *FileService) MaxUploadSize() int64 {
return s.maxUploadSize
}
// Upload stores a file's content and creates its metadata record.
// The MIME type is always detected server-side from the file content.
func (s *FileService) Upload(ctx context.Context, userID string, parentID *string, fileName string, reader io.Reader) (*FileInfo, error) {
@@ -87,14 +95,16 @@ func (s *FileService) Upload(ctx context.Context, userID string, parentID *strin
return nil, model.NewInternalError("check name conflict", err)
}
// Limit the reader if a max upload size is configured.
if s.maxUploadSize > 0 {
reader = io.LimitReader(reader, s.maxUploadSize+1)
reader = &uploadLimitReader{reader: reader, remaining: s.maxUploadSize}
}
// Detect MIME type from file content.
head := make([]byte, 512)
n, readErr := io.ReadFull(reader, head)
if isUploadTooLargeError(readErr) {
return nil, uploadTooLargeError(s.maxUploadSize)
}
if readErr != nil && readErr != io.ErrUnexpectedEOF && readErr != io.EOF {
return nil, model.NewInternalError("read for mime detection", readErr)
}
@@ -103,21 +113,30 @@ func (s *FileService) Upload(ctx context.Context, userID string, parentID *strin
reader = io.MultiReader(bytes.NewReader(head), reader)
fileID := uuid.NewString()
storagePath := fmt.Sprintf("%s/%s", userID, fileID)
stagedPath := fmt.Sprintf("staging/%s/%s", userID, uuid.NewString())
storagePath := fmt.Sprintf("data/%s/%s", userID, fileID)
// Compute SHA-256 hash while writing to storage.
// Compute SHA-256 hash while writing to staging storage.
hasher := sha256.New()
teeReader := io.TeeReader(reader, hasher)
written, err := s.storage.Save(ctx, storagePath, teeReader)
written, err := s.storage.SaveStaged(ctx, stagedPath, teeReader)
if err != nil {
return nil, model.NewInternalError("save file", err)
_ = s.storage.DeleteStaged(ctx, stagedPath)
if isUploadTooLargeError(err) {
return nil, uploadTooLargeError(s.maxUploadSize)
}
return nil, model.NewInternalError("save staged file", err)
}
if s.maxUploadSize > 0 && written > s.maxUploadSize {
// Clean up the oversized file.
_ = s.storage.Delete(ctx, storagePath)
return nil, &model.AppError{Status: http.StatusBadRequest, Message: fmt.Sprintf("file size exceeds the maximum allowed size of %d bytes", s.maxUploadSize)}
_ = s.storage.DeleteStaged(ctx, stagedPath)
return nil, uploadTooLargeError(s.maxUploadSize)
}
if err := s.storage.PromoteStaged(ctx, stagedPath, storagePath); err != nil {
_ = s.storage.DeleteStaged(ctx, stagedPath)
return nil, model.NewInternalError("promote staged file", err)
}
file := &model.File{
@@ -134,7 +153,7 @@ func (s *FileService) Upload(ctx context.Context, userID string, parentID *strin
}
if err := s.fileRepo.Create(ctx, file); err != nil {
// Compensate: remove the stored file.
// Compensate: remove the promoted file.
_ = s.storage.Delete(ctx, storagePath)
if errors.Is(err, model.ErrDuplicate) {
return nil, &model.AppError{Status: http.StatusConflict, Message: "a file with this name already exists in this directory"}
@@ -145,6 +164,53 @@ func (s *FileService) Upload(ctx context.Context, userID string, parentID *strin
return modelToFileInfo(file), nil
}
type uploadLimitReader struct {
reader io.Reader
remaining int64
}
func (r *uploadLimitReader) Read(p []byte) (int, error) {
if len(p) == 0 {
return 0, nil
}
if r.remaining == 0 {
var one [1]byte
n, err := r.reader.Read(one[:])
if n > 0 {
p[0] = one[0]
return 1, errUploadTooLarge
}
return 0, err
}
if int64(len(p)) > r.remaining {
p = p[:r.remaining]
}
n, err := r.reader.Read(p)
r.remaining -= int64(n)
return n, err
}
func isUploadTooLargeError(err error) bool {
if err == nil {
return false
}
if errors.Is(err, errUploadTooLarge) {
return true
}
var maxBytesErr *http.MaxBytesError
return errors.As(err, &maxBytesErr)
}
func uploadTooLargeError(maxUploadSize int64) *model.AppError {
if maxUploadSize > 0 {
return &model.AppError{Status: http.StatusRequestEntityTooLarge, Message: fmt.Sprintf("file size exceeds the maximum allowed size of %d bytes", maxUploadSize)}
}
return &model.AppError{Status: http.StatusRequestEntityTooLarge, Message: "request body is too large"}
}
// Download returns a reader for the file's content and its metadata.
// The caller must close the returned ReadCloser.
func (s *FileService) Download(ctx context.Context, userID, fileID string) (io.ReadCloser, *FileInfo, error) {