fix(file)!: stream uploads through staged storage
- fix: replace multipart form parsing with streaming multipart reads and apply request body limits when max_upload_size is configured. - refactor: route uploads through staging paths before promotion to long-term data paths, keeping incomplete uploads out of durable storage records. - test: cover oversized uploads, unlimited uploads, staged cleanup, and local storage promotion boundaries. - docs: document the staged upload model and multipart parent_id query parameter.
This commit is contained in:
+76
-10
@@ -41,6 +41,8 @@ type FileList struct {
|
||||
Total int64 `json:"total"`
|
||||
}
|
||||
|
||||
var errUploadTooLarge = errors.New("upload exceeds maximum size")
|
||||
|
||||
// FileService handles file management business logic.
|
||||
type FileService struct {
|
||||
fileRepo repository.FileRepository
|
||||
@@ -67,6 +69,12 @@ func NewFileService(
|
||||
}
|
||||
}
|
||||
|
||||
// MaxUploadSize returns the configured maximum upload size in bytes.
|
||||
// A value of 0 means uploads are not size-limited by MyGO.
|
||||
func (s *FileService) MaxUploadSize() int64 {
|
||||
return s.maxUploadSize
|
||||
}
|
||||
|
||||
// Upload stores a file's content and creates its metadata record.
|
||||
// The MIME type is always detected server-side from the file content.
|
||||
func (s *FileService) Upload(ctx context.Context, userID string, parentID *string, fileName string, reader io.Reader) (*FileInfo, error) {
|
||||
@@ -87,14 +95,16 @@ func (s *FileService) Upload(ctx context.Context, userID string, parentID *strin
|
||||
return nil, model.NewInternalError("check name conflict", err)
|
||||
}
|
||||
|
||||
// Limit the reader if a max upload size is configured.
|
||||
if s.maxUploadSize > 0 {
|
||||
reader = io.LimitReader(reader, s.maxUploadSize+1)
|
||||
reader = &uploadLimitReader{reader: reader, remaining: s.maxUploadSize}
|
||||
}
|
||||
|
||||
// Detect MIME type from file content.
|
||||
head := make([]byte, 512)
|
||||
n, readErr := io.ReadFull(reader, head)
|
||||
if isUploadTooLargeError(readErr) {
|
||||
return nil, uploadTooLargeError(s.maxUploadSize)
|
||||
}
|
||||
if readErr != nil && readErr != io.ErrUnexpectedEOF && readErr != io.EOF {
|
||||
return nil, model.NewInternalError("read for mime detection", readErr)
|
||||
}
|
||||
@@ -103,21 +113,30 @@ func (s *FileService) Upload(ctx context.Context, userID string, parentID *strin
|
||||
reader = io.MultiReader(bytes.NewReader(head), reader)
|
||||
|
||||
fileID := uuid.NewString()
|
||||
storagePath := fmt.Sprintf("%s/%s", userID, fileID)
|
||||
stagedPath := fmt.Sprintf("staging/%s/%s", userID, uuid.NewString())
|
||||
storagePath := fmt.Sprintf("data/%s/%s", userID, fileID)
|
||||
|
||||
// Compute SHA-256 hash while writing to storage.
|
||||
// Compute SHA-256 hash while writing to staging storage.
|
||||
hasher := sha256.New()
|
||||
teeReader := io.TeeReader(reader, hasher)
|
||||
|
||||
written, err := s.storage.Save(ctx, storagePath, teeReader)
|
||||
written, err := s.storage.SaveStaged(ctx, stagedPath, teeReader)
|
||||
if err != nil {
|
||||
return nil, model.NewInternalError("save file", err)
|
||||
_ = s.storage.DeleteStaged(ctx, stagedPath)
|
||||
if isUploadTooLargeError(err) {
|
||||
return nil, uploadTooLargeError(s.maxUploadSize)
|
||||
}
|
||||
return nil, model.NewInternalError("save staged file", err)
|
||||
}
|
||||
|
||||
if s.maxUploadSize > 0 && written > s.maxUploadSize {
|
||||
// Clean up the oversized file.
|
||||
_ = s.storage.Delete(ctx, storagePath)
|
||||
return nil, &model.AppError{Status: http.StatusBadRequest, Message: fmt.Sprintf("file size exceeds the maximum allowed size of %d bytes", s.maxUploadSize)}
|
||||
_ = s.storage.DeleteStaged(ctx, stagedPath)
|
||||
return nil, uploadTooLargeError(s.maxUploadSize)
|
||||
}
|
||||
|
||||
if err := s.storage.PromoteStaged(ctx, stagedPath, storagePath); err != nil {
|
||||
_ = s.storage.DeleteStaged(ctx, stagedPath)
|
||||
return nil, model.NewInternalError("promote staged file", err)
|
||||
}
|
||||
|
||||
file := &model.File{
|
||||
@@ -134,7 +153,7 @@ func (s *FileService) Upload(ctx context.Context, userID string, parentID *strin
|
||||
}
|
||||
|
||||
if err := s.fileRepo.Create(ctx, file); err != nil {
|
||||
// Compensate: remove the stored file.
|
||||
// Compensate: remove the promoted file.
|
||||
_ = s.storage.Delete(ctx, storagePath)
|
||||
if errors.Is(err, model.ErrDuplicate) {
|
||||
return nil, &model.AppError{Status: http.StatusConflict, Message: "a file with this name already exists in this directory"}
|
||||
@@ -145,6 +164,53 @@ func (s *FileService) Upload(ctx context.Context, userID string, parentID *strin
|
||||
return modelToFileInfo(file), nil
|
||||
}
|
||||
|
||||
type uploadLimitReader struct {
|
||||
reader io.Reader
|
||||
remaining int64
|
||||
}
|
||||
|
||||
func (r *uploadLimitReader) Read(p []byte) (int, error) {
|
||||
if len(p) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
if r.remaining == 0 {
|
||||
var one [1]byte
|
||||
n, err := r.reader.Read(one[:])
|
||||
if n > 0 {
|
||||
p[0] = one[0]
|
||||
return 1, errUploadTooLarge
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if int64(len(p)) > r.remaining {
|
||||
p = p[:r.remaining]
|
||||
}
|
||||
|
||||
n, err := r.reader.Read(p)
|
||||
r.remaining -= int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func isUploadTooLargeError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if errors.Is(err, errUploadTooLarge) {
|
||||
return true
|
||||
}
|
||||
var maxBytesErr *http.MaxBytesError
|
||||
return errors.As(err, &maxBytesErr)
|
||||
}
|
||||
|
||||
func uploadTooLargeError(maxUploadSize int64) *model.AppError {
|
||||
if maxUploadSize > 0 {
|
||||
return &model.AppError{Status: http.StatusRequestEntityTooLarge, Message: fmt.Sprintf("file size exceeds the maximum allowed size of %d bytes", maxUploadSize)}
|
||||
}
|
||||
return &model.AppError{Status: http.StatusRequestEntityTooLarge, Message: "request body is too large"}
|
||||
}
|
||||
|
||||
// Download returns a reader for the file's content and its metadata.
|
||||
// The caller must close the returned ReadCloser.
|
||||
func (s *FileService) Download(ctx context.Context, userID, fileID string) (io.ReadCloser, *FileInfo, error) {
|
||||
|
||||
Reference in New Issue
Block a user