fix(file)!: stream uploads through staged storage

- fix: replace multipart form parsing with streaming multipart reads and
  apply request body limits when max_upload_size is configured.
- refactor: route uploads through staging paths before promotion to
  long-term data paths, keeping incomplete uploads out of durable
  storage records.
- test: cover oversized uploads, unlimited uploads, staged cleanup, and
  local storage promotion boundaries.
- docs: document the staged upload model and multipart parent_id query
  parameter.
This commit is contained in:
2026-07-05 17:18:19 +08:00
parent bfeb4b26e4
commit b988b4b15e
10 changed files with 457 additions and 50 deletions
+52 -3
View File
@@ -43,7 +43,7 @@ func newMemStorage() *memStorage {
return &memStorage{files: make(map[string][]byte)}
}
func (s *memStorage) Save(_ context.Context, path string, reader io.Reader) (int64, error) {
func (s *memStorage) SaveStaged(_ context.Context, path string, reader io.Reader) (int64, error) {
data, err := io.ReadAll(reader)
if err != nil {
return 0, err
@@ -54,6 +54,19 @@ func (s *memStorage) Save(_ context.Context, path string, reader io.Reader) (int
return int64(len(data)), nil
}
func (s *memStorage) PromoteStaged(_ context.Context, stagedPath, finalPath string) error {
s.mu.Lock()
defer s.mu.Unlock()
data, ok := s.files[stagedPath]
if !ok {
return io.ErrUnexpectedEOF
}
s.files[finalPath] = data
delete(s.files, stagedPath)
return nil
}
func (s *memStorage) Open(_ context.Context, path string) (io.ReadCloser, error) {
s.mu.RLock()
data, ok := s.files[path]
@@ -64,6 +77,10 @@ func (s *memStorage) Open(_ context.Context, path string) (io.ReadCloser, error)
return io.NopCloser(bytes.NewReader(data)), nil
}
func (s *memStorage) DeleteStaged(ctx context.Context, path string) error {
return s.Delete(ctx, path)
}
func (s *memStorage) Delete(_ context.Context, path string) error {
s.mu.Lock()
delete(s.files, path)
@@ -74,6 +91,10 @@ func (s *memStorage) Delete(_ context.Context, path string) error {
var _ storage.StorageBackend = (*memStorage)(nil)
func setupFileServiceWithStorage(t *testing.T) (*FileService, *memStorage) {
return setupFileServiceWithStorageAndMaxUploadSize(t, 0)
}
func setupFileServiceWithStorageAndMaxUploadSize(t *testing.T, maxUploadSize int64) (*FileService, *memStorage) {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
@@ -87,7 +108,7 @@ func setupFileServiceWithStorage(t *testing.T) (*FileService, *memStorage) {
fileRepo := repository.NewFileRepository(db)
store := newMemStorage()
return NewFileService(fileRepo, store, 0, nil), store // 0 = unlimited, nil logger defaults to slog.Default()
return NewFileService(fileRepo, store, maxUploadSize, nil), store // nil logger defaults to slog.Default()
}
func setupFileService(t *testing.T) *FileService {
@@ -125,6 +146,34 @@ func TestFileService_Upload(t *testing.T) {
}
}
func TestFileService_UploadRejectsOversizedFileAndCleansStaging(t *testing.T) {
svc, store := setupFileServiceWithStorageAndMaxUploadSize(t, 5)
ctx := context.Background()
_, err := svc.Upload(ctx, "user1", nil, "too-large.txt", strings.NewReader("123456"))
assertAppError(t, err, http.StatusRequestEntityTooLarge)
store.mu.RLock()
defer store.mu.RUnlock()
if len(store.files) != 0 {
t.Fatalf("storage files = %#v, want empty after oversized upload", store.files)
}
}
func TestFileService_UploadUnlimitedWhenMaxUploadSizeIsZero(t *testing.T) {
svc := setupFileService(t)
ctx := context.Background()
content := strings.Repeat("a", 128)
info, err := svc.Upload(ctx, "user1", nil, "unlimited.txt", strings.NewReader(content))
if err != nil {
t.Fatalf("Upload = %v", err)
}
if info.Size != int64(len(content)) {
t.Errorf("Size = %d, want %d", info.Size, len(content))
}
}
func TestFileService_UploadIntoDirectory(t *testing.T) {
svc := setupFileService(t)
ctx := context.Background()
@@ -498,7 +547,7 @@ func TestFileService_SoftDeleteKeepsStorage(t *testing.T) {
t.Fatalf("Delete = %v", err)
}
storagePath := "user1/" + info.ID
storagePath := "data/user1/" + info.ID
reader, err := store.Open(ctx, storagePath)
if err != nil {
t.Fatalf("storage should retain content after soft-delete, got: %v", err)