fix(file)!: stream uploads through staged storage
- fix: replace multipart form parsing with streaming multipart reads and apply request body limits when max_upload_size is configured. - refactor: route uploads through staging paths before promotion to long-term data paths, keeping incomplete uploads out of durable storage records. - test: cover oversized uploads, unlimited uploads, staged cleanup, and local storage promotion boundaries. - docs: document the staged upload model and multipart parent_id query parameter.
This commit is contained in:
@@ -43,7 +43,7 @@ func newMemStorage() *memStorage {
|
||||
return &memStorage{files: make(map[string][]byte)}
|
||||
}
|
||||
|
||||
func (s *memStorage) Save(_ context.Context, path string, reader io.Reader) (int64, error) {
|
||||
func (s *memStorage) SaveStaged(_ context.Context, path string, reader io.Reader) (int64, error) {
|
||||
data, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
@@ -54,6 +54,19 @@ func (s *memStorage) Save(_ context.Context, path string, reader io.Reader) (int
|
||||
return int64(len(data)), nil
|
||||
}
|
||||
|
||||
func (s *memStorage) PromoteStaged(_ context.Context, stagedPath, finalPath string) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
data, ok := s.files[stagedPath]
|
||||
if !ok {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
s.files[finalPath] = data
|
||||
delete(s.files, stagedPath)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *memStorage) Open(_ context.Context, path string) (io.ReadCloser, error) {
|
||||
s.mu.RLock()
|
||||
data, ok := s.files[path]
|
||||
@@ -64,6 +77,10 @@ func (s *memStorage) Open(_ context.Context, path string) (io.ReadCloser, error)
|
||||
return io.NopCloser(bytes.NewReader(data)), nil
|
||||
}
|
||||
|
||||
func (s *memStorage) DeleteStaged(ctx context.Context, path string) error {
|
||||
return s.Delete(ctx, path)
|
||||
}
|
||||
|
||||
func (s *memStorage) Delete(_ context.Context, path string) error {
|
||||
s.mu.Lock()
|
||||
delete(s.files, path)
|
||||
@@ -74,6 +91,10 @@ func (s *memStorage) Delete(_ context.Context, path string) error {
|
||||
var _ storage.StorageBackend = (*memStorage)(nil)
|
||||
|
||||
func setupFileServiceWithStorage(t *testing.T) (*FileService, *memStorage) {
|
||||
return setupFileServiceWithStorageAndMaxUploadSize(t, 0)
|
||||
}
|
||||
|
||||
func setupFileServiceWithStorageAndMaxUploadSize(t *testing.T, maxUploadSize int64) (*FileService, *memStorage) {
|
||||
t.Helper()
|
||||
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
@@ -87,7 +108,7 @@ func setupFileServiceWithStorage(t *testing.T) (*FileService, *memStorage) {
|
||||
fileRepo := repository.NewFileRepository(db)
|
||||
store := newMemStorage()
|
||||
|
||||
return NewFileService(fileRepo, store, 0, nil), store // 0 = unlimited, nil logger defaults to slog.Default()
|
||||
return NewFileService(fileRepo, store, maxUploadSize, nil), store // nil logger defaults to slog.Default()
|
||||
}
|
||||
|
||||
func setupFileService(t *testing.T) *FileService {
|
||||
@@ -125,6 +146,34 @@ func TestFileService_Upload(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileService_UploadRejectsOversizedFileAndCleansStaging(t *testing.T) {
|
||||
svc, store := setupFileServiceWithStorageAndMaxUploadSize(t, 5)
|
||||
ctx := context.Background()
|
||||
|
||||
_, err := svc.Upload(ctx, "user1", nil, "too-large.txt", strings.NewReader("123456"))
|
||||
assertAppError(t, err, http.StatusRequestEntityTooLarge)
|
||||
|
||||
store.mu.RLock()
|
||||
defer store.mu.RUnlock()
|
||||
if len(store.files) != 0 {
|
||||
t.Fatalf("storage files = %#v, want empty after oversized upload", store.files)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileService_UploadUnlimitedWhenMaxUploadSizeIsZero(t *testing.T) {
|
||||
svc := setupFileService(t)
|
||||
ctx := context.Background()
|
||||
|
||||
content := strings.Repeat("a", 128)
|
||||
info, err := svc.Upload(ctx, "user1", nil, "unlimited.txt", strings.NewReader(content))
|
||||
if err != nil {
|
||||
t.Fatalf("Upload = %v", err)
|
||||
}
|
||||
if info.Size != int64(len(content)) {
|
||||
t.Errorf("Size = %d, want %d", info.Size, len(content))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileService_UploadIntoDirectory(t *testing.T) {
|
||||
svc := setupFileService(t)
|
||||
ctx := context.Background()
|
||||
@@ -498,7 +547,7 @@ func TestFileService_SoftDeleteKeepsStorage(t *testing.T) {
|
||||
t.Fatalf("Delete = %v", err)
|
||||
}
|
||||
|
||||
storagePath := "user1/" + info.ID
|
||||
storagePath := "data/user1/" + info.ID
|
||||
reader, err := store.Open(ctx, storagePath)
|
||||
if err != nil {
|
||||
t.Fatalf("storage should retain content after soft-delete, got: %v", err)
|
||||
|
||||
Reference in New Issue
Block a user