Much optimized upload process

This commit is contained in:
Rostislav Raykov
2025-03-29 15:14:51 +02:00
parent eabad65167
commit 4e8a1c4bcf
7 changed files with 247 additions and 229 deletions

View File

@@ -3,6 +3,8 @@ package org.rostislav.quickdrop.controller;
import jakarta.servlet.http.HttpServletRequest;
import org.rostislav.quickdrop.entity.FileEntity;
import org.rostislav.quickdrop.entity.ShareTokenEntity;
import org.rostislav.quickdrop.model.FileUploadRequest;
import org.rostislav.quickdrop.service.AsyncFileMergeService;
import org.rostislav.quickdrop.service.FileService;
import org.rostislav.quickdrop.service.SessionService;
import org.rostislav.quickdrop.util.FileUtils;
@@ -16,8 +18,8 @@ import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBo
import java.io.IOException;
import java.time.LocalDate;
import java.util.HashMap;
import java.util.Map;
import static org.springframework.http.ResponseEntity.ok;
@RestController
@RequestMapping("/api/file")
@@ -25,10 +27,12 @@ public class FileRestController {
private static final Logger logger = LoggerFactory.getLogger(FileRestController.class);
private final FileService fileService;
private final SessionService sessionService;
private final AsyncFileMergeService asyncFileMergeService;
public FileRestController(FileService fileService, SessionService sessionService) {
public FileRestController(FileService fileService, SessionService sessionService, AsyncFileMergeService asyncFileMergeService) {
this.fileService = fileService;
this.sessionService = sessionService;
this.asyncFileMergeService = asyncFileMergeService;
}
@PostMapping("/upload-chunk")
@@ -41,29 +45,22 @@ public class FileRestController {
@RequestParam(value = "keepIndefinitely", defaultValue = "false") Boolean keepIndefinitely,
@RequestParam(value = "password", required = false) String password,
@RequestParam(value = "hidden", defaultValue = "false") Boolean hidden) {
if (chunkNumber == 0) {
logger.info("Upload started for file: {}", fileName);
}
try {
logger.info("Saving chunk {} of {}", chunkNumber, totalChunks);
fileService.saveFileChunk(file, fileName, chunkNumber);
if (chunkNumber + 1 == totalChunks) {
logger.info("All chunks uploaded for file: {} - Finalizing", fileName);
return fileService.finalizeFile(fileName, totalChunks, description, keepIndefinitely, password, hidden);
}
logger.info("Submitting chunk {} of {} for file: {}", chunkNumber, totalChunks, fileName);
FileUploadRequest fileUploadRequest = new FileUploadRequest(description, keepIndefinitely, password, hidden, fileName, totalChunks);
FileEntity fileEntity = asyncFileMergeService.submitChunk(fileUploadRequest, file, chunkNumber);
return ResponseEntity.ok(fileEntity);
} catch (IOException e) {
fileService.deleteChunkFilesFromTemp(fileName);
fileService.deleteFullFileFromTemp(fileName);
logger.error("Error processing chunk {} for file {}: {}", chunkNumber, fileName, e.getMessage());
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
.body("{\"error\": \"Error processing chunk\"}");
}
Map<String, String> response = new HashMap<>();
response.put("message", "Chunk " + chunkNumber + " uploaded successfully");
return ResponseEntity.ok(response);
}
@PostMapping("/share/{uuid}")
@@ -84,7 +81,7 @@ public class FileRestController {
token = fileService.generateShareToken(uuid, expirationDate, numberOfDownloads);
}
String shareLink = FileUtils.getShareLink(request, token.shareToken);
return ResponseEntity.ok(shareLink);
return ok(shareLink);
}
@GetMapping("/download/{uuid}/{token}")
@@ -97,7 +94,7 @@ public class FileRestController {
FileEntity fileEntity = fileService.getFile(uuid);
return ResponseEntity.ok()
return ok()
.header("Content-Disposition", "attachment; filename=\"" + fileEntity.name + "\"")
.header("Content-Type", "application/octet-stream")
.body(responseBody);

View File

@@ -0,0 +1,19 @@
package org.rostislav.quickdrop.model;
import java.io.File;
public class ChunkInfo {
public int chunkNumber;
public File chunkFile;
public boolean isLastChunk;
public ChunkInfo() {
}
public ChunkInfo(int chunkNumber, File chunkFile, boolean isLastChunk) {
this.chunkNumber = chunkNumber;
this.chunkFile = chunkFile;
this.isLastChunk = isLastChunk;
}
}

View File

@@ -1,6 +1,8 @@
package org.rostislav.quickdrop.model;
public class FileUploadRequest {
public String fileName;
public int totalChunks;
public String description;
public boolean keepIndefinitely;
public String password;
@@ -9,10 +11,12 @@ public class FileUploadRequest {
public FileUploadRequest() {
}
public FileUploadRequest(String description, boolean keepIndefinitely, String password, boolean hidden) {
public FileUploadRequest(String description, boolean keepIndefinitely, String password, boolean hidden, String fileName, int totalChunks) {
this.description = description;
this.keepIndefinitely = keepIndefinitely;
this.password = password;
this.hidden = hidden;
this.fileName = fileName;
this.totalChunks = totalChunks;
}
}

View File

@@ -0,0 +1,139 @@
package org.rostislav.quickdrop.service;
import org.rostislav.quickdrop.entity.FileEntity;
import org.rostislav.quickdrop.model.ChunkInfo;
import org.rostislav.quickdrop.model.FileUploadRequest;
import org.rostislav.quickdrop.repository.FileRepository;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import java.io.*;
import java.nio.file.Paths;
import java.util.UUID;
import java.util.concurrent.*;
@Service
public class AsyncFileMergeService {
private static final Logger logger = LoggerFactory.getLogger(AsyncFileMergeService.class);
private final ConcurrentMap<String, MergeTask> mergeTasks = new ConcurrentHashMap<>();
private final ExecutorService executorService = Executors.newCachedThreadPool();
private final ApplicationSettingsService applicationSettingsService;
private final FileEncryptionService fileEncryptionService;
private final FileService fileService;
private final File tempDir = new File(System.getProperty("java.io.tmpdir"));
private final FileRepository fileRepository;
public AsyncFileMergeService(ApplicationSettingsService applicationSettingsService,
FileEncryptionService fileEncryptionService,
FileService fileService, FileRepository fileRepository) {
this.applicationSettingsService = applicationSettingsService;
this.fileEncryptionService = fileEncryptionService;
this.fileService = fileService;
this.fileRepository = fileRepository;
}
public FileEntity submitChunk(FileUploadRequest request, MultipartFile multipartChunk, int chunkNumber) throws IOException {
File savedChunk = new File(tempDir, request.fileName + "_chunk_" + chunkNumber);
multipartChunk.transferTo(savedChunk);
logger.info("Chunk {} for file {} saved to {}", chunkNumber, request.fileName, savedChunk.getAbsolutePath());
MergeTask mergeTask = mergeTasks.computeIfAbsent(request.fileName, key -> {
MergeTask task = new MergeTask(request);
executorService.submit(task);
return task;
});
boolean isLastChunk = (chunkNumber == request.totalChunks - 1);
mergeTask.enqueueChunk(new ChunkInfo(chunkNumber, savedChunk, isLastChunk));
if (isLastChunk) {
try {
return mergeTask.getMergeCompletionFuture().get();
} catch (InterruptedException | ExecutionException e) {
logger.error("Error waiting for merge completion: {}", e.getMessage());
Thread.currentThread().interrupt();
throw new IOException("Merge task interrupted", e);
}
}
return null;
}
private void cleanUpChunks(FileUploadRequest request) {
for (int i = 0; i < request.totalChunks; i++) {
File chunkFile = new File(tempDir, request.fileName + "_chunk_" + i);
if (chunkFile.exists() && !chunkFile.delete()) {
logger.warn("Failed to delete chunk file: {}", chunkFile.getAbsolutePath());
}
logger.info("Cleaning up chunk {}", i);
}
}
private class MergeTask implements Runnable {
private final BlockingQueue<ChunkInfo> queue = new LinkedBlockingQueue<>();
private final CompletableFuture<FileEntity> mergeCompletionFuture = new CompletableFuture<>();
private final FileUploadRequest request;
private int processedChunks = 0;
private String uuid;
MergeTask(FileUploadRequest request) {
this.request = request;
do {
uuid = UUID.randomUUID().toString();
} while (fileRepository.findByUUID(uuid).isPresent());
}
public void enqueueChunk(ChunkInfo chunkInfo) {
queue.add(chunkInfo);
}
public CompletableFuture<FileEntity> getMergeCompletionFuture() {
return mergeCompletionFuture;
}
@Override
public void run() {
File finalFile = Paths.get(applicationSettingsService.getFileStoragePath(), uuid).toFile();
try (OutputStream finalOut = fileService.shouldEncrypt(request) ?
fileEncryptionService.getEncryptedOutputStream(finalFile, request.password) :
new BufferedOutputStream(new FileOutputStream(finalFile, true))) {
while (processedChunks < request.totalChunks) {
ChunkInfo info = queue.take();
try (InputStream in = new BufferedInputStream(new FileInputStream(info.chunkFile))) {
in.transferTo(finalOut);
}
if (!info.chunkFile.delete()) {
logger.warn("Failed to delete chunk file: {}", info.chunkFile.getAbsolutePath());
}
processedChunks++;
logger.info("Merged chunk {} for file {}", info.chunkNumber, request.fileName);
if (info.isLastChunk) {
break;
}
}
logger.info("All {} chunks merged for file {}", request.totalChunks, request.fileName);
FileEntity fileEntity = fileService.saveFile(finalFile, request, uuid);
if (fileEntity != null) {
logger.info("File {} saved successfully with UUID {}", request.fileName, fileEntity.uuid);
} else {
logger.error("Saving file {} failed", request.fileName);
}
mergeCompletionFuture.complete(fileEntity);
} catch (Exception e) {
logger.error("Error merging chunks for file {}: {}", request.fileName, e.getMessage());
mergeCompletionFuture.completeExceptionally(e);
cleanUpChunks(request);
e.printStackTrace();
} finally {
mergeTasks.remove(request.fileName);
}
}
}
}

View File

@@ -6,10 +6,7 @@ import javax.crypto.*;
import javax.crypto.spec.IvParameterSpec;
import javax.crypto.spec.PBEKeySpec;
import javax.crypto.spec.SecretKeySpec;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.*;
import java.security.InvalidAlgorithmParameterException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
@@ -27,7 +24,7 @@ public class FileEncryptionService {
public SecretKey generateKeyFromPassword(String password, byte[] salt)
throws NoSuchAlgorithmException, InvalidKeySpecException {
PBEKeySpec spec = new PBEKeySpec(password.toCharArray(), salt, ITERATION_COUNT, KEY_LENGTH);
SecretKeyFactory keyFactory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA256");
SecretKeyFactory keyFactory = SecretKeyFactory.getInstance(KEY_DERIVATION_ALGORITHM);
byte[] keyBytes = keyFactory.generateSecret(spec).getEncoded();
return new SecretKeySpec(keyBytes, "AES");
}
@@ -38,30 +35,6 @@ public class FileEncryptionService {
return bytes;
}
public void encryptFile(File inputFile, File outputFile, String password) throws NoSuchAlgorithmException, InvalidKeySpecException, NoSuchPaddingException, IOException, InvalidAlgorithmParameterException, InvalidKeyException {
byte[] salt = generateRandomBytes();
byte[] iv = generateRandomBytes();
SecretKey secretKey = generateKeyFromPassword(password, salt);
IvParameterSpec ivSpec = new IvParameterSpec(iv);
Cipher cipher = Cipher.getInstance(ALGORITHM);
cipher.init(Cipher.ENCRYPT_MODE, secretKey, ivSpec);
try (FileOutputStream fos = new FileOutputStream(outputFile);
CipherOutputStream cos = new CipherOutputStream(fos, cipher);
FileInputStream fis = new FileInputStream(inputFile)) {
fos.write(salt);
fos.write(iv);
byte[] buffer = new byte[8192];
int bytesRead;
while ((bytesRead = fis.read(buffer)) != -1) {
cos.write(buffer, 0, bytesRead);
}
}
}
@SuppressWarnings("ResultOfMethodCallIgnored")
public void decryptFile(File inputFile, File outputFile, String password) throws IOException, NoSuchAlgorithmException, InvalidKeySpecException, NoSuchPaddingException, InvalidAlgorithmParameterException, InvalidKeyException {
try (FileInputStream fis = new FileInputStream(inputFile)) {
@@ -87,4 +60,20 @@ public class FileEncryptionService {
}
}
}
public OutputStream getEncryptedOutputStream(File finalFile, String password) throws Exception {
FileOutputStream fos = new FileOutputStream(finalFile, true);
byte[] salt = generateRandomBytes();
byte[] iv = generateRandomBytes();
fos.write(salt);
fos.write(iv);
SecretKey secretKey = generateKeyFromPassword(password, salt);
IvParameterSpec ivSpec = new IvParameterSpec(iv);
Cipher cipher = Cipher.getInstance(ALGORITHM);
cipher.init(Cipher.ENCRYPT_MODE, secretKey, ivSpec);
return new CipherOutputStream(fos, cipher);
}
}

View File

@@ -22,16 +22,16 @@ import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.security.crypto.password.PasswordEncoder;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;
import java.io.*;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.time.LocalDate;
import java.util.List;
import java.util.Optional;
@@ -47,7 +47,6 @@ public class FileService {
private final PasswordEncoder passwordEncoder;
private final ApplicationSettingsService applicationSettingsService;
private final DownloadLogRepository downloadLogRepository;
private final File tempDir = Paths.get(System.getProperty("java.io.tmpdir")).toFile();
private final SessionService sessionService;
private final RenewalLogRepository renewalLogRepository;
private final FileEncryptionService fileEncryptionService;
@@ -68,7 +67,7 @@ public class FileService {
private static StreamingResponseBody getStreamingResponseBody(Path outputFile, FileEntity fileEntity) {
return outputStream -> {
try (FileInputStream inputStream = new FileInputStream(outputFile.toFile())) {
byte[] buffer = new byte[1024];
byte[] buffer = new byte[8192];
int bytesRead;
while ((bytesRead = inputStream.read(buffer)) != -1) {
outputStream.write(buffer, 0, bytesRead);
@@ -88,123 +87,6 @@ public class FileService {
};
}
public void saveFileChunk(MultipartFile file, String fileName, int chunkNumber) throws IOException {
File chunkFile = new File(tempDir, getFileChunkName(fileName, chunkNumber));
try (FileOutputStream fos = new FileOutputStream(chunkFile)) {
fos.write(file.getBytes());
}
}
public FileEntity assembleChunks(String fileName, int totalChunks, FileUploadRequest fileUploadRequest) throws IOException {
File finalFile = new File(tempDir, fileName);
if (!finalFile.createNewFile()) {
throw new IOException("Failed to create new file");
}
mergeChunksIntoFile(finalFile, fileName, totalChunks);
deleteChunkFilesFromTemp(fileName);
return saveFile(finalFile, fileUploadRequest);
}
private void mergeChunksIntoFile(File finalFile, String fileName, int totalChunks) throws IOException {
try (FileOutputStream fos = new FileOutputStream(finalFile)) {
for (int i = 0; i < totalChunks; i++) {
File chunk = new File(tempDir, fileName + "_chunk_" + i);
try {
Files.copy(chunk.toPath(), fos);
Files.delete(chunk.toPath());
} catch (IOException ex) {
logger.error("Error processing chunk {}: {}", i, ex.getMessage());
throw ex;
}
}
logger.info("All chunks merged into file: {}", finalFile);
}
}
public void deleteChunkFilesFromTemp(String fileName) {
File[] tempFiles = tempDir.listFiles((dir, name) -> name.startsWith(fileName + "_chunk_"));
if (tempFiles != null) {
for (File tempFile : tempFiles) {
if (tempFile.delete()) {
logger.info("Deleted temp file: {}", tempFile);
} else {
logger.error("Failed to delete temp file: {}", tempFile);
}
}
}
}
public void deleteFullFileFromTemp(String fileName) {
Path assembledFilePath = Paths.get(tempDir.getAbsolutePath(), fileName);
try {
if (Files.exists(assembledFilePath)) {
Files.delete(assembledFilePath);
logger.info("Deleted assembled file: {}", assembledFilePath);
}
} catch (IOException e) {
logger.error("Failed to delete assembled file: {}", assembledFilePath, e);
}
}
public FileEntity saveFile(File file, FileUploadRequest fileUploadRequest) {
if (!validateObjects(file, fileUploadRequest)) {
return null;
}
logger.info("Saving file: {}", file.getName());
String uuid = UUID.randomUUID().toString();
while (fileRepository.findByUUID(uuid).isPresent()) {
uuid = UUID.randomUUID().toString();
}
Path targetPath = Path.of(applicationSettingsService.getFileStoragePath(), uuid);
FileEntity fileEntity = populateFileEntity(file, fileUploadRequest, uuid);
if (fileEntity.encrypted) {
if (!saveEncryptedFile(targetPath, file, fileUploadRequest)) return null;
} else {
if (!moveAndRenameUnencryptedFile(file, targetPath)) return null;
}
logger.info("FileEntity inserted into database: {}", fileEntity);
return fileRepository.save(fileEntity);
}
public List<FileEntity> getFiles() {
return fileRepository.findAll();
}
private FileEntity populateFileEntity(File file, FileUploadRequest request, String uuid) {
FileEntity fileEntity = new FileEntity();
fileEntity.name = file.getName();
fileEntity.uuid = uuid;
fileEntity.description = request.description;
fileEntity.size = file.length();
fileEntity.keepIndefinitely = request.keepIndefinitely;
fileEntity.hidden = request.hidden;
fileEntity.encrypted = request.password != null && !request.password.isBlank() && applicationSettingsService.isEncryptionEnabled();
if (request.password != null && !request.password.isBlank()) {
fileEntity.passwordHash = passwordEncoder.encode(request.password);
}
return fileEntity;
}
public FileEntity getFile(Long id) {
return fileRepository.findById(id).orElse(null);
}
public FileEntity getFile(String uuid) {
return fileRepository.findByUUID(uuid).orElse(null);
}
private static RequesterInfo getRequesterInfo(HttpServletRequest request) {
String forwardedFor = request.getHeader("X-Forwarded-For");
String realIp = request.getHeader("X-Real-IP");
@@ -223,6 +105,48 @@ public class FileService {
return new RequesterInfo(ipAddress, userAgent);
}
public FileEntity saveFile(File file, FileUploadRequest fileUploadRequest, String uuid) {
if (!validateObjects(file, fileUploadRequest)) {
return null;
}
logger.info("Saving file: {}", file.getName());
FileEntity fileEntity = populateFileEntity(file, fileUploadRequest, uuid);
logger.info("FileEntity inserted into database: {}", fileEntity);
return fileRepository.save(fileEntity);
}
public List<FileEntity> getFiles() {
return fileRepository.findAll();
}
public boolean shouldEncrypt(FileUploadRequest request) {
return request.password != null && !request.password.isBlank() && applicationSettingsService.isEncryptionEnabled();
}
private FileEntity populateFileEntity(File file, FileUploadRequest request, String uuid) {
FileEntity fileEntity = new FileEntity();
fileEntity.name = request.fileName;
fileEntity.uuid = uuid;
fileEntity.description = request.description;
fileEntity.size = file.length();
fileEntity.keepIndefinitely = request.keepIndefinitely;
fileEntity.hidden = request.hidden;
fileEntity.encrypted = shouldEncrypt(request);
if (request.password != null && !request.password.isBlank()) {
fileEntity.passwordHash = passwordEncoder.encode(request.password);
}
return fileEntity;
}
public FileEntity getFile(String uuid) {
return fileRepository.findByUUID(uuid).orElse(null);
}
public boolean deleteFileFromFileSystem(String uuid) {
Path path = Path.of(applicationSettingsService.getFileStoragePath(), uuid);
try {
@@ -407,56 +331,6 @@ public class FileService {
logger.info("Share token updated/invalidated. File streamed successfully: {}", fileEntity.name);
}
public ResponseEntity<?> finalizeFile(String fileName, int totalChunks, String description,
Boolean keepIndefinitely, String password, Boolean hidden) throws IOException {
FileUploadRequest fileUploadRequest = new FileUploadRequest(description, keepIndefinitely, password, hidden);
FileEntity fileEntity = assembleChunks(fileName, totalChunks, fileUploadRequest);
if (fileEntity != null) {
return ResponseEntity.ok(fileEntity);
}
return ResponseEntity.badRequest().build();
}
private boolean saveEncryptedFile(Path savePath, File file, FileUploadRequest fileUploadRequest) {
try {
Path encryptedFile = Files.createFile(savePath);
logger.info("Encrypting file: {}", encryptedFile);
fileEncryptionService.encryptFile(file, encryptedFile.toFile(), fileUploadRequest.password);
logger.info("Encrypted file saved: {}", encryptedFile);
} catch (
Exception e) {
logger.error("Error encrypting file: {}", e.getMessage());
return false;
}
try {
Files.delete(file.toPath());
logger.info("Temp file deleted: {}", file);
} catch (
Exception e) {
logger.error("Error deleting temp file: {}", e.getMessage());
return false;
}
return true;
}
private boolean moveAndRenameUnencryptedFile(File file, Path path) {
for (int retry = 0; retry < 3; retry++) {
try {
Files.move(file.toPath(), path, StandardCopyOption.REPLACE_EXISTING);
logger.info("File moved successfully: {}", path);
return true;
} catch (IOException e) {
logger.error("Attempt {}/3 failed to move file {}: {}", retry + 1, file.getName(), e.getMessage());
}
}
logger.error("Failed to move file after 3 attempts: {}", file.getName());
return false;
}
public FileEntity updateKeepIndefinitely(String uuid, boolean keepIndefinitely, HttpServletRequest request) {
Optional<FileEntity> referenceById = fileRepository.findByUUID(uuid);
if (referenceById.isEmpty()) {
@@ -535,13 +409,6 @@ public class FileService {
return shareTokenRepository.getShareTokenEntityByToken(token);
}
private record RequesterInfo(String ipAddress, String userAgent) {
}
private String getFileChunkName(String fileName, int chunkNumber) {
return fileName + "_chunk_" + chunkNumber;
}
private Path decryptFileIfNeeded(FileEntity fileEntity, Path filePath, String password) {
if (!fileEntity.encrypted) {
return filePath;
@@ -575,4 +442,7 @@ public class FileService {
public boolean fileExistsInFileSystem(String uuid) {
return Files.exists(Path.of(applicationSettingsService.getFileStoragePath(), uuid));
}
private record RequesterInfo(String ipAddress, String userAgent) {
}
}

View File

@@ -17,4 +17,4 @@ server.error.path=/error
spring.flyway.baseline-on-migrate=true
spring.flyway.baseline-version=1
spring.flyway.locations=classpath:db/migration
app.version=1.4.2
app.version=1.4.3